// Copyright (c) 2018, Compiler Explorer Authors // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are met: // // * Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE // ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE // LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR // CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF // SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE // POSSIBILITY OF SUCH DAMAGE. import assert from 'assert'; import {DynamoDB} from '@aws-sdk/client-dynamodb'; import _ from 'underscore'; import {unwrap} from '../assert.js'; import {logger} from '../logger.js'; import {S3Bucket} from '../s3-handler.js'; import {anonymizeIp} from '../utils.js'; import {StorageBase} from './base.js'; /* * NEVER CHANGE THIS VALUE * * This value does not control the length of generated * short links. It is purely an implementation detail * of the underlying dynamodb table, and changing it * will break all existing links in the database. */ const PREFIX_LENGTH = 6; /* * This value can be changed freely to control the minimum * generated link length. * * Changing it will have no impact on existing links. */ const MIN_STORED_ID_LENGTH = 9; assert(MIN_STORED_ID_LENGTH >= PREFIX_LENGTH, 'MIN_STORED_ID_LENGTH must be at least PREFIX_LENGTH'); export class StorageS3 extends StorageBase { static get key() { return 's3'; } protected readonly prefix: string; protected readonly table: string; protected readonly s3: S3Bucket; protected readonly dynamoDb: DynamoDB; constructor(httpRootDir, compilerProps, awsProps) { super(httpRootDir, compilerProps); const region = awsProps('region'); const bucket = awsProps('storageBucket'); this.prefix = awsProps('storagePrefix'); this.table = awsProps('storageDynamoTable'); logger.info( `Using s3 storage solution on ${region}, bucket ${bucket}, ` + `prefix ${this.prefix}, dynamo table ${this.table}`, ); this.s3 = new S3Bucket(bucket, region); this.dynamoDb = new DynamoDB({region: region}); } async storeItem(item, req) { logger.info(`Storing item ${item.prefix}`); const now = new Date(); let ip = req.get('X-Forwarded-For') || anonymizeIp(req.ip); const commaIndex = ip.indexOf(','); if (commaIndex > 0) { // Anonymize only client IP ip = `${anonymizeIp(ip.substring(0, commaIndex))}${ip.substring(commaIndex, ip.length)}`; } now.setSeconds(0, 0); try { await Promise.all([ this.dynamoDb.putItem({ TableName: this.table, Item: { prefix: {S: item.prefix}, unique_subhash: {S: item.uniqueSubHash}, full_hash: {S: item.fullHash}, stats: { M: {clicks: {N: '0'}}, }, creation_ip: {S: ip}, creation_date: {S: now.toISOString()}, }, }), this.s3.put(item.fullHash, item.config, this.prefix, {}), ]); return item; } catch (err) { logger.error('Unable to store item', {item, err}); throw err; } } async findUniqueSubhash(hash) { const prefix = hash.substring(0, PREFIX_LENGTH); const data = await this.dynamoDb.query({ TableName: this.table, ProjectionExpression: 'unique_subhash, full_hash', KeyConditionExpression: 'prefix = :prefix', ExpressionAttributeValues: {':prefix': {S: prefix}}, }); const subHashes = _.chain(data.Items).pluck('unique_subhash').pluck('S').value(); const fullHashes = _.chain(data.Items).pluck('full_hash').pluck('S').value(); for (let i = MIN_STORED_ID_LENGTH; i < hash.length - 1; i++) { const subHash = hash.substring(0, i); // Check if the current base is present in the subHashes array const index = _.indexOf(subHashes, subHash, true); if (index === -1) { // Current base is not present, we have a new config in our hands return { prefix: prefix, uniqueSubHash: subHash, alreadyPresent: false, }; } else { const itemHash = fullHashes[index]; /* If the hashes coincide, it means this config has already been stored. * Else, keep looking */ if (itemHash === hash) { return { prefix: prefix, uniqueSubHash: subHash, alreadyPresent: true, }; } } } throw new Error(`Could not find unique subhash for hash "${hash}"`); } getKeyStruct(id) { return { prefix: {S: id.substring(0, PREFIX_LENGTH)}, unique_subhash: {S: id}, }; } async expandId(id: string) { // By just getting the item and not trying to update it, we save an update when the link does not exist // for which we have less resources allocated, but get one extra read (But we do have more reserved for it) const item = await this.dynamoDb.getItem({ TableName: this.table, Key: this.getKeyStruct(id), }); const attributes = item.Item; if (!attributes) throw new Error(`ID ${id} not present in links table`); const result = await this.s3.get(unwrap(attributes.full_hash.S), this.prefix); // If we're here, we are pretty confident there is a match. But never hurts to double check if (!result.hit) throw new Error(`ID ${id} not present in storage`); const metadata = attributes.named_metadata ? attributes.named_metadata.M : null; return { config: unwrap(result.data).toString(), specialMetadata: metadata, }; } async incrementViewCount(id) { try { await this.dynamoDb.updateItem({ TableName: this.table, Key: this.getKeyStruct(id), UpdateExpression: 'SET stats.clicks = stats.clicks + :inc', ExpressionAttributeValues: { ':inc': {N: '1'}, }, ReturnValues: 'NONE', }); } catch (err) { // Swallow up errors logger.error(`Error when incrementing view count for ${id}`, err); } } }