diff --git a/packages/datadog-plugin-aws-sdk/src/services/dynamodb.js b/packages/datadog-plugin-aws-sdk/src/services/dynamodb.js index 4097586b2c5..cbca2192ad6 100644 --- a/packages/datadog-plugin-aws-sdk/src/services/dynamodb.js +++ b/packages/datadog-plugin-aws-sdk/src/services/dynamodb.js @@ -1,6 +1,9 @@ 'use strict' const BaseAwsSdkPlugin = require('../base') +const log = require('../../../dd-trace/src/log') +const { DYNAMODB_PTR_KIND, SPAN_POINTER_DIRECTION } = require('../../../dd-trace/src/constants') +const { extractPrimaryKeys, generatePointerHash } = require('../util') class DynamoDb extends BaseAwsSdkPlugin { static get id () { return 'dynamodb' } @@ -48,6 +51,157 @@ class DynamoDb extends BaseAwsSdkPlugin { return tags } + + addSpanPointers (span, response) { + const request = response?.request + const operationName = request?.operation + + const hashes = [] + switch (operationName) { + case 'putItem': { + const hash = DynamoDb.calculatePutItemHash( + request?.params?.TableName, + request?.params?.Item, + this.getPrimaryKeyConfig() + ) + if (hash) hashes.push(hash) + break + } + case 'updateItem': + case 'deleteItem': { + const hash = DynamoDb.calculateHashWithKnownKeys(request?.params?.TableName, request?.params?.Key) + if (hash) hashes.push(hash) + break + } + case 'transactWriteItems': { + const transactItems = request?.params?.TransactItems || [] + for (const item of transactItems) { + if (item.Put) { + const hash = + DynamoDb.calculatePutItemHash(item.Put.TableName, item.Put.Item, this.getPrimaryKeyConfig()) + if (hash) hashes.push(hash) + } else if (item.Update || item.Delete) { + const operation = item.Update ? item.Update : item.Delete + const hash = DynamoDb.calculateHashWithKnownKeys(operation.TableName, operation.Key) + if (hash) hashes.push(hash) + } + } + break + } + case 'batchWriteItem': { + const requestItems = request?.params.RequestItems || {} + for (const [tableName, operations] of Object.entries(requestItems)) { + if (!Array.isArray(operations)) continue + for (const operation of operations) { + if (operation?.PutRequest) { + const hash = + DynamoDb.calculatePutItemHash(tableName, operation.PutRequest.Item, this.getPrimaryKeyConfig()) + if (hash) hashes.push(hash) + } else if (operation?.DeleteRequest) { + const hash = DynamoDb.calculateHashWithKnownKeys(tableName, operation.DeleteRequest.Key) + if (hash) hashes.push(hash) + } + } + } + break + } + } + + for (const hash of hashes) { + span.addSpanPointer(DYNAMODB_PTR_KIND, SPAN_POINTER_DIRECTION.DOWNSTREAM, hash) + } + } + + /** + * Parses primary key config from the `DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS` env var. + * Only runs when needed, and warns when missing or invalid config. + * @returns {Object|undefined} Parsed config from env var or undefined if empty/missing/invalid config. + */ + getPrimaryKeyConfig () { + if (this.dynamoPrimaryKeyConfig) { + // Return cached config if it exists + return this.dynamoPrimaryKeyConfig + } + + const configStr = this._tracerConfig?.aws?.dynamoDb?.tablePrimaryKeys + if (!configStr) { + log.warn('Missing DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS env variable. ' + + 'Please add your table\'s primary keys under this env variable.') + return + } + + try { + const parsedConfig = JSON.parse(configStr) + const config = {} + for (const [tableName, primaryKeys] of Object.entries(parsedConfig)) { + if (Array.isArray(primaryKeys) && primaryKeys.length > 0 && primaryKeys.length <= 2) { + config[tableName] = new Set(primaryKeys) + } else { + log.warn(`Invalid primary key configuration for table: ${tableName}.` + + 'Please fix the DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS env var.') + } + } + + this.dynamoPrimaryKeyConfig = config + return config + } catch (err) { + log.warn('Failed to parse DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS:', err.message) + } + } + + /** + * Calculates a hash for DynamoDB PutItem operations using table's configured primary keys. + * @param {string} tableName - Name of the DynamoDB table. + * @param {Object} item - Complete PutItem item parameter to be put. + * @param {Object.>} primaryKeyConfig - Mapping of table names to Sets of primary key names + * loaded from DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS. + * @returns {string|undefined} Hash combining table name and primary key/value pairs, or undefined if unable. + */ + static calculatePutItemHash (tableName, item, primaryKeyConfig) { + if (!tableName || !item) { + log.debug('Unable to calculate hash because missing required parameters') + return + } + if (!primaryKeyConfig) { + log.warn('Missing DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS env variable') + return + } + const primaryKeySet = primaryKeyConfig[tableName] + if (!primaryKeySet || !(primaryKeySet instanceof Set) || primaryKeySet.size === 0 || primaryKeySet.size > 2) { + log.warn( + `span pointers: failed to extract PutItem span pointer: table ${tableName} ` + + 'not found in primary key names or the DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS env var was invalid.' + + 'Please update the env var.' + ) + return + } + const keyValues = extractPrimaryKeys(primaryKeySet, item) + if (keyValues) { + return generatePointerHash([tableName, ...keyValues]) + } + } + + /** + * Calculates a hash for DynamoDB operations that have keys provided (UpdateItem, DeleteItem). + * @param {string} tableName - Name of the DynamoDB table. + * @param {Object} keysObject - Object containing primary key/value attributes in DynamoDB format. + * (e.g., { userId: { S: "123" }, sortKey: { N: "456" } }) + * @returns {string|undefined} Hash value combining table name and primary key/value pairs, or undefined if unable. + * + * @example + * calculateHashWithKnownKeys('UserTable', { userId: { S: "user123" }, timestamp: { N: "1234567" } }) + */ + static calculateHashWithKnownKeys (tableName, keysObject) { + if (!tableName || !keysObject) { + log.debug('Unable to calculate hash because missing parameters') + return + } + const keyNamesSet = new Set(Object.keys(keysObject)) + const keyValues = extractPrimaryKeys(keyNamesSet, keysObject) + if (keyValues) { + return generatePointerHash([tableName, ...keyValues]) + } + } } module.exports = DynamoDb diff --git a/packages/datadog-plugin-aws-sdk/src/services/s3.js b/packages/datadog-plugin-aws-sdk/src/services/s3.js index 5fcfb6ed165..d860223d67b 100644 --- a/packages/datadog-plugin-aws-sdk/src/services/s3.js +++ b/packages/datadog-plugin-aws-sdk/src/services/s3.js @@ -2,7 +2,7 @@ const BaseAwsSdkPlugin = require('../base') const log = require('../../../dd-trace/src/log') -const { generatePointerHash } = require('../../../dd-trace/src/util') +const { generatePointerHash } = require('../util') const { S3_PTR_KIND, SPAN_POINTER_DIRECTION } = require('../../../dd-trace/src/constants') class S3 extends BaseAwsSdkPlugin { diff --git a/packages/datadog-plugin-aws-sdk/src/util.js b/packages/datadog-plugin-aws-sdk/src/util.js new file mode 100644 index 00000000000..4bb7e86c8cd --- /dev/null +++ b/packages/datadog-plugin-aws-sdk/src/util.js @@ -0,0 +1,92 @@ +'use strict' + +const crypto = require('crypto') +const log = require('../../dd-trace/src/log') + +/** + * Generates a unique hash from an array of strings by joining them with | before hashing. + * Used to uniquely identify AWS requests for span pointers. + * @param {string[]} components - Array of strings to hash + * @returns {string} A 32-character hash uniquely identifying the components + */ +function generatePointerHash (components) { + // If passing S3's ETag as a component, make sure any quotes have already been removed! + const dataToHash = components.join('|') + const hash = crypto.createHash('sha256').update(dataToHash).digest('hex') + return hash.substring(0, 32) +} + +/** + * Encodes a DynamoDB attribute value to Buffer for span pointer hashing. + * @param {Object} valueObject - DynamoDB value in AWS format ({ S: string } or { N: string } or { B: Buffer }) + * @returns {Buffer|undefined} Encoded value as Buffer, or undefined if invalid input. + * + * @example + * encodeValue({ S: "user123" }) -> Buffer("user123") + * encodeValue({ N: "42" }) -> Buffer("42") + * encodeValue({ B: Buffer([1, 2, 3]) }) -> Buffer([1, 2, 3]) + */ +function encodeValue (valueObject) { + if (!valueObject) { + return + } + + try { + const type = Object.keys(valueObject)[0] + const value = valueObject[type] + + switch (type) { + case 'S': + return Buffer.from(value) + case 'N': + return Buffer.from(value.toString()) + case 'B': + return Buffer.isBuffer(value) ? value : Buffer.from(value) + default: + log.debug(`Found unknown type while trying to create DynamoDB span pointer: ${type}`) + } + } catch (err) { + log.debug(`Failed to encode value while trying to create DynamoDB span pointer: ${err.message}`) + } +} + +/** + * Extracts and encodes primary key values from a DynamoDB item. + * Handles tables with single-key and two-key scenarios. + * + * @param {Set} keySet - Set of primary key names. + * @param {Object} keyValuePairs - Object containing key/value pairs. + * @returns {Array|undefined} [key1Name, key1Value, key2Name, key2Value], or undefined if invalid input. + * key2 entries are empty strings in the single-key case. + * @example + * extractPrimaryKeys(new Set(['userId']), {userId: {S: "user123"}}) + * // Returns ["userId", Buffer("user123"), "", ""] + * extractPrimaryKeys(new Set(['userId', 'timestamp']), {userId: {S: "user123"}, timestamp: {N: "1234}}) + * // Returns ["timestamp", Buffer.from("1234"), "userId", Buffer.from("user123")] + */ +const extractPrimaryKeys = (keySet, keyValuePairs) => { + const keyNames = Array.from(keySet) + if (keyNames.length === 0) { + return + } + + if (keyNames.length === 1) { + const value = encodeValue(keyValuePairs[keyNames[0]]) + if (value) { + return [keyNames[0], value, '', ''] + } + } else { + const [key1, key2] = keyNames.sort() + const value1 = encodeValue(keyValuePairs[key1]) + const value2 = encodeValue(keyValuePairs[key2]) + if (value1 && value2) { + return [key1, value1, key2, value2] + } + } +} + +module.exports = { + generatePointerHash, + encodeValue, + extractPrimaryKeys +} diff --git a/packages/datadog-plugin-aws-sdk/test/dynamodb.spec.js b/packages/datadog-plugin-aws-sdk/test/dynamodb.spec.js new file mode 100644 index 00000000000..7fba9babfb0 --- /dev/null +++ b/packages/datadog-plugin-aws-sdk/test/dynamodb.spec.js @@ -0,0 +1,831 @@ +'use strict' + +const agent = require('../../dd-trace/test/plugins/agent') +const { setup } = require('./spec_helpers') +const axios = require('axios') +const { DYNAMODB_PTR_KIND, SPAN_POINTER_DIRECTION } = require('../../dd-trace/src/constants') +const DynamoDb = require('../src/services/dynamodb') +const { generatePointerHash } = require('../src/util') + +/* eslint-disable no-console */ +async function resetLocalStackDynamo () { + try { + await axios.post('http://localhost:4566/reset') + console.log('LocalStack Dynamo reset successful') + } catch (error) { + console.error('Error resetting LocalStack Dynamo:', error.message) + } +} + +describe('Plugin', () => { + describe('aws-sdk (dynamodb)', function () { + setup() + + withVersions('aws-sdk', ['aws-sdk', '@aws-sdk/smithy-client'], (version, moduleName) => { + let tracer + let AWS + let dynamo + + const dynamoClientName = moduleName === '@aws-sdk/smithy-client' ? '@aws-sdk/client-dynamodb' : 'aws-sdk' + + // Test both cases: tables with only partition key and with partition+sort key. + const oneKeyTableName = 'OneKeyTable' + const twoKeyTableName = 'TwoKeyTable' + + describe('with configuration', () => { + before(() => { + tracer = require('../../dd-trace') + tracer.init() + return agent.load('aws-sdk') + }) + + before(async () => { + AWS = require(`../../../versions/${dynamoClientName}@${version}`).get() + dynamo = new AWS.DynamoDB({ endpoint: 'http://127.0.0.1:4566', region: 'us-east-1' }) + + const deleteTable = async (tableName) => { + if (dynamoClientName === '@aws-sdk/client-dynamodb') { + try { + await dynamo.deleteTable({ TableName: tableName }) + await new Promise(resolve => setTimeout(resolve, 1000)) + } catch (err) { + if (err.name !== 'ResourceNotFoundException') { + throw err + } + } + } else { + try { + if (typeof dynamo.deleteTable({}).promise === 'function') { + await dynamo.deleteTable({ TableName: tableName }).promise() + await dynamo.waitFor('tableNotExists', { TableName: tableName }).promise() + } else { + await new Promise((resolve, reject) => { + dynamo.deleteTable({ TableName: tableName }, (err) => { + if (err && err.code !== 'ResourceNotFoundException') { + reject(err) + } else { + resolve() + } + }) + }) + } + } catch (err) { + if (err.code !== 'ResourceNotFoundException') { + throw err + } + } + } + } + + const createTable = async (params) => { + if (dynamoClientName === '@aws-sdk/client-dynamodb') { + await dynamo.createTable(params) + } else { + if (typeof dynamo.createTable({}).promise === 'function') { + await dynamo.createTable(params).promise() + } else { + await new Promise((resolve, reject) => { + dynamo.createTable(params, (err, data) => { + if (err) reject(err) + else resolve(data) + }) + }) + } + } + } + + // Delete existing tables + await deleteTable(oneKeyTableName) + await deleteTable(twoKeyTableName) + + // Create tables + await createTable({ + TableName: oneKeyTableName, + KeySchema: [{ AttributeName: 'name', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'name', AttributeType: 'S' }], + ProvisionedThroughput: { ReadCapacityUnits: 5, WriteCapacityUnits: 5 } + }) + + await createTable({ + TableName: twoKeyTableName, + KeySchema: [ + { AttributeName: 'id', KeyType: 'HASH' }, + { AttributeName: 'binary', KeyType: 'RANGE' } + ], + AttributeDefinitions: [ + { AttributeName: 'id', AttributeType: 'N' }, + { AttributeName: 'binary', AttributeType: 'B' } + ], + ProvisionedThroughput: { ReadCapacityUnits: 5, WriteCapacityUnits: 5 } + }) + }) + + after(async () => { + await resetLocalStackDynamo() + return agent.close({ ritmReset: false }) + }) + + describe('span pointers', () => { + beforeEach(() => { + DynamoDb.dynamoPrimaryKeyConfig = null + delete process.env.DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS + }) + + function testSpanPointers ({ expectedHashes, operation }) { + let expectedLength = 0 + if (expectedHashes) { + expectedLength = Array.isArray(expectedHashes) ? expectedHashes.length : 1 + } + return (done) => { + operation((err) => { + if (err) { + return done(err) + } + + agent.use(traces => { + try { + const span = traces[0][0] + const links = JSON.parse(span.meta?.['_dd.span_links'] || '[]') + expect(links).to.have.lengthOf(expectedLength) + + if (expectedHashes) { + if (Array.isArray(expectedHashes)) { + expectedHashes.forEach((hash, i) => { + expect(links[i].attributes['ptr.hash']).to.equal(hash) + }) + } else { + expect(links[0].attributes).to.deep.equal({ + 'ptr.kind': DYNAMODB_PTR_KIND, + 'ptr.dir': SPAN_POINTER_DIRECTION.DOWNSTREAM, + 'ptr.hash': expectedHashes, + 'link.kind': 'span-pointer' + }) + } + } + return done() + } catch (error) { + return done(error) + } + }).catch(error => { + done(error) + }) + }) + } + } + + describe('1-key table', () => { + it('should add span pointer for putItem when config is valid', () => { + testSpanPointers({ + expectedHashes: '27f424c8202ab35efbf8b0b444b1928f', + operation: (callback) => { + process.env.DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS = + '{"OneKeyTable": ["name"]}' + dynamo.putItem({ + TableName: oneKeyTableName, + Item: { + name: { S: 'test1' }, + foo: { S: 'bar1' } + } + }, callback) + } + }) + }) + + it('should not add links or error for putItem when config is invalid', () => { + testSpanPointers({ + operation: (callback) => { + process.env.DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS = '{"DifferentTable": ["test"]}' + dynamo.putItem({ + TableName: oneKeyTableName, + Item: { + name: { S: 'test2' }, + foo: { S: 'bar2' } + } + }, callback) + } + }) + }) + + it('should not add links or error for putItem when config is missing', () => { + testSpanPointers({ + operation: (callback) => { + process.env.DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS = null + dynamo.putItem({ + TableName: oneKeyTableName, + Item: { + name: { S: 'test3' }, + foo: { S: 'bar3' } + } + }, callback) + } + }) + }) + + it('should add span pointer for updateItem', () => { + testSpanPointers({ + expectedHashes: '27f424c8202ab35efbf8b0b444b1928f', + operation: (callback) => { + dynamo.updateItem({ + TableName: oneKeyTableName, + Key: { name: { S: 'test1' } }, + AttributeUpdates: { + foo: { + Action: 'PUT', + Value: { S: 'bar4' } + } + } + }, callback) + } + }) + }) + + it('should add span pointer for deleteItem', () => { + testSpanPointers({ + expectedHashes: '27f424c8202ab35efbf8b0b444b1928f', + operation: (callback) => { + dynamo.deleteItem({ + TableName: oneKeyTableName, + Key: { name: { S: 'test1' } } + }, callback) + } + }) + }) + + it('should add span pointers for transactWriteItems', () => { + // Skip for older versions that don't support transactWriteItems + if (typeof dynamo.transactWriteItems !== 'function') { + return + } + testSpanPointers({ + expectedHashes: [ + '955ab85fc7d1d63fe4faf18696514f13', + '856c95a173d9952008a70283175041fc', + '9682c132f1900106a792f166d0619e0b' + ], + operation: (callback) => { + process.env.DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS = '{"OneKeyTable": ["name"]}' + dynamo.transactWriteItems({ + TransactItems: [ + { + Put: { + TableName: oneKeyTableName, + Item: { + name: { S: 'test4' }, + foo: { S: 'bar4' } + } + } + }, + { + Update: { + TableName: oneKeyTableName, + Key: { name: { S: 'test2' } }, + UpdateExpression: 'SET foo = :newfoo', + ExpressionAttributeValues: { + ':newfoo': { S: 'bar5' } + } + } + }, + { + Delete: { + TableName: oneKeyTableName, + Key: { name: { S: 'test3' } } + } + } + ] + }, callback) + } + }) + }) + + it('should add span pointers for batchWriteItem', () => { + // Skip for older versions that don't support batchWriteItem + if (typeof dynamo.batchWriteItem !== 'function') { + return + } + testSpanPointers({ + expectedHashes: [ + '955ab85fc7d1d63fe4faf18696514f13', + '9682c132f1900106a792f166d0619e0b' + ], + operation: (callback) => { + process.env.DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS = '{"OneKeyTable": ["name"]}' + dynamo.batchWriteItem({ + RequestItems: { + [oneKeyTableName]: [ + { + PutRequest: { + Item: { + name: { S: 'test4' }, + foo: { S: 'bar4' } + } + } + }, + { + DeleteRequest: { + Key: { + name: { S: 'test3' } + } + } + } + ] + } + }, callback) + } + }) + }) + }) + + describe('2-key table', () => { + it('should add span pointer for putItem when config is valid', () => { + testSpanPointers({ + expectedHashes: 'cc32f0e49ee05d3f2820ccc999bfe306', + operation: (callback) => { + process.env.DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS = '{"TwoKeyTable": ["id", "binary"]}' + dynamo.putItem({ + TableName: twoKeyTableName, + Item: { + id: { N: '1' }, + binary: { B: Buffer.from('Hello world 1') } + } + }, callback) + } + }) + }) + + it('should not add links or error for putItem when config is invalid', () => { + testSpanPointers({ + operation: (callback) => { + process.env.DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS = '{"DifferentTable": ["test"]}' + dynamo.putItem({ + TableName: twoKeyTableName, + Item: { + id: { N: '2' }, + binary: { B: Buffer.from('Hello world 2') } + } + }, callback) + } + }) + }) + + it('should not add links or error for putItem when config is missing', () => { + testSpanPointers({ + operation: (callback) => { + process.env.DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS = null + dynamo.putItem({ + TableName: twoKeyTableName, + Item: { + id: { N: '3' }, + binary: { B: Buffer.from('Hello world 3') } + } + }, callback) + } + }) + }) + + it('should add span pointer for updateItem', function (done) { + dynamo.putItem({ + TableName: twoKeyTableName, + Item: { + id: { N: '100' }, + binary: { B: Buffer.from('abc') } + } + }, async function (err) { + if (err) { + return done(err) + } + await new Promise(resolve => setTimeout(resolve, 100)) + testSpanPointers({ + expectedHashes: '5dac7d25254d596482a3c2c187e51046', + operation: (callback) => { + dynamo.updateItem({ + TableName: twoKeyTableName, + Key: { + id: { N: '100' }, + binary: { B: Buffer.from('abc') } + }, + AttributeUpdates: { + someOtherField: { + Action: 'PUT', + Value: { S: 'new value' } + } + } + }, callback) + } + })(done) + }) + }) + + it('should add span pointer for deleteItem', function (done) { + dynamo.putItem({ + TableName: twoKeyTableName, + Item: { + id: { N: '200' }, + binary: { B: Buffer.from('Hello world') } + } + }, async function (err) { + if (err) return done(err) + await new Promise(resolve => setTimeout(resolve, 100)) + testSpanPointers({ + expectedHashes: 'c356b0dd48c734d889e95122750c2679', + operation: (callback) => { + dynamo.deleteItem({ + TableName: twoKeyTableName, + Key: { + id: { N: '200' }, + binary: { B: Buffer.from('Hello world') } + } + }, callback) + } + })(done) + }) + }) + + it('should add span pointers for transactWriteItems', () => { + // Skip for older versions that don't support transactWriteItems + if (typeof dynamo.transactWriteItems !== 'function') { + return + } + testSpanPointers({ + expectedHashes: [ + 'dd071963cd90e4b3088043f0b9a9f53c', + '7794824f72d673ac7844353bc3ea25d9', + '8a6f801cc4e7d1d5e0dd37e0904e6316' + ], + operation: (callback) => { + process.env.DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS = '{"TwoKeyTable": ["id", "binary"]}' + dynamo.transactWriteItems({ + TransactItems: [ + { + Put: { + TableName: twoKeyTableName, + Item: { + id: { N: '4' }, + binary: { B: Buffer.from('Hello world 4') } + } + } + }, + { + Update: { + TableName: twoKeyTableName, + Key: { + id: { N: '2' }, + binary: { B: Buffer.from('Hello world 2') } + }, + AttributeUpdates: { + someOtherField: { + Action: 'PUT', + Value: { S: 'new value' } + } + } + } + }, + { + Delete: { + TableName: twoKeyTableName, + Key: { + id: { N: '3' }, + binary: { B: Buffer.from('Hello world 3') } + } + } + } + ] + }, callback) + } + }) + }) + + it('should add span pointers for batchWriteItem', () => { + // Skip for older versions that don't support batchWriteItem + if (typeof dynamo.batchWriteItem !== 'function') { + return + } + testSpanPointers({ + expectedHashes: [ + '1f64650acbe1ae4d8413049c6bd9bbe8', + '8a6f801cc4e7d1d5e0dd37e0904e6316' + ], + operation: (callback) => { + process.env.DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS = '{"TwoKeyTable": ["id", "binary"]}' + dynamo.batchWriteItem({ + RequestItems: { + [twoKeyTableName]: [ + { + PutRequest: { + Item: { + id: { N: '5' }, + binary: { B: Buffer.from('Hello world 5') } + } + } + }, + { + DeleteRequest: { + Key: { + id: { N: '3' }, + binary: { B: Buffer.from('Hello world 3') } + } + } + } + ] + } + }, callback) + } + }) + }) + }) + }) + }) + }) + + describe('getPrimaryKeyConfig', () => { + let dynamoDbInstance + + beforeEach(() => { + dynamoDbInstance = new DynamoDb() + dynamoDbInstance.dynamoPrimaryKeyConfig = null + dynamoDbInstance._tracerConfig = {} + }) + + it('should return cached config if available', () => { + const cachedConfig = { Table1: new Set(['key1']) } + dynamoDbInstance.dynamoPrimaryKeyConfig = cachedConfig + + const result = dynamoDbInstance.getPrimaryKeyConfig() + expect(result).to.equal(cachedConfig) + }) + + it('should return undefined when config str is missing', () => { + const result = dynamoDbInstance.getPrimaryKeyConfig() + expect(result).to.be.undefined + }) + + it('should parse valid config with single table', () => { + const configStr = '{"Table1": ["key1", "key2"]}' + dynamoDbInstance._tracerConfig = { aws: { dynamoDb: { tablePrimaryKeys: configStr } } } + + const result = dynamoDbInstance.getPrimaryKeyConfig() + expect(result).to.deep.equal({ + Table1: new Set(['key1', 'key2']) + }) + }) + + it('should parse valid config with multiple tables', () => { + const configStr = '{"Table1": ["key1"], "Table2": ["key2", "key3"]}' + dynamoDbInstance._tracerConfig = { aws: { dynamoDb: { tablePrimaryKeys: configStr } } } + + const result = dynamoDbInstance.getPrimaryKeyConfig() + expect(result).to.deep.equal({ + Table1: new Set(['key1']), + Table2: new Set(['key2', 'key3']) + }) + }) + }) + + describe('calculatePutItemHash', () => { + it('generates correct hash for single string key', () => { + const tableName = 'UserTable' + const item = { userId: { S: 'user123' }, name: { S: 'John' } } + const keyConfig = { UserTable: new Set(['userId']) } + + const actualHash = DynamoDb.calculatePutItemHash(tableName, item, keyConfig) + const expectedHash = generatePointerHash([tableName, 'userId', 'user123', '', '']) + expect(actualHash).to.equal(expectedHash) + }) + + it('generates correct hash for single number key', () => { + const tableName = 'OrderTable' + const item = { orderId: { N: '98765' }, total: { N: '50.00' } } + const keyConfig = { OrderTable: new Set(['orderId']) } + + const actualHash = DynamoDb.calculatePutItemHash(tableName, item, keyConfig) + const expectedHash = generatePointerHash([tableName, 'orderId', '98765', '', '']) + expect(actualHash).to.equal(expectedHash) + }) + + it('generates correct hash for single binary key', () => { + const tableName = 'BinaryTable' + const binaryData = Buffer.from([1, 2, 3]) + const item = { binaryId: { B: binaryData }, data: { S: 'test' } } + const keyConfig = { BinaryTable: new Set(['binaryId']) } + + const actualHash = DynamoDb.calculatePutItemHash(tableName, item, keyConfig) + const expectedHash = generatePointerHash([tableName, 'binaryId', binaryData, '', '']) + expect(actualHash).to.equal(expectedHash) + }) + + it('generates correct hash for string-string key', () => { + const tableName = 'UserEmailTable' + const item = { + userId: { S: 'user123' }, + email: { S: 'test@example.com' }, + verified: { BOOL: true } + } + const keyConfig = { UserEmailTable: new Set(['userId', 'email']) } + + const actualHash = DynamoDb.calculatePutItemHash(tableName, item, keyConfig) + const expectedHash = generatePointerHash([tableName, 'email', 'test@example.com', 'userId', 'user123']) + expect(actualHash).to.equal(expectedHash) + }) + + it('generates correct hash for string-number key', () => { + const tableName = 'UserActivityTable' + const item = { + userId: { S: 'user123' }, + timestamp: { N: '1234567' }, + action: { S: 'login' } + } + const keyConfig = { UserActivityTable: new Set(['userId', 'timestamp']) } + + const actualHash = DynamoDb.calculatePutItemHash(tableName, item, keyConfig) + const expectedHash = generatePointerHash([tableName, 'timestamp', '1234567', 'userId', 'user123']) + expect(actualHash).to.equal(expectedHash) + }) + + it('generates correct hash for binary-binary key', () => { + const tableName = 'BinaryTable' + const binary1 = Buffer.from('abc') + const binary2 = Buffer.from('1ef230') + const item = { + key1: { B: binary1 }, + key2: { B: binary2 }, + data: { S: 'test' } + } + const keyConfig = { BinaryTable: new Set(['key1', 'key2']) } + + const actualHash = DynamoDb.calculatePutItemHash(tableName, item, keyConfig) + const expectedHash = generatePointerHash([tableName, 'key1', binary1, 'key2', binary2]) + expect(actualHash).to.equal(expectedHash) + }) + + it('generates unique hashes for different tables', () => { + const item = { userId: { S: 'user123' } } + const keyConfig = { + Table1: new Set(['userId']), + Table2: new Set(['userId']) + } + + const hash1 = DynamoDb.calculatePutItemHash('Table1', item, keyConfig) + const hash2 = DynamoDb.calculatePutItemHash('Table2', item, keyConfig) + expect(hash1).to.not.equal(hash2) + }) + + describe('edge cases', () => { + it('returns undefined for unknown table', () => { + const tableName = 'UnknownTable' + const item = { userId: { S: 'user123' } } + const keyConfig = { KnownTable: new Set(['userId']) } + + const result = DynamoDb.calculatePutItemHash(tableName, item, keyConfig) + expect(result).to.be.undefined + }) + + it('returns undefined for empty primary key config', () => { + const tableName = 'UserTable' + const item = { userId: { S: 'user123' } } + + const result = DynamoDb.calculatePutItemHash(tableName, item, {}) + expect(result).to.be.undefined + }) + + it('returns undefined for invalid primary key config', () => { + const tableName = 'UserTable' + const item = { userId: { S: 'user123' } } + const invalidConfig = { UserTable: ['userId'] } // Array instead of Set + + const result = DynamoDb.calculatePutItemHash(tableName, item, invalidConfig) + expect(result).to.be.undefined + }) + + it('returns undefined when missing attributes in item', () => { + const tableName = 'UserTable' + const item = { someOtherField: { S: 'value' } } + const keyConfig = { UserTable: new Set(['userId']) } + + const actualHash = DynamoDb.calculatePutItemHash(tableName, item, keyConfig) + expect(actualHash).to.be.undefined + }) + + it('returns undefined for Set with more than 2 keys', () => { + const tableName = 'TestTable' + const item = { key1: { S: 'value1' }, key2: { S: 'value2' }, key3: { S: 'value3' } } + const keyConfig = { TestTable: new Set(['key1', 'key2', 'key3']) } + + const result = DynamoDb.calculatePutItemHash(tableName, item, keyConfig) + expect(result).to.be.undefined + }) + + it('returns undefined for empty keyConfig', () => { + const result = DynamoDb.calculatePutItemHash('TestTable', {}, {}) + expect(result).to.be.undefined + }) + + it('returns undefined for undefined keyConfig', () => { + const result = DynamoDb.calculatePutItemHash('TestTable', {}, undefined) + expect(result).to.be.undefined + }) + }) + }) + + describe('calculateHashWithKnownKeys', () => { + it('generates correct hash for single string key', () => { + const tableName = 'UserTable' + const keys = { userId: { S: 'user123' } } + const actualHash = DynamoDb.calculateHashWithKnownKeys(tableName, keys) + const expectedHash = generatePointerHash([tableName, 'userId', 'user123', '', '']) + expect(actualHash).to.equal(expectedHash) + }) + + it('generates correct hash for single number key', () => { + const tableName = 'OrderTable' + const keys = { orderId: { N: '98765' } } + const actualHash = DynamoDb.calculateHashWithKnownKeys(tableName, keys) + const expectedHash = generatePointerHash([tableName, 'orderId', '98765', '', '']) + expect(actualHash).to.equal(expectedHash) + }) + + it('generates correct hash for single binary key', () => { + const tableName = 'BinaryTable' + const binaryData = Buffer.from([1, 2, 3]) + const keys = { binaryId: { B: binaryData } } + const actualHash = DynamoDb.calculateHashWithKnownKeys(tableName, keys) + const expectedHash = generatePointerHash([tableName, 'binaryId', binaryData, '', '']) + expect(actualHash).to.equal(expectedHash) + }) + + it('generates correct hash for string-string key', () => { + const tableName = 'UserEmailTable' + const keys = { + userId: { S: 'user123' }, + email: { S: 'test@example.com' } + } + const actualHash = DynamoDb.calculateHashWithKnownKeys(tableName, keys) + const expectedHash = generatePointerHash([tableName, 'email', 'test@example.com', 'userId', 'user123']) + expect(actualHash).to.equal(expectedHash) + }) + + it('generates correct hash for string-number key', () => { + const tableName = 'UserActivityTable' + const keys = { + userId: { S: 'user123' }, + timestamp: { N: '1234567' } + } + const actualHash = DynamoDb.calculateHashWithKnownKeys(tableName, keys) + const expectedHash = generatePointerHash([tableName, 'timestamp', '1234567', 'userId', 'user123']) + expect(actualHash).to.equal(expectedHash) + }) + + it('generates correct hash for binary-binary key', () => { + const tableName = 'BinaryTable' + const binary1 = Buffer.from('abc') + const binary2 = Buffer.from('1ef230') + const keys = { + key1: { B: binary1 }, + key2: { B: binary2 } + } + const actualHash = DynamoDb.calculateHashWithKnownKeys(tableName, keys) + const expectedHash = generatePointerHash([tableName, 'key1', binary1, 'key2', binary2]) + expect(actualHash).to.equal(expectedHash) + }) + + it('generates unique hashes', () => { + const keys = { userId: { S: 'user123' } } + const hash1 = DynamoDb.calculateHashWithKnownKeys('Table1', keys) + const hash2 = DynamoDb.calculateHashWithKnownKeys('Table2', keys) + expect(hash1).to.not.equal(hash2) + }) + + describe('edge cases', () => { + it('handles empty keys object', () => { + const tableName = 'UserTable' + const hash = DynamoDb.calculateHashWithKnownKeys(tableName, {}) + expect(hash).to.be.undefined + }) + + it('handles invalid key types', () => { + const tableName = 'UserTable' + const keys = { userId: { INVALID: 'user123' } } + const hash = DynamoDb.calculateHashWithKnownKeys(tableName, keys) + expect(hash).to.be.undefined + }) + + it('handles null keys object', () => { + const hash = DynamoDb.calculateHashWithKnownKeys('TestTable', null) + expect(hash).to.be.undefined + }) + + it('handles undefined keys object', () => { + const hash = DynamoDb.calculateHashWithKnownKeys('TestTable', undefined) + expect(hash).to.be.undefined + }) + + it('handles mixed valid and invalid key types', () => { + const keys = { + validKey: { S: 'test' }, + invalidKey: { INVALID: 'value' } + } + const hash = DynamoDb.calculateHashWithKnownKeys('TestTable', keys) + expect(hash).to.be.undefined + }) + }) + }) + }) +}) diff --git a/packages/datadog-plugin-aws-sdk/test/util.spec.js b/packages/datadog-plugin-aws-sdk/test/util.spec.js new file mode 100644 index 00000000000..68bf57a7bfc --- /dev/null +++ b/packages/datadog-plugin-aws-sdk/test/util.spec.js @@ -0,0 +1,213 @@ +const { generatePointerHash, encodeValue, extractPrimaryKeys } = require('../src/util') + +describe('generatePointerHash', () => { + describe('should generate a valid hash for S3 object with', () => { + it('basic values', () => { + const hash = generatePointerHash(['some-bucket', 'some-key.data', 'ab12ef34']) + expect(hash).to.equal('e721375466d4116ab551213fdea08413') + }) + + it('non-ascii key', () => { + const hash = generatePointerHash(['some-bucket', 'some-key.你好', 'ab12ef34']) + expect(hash).to.equal('d1333a04b9928ab462b5c6cadfa401f4') + }) + + it('multipart-upload', () => { + const hash = generatePointerHash(['some-bucket', 'some-key.data', 'ab12ef34-5']) + expect(hash).to.equal('2b90dffc37ebc7bc610152c3dc72af9f') + }) + }) + + describe('should generate a valid hash for DynamoDB item with', () => { + it('one string primary key', () => { + const hash = generatePointerHash(['some-table', 'some-key', 'some-value', '', '']) + expect(hash).to.equal('7f1aee721472bcb48701d45c7c7f7821') + }) + + it('one buffered binary primary key', () => { + const hash = generatePointerHash(['some-table', 'some-key', Buffer.from('some-value'), '', '']) + expect(hash).to.equal('7f1aee721472bcb48701d45c7c7f7821') + }) + + it('one number primary key', () => { + const hash = generatePointerHash(['some-table', 'some-key', '123.456', '', '']) + expect(hash).to.equal('434a6dba3997ce4dbbadc98d87a0cc24') + }) + + it('one buffered number primary key', () => { + const hash = generatePointerHash(['some-table', 'some-key', Buffer.from('123.456'), '', '']) + expect(hash).to.equal('434a6dba3997ce4dbbadc98d87a0cc24') + }) + + it('string and number primary key', () => { + // sort primary keys lexicographically + const hash = generatePointerHash(['some-table', 'other-key', '123', 'some-key', 'some-value']) + expect(hash).to.equal('7aa1b80b0e49bd2078a5453399f4dd67') + }) + + it('buffered string and number primary key', () => { + const hash = generatePointerHash([ + 'some-table', + 'other-key', + Buffer.from('123'), + 'some-key', Buffer.from('some-value') + ]) + expect(hash).to.equal('7aa1b80b0e49bd2078a5453399f4dd67') + }) + }) +}) + +describe('encodeValue', () => { + describe('basic type handling', () => { + it('handles string (S) type correctly', () => { + const result = encodeValue({ S: 'hello world' }) + expect(Buffer.isBuffer(result)).to.be.true + expect(result).to.deep.equal(Buffer.from('hello world')) + }) + + it('handles number (N) as string type correctly', () => { + const result = encodeValue({ N: '123.45' }) + expect(Buffer.isBuffer(result)).to.be.true + expect(result).to.deep.equal(Buffer.from('123.45')) + }) + + it('handles number (N) as type string or number the same', () => { + const result1 = encodeValue({ N: 456.78 }) + const result2 = encodeValue({ N: '456.78' }) + expect(Buffer.isBuffer(result1)).to.be.true + expect(result1).to.deep.equal(result2) + }) + + it('handles binary (B) type correctly', () => { + const binaryData = Buffer.from([1, 2, 3]) + const result = encodeValue({ B: binaryData }) + expect(Buffer.isBuffer(result)).to.be.true + expect(result).to.deep.equal(binaryData) + }) + }) + + describe('edge cases', () => { + it('returns undefined for null input', () => { + const result = encodeValue(null) + expect(result).to.be.undefined + }) + + it('returns undefined for undefined input', () => { + const result = encodeValue(undefined) + expect(result).to.be.undefined + }) + + it('returns undefined for unsupported type', () => { + const result = encodeValue({ A: 'abc' }) + expect(result).to.be.undefined + }) + + it('returns undefined for malformed input', () => { + const result = encodeValue({}) + expect(result).to.be.undefined + }) + + it('handles empty string values', () => { + const result = encodeValue({ S: '' }) + expect(Buffer.isBuffer(result)).to.be.true + expect(result.length).to.equal(0) + }) + + it('handles empty buffer', () => { + const result = encodeValue({ B: Buffer.from([]) }) + expect(Buffer.isBuffer(result)).to.be.true + expect(result.length).to.equal(0) + }) + }) +}) + +describe('extractPrimaryKeys', () => { + describe('single key table', () => { + it('handles string key', () => { + const keySet = new Set(['userId']) + const item = { userId: { S: 'user123' } } + const result = extractPrimaryKeys(keySet, item) + expect(result).to.deep.equal(['userId', Buffer.from('user123'), '', '']) + }) + + it('handles number key', () => { + const keySet = new Set(['timestamp']) + const item = { timestamp: { N: '1234567' } } + const result = extractPrimaryKeys(keySet, item) + expect(result).to.deep.equal(['timestamp', Buffer.from('1234567'), '', '']) + }) + + it('handles binary key', () => { + const keySet = new Set(['binaryId']) + const binaryData = Buffer.from([1, 2, 3]) + const item = { binaryId: { B: binaryData } } + const result = extractPrimaryKeys(keySet, item) + expect(result).to.deep.equal(['binaryId', binaryData, '', '']) + }) + }) + + describe('double key table', () => { + it('handles and sorts string-string keys', () => { + const keySet = new Set(['userId', 'email']) + const item = { + userId: { S: 'user123' }, + email: { S: 'test@example.com' } + } + const result = extractPrimaryKeys(keySet, item) + expect(result).to.deep.equal(['email', Buffer.from('test@example.com'), 'userId', Buffer.from('user123')]) + }) + + it('handles and sorts string-number keys', () => { + const keySet = new Set(['timestamp', 'userId']) + const item = { + timestamp: { N: '1234567' }, + userId: { S: 'user123' } + } + const result = extractPrimaryKeys(keySet, item) + expect(result).to.deep.equal(['timestamp', Buffer.from('1234567'), 'userId', Buffer.from('user123')]) + }) + }) + + describe('edge cases', () => { + it('returns undefined when missing values', () => { + const keySet = new Set(['userId', 'timestamp']) + const item = { userId: { S: 'user123' } } // timestamp missing + const result = extractPrimaryKeys(keySet, item) + expect(result).to.be.undefined + }) + + it('returns undefined when invalid value types', () => { + const keySet = new Set(['userId', 'timestamp']) + const item = { + userId: { S: 'user123' }, + timestamp: { INVALID: '1234567' } + } + const result = extractPrimaryKeys(keySet, item) + expect(result).to.be.undefined + }) + + it('handles empty Set input', () => { + const result = extractPrimaryKeys(new Set([]), {}) + expect(result).to.be.undefined + }) + + it('returns undefined when null values in item', () => { + const keySet = new Set(['key1', 'key2']) + const item = { + key1: null, + key2: { S: 'value2' } + } + const result = extractPrimaryKeys(keySet, item) + expect(result).to.be.undefined + }) + + it('returns undefined when undefined values in item', () => { + const keySet = new Set(['key1', 'key2']) + const item = { + key2: { S: 'value2' } + } + const result = extractPrimaryKeys(keySet, item) + expect(result).to.be.undefined + }) + }) +}) diff --git a/packages/dd-trace/src/config.js b/packages/dd-trace/src/config.js index a46cc3153fc..a16df70ee07 100644 --- a/packages/dd-trace/src/config.js +++ b/packages/dd-trace/src/config.js @@ -566,6 +566,7 @@ class Config { this._setValue(defaults, 'url', undefined) this._setValue(defaults, 'version', pkg.version) this._setValue(defaults, 'instrumentation_config_id', undefined) + this._setValue(defaults, 'aws.dynamoDb.tablePrimaryKeys', undefined) } _applyEnvironment () { @@ -590,6 +591,7 @@ class Config { DD_APPSEC_RASP_ENABLED, DD_APPSEC_TRACE_RATE_LIMIT, DD_APPSEC_WAF_TIMEOUT, + DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS, DD_CRASHTRACKING_ENABLED, DD_CODE_ORIGIN_FOR_SPANS_ENABLED, DD_DATA_STREAMS_ENABLED, @@ -879,6 +881,7 @@ class Config { this._setBoolean(env, 'tracing', DD_TRACING_ENABLED) this._setString(env, 'version', DD_VERSION || tags.version) this._setBoolean(env, 'inferredProxyServicesEnabled', DD_TRACE_INFERRED_PROXY_SERVICES_ENABLED) + this._setString(env, 'aws.dynamoDb.tablePrimaryKeys', DD_AWS_SDK_DYNAMODB_TABLE_PRIMARY_KEYS) } _applyOptions (options) { diff --git a/packages/dd-trace/src/constants.js b/packages/dd-trace/src/constants.js index 4e7faf669d4..3c93480df9f 100644 --- a/packages/dd-trace/src/constants.js +++ b/packages/dd-trace/src/constants.js @@ -47,6 +47,7 @@ module.exports = { SCHEMA_NAME: 'schema.name', GRPC_CLIENT_ERROR_STATUSES: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], GRPC_SERVER_ERROR_STATUSES: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], + DYNAMODB_PTR_KIND: 'aws.dynamodb.item', S3_PTR_KIND: 'aws.s3.object', SPAN_POINTER_DIRECTION: Object.freeze({ UPSTREAM: 'u', diff --git a/packages/dd-trace/src/util.js b/packages/dd-trace/src/util.js index 8cfa3d6f58c..de3618fcd27 100644 --- a/packages/dd-trace/src/util.js +++ b/packages/dd-trace/src/util.js @@ -1,6 +1,5 @@ 'use strict' -const crypto = require('crypto') const path = require('path') function isTrue (str) { @@ -78,25 +77,11 @@ function hasOwn (object, prop) { return Object.prototype.hasOwnProperty.call(object, prop) } -/** - * Generates a unique hash from an array of strings by joining them with | before hashing. - * Used to uniquely identify AWS requests for span pointers. - * @param {string[]} components - Array of strings to hash - * @returns {string} A 32-character hash uniquely identifying the components - */ -function generatePointerHash (components) { - // If passing S3's ETag as a component, make sure any quotes have already been removed! - const dataToHash = components.join('|') - const hash = crypto.createHash('sha256').update(dataToHash).digest('hex') - return hash.substring(0, 32) -} - module.exports = { isTrue, isFalse, isError, globMatch, calculateDDBasePath, - hasOwn, - generatePointerHash + hasOwn } diff --git a/packages/dd-trace/test/plugins/externals.json b/packages/dd-trace/test/plugins/externals.json index c3fc12fb176..73a61536476 100644 --- a/packages/dd-trace/test/plugins/externals.json +++ b/packages/dd-trace/test/plugins/externals.json @@ -30,6 +30,10 @@ "name": "@aws-sdk/client-s3", "versions": [">=3"] }, + { + "name": "@aws-sdk/client-dynamodb", + "versions": [">=3"] + }, { "name": "@aws-sdk/client-sfn", "versions": [">=3"] diff --git a/packages/dd-trace/test/util.spec.js b/packages/dd-trace/test/util.spec.js index 40b209a96cf..f32b47c0cee 100644 --- a/packages/dd-trace/test/util.spec.js +++ b/packages/dd-trace/test/util.spec.js @@ -3,7 +3,6 @@ require('./setup/tap') const { isTrue, isFalse, globMatch } = require('../src/util') -const { generatePointerHash } = require('../src/util') const TRUES = [ 1, @@ -69,20 +68,3 @@ describe('util', () => { }) }) }) - -describe('generatePointerHash', () => { - it('should generate a valid hash for a basic S3 object', () => { - const hash = generatePointerHash(['some-bucket', 'some-key.data', 'ab12ef34']) - expect(hash).to.equal('e721375466d4116ab551213fdea08413') - }) - - it('should generate a valid hash for an S3 object with a non-ascii key', () => { - const hash1 = generatePointerHash(['some-bucket', 'some-key.你好', 'ab12ef34']) - expect(hash1).to.equal('d1333a04b9928ab462b5c6cadfa401f4') - }) - - it('should generate a valid hash for multipart-uploaded S3 object', () => { - const hash1 = generatePointerHash(['some-bucket', 'some-key.data', 'ab12ef34-5']) - expect(hash1).to.equal('2b90dffc37ebc7bc610152c3dc72af9f') - }) -})