[#155] Support SQL UID

* Update gRPC idl
* IdGenerator.stringValueOfNext
* Change to spanId string type value
* Support SqlUidMetaData gRPC stub
* Profiler SQL UID by config
* Support UID array id
* sqluid functional test
* Support murmur128 hash algorithm
* Add murmur-128 package to NOTICE
* Add murmur3-128 source code to repository because of  ESM library
* parsing result factory pattern
* Add BytesStringStringValue
* Fix coverage github actions
* Uid annotationKey
* Update PINPOINT_PROFILER_SQL_STAT ENV in README.md

Author: feelform

NOTICE

@@ -146,4 +146,10 @@ spring-projects/spring-framework
 https://github.com/spring-projects/spring-framework
 Apache License 2.0, https://github.com/spring-projects/spring-framework/blob/master/LICENSE.txt
 
-=====
+=====
+
+murmur-128 v1.0.0
+https://github.com/LinusU/murmur-128
+MIT https://github.com/LinusU/murmur-128/blob/master/package.json
+MIT https://github.com/LinusU/fmix/blob/master/package.json
+MIT https://github.com/LinusU/encode-utf8/blob/master/package.json

README.md

@@ -62,6 +62,7 @@ PINPOINT_ENABLE | true | If you set it to false, the agent will not work.
 PINPOINT_CONTAINER | false | Whether to use docker or kubernetes. If the PINPOINT_CONTAINER environment variable is not set, the agent analyzes the'/.dockerenv' and'/proc/self/cgroup' files to determine whether to use the Docker container. If the KUBERNETES_SERVICE_HOST environment variable exists, it is determined that it is the kubernetes environment and changes it to the true value.
 PINPOINT_TRACE_EXCLUSION_URL_PATTERN |  | comma-separated string. ex) `/health_check,/admin/**` or [Unit tests](https://github.com/pinpoint-apm/pinpoint-node-agent/blob/01fcbdefe5a0ffba9c957bee0da3fb7397638182/test/utils/ant-path-matcher.test.js#L332)
 PINPOINT_TRACE_EXCLUSION_URL_CACHE_SIZE | | If the app is designed so that the pathname of the URL is fixed, if the cache size is set, the pathname of the frequently used URL does not match with patterns. In case of using query for pathname like `/user/1000`, cache is unnecessarily. [Unit tests](https://github.com/pinpoint-apm/pinpoint-node-agent/blob/01fcbdefe5a0ffba9c957bee0da3fb7397638182/test/utils/ant-path-matcher.test.js#L447)
+PINPOINT_PROFILER_SQL_STAT | false | SQL uid
 
 ### Agent ID
 The agent ID is used as the identifier per the server or node. You need to set the hostname or node identifier(The maximum length is 24) on the server.

grpc/grpc-idl

@@ -13,6 +13,7 @@ const StringMetaInfo = require('../data/dto/string-meta-info')
 const Span = require('../context/span')
 const SpanChunk = require('../context/span-chunk')
 const SqlMetaData = require('./sql-meta-data')
+const SqlUidMetaData = require('./sql-uid-meta-data')
 
 class DataSender {
   constructor(config, dataSender) {
@@ -45,6 +46,8 @@ class DataSender {
       this.dataSender.sendSpanChunk(data)
     } else if (data instanceof SqlMetaData) {
       this.dataSender.sendSqlMetaInfo(data)
+    } else if (data instanceof SqlUidMetaData) {
+      this.dataSender.sendSqlUidMetaData(data)
     }
   }
 

lib/client/data-sender.js

@@ -73,6 +73,7 @@ class GrpcDataSender {
     this.requestApiMetaData = new GrpcUnaryRPC('requestApiMetaData', this.metadataClient, this.metadataClient.requestApiMetaData, DEFAULT_METADATA_RETRY_DELAY_MILLIS, DEFAULT_METADATA_RETRY_MAX_COUNT)
     this.requestStringMetaData = new GrpcUnaryRPC('requestStringMetaData', this.metadataClient, this.metadataClient.requestStringMetaData, DEFAULT_METADATA_RETRY_DELAY_MILLIS, DEFAULT_METADATA_RETRY_MAX_COUNT)
     this.requestSqlMetaData = new GrpcUnaryRPC('requestSqlMetaData', this.metadataClient, this.metadataClient.requestSqlMetaData, DEFAULT_METADATA_RETRY_DELAY_MILLIS, DEFAULT_METADATA_RETRY_MAX_COUNT)
+    this.requestSqlUidMetaData = new GrpcUnaryRPC('requestSqlUidMetaData', this.metadataClient, this.metadataClient.requestSqlUidMetaData, DEFAULT_METADATA_RETRY_DELAY_MILLIS, DEFAULT_METADATA_RETRY_MAX_COUNT)
   }
 
   initializeSpanStream(collectorIp, collectorSpanPort, headerInterceptor, config) {
@@ -178,6 +179,21 @@ class GrpcDataSender {
     })
   }
 
+  sendSqlUidMetaData(sqlMetaData, callback) {
+    const pSqlMetaData = sqlMetaData.valueOfProtocolBuffer()
+    if (log.isDebug()) {
+      log.debug(`sendSqlMetaInfo sqlMetaData: ${JSON.stringify(pSqlMetaData.toObject())}`)
+    }
+    this.requestSqlUidMetaData.request(pSqlMetaData, (err, response) => {
+      if (err) {
+        log.error(err)
+      }
+      if (callback) {
+        callback(err, response)
+      }
+    })
+  }
+
   sendSpan(span) {
     try {
       const pSpan = span.spanMessage

lib/client/grpc-data-sender.js

@@ -0,0 +1,24 @@
+/**
+ * Pinpoint Node.js Agent
+ * Copyright 2020-present NAVER Corp.
+ * Apache License v2.0
+ */
+
+'use strict'
+const spanMessages = require('../data/v1/Span_pb')
+
+class SqlUidMetaData {
+    constructor(parsingResult) {
+        this.sqlUid = parsingResult.getId()
+        this.sql = parsingResult.getSql()
+    }
+
+    valueOfProtocolBuffer() {
+        const pSqlMetaData = new spanMessages.PSqlUidMetaData()
+        pSqlMetaData.setSqluid(this.sqlUid)
+        pSqlMetaData.setSql(this.sql)
+        return pSqlMetaData
+    }
+}
+
+module.exports = SqlUidMetaData

lib/client/sql-uid-meta-data.js

@@ -57,6 +57,7 @@ const ENV_MAP = {
   traceExclusionUrlPatterns: valueOfString('PINPOINT_TRACE_EXCLUSION_URL_PATTERN'),
   traceExclusionUrlCacheSize: valueOfNumber('PINPOINT_TRACE_EXCLUSION_URL_CACHE_SIZE'),
   traceLocationAndFileNameOfCallSite: valueOfBoolean('PINPOINT_TRACE_LOCATION_AND_FILENAME_OF_CALL_SITE'),
+  profilerSqlStat: valueOfBoolean('PINPOINT_PROFILER_SQL_STAT'),
 }
 
 const CONFIG_FILE_MAP = {
@@ -83,7 +84,8 @@ const CONFIG_FILE_MAP = {
   traceExclusionUrlPatterns: 'trace-exclusion-url.pattern',
   traceExclusionUrlCacheSize: 'trace-exclusion-url.cache-size',
   streamDeadlineMinutesClientSide: 'stream-deadline-minutes.client-side',
-  traceLocationAndFileNameOfCallSite: 'trace-location-and-filename-of-call-site'
+  traceLocationAndFileNameOfCallSite: 'trace-location-and-filename-of-call-site',
+  profilerSqlStat: 'profiler-sql-stat'
 }
 
 let agentConfig = null

lib/config.js

@@ -16,6 +16,7 @@ module.exports = {
 
   SQL_ID: new AnnotationKey(20, "SQL-ID"),
   SQL_BINDVALUE: new AnnotationKey(24, "SQL-BindValue", AnnotationKeyProperty.VIEW_IN_RECORD_SET),
+  SQL_UID: new AnnotationKey(25, 'SQL-UID'),
 
   HTTP_URL: new AnnotationKey(40, 'http.url'),
   HTTP_PARAM: new AnnotationKey(41, 'http.param', AnnotationKeyProperty.VIEW_IN_RECORD_SET),

lib/constant/annotation-key.js

@@ -14,6 +14,10 @@ class IdGenerator {
   get next () {
     return Math.floor(Math.random() * this.MAX_NUM)
   }
+
+  stringValueOfNext() {
+    return this.next.toString()
+  }
 }
 
 module.exports = new IdGenerator()

lib/context/id-generator.js

@@ -13,7 +13,6 @@ const AnnotationKeyUtils = require('./annotation-key-utils')
 const log = require('../utils/logger')
 const sqlMetaDataService = require('../instrumentation/sql/sql-metadata-service')
 const Annotations = require('../instrumentation/context/annotation/annotations')
-const IntStringStringValue = require('../instrumentation/context/annotation/int-string-string-value')
 
 // https://github.com/pinpoint-apm/pinpoint/blob/master/bootstraps/bootstrap-core/src/main/java/com/navercorp/pinpoint/bootstrap/context/SpanEventRecorder.java
 class SpanEventRecorder {
@@ -172,11 +171,7 @@ class SpanEventRecorder {
       bindString = null
     }
 
-    this.recordSqlParam(new IntStringStringValue(parsingResult.getId(), parsingResult.getOutput(), bindString))
-  }
-
-  recordSqlParam(intStringStringValue) {
-    const annotation = Annotations.of(annotationKey.SQL_ID.getCode(), intStringStringValue)
+    const annotation = parsingResult.newSqlAnnotation(bindString)
     this.spanEvent.addAnnotation(annotation)
   }
 

lib/context/span-event-recorder.js

@@ -64,8 +64,8 @@ class TraceContext {
   newTraceObject(sampling) {
     const transactionId = new TransactionId(this.agentInfo.agentId,
       this.agentInfo.agentStartTime.toString())
-    const spanId = IdGenerator.next
-    const traceId = new TraceId(transactionId, spanId.toString())
+    const spanId = IdGenerator.stringValueOfNext()
+    const traceId = new TraceId(transactionId, spanId)
     return this.createTraceObject(traceId, sampling)
   }