Add tools and instructions for REST API traffic replay

.github/dependabot.yml

@@ -35,3 +35,9 @@ updates:
     package-ecosystem: "gomod"
     schedule:
       interval: "weekly"
+
+  - directory: "/hedera-mirror-test/traffic-replay/log-downloader"
+    open-pull-requests-limit: 10
+    package-ecosystem: "npm"
+    schedule:
+      interval: "weekly"

build.gradle.kts

@@ -291,6 +291,10 @@ tasks.register("release") {
             "(?<=\"@hashgraph/(check-state-proof|mirror-rest|mirror-monitor)\",\\s{3,7}\"version\": \")[^\"]+"
         )
         replaceVersion("hedera-mirror-rest/**/openapi.yml", "(?<=^  version: ).+")
+        replaceVersion(
+            "hedera-mirror-test/traffic-replay/log-downloader/package*.json",
+            "(?<=\"@hashgraph/mirror-log-downloader\",\\s{3,7}\"version\": \")[^\"]+"
+        )
     }
 }
 

hedera-mirror-test/traffic-replay/README.md

@@ -0,0 +1,46 @@
+# REST API Traffic Replay
+
+This documents describes how to use the tools and [goreplay](https://github.com/buger/goreplay) to replay REST API
+traffic to a target server of choice.
+
+## Prerequisites
+
+- gcloud - follow the official [installation guide](https://cloud.google.com/sdk/docs/install) to install the gcloud
+  command line tool
+- goreplay - either download the v1.3.3 binary from the official [release](https://github.com/buger/goreplay/releases/tag/1.3.3)
+  or clone the source and build it.
+- nodejs >= 18
+- read access to the google cloud logging service in the project which hosts the GKE cluster with mirrornode REST
+  service
+
+## Preparation
+
+The `log-downloader` tool requires google cloud Application Default Credentials (ADC) to authenticate itself to call
+Google APIs. To use your own user credentials, run
+
+```bash
+gcloud auth application-default login
+```
+
+and follow the prompts to approve and save the credentials to the designated location.
+
+## Replay traffic
+
+There are two simple steps. And step 2 can be repeated to replay the traffic with existing log files in goreplay format.
+
+1. Download and convert the logs using `log-downloader`. The tool has pre-configured logging filter support for
+   `mainnet-na`, `mainnet-eu`, and `testnet-eu`. It requires the user to provide a log start date and optional duration
+   (default to 10s). The logs will be converted and saved to a disk file. Example usage:
+
+   ```bash
+   $ node log-downloader/index.js --filter $CLUSTER_FILTER -f 2024-10-05T00:00:00Z -d 10m -o demo-2024-10-05-gor.log -p demo-project
+   ```
+
+   Please refer to internal documentation for the available cluster filters.
+
+2. Replay the traffic using goreplay. Example usage:
+   ```bash
+   $ gor --input-file demo-2024-10-05-gor.log --output-http http://localhost:8080
+   ```
+   By default, goreplay doesn't show progress at all, if needed, add `--verbose 3` to show logs for each replayed
+   request.

hedera-mirror-test/traffic-replay/log-downloader/converter.js

@@ -0,0 +1,106 @@
+/*
+ * Copyright (C) 2024 Hedera Hashgraph, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs';
+
+class GoReplayConverter {
+  static #CRLF = '\r\n';
+  static #HTTP_HEADERS = ['Host: 127.0.0.1:80', 'User-Agent: curl/8.8.0', 'Accept: */*', GoReplayConverter.#CRLF].join(
+    GoReplayConverter.#CRLF
+  );
+  static #INPUT_LINE_REGEX = /^([\d\-TZ:.]+) .* GET (.*) in \d+ ms: .*$/;
+  static #LOG_INTERVAL = 5; // log every 5 seconds
+  static #PAYLOAD_SEPARATOR = '\n🐵🙈🙉\n';
+  static #LOG_SUFFIX = `${GoReplayConverter.#HTTP_HEADERS}${GoReplayConverter.#PAYLOAD_SEPARATOR}`; // just hardcode it
+  static #PAYLOAD_TYPE = 1; // request
+  static #REQUEST_DURATION = 0; // hardcode it to 0 since it's not used in replay
+
+  #count = 0;
+  #lastLogTimestamp = 0;
+  #lastStatSeconds;
+  #startSeconds;
+  #outputStream;
+
+  constructor(outputFile) {
+    this.#outputStream = fs.createWriteStream(outputFile);
+  }
+
+  accept(line) {
+    if (!this.#startSeconds) {
+      this.#startSeconds = getEpochSeconds();
+      this.#lastStatSeconds = this.#startSeconds;
+    }
+
+    const converted = this.#convertLine(line);
+    if (!converted) {
+      return;
+    }
+
+    this.#outputStream.write(converted);
+    this.#recordOne();
+  }
+
+  close() {
+    this.#outputStream.close();
+    this.#showStat(true);
+  }
+
+  #convertLine(line) {
+    const match = line.match(GoReplayConverter.#INPUT_LINE_REGEX);
+    if (!match) {
+      return null;
+    }
+
+    const epochMs = new Date(match[1]).getTime();
+    const requestUrl = match[2];
+    // Logs are gathered from multiple pods in a distributed fashion, so it's possible that the timestamps from the
+    // ordered logs are out of order. Workaround it by make sure the timestamp doesn't go backwards and the impact to
+    // traffic replay is negligible.
+    const logTimestamp = epochMs > this.#lastLogTimestamp ? epochMs : this.#lastLogTimestamp;
+    this.#lastLogTimestamp = logTimestamp;
+    // The time in rest api log is at millis granularity, but goreplay requires nanos, so just suffix with 000000
+    return `${GoReplayConverter.#PAYLOAD_TYPE} ${getUUID()} ${logTimestamp}000000 ${
+      GoReplayConverter.#REQUEST_DURATION
+    }\nGET ${requestUrl} HTTP/1.1${GoReplayConverter.#CRLF}${GoReplayConverter.#LOG_SUFFIX}`;
+  }
+
+  #recordOne() {
+    this.#count++;
+    if (getElapsed(this.#lastStatSeconds) >= GoReplayConverter.#LOG_INTERVAL) {
+      this.#lastStatSeconds = getEpochSeconds();
+      this.#showStat();
+    }
+  }
+
+  #showStat(final = false) {
+    const elapsed = getElapsed(this.#startSeconds);
+    const rate = this.#count / elapsed;
+    const prefix = final ? 'Completed processing of' : 'Processed';
+    log(`${prefix} ${this.#count} lines in ${toThousandth(elapsed)} seconds at average rate of ${toThousandth(rate)}`);
+  }
+}
+
+const getElapsed = (lastSeconds) => getEpochSeconds() - lastSeconds;
+
+const getEpochSeconds = () => Date.now() / 1000;
+
+const getUUID = () => Buffer.from(Array.from({length: 12}, randomByte)).toString('hex');
+
+const randomByte = () => Math.floor(Math.random() * 256);
+
+const toThousandth = (value) => value.toFixed(3);
+
+export default GoReplayConverter;

hedera-mirror-test/traffic-replay/log-downloader/downloader.js

@@ -0,0 +1,71 @@
+/*
+ * Copyright (C) 2024 Hedera Hashgraph, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {Logging} from '@google-cloud/logging';
+
+class Downloader {
+  #completed = false;
+  #filter;
+  #projectId;
+  #sinks;
+
+  constructor(filter, projectId, ...sinks) {
+    this.#filter = filter;
+    this.#projectId = projectId;
+    this.#sinks = sinks;
+  }
+
+  async download() {
+    if (this.#completed) {
+      return;
+    }
+
+    log(`Downloading logs for project '${this.#projectId}' with filter '${this.#filter}'`);
+
+    const logging = new Logging({projectId: this.#projectId});
+    let pageToken = undefined;
+    let count = 0;
+    let requestIndex = 1;
+    const sinks = this.#sinks;
+
+    while (true) {
+      const request = {
+        filter: this.#filter,
+        orderBy: 'timestamp asc',
+        pageSize: 100000,
+        pageToken,
+        resourceNames: [`projects/${this.#projectId}`],
+      };
+      const response = (await logging.getEntries(request))[2];
+      const entries = response.entries;
+      pageToken = response.nextPageToken;
+
+      entries.forEach((entry) => sinks.forEach((sink) => sink.accept(entry.textPayload)));
+
+      count += entries.length;
+      log(`Request #${requestIndex++} - downloaded ${entries.length} entries`);
+
+      if (!pageToken) {
+        break;
+      }
+    }
+
+    sinks.forEach((sink) => sink.close());
+    this.#completed = true;
+  }
+}
+
+export default Downloader;

hedera-mirror-test/traffic-replay/log-downloader/index.js

@@ -0,0 +1,79 @@
+/*
+ * Copyright (C) 2024 Hedera Hashgraph, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {hideBin} from 'yargs/helpers';
+import parseDuration from 'parse-duration';
+import yargs from 'yargs';
+
+import Downloader from './downloader.js';
+import GoReplayConverter from './converter.js';
+
+global.log = (msg) => {
+  const timestamp = new Date().toISOString();
+  console.log(`${timestamp} ${msg}`);
+};
+
+const main = async () => {
+  const args = yargs(hideBin(process.argv))
+    .option('duration', {
+      alias: 'd',
+      default: '10s',
+      demandOption: true,
+      description: 'Duration to download logs, e.g., 1s, 10m, 1h',
+      type: 'string',
+    })
+    .option('filter', {
+      demandOption: true,
+      description: 'Google Cloud logging filter for the resource to get logs from',
+      type: 'string',
+    })
+    .option('from', {
+      alias: 'f',
+      demandOption: true,
+      description: 'From time to download logs, in ISO 8601 format, e.g. 2024-10-01T10:05:30Z',
+      type: 'string',
+    })
+    .option('output-file', {
+      alias: 'o',
+      demandOption: true,
+      description: 'Output file',
+      type: 'string',
+    })
+    .option('project', {
+      alias: 'p',
+      demandOption: true,
+      description: 'Google Cloud project id',
+      type: 'string',
+    })
+    .parse();
+
+  const fromDate = new Date(args.from);
+  const toDate = new Date(fromDate.getTime() + parseDuration(args.duration));
+
+  const filter = [
+    args.filter,
+    `resource.labels.project_id="${args.project}"`,
+    `timestamp>="${fromDate.toISOString()}"`,
+    `timestamp<="${toDate.toISOString()}"`,
+  ].join(' ');
+
+  const converter = new GoReplayConverter(args.outputFile);
+  const downloader = new Downloader(filter, args.project, converter);
+
+  await downloader.download();
+};
+
+await main();