Add load-test-api CI job (#2358)

* Add `load-test-api` CI job Signed-off-by: wslulciuc <willy@datakin.com> * Fix path to load script Signed-off-by: wslulciuc <willy@datakin.com> * Add `get-k6.sh` Signed-off-by: wslulciuc <willy@datakin.com> * Add get-k6.sh Signed-off-by: wslulciuc <willy@datakin.com> * Move load-testing/ under api/ Signed-off-by: wslulciuc <willy@datakin.com> * Create `/root/.gnupg/` Signed-off-by: wslulciuc <willy@datakin.com> * Add load test plan to `.circleci/api-load-test.sh` Signed-off-by: wslulciuc <willy@datakin.com> * continued: Add load test plan to `.circleci/api-load-test.sh` Signed-off-by: wslulciuc <willy@datakin.com> * Run http server in detach mode Signed-off-by: wslulciuc <willy@datakin.com> * Log steps in `.circleci/api-load-test.sh` Signed-off-by: wslulciuc <willy@datakin.com> * continued: Log steps in `.circleci/api-load-test.sh` Signed-off-by: wslulciuc <willy@datakin.com> * Fix persist workspace path Signed-off-by: wslulciuc <willy@datakin.com> * Use db container only Signed-off-by: wslulciuc <willy@datakin.com> * Install jdk17 Signed-off-by: wslulciuc <willy@datakin.com> * Fix paths Signed-off-by: wslulciuc <willy@datakin.com> * Debug Signed-off-by: wslulciuc <willy@datakin.com> * continued: Debug Signed-off-by: wslulciuc <willy@datakin.com> * continued: Debug Signed-off-by: wslulciuc <willy@datakin.com> * Add postgres host override in marquez.dev.yml Signed-off-by: wslulciuc <willy@datakin.com> * continued: Add postgres host override in marquez.dev.yml Signed-off-by: wslulciuc <willy@datakin.com> * continued: Add postgres host override in marquez.dev.yml Signed-off-by: wslulciuc <willy@datakin.com> * continued: Add postgres host override in marquez.dev.yml Signed-off-by: wslulciuc <willy@datakin.com> * continued: Add postgres host override in marquez.dev.yml Signed-off-by: wslulciuc <willy@datakin.com> * WIP Signed-off-by: wslulciuc <willy@datakin.com> * Wait for http API server Signed-off-by: wslulciuc <willy@datakin.com> * Fix mqz host in load test Signed-off-by: wslulciuc <willy@datakin.com> * Use mqz dev port Signed-off-by: wslulciuc <willy@datakin.com> * Set mqz admin port Signed-off-by: wslulciuc <willy@datakin.com> * Use default dev admin port Signed-off-by: wslulciuc <willy@datakin.com> * Save http logs Signed-off-by: wslulciuc <willy@datakin.com> * continued: Save http logs Signed-off-by: wslulciuc <willy@datakin.com> * continued: Save http logs Signed-off-by: wslulciuc <willy@datakin.com> * Display CPU/MEM info Signed-off-by: wslulciuc <willy@datakin.com> * Require `build-api` on `load-test-api` Signed-off-by: wslulciuc <willy@datakin.com> * Display OL events stats Signed-off-by: wslulciuc <willy@datakin.com> * Use `METADATA_STATS_QUERY` Signed-off-by: wslulciuc <willy@datakin.com> * Remove unused method Signed-off-by: wslulciuc <willy@datakin.com> Signed-off-by: wslulciuc <willy@datakin.com>
MarquezProject · Jan 17, 2023 · 656b2e6 · 656b2e6
1 parent 3ca46c1
commit 656b2e6
Show file tree

Hide file tree

Showing 10 changed files with 170 additions and 19 deletions.
diff --git a/.circleci/api-load-test.sh b/.circleci/api-load-test.sh
@@ -0,0 +1,93 @@
+#!/bin/bash
+#
+# Copyright 2018-2023 contributors to the Marquez project
+# SPDX-License-Identifier: Apache-2.0
+#
+# A script used in CI to load test HTTP API server by:
+#   (1) Starting HTTP API server
+#   (2) Generating random dataset, job, and run metadata
+#   (3) Running load test using k6
+#   (4) Writing load test results to 'k6/results' for analysis
+#
+# Usage: $ ./api-load-test.sh
+
+set -e
+
+# Build version of Marquez
+readonly MARQUEZ_VERSION="0.30.0-SNAPSHOT"
+# Fully qualified path to marquez.jar
+readonly MARQUEZ_JAR="api/build/libs/marquez-api-${MARQUEZ_VERSION}.jar"
+
+readonly MARQUEZ_HOST="localhost"
+readonly MARQUEZ_ADMIN_PORT=5001 # Use default 'dev' admin port
+readonly MARQUEZ_URL="http://${MARQUEZ_HOST}:${MARQUEZ_ADMIN_PORT}"
+readonly MARQUEZ_DB="marquez-db"
+
+readonly METADATA_FILE="api/load-testing/metadata.json"
+readonly METADATA_STATS_QUERY=$(cat <<-END
+  SELECT run_uuid,COUNT(*)
+    FROM lineage_events
+   GROUP BY run_uuid;
+END
+)
+
+log() {
+  echo -e "\033[1m>>\033[0m ${1}"
+}
+
+cpu_and_mem_info() {
+  log "CPU info:"
+  cat /proc/cpuinfo
+  log "MEM info:"
+  cat /proc/meminfo
+}
+
+ol_events_stats() {
+    # Query db for OL events stats
+  log "load test metadata stats:"
+  docker exec "${MARQUEZ_DB}" \
+    psql -U marquez -c "${METADATA_STATS_QUERY}"
+}
+
+# Change working directory to project root
+project_root=$(git rev-parse --show-toplevel)
+cd "${project_root}"
+
+# (1) Start db
+log "start db:"
+docker-compose -f docker-compose.db.yml up --detach
+
+# (2) Build HTTP API server
+log "build http API server..."
+./gradlew --no-daemon :api:build -x test > /dev/null 2>&1
+
+# (3) Start HTTP API server
+log "start http API server..."
+mkdir marquez && \
+  java -jar "${MARQUEZ_JAR}" server marquez.dev.yml > marquez/http.log 2>&1 &
+
+# (4) Wait for HTTP API server
+log "waiting for http API server (${MARQUEZ_URL})..."
+until curl --output /dev/null --silent --head --fail "${MARQUEZ_URL}/ping"; do
+    sleep 5
+done
+# When available, print status
+log "http API server is ready!"
+
+# (5) Use metadata command to generate random dataset, job, and run metadata
+log "generate load test metadata (${METADATA_FILE}):"
+java -jar "${MARQUEZ_JAR}" metadata --runs 10 --bytes-per-event 16384 --output "${METADATA_FILE}"
+
+# Display CPU/MEM
+cpu_and_mem_info
+
+# (6) Run load test
+log "start load test:"
+mkdir -p k6/results && \
+  k6 run --vus 25 --duration 30s api/load-testing/http.js \
+    --out json=k6/results/full.json --summary-export=k6/results/summary.json
+
+# Display OL event stats
+ol_events_stats
+
+echo "DONE!"
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -15,7 +15,6 @@ only-on-release: &only-on-release
       ignore: /.*/
 
 orbs:
-  # https://circleci.com/orbs/registry/orb/codecov/codecov
   codecov: codecov/codecov@3.2.3
 
 jobs:
@@ -148,6 +147,21 @@ jobs:
       - run: npm install --prefix=${HOME}/.local --global redoc-cli
       - run: redoc-cli bundle spec/openapi.yml
 
+  load-test-api:
+    working_directory: ~/marquez
+    machine:
+      image: ubuntu-2004:current
+    steps:
+      - checkout
+      - run: ./.circleci/get-docker-compose.sh
+      - run: ./.circleci/get-jdk17.sh
+      - run: ./.circleci/get-k6.sh
+      - run: ./.circleci/api-load-test.sh
+      - store_artifacts:
+          path: marquez
+      - store_artifacts:
+          path: k6
+
   migrate-db:
     working_directory: ~/marquez
     machine:
@@ -204,6 +218,9 @@ workflows:
       - unit-test-web
       - unit-test-client-python
       - lint-spec-api
+      - load-test-api:
+          requires:
+            - build-api
       - migrate-db:
           requires:
             - build-api

diff --git a/.circleci/db-migration.sh b/.circleci/db-migration.sh
@@ -1,11 +1,11 @@
 #!/bin/bash
 #
-# Copyright 2018-2022 contributors to the Marquez project
+# Copyright 2018-2023 contributors to the Marquez project
 # SPDX-License-Identifier: Apache-2.0
 #
 # A script used in CI to test database migrations by:
 #   (1) Applying db migrations on latest Marquez release
-#   (2) Take a backup of db from Step 1
+#   (2) Taking a backup of db from Step 1
 #   (3) Applying db migrations on latest Marquez build using backup
 #
 # Usage: $ ./db-migration.sh

diff --git a/.circleci/get-jdk17.sh b/.circleci/get-jdk17.sh
@@ -1,19 +1,12 @@
 #!/bin/bash
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Copyright 2018-2023 contributors to the Marquez project
+# SPDX-License-Identifier: Apache-2.0
 #
 # Usage: $ ./get-jdk17.sh
 
+set -e
+
 wget -qO - https://adoptium.jfrog.io/adoptium/api/gpg/key/public | sudo apt-key add -
 sudo add-apt-repository --yes https://adoptium.jfrog.io/adoptium/deb
 sudo apt-get update --allow-releaseinfo-change && sudo apt-get install --yes temurin-17-jdk

diff --git a/.circleci/get-k6.sh b/.circleci/get-k6.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Copyright 2018-2023 contributors to the Marquez project
+# SPDX-License-Identifier: Apache-2.0
+#
+# Usage: $ ./get-k6.sh
+
+set -e
+
+# Delete existing key (if present)
+sudo apt-key del k6
+
+# Add k6 key and update the repository
+sudo gpg -k && \
+  sudo gpg --no-default-keyring --keyring /usr/share/keyrings/k6-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C5AD17C747E3415A3642D57D77C6C491D6AC1D69
+echo "deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main" | sudo tee /etc/apt/sources.list.d/k6.list
+
+# Install k6, then verify
+sudo snap install k6 && k6 version
+
+echo "DONE!"
diff --git a/api/load-testing/http.js b/api/load-testing/http.js
@@ -0,0 +1,27 @@
+import { SharedArray } from 'k6/data';
+import http from 'k6/http';
+import { check, sleep } from 'k6';
+import { Rate } from 'k6/metrics';
+
+export const errorRate = new Rate('errors');
+
+const metadata = new SharedArray('metadata', function () {
+  return JSON.parse(open('./metadata.json'));
+});
+
+export default function () {
+  const url = 'http://localhost:5000/api/v1/lineage';
+  const params = {
+    headers: {
+      'Content-Type': 'application/json',
+    },
+  };
+
+  var ol_event = metadata[__VU-1]
+
+  check(http.post(url, JSON.stringify(ol_event), params), {
+    'status is 201': (r) => r.status == 201,
+  }) || errorRate.add(1);
+
+  sleep(1);
+}
diff --git a/docs/load-testing-architecture.png → ...oad-testing/load-testing-architecture.png b/docs/load-testing-architecture.png → ...oad-testing/load-testing-architecture.png
diff --git a/docs/load-testing.md → api/load-testing/load-testing.md b/docs/load-testing.md → api/load-testing/load-testing.md
@@ -90,5 +90,5 @@ export default function () {
    > **Note:** To learn how to run tests locally with `k6`, see [_Running k6_](https://k6.io/docs/getting-started/running-k6).
 
 ----
-SPDX-License-Identifier: Apache-2.0 
-Copyright 2018-2023 contributors to the Marquez project.
+SPDX-License-Identifier: Apache-2.0
+Copyright 2018-2023 contributors to the Marquez project.
diff --git a/docker-compose.db.yml b/docker-compose.db.yml
@@ -9,7 +9,7 @@ services:
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=password
       - MARQUEZ_DB=marquez
-      - MARQUEZ_USER=buendia
-      - MARQUEZ_PASSWORD=macondo
+      - MARQUEZ_USER=marquez
+      - MARQUEZ_PASSWORD=marquez
     volumes:
       - ./docker/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh
diff --git a/marquez.dev.yml b/marquez.dev.yml
@@ -9,7 +9,7 @@ server:
 
 db:
   driverClass: org.postgresql.Driver
-  url: jdbc:postgresql://postgres:5432/marquez
+  url: jdbc:postgresql://${POSTGRES_HOST:-localhost}:5432/marquez
   user: marquez
   password: marquez