From a7e832aadcfa97f6c281ef84c45024777ab1b79d Mon Sep 17 00:00:00 2001 From: Marcin Sobczyk Date: Fri, 13 Oct 2023 13:17:03 +0200 Subject: [PATCH] Use common cache with haf Please enter the commit message for your changes. Lines starting --- .gitlab-ci.yaml | 56 +++++++++++++++++++++++++++---------------------- haf | 2 +- 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/.gitlab-ci.yaml b/.gitlab-ci.yaml index 57f0187fb..6f6131e96 100644 --- a/.gitlab-ci.yaml +++ b/.gitlab-ci.yaml @@ -22,15 +22,13 @@ variables: GIT_STRATEGY: clone GIT_SUBMODULE_STRATEGY: recursive CI_DEBUG_SERVICES: "false" #All the service logs should be saved as artifacts, so it's fine to turn this off. - DATA_CACHE_HAF_TEMPLATE: /cache/replay_data_hivemind_haf - DATA_CACHE_HAF: "/cache/replay_data_hivemind_haf_${CI_PIPELINE_ID}" - DATA_CACHE_HAF_DATADIR: $DATA_CACHE_HAF/datadir - DATA_CACHE_HAF_SHM_DIR: $DATA_CACHE_HAF/shm_dir + DATA_CACHE_HAF_PREFIX: "/cache/replay_data_haf" BLOCK_LOG_SOURCE_DIR_5M: /blockchain/block_log_5m + DATA_CACHE_HIVEMIND: "/cache/replay_data_hivemind_$CI_COMMIT_SHA" include: - project: hive/haf - ref: 365b6663b6ebc0cd00fe01f822a02ed16c294df5 # develop + ref: 0ae69352415622e26923195e8c354d4c74b9483c # develop file: /scripts/ci-helpers/prepare_data_image_job.yml workflow: @@ -307,9 +305,10 @@ download_await: variables: GIT_STRATEGY: none AWAIT_PACKAGE_URL: $CI_API_V4_URL/projects/440/packages/generic/await/v1.3.2.1/await + DATA_CACHE_HAF: "${DATA_CACHE_HAF_PREFIX}_${HAF_COMMIT}" needs: - job: prepare_haf_data - artifacts: false + artifacts: true script: - | wget --header "JOB-TOKEN: $CI_JOB_TOKEN" "$AWAIT_PACKAGE_URL" -O "${DATA_CACHE_HAF}/await" @@ -337,8 +336,9 @@ sync: host haf_block_log haf_app_admin all trust\n host haf_block_log haf_admin all trust\n " - DATADIR: $DATA_CACHE_HAF_DATADIR - SHM_DIR: $DATA_CACHE_HAF_SHM_DIR + DATA_SOURCE: $DATA_CACHE_HAF + DATADIR: $DATA_CACHE_HIVEMIND_DATADIR + SHM_DIR: $DATA_CACHE_HIVEMIND_SHM_DIR LOG_FILE: $CI_JOB_NAME.log command: ["--replay-blockchain", "--stop-replay-at-block=5000000"] variables: @@ -348,6 +348,9 @@ sync: WORKING_DIR: /home/hivemind LOG_PATH: $CI_PROJECT_DIR/hivemind-sync.log ADD_MOCKS: "true" + DATA_CACHE_HAF: "${DATA_CACHE_HAF_PREFIX}_${HAF_COMMIT}" + DATA_CACHE_HIVEMIND_DATADIR: "${DATA_CACHE_HIVEMIND}/datadir" + DATA_CACHE_HIVEMIND_SHM_DIR: "${DATA_CACHE_HIVEMIND_DATADIR}/blockchain" script: - | sleep 20s @@ -370,7 +373,7 @@ sync: pushd +2 ${WORKING_DIR}/app/ci/collect-db-stats.sh after_script: - - cp "$DATA_CACHE_HAF_DATADIR/$CI_JOB_NAME.log" "haf-$CI_JOB_NAME.log" || true # in after_script, so it's done even if the job fails + - cp "$DATA_CACHE_HIVEMIND_DATADIR/$CI_JOB_NAME.log" "haf-$CI_JOB_NAME.log" || true # in after_script, so it's done even if the job fails artifacts: when: always expire_in: 7 days @@ -404,8 +407,8 @@ e2e_benchmark: host haf_block_log haf_app_admin all trust\n host haf_block_log haf_admin all trust\n " - DATADIR: $DATA_CACHE_HAF_DATADIR - SHM_DIR: $DATA_CACHE_HAF_SHM_DIR + DATADIR: $DATA_CACHE_HIVEMIND_DATADIR + SHM_DIR: $DATA_CACHE_HIVEMIND_SHM_DIR LOG_FILE: $CI_JOB_NAME.log PGCTLTIMEOUT: 600 # give PostgreSQL more time to start if GitLab shut it down improperly after the sync job entrypoint: @@ -420,9 +423,9 @@ e2e_benchmark: - *hivemind-serve-script variables: WORKING_DIR: /home/hivemind - LOG_PATH: $DATA_CACHE_HAF_DATADIR/hivemind-server.log - AWAIT_LOG_PATH: $DATA_CACHE_HAF_DATADIR/hivemind-server-await.log - REQUEST_PATH_LOG_PATH: $DATA_CACHE_HAF_DATADIR/request_process_times_smoketests.log + LOG_PATH: $DATA_CACHE_HIVEMIND_DATADIR/hivemind-server.log + AWAIT_LOG_PATH: $DATA_CACHE_HIVEMIND_DATADIR/hivemind-server-await.log + REQUEST_PATH_LOG_PATH: $DATA_CACHE_HIVEMIND_DATADIR/request_process_times_smoketests.log - name: $HIVEMIND_IMAGE alias: hivemind-benchmark # cannot be a variable entrypoint: @@ -431,14 +434,17 @@ e2e_benchmark: - *hivemind-serve-script variables: WORKING_DIR: /home/hivemind - LOG_PATH: $DATA_CACHE_HAF_DATADIR/hivemind-benchmark-server.log - AWAIT_LOG_PATH: $DATA_CACHE_HAF_DATADIR/hivemind-benchmark-server-await.log - REQUEST_PATH_LOG_PATH: $DATA_CACHE_HAF_DATADIR/request_process_times.log + LOG_PATH: $DATA_CACHE_HIVEMIND_DATADIR/hivemind-benchmark-server.log + AWAIT_LOG_PATH: $DATA_CACHE_HIVEMIND_DATADIR/hivemind-benchmark-server-await.log + REQUEST_PATH_LOG_PATH: $DATA_CACHE_HIVEMIND_DATADIR/request_process_times.log variables: HIVED_UID: $HIVED_UID JOB_TOKEN: $CI_JOB_TOKEN RUNNER_HIVEMIND_BENCHMARK_URL: http://$RUNNER_HIVEMIND_BENCHMARK_SERVER_HOSTNAME RUNNER_HIVEMIND_SMOKETEST_URL: http://$RUNNER_HIVEMIND_SMOKETEST_SERVER_HOSTNAME + DATA_CACHE_HAF: "${DATA_CACHE_HAF_PREFIX}_${HAF_COMMIT}" + DATA_CACHE_HIVEMIND_DATADIR: "${DATA_CACHE_HIVEMIND}/datadir" + DATA_CACHE_HIVEMIND_SHM_DIR: "${DATA_CACHE_HIVEMIND_DATADIR}/blockchain" script: - | echo "HAF image name $HAF_IMAGE_NAME" @@ -466,14 +472,14 @@ e2e_benchmark: after_script: - | echo -e "\e[0Ksection_start:$(date +%s):logs[collapsed=true]\r\e[0KCollecting logs..." - ls -lah "${DATA_CACHE_HAF_DATADIR}" - cp "${DATA_CACHE_HAF_DATADIR}/${CI_JOB_NAME}.log" "haf-$CI_JOB_NAME.log" || true - cp "${DATA_CACHE_HAF_DATADIR}/request_process_times_smoketests.log" request_process_times_smoketests.log || true - cp "${DATA_CACHE_HAF_DATADIR}/hivemind-server.log" hivemind-server.log || true - cp "${DATA_CACHE_HAF_DATADIR}/hivemind-server-await.log" hivemind-server-await.log || true - cp "${DATA_CACHE_HAF_DATADIR}/request_process_times.log" request_process_times.log || true - cp "${DATA_CACHE_HAF_DATADIR}/hivemind-benchmark-server.log" hivemind-benchmark-server.log || true - cp "${DATA_CACHE_HAF_DATADIR}/hivemind-benchmark-server-await.log" hivemind-benchmark-server-await.log || true + ls -lah "${DATA_CACHE_HIVEMIND_DATADIR}" + cp "${DATA_CACHE_HIVEMIND_DATADIR}/${CI_JOB_NAME}.log" "haf-$CI_JOB_NAME.log" || true + cp "${DATA_CACHE_HIVEMIND_DATADIR}/request_process_times_smoketests.log" request_process_times_smoketests.log || true + cp "${DATA_CACHE_HIVEMIND_DATADIR}/hivemind-server.log" hivemind-server.log || true + cp "${DATA_CACHE_HIVEMIND_DATADIR}/hivemind-server-await.log" hivemind-server-await.log || true + cp "${DATA_CACHE_HIVEMIND_DATADIR}/request_process_times.log" request_process_times.log || true + cp "${DATA_CACHE_HIVEMIND_DATADIR}/hivemind-benchmark-server.log" hivemind-benchmark-server.log || true + cp "${DATA_CACHE_HIVEMIND_DATADIR}/hivemind-benchmark-server-await.log" hivemind-benchmark-server-await.log || true echo -e "\e[0Ksection_end:$(date +%s):logs\r\e[0K" - | echo -e "\e[0Ksection_start:$(date +%s):dotenv[collapsed=true]\r\e[0KPreparing dotenv file..." diff --git a/haf b/haf index 6fd174c2c..0ae693524 160000 --- a/haf +++ b/haf @@ -1 +1 @@ -Subproject commit 6fd174c2c7774fcc343208c52ee1b60d6143e231 +Subproject commit 0ae69352415622e26923195e8c354d4c74b9483c