Use common cache with haf

 Please enter the commit message for your changes. Lines starting

.gitlab-ci.yaml

@@ -22,15 +22,13 @@ variables:
  GIT_STRATEGY: clone
  GIT_SUBMODULE_STRATEGY: recursive
  CI_DEBUG_SERVICES: "false" #All the service logs should be saved as artifacts, so it's fine to turn this off.
- DATA_CACHE_HAF_TEMPLATE: /cache/replay_data_hivemind_haf
- DATA_CACHE_HAF: "/cache/replay_data_hivemind_haf_${CI_PIPELINE_ID}"
- DATA_CACHE_HAF_DATADIR: $DATA_CACHE_HAF/datadir
- DATA_CACHE_HAF_SHM_DIR: $DATA_CACHE_HAF/shm_dir
+ DATA_CACHE_HAF_PREFIX: "/cache/replay_data_haf"
  BLOCK_LOG_SOURCE_DIR_5M: /blockchain/block_log_5m
+ DATA_CACHE_HIVEMIND: "/cache/replay_data_hivemind_$CI_COMMIT_SHA"
 
 include:
  - project: hive/haf
- ref: 365b6663b6ebc0cd00fe01f822a02ed16c294df5 # develop
+ ref: 0ae69352415622e26923195e8c354d4c74b9483c # develop
  file: /scripts/ci-helpers/prepare_data_image_job.yml
 
 workflow:
@@ -307,9 +305,10 @@ download_await:
  variables:
  GIT_STRATEGY: none
  AWAIT_PACKAGE_URL: $CI_API_V4_URL/projects/440/packages/generic/await/v1.3.2.1/await
+ DATA_CACHE_HAF: "${DATA_CACHE_HAF_PREFIX}_${HAF_COMMIT}"
  needs:
  - job: prepare_haf_data
- artifacts: false
+ artifacts: true
  script:
  - |
  wget --header "JOB-TOKEN: $CI_JOB_TOKEN" "$AWAIT_PACKAGE_URL" -O "${DATA_CACHE_HAF}/await"
@@ -337,8 +336,9 @@ sync:
  host haf_block_log haf_app_admin all trust\n
  host haf_block_log haf_admin all trust\n
  "
- DATADIR: $DATA_CACHE_HAF_DATADIR
- SHM_DIR: $DATA_CACHE_HAF_SHM_DIR
+ DATA_SOURCE: $DATA_CACHE_HAF
+ DATADIR: $DATA_CACHE_HIVEMIND_DATADIR
+ SHM_DIR: $DATA_CACHE_HIVEMIND_SHM_DIR
  LOG_FILE: $CI_JOB_NAME.log
  command: ["--replay-blockchain", "--stop-replay-at-block=5000000"]
  variables:
@@ -348,6 +348,9 @@ sync:
  WORKING_DIR: /home/hivemind
  LOG_PATH: $CI_PROJECT_DIR/hivemind-sync.log
  ADD_MOCKS: "true"
+ DATA_CACHE_HAF: "${DATA_CACHE_HAF_PREFIX}_${HAF_COMMIT}"
+ DATA_CACHE_HIVEMIND_DATADIR: "${DATA_CACHE_HIVEMIND}/datadir"
+ DATA_CACHE_HIVEMIND_SHM_DIR: "${DATA_CACHE_HIVEMIND_DATADIR}/blockchain"
  script:
  - |
  sleep 20s
@@ -370,7 +373,7 @@ sync:
  pushd +2
  ${WORKING_DIR}/app/ci/collect-db-stats.sh
  after_script:
- - cp "$DATA_CACHE_HAF_DATADIR/$CI_JOB_NAME.log" "haf-$CI_JOB_NAME.log" || true # in after_script, so it's done even if the job fails
+ - cp "$DATA_CACHE_HIVEMIND_DATADIR/$CI_JOB_NAME.log" "haf-$CI_JOB_NAME.log" || true # in after_script, so it's done even if the job fails
  artifacts:
  when: always
  expire_in: 7 days
@@ -404,8 +407,8 @@ e2e_benchmark:
  host haf_block_log haf_app_admin all trust\n
  host haf_block_log haf_admin all trust\n
  "
- DATADIR: $DATA_CACHE_HAF_DATADIR
- SHM_DIR: $DATA_CACHE_HAF_SHM_DIR
+ DATADIR: $DATA_CACHE_HIVEMIND_DATADIR
+ SHM_DIR: $DATA_CACHE_HIVEMIND_SHM_DIR
  LOG_FILE: $CI_JOB_NAME.log
  PGCTLTIMEOUT: 600 # give PostgreSQL more time to start if GitLab shut it down improperly after the sync job
  entrypoint:
@@ -420,9 +423,9 @@ e2e_benchmark:
  - *hivemind-serve-script
  variables:
  WORKING_DIR: /home/hivemind
- LOG_PATH: $DATA_CACHE_HAF_DATADIR/hivemind-server.log
- AWAIT_LOG_PATH: $DATA_CACHE_HAF_DATADIR/hivemind-server-await.log
- REQUEST_PATH_LOG_PATH: $DATA_CACHE_HAF_DATADIR/request_process_times_smoketests.log
+ LOG_PATH: $DATA_CACHE_HIVEMIND_DATADIR/hivemind-server.log
+ AWAIT_LOG_PATH: $DATA_CACHE_HIVEMIND_DATADIR/hivemind-server-await.log
+ REQUEST_PATH_LOG_PATH: $DATA_CACHE_HIVEMIND_DATADIR/request_process_times_smoketests.log
  - name: $HIVEMIND_IMAGE
  alias: hivemind-benchmark # cannot be a variable
  entrypoint:
@@ -431,14 +434,17 @@ e2e_benchmark:
  - *hivemind-serve-script
  variables:
  WORKING_DIR: /home/hivemind
- LOG_PATH: $DATA_CACHE_HAF_DATADIR/hivemind-benchmark-server.log
- AWAIT_LOG_PATH: $DATA_CACHE_HAF_DATADIR/hivemind-benchmark-server-await.log
- REQUEST_PATH_LOG_PATH: $DATA_CACHE_HAF_DATADIR/request_process_times.log
+ LOG_PATH: $DATA_CACHE_HIVEMIND_DATADIR/hivemind-benchmark-server.log
+ AWAIT_LOG_PATH: $DATA_CACHE_HIVEMIND_DATADIR/hivemind-benchmark-server-await.log
+ REQUEST_PATH_LOG_PATH: $DATA_CACHE_HIVEMIND_DATADIR/request_process_times.log
  variables:
  HIVED_UID: $HIVED_UID
  JOB_TOKEN: $CI_JOB_TOKEN
  RUNNER_HIVEMIND_BENCHMARK_URL: http://$RUNNER_HIVEMIND_BENCHMARK_SERVER_HOSTNAME
  RUNNER_HIVEMIND_SMOKETEST_URL: http://$RUNNER_HIVEMIND_SMOKETEST_SERVER_HOSTNAME
+ DATA_CACHE_HAF: "${DATA_CACHE_HAF_PREFIX}_${HAF_COMMIT}"
+ DATA_CACHE_HIVEMIND_DATADIR: "${DATA_CACHE_HIVEMIND}/datadir"
+ DATA_CACHE_HIVEMIND_SHM_DIR: "${DATA_CACHE_HIVEMIND_DATADIR}/blockchain"
  script:
  - |
  echo "HAF image name $HAF_IMAGE_NAME"
@@ -466,14 +472,14 @@ e2e_benchmark:
  after_script:
  - |
  echo -e "\e[0Ksection_start:$(date +%s):logs[collapsed=true]\r\e[0KCollecting logs..."
- ls -lah "${DATA_CACHE_HAF_DATADIR}"
- cp "${DATA_CACHE_HAF_DATADIR}/${CI_JOB_NAME}.log" "haf-$CI_JOB_NAME.log" || true
- cp "${DATA_CACHE_HAF_DATADIR}/request_process_times_smoketests.log" request_process_times_smoketests.log || true
- cp "${DATA_CACHE_HAF_DATADIR}/hivemind-server.log" hivemind-server.log || true
- cp "${DATA_CACHE_HAF_DATADIR}/hivemind-server-await.log" hivemind-server-await.log || true
- cp "${DATA_CACHE_HAF_DATADIR}/request_process_times.log" request_process_times.log || true
- cp "${DATA_CACHE_HAF_DATADIR}/hivemind-benchmark-server.log" hivemind-benchmark-server.log || true
- cp "${DATA_CACHE_HAF_DATADIR}/hivemind-benchmark-server-await.log" hivemind-benchmark-server-await.log || true
+ ls -lah "${DATA_CACHE_HIVEMIND_DATADIR}"
+ cp "${DATA_CACHE_HIVEMIND_DATADIR}/${CI_JOB_NAME}.log" "haf-$CI_JOB_NAME.log" || true
+ cp "${DATA_CACHE_HIVEMIND_DATADIR}/request_process_times_smoketests.log" request_process_times_smoketests.log || true
+ cp "${DATA_CACHE_HIVEMIND_DATADIR}/hivemind-server.log" hivemind-server.log || true
+ cp "${DATA_CACHE_HIVEMIND_DATADIR}/hivemind-server-await.log" hivemind-server-await.log || true
+ cp "${DATA_CACHE_HIVEMIND_DATADIR}/request_process_times.log" request_process_times.log || true
+ cp "${DATA_CACHE_HIVEMIND_DATADIR}/hivemind-benchmark-server.log" hivemind-benchmark-server.log || true
+ cp "${DATA_CACHE_HIVEMIND_DATADIR}/hivemind-benchmark-server-await.log" hivemind-benchmark-server-await.log || true
  echo -e "\e[0Ksection_end:$(date +%s):logs\r\e[0K"
  - |
  echo -e "\e[0Ksection_start:$(date +%s):dotenv[collapsed=true]\r\e[0KPreparing dotenv file..."