From c529d1d962b45b3d49ae820b9a8319a1796a08cb Mon Sep 17 00:00:00 2001 From: alabdao <139594838+alabdao@users.noreply.github.com> Date: Mon, 23 Oct 2023 15:28:27 -0400 Subject: [PATCH] [LAB-723] Update CI to use Bacalhau 1.1.2 (#722) --- .github/workflows/ci.yml | 8 +++-- Dockerfile | 2 +- docker-compose.private.yml | 12 ------- docker-compose.yml | 36 +++++++------------ docker/images/ipfs/Dockerfile | 2 ++ .../ipfs/container-init.d/006-datastore.sh | 2 +- 6 files changed, 21 insertions(+), 41 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce372bd2c..0a511189f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,7 +106,7 @@ jobs: - ubuntu-22.04 # - macos-13 bacalhau_version: - - 1.1.0 + - 1.1.2 runs-on: ${{ matrix.os }} environment: ci steps: @@ -137,10 +137,11 @@ jobs: - ubuntu-22.04 # - macos-13 bacalhau_version: - - 1.1.0 + - 1.1.2 runs-on: ${{ matrix.os }} environment: ci env: + BACALHAU_VERSION: ${{matrix.bacalhau_version}} # Setting it at workflow level to be used by all the steps BACALHAU_API_HOST: "127.0.0.1" LOG_LEVEL: trace @@ -277,10 +278,11 @@ jobs: - ubuntu-22.04 # - macos-13 bacalhau_version: - - 1.1.0 + - 1.1.2 runs-on: ${{ matrix.os }} environment: ci env: + BACALHAU_VERSION: ${{matrix.bacalhau_version}} # Setting it at workflow level to be used by all the steps BACALHAU_API_HOST: "127.0.0.1" LOG_LEVEL: trace diff --git a/Dockerfile b/Dockerfile index 6a71e937a..50ca06704 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM golang:1.20-buster as builder -ARG BACALHAU_VERSION=1.1.1 +ARG BACALHAU_VERSION=1.1.2 # Install deps RUN apt-get update && apt-get install -y --no-install-recommends \ diff --git a/docker-compose.private.yml b/docker-compose.private.yml index 352f33d72..062ab5dd8 100644 --- a/docker-compose.private.yml +++ b/docker-compose.private.yml @@ -6,13 +6,6 @@ services: ipfs: - image: private_ipfs - build: - context: ./docker/images/ipfs - dockerfile: ./Dockerfile - cache_from: - - quay.io/labdao/ipfs:latest - target: ipfs-with-utils environment: IPFS_S3_ENABLED: "false" IPFS_SWARM_KEY_BASE64: "L2tleS9zd2FybS9wc2svMS4wLjAvCi9iYXNlMTYvCjk0Y2Y4ODFiMDZmZDI5YTgxNDVlMmY2MjNiMmRjNGMwNTU2Y2QxNTIwNWM4YjhjMzg0YWEzOThkY2U4YWFhMzYK" @@ -21,8 +14,6 @@ services: IPFS_DEBUG: "true" IPFS_LOGGING: "debug" IPFS_GATEWAY_PORT: "8888" - volumes: - - privateipfs:/data/ipfs backend: environment: @@ -34,6 +25,3 @@ services: LIBP2P_FORCE_PNET: "1" IPFS_DEBUG: "true" IPFS_LOGGING: "debug" - -volumes: - privateipfs: diff --git a/docker-compose.yml b/docker-compose.yml index 3cf3579ea..63f1e326c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,23 +3,10 @@ version: "3.7" services: - # sidecar container for setting up required key material - requester_setup: - image: "alpine" - command: > - sh -c ' - mkdir -p /home/nonroot/.bacalhau && - echo CAASqAkwggSkAgEAAoIBAQDQubr6zD3DLEQ07K2wPUQEFfVQLfqTQihc8lQOWuihElXcqZJLajK4od5zc+nCQ/jyUmrmOsYd7aKBmXfyTwOBwCaXEpqqxlP+Pal/Ruh+Ptpfl11KP4sHg3P5sYM/bMRJrEtg1k3mbaToioH5I9GrH7zv5LDHUSezbYFCJ4ZyakUgal+GXPx+TpbdhIx93pZXPHzxgd/m1rbSyiR7sUqXAokzKf2Qv1sOnGrHHcKy8DiSrXo+YUyDRC3lQKwaSfyrBFq1BLqE+E383zYIc5yhcV3kSGivQ2v9o67jEbss3f3ogEfXplsAnBh1S+RfcfneUaRs/UC/+8kyWAC0sU1ZAgMBAAECggEAHZZWZPrSvnnl6UKrlFH69EFQlt8NqBY2toY7WBWW6E47EZ6P+FTDf9yiNed2eW09OsxoDt7B+rgW0xw4HocIzjm4vq2Kcy7N9To0brBUgKDUP7yrIr1DLft6UyjQnAsFABSGUuHOBR4BxFzt860aKZHELJVUYcylPhAFxKTpsAoSy68/Zfkz5wjGtC/ImcHIwGub7NKNmbouQQe6p4beOkYmyVsX6RL/7RPMi7Jvspq0nj0V0j/T13XBUH4b862cqs54HuJQjI9j4QijA9SuCySXQfJw2If8sM7z/3w+IwYhp2JIu4oMTIppCvTBgzAmh+7OMqfOlL7av9iMmqQiAQKBgQDyDtX4TxN1JiFdJg8L3AV8e53GE2eAGY7zpiN7CDrACDXeyrni8fIVz4wY5Pf7+E49dI1S67dsyHVb2dK6Gg4tyECXoRHYPMRktw8LJf9QfCfYsGrpoj/yq2UZ5SqU1Uh70h3b7W3vdZXPZO1BBzT2iwTPGQl/0I0bunSTl72gYQKBgQDcv2etOyDp72NaiZduLpia0RVc92MAZLyoWyq4rscfjU0MvkCPyTa6sKjF2uaUDS5R8yVHB/6+2Mzs4LuWWpNtp6b3rG88CXxfbGDIYn2BOXDqQ9s2/ceISkNMq+NbfcUbbNvG920Ot20VKjkl9s7LcEC0cLiVU14oWd8BOV6v+QKBgQDWOk8g7ktYOSD6Ib7bTiUE4RrnvTCy7OvZs8ZfEw97+UhEH8OYmW0Z5JAnUC4xsLb+KPMS2k+CVX+YhRW9Y2X9GugfyovShyWBxYno4f4uq1NQsgJNOC3EhfB6lJm1E3aNL48BoAbWEYk+/iIElW5IbQ0ahRwlJNxeGA0ouvk2wQKBgQDZYfwdEa9V8BItJHXlYSmSu/eHt27dAlbui+jEOgmJPsLzm59EeWfV7QYm4WxTu68zsHezPWJWWnVjh+PzKnsobwLzv/FCdCzwY2/jpWnmicRyTiRgQQXvZwJjTJn2jSI/4bGEfD//UA6x2wSwKB55JK/zgVWkZ5a/sdD0CaT/MQKBgA7CMHQIjXl8FqATmBgiejGV6BnbUHko4m0Ab7fQUZY1qlgKrqcXgFmMQkb+p8Og1YeVn8Qy8xuvueAfwemA8OxvLuQNzQcA2qhGZID4Tu2Ng+XIkNxMj1gJelppqVeDnZSFr3TbZcSXIVhk1idAnpGDilS8cOO6LUY/hH/IVGGe > /home/nonroot/.bacalhau/libp2p_private_key && - chown -R 65532:65532 /home/nonroot/' - volumes: - - requestervolume:/home/nonroot/ - requester: - image: "ghcr.io/bacalhau-project/bacalhau:v1.1.1" + image: "ghcr.io/bacalhau-project/bacalhau:v${BACALHAU_VERSION:-1.1.2}" hostname: requester - command: serve --ipfs-connect '/dns4/ipfs/tcp/5001' --node-type requester --private-internal-ipfs=false --peer none --labels "owner=labdao" - volumes: - - requestervolume:/home/nonroot/ + command: serve --ipfs-connect '/dns4/ipfs/tcp/5001' --node-type requester --private-internal-ipfs=false --peer none --labels "owner=labdao" --job-selection-probe-http "http://receptor:8080/judge" environment: # BACALHAU_ENVIRONMENT: production LOG_LEVEL: trace @@ -29,8 +16,6 @@ services: depends_on: ipfs: condition: service_healthy - requester_setup: - condition: service_completed_successfully # sidecar container for requester healthcheck requester_health: @@ -47,10 +32,10 @@ services: start_period: 10s compute: - image: "ghcr.io/bacalhau-project/bacalhau:v1.1.1" + image: "ghcr.io/bacalhau-project/bacalhau:v${BACALHAU_VERSION:-1.1.2}" hostname: compute user: root - command: serve --ipfs-connect '/dns4/ipfs/tcp/5001' --node-type compute --labels "owner=labdao" --private-internal-ipfs=false --peer "/dns4/requester/tcp/1235/p2p/Qmd2oBsjjvtUYRmb7zhDEPhSf9YbxA1a7ZLZ6hyCwMqpnh" + command: serve --ipfs-connect '/dns4/ipfs/tcp/5001' --node-type compute --labels "owner=labdao" --private-internal-ipfs=false --peer "/dns4/requester/tcp/1234/http" --job-selection-probe-http "http://receptor:8080/judge" --max-job-execution-timeout "24h" --job-selection-accept-networked --job-selection-data-locality anywhere environment: LOG_LEVEL: trace DOCKER_DEFAULT_PLATFORM: linux/amd64 @@ -104,6 +89,8 @@ services: context: . cache_from: - quay.io/labdao/backend:latest + args: + BACALHAU_VERSION: ${BACALHAU_VERSION:-1.1.2} ports: - 8080:8080 environment: @@ -171,20 +158,21 @@ services: condition: service_healthy ipfs: - image: "ipfs/kubo:v0.18.0" hostname: ipfs + build: + context: ./docker/images/ipfs + dockerfile: ./Dockerfile + cache_from: + - quay.io/labdao/ipfs:latest environment: IPFS_LOGGING: "debug" IPFS_GATEWAY_PORT: "8888" - volumes: - - ipfs:/data/ipfs + IPFS_DEBUG: "true" ports: - 4001:4001 - 5001:5001 - 8888:8888 volumes: - requestervolume: - ipfs: dbdata-backend: dbdata-receptor: diff --git a/docker/images/ipfs/Dockerfile b/docker/images/ipfs/Dockerfile index ea106b504..2e7db5738 100644 --- a/docker/images/ipfs/Dockerfile +++ b/docker/images/ipfs/Dockerfile @@ -38,6 +38,8 @@ EXPOSE 8081 ENV IPFS_PATH /data/ipfs +ENV IPFS_PROFILE "server" + # The default logging level ENV IPFS_LOGGING "" diff --git a/docker/images/ipfs/container-init.d/006-datastore.sh b/docker/images/ipfs/container-init.d/006-datastore.sh index b1d41abd8..1d690c3c0 100644 --- a/docker/images/ipfs/container-init.d/006-datastore.sh +++ b/docker/images/ipfs/container-init.d/006-datastore.sh @@ -9,7 +9,7 @@ if [ -n "${IPFS_DATASTORE_STORAGEMAX}" ]; then ipfs config Datastore.StorageMax "${IPFS_DATASTORE_STORAGEMAX}" fi -if [ -N "${IPFS_DATASTORE_STORAGEGCWATERMARK}" ]; then +if [ -n "${IPFS_DATASTORE_STORAGEGCWATERMARK}" ]; then ipfs config Datastore.StorageGCWatermark "${IPFS_DATASTORE_STORAGEGCWATERMARK}" fi