.gitlab-ci.yml

# .gitlab-ci.yml
#
# polkadot
#
# pipelines can be triggered manually in the web
# setting DEPLOY_TAG will only deploy the tagged image
#
# please do not add new jobs without "rules:" and "*-env". There are &rules-test for everything,
# &rules-pr-only and &rules-build presets. And "kubernetes-env" with "docker-env" to set a runner
# which executes the job.

stages:
  - test
  - build
  - publish
  - deploy

image:                             paritytech/ci-linux:production

workflow:
  rules:
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH

variables:
  GIT_STRATEGY:                    fetch
  GIT_DEPTH:                       100
  CI_SERVER_NAME:                  "GitLab CI"
  DOCKER_OS:                       "debian:stretch"
  ARCH:                            "x86_64"

default:
  cache:                           {}

.collect-artifacts:                &collect-artifacts
  artifacts:
    name:                          "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
    when:                          on_success
    expire_in:                     28 days
    paths:
      - artifacts/

.kubernetes-env:                   &kubernetes-env
  tags:
    - kubernetes-parity-build
  interruptible:                   true

.docker-env:                       &docker-env
  retry:
    max: 2
    when:
      - runner_system_failure
      - unknown_failure
      - api_failure
  interruptible:                   true
  tags:
    - linux-docker

.compiler-info:                    &compiler-info
  before_script:
    - rustup show
    - cargo --version
    - sccache -s

.rules-build:                      &rules-build
  rules:
    # Due to https://gitlab.com/gitlab-org/gitlab/-/issues/31264 there's no way to setup a manual
    # build job so that publish-docker-rococo would "needs" build-linux-rococo job. This leads
    # either to blocked or to forever running pipeline. It was decided to run these jobs from UI
    # and on schedule.
    #
    # $PIPELINE should be passed in https://gitlab.parity.io/parity/polkadot/-/pipeline_schedules
    # or other trigger to avoid running these jobs and run just those allowing this variable.
    - if: $PIPELINE == "rococo"
      when: never
    - if: $CI_PIPELINE_SOURCE == "web"
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_REF_NAME == "master"
    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1

.rules-test:                       &rules-test
  # these jobs run always*
  rules:
    - if: $PIPELINE == "rococo"
      when: never
    - when: always

.pr-only:                          &rules-pr-only
  # these jobs run only on PRs
  rules:
    - if: $PIPELINE == "rococo"
      when: never
    - if: $CI_PIPELINE_SOURCE == "schedule"
      when: never
    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs

#### stage:                        test

check-runtime:
  stage:                           test
  image:                           paritytech/tools:latest
  <<:                              *kubernetes-env
  <<:                              *rules-pr-only
  variables:
    GITLAB_API:                    "https://gitlab.parity.io/api/v4"
    GITHUB_API_PROJECT:            "parity%2Finfrastructure%2Fgithub-api"
  script:
    - ./scripts/gitlab/check_runtime.sh
  allow_failure:                   true

check-line-width:
  stage:                           test
  image:                           paritytech/tools:latest
  <<:                              *kubernetes-env
  <<:                              *rules-pr-only
  script:
    - ./scripts/gitlab/check_line_width.sh
  allow_failure:                   true

test-deterministic-wasm:
  stage:                           test
  <<:                              *rules-test
  <<:                              *docker-env
  <<:                              *compiler-info
  script:
    - ./scripts/gitlab/test_deterministic_wasm.sh

test-linux-stable:
  stage:                           test
  <<:                              *rules-test
  <<:                              *docker-env
  <<:                              *compiler-info
  variables:
    RUST_TOOLCHAIN: stable
    # Enable debug assertions since we are running optimized builds for testing
    # but still want to have debug assertions.
    RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
    TARGET: native
  artifacts:
    paths:
      - ./target/release/polkadot
  script:
    - ./scripts/gitlab/test_linux_stable.sh
    - sccache -s

check-web-wasm:
  stage:                           test
  <<:                              *rules-test
  <<:                              *docker-env
  <<:                              *compiler-info
  script:
    # WASM support is in progress. As more and more crates support WASM, we should
    # add entries here. See https://github.com/paritytech/polkadot/issues/625
    - ./scripts/gitlab/check_web_wasm.sh
    - sccache -s

check-runtime-benchmarks:
  stage:                           test
  <<:                              *rules-test
  <<:                              *docker-env
  <<:                              *compiler-info
  script:
    # Check that the node will compile with `runtime-benchmarks` feature flag.
    - ./scripts/gitlab/check_runtime_benchmarks.sh
    - sccache -s

#### stage:                        build

check-transaction-versions:
  image:                           node:15
  stage:                           build
  <<:                              *rules-test
  <<:                              *docker-env
  needs:
    - job:                         test-linux-stable
      artifacts:                   false
  before_script:
    - npm install --ignore-scripts -g @polkadot/metadata-cmp
    - git fetch origin release
  script:
    - scripts/gitlab/check_extrinsics_ordering.sh

.pack-artifacts:                   &pack-artifacts
  - mkdir -p ./artifacts
  - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name
  - mv ./target/release/polkadot ./artifacts/.
  - sha256sum ./artifacts/polkadot | tee ./artifacts/polkadot.sha256
  - if [ "${CI_COMMIT_TAG}" ]; then
      EXTRATAG="latest";
    else
      EXTRATAG="$(./artifacts/polkadot --version |
        sed -n -r 's/^polkadot ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p')";
      EXTRATAG="${CI_COMMIT_REF_NAME}-${EXTRATAG}-$(cut -c 1-8 ./artifacts/polkadot.sha256)";
    fi
  - echo "Polkadot version = ${VERSION} (EXTRATAG ${EXTRATAG})"
  - echo -n ${VERSION} > ./artifacts/VERSION
  - echo -n ${EXTRATAG} > ./artifacts/EXTRATAG
  - cp -r scripts/docker/* ./artifacts

build-linux-release:
  stage:                           build
  <<:                              *collect-artifacts
  <<:                              *docker-env
  <<:                              *compiler-info
  rules:
    # .rules-test with manual on PRs
    - if: $PIPELINE == "rococo"
      when: never
    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
      when: manual
      allow_failure: true
    - when: always
  script:
    - time cargo build --release --verbose
    - sccache -s
    - *pack-artifacts

build-linux-rococo:
  stage:                           build
  <<:                              *collect-artifacts
  <<:                              *docker-env
  <<:                              *compiler-info
  rules:
    - if: $PIPELINE == "rococo"
  script:
    - time cargo build --release --verbose --features=real-overseer
    - sccache -s
    - *pack-artifacts

generate-impl-guide:
  stage:                           build
  <<:                              *rules-test
  <<:                              *docker-env
  image:
    name: michaelfbryan/mdbook-docker-image:latest
    entrypoint: [""]
  script:
    - mdbook build roadmap/implementers-guide

#### stage:                        publish

.build-push-docker-image:          &build-push-docker-image
  <<:                              *kubernetes-env
  <<:                              *collect-artifacts
  image:                           quay.io/buildah/stable
  before_script:                   &check-versions
    - test -s ./artifacts/VERSION || exit 1
    - test -s ./artifacts/EXTRATAG || exit 1
    - VERSION="$(cat ./artifacts/VERSION)"
    - EXTRATAG="$(cat ./artifacts/EXTRATAG)"
    - echo "Polkadot version = ${VERSION} (EXTRATAG ${EXTRATAG})"
  script:
    - test "$Docker_Hub_User_Parity" -a "$Docker_Hub_Pass_Parity" ||
        ( echo "no docker credentials provided"; exit 1 )
    - cd ./artifacts
    - buildah bud
        --format=docker
        --build-arg VCS_REF="${CI_COMMIT_SHA}"
        --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
        --tag "$IMAGE_NAME:$VERSION"
        --tag "$IMAGE_NAME:$EXTRATAG" .
    # The job will success only on the protected branch
    - echo "$Docker_Hub_Pass_Parity" |
        buildah login --username "$Docker_Hub_User_Parity" --password-stdin docker.io
    - buildah info
    - buildah push --format=v2s2 "$IMAGE_NAME:$VERSION"
    - buildah push --format=v2s2 "$IMAGE_NAME:$EXTRATAG"
  after_script:
    - buildah logout "$IMAGE_NAME"
    # only VERSION information is needed for the deployment
    - find ./artifacts/ -depth -not -name VERSION -not -name artifacts -delete

publish-docker-polkadot:
  stage:                           publish
  <<:                              *build-push-docker-image
  # Don't run on releases - this is handled by the Github Action here:
  # .github/workflows/publish-docker-release.yml
  rules:
    - if: $PIPELINE == "rococo"
      when: never
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_PIPELINE_SOURCE == "web"
    - if: $CI_COMMIT_REF_NAME == "master"
  needs:
    - job:                         build-linux-release
      artifacts:                   true
  variables:
    GIT_STRATEGY:                  none
    # DOCKERFILE:                  scripts/docker/Dockerfile
    IMAGE_NAME:                    docker.io/parity/polkadot

publish-docker-rococo:
  stage:                           publish
  <<:                              *build-push-docker-image
  rules:
    - if: $PIPELINE == "rococo"
  needs:
    - job:                         build-linux-rococo
      artifacts:                   true
  variables:
    GIT_STRATEGY:                  none
    # DOCKERFILE:                  scripts/docker/Dockerfile
    IMAGE_NAME:                    docker.io/parity/rococo

publish-s3-release:
  stage:                           publish
  <<:                              *rules-build
  needs:
    - job:                         build-linux-release
      artifacts:                   true
  <<:                              *kubernetes-env
  image:                           paritytech/awscli:latest
  variables:
    GIT_STRATEGY:                  none
    BUCKET:                        "releases.parity.io"
    PREFIX:                        "polkadot/${ARCH}-${DOCKER_OS}"
  before_script:
    - *check-versions
  script:
    - echo "uploading objects to https://${BUCKET}/${PREFIX}/${VERSION}"
    - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/${VERSION}/
    - echo "update objects at https://${BUCKET}/${PREFIX}/${EXTRATAG}"
    - find ./artifacts -type f | while read file; do
      name="${file#./artifacts/}";
      aws s3api copy-object
        --copy-source ${BUCKET}/${PREFIX}/${VERSION}/${name}
        --bucket ${BUCKET} --key ${PREFIX}/${EXTRATAG}/${name};
      done
    - |
      cat <<-EOM
      |
      |  polkadot binary paths:
      |
      |  - https://${BUCKET}/${PREFIX}/${EXTRATAG}/polkadot
      |  - https://${BUCKET}/${PREFIX}/${VERSION}/polkadot
      |
      EOM
  after_script:
    - aws s3 ls s3://${BUCKET}/${PREFIX}/${EXTRATAG}/
        --recursive --human-readable --summarize

#### stage:                        deploy

deploy-polkasync-kusama:
  stage:                           deploy
  <<:                              *rules-build
  variables:
    POLKADOT_CI_COMMIT_NAME:       "${CI_COMMIT_REF_NAME}"
    POLKADOT_CI_COMMIT_REF:        "${CI_COMMIT_REF}"
  allow_failure:                   true
  trigger:                         "parity/infrastructure/parity-testnet"

#### stage:                        .post

check-labels:
  stage:                           .post
  image:                           paritytech/tools:latest
  <<:                              *rules-pr-only
  <<:                              *kubernetes-env
  script:
    - ./scripts/gitlab/check_labels.sh