azure-pipelines.yml

# Trigger on pushes to `master`
trigger:
- master

# Trigger on PRs against `master`
pr:
- master

variables:
  # We run off of the latest `master`
  BINARYBUILDER_IMAGE_NAME: staticfloat/binarybuilder.jl:master
  # Things we pretty much always want when running with Docker
  BASE_DOCKER_OPTS: -t -v /data/staticfloat/yggstorage:/storage -e GITHUB_TOKEN=$(GITHUB_TOKEN) -e TERM=xterm-16color

pool: Default

jobs:
- job: generator
  steps:
  - checkout: self
    fetchDepth: 99999
    clean: true
  - bash: |
      # Normally we look at the last pushed commit
      COMPARE_AGAINST="HEAD~1"

      # If we're on a PR though, we look at the entire branch at once
      if [[ $(Build.Reason) == "PullRequest" ]]; then
          COMPARE_AGAINST="remotes/origin/$(System.PullRequest.TargetBranch)"
      fi

      # Get the directories holding all changed files
      PROJECTS=$(git diff-tree --no-commit-id --name-only -r HEAD "${COMPARE_AGAINST}" | grep -E ".+/.+" | cut -d/ -f1,2 | sort -u)

      # LLVM is special, we won't build it automatically since it takes special attention
      EXCLUDED_NAMES=" LLVM "

      # This is the dynamic mapping we're going to build up, if it's empty we don't do anything
      PROJECTS_ACCEPTED=""
      for PROJECT in ${PROJECTS}; do
          NAME=$(basename "${PROJECT}")
          echo "Considering ${PROJECT}"
          # Only accept things that contain a `build_tarballs.jl`
          if [[ ! -f "${PROJECT}/build_tarballs.jl" ]]; then
              echo " --> Skipping as it does not have a build_tarballs.jl"
              continue
          fi

          # Ignore RootFS stuff, we'll do that manually
          if [[ "${PROJECT}" == "0_RootFS/"* ]]; then
              echo " --> Skipping as it's within 0_RootFS/"
              continue
          fi

          # Ignore stuff in our excluded projects
          if [[ "${EXCLUDED_NAMES}" == *" ${NAME} "* ]]; then
              echo " --> Skipping as it's excluded"
              continue
          fi

          # Otherwise, emit a build with `PROJECT` set to `${PROJECT}`
          echo " --> Accepted!"
          PROJECTS_ACCEPTED="${PROJECTS_ACCEPTED} ${PROJECT}"
      done
      if [[ -n "${PROJECTS_ACCEPTED}" ]]; then
          # Pull down the latest source BB image, since we're gonna do some building
          docker pull $(BINARYBUILDER_IMAGE_NAME)

          # Create a docker image that sucks in the current Yggdrasil tree
          echo "FROM $(BINARYBUILDER_IMAGE_NAME)" > builder.Dockerfile
          echo "ADD . /workspace" >> builder.Dockerfile

          # Ignore 0_RootFS and .git to make things nice and fast:
          echo "0_RootFS/*" >> .dockerignore
          echo ".git/*" >> .dockerignore

          # Build it, tag it with a unique tag name
          docker build --rm -t bb_worker:$(Build.SourceVersion) -f builder.Dockerfile .
          
          # Next, for each project, download its sources
          for PROJECT in ${PROJECTS_ACCEPTED}; do
              NAME=$(basename ${PROJECT})
              DOCKER_OPTS="$(BASE_DOCKER_OPTS) -w /workspace/${PROJECT} bb_worker:$(Build.SourceVersion)"

              # First, we get build metadata in the form of a JSON object:
              CONTAINER_NAME=$(echo meta_json-${NAME}-$(Build.SourceVersion) | tr '[@]' '_')
              docker run --name=${CONTAINER_NAME} ${DOCKER_OPTS} bash -c " \
                  echo \"Generating meta.json...\"; \
                  julia --color=yes ./build_tarballs.jl --meta-json=./meta.json; \
                  echo \"Downloading sources...\"; \
                  julia --color=yes /workspace/.ci/download_sources.jl ./meta.json ./platforms.list; \
              "
              docker cp ${CONTAINER_NAME}:/workspace/${PROJECT}/platforms.list ${NAME}.platforms.list
              docker cp ${CONTAINER_NAME}:/workspace/${PROJECT}/meta.json ${NAME}.meta.json

              # Remove that container, but do it silently!
              docker rm ${CONTAINER_NAME} 2>/dev/null >/dev/null
          done

          # Emit project variable declarations
          echo -n "##vso[task.setVariable variable=projects;isOutput=true]{"
          for PROJECT in ${PROJECTS_ACCEPTED}; do
              NAME=$(basename ${PROJECT})
              echo -n "'${NAME}':{'NAME': '${NAME}', 'PROJECT':'${PROJECT}'}, "
          done
          echo "}"

          # Emit project/platform joint variable declarations
          echo -n "##vso[task.setVariable variable=projplatforms;isOutput=true]{"
          for PROJECT in ${PROJECTS_ACCEPTED}; do
              NAME=$(basename ${PROJECT})

              # Load in the platforms
              PLATFORMS=$(cat ${NAME}.platforms.list)
              if [[ -n "${PLATFORMS}" ]]; then
                  for PLATFORM in ${PLATFORMS}; do
                      echo -n "'${NAME}-${PLATFORM}':{'NAME': '${NAME}', 'PROJECT':'${PROJECT}', 'PLATFORM':'${PLATFORM}'}, "
                  done
              else
                  # If we were unable to determine the proper platforms, then create a single output with empty platform
                  echo -n "'${NAME}-${PLATFORM}':{'NAME': '${NAME}', 'PROJECT':'${PROJECT}', 'PLATFORM':''}, "
              fi
          done
          echo "}"
      fi
    name: mtrx

- job: build
  dependsOn: generator
  timeoutInMinutes: 180
  cancelTimeoutInMinutes: 2
  strategy:
    matrix: $[ dependencies.generator.outputs['mtrx.projplatforms'] ]
  variables:
    projplatforms: $[ dependencies.generator.outputs['mtrx.projplatforms'] ]
  steps:
  - script: |
      # If we're on master and this is not a pull request, then DEPLOY.  NOTE: A
      # manual rebuild of a PR in Azure web interface is not a `PullRequest`
      DEPLOY=""
      if [[ "$(Build.Reason)" != "PullRequest" ]] && [[ "$(Build.SourceBranch)" == "refs/heads/master" ]] ; then
          DEPLOY="--deploy"
      fi

      # Run inside of that just-built Yggdrasil image, mapping /storage in
      DOCKER_OPTS="$(BASE_DOCKER_OPTS) --rm --privileged -w /workspace/$(PROJECT)"
      docker run ${DOCKER_OPTS} bb_worker:$(Build.SourceVersion) julia --color=yes ./build_tarballs.jl --verbose ${DEPLOY} $(PLATFORM)
    displayName: "run build_tarballs.jl"
    condition: ne(variables['projplatforms'], '')

- job: register
  dependsOn:
  - generator
  - build
  strategy:
    matrix: $[ dependencies.generator.outputs['mtrx.projects'] ]
    maxParallel: 1
  variables:
    projects: $[ dependencies.generator.outputs['mtrx.projects'] ]
  steps:
  - script: |
      docker run $(BASE_DOCKER_OPTS) --rm -w /workspace/$(PROJECT) bb_worker:$(Build.SourceVersion) bash -c " \
          echo \"Generating meta.json...\"; \
          julia --color=yes ./build_tarballs.jl --meta-json=./meta.json; \
          echo \"Registering $(NAME)...\"; \
          julia --color=yes /workspace/.ci/register_package.jl meta.json --verbose; \
      "
    displayName: "register JLL package"
    # We only register if this is on `master`; same as setting `${DEPLOY}` above.
    condition: and(and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/master')), ne(variables['projects'], ''))

- job: cleanup
  dependsOn:
  - build
  - register
  condition: always()
  steps:
  - script: |
      docker rmi bb_worker:$(Build.SourceVersion) || true
    displayName: "Cleanup docker image"
    # Use always() so that we run regardless of whether previous steps were successful or not
  - script: |
      rm -f *.meta.json *.platform.list
    displayName: "Cleanup meta files"