Set build type for scheduled builds based on doc repo changes (#2931)

* check if docs repo has changed

* more

* updates so I can test it

* build commit

* lbc

* gpb

* more testing

* a

* b

* mmhm

* use auth token

* fix quotes

* c

* aha!

* no way this works

* forgot the squigles

* back up

* perhaps

* lsc

* no squigs

* jq isn’t working

* quotes

* try a file

* ugh

* d

* e

* move to file, echo all the things

* Fixing up the script - mostly adding curl and jq flags

* wire up the rebuild

* fix script typo

* export rebuild var

* Make the script more generic

* Set REBUILD value to false since empty strings are not accepted

* Remove early exits used for testing purposes

* Try setting concurrency group name based on branch

* uncomment

---------

Co-authored-by: nassimkammah <nassim.kammah@elastic.co>

.buildkite/build_pipeline.yml

@@ -25,6 +25,9 @@ steps:
             value: ""
         hint: "Should we ignore checking broken links? Should we allow to run the build without failing if there's a broken link? Ignoring broken links is dangerous not just because bad links will leak into the public site but because subsequent builds and pull requests that do not fix the links fail."
   - wait
+  - label: "Full rebuild or incremental build?"
+    if: build.source == "schedule"
+    command: ".buildkite/scripts/compare_commits.sh"
   - label: ":white_check_mark: Build docs"
     command: |
       export REBUILD="$(buildkite-agent meta-data get REBUILD --default '' --log-level fatal)"
@@ -34,7 +37,7 @@ steps:
       provider: "gcp"
       image: family/docs-ubuntu-2204
       machineType: ${BUILD_MACHINE_TYPE}
-    concurrency_group: build-docs
+    concurrency_group: build-docs-${BUILDKITE_BRANCH}
     concurrency: 1
 notify:
   - email: "docs-status@elastic.co"

.buildkite/hooks/pre-command

@@ -34,6 +34,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "docs-build-pr" ]];then
     fi
 elif [[ "$BUILDKITE_PIPELINE_SLUG" == "docs-build" ]];then
     export BUILD_MACHINE_TYPE="n2-highcpu-32"
+    export BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field=value secret/ci/elastic-docs/buildkite_token)
 elif [[ "$BUILDKITE_PIPELINE_SLUG" == "docs-build-air-gapped" ]] && [[ "$BUILDKITE_STEP_KEY" == "publish-air-gapped-doc" ]]; then
     export DOCKER_USERNAME=$(retry 5 vault kv get -field=username secret/ci/elastic-docs/docker.elastic.co)
     export DOCKER_PASSWORD=$(retry 5 vault kv get -field=password secret/ci/elastic-docs/docker.elastic.co)

.buildkite/scripts/compare_commits.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+last_successful_build_url="https://api.buildkite.com/v2/organizations/elastic/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds?branch=${BUILDKITE_BRANCH}&state=passed"
+LAST_SUCCESSFUL_COMMIT=$(curl -s -H "Authorization: Bearer ${BUILDKITE_API_TOKEN}" $last_successful_build_url | jq -r '.[0].commit')
+
+echo "Comparing the current docs build commit ${BUILDKITE_COMMIT} to the last successful build commit ${LAST_SUCCESSFUL_COMMIT}"
+if [[ "$BUILDKITE_COMMIT" == "$LAST_SUCCESSFUL_COMMIT" ]]; then
+  echo "The docs repo has not changed since the last build."
+  buildkite-agent meta-data set "REBUILD" "false"
+else
+  echo "The docs repo has changed since the last build."
+  buildkite-agent meta-data set "REBUILD" "rebuild"
+fi