.gitlab-ci.yml

stages:
  - bootstrap
  - build
  - post-deploy
  - cleanup

variables:
  PREVIEW_CONFIG: "config/preview.yaml"
  LIVE_CONFIG: "config/live.yaml"
  ARTIFACT_RESOURCE: "public"
  LIVE_DOMAIN: "https://docs.datadoghq.com/"
  PREVIEW_DOMAIN: "https://docs-staging.datadoghq.com/"
  # datadog-ci
  DATADOG_SITE: 'datadoghq.com'
  DATADOG_SUBDOMAIN: 'dd-corpsite'
  FORCE_COLOR: '1'
  GIT_DEPTH: 50
  # gitlab checkout
  DOCS_REPO_NAME: documentation
  REPO_URL: "https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/${DOCS_REPO_NAME}.git"
  # yarn/node cache dirs
  YARN_CACHE_FOLDER: "${CI_PROJECT_DIR}/.yarn/cache/"

# ================== copy scripts =============== #
before_script:
  - "[ -d local/bin ] && find local/bin/ -type f -exec cp {} /usr/local/bin \\;"  # load scripts
  - "[ -d local/etc ] && find local/etc/ -type f -exec cp {} /etc \\;"  # load configs
  - "[ -f /usr/local/bin/helpers.sh ] && source /usr/local/bin/helpers.sh"  # source helpers so they are available in the container
  - "[ -f /etc/profile ] && source /etc/profile" # for golang on path
  - mkdir -p logs

# set policy for the common case
# yarn cache is used in a lot but only built during build job so lets default to pull and explicit push in build job
.yarn_cache: &yarn_cache
  - key:
      files:
        - yarn.lock
    paths:
      - package.json
      - yarn.lock
      - ${YARN_CACHE_FOLDER}
    policy: pull

.yarn_cache_pull_push: &yarn_cache_pull_push
  - key:
      files:
        - yarn.lock
    paths:
      - package.json
      - yarn.lock
      - ${YARN_CACHE_FOLDER}
    policy: pull-push

# ================== templates ================== #
.base_template: &base_template
  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/corp-ci:master
  tags:
    - "runner:main"
  except:
    - tags
  only:
    - branches

# ================== preview ================== #
# If the branch has a name of <slack-user>/<feature-name> then ci builds a preview site
build_preview:
  <<: *base_template
  stage: build
  cache:
    - *yarn_cache_pull_push
  environment: "preview"
  variables:
    URL: ${PREVIEW_DOMAIN}
    CONFIG: ${PREVIEW_CONFIG}
    MESSAGE: ":gift_heart: Your preview site is available!\nNow running tests..."
    CONFIGURATION_FILE: "./local/bin/py/build/configurations/pull_config_preview.yaml"
    LOCAL: "False"
  script:
    - post_dd_event "documentation deploy ${CI_COMMIT_REF_NAME} started" "${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}" "info"
    - yarn install --immutable
    - touch Makefile.config
    - make BRANCH=${CI_COMMIT_REF_NAME} examples
    - sync_integration_descriptions
    - ./local/bin/py/build/pull_rbac.py $(get_secret 'dd-api-key') $(get_secret 'dd-app-key')
    - placehold_translations
    - update_preview_baseurl
    - ./node_modules/.bin/jest --testPathPattern=assets/scripts/tests/
    - yarn run prebuild
    - yarn run build:preview
    # remove service_checks json as we don't need to s3 push that..
    - rm -rf data/service_checks
    - in-isolation deploy_preview > logs/deploy.txt
    # remove static images so we can skip from artifact passed in gitlab
    - remove_static_from_repo
    - notify_slack "<https://github.com/DataDog/documentation/commit/${CI_COMMIT_SHA}|${CI_COMMIT_REF_NAME}> is ready for preview. <${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}| checks running>." "#31b834"
  artifacts:
    paths:
      - ${ARTIFACT_RESOURCE}
      - ./integrations_data
      - .github/
      - .vale.ini
      - .htmltest.yml
      - ./logs
  only:
    - /.+?\/[a-zA-Z0-9_-]+/
  interruptible: true

link_checks_preview:
  <<: *base_template
  stage: post-deploy
  cache: {}
  environment: "preview"
  variables:
    URL: ${PREVIEW_DOMAIN}
    GIT_STRATEGY: none
  script:
    - post_dd_event "documentation htmltest ${CI_COMMIT_REF_NAME} started" "${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}" "info"
    - find ./public -name "*.html" -exec sed -i -e "s#https://docs-staging.datadoghq.com/$CI_COMMIT_REF_NAME/#/#g" {} +
    - find ./public -name "*.html" -exec sed -i -e "s#/api/v1/#/api/latest/#g" {} +
    - find ./public -name "*.html" -exec sed -i -e "s#/api/v2/#/api/latest/#g" {} +
    - htmltest
  after_script:
    - "[ -f /usr/local/bin/helpers.sh ] && source /usr/local/bin/helpers.sh"  # source helpers
    - RESULT="⛈ htmltest result; the following issues were found:\n$(tail -n +2 ./tmp/.htmltest/htmltest.log)\n--------------------" # output all but first line
    - if grep -q "not exist" tmp/.htmltest/htmltest.log; then notify_slack "${RESULT}" "#31b834"; fi
  only:
    - /.+?\/[a-zA-Z0-9_-]+/
  artifacts:
    paths:
      - ./tmp/.htmltest
  interruptible: true

missing_tms_preview:
  <<: *base_template
  stage: post-deploy
  environment: "preview"
  allow_failure: true
  cache: {}
  variables:
    GIT_STRATEGY: none
  dependencies:
    - build_preview
  script:
    - check_missing_tms
  only:
    - /.+?\/[a-zA-Z0-9_-]+/
  interruptible: true

index_algolia_preview:
  image:
    name: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docsearch-scraper:v2978988-4692ed5-0.8
    entrypoint: [""] # force an empty entrypoint (see https://gitlab.com/gitlab-org/gitlab-runner/issues/2692)
  tags: ["runner:main"]
  stage: post-deploy
  cache:
    - *yarn_cache
  environment: "preview"
  timeout: 1h 30m
  script:
    - yarn install --immutable
    - |-
        echo APPLICATION_ID=$(get_secret 'ci.documentation.algolia_preview_application_id') > .env
        echo API_KEY=$(get_secret 'ci.documentation.algolia_preview_api_key') >> .env
        export DISPLAY=:99
        Xvfb :99 -ac &
        echo "n" | docsearch run ./local/etc/algolia/docs_datadoghq_preview.json
  dependencies:
    - build_preview
  when: manual
  only:
    - /.+?\/[a-zA-Z0-9_-]+/
  interruptible: true

replica_algolia_preview:
  <<: *base_template
  stage: post-deploy
  cache:
    - *yarn_cache
  environment: "preview"
  script:
    - yarn install --immutable
    - ALGOLIA_APP_ID=$(get_secret 'algolia_preview_application_id') ALGOLIA_ADMIN_KEY=$(get_secret 'algolia_preview_api_key') yarn run algolia:config
  dependencies:
    - build_preview
  when: manual
  only:
    - /.+?\/[a-zA-Z0-9_-]+/
  interruptible: true

sourcemaps_preview:
  <<: *base_template
  stage: post-deploy
  cache:
    - *yarn_cache
  environment: "preview"
  script:
    - yarn install --immutable
    # Rename any js.map files to include SHA if correlating JS file is fingerprinted.  Hugo's asset build system seems to lose the fingerprint value when generating source maps.
    - find ./public -type f -name "*.*.js" -exec bash -c 'x={}; hash=$(basename $x | cut -d '.' -f 2); map=${x/$hash./}.map; mv $map ${map/.js/.$hash.js}' \;
    - DATADOG_API_KEY="$(get_secret 'dd-api-key')" ./node_modules/.bin/datadog-ci sourcemaps upload ./public/static --service docs --minified-path-prefix "https://docs-staging.datadoghq.com/${CI_COMMIT_REF_NAME}/static/" --release-version "${CI_COMMIT_SHORT_SHA}"
  dependencies:
    - build_preview
  allow_failure: true
  only:
    - /.+?\/[a-zA-Z0-9_-]+/
  interruptible: true

# ================== live ================== #
build_live:
  <<: *base_template
  stage: build
  cache:
    - *yarn_cache_pull_push
  environment: "live"
  variables:
    CONFIG: ${LIVE_CONFIG}
    URL: ${LIVE_DOMAIN}
    UNTRACKED_EXTRAS: "data"
    CONFIGURATION_FILE: "./local/bin/py/build/configurations/pull_config.yaml"
    LOCAL: "False"
  script:
    - post_dd_event "documentation deploy ${CI_COMMIT_REF_NAME} started" "${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}" "info"
    - notify_slack "<https://github.com/DataDog/documentation/commit/${CI_COMMIT_SHA}|${CI_COMMIT_SHA:0:8}> sent to gitlab for production deployment. <${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}|details>" "#FFD700"
    - yarn install --immutable
    - touch Makefile.config
    - make clean-examples
    - make examples
    - sync_integration_descriptions
    - sync_integration_metrics
    - ./local/bin/py/build/pull_rbac.py $(get_secret 'dd-api-key') $(get_secret 'dd-app-key')
    - placehold_translations
    - yarn run prebuild
    - yarn run build:live
    - in-isolation deploy_live
    - create_artifact
    - create_artifact_ignored
    - remove_static_from_repo
  artifacts:
    when: on_success
    paths:
      - ${ARTIFACT_RESOURCE}
      - ./integrations_data
      - content/en
      - .htmltest.yml
    expire_in: 1 week
  only:
    - master

index_internal_doc:
  stage: post-deploy
  cache: {}
  environment: "live"
  image: registry.ddbuild.io/datadoc/webapp:prod-released
  dependencies:
    - build_live
  only:
    - master
  except: [ tags, schedules ]
  tags: ["runner:main"]
  script:
    - INDEXING_SERVICE=https://datadoc.ddbuild.io index-repository hugo https://docs.datadoghq.com/ "Datadog" "Public Documentation" --source_folder=content/en --tags="team:Documentation,source:hugo-website,visibility:public,github-repository:documentation"

test_live_link_checks:
  <<: *base_template
  stage: post-deploy
  cache: {}
  environment: "live"
  variables:
    URL: ${LIVE_DOMAIN}
    GIT_STRATEGY: none
  script:
    - post_dd_event "documentation htmltest ${CI_COMMIT_REF_NAME} started" "${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}" "info"
    - find ./public -name "*.html" -exec sed -i -e "s#https://docs.datadoghq.com/#/#g" {} +
    - find ./public -name "*.html" -exec sed -i -e "s#/api/v1/#/api/latest/#g" {} +
    - find ./public -name "*.html" -exec sed -i -e "s#/api/v2/#/api/latest/#g" {} +
    - htmltest
  after_script:
    - "[ -f /usr/local/bin/helpers.sh ] && source /usr/local/bin/helpers.sh"  # source helpers
    - RESULT="⛈ htmltest result; the following issues were found:\n$(tail -n +2 ./tmp/.htmltest/htmltest.log)\n--------------------" # output all but first line
    - if grep -q "not exist" tmp/.htmltest/htmltest.log; then notify_slack "${RESULT}" "#31b834"; fi
  only:
    - master
  artifacts:
    paths:
      - ./tmp/.htmltest

index_algolia:
  image:
    name: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docsearch-scraper:v2978988-4692ed5-0.8
    entrypoint: [ "" ] # force an empty entrypoint (see https://gitlab.com/gitlab-org/gitlab-runner/issues/2692)
  tags: ["runner:main"]
  stage: post-deploy
  environment: "live"
  timeout: 1h 45m
  script:
    - |-
      echo APPLICATION_ID=$(get_secret 'ci.documentation.algolia_docsearch_application_id') > .env
      echo API_KEY=$(get_secret 'ci.documentation.algolia_docsearch_api_key') >> .env
      export DISPLAY=:99
      Xvfb :99 -ac &
      echo "n" | docsearch run ./local/etc/algolia/docs_datadoghq.json
  dependencies:
    - build_live
  only:
    - schedules

replica_algolia:
  <<: *base_template
  stage: post-deploy
  cache:
    - *yarn_cache
  environment: "live"
  script:
    - yarn install --immutable
    - ALGOLIA_APP_ID=$(get_secret 'algolia_docsearch_application_id') ALGOLIA_ADMIN_KEY=$(get_secret 'algolia_docsearch_api_key') yarn run algolia:config
  dependencies:
    - build_live
  only:
    - schedules

sourcemaps_live:
  <<: *base_template
  stage: post-deploy
  cache:
    - *yarn_cache
  environment: "live"
  script:
    - yarn install --immutable
    # Rename any js.map files to include SHA if correlating JS file is fingerprinted. Hugo's asset build system seems to lose the fingerprint value when generating source maps.
    - find ./public -type f -name "*.*.js" -exec bash -c 'x={}; hash=$(basename $x | cut -d '.' -f 2); map=${x/$hash./}.map; mv $map ${map/.js/.$hash.js}' \;
    - DATADOG_API_KEY="$(get_secret 'dd-api-key')" ./node_modules/.bin/datadog-ci sourcemaps upload ./public/static --service docs --minified-path-prefix "https://docs.datadoghq.com/static/" --release-version "${CI_COMMIT_SHORT_SHA}"
  dependencies:
    - build_live
  allow_failure: true
  only:
    - master

evaluate_missing_metrics:
  <<: *base_template
  stage: post-deploy
  cache: {}
  environment: "live"
  script:
    - chmod +x ./local/bin/py/missing_metrics.py && ./local/bin/py/missing_metrics.py -k $(get_secret 'dd-demo-api-key') -p $(get_secret 'dd-demo-app-key') -a $(get_secret 'dd-api-key') -b $(get_secret 'dd-app-key')
  when: manual
  only:
    - master

# Uploads git ignored source (EN) files to Transifex as part of the nightly build.
# The translation syncing pipeline is triggered by github webhook, but these files are not in git.
# This allows some dynamically built content (such as integrations) to be localized automatically.
manage_ignored_translations:on-schedule:
  <<: *base_template
  stage: post-deploy
  cache: {}
  environment: "live"
  only:
    - schedules
  script:
    - echo "Downloading ignored translation source files..."
    - manage_ignored_translation_files

test_missing_tms_live:
  <<: *base_template
  stage: post-deploy
  cache: {}
  environment: "live"
  variables:
    GIT_STRATEGY: none
  dependencies:
    - build_live
  script:
    - check_missing_tms
  only:
    - master

test_pa11y:
  <<: *base_template
  stage: post-deploy
  cache:
    - *yarn_cache
  environment: "live"
  dependencies:
    - build_live
  script:
    - yarn install --immutable
    - node ./local/bin/js/pa11y.js --env live --dd_api_key $(get_secret 'dd-api-key')
  only:
    - schedules

tag_successful_live_pipeline:
  <<: *base_template
  stage: cleanup
  cache: {}
  environment: "live"
  when: on_success
  script:
    - tag_successful_pipeline
  only:
    - master

post_success_event_to_dd_live:
  <<: *base_template
  stage: cleanup
  cache: {}
  environment: "live"
  variables:
    GIT_STRATEGY: none
  when: on_success
  script:
    - post_dd_event "documentation deploy ${CI_COMMIT_REF_NAME} succeeded" "${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}" "success"
    - post_dd_metric "documentation.pipeline.completed" "1" "" "success"
  only:
    - master

post_failure_event_to_dd_live:
  <<: *base_template
  stage: cleanup
  cache: {}
  environment: "live"
  variables:
    GIT_STRATEGY: none
  when: on_failure
  script:
    - post_dd_event "documentation deploy ${CI_COMMIT_REF_NAME} failed" "${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}" "error"
    - post_dd_metric "documentation.pipeline.completed" "0" "" "failure"
  only:
    - master