diff --git a/.circleci/bin/closepr b/.circleci/bin/closepr new file mode 100755 index 0000000..9368109 --- /dev/null +++ b/.circleci/bin/closepr @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +[[ "$TRACE" ]] && set -o xtrace +set -o errexit +set -o nounset +set -o pipefail + +close_pr() { + gh pr close "$CIRCLE_PULL_REQUEST" +} + +delete_branch() { + git push origin --delete "$CIRCLE_BRANCH" +} + +main() { + .circleci/bin/configuregit + close_pr + delete_branch +} + +main diff --git a/.circleci/bin/configuregit b/.circleci/bin/configuregit new file mode 100755 index 0000000..5fa07d5 --- /dev/null +++ b/.circleci/bin/configuregit @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +[[ "$TRACE" ]] && set -o xtrace +set -o errexit +set -o nounset +set -o pipefail + +git config --global user.email "noreply@trade-tariff.service.gov.uk" +git config --global user.name "trade-tariff-bot" diff --git a/.circleci/bin/createpr b/.circleci/bin/createpr new file mode 100755 index 0000000..7045199 --- /dev/null +++ b/.circleci/bin/createpr @@ -0,0 +1,94 @@ +#!/usr/bin/env bash + +[[ "$TRACE" ]] && set -o xtrace +set -o errexit +set -o nounset +set -o pipefail + +EPOCH=$(date +%s) + +update_patch_version() { + local raw_current_version + local current_version + local current_major_version + local current_minor_version + local current_patch_version + local new_patch_version + local new_version + + raw_current_version=$(cat search-config.toml | grep version) + current_version=$(cat search-config.toml | grep version | cut -d= -f2 | tr -d "[:space:]" | tr -d '"') + current_major_version=$(echo $current_version | cut -d. -f1) + current_minor_version=$(echo $current_version | cut -d. -f2) + current_patch_version=$(echo $current_version | cut -d. -f3) + new_patch_version=$((current_patch_version + 1)) + new_version=$(echo "$current_major_version.$current_minor_version.$new_patch_version") + + sed -i "s/$raw_current_version/version = \"$new_version\"/g" search-config.toml +} + +get_branch_name() { + echo "automated-model-run-$EPOCH" +} + +get_model_version() { + echo "$(cat search-config.toml | grep version | cut -d= -f2 | tr -d "[:space:]" | tr -d '"')" +} + +checkout_new_branch() { + git checkout -b "$(get_branch_name)" +} + +stage_commit_and_push_changes() { + git add search-config.toml + git commit -m "BAU: Automated model run $EPOCH." + git push origin "$(get_branch_name)" +} + +open_pr() { + local title + local description + local base_branch="main" + local branch_name + + title="🔄 BAU: Automated model run updated to model version $(get_model_version)" + description=$( + cat < MODEL_VERSION + echo "$VERSION_PREFIX" >MODEL_VERSION } validate_model diff --git a/.circleci/bin/ghinstall b/.circleci/bin/ghinstall new file mode 100755 index 0000000..b1863f8 --- /dev/null +++ b/.circleci/bin/ghinstall @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +sudo apt-get install dirmngr + +curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo gpg --dearmor -o /usr/share/keyrings/githubcli-archive-keyring.gpg + +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null + +sudo apt update +sudo apt install gh + +if [ -z "${GITHUB_TOKEN:-}" ]; then + echo "GITHUB_TOKEN is not set" + exit 1 +fi + +echo "${GITHUB_TOKEN}" | env -u GITHUB_TOKEN gh auth login --with-token --hostname github.com diff --git a/.circleci/bin/isautomatedbuild b/.circleci/bin/isautomatedbuild new file mode 100755 index 0000000..681f689 --- /dev/null +++ b/.circleci/bin/isautomatedbuild @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail +set -o noclobber + +get_current_head_tags() { + git tag --points-at HEAD +} + +is_automated_build() { + if [[ "$(get_current_head_tags)" == *"automated-model-run"* ]]; then + return 0 + else + return 1 + fi +} diff --git a/.circleci/bin/mergepr b/.circleci/bin/mergepr new file mode 100755 index 0000000..a10aa51 --- /dev/null +++ b/.circleci/bin/mergepr @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +[[ "$TRACE" ]] && set -o xtrace +set -o errexit +set -o nounset +set -o pipefail + +source .circleci/bin/isautomatedbuild + +merge_pr() { + gh pr merge "$CIRCLE_PULL_REQUEST" +} + +main() { + .circleci/bin/configuregit + + if is_automated_build; then + echo "Automated build, merging PR" + + merge_pr + fi +} + +main diff --git a/.circleci/config.yml b/.circleci/config.yml index 01e1fd6..b96b3aa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -177,6 +177,13 @@ jobs: command: .circleci/bin/buildami no_output_timeout: 30m + automated-build-model: + executor: base + steps: + - checkout + - run: ".circleci/bin/ghinstall" + - run: ".circleci/bin/createpr" + build-model: executor: base steps: @@ -207,6 +214,13 @@ jobs: name: "Clear Lingering Model Instances" command: .circleci/bin/clearinstances + merge-automated-model-pr: + executor: base + steps: + - checkout + - run: ".circleci/bin/ghinstall" + - run: ".circleci/bin/mergepr" + workflows: version: 2 @@ -288,6 +302,15 @@ workflows: - deploy-development <<: *filter-not-main + - merge-automated-model-pr: + name: merge-automated-model-pr + context: trade-tariff-fpo-models-development + requires: + - smoketest-development + - e2etest-development + - perftest-development + <<: *filter-not-main + deploy-to-staging-and-production: jobs: - fetch-model: @@ -368,3 +391,12 @@ workflows: jobs: - clear-model-instances: context: trade-tariff-fpo-models-development + + bi-weekly-automated-model-build: + triggers: + - schedule: + cron: "0 0 1,15 * *" # 1st and 15th of every month at midnight UTC + <<: *filter-main + jobs: + - automated-build-model: + context: trade-tariff-fpo-models-development diff --git a/requirements.txt b/requirements.txt index 3a41266..ab681b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ distlib==0.3.8 filelock==3.13.4 fsspec==2024.3.1 h11==0.14.0 -huggingface-hub==0.22.2 +huggingface-hub==0.25.2 identify==2.5.35 idna==3.7 Jinja2==3.1.4 diff --git a/requirements_lambda.txt b/requirements_lambda.txt index 7782111..9a6a295 100644 --- a/requirements_lambda.txt +++ b/requirements_lambda.txt @@ -1,6 +1,6 @@ aws-lambda-powertools==2.28.1 jmespath==1.0.1 lingua-language-detector==2.0.2 -sentence-transformers==2.2.2 +sentence-transformers==2.6.1 sentry-sdk==2.8.0 toml==0.10.2