diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml new file mode 100644 index 000000000..70b993b61 --- /dev/null +++ b/.github/workflows/cleanup.yml @@ -0,0 +1,33 @@ +name: Cleanup old dev versions + +# Once a day: +# - remove old dev versions +# - execute build of documentation to update download links + +on: + workflow_dispatch: + + schedule: + - cron: "1 3 * * *" + +jobs: + update-package: + name: Cleanup old versions + + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Cleanup + env: + DO_SPACE_ACCESS_KEY: ${{ secrets.DO_SPACE_ACCESS_KEY }} + DO_SPACE_SECRET_KEY: ${{ secrets.DO_SPACE_SECRET_KEY }} + RTD_API_TOKEN: ${{ secrets.RTD_API_TOKEN }} + run: | + cd builder/cleanup + pip install -r requirements.txt + ./cleanup.py + - name: Update documention + curl -vL -X POST -H "Authorization: $Token RTD_API_TOKEN" \ + https://readthedocs.org/api/v3/projects/nethsecurity-docs/versions/latest/builds diff --git a/builder/cleanup/README.md b/builder/cleanup/README.md new file mode 100644 index 000000000..53bc546fc --- /dev/null +++ b/builder/cleanup/README.md @@ -0,0 +1,10 @@ +# Cleanup + +This script is used to remove old image versions from the CDN. + +## How to use it + +``` +pip install -r requirements.txt +DO_SPACE_ACCESS_KEY=xxx DO_SPACE_SECRET_KEY=yyy ./cleanup.py +``` diff --git a/builder/cleanup/cleanup.py b/builder/cleanup/cleanup.py new file mode 100755 index 000000000..2277b2f29 --- /dev/null +++ b/builder/cleanup/cleanup.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +# +# Cleanup old development builds from DigitalOcean Spaces: +# - Keep all tagged releases +# - Keep at least 3 versions of each sub release +# + +import os +import re +import boto3 +import semver + +def owrt_version(release): + match = re.match(r"^([\d.]+)(?:-(.+))?$", release) + if match: + version = match.group(1).split(".") + return (int(version[0]), int(version[1]) if len(version) > 1 else 0, + int(version[2]) if len(version) > 2 else 0) + else: + raise ValueError(f"Invalid OpenWrt release format: {release}") + +def ns_version(version): + try: + return semver.VersionInfo.parse(version) + except ValueError: + return semver.VersionInfo.parse('0.0.0') + +region = "ams3" +bucket_name = "nethsecurity" +s3_client = boto3.session.Session().client( + 's3', + region_name=region, + endpoint_url='https://' + region + '.digitaloceanspaces.com', + aws_access_key_id=os.environ['DO_SPACE_ACCESS_KEY'], + aws_secret_access_key=os.environ['DO_SPACE_SECRET_KEY'] +) +prefix = 'dev' +unordered_versions = [] +max_versions = 3 +response = s3_client.list_objects_v2(Bucket=bucket_name, Prefix=f'{prefix}/', Delimiter='/') +for o in response.get('CommonPrefixes'): + entry = entry = o.get('Prefix').removeprefix(f'{prefix}/').rstrip('/') + if entry.startswith('8-'): + unordered_versions.append(entry) +# Sort by OpenWrt release +owrt_sorted = sorted(unordered_versions, key=lambda x: owrt_version(x)) +# Sort by NethSecurity release +sorted_dev = sorted(owrt_sorted, key=lambda v: ns_version(v)) +to_delete = [] +for version in sorted_dev: + sversion = ns_version(version[13:]) + # keep at least max_versions of each sub release like 8-23.05.2-ns.0.0.1-beta1-3-g4c5b89a + if '-' in sversion.prerelease and len(to_delete) < max_versions: + to_delete.append(version) + +for d in to_delete: + print(f"Deleting {d} ...") + objects_to_delete = s3_client.list_objects(Bucket=bucket_name, Prefix=f"{prefix}/{d}/") + delete_keys = {'Objects' : []} + delete_keys['Objects'] = [{'Key' : k} for k in [obj['Key'] for obj in objects_to_delete.get('Contents', [])] ] + #s3_client.delete_objects(Bucket=bucket_name, Delete=delete_keys) diff --git a/builder/cleanup/requirements.txt b/builder/cleanup/requirements.txt new file mode 100644 index 000000000..967d1230c --- /dev/null +++ b/builder/cleanup/requirements.txt @@ -0,0 +1 @@ +boto3==1.34.33