diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..2c9f817 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,25 @@ +# Ignore all differences in line endings +* text=auto eol=lf +*.md text eol=lf +*.py text eol=lf +*.sh text eol=lf +*.yml text eol=lf +*.yaml text eol=lf +*.json text eol=lf +*.properties text eol=lf +*.conf text eol=lf +*.ipynb text eol=lf +Dockerfile* text eol=lf +.gitattributes text eol=lf +.gitignore text eol=lf +.dockerignore text eol=lf + +# Files using LFS to track +*.tgz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.jsonl filter=lfs diff=lfs merge=lfs -text +*.xlsx filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.png filter=lfs diff=lfs merge=lfs -text +*.jpg filter=lfs diff=lfs merge=lfs -text +*.jpeg filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml new file mode 100644 index 0000000..6f538d8 --- /dev/null +++ b/.github/workflows/build-docker.yml @@ -0,0 +1,47 @@ +name: build-docker-images + +on: + push: + branches: [ main ] + paths-ignore: + - "*.md" + + pull_request: + branches: [ main ] + paths-ignore: + - "*.md" + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +env: + REGISTRY_URL: "docker.io" # docker.io or other registry URL, DOCKER_REGISTRY_USER/DOCKER_REGISTRY_PASSWORD to be set in CI env. + BUILDKIT_PROGRESS: "plain" # Full logs for CI build. + + # DOCKER_REGISTRY_USER and DOCKER_REGISTRY_PASSWORD is required for docker image push, they should be set in CI secrets. + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }} + + # used to sync image to mirror registry + DOCKER_MIRROR_REGISTRY_USERNAME: ${{ secrets.DOCKER_MIRROR_REGISTRY_USERNAME }} + DOCKER_MIRROR_REGISTRY_PASSWORD: ${{ secrets.DOCKER_MIRROR_REGISTRY_PASSWORD }} + +jobs: + docker_kit: + name: 'docker-kit' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: | + source ./tool.sh && build_image docker-kit latest docker_docker_kit/Dockerfile && push_image + + sync_images: + needs: ['docker_kit'] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: | + source ./tool.sh + setup_docker_syncer + echo "Syncing image to mirror registry..." + python docker_docker_kit/work/image-syncer/run_jobs.py diff --git a/docker_docker_kit/Dockerfile b/docker_docker_kit/Dockerfile new file mode 100644 index 0000000..5f91212 --- /dev/null +++ b/docker_docker_kit/Dockerfile @@ -0,0 +1,17 @@ +# Distributed under the terms of the Modified BSD License. + +ARG BASE_NAMESPACE +ARG BASE_IMG="base" +FROM ${BASE_NAMESPACE:+$BASE_NAMESPACE/}${BASE_IMG} + +LABEL maintainer="haobibo@gmail.com" + +COPY work /opt/utils/ + +RUN source /opt/utils/script-setup.sh \ + && setup_docker_compose && setup_docker_syncer \ + && pip install -U PyYaml \ + && ln -sf /usr/bin/image-syncer /opt/utils/image-syncer/ + +# Clean up and display components version information... +RUN source /opt/utils/script-utils.sh && install__clean && list_installed_packages diff --git a/docker_docker_kit/work/image-syncer/run_jobs.py b/docker_docker_kit/work/image-syncer/run_jobs.py new file mode 100644 index 0000000..108564c --- /dev/null +++ b/docker_docker_kit/work/image-syncer/run_jobs.py @@ -0,0 +1,50 @@ +import os +import json +import sys +import yaml + +import run_sync + + +def get_job_names_from_yaml(file_path): + """Get all job names from GitHub Actions config file""" + with open(file_path, 'r') as file: + try: + yaml_content = yaml.safe_load(file) + # GitHub Actions YAML file structure has a 'jobs' key at its root + jobs = yaml_content.get('jobs', {}) + for _, v in jobs.items(): + name = v.get('name') + if name is not None: + yield name + except yaml.YAMLError as exc: + print(f"Error parsing YAML file: {exc}") + return [] + + +def main(): + namespace = os.environ.get('IMG_NAMESPACE') + if namespace is None: + print('Using default IMG_NAMESPACE=library !') + namespace = 'library' + + images = [] + job_names = get_job_names_from_yaml('.github/workflows/build-docker.yml') + for name in job_names: + images.extend(name.split(',')) + + for image in images: + img = '/'.join([namespace, image]) + print("Docker image sync job name found:", img) + configs = run_sync.generate(image=img, tags=None) + for _, c in enumerate(configs): + s_config = json.dumps(c, ensure_ascii=False, sort_keys=True) + print('Config item:', json.dumps(c, ensure_ascii=False, sort_keys=True)) + ret = run_sync.sync_image(cfg=c) + if ret != 0: + return ret + return ret + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/docker_docker_kit/work/image-syncer/run_sync.py b/docker_docker_kit/work/image-syncer/run_sync.py new file mode 100644 index 0000000..1585680 --- /dev/null +++ b/docker_docker_kit/work/image-syncer/run_sync.py @@ -0,0 +1,64 @@ +import argparse +import json +import os +import subprocess +import sys +import tempfile + + +def generate(image: str, target_registries: list = None, tags: list = None, target_image: str = None): + """Generate a config item which will be used by `image-syncer`.""" + uname = os.environ.get('DOCKER_MIRROR_REGISTRY_USERNAME', None) + passwd = os.environ.get('DOCKER_MIRROR_REGISTRY_PASSWORD', None) + + if uname is None or passwd is None: + print('ENV variable required: DOCKER_MIRROR_REGISTRY_USERNAME and DOCKER_MIRROR_REGISTRY_PASSWORD !') + sys.exit(-2) + + if target_registries is None: + # , 'cn-shanghai', 'cn-shenzhen', 'cn-chengdu', 'cn-hongkong', 'us-west-1', eu-central-1 + destinations = ['cn-beijing', 'cn-hangzhou'] + target_registries = ['registry.%s.aliyuncs.com' % i for i in destinations] + + for dest in target_registries: + src = "%s:%s" % (image, tags) if tags is not None else image + yield { + 'auth': { + dest: {"username": uname, "password": passwd} + }, + 'images': { + src: "%s/%s" % (dest, target_image or image) + } + } + + +def sync_image(cfg: dict): + """Run the sync task using `image-syncer` with given config item.""" + with tempfile.NamedTemporaryFile(mode='wt', encoding='UTF-8', suffix='.json') as fp: + json.dump(cfg, fp, ensure_ascii=False, indent=2, sort_keys=True) + fp.flush() + ret = 0 + try: + subprocess.run(['image-syncer', '--proc=16', '--retries=2', '--config=' + fp.name], check=True) + except subprocess.CalledProcessError as e: + ret = e.returncode + print(e) + return ret + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('img', type=str, help='Source image, with or without tag') + parser.add_argument('--tags', type=str, action='extend', nargs='*', help='Tags to sync, optional.') + parser.add_argument('--dest-image', type=str, help='Target image name, with our without tag') + parser.add_argument('--dest-registry', type=str, action='extend', nargs='*', help='tTarget registry URL') + args = parser.parse_args() + + configs = generate(image=args.img, tags=args.tags, target_registries=args.dest_registry, target_image=args.dest_image) + for _, c in enumerate(configs): + ret = sync_image(cfg=c) + return ret + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tool.sh b/tool.sh new file mode 100644 index 0000000..6e5198a --- /dev/null +++ b/tool.sh @@ -0,0 +1,105 @@ +#!/bin/bash +set -xu + +CI_PROJECT_NAME=${GITHUB_REPOSITORY:-"QPod/dev-lab"} +CI_PROJECT_BRANCH=${GITHUB_HEAD_REF:-"main"} +CI_PROJECT_SPACE=$(echo "${CI_PROJECT_BRANCH}" | cut -f1 -d'/') + +if [ "${CI_PROJECT_BRANCH}" = "main" ] ; then + # If on the main branch, docker images namespace will be same as CI_PROJECT_NAME's name space + export CI_PROJECT_NAMESPACE="$(dirname ${CI_PROJECT_NAME})" ; +else + # not main branch, docker namespace = {CI_PROJECT_NAME's name space} + "0" + {1st substr before / in CI_PROJECT_SPACE} + export CI_PROJECT_NAMESPACE="$(dirname ${CI_PROJECT_NAME})0${CI_PROJECT_SPACE}" ; +fi + +export IMG_NAMESPACE=$(echo "${CI_PROJECT_NAMESPACE}" | awk '{print tolower($0)}') +export IMG_PREFIX=$(echo "${REGISTRY_URL:-"docker.io"}/${IMG_NAMESPACE}" | awk '{print tolower($0)}') + +echo "--------> CI_PROJECT_NAMESPACE=${CI_PROJECT_NAMESPACE}" +echo "--------> DOCKER_IMG_NAMESPACE=${IMG_NAMESPACE}" +echo "--------> DOCKER_IMG_PREFIX=${IMG_PREFIX}" + + +if [ -f /etc/docker/daemon.json ]; then + jq '.experimental=true | ."data-root"="/mnt/docker"' /etc/docker/daemon.json > /tmp/daemon.json && sudo mv /tmp/daemon.json /etc/docker/ \ + && ( sudo service docker restart || true ) +fi +cat /etc/docker/daemon.json +docker info + +build_image() { + echo "$@" ; + IMG=$1; TAG=$2; FILE=$3; shift 3; VER=$(date +%Y.%m%d.%H%M); WORKDIR="$(dirname $FILE)"; + docker build --compress --force-rm=true -t "${IMG_PREFIX}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${IMG_PREFIX}" "$@" "${WORKDIR}" ; + docker tag "${IMG_PREFIX}/${IMG}:${TAG}" "${IMG_PREFIX}/${IMG}:${VER}" ; +} + +build_image_no_tag() { + echo "$@" ; + IMG=$1; TAG=$2; FILE=$3; shift 3; WORKDIR="$(dirname $FILE)"; + docker build --compress --force-rm=true -t "${IMG_PREFIX}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${IMG_PREFIX}" "$@" "${WORKDIR}" ; +} + +build_image_common() { + echo "$@" ; + IMG=$1; TAG=$2; FILE=$3; shift 3; VER=$(date +%Y.%m%d.%H%M); WORKDIR="$(dirname $FILE)"; + docker build --compress --force-rm=true -t "${IMG_PREFIX}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${IMG_PREFIX}" "$@" "${WORKDIR}" ; + docker tag "${IMG_PREFIX}/${IMG}:${TAG}" "${IMG_PREFIX}/${IMG}:${VER}" ; +} + +alias_image() { + IMG_1=$1; TAG_1=$2; IMG_2=$3; TAG_2=$4; shift 4; VER=$(date +%Y.%m%d.%H%M); + docker tag "${IMG_PREFIX}/${IMG_1}:${TAG_1}" "${IMG_PREFIX}/${IMG_2}:${TAG_2}" ; + docker tag "${IMG_PREFIX}/${IMG_2}:${TAG_2}" "${IMG_PREFIX}/${IMG_2}:${VER}" ; +} + +push_image() { + KEYWORD="${1:-second}"; + docker image prune --force && docker images | sort; + IMAGES=$(docker images | grep "${KEYWORD}" | awk '{print $1 ":" $2}') ; + echo "$DOCKER_REGISTRY_PASSWORD" | docker login "${REGISTRY_URL}" -u "$DOCKER_REGISTRY_USER" --password-stdin ; + for IMG in $(echo "${IMAGES}" | tr " " "\n") ; + do + docker push "${IMG}"; + status=$?; + echo "[${status}] Image pushed > ${IMG}"; + done +} + +clear_images() { + KEYWORD=${1:-'days ago\|weeks ago\|months ago\|years ago'}; # if no keyword is provided, clear all images build days ago + IMGS_1=$(docker images | grep "${KEYWORD}" | awk '{print $1 ":" $2}') ; + IMGS_2=$(docker images | grep "${KEYWORD}" | awk '{print $3}') ; + + for IMG in $(echo "$IMGS_1 $IMGS_2" | tr " " "\n") ; do + docker rmi "${IMG}" || true; status=$?; echo "[${status}] image removed > ${IMG}"; + done + docker image prune --force && docker images ; +} + +setup_docker_syncer() { + ARCH="amd64" \ + && SYNCER_VERSION="$(curl -sL https://github.com/AliyunContainerService/image-syncer/releases.atom | grep 'releases/tag' | head -1 | grep -Po '\d[.\d]+')" \ + && SYNCER_URL="https://github.com/AliyunContainerService/image-syncer/releases/download/v${SYNCER_VERSION}/image-syncer-v${SYNCER_VERSION}-linux-${ARCH}.tar.gz" \ + && echo "Downloading image-syncer from: ${SYNCER_URL}" \ + && curl -o /tmp/image_syncer.tgz -sL ${SYNCER_URL} \ + && mkdir -pv /tmp/image_syncer && tar -zxvf /tmp/image_syncer.tgz -C /tmp/image_syncer \ + && sudo chmod +x /tmp/image_syncer/image-syncer \ + && sudo mv /tmp/image_syncer/image-syncer /usr/bin/ \ + && rm -rf /tmp/image_syncer* \ + && echo "@ image-syncer installed to: $(which image-syncer)" +} + + +remove_folder() { + sudo du -h -d1 "$1" || true ; + sudo rm -rf "$1" || true ; +} + +free_diskspace() { + remove_folder /usr/share/dotnet + remove_folder /usr/local/lib/android + # remove_folder /var/lib/docker + df -h +}