diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 6f538d8..e94840f 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -27,21 +27,66 @@ env: DOCKER_MIRROR_REGISTRY_PASSWORD: ${{ secrets.DOCKER_MIRROR_REGISTRY_PASSWORD }} jobs: - docker_kit: - name: 'docker-kit' + ## Clash + docker_clash: + name: 'app-clash' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - run: | - source ./tool.sh && build_image docker-kit latest docker_docker_kit/Dockerfile && push_image + source ./tool.sh && build_image app-clash latest docker_app_clash/Dockerfile && push_image + ## DevBox - base + qpod_base-dev: + name: 'developer,base-dev' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: | + source ./tool.sh && free_diskspace + build_image base-dev latest docker_devbox/Dockerfile \ + --build-arg "BASE_IMG=node" \ + --build-arg "ARG_PROFILE_JUPYTER=base,kernels,extensions" \ + --build-arg "ARG_PROFILE_VSCODE=base" + alias_image base-dev latest developer latest && push_image dev + ## DevBox - full stack + qpod_full-stack-dev: + name: 'full-stack-dev' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: | + source ./tool.sh && free_diskspace + build_image full-stack-dev latest docker_devbox/Dockerfile \ + --build-arg "BASE_IMG=full-stack" \ + --build-arg "ARG_PROFILE_JUPYTER=base,kernels,extensions" \ + --build-arg "ARG_PROFILE_VSCODE=base" + push_image + ## DevBox - cuda + qpod_cuda-dev: + name: 'full-cuda,cuda-dev' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: | + source ./tool.sh && free_diskspace + build_image cuda-dev latest docker_devbox/Dockerfile \ + --build-arg "BASE_IMG=core-cuda" \ + --build-arg "ARG_PROFILE_JUPYTER=base,kernels,extensions" \ + --build-arg "ARG_PROFILE_VSCODE=base" + alias_image cuda-dev latest full-cuda latest && push_image + + ## Sync all images in this build (listed by "names") to mirror registry. sync_images: - needs: ['docker_kit'] + needs: ["qpod_cuda-dev"] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - run: | source ./tool.sh - setup_docker_syncer - echo "Syncing image to mirror registry..." - python docker_docker_kit/work/image-syncer/run_jobs.py + printenv > /tmp/docker.env + docker run --rm \ + --env-file /tmp/docker.env \ + -v $(pwd):/tmp \ + -w /tmp \ + ${DOCKER_IMG_PREFIX:-qpod}/docker-kit python /opt/utils/image-syncer/run_jobs.py diff --git a/README.md b/README.md index c4b0c5e..c8bc52f 100644 --- a/README.md +++ b/README.md @@ -1 +1,46 @@ -# dev-lab \ No newline at end of file +# QPod Lab Dev - Docker Image Stack + +[![License](https://img.shields.io/badge/License-BSD%203--Clause-green.svg)](https://opensource.org/licenses/BSD-3-Clause) +[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/QPod/lab-dev/build-docker.yml?branch=main)](https://github.com/QPod/lab-dev/actions/workflows/build-docker.yml) +[![Recent Code Update](https://img.shields.io/github/last-commit/QPod/lab-dev.svg)](https://github.com/QPod/lab-dev/stargazers) +[![Visit Images on DockerHub](https://img.shields.io/badge/DockerHub-Images-green)](https://hub.docker.com/u/qpod) + +Please generously STAR★ our project or donate to us! [![GitHub Starts](https://img.shields.io/github/stars/QPod/lab-dev.svg?label=Stars&style=social)](https://github.com/QPod/lab-dev/stargazers) +[![Donate-PayPal](https://img.shields.io/badge/Donate-PayPal-blue.svg)](https://paypal.me/haobibo) +[![Donate-AliPay](https://img.shields.io/badge/Donate-Alipay-blue.svg)](https://raw.githubusercontent.com/wiki/haobibo/resources/img/Donate-AliPay.png) +[![Donate-WeChat](https://img.shields.io/badge/Donate-WeChat-green.svg)](https://raw.githubusercontent.com/wiki/haobibo/resources/img/Donate-WeChat.png) + +Discussion and contributions are welcome: +[![Join the Discord Chat](https://img.shields.io/badge/Discuss_on-Discord-green)](https://discord.gg/kHUzgQxgbJ) +[![Open an Issue on GitHub](https://img.shields.io/github/issues/QPod/lab-foundation)](https://github.com/QPod/lab-foundation/issues) + +## Lab Dev - Building Blocks and IDEs for Application Development + +`QPod lab-dev` ( [DockerHub](https://hub.docker.com/u/qpod) | [GitHub](https://github.com/QPod/lab-dev) ) provides Building Blocks and IDEs for Application Development. + +## Documentation & Tutorial + +[Wiki & Document](https://qpod.github.io/) | [中文使用指引(含中国网络镜像)](https://qpod.github.io/docs/intro-cn) + +## Screenshot and Arch Diagram + +![Screenshot of QPod](https://raw.githubusercontent.com/wiki/QPod/qpod-hub/img/QPod-screenshot.webp "Screenshot of QPod") + +![Arch Diagram for QPod DevBox](https://raw.githubusercontent.com/wiki/QPod/docker-images/img/QPod-arch.svg "Arch Diagram for QPod DevBox") + +## Develop and Debug + +```bash +IMG="qpod/developer" +# IMG="registry.cn-hangzhou.aliyuncs.com/qpod/developer" + +docker run -d --restart=always \ + --name=QPod-lab-dev \ + --hostname=QPod \ + -p 18888-18890:8888-8890 \ + -v $(pwd):/root/lab-dev \ + -w /root/lab-dev \ + $IMG + +sleep 5s && docker logs QPod-lab-dev 2>&1|grep token= +``` diff --git a/docker_app_clash/Dockerfile b/docker_app_clash/Dockerfile new file mode 100644 index 0000000..a05a5fc --- /dev/null +++ b/docker_app_clash/Dockerfile @@ -0,0 +1,12 @@ +FROM alpine AS builder + +# https://github.com/haishanh/yacd +WORKDIR /tmp +RUN wget https://github.com/haishanh/yacd/archive/gh-pages.zip \ + && unzip ./*.zip && rm -rf *.zip \ + && mv ./* ./ui + +# https://github.com/Dreamacro/clash +FROM dreamacro/clash-premium +COPY --from=builder /tmp/* /root/.config/clash/ui +COPY config.yaml /root/.config/clash/ diff --git a/docker_app_clash/config.yaml b/docker_app_clash/config.yaml new file mode 100644 index 0000000..6181fcf --- /dev/null +++ b/docker_app_clash/config.yaml @@ -0,0 +1,20 @@ +# https://github.com/Dreamacro/clash/wiki/configuration#all-configuration-options +mixed-port: 7890 +external-controller: :9090 +allow-lan: true +bind-address: '*' +log-level: debug +mode: Rule +external-ui: ui + +rules: + - DOMAIN-SUFFIX,local,DIRECT + - IP-CIDR,192.168.0.0/16,DIRECT,no-resolve + - IP-CIDR,10.0.0.0/8,DIRECT,no-resolve + - IP-CIDR,172.16.0.0/12,DIRECT,no-resolve + - IP-CIDR,127.0.0.0/8,DIRECT,no-resolve + - IP-CIDR,100.64.0.0/10,DIRECT,no-resolve + - IP-CIDR6,::1/128,DIRECT,no-resolve + - IP-CIDR6,fc00::/7,DIRECT,no-resolve + - IP-CIDR6,fe80::/10,DIRECT,no-resolve + - IP-CIDR6,fd00::/8,DIRECT,no-resolve diff --git a/docker_app_clash/docker-compose.yml b/docker_app_clash/docker-compose.yml new file mode 100644 index 0000000..6207ce4 --- /dev/null +++ b/docker_app_clash/docker-compose.yml @@ -0,0 +1,17 @@ +version: '3.9' + +# ref1: https://github.com/Dreamacro/clash/wiki/Run-clash-in-docker +# ref2: https://github.com/Dreamacro/clash/wiki/configuration + +services: + svc-clash: + image: 0x06/clash-premium + container_name: svc-clash + restart: unless-stopped + volumes: + - ./config.yaml:/root/.config/clash/config.yaml:ro + ports: + - "7890:7890" + - "9999:9090" + # When your system is Linux, you can use `network_mode: "host"` directly. + # network_mode: "host" diff --git a/docker_devbox/Dockerfile b/docker_devbox/Dockerfile new file mode 100644 index 0000000..2097d9c --- /dev/null +++ b/docker_devbox/Dockerfile @@ -0,0 +1,48 @@ +# Distributed under the terms of the Modified BSD License. + +ARG BASE_NAMESPACE +ARG BASE_IMG="core" +FROM ${BASE_NAMESPACE:+$BASE_NAMESPACE/}${BASE_IMG} + +LABEL maintainer="haobibo@gmail.com" + +# base,kernels,extensions +ARG ARG_PROFILE_JUPYTER=base + +# base +ARG ARG_PROFILE_VSCODE=base + +ARG ARG_KEEP_NODEJS=true + +COPY work /opt/utils/ + +# Setup Jupyter: Basic Configurations and Extensions... +RUN mkdir -pv /opt/conda/etc/jupyter/ \ + && mv /opt/utils/etc_jupyter/* /opt/conda/etc/jupyter/ && rm -rf /opt/utils/etc_jupyter \ + && mv /opt/utils/start-*.sh /usr/local/bin/ && chmod +x /usr/local/bin/start-*.sh \ + && source /opt/utils/script-devbox-jupyter.sh \ + && for profile in $(echo $ARG_PROFILE_JUPYTER | tr "," "\n") ; do ( setup_jupyter_${profile} || true ) ; done + +# If not keeping NodeJS, remove NoedJS to reduce image size +RUN ${ARG_KEEP_NODEJS:-true} || ( echo "Removing Node/NPM..." && rm -rf /usr/bin/node /usr/bin/npm /usr/bin/npx /opt/node ) + +# If installing coder-server # https://github.com/cdr/code-server/releases +RUN source /opt/utils/script-devbox-vscode.sh \ + && for profile in $(echo $ARG_PROFILE_VSCODE | tr "," "\n") ; do ( setup_vscode_${profile} || true ) ; done + +# Clean up and display components version information... +RUN source /opt/utils/script-utils.sh && install__clean && list_installed_packages + +WORKDIR $HOME_DIR +EXPOSE 8888 + +ENTRYPOINT ["tini", "-g", "--"] +CMD ["start-notebook.sh"] + +# '-c' option make bash commands are read from string. +# If there are arguments after the string, they are assigned to the positional parameters, starting with $0. +# '-o pipefail' prevents errors in a pipeline from being masked. +# If any command in a pipeline fails, that return code will be used as the return code of the whole pipeline. +# '--login': make bash first reads and executes commands from the file /etc/profile, if that file exists. +# After that, it looks for ~/.bash_profile, ~/.bash_login, and ~/.profile, in that order, and reads and executes commands from the first one that exists and is readable. +SHELL ["/bin/bash", "--login", "-o", "pipefail", "-c"] diff --git a/docker_devbox/work/etc_jupyter/jupyter_server_config.json b/docker_devbox/work/etc_jupyter/jupyter_server_config.json new file mode 100644 index 0000000..3196589 --- /dev/null +++ b/docker_devbox/work/etc_jupyter/jupyter_server_config.json @@ -0,0 +1,11 @@ +{ + "ServerApp": { + "ip": "0.0.0.0", + "port": 8888, + "root_dir": "/root", + "default_url": "/lab", + "allow_root": true, + "allow_origin": "*", + "open_browser": false + } +} diff --git a/docker_devbox/work/install_list_JPY_extend.apt b/docker_devbox/work/install_list_JPY_extend.apt new file mode 100644 index 0000000..aeea759 --- /dev/null +++ b/docker_devbox/work/install_list_JPY_extend.apt @@ -0,0 +1,5 @@ +% This file contains apt packages to be installed with apt-get install line by line. +% Use percent char as line comment separator. + +libcairo2-dev libjpeg8-dev libpango1.0-dev libgif-dev % optional libs to install JS lib canvas +libzmq3-dev % required to build some jupyter kernels like libpango diff --git a/docker_devbox/work/install_list_JPY_extend.pip b/docker_devbox/work/install_list_JPY_extend.pip new file mode 100644 index 0000000..9315570 --- /dev/null +++ b/docker_devbox/work/install_list_JPY_extend.pip @@ -0,0 +1,10 @@ +% This file contains python packages to be installed with pip line by line. +% Use percent char as line comment separator. + +ipyparallel % https://github.com/ipython/ipyparallel +ipynb % https://github.com/ipython/ipynb +jupyter-resource-usage % https://github.com/jupyter-server/jupyter-resource-usage +jupyterlab_server[openapi] % https://github.com/jupyterlab/jupyterlab_server +jupyterlab-git % https://github.com/jupyterlab/jupyterlab-git +% jupyter-collaboration % https://github.com/jupyterlab/jupyter-collaboration +% jupyterlab-latex % bug on pypi version: https://github.com/jupyterlab/jupyterlab-latex diff --git a/docker_devbox/work/script-devbox-jupyter.sh b/docker_devbox/work/script-devbox-jupyter.sh new file mode 100644 index 0000000..de56f61 --- /dev/null +++ b/docker_devbox/work/script-devbox-jupyter.sh @@ -0,0 +1,81 @@ +source /opt/utils/script-utils.sh + + +setup_jupyter_base() { + pip install -Uq --pre jupyterhub jupyterlab notebook ipywidgets jupyter-server-proxy \ + && echo "@ Version of Jupyter Server: $(jupyter server --version)" \ + && echo "@ Version of Jupyter Lab: $(jupyter lab --version)" \ + && echo "@ Version of Jupyter Notebook: $(jupyter notebook --version)" \ + && echo "@ Version of JupyterHub: $(jupyterhub --version)" +} + + +setup_jupyter_kernels() { + echo "@ Jupyter Kernels RefList: https://github.com/jupyter/jupyter/wiki/Jupyter-kernels" + + echo "@ Install Jupyter Kernel for Bash" \ + && pip install -Uq bash_kernel && python -m bash_kernel.install --sys-prefix + + ## checked @ 2024-0307 + which npm \ + && echo "@ Install Jupyter Kernel for JavaScript/TypeScript: https://github.com/yunabe/tslab" \ + && npm install -g tslab \ + && tslab install --sys-prefix --python /opt/conda/bin/python --binary $(which tslab) + ## ref: https://github.com/n-riesco/ijavascript (#TODO: not working for now) + # && npm_config_zmq_external=true npm install -g --unsafe-perm ijavascript \ + # && /opt/node/bin/ijsinstall --install=global --spec-path=full \ + # && mv /usr/local/share/jupyter/kernels/javascript /opt/conda/share/jupyter/kernels/ + + ## checked @ 2024-0307 # TODO: help func requries proxy + which R \ + && echo "@ Install Jupyter Kernel for R: https://github.com/melff/RKernel" \ + && R -e "devtools::install_github('melff/RKernel/pkg')" \ + && R -e "RKernel::installspec(user=FALSE)" \ + && mv /usr/local/share/jupyter/kernels/rkernel /opt/conda/share/jupyter/kernels/ + + ## checked @ 2024-0307 + which go \ + && echo "@ Install Jupyter Kernel for Golang: https://github.com/janpfeifer/gonb" \ + && export GOPATH=/opt/go/path \ + && go install github.com/janpfeifer/gonb@latest \ + && go install golang.org/x/tools/cmd/goimports@latest \ + && go install golang.org/x/tools/gopls@latest \ + && $GOPATH/bin/gonb --install \ + && mv ~/.local/share/jupyter/kernels/gonb /opt/conda/share/jupyter/kernels/ + + ## TODO: buggy, cargo install sccache + which rustc \ + && echo "@ Install Jupyter Kernel for Rust: https://github.com/evcxr/evcxr/issues/140" \ + && cargo install --locked evcxr_jupyter \ + && evcxr_jupyter --install --sys-prefix \ + && mv ~/.local/share/jupyter/kernels/rust /opt/conda/share/jupyter/kernels/ + + ## TODO: buggy + which java \ + && pip install -Uq pandas py4j + #&& pip install beakerx && beakerx install \ + #&& jupyter labextension install beakerx-jupyterlab + + ## checked @ 2024-0307 + which julia \ + && echo "@ Install Jupyter Kernel for Julia: https://github.com/JuliaLang/IJulia.jl" \ + && julia -e "using Pkg; Pkg.add(\"IJulia\"); Pkg.precompile();" \ + && mv ~/.local/share/jupyter/kernels/julia* /opt/conda/share/jupyter/kernels/ + + ## https://github.com/jupyter-xeus/xeus-octave # TODO: to check + which octave \ + && export PATH=/opt/octave/bin:$PATH \ + && pip install -Uq xeus-python + + echo "@ Installed Jupyter Kernels:" && jupyter kernelspec list +} + + +setup_jupyter_extensions() { + install_apt /opt/utils/install_list_JPY_extend.apt \ + && install_pip /opt/utils/install_list_JPY_extend.pip + + echo "@ Jupyter Server Extension list: " && jupyter server extension list \ + && echo "@ Jupyter Lab Extension list: " && jupyter labextension list \ + && echo "@ Jupyter Notebook Extension list: " && jupyter notebook extension list +} diff --git a/docker_devbox/work/script-devbox-vscode.sh b/docker_devbox/work/script-devbox-vscode.sh new file mode 100644 index 0000000..17c9add --- /dev/null +++ b/docker_devbox/work/script-devbox-vscode.sh @@ -0,0 +1,13 @@ +source /opt/utils/script-utils.sh + +# ref: https://github.com/coder/code-server + +setup_vscode_base() { + VERSION_CODER=$(curl -sL https://github.com/cdr/code-server/releases.atom | grep "releases/tag" | head -1 | grep -Po '(\d[\d|.]+)') \ + && install_tar_gz "https://github.com/cdr/code-server/releases/download/v${VERSION_CODER}/code-server-${VERSION_CODER}-linux-amd64.tar.gz" \ + && mv /opt/code-server* /opt/code-server \ + && ln -s /opt/code-server/bin/code-server /usr/bin/ \ + && printf "#!/bin/bash\n/opt/code-server/bin/code-server --port=8888 --auth=none --disable-telemetry ${HOME}\n" > /usr/local/bin/start-code-server.sh \ + && chmod u+x /usr/local/bin/start-code-server.sh \ + && echo "@ coder-server Version: $(code-server -v)" +} diff --git a/docker_devbox/work/start--pre.sh b/docker_devbox/work/start--pre.sh new file mode 100644 index 0000000..692a5fd --- /dev/null +++ b/docker_devbox/work/start--pre.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -e + +# Utilities function to compare version. +ver_gte() { [ "$2" = "`echo -e "$1\n$2" | sort -V | head -n1`" ]; } + +# Generate a SSH id for git if it does not exist. +[ -e ~/.ssh/id_rsa.pub ] || ssh-keygen -t rsa -b 4096 -N "" -C `hostname -f` -f ~/.ssh/id_rsa + +# Generate a self-signed certificate for notebook if it does not exist (only when GEN_CERT or USE_SSL is set to yes). +NOTEBOOK_PEM_FILE="/opt/conda/etc/jupyter/notebook.pem" +( [ -n "${GEN_CERT:+x}" ] || [ -n "${USE_SSL:+x}" ] ) && [ ! -f ${NOTEBOOK_PEM_FILE} ] && ( openssl req -new -newkey rsa:2048 \ + -days 356 -nodes -x509 -subj "/C=XX/ST=XX/L=XX/O=generated/CN=generated" -keyout $NOTEBOOK_PEM_FILE -out $NOTEBOOK_PEM_FILE \ + && chmod 600 $NOTEBOOK_PEM_FILE ) + +# Print something so running this script returns a non-zero return code +echo "Pre-start work done!" diff --git a/docker_devbox/work/start-jupyterlab.sh b/docker_devbox/work/start-jupyterlab.sh new file mode 100644 index 0000000..714fb29 --- /dev/null +++ b/docker_devbox/work/start-jupyterlab.sh @@ -0,0 +1,14 @@ +#!/bin/bash +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +. $DIR/start--pre.sh + +NOTEBOOK_ARGS="" +[ -n "${USE_SSL:+x}" ] && NOTEBOOK_ARGS="${NOTEBOOK_ARGS} --NotebookApp.certfile=${NOTEBOOK_PEM_FILE}" + + +if [[ ! -z "${JUPYTERHUB_API_TOKEN}" ]]; then + # launched by JupyterHub, use single-user entrypoint + exec $DIR/start-singleuser.sh $* +else + jupyter lab ${NOTEBOOK_ARGS} $* +fi diff --git a/docker_devbox/work/start-notebook.sh b/docker_devbox/work/start-notebook.sh new file mode 100644 index 0000000..d1b53a8 --- /dev/null +++ b/docker_devbox/work/start-notebook.sh @@ -0,0 +1,14 @@ +#!/bin/bash +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +. $DIR/start--pre.sh + +NOTEBOOK_ARGS="" +[ -n "${USE_SSL:+x}" ] && NOTEBOOK_ARGS="${NOTEBOOK_ARGS} --NotebookApp.certfile=${NOTEBOOK_PEM_FILE}" + + +if [[ ! -z "${JUPYTERHUB_API_TOKEN}" ]]; then + # launched by JupyterHub, use single-user entrypoint + exec $DIR/start-singleuser.sh $* +else + jupyter notebook ${NOTEBOOK_ARGS} $* +fi diff --git a/docker_devbox/work/start-singleuser.sh b/docker_devbox/work/start-singleuser.sh new file mode 100644 index 0000000..7ebf9a9 --- /dev/null +++ b/docker_devbox/work/start-singleuser.sh @@ -0,0 +1,34 @@ +#!/bin/bash +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +. $DIR/start--pre.sh + +if [[ "$NOTEBOOK_ARGS $@" != *"--ip="* ]]; then + NOTEBOOK_ARGS="--ip=0.0.0.0 $NOTEBOOK_ARGS" +fi + + +# handle some deprecated environment variables from DockerSpawner < 0.8. +# These won't be passed from DockerSpawner 0.9, so avoid specifying --arg=empty-string +if [ ! -z "$NOTEBOOK_DIR" ]; then + NOTEBOOK_ARGS="--notebook-dir='$NOTEBOOK_DIR' $NOTEBOOK_ARGS" +fi +if [ ! -z "$JPY_PORT" ]; then + NOTEBOOK_ARGS="--port=$JPY_PORT $NOTEBOOK_ARGS" +fi +if [ ! -z "$JPY_USER" ]; then + NOTEBOOK_ARGS="--user=$JPY_USER $NOTEBOOK_ARGS" +fi +if [ ! -z "$JPY_COOKIE_NAME" ]; then + NOTEBOOK_ARGS="--cookie-name=$JPY_COOKIE_NAME $NOTEBOOK_ARGS" +fi +if [ ! -z "$JPY_BASE_URL" ]; then + NOTEBOOK_ARGS="--base-url=$JPY_BASE_URL $NOTEBOOK_ARGS" +fi +if [ ! -z "$JPY_HUB_PREFIX" ]; then + NOTEBOOK_ARGS="--hub-prefix=$JPY_HUB_PREFIX $NOTEBOOK_ARGS" +fi +if [ ! -z "$JPY_HUB_API_URL" ]; then + NOTEBOOK_ARGS="--hub-api-url=$JPY_HUB_API_URL $NOTEBOOK_ARGS" +fi + +exec jupyterhub-singleuser ${NOTEBOOK_ARGS} $@ diff --git a/docker_docker_kit/Dockerfile b/docker_docker_kit/Dockerfile deleted file mode 100644 index 5f91212..0000000 --- a/docker_docker_kit/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -# Distributed under the terms of the Modified BSD License. - -ARG BASE_NAMESPACE -ARG BASE_IMG="base" -FROM ${BASE_NAMESPACE:+$BASE_NAMESPACE/}${BASE_IMG} - -LABEL maintainer="haobibo@gmail.com" - -COPY work /opt/utils/ - -RUN source /opt/utils/script-setup.sh \ - && setup_docker_compose && setup_docker_syncer \ - && pip install -U PyYaml \ - && ln -sf /usr/bin/image-syncer /opt/utils/image-syncer/ - -# Clean up and display components version information... -RUN source /opt/utils/script-utils.sh && install__clean && list_installed_packages diff --git a/docker_docker_kit/work/image-syncer/run_jobs.py b/docker_docker_kit/work/image-syncer/run_jobs.py deleted file mode 100644 index 108564c..0000000 --- a/docker_docker_kit/work/image-syncer/run_jobs.py +++ /dev/null @@ -1,50 +0,0 @@ -import os -import json -import sys -import yaml - -import run_sync - - -def get_job_names_from_yaml(file_path): - """Get all job names from GitHub Actions config file""" - with open(file_path, 'r') as file: - try: - yaml_content = yaml.safe_load(file) - # GitHub Actions YAML file structure has a 'jobs' key at its root - jobs = yaml_content.get('jobs', {}) - for _, v in jobs.items(): - name = v.get('name') - if name is not None: - yield name - except yaml.YAMLError as exc: - print(f"Error parsing YAML file: {exc}") - return [] - - -def main(): - namespace = os.environ.get('IMG_NAMESPACE') - if namespace is None: - print('Using default IMG_NAMESPACE=library !') - namespace = 'library' - - images = [] - job_names = get_job_names_from_yaml('.github/workflows/build-docker.yml') - for name in job_names: - images.extend(name.split(',')) - - for image in images: - img = '/'.join([namespace, image]) - print("Docker image sync job name found:", img) - configs = run_sync.generate(image=img, tags=None) - for _, c in enumerate(configs): - s_config = json.dumps(c, ensure_ascii=False, sort_keys=True) - print('Config item:', json.dumps(c, ensure_ascii=False, sort_keys=True)) - ret = run_sync.sync_image(cfg=c) - if ret != 0: - return ret - return ret - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/docker_docker_kit/work/image-syncer/run_sync.py b/docker_docker_kit/work/image-syncer/run_sync.py deleted file mode 100644 index 1585680..0000000 --- a/docker_docker_kit/work/image-syncer/run_sync.py +++ /dev/null @@ -1,64 +0,0 @@ -import argparse -import json -import os -import subprocess -import sys -import tempfile - - -def generate(image: str, target_registries: list = None, tags: list = None, target_image: str = None): - """Generate a config item which will be used by `image-syncer`.""" - uname = os.environ.get('DOCKER_MIRROR_REGISTRY_USERNAME', None) - passwd = os.environ.get('DOCKER_MIRROR_REGISTRY_PASSWORD', None) - - if uname is None or passwd is None: - print('ENV variable required: DOCKER_MIRROR_REGISTRY_USERNAME and DOCKER_MIRROR_REGISTRY_PASSWORD !') - sys.exit(-2) - - if target_registries is None: - # , 'cn-shanghai', 'cn-shenzhen', 'cn-chengdu', 'cn-hongkong', 'us-west-1', eu-central-1 - destinations = ['cn-beijing', 'cn-hangzhou'] - target_registries = ['registry.%s.aliyuncs.com' % i for i in destinations] - - for dest in target_registries: - src = "%s:%s" % (image, tags) if tags is not None else image - yield { - 'auth': { - dest: {"username": uname, "password": passwd} - }, - 'images': { - src: "%s/%s" % (dest, target_image or image) - } - } - - -def sync_image(cfg: dict): - """Run the sync task using `image-syncer` with given config item.""" - with tempfile.NamedTemporaryFile(mode='wt', encoding='UTF-8', suffix='.json') as fp: - json.dump(cfg, fp, ensure_ascii=False, indent=2, sort_keys=True) - fp.flush() - ret = 0 - try: - subprocess.run(['image-syncer', '--proc=16', '--retries=2', '--config=' + fp.name], check=True) - except subprocess.CalledProcessError as e: - ret = e.returncode - print(e) - return ret - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('img', type=str, help='Source image, with or without tag') - parser.add_argument('--tags', type=str, action='extend', nargs='*', help='Tags to sync, optional.') - parser.add_argument('--dest-image', type=str, help='Target image name, with our without tag') - parser.add_argument('--dest-registry', type=str, action='extend', nargs='*', help='tTarget registry URL') - args = parser.parse_args() - - configs = generate(image=args.img, tags=args.tags, target_registries=args.dest_registry, target_image=args.dest_image) - for _, c in enumerate(configs): - ret = sync_image(cfg=c) - return ret - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/tool.sh b/tool.sh index 6e5198a..eac3c98 100644 --- a/tool.sh +++ b/tool.sh @@ -1,7 +1,7 @@ #!/bin/bash set -xu -CI_PROJECT_NAME=${GITHUB_REPOSITORY:-"QPod/dev-lab"} +CI_PROJECT_NAME=${GITHUB_REPOSITORY:-"QPod/lab-dev"} CI_PROJECT_BRANCH=${GITHUB_HEAD_REF:-"main"} CI_PROJECT_SPACE=$(echo "${CI_PROJECT_BRANCH}" | cut -f1 -d'/')