From 2e1c23c291a63cb0ed7bd5b64a9d064a821175c0 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Mon, 5 Aug 2024 13:58:55 +0800 Subject: [PATCH 1/6] fix docker cache --- tools/ci_build/get_docker_image.py | 18 +++++++++++++++--- .../templates/c-api-linux-cpu.yml | 8 ++++---- .../templates/get-docker-image-steps.yml | 1 + .../inference/aarch64/default/cpu/Dockerfile | 2 +- .../inference/x86_64/default/cpu/Dockerfile | 2 +- 5 files changed, 22 insertions(+), 9 deletions(-) diff --git a/tools/ci_build/get_docker_image.py b/tools/ci_build/get_docker_image.py index 99ecaf677f339..0b244026debf8 100755 --- a/tools/ci_build/get_docker_image.py +++ b/tools/ci_build/get_docker_image.py @@ -98,17 +98,24 @@ def main(): ) if use_container_registry: + run(args.docker_path, + "buildx", + "create", + "--driver=docker-container", + "--name=container_builder" + ) run( args.docker_path, "--log-level", "error", "buildx", "build", - "--push", + "--load", "--tag", full_image_name, - "--cache-from", - full_image_name, + "--cache-from=type=registry,ref=" + full_image_name, + "--builder", + "container_builder", "--build-arg", "BUILDKIT_INLINE_CACHE=1", *shlex.split(args.docker_build_args), @@ -116,6 +123,11 @@ def main(): args.dockerfile, args.context, ) + run( + args.docker_path, + "push", + full_image_name, + ) elif args.use_imagecache: log.info("Building image with pipeline cache...") run( diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml index e2b71c5c55fd2..0f4328f75e1bd 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml @@ -51,15 +51,15 @@ jobs: Dockerfile: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile Context: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=${{parameters.BaseImage}}" - Repository: onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}} - + Repository: onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}}_packaging + - ${{ if eq(parameters.OnnxruntimeArch, 'aarch64') }}: - template: get-docker-image-steps.yml parameters: Dockerfile: tools/ci_build/github/linux/docker/inference/aarch64/default/cpu/Dockerfile Context: tools/ci_build/github/linux/docker/inference/aarch64/default/cpu DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=${{parameters.BaseImage}}" - Repository: onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}} + Repository: onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}}_packaging UpdateDepsTxt: false - task: CmdLine@2 @@ -67,7 +67,7 @@ jobs: script: | mkdir -p $HOME/.onnx docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ - --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}} /bin/bash -c "python3.9 \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}}_packaging /bin/bash -c "python3.9 \ /onnxruntime_src/tools/ci_build/build.py --enable_lto --build_java --build_nodejs --build_dir /build --config Release \ --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/installed" workingDirectory: $(Build.SourcesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml b/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml index 94cdf042ec62b..da59bd1f4b0ea 100644 --- a/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml @@ -53,6 +53,7 @@ steps: displayName: patch manylinux - script: | + docker version -- docker image ls docker system df displayName: Check Docker Images diff --git a/tools/ci_build/github/linux/docker/inference/aarch64/default/cpu/Dockerfile b/tools/ci_build/github/linux/docker/inference/aarch64/default/cpu/Dockerfile index 2cd054e6246bc..ca00050121d67 100644 --- a/tools/ci_build/github/linux/docker/inference/aarch64/default/cpu/Dockerfile +++ b/tools/ci_build/github/linux/docker/inference/aarch64/default/cpu/Dockerfile @@ -5,7 +5,7 @@ ARG BASEIMAGE=arm64v8/almalinux:8 FROM $BASEIMAGE -ENV PATH /opt/rh/gcc-toolset-12/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +ENV PATH=/opt/rh/gcc-toolset-12/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ENV LANG=en_US.UTF-8 ENV LC_ALL=en_US.UTF-8 diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile b/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile index caf9583807b62..ef28dde67617f 100644 --- a/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile +++ b/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile @@ -5,7 +5,7 @@ ARG BASEIMAGE=amd64/almalinux:8 FROM $BASEIMAGE -ENV PATH /usr/lib/jvm/msopenjdk-11/bin:/opt/rh/gcc-toolset-12/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +ENV PATH=/usr/lib/jvm/msopenjdk-11/bin:/opt/rh/gcc-toolset-12/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ENV LANG=en_US.UTF-8 ENV LC_ALL=en_US.UTF-8 ENV JAVA_HOME=/usr/lib/jvm/msopenjdk-11 From 10dc0f9fba3c8d102a30e381d294e00ddbc62a54 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Mon, 5 Aug 2024 14:37:06 +0800 Subject: [PATCH 2/6] typo --- .../github/azure-pipelines/templates/get-docker-image-steps.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml b/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml index da59bd1f4b0ea..33194a2ce644c 100644 --- a/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml @@ -53,7 +53,7 @@ steps: displayName: patch manylinux - script: | - docker version -- + docker version docker image ls docker system df displayName: Check Docker Images From 47b49ce4a687dc6d260302d1737716995bcc53fd Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Mon, 5 Aug 2024 15:31:50 +0800 Subject: [PATCH 3/6] lint --- tools/ci_build/get_docker_image.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tools/ci_build/get_docker_image.py b/tools/ci_build/get_docker_image.py index 0b244026debf8..e569d00d55e16 100755 --- a/tools/ci_build/get_docker_image.py +++ b/tools/ci_build/get_docker_image.py @@ -98,12 +98,7 @@ def main(): ) if use_container_registry: - run(args.docker_path, - "buildx", - "create", - "--driver=docker-container", - "--name=container_builder" - ) + run(args.docker_path, "buildx", "create", "--driver=docker-container", "--name=container_builder") run( args.docker_path, "--log-level", From b8bcfd02249b941f7ce892ad80d7fcd34a981dba Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Tue, 6 Aug 2024 11:10:07 +0800 Subject: [PATCH 4/6] no cache in A100 --- .../azure-pipelines/bigmodels-ci-pipeline.yml | 1 + .../templates/get-docker-image-steps.yml | 52 +++---------------- 2 files changed, 7 insertions(+), 46 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml index a66828ee5e188..4a3532dd57fa3 100644 --- a/tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml @@ -321,6 +321,7 @@ stages: --build-arg TRT_VERSION=${{ variables.linux_trt_version }} " Repository: onnxruntimeubi8packagestest_torch + UseImageCacheContainerRegistry: false UpdateDepsTxt: false - task: DownloadPackage@1 diff --git a/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml b/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml index 33194a2ce644c..812f6bf4e54e8 100644 --- a/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml @@ -71,53 +71,13 @@ steps: --repository "${{ parameters.Repository }}" displayName: "Get ${{ parameters.Repository }} image for ${{ parameters.Dockerfile }}" ContainerRegistry: onnxruntimebuildcache -- ${{ if eq(parameters.UseImageCacheContainerRegistry, false) }}: - - task: Cache@2 - displayName: Cache Docker Image Task - inputs: - key: ' "${{ parameters.Repository }}" | "$(Build.SourceVersion)" ' - path: ${{ parameters.IMAGE_CACHE_DIR }} - restoreKeys: | - "${{ parameters.Repository }}" | "$(Build.SourceVersion)" - "${{ parameters.Repository }}" - cacheHitVar: CACHE_RESTORED - condition: eq('${{ parameters.UsePipelineCache }}', 'true') - - - script: | - test -f ${{ parameters.IMAGE_CACHE_DIR }}/cache.tar && docker load -i ${{ parameters.IMAGE_CACHE_DIR }}/cache.tar - docker image ls - displayName: Docker restore - condition: eq('${{ parameters.UsePipelineCache }}', 'true') - - - script: | - if [ ${{ parameters.UsePipelineCache}} ] - then - use_imagecache="--use_imagecache" - else - use_imagecache="" - fi - ${{ parameters.ScriptName }} \ - --dockerfile "${{ parameters.Dockerfile }}" \ - --context "${{ parameters.Context }}" \ - --docker-build-args "${{ parameters.DockerBuildArgs }}" \ - --repository "${{ parameters.Repository }}" \ - $use_imagecache - displayName: "Get ${{ parameters.Repository }} image for ${{ parameters.Dockerfile }}" - - script: | - set -ex - mkdir -p "${{ parameters.IMAGE_CACHE_DIR }}" - docker save -o "${{ parameters.IMAGE_CACHE_DIR }}/cache.tar" ${{ parameters.Repository }} - docker image ls - docker system df - displayName: Docker save - condition: eq('${{ parameters.UsePipelineCache }}', 'true') - - - script: | - echo ${{ parameters.IMAGE_CACHE_DIR }} - ls -lah ${{ parameters.IMAGE_CACHE_DIR }} - displayName: Display docker dir - condition: eq('${{ parameters.UsePipelineCache }}', 'true') +- script: | + docker version + docker image ls + docker system df + df -h + displayName: Check Docker Images - ${{ if and(eq(parameters.UpdateDepsTxt, true), or(eq(variables['System.CollectionId'], 'f3ad12f2-e480-4533-baf2-635c95467d29'),eq(variables['System.CollectionId'], 'bc038106-a83b-4dab-9dd3-5a41bc58f34c'))) }}: - task: PythonScript@0 From 5edd429d87755d061395e9e473b7b604765907c1 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Tue, 6 Aug 2024 11:53:44 +0800 Subject: [PATCH 5/6] update --- .../azure-pipelines/templates/get-docker-image-steps.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml b/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml index 812f6bf4e54e8..97fbcafaa8bc1 100644 --- a/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml @@ -71,6 +71,14 @@ steps: --repository "${{ parameters.Repository }}" displayName: "Get ${{ parameters.Repository }} image for ${{ parameters.Dockerfile }}" ContainerRegistry: onnxruntimebuildcache +- ${{ if eq(parameters.UseImageCacheContainerRegistry, false) }}: + - script: | + ${{ parameters.ScriptName }} \ + --dockerfile "${{ parameters.Dockerfile }}" \ + --context "${{ parameters.Context }}" \ + --docker-build-args "${{ parameters.DockerBuildArgs }}" \ + --repository "${{ parameters.Repository }}" + displayName: "Get ${{ parameters.Repository }} image for ${{ parameters.Dockerfile }}" - script: | docker version From 0813094a900164dc5b86bd12a2a773498ff087c1 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Tue, 6 Aug 2024 12:00:30 +0800 Subject: [PATCH 6/6] update1 --- tools/ci_build/get_docker_image.py | 19 ------------------- .../templates/get-docker-image-steps.yml | 19 ++++++++++++------- 2 files changed, 12 insertions(+), 26 deletions(-) diff --git a/tools/ci_build/get_docker_image.py b/tools/ci_build/get_docker_image.py index e569d00d55e16..a3f603b0beda4 100755 --- a/tools/ci_build/get_docker_image.py +++ b/tools/ci_build/get_docker_image.py @@ -123,25 +123,6 @@ def main(): "push", full_image_name, ) - elif args.use_imagecache: - log.info("Building image with pipeline cache...") - run( - args.docker_path, - "--log-level", - "error", - "buildx", - "build", - "--tag", - full_image_name, - "--cache-from", - full_image_name, - "--build-arg", - "BUILDKIT_INLINE_CACHE=1", - *shlex.split(args.docker_build_args), - "-f", - args.dockerfile, - args.context, - ) else: log.info("Building image...") run( diff --git a/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml b/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml index 97fbcafaa8bc1..5b6769685a972 100644 --- a/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml @@ -72,13 +72,18 @@ steps: displayName: "Get ${{ parameters.Repository }} image for ${{ parameters.Dockerfile }}" ContainerRegistry: onnxruntimebuildcache - ${{ if eq(parameters.UseImageCacheContainerRegistry, false) }}: - - script: | - ${{ parameters.ScriptName }} \ - --dockerfile "${{ parameters.Dockerfile }}" \ - --context "${{ parameters.Context }}" \ - --docker-build-args "${{ parameters.DockerBuildArgs }}" \ - --repository "${{ parameters.Repository }}" - displayName: "Get ${{ parameters.Repository }} image for ${{ parameters.Dockerfile }}" + # the difference is no --container-registry + - template: with-container-registry-steps.yml + parameters: + Steps: + - script: | + ${{ parameters.ScriptName }} \ + --dockerfile "${{ parameters.Dockerfile }}" \ + --context "${{ parameters.Context }}" \ + --docker-build-args "${{ parameters.DockerBuildArgs }}" \ + --repository "${{ parameters.Repository }}" + displayName: "Get ${{ parameters.Repository }} image for ${{ parameters.Dockerfile }}" + ContainerRegistry: onnxruntimebuildcache - script: | docker version