From 65288857bc68bf44d2de531d9d28bbfcad81370d Mon Sep 17 00:00:00 2001 From: satackey <21271711+satackey@users.noreply.github.com> Date: Sat, 15 Aug 2020 16:26:06 +0900 Subject: [PATCH 1/8] Setup Windows matrix --- .github/workflows/release.yml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ffeac0b0..d2c31a0b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -48,6 +48,13 @@ jobs: 'test_project_scratch', 'hello-world', ], + os: [ + 'ubuntu-latest', + 'windows-latest', + ], + branch: [ + context.payload.ref.replace('refs/heads/', '') + ], include: [ { inspect_image: 'test_project_scratch', @@ -69,18 +76,13 @@ jobs: needs: build strategy: matrix: ${{ fromJSON(needs.build.outputs.matrix) }} - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - run: ${{ matrix.prepare_command }} - - name: Extract - id: extract - run: | - echo "##[set-output name=branch;]${GITHUB_REF#refs/heads/}" - - name: Download action uses: actions/download-artifact@v2 with: @@ -88,7 +90,7 @@ jobs: path: action-dlc - uses: ./action-dlc - name: Run satackey/action-docker-layer-caching@${{ steps.extract.outputs.branch }} + name: Run satackey/action-docker-layer-caching@${{ matrix.branch }} with: key: docker-layer-caching-${{ matrix.inspect_image }}-sha:${{ github.sha }}-{hash} @@ -98,7 +100,7 @@ jobs: needs: [build, test_saving] strategy: matrix: ${{ fromJSON(needs.build.outputs.matrix) }} - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 @@ -116,7 +118,7 @@ jobs: path: action-dlc - uses: ./action-dlc - name: Run satackey/action-docker-layer-caching@${{ steps.extract.outputs.branch }} + name: Run satackey/action-docker-layer-caching@${{ matrix.branch }} with: key: never-restored-docker-layer-caching-${{ matrix.inspect_image }}-sha:${{ github.sha }}-{hash} restore-keys: docker-layer-caching-${{ matrix.inspect_image }}-sha:${{ github.sha }}- From 56ca79f27fcc0b8987e47814fd95957f91db71a8 Mon Sep 17 00:00:00 2001 From: satackey <21271711+satackey@users.noreply.github.com> Date: Sat, 15 Aug 2020 16:29:36 +0900 Subject: [PATCH 2/8] Use bash by defualt --- .github/workflows/release.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d2c31a0b..39636e63 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,6 +7,10 @@ on: pull_request: delete: +defaults: + run: + shell: bash + jobs: build: runs-on: ubuntu-latest From 5f8502e07c3e37c29acb4b112fc64e24865bf04d Mon Sep 17 00:00:00 2001 From: satackey <21271711+satackey@users.noreply.github.com> Date: Sat, 15 Aug 2020 16:36:46 +0900 Subject: [PATCH 3/8] exclude docker-compose windows test --- .github/workflows/release.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 39636e63..a45a3fd3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -70,6 +70,12 @@ jobs: build_command: 'docker pull hello-world', }, ], + exclude: [ + { + inspect_image: 'test_project_scratch', + os: 'windows-latest', + }, + ], } outputs: From 8edf211581c9e74702eb1c015a0402384d864748 Mon Sep 17 00:00:00 2001 From: satackey <21271711+satackey@users.noreply.github.com> Date: Sat, 15 Aug 2020 16:43:00 +0900 Subject: [PATCH 4/8] Update exec --- src/ImageDetector.ts | 2 +- src/LayerCache.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ImageDetector.ts b/src/ImageDetector.ts index 1ab23033..f93d9bab 100644 --- a/src/ImageDetector.ts +++ b/src/ImageDetector.ts @@ -5,7 +5,7 @@ export class ImageDetector { async getExistingImages(): Promise { const existingSet = new Set([]) const ids = (await exec.exec(`docker image ls -q`, [], { silent: true })).stdoutStr.split(`\n`).filter(id => id !== ``) - const repotags = (await exec.exec(`sh -c "docker image ls --format '{{ .Repository }}:{{ .Tag }}' --filter 'dangling=false'"`, [], { silent: true })).stdoutStr.split(`\n`).filter(id => id !== ``); + const repotags = (await exec.exec(`docker`, `image ls --format {{.Repository}}:{{.Tag}} --filter dangling=false`.split(' '), { silent: true })).stdoutStr.split(`\n`).filter(id => id !== ``); core.debug(JSON.stringify({ log: "getExistingImages", ids, repotags })); ([...ids, ...repotags]).forEach(image => existingSet.add(image)) core.debug(JSON.stringify({ existingSet })) diff --git a/src/LayerCache.ts b/src/LayerCache.ts index 54c7464f..120aca89 100644 --- a/src/LayerCache.ts +++ b/src/LayerCache.ts @@ -49,8 +49,8 @@ class LayerCache { } private async saveImageAsUnpacked() { - await this.exec('mkdir -p', [this.getSavedImageTarDir()], { silent: true }) - await this.exec(`sh -c`, [`docker save '${(await this.makeRepotagsDockerSaveArgReady(this.ids)).join(`' '`)}' | tar xf - -C ${this.getSavedImageTarDir()}`]) + await fs.mkdir(this.getSavedImageTarDir(), { recursive: true }) + await this.exec(`sh -c`, [`docker save '${(await this.makeRepotagsDockerSaveArgReady(this.ids)).join(`' '`)}' | tar xf - -C .`], { cwd: this.getSavedImageTarDir() }) } private async makeRepotagsDockerSaveArgReady(repotags: string[]): Promise { From 95f09d2618739a6b9fccb2af53a5a2b2bde69c92 Mon Sep 17 00:00:00 2001 From: satackey <21271711+satackey@users.noreply.github.com> Date: Sat, 15 Aug 2020 16:45:59 +0900 Subject: [PATCH 5/8] Add ${{ matrix.os }} to cache key --- .github/workflows/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a45a3fd3..a600c0b2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -102,7 +102,7 @@ jobs: - uses: ./action-dlc name: Run satackey/action-docker-layer-caching@${{ matrix.branch }} with: - key: docker-layer-caching-${{ matrix.inspect_image }}-sha:${{ github.sha }}-{hash} + key: docker-layer-caching-${{ matrix.os }}-${{ matrix.inspect_image }}-sha:${{ github.sha }}-{hash} - run: ${{ matrix.build_command }} @@ -130,8 +130,8 @@ jobs: - uses: ./action-dlc name: Run satackey/action-docker-layer-caching@${{ matrix.branch }} with: - key: never-restored-docker-layer-caching-${{ matrix.inspect_image }}-sha:${{ github.sha }}-{hash} - restore-keys: docker-layer-caching-${{ matrix.inspect_image }}-sha:${{ github.sha }}- + key: never-restored-docker-layer-caching-${{ matrix.os }}-${{ matrix.inspect_image }}-sha:${{ github.sha }}-{hash} + restore-keys: docker-layer-caching-${{ matrix.os }}-${{ matrix.inspect_image }}-sha:${{ github.sha }}- skip-save: 'true' - name: Show cached image info From d61cc582ceb616be15a87aa8e9f7860115815b97 Mon Sep 17 00:00:00 2001 From: satackey <21271711+satackey@users.noreply.github.com> Date: Sat, 15 Aug 2020 17:33:48 +0900 Subject: [PATCH 6/8] debug --- .github/workflows/release.yml | 1 + src/LayerCache.ts | 18 +++++++++++------- src/Tar.ts | 1 + 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a600c0b2..46d67254 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -111,6 +111,7 @@ jobs: strategy: matrix: ${{ fromJSON(needs.build.outputs.matrix) }} runs-on: ${{ matrix.os }} + steps: - uses: actions/checkout@v2 diff --git a/src/LayerCache.ts b/src/LayerCache.ts index 120aca89..cc0999b0 100644 --- a/src/LayerCache.ts +++ b/src/LayerCache.ts @@ -133,14 +133,15 @@ class LayerCache { } } - private async storeSingleLayerBy(id: string): Promise { - const path = this.genSingleLayerStorePath(id) - const key = await this.generateSingleLayerSaveKey(id) + private async storeSingleLayerBy(layerId: string): Promise { + const path = this.genSingleLayerStorePath(layerId) + const key = await this.generateSingleLayerSaveKey(layerId) - core.info(`Start storing layer cache: ${key}`) + core.info(`Start storing layer cache: ${JSON.stringify({ layerId, key })}`) const cacheId = await LayerCache.dismissError(cache.saveCache([path], key), LayerCache.ERROR_CACHE_ALREAD_EXISTS_STR, -1) - core.info(`Stored layer cache, key: ${key}, id: ${cacheId}`) + core.info(`Stored layer cache: ${JSON.stringify({ key, cacheId })}`) + core.debug(JSON.stringify({ log: `storeSingleLayerBy`, layerId, path, key, cacheId})) return cacheId } @@ -201,9 +202,12 @@ class LayerCache { } private async restoreSingleLayerBy(id: string): Promise { - core.debug(JSON.stringify({ log: `restoreSingleLayerBy`, id })) + const path = this.genSingleLayerStorePath(id) + const key = await this.recoverSingleLayerKey(id) + + core.debug(JSON.stringify({ log: `restoreSingleLayerBy`, id, path, key })) - const result = await cache.restoreCache([this.genSingleLayerStorePath(id)], await this.recoverSingleLayerKey(id)) + const result = await cache.restoreCache([path], key) if (result == null) { throw new Error(`${LayerCache.ERROR_LAYER_CACHE_NOT_FOUND_STR}: ${JSON.stringify({ id })}`) diff --git a/src/Tar.ts b/src/Tar.ts index 99653304..c40ebc2a 100644 --- a/src/Tar.ts +++ b/src/Tar.ts @@ -16,6 +16,7 @@ export function assertManifests(x: unknown): asserts x is Manifests { export async function loadRawManifests(path: string) { return (await fs.readFile(`${path}/manifest.json`)).toString() } + export async function loadManifests(path: string) { const raw = await loadRawManifests(path) const manifests = JSON.parse(raw.toString()) From bc7009fb01e96a858e362383c63406e5c003a558 Mon Sep 17 00:00:00 2001 From: satackey <21271711+satackey@users.noreply.github.com> Date: Sat, 15 Aug 2020 19:36:49 +0900 Subject: [PATCH 7/8] mkdir restoreSingleLayerBy --- src/LayerCache.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/LayerCache.ts b/src/LayerCache.ts index cc0999b0..7f53ad6d 100644 --- a/src/LayerCache.ts +++ b/src/LayerCache.ts @@ -178,8 +178,9 @@ class LayerCache { } private async restoreLayers(): Promise { - const pool = new PromisePool(this.concurrency) + + const pool = new PromisePool(this.concurrency) const tasks = (await this.getLayerIds()).map( layerId => pool.open(() => this.restoreSingleLayerBy(layerId)) ) @@ -204,9 +205,11 @@ class LayerCache { private async restoreSingleLayerBy(id: string): Promise { const path = this.genSingleLayerStorePath(id) const key = await this.recoverSingleLayerKey(id) + const dir = path.replace(/[^/\\]+$/, ``) - core.debug(JSON.stringify({ log: `restoreSingleLayerBy`, id, path, key })) + core.debug(JSON.stringify({ log: `restoreSingleLayerBy`, id, path, dir, key })) + await fs.mkdir(dir, { recursive: true }) const result = await cache.restoreCache([path], key) if (result == null) { @@ -247,7 +250,7 @@ class LayerCache { } genSingleLayerStorePath(id: string) { - return `${this.getLayerCachesDir()}/${id}/layer.tar` + return path.resolve(`${this.getLayerCachesDir()}/${id}/layer.tar`) } async generateRootHashFromManifest(): Promise { From a26d576c0fae03611f4b910fa6915b1531d3cb98 Mon Sep 17 00:00:00 2001 From: satackey <21271711+satackey@users.noreply.github.com> Date: Sat, 15 Aug 2020 20:23:14 +0900 Subject: [PATCH 8/8] fix matrix --- .github/workflows/release.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 46d67254..b6b8e475 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -56,9 +56,6 @@ jobs: 'ubuntu-latest', 'windows-latest', ], - branch: [ - context.payload.ref.replace('refs/heads/', '') - ], include: [ { inspect_image: 'test_project_scratch', @@ -68,7 +65,9 @@ jobs: inspect_image: 'hello-world', prepare_command: ':', build_command: 'docker pull hello-world', - }, + }, { + branch: process.env.GITHUB_REF.replace('refs/heads/', '') + } ], exclude: [ {