From 0f71f6bf5d0e3bd565cb44da1a13f07a111b1a34 Mon Sep 17 00:00:00 2001 From: Dave Hadka Date: Thu, 4 Jun 2020 15:44:38 -0500 Subject: [PATCH 1/4] Adds option to download using AzCopy --- .github/workflows/cache-tests.yml | 6 ++++ .../cache/src/internal/cacheHttpClient.ts | 31 +++++++++++++++++-- packages/cache/src/internal/cacheUtils.ts | 11 +++++++ 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cache-tests.yml b/.github/workflows/cache-tests.yml index 10120ddc46..460c8ce962 100644 --- a/.github/workflows/cache-tests.yml +++ b/.github/workflows/cache-tests.yml @@ -62,6 +62,12 @@ jobs: run: | node -e "Promise.resolve(require('./packages/cache/lib/cache').restoreCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}'))" + - name: Restore cache using restoreCache() with AzCopy enabled + env: + USE_AZCOPY: true + run: | + node -e "Promise.resolve(require('./packages/cache/lib/cache').restoreCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}'))" + - name: Verify cache shell: bash run: | diff --git a/packages/cache/src/internal/cacheHttpClient.ts b/packages/cache/src/internal/cacheHttpClient.ts index 21af8031d0..69c06e6475 100644 --- a/packages/cache/src/internal/cacheHttpClient.ts +++ b/packages/cache/src/internal/cacheHttpClient.ts @@ -1,4 +1,5 @@ import * as core from '@actions/core' +import {exec} from '@actions/exec' import {HttpClient, HttpCodes} from '@actions/http-client' import {BearerCredentialHandler} from '@actions/http-client/auth' import { @@ -9,6 +10,7 @@ import { import * as crypto from 'crypto' import * as fs from 'fs' import * as stream from 'stream' +import {URL} from 'url' import * as util from 'util' import * as utils from './cacheUtils' @@ -220,7 +222,7 @@ async function pipeResponseToStream( await pipeline(response.message, output) } -export async function downloadCache( +async function downloadCacheHttpClient( archiveLocation: string, archivePath: string ): Promise { @@ -256,6 +258,31 @@ export async function downloadCache( } } +export async function downloadCache( + archiveLocation: string, + archivePath: string +): Promise { + const archiveUrl = new URL(archiveLocation) + const useAzCopy = process.env['USE_AZCOPY'] ?? '' + + // Use AzCopy to download caches hosted on Azure to improve reliability. + if ( + archiveUrl.hostname.endsWith('.blob.core.windows.net') && + useAzCopy === 'true' + ) { + const command = await utils.getAzCopyCommand() + + if (command) { + core.info(`Downloading cache using ${command}...`) + await exec(command, ['copy', archiveLocation, archivePath]) + return + } + } + + // Otherwise, download using the Actions http-client. + await downloadCacheHttpClient(archiveLocation, archivePath) +} + // Reserve Cache export async function reserveCache( key: string, @@ -360,7 +387,7 @@ async function uploadFile( }) .on('error', error => { throw new Error( - `Cache upload failed because file read failed with ${error.Message}` + `Cache upload failed because file read failed with ${error.message}` ) }), start, diff --git a/packages/cache/src/internal/cacheUtils.ts b/packages/cache/src/internal/cacheUtils.ts index b279369514..3e1a0de647 100644 --- a/packages/cache/src/internal/cacheUtils.ts +++ b/packages/cache/src/internal/cacheUtils.ts @@ -113,3 +113,14 @@ export async function isGnuTarInstalled(): Promise { const versionOutput = await getVersion('tar') return versionOutput.toLowerCase().includes('gnu tar') } + +export async function getAzCopyCommand(): Promise { + // On Ubuntu, azcopy points to an earlier version, so prefer the azcopy10 alias + if ((await getVersion('azcopy10')).toLowerCase().includes('azcopy')) { + return 'azcopy10' + } else if ((await getVersion('azcopy')).toLowerCase().includes('azcopy')) { + return 'azcopy' + } else { + return undefined + } +} From 824c8852009e1c46ce1393cec00d63f7bf3dcf3f Mon Sep 17 00:00:00 2001 From: Dave Hadka Date: Thu, 4 Jun 2020 16:37:36 -0500 Subject: [PATCH 2/4] Bump version number and add release notes --- packages/cache/RELEASES.md | 5 ++++- packages/cache/package.json | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/cache/RELEASES.md b/packages/cache/RELEASES.md index 0e623d9e4c..b191601399 100644 --- a/packages/cache/RELEASES.md +++ b/packages/cache/RELEASES.md @@ -8,4 +8,7 @@ - Fixes issues with the zstd compression algorithm on Windows and Ubuntu 16.04 [#469](https://github.com/actions/toolkit/pull/469) ### 0.2.1 -- Fix to await async function getCompressionMethod \ No newline at end of file +- Fix to await async function getCompressionMethod + +### 0.3.0 +- Downloads Azure-hosted caches using AzCopy when available \ No newline at end of file diff --git a/packages/cache/package.json b/packages/cache/package.json index 6e0a31818d..24147547ad 100644 --- a/packages/cache/package.json +++ b/packages/cache/package.json @@ -1,6 +1,6 @@ { "name": "@actions/cache", - "version": "0.2.1", + "version": "0.3.0", "preview": true, "description": "Actions cache lib", "keywords": [ From 353580da2caa15871e46dd7ba6e45c78d6a993ca Mon Sep 17 00:00:00 2001 From: Dave Hadka Date: Thu, 4 Jun 2020 17:07:54 -0500 Subject: [PATCH 3/4] Ensure we use at least v10 --- packages/cache/src/internal/cacheUtils.ts | 24 +++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/packages/cache/src/internal/cacheUtils.ts b/packages/cache/src/internal/cacheUtils.ts index 3e1a0de647..ff14fa7955 100644 --- a/packages/cache/src/internal/cacheUtils.ts +++ b/packages/cache/src/internal/cacheUtils.ts @@ -115,12 +115,24 @@ export async function isGnuTarInstalled(): Promise { } export async function getAzCopyCommand(): Promise { - // On Ubuntu, azcopy points to an earlier version, so prefer the azcopy10 alias - if ((await getVersion('azcopy10')).toLowerCase().includes('azcopy')) { + // Always prefer the azcopy10 alias first, which is the correct version on Ubuntu. + if ((await getVersion('azcopy10')).toLowerCase().startsWith('azcopy version')) { return 'azcopy10' - } else if ((await getVersion('azcopy')).toLowerCase().includes('azcopy')) { - return 'azcopy' - } else { - return undefined } + + // Fall back to any azcopy that is version 10 or newer. + const versionOutput = await getVersion('azcopy') + + if (versionOutput.toLowerCase().startsWith('azcopy version')) { + const version = versionOutput.substring(15) + + if (semver.gte(version, '10.0.0')) { + return 'azcopy' + } else { + core.debug(`Found azcopy but version is not supported: ${version}`) + } + } + + // Otherwise, azcopy is not available. + return undefined } From b6f111e94c27425d53bb3dd85c943656c983d263 Mon Sep 17 00:00:00 2001 From: Dave Hadka Date: Thu, 4 Jun 2020 17:12:42 -0500 Subject: [PATCH 4/4] Negate env var so it disables AzCopy --- .github/workflows/cache-tests.yml | 8 ++++---- packages/cache/src/internal/cacheHttpClient.ts | 6 +++--- packages/cache/src/internal/cacheUtils.ts | 6 ++++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/cache-tests.yml b/.github/workflows/cache-tests.yml index 460c8ce962..dd90eb2109 100644 --- a/.github/workflows/cache-tests.yml +++ b/.github/workflows/cache-tests.yml @@ -58,13 +58,13 @@ jobs: run: | node -e "Promise.resolve(require('./packages/cache/lib/cache').saveCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}'))" - - name: Restore cache using restoreCache() + - name: Restore cache using restoreCache() with http-client + env: + DISABLE_AZCOPY: true run: | node -e "Promise.resolve(require('./packages/cache/lib/cache').restoreCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}'))" - - name: Restore cache using restoreCache() with AzCopy enabled - env: - USE_AZCOPY: true + - name: Restore cache using restoreCache() with AzCopy run: | node -e "Promise.resolve(require('./packages/cache/lib/cache').restoreCache(['test-cache','~/test-cache'],'test-${{ runner.os }}-${{ github.run_id }}'))" diff --git a/packages/cache/src/internal/cacheHttpClient.ts b/packages/cache/src/internal/cacheHttpClient.ts index 69c06e6475..da4954be5a 100644 --- a/packages/cache/src/internal/cacheHttpClient.ts +++ b/packages/cache/src/internal/cacheHttpClient.ts @@ -263,12 +263,12 @@ export async function downloadCache( archivePath: string ): Promise { const archiveUrl = new URL(archiveLocation) - const useAzCopy = process.env['USE_AZCOPY'] ?? '' + const disableAzCopy = process.env['DISABLE_AZCOPY'] ?? '' - // Use AzCopy to download caches hosted on Azure to improve reliability. + // Use AzCopy to download caches hosted on Azure to improve speed and reliability. if ( archiveUrl.hostname.endsWith('.blob.core.windows.net') && - useAzCopy === 'true' + disableAzCopy !== 'true' ) { const command = await utils.getAzCopyCommand() diff --git a/packages/cache/src/internal/cacheUtils.ts b/packages/cache/src/internal/cacheUtils.ts index ff14fa7955..2a1a6e3c64 100644 --- a/packages/cache/src/internal/cacheUtils.ts +++ b/packages/cache/src/internal/cacheUtils.ts @@ -116,12 +116,14 @@ export async function isGnuTarInstalled(): Promise { export async function getAzCopyCommand(): Promise { // Always prefer the azcopy10 alias first, which is the correct version on Ubuntu. - if ((await getVersion('azcopy10')).toLowerCase().startsWith('azcopy version')) { + let versionOutput = await getVersion('azcopy10') + + if (versionOutput.toLowerCase().startsWith('azcopy version')) { return 'azcopy10' } // Fall back to any azcopy that is version 10 or newer. - const versionOutput = await getVersion('azcopy') + versionOutput = await getVersion('azcopy') if (versionOutput.toLowerCase().startsWith('azcopy version')) { const version = versionOutput.substring(15)