From 0e0e9db21dfbf688f2e59790dde35fba535bc2b5 Mon Sep 17 00:00:00 2001 From: Thomas Boop Date: Fri, 4 Jun 2021 00:34:29 -0400 Subject: [PATCH 01/10] add hash files to the toolkit --- packages/glob/__tests__/hash-files.test.ts | 79 +++++++++++++++++++ packages/glob/src/glob.ts | 20 +++++ .../glob/src/internal-hash-file-options.ts | 12 +++ packages/glob/src/internal-hash-files.ts | 42 ++++++++++ 4 files changed, 153 insertions(+) create mode 100644 packages/glob/__tests__/hash-files.test.ts create mode 100644 packages/glob/src/internal-hash-file-options.ts create mode 100644 packages/glob/src/internal-hash-files.ts diff --git a/packages/glob/__tests__/hash-files.test.ts b/packages/glob/__tests__/hash-files.test.ts new file mode 100644 index 0000000000..e860042693 --- /dev/null +++ b/packages/glob/__tests__/hash-files.test.ts @@ -0,0 +1,79 @@ +import * as io from '../../io/src/io' +import * as path from 'path' +import {hashFiles} from '../src/glob' +import {promises as fs} from 'fs' + +const IS_WINDOWS = process.platform === 'win32' + +/** + * These test focus on the ability of globber to find files + * and not on the pattern matching aspect + */ +describe('globber', () => { + beforeAll(async () => { + await io.rmRF(getTestTemp()) + }) + + it('basic hashfiles test', async () => { + const root = path.join(getTestTemp(), 'basic-hashfiles') + await fs.mkdir(path.join(root), {recursive: true}) + await fs.writeFile(path.join(root, 'test.txt'), 'test file content') + const hash = await hashFiles(`${root}/*`) + expect(hash).toEqual( + 'd8a411e8f8643821bed189e627ff57151918aa554c00c10b31c693ab2dded273' + ) + }) + + it('basic hashfiles no match should return empty string', async () => { + const root = path.join(getTestTemp(), 'empty-hashfiles') + const hash = await hashFiles(`${root}/*`) + expect(hash).toEqual('') + }) + + it('correctly follows followSymbolicLinks being set', async () => { + // Create the following layout: + // + // /folder-a + // /folder-a/file + // /symDir -> /folder-a + const root = path.join( + getTestTemp(), + 'defaults-to-follow-symbolic-links-true' + ) + await fs.mkdir(path.join(root, 'folder-a'), {recursive: true}) + await createSymlinkDir( + path.join(root, 'folder-a'), + path.join(root, 'symDir') + ) + await fs.writeFile(path.join(root, 'folder-a', 'file'), 'test file content') + const testPath = path.join(root, 'symdir') + const hashNoOptions = await hashFiles(testPath) + const hashSymbolicFalse = await hashFiles(testPath, { + followSymbolicLinks: false + }) + const hashSymbolicTrue = await hashFiles(testPath, { + followSymbolicLinks: true + }) + expect(hashNoOptions).toEqual( + 'd8a411e8f8643821bed189e627ff57151918aa554c00c10b31c693ab2dded273' + ) + expect(hashSymbolicFalse).toEqual('') + expect(hashNoOptions).toEqual(hashSymbolicTrue) + }) +}) + +function getTestTemp(): string { + return path.join(__dirname, '_temp', 'hash_files') +} + +/** + * Creates a symlink directory on OSX/Linux, and a junction point directory on Windows. + * A symlink directory is not created on Windows since it requires an elevated context. + */ +async function createSymlinkDir(real: string, link: string): Promise { + if (IS_WINDOWS) { + await fs.symlink(real, link, 'junction') + } else { + await fs.symlink(real, link) + } +} diff --git a/packages/glob/src/glob.ts b/packages/glob/src/glob.ts index 02fc9cc199..e1098885da 100644 --- a/packages/glob/src/glob.ts +++ b/packages/glob/src/glob.ts @@ -1,5 +1,7 @@ import {Globber, DefaultGlobber} from './internal-globber' import {GlobOptions} from './internal-glob-options' +import {HashFileOptions} from './internal-hash-file-options' +import {hashFiles as _hashFiles} from './internal-hash-files' export {Globber, GlobOptions} @@ -15,3 +17,21 @@ export async function create( ): Promise { return await DefaultGlobber.create(patterns, options) } + +/** + * Computes the sha256 hash of a glob + * + * @param patterns Patterns separated by newlines + * @param options Glob options + */ +export async function hashFiles( + patterns: string, + options?: HashFileOptions +): Promise { + let followSymbolicLinks = true + if (options && typeof options.followSymbolicLinks === 'boolean') { + followSymbolicLinks = options.followSymbolicLinks + } + const globber = await create(patterns, {followSymbolicLinks}) + return _hashFiles(globber) +} diff --git a/packages/glob/src/internal-hash-file-options.ts b/packages/glob/src/internal-hash-file-options.ts new file mode 100644 index 0000000000..00f9bb5f77 --- /dev/null +++ b/packages/glob/src/internal-hash-file-options.ts @@ -0,0 +1,12 @@ +/** + * Options to control globbing behavior + */ +export interface HashFileOptions { + /** + * Indicates whether to follow symbolic links. Generally should set to false + * when deleting files. + * + * @default true + */ + followSymbolicLinks?: boolean +} diff --git a/packages/glob/src/internal-hash-files.ts b/packages/glob/src/internal-hash-files.ts new file mode 100644 index 0000000000..a87dcd8779 --- /dev/null +++ b/packages/glob/src/internal-hash-files.ts @@ -0,0 +1,42 @@ +import * as crypto from 'crypto' +import * as core from '@actions/core' +import * as fs from 'fs' +import * as stream from 'stream' +import * as util from 'util' +import * as path from 'path' +import {Globber} from './glob' + +export async function hashFiles(globber: Globber): Promise { + let hasMatch = false + const githubWorkspace = process.env['GITHUB_WORKSPACE'] ?? process.cwd() + const result = crypto.createHash('sha256') + let count = 0 + for await (const file of globber.globGenerator()) { + core.debug(file) + if (!file.startsWith(`${githubWorkspace}${path.sep}`)) { + core.debug(`Ignore '${file}' since it is not under GITHUB_WORKSPACE.`) + continue + } + if (fs.statSync(file).isDirectory()) { + core.debug(`Skip directory '${file}'.`) + continue + } + const hash = crypto.createHash('sha256') + const pipeline = util.promisify(stream.pipeline) + await pipeline(fs.createReadStream(file), hash) + result.write(hash.digest()) + count++ + if (!hasMatch) { + hasMatch = true + } + } + result.end() + + if (hasMatch) { + core.debug(`Found ${count} files to hash.`) + return result.digest('hex') + } else { + core.error(`No matches found for glob`) + return '' + } +} From d3fd3dc4316f621f888d58527baf39cd5e03760e Mon Sep 17 00:00:00 2001 From: Thomas Boop Date: Fri, 4 Jun 2021 00:54:35 -0400 Subject: [PATCH 02/10] slight formatting updates --- packages/glob/__tests__/hash-files.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/glob/__tests__/hash-files.test.ts b/packages/glob/__tests__/hash-files.test.ts index e860042693..48d8d776a9 100644 --- a/packages/glob/__tests__/hash-files.test.ts +++ b/packages/glob/__tests__/hash-files.test.ts @@ -40,13 +40,13 @@ describe('globber', () => { getTestTemp(), 'defaults-to-follow-symbolic-links-true' ) - await fs.mkdir(path.join(root, 'folder-a'), {recursive: true}) + await fs.mkdir(path.join(root, 'realdir'), {recursive: true}) await createSymlinkDir( - path.join(root, 'folder-a'), + path.join(root, 'realdir'), path.join(root, 'symDir') ) - await fs.writeFile(path.join(root, 'folder-a', 'file'), 'test file content') - const testPath = path.join(root, 'symdir') + await fs.writeFile(path.join(root, 'realdir', 'file'), 'test file content') + const testPath = path.join(root, 'symdir', '*') const hashNoOptions = await hashFiles(testPath) const hashSymbolicFalse = await hashFiles(testPath, { followSymbolicLinks: false From 6b1b93f3abf51d16d92e49eafdd8b1408ca198c4 Mon Sep 17 00:00:00 2001 From: Thomas Boop Date: Fri, 4 Jun 2021 01:08:41 -0400 Subject: [PATCH 03/10] create file then symdir --- packages/glob/__tests__/hash-files.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/glob/__tests__/hash-files.test.ts b/packages/glob/__tests__/hash-files.test.ts index 48d8d776a9..2e851c9d6c 100644 --- a/packages/glob/__tests__/hash-files.test.ts +++ b/packages/glob/__tests__/hash-files.test.ts @@ -41,11 +41,11 @@ describe('globber', () => { 'defaults-to-follow-symbolic-links-true' ) await fs.mkdir(path.join(root, 'realdir'), {recursive: true}) + await fs.writeFile(path.join(root, 'realdir', 'file'), 'test file content') await createSymlinkDir( path.join(root, 'realdir'), path.join(root, 'symDir') ) - await fs.writeFile(path.join(root, 'realdir', 'file'), 'test file content') const testPath = path.join(root, 'symdir', '*') const hashNoOptions = await hashFiles(testPath) const hashSymbolicFalse = await hashFiles(testPath, { From 7e6460102132c1b0142287bc5a00235f67be6873 Mon Sep 17 00:00:00 2001 From: Thomas Boop Date: Fri, 4 Jun 2021 01:15:00 -0400 Subject: [PATCH 04/10] split out tests --- packages/glob/__tests__/hash-files.test.ts | 60 ++++++++++++++++++---- 1 file changed, 49 insertions(+), 11 deletions(-) diff --git a/packages/glob/__tests__/hash-files.test.ts b/packages/glob/__tests__/hash-files.test.ts index 2e851c9d6c..bd2b192c8d 100644 --- a/packages/glob/__tests__/hash-files.test.ts +++ b/packages/glob/__tests__/hash-files.test.ts @@ -30,7 +30,7 @@ describe('globber', () => { expect(hash).toEqual('') }) - it('correctly follows followSymbolicLinks being set', async () => { + it('followSymbolicLinks defaults to true', async () => { // Create the following layout: // // /folder-a @@ -47,18 +47,56 @@ describe('globber', () => { path.join(root, 'symDir') ) const testPath = path.join(root, 'symdir', '*') - const hashNoOptions = await hashFiles(testPath) - const hashSymbolicFalse = await hashFiles(testPath, { - followSymbolicLinks: false - }) - const hashSymbolicTrue = await hashFiles(testPath, { - followSymbolicLinks: true - }) - expect(hashNoOptions).toEqual( + const hash = await hashFiles(testPath) + expect(hash).toEqual( + 'd8a411e8f8643821bed189e627ff57151918aa554c00c10b31c693ab2dded273' + ) + }) + + it('followSymbolicLinks set to true', async () => { + // Create the following layout: + // + // /folder-a + // /folder-a/file + // /symDir -> /folder-a + const root = path.join( + getTestTemp(), + 'set-to-true' + ) + await fs.mkdir(path.join(root, 'realdir'), {recursive: true}) + await fs.writeFile(path.join(root, 'realdir', 'file'), 'test file content') + await createSymlinkDir( + path.join(root, 'realdir'), + path.join(root, 'symDir') + ) + const testPath = path.join(root, 'symdir', '*') + const hash = await hashFiles(testPath, {followSymbolicLinks: true}) + expect(hash).toEqual( 'd8a411e8f8643821bed189e627ff57151918aa554c00c10b31c693ab2dded273' ) - expect(hashSymbolicFalse).toEqual('') - expect(hashNoOptions).toEqual(hashSymbolicTrue) + }) + + it('followSymbolicLinks set to false', async () => { + // Create the following layout: + // + // /folder-a + // /folder-a/file + // /symDir -> /folder-a + const root = path.join( + getTestTemp(), + 'set-to-false' + ) + await fs.mkdir(path.join(root, 'realdir'), {recursive: true}) + await fs.writeFile(path.join(root, 'realdir', 'file'), 'test file content') + await createSymlinkDir( + path.join(root, 'realdir'), + path.join(root, 'symDir') + ) + const testPath = path.join(root, 'symdir', '*') + const hash = await hashFiles(testPath, {followSymbolicLinks: false}) + expect(hash).toEqual( + '' + ) }) }) From 52c5a709bdc17f2961df85829bd931d51dd83139 Mon Sep 17 00:00:00 2001 From: Thomas Boop Date: Fri, 4 Jun 2021 09:27:30 -0400 Subject: [PATCH 05/10] fix lint! --- packages/glob/__tests__/hash-files.test.ts | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/packages/glob/__tests__/hash-files.test.ts b/packages/glob/__tests__/hash-files.test.ts index bd2b192c8d..656c23a572 100644 --- a/packages/glob/__tests__/hash-files.test.ts +++ b/packages/glob/__tests__/hash-files.test.ts @@ -59,10 +59,7 @@ describe('globber', () => { // /folder-a // /folder-a/file // /symDir -> /folder-a - const root = path.join( - getTestTemp(), - 'set-to-true' - ) + const root = path.join(getTestTemp(), 'set-to-true') await fs.mkdir(path.join(root, 'realdir'), {recursive: true}) await fs.writeFile(path.join(root, 'realdir', 'file'), 'test file content') await createSymlinkDir( @@ -82,10 +79,7 @@ describe('globber', () => { // /folder-a // /folder-a/file // /symDir -> /folder-a - const root = path.join( - getTestTemp(), - 'set-to-false' - ) + const root = path.join(getTestTemp(), 'set-to-false') await fs.mkdir(path.join(root, 'realdir'), {recursive: true}) await fs.writeFile(path.join(root, 'realdir', 'file'), 'test file content') await createSymlinkDir( @@ -94,9 +88,7 @@ describe('globber', () => { ) const testPath = path.join(root, 'symdir', '*') const hash = await hashFiles(testPath, {followSymbolicLinks: false}) - expect(hash).toEqual( - '' - ) + expect(hash).toEqual('') }) }) From 7cb8a4ca14a0c6baa79c078e01d096bbb4487c13 Mon Sep 17 00:00:00 2001 From: Thomas Boop Date: Fri, 4 Jun 2021 13:44:10 -0400 Subject: [PATCH 06/10] fix tests on ubuntu --- packages/glob/__tests__/hash-files.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/glob/__tests__/hash-files.test.ts b/packages/glob/__tests__/hash-files.test.ts index 656c23a572..8195ce4a42 100644 --- a/packages/glob/__tests__/hash-files.test.ts +++ b/packages/glob/__tests__/hash-files.test.ts @@ -46,7 +46,7 @@ describe('globber', () => { path.join(root, 'realdir'), path.join(root, 'symDir') ) - const testPath = path.join(root, 'symdir', '*') + const testPath = path.join(root, 'symdir') const hash = await hashFiles(testPath) expect(hash).toEqual( 'd8a411e8f8643821bed189e627ff57151918aa554c00c10b31c693ab2dded273' @@ -66,7 +66,7 @@ describe('globber', () => { path.join(root, 'realdir'), path.join(root, 'symDir') ) - const testPath = path.join(root, 'symdir', '*') + const testPath = path.join(root, 'symdir') const hash = await hashFiles(testPath, {followSymbolicLinks: true}) expect(hash).toEqual( 'd8a411e8f8643821bed189e627ff57151918aa554c00c10b31c693ab2dded273' @@ -86,7 +86,7 @@ describe('globber', () => { path.join(root, 'realdir'), path.join(root, 'symDir') ) - const testPath = path.join(root, 'symdir', '*') + const testPath = path.join(root, 'symdir') const hash = await hashFiles(testPath, {followSymbolicLinks: false}) expect(hash).toEqual('') }) From b5dfd33c3f5c93dc35d5cd71005731e3637e9ea1 Mon Sep 17 00:00:00 2001 From: Thomas Boop Date: Fri, 4 Jun 2021 15:39:47 -0400 Subject: [PATCH 07/10] minor updates --- packages/glob/__tests__/hash-files.test.ts | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/packages/glob/__tests__/hash-files.test.ts b/packages/glob/__tests__/hash-files.test.ts index 8195ce4a42..e09335cf95 100644 --- a/packages/glob/__tests__/hash-files.test.ts +++ b/packages/glob/__tests__/hash-files.test.ts @@ -31,22 +31,17 @@ describe('globber', () => { }) it('followSymbolicLinks defaults to true', async () => { - // Create the following layout: - // - // /folder-a - // /folder-a/file - // /symDir -> /folder-a const root = path.join( getTestTemp(), 'defaults-to-follow-symbolic-links-true' ) await fs.mkdir(path.join(root, 'realdir'), {recursive: true}) - await fs.writeFile(path.join(root, 'realdir', 'file'), 'test file content') + await fs.writeFile(path.join(root, 'realdir', 'file.txt'), 'test file content') await createSymlinkDir( path.join(root, 'realdir'), path.join(root, 'symDir') ) - const testPath = path.join(root, 'symdir') + const testPath = path.join(root, `symDir`) const hash = await hashFiles(testPath) expect(hash).toEqual( 'd8a411e8f8643821bed189e627ff57151918aa554c00c10b31c693ab2dded273' @@ -54,11 +49,6 @@ describe('globber', () => { }) it('followSymbolicLinks set to true', async () => { - // Create the following layout: - // - // /folder-a - // /folder-a/file - // /symDir -> /folder-a const root = path.join(getTestTemp(), 'set-to-true') await fs.mkdir(path.join(root, 'realdir'), {recursive: true}) await fs.writeFile(path.join(root, 'realdir', 'file'), 'test file content') @@ -66,7 +56,7 @@ describe('globber', () => { path.join(root, 'realdir'), path.join(root, 'symDir') ) - const testPath = path.join(root, 'symdir') + const testPath = path.join(root, `symDir`) const hash = await hashFiles(testPath, {followSymbolicLinks: true}) expect(hash).toEqual( 'd8a411e8f8643821bed189e627ff57151918aa554c00c10b31c693ab2dded273' From 442ffe83a0958429c0b9c36d8c9f473355d06fec Mon Sep 17 00:00:00 2001 From: Thomas Boop Date: Mon, 7 Jun 2021 09:32:48 -0400 Subject: [PATCH 08/10] run linter --- packages/glob/__tests__/hash-files.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/glob/__tests__/hash-files.test.ts b/packages/glob/__tests__/hash-files.test.ts index e09335cf95..743055160c 100644 --- a/packages/glob/__tests__/hash-files.test.ts +++ b/packages/glob/__tests__/hash-files.test.ts @@ -36,7 +36,10 @@ describe('globber', () => { 'defaults-to-follow-symbolic-links-true' ) await fs.mkdir(path.join(root, 'realdir'), {recursive: true}) - await fs.writeFile(path.join(root, 'realdir', 'file.txt'), 'test file content') + await fs.writeFile( + path.join(root, 'realdir', 'file.txt'), + 'test file content' + ) await createSymlinkDir( path.join(root, 'realdir'), path.join(root, 'symDir') From 656d504347c01763ef6d1dddf630d4878a772abf Mon Sep 17 00:00:00 2001 From: Thomas Boop Date: Mon, 7 Jun 2021 11:11:12 -0400 Subject: [PATCH 09/10] add multipath test --- packages/glob/__tests__/hash-files.test.ts | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/packages/glob/__tests__/hash-files.test.ts b/packages/glob/__tests__/hash-files.test.ts index 743055160c..cb912ca005 100644 --- a/packages/glob/__tests__/hash-files.test.ts +++ b/packages/glob/__tests__/hash-files.test.ts @@ -83,6 +83,30 @@ describe('globber', () => { const hash = await hashFiles(testPath, {followSymbolicLinks: false}) expect(hash).toEqual('') }) + + it('multipath test basic', async () => { + // Create the following layout: + // + // /folder-a + // /folder-a/file + // /symDir -> /folder-a + const root = path.join(getTestTemp(), 'set-to-false') + await fs.mkdir(path.join(root, 'dir1'), {recursive: true}) + await fs.mkdir(path.join(root, 'dir2'), {recursive: true}) + await fs.writeFile( + path.join(root, 'dir1', 'testfile1.txt'), + 'test file content' + ) + await fs.writeFile( + path.join(root, 'dir2', 'testfile2.txt'), + 'test file content' + ) + const testPath = `${path.join(root, 'dir1')}\n${path.join(root, 'dir2')}` + const hash = await hashFiles(testPath) + expect(hash).toEqual( + '4e911ea5824830b6a3ec096c7833d5af8381c189ffaa825c3503a5333a73eadc' + ) + }) }) function getTestTemp(): string { From b96cb32f17f1a12b8bcbd201543e229843b64c17 Mon Sep 17 00:00:00 2001 From: Thomas Boop Date: Mon, 7 Jun 2021 14:13:01 -0400 Subject: [PATCH 10/10] change to warning --- packages/glob/src/internal-hash-files.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/glob/src/internal-hash-files.ts b/packages/glob/src/internal-hash-files.ts index a87dcd8779..be9e3ba1ee 100644 --- a/packages/glob/src/internal-hash-files.ts +++ b/packages/glob/src/internal-hash-files.ts @@ -36,7 +36,7 @@ export async function hashFiles(globber: Globber): Promise { core.debug(`Found ${count} files to hash.`) return result.digest('hex') } else { - core.error(`No matches found for glob`) + core.warning(`No matches found for glob`) return '' } }