From 790e3f64a0135734d73fb53e289f51313dc87020 Mon Sep 17 00:00:00 2001 From: AlexandrKravchuk Date: Tue, 30 Jun 2020 16:35:08 +0300 Subject: [PATCH] fix DTS 1674974; build-task-team#10 --- .../FileExtractor/JavaFilesExtractor.ts | 91 ++++++++++++------- .../resources.resjson/en-US/resources.resjson | 4 +- .../JavaToolInstallerV0/javatoolinstaller.ts | 31 ++++--- Tasks/JavaToolInstallerV0/task.json | 6 +- Tasks/JavaToolInstallerV0/task.loc.json | 6 +- 5 files changed, 89 insertions(+), 49 deletions(-) diff --git a/Tasks/JavaToolInstallerV0/FileExtractor/JavaFilesExtractor.ts b/Tasks/JavaToolInstallerV0/FileExtractor/JavaFilesExtractor.ts index 26eb9387dfaf..7072d67cbc4c 100644 --- a/Tasks/JavaToolInstallerV0/FileExtractor/JavaFilesExtractor.ts +++ b/Tasks/JavaToolInstallerV0/FileExtractor/JavaFilesExtractor.ts @@ -3,6 +3,7 @@ import * as os from 'os'; import * as path from 'path'; import * as taskLib from 'azure-pipelines-task-lib/task'; import * as toolLib from 'azure-pipelines-tool-lib/tool'; +import {getFileEnding} from '../javatoolinstaller'; export const BIN_FOLDER = 'bin'; @@ -34,7 +35,7 @@ export class JavaFilesExtractor { } } - private isTar(file): boolean { + private static isTar(file): boolean { const name = file.toLowerCase(); // standard gnu-tar extension formats with recognized auto compression formats // https://www.gnu.org/software/tar/manual/html_section/tar_69.html @@ -62,6 +63,8 @@ export class JavaFilesExtractor { const sevenZip = taskLib.tool(this.getSevenZipLocation()); sevenZip.arg('x'); sevenZip.arg('-o' + destinationFolder); + // skip if exists + sevenZip.arg('-aos'); sevenZip.arg(file); const execResult = sevenZip.execSync(); if (execResult.code != taskLib.TaskResult.Succeeded) { @@ -80,7 +83,7 @@ export class JavaFilesExtractor { if (this.win) { if ('.tar' === fileEnding) { // a simple tar this.sevenZipExtract(file, this.destinationFolder); - } else if (this.isTar(file)) { // a compressed tar, e.g. 'fullFilePath/test.tar.gz' + } else if (JavaFilesExtractor.isTar(file)) { // a compressed tar, e.g. 'fullFilePath/test.tar.gz' // e.g. 'fullFilePath/test.tar.gz' --> 'test.tar.gz' const shortFileName = path.basename(file); // e.g. 'destinationFolder/_test.tar.gz_' @@ -118,19 +121,18 @@ export class JavaFilesExtractor { } // This method recursively finds all .pack files under fsPath and unpacks them with the unpack200 tool - private unpackJars(fsPath: string, javaBinPath: string) { + public static unpackJars(fsPath: string, javaBinPath: string) { if (fs.existsSync(fsPath)) { if (fs.lstatSync(fsPath).isDirectory()) { - let self = this; - fs.readdirSync(fsPath).forEach(function(file,index){ + fs.readdirSync(fsPath).forEach(function(file){ const curPath = path.join(fsPath, file); - self.unpackJars(curPath, javaBinPath); + JavaFilesExtractor.unpackJars(curPath, javaBinPath); }); } else if (path.extname(fsPath).toLowerCase() === '.pack') { - // Unpack the pack file synchonously + // Unpack the pack file synchronously const p = path.parse(fsPath); const toolName = process.platform.match(/^win/i) ? 'unpack200.exe' : 'unpack200'; - const args = process.platform.match(/^win/i) ? '-r -v -l ""' : ''; + const args = process.platform.match(/^win/i) ? '-r -v -l ""' : ''; const name = path.join(p.dir, p.name); taskLib.execSync(path.join(javaBinPath, toolName), `${args} "${name}.pack" "${name}.jar"`); } @@ -142,12 +144,13 @@ export class JavaFilesExtractor { * @param pathsArray - contains paths to all the files inside the structure * @param root - path to the directory we want to get the structure of */ - private static sliceStructure(pathsArray: Array, root: string = pathsArray[0]): Array{ + public static sliceStructure(pathsArray: Array, root: string = pathsArray[0]): Array{ const dirPathLength = root.length; const structureObject: IDirectoriesDictionary = {}; for(let i = 0; i < pathsArray.length; i++){ const pathStr = pathsArray[i]; const cleanPathStr = pathStr.slice(dirPathLength + 1); + if (cleanPathStr === '') continue; const dirPathArray = cleanPathStr.split(path.sep); // Create the list of unique values structureObject[dirPathArray[0]] = null; @@ -155,11 +158,52 @@ export class JavaFilesExtractor { return Object.keys(structureObject); } + /** + * Returns name w/o extension + * @param name - name of the file + */ + public static getStrippedName(name: string): string { + const fileBaseName = path.basename(name), + fileExtension = getFileEnding(fileBaseName); + return fileBaseName.substring(0, fileBaseName.length-fileExtension.length); + } + + /** + * Returns path to JAVA_HOME, or throw exception if the extracted archive isn't valid + * @param pathToStructure - path to files extracted from the JDK archive + */ + public static getJavaHomeFromStructure(pathToStructure): string { + const structure = taskLib.find(pathToStructure); + const rootDirectoriesArray = JavaFilesExtractor.sliceStructure(structure); + let jdkDirectory; + if (rootDirectoriesArray.find(dir => dir === BIN_FOLDER)){ + jdkDirectory = pathToStructure; + } else { + jdkDirectory = path.join(pathToStructure, rootDirectoriesArray[0]); + const ifBinExistsInside = fs.existsSync(path.join(jdkDirectory, BIN_FOLDER)); + if (rootDirectoriesArray.length > 1 || !ifBinExistsInside){ + throw new Error(taskLib.loc('WrongArchiveStructure')); + } + } + return jdkDirectory; + } + + /** + * Validate files structure if it can be a JDK, then set JAVA_HOME and returns it. + * @param pathToExtractedJDK - path to files extracted from the JDK archive + * @param withValidation - validate files and search bin inside + */ + public static setJavaHome(pathToExtractedJDK: string, withValidation: boolean = true): string { + let jdkDirectory = withValidation ? + JavaFilesExtractor.getJavaHomeFromStructure(pathToExtractedJDK) : + pathToExtractedJDK; + console.log(taskLib.loc('SetJavaHome', jdkDirectory)); + taskLib.setVariable('JAVA_HOME', jdkDirectory); + return jdkDirectory; + } + public async unzipJavaDownload(repoRoot: string, fileEnding: string, extractLocation: string): Promise { this.destinationFolder = extractLocation; - let initialDirectoriesList: string[]; - let finalDirectoriesList: string[]; - let jdkDirectory: string; // Create the destination folder if it doesn't exist if (!taskLib.exist(this.destinationFolder)) { @@ -167,30 +211,13 @@ export class JavaFilesExtractor { taskLib.mkdirP(this.destinationFolder); } - initialDirectoriesList = taskLib.find(this.destinationFolder).filter(x => taskLib.stats(x).isDirectory()); - const jdkFile = path.normalize(repoRoot); const stats = taskLib.stats(jdkFile); if (stats.isFile()) { await this.extractFiles(jdkFile, fileEnding); - finalDirectoriesList = taskLib.find(this.destinationFolder).filter(x => taskLib.stats(x).isDirectory()); - const freshExtractedDirectories = finalDirectoriesList.filter(dir => initialDirectoriesList.indexOf(dir) < 0); - const rootStructure = JavaFilesExtractor.sliceStructure(freshExtractedDirectories, this.destinationFolder); - let jdkDirectory = freshExtractedDirectories[0]; - if (rootStructure.find(dir => dir === BIN_FOLDER)){ - // In case of 'bin' folder was extracted directly to the destination folder - jdkDirectory = path.join(jdkDirectory, '..'); - } else { - // Check if bin is inside the current JDK directory - const ifBinExists = fs.existsSync(path.join(jdkDirectory, BIN_FOLDER)); - if (rootStructure.length > 1 || !ifBinExists){ - throw new Error(taskLib.loc('WrongArchiveStructure')); - } - } - taskLib.debug(`Using folder "${jdkDirectory}" for JDK`); - this.unpackJars(jdkDirectory, path.join(jdkDirectory, 'bin')); - return jdkDirectory; } + const jdkDirectory = JavaFilesExtractor.getJavaHomeFromStructure(this.destinationFolder); + JavaFilesExtractor.unpackJars(jdkDirectory, path.join(jdkDirectory, BIN_FOLDER)); + return jdkDirectory; } - } diff --git a/Tasks/JavaToolInstallerV0/Strings/resources.resjson/en-US/resources.resjson b/Tasks/JavaToolInstallerV0/Strings/resources.resjson/en-US/resources.resjson index c821a77d5e0a..a9ce6c46be26 100644 --- a/Tasks/JavaToolInstallerV0/Strings/resources.resjson/en-US/resources.resjson +++ b/Tasks/JavaToolInstallerV0/Strings/resources.resjson/en-US/resources.resjson @@ -11,7 +11,7 @@ "loc.input.label.jdkSourceOption": "JDK source", "loc.input.help.jdkSourceOption": "Source for the compressed JDK.", "loc.input.label.jdkFile": "JDK file", - "loc.input.help.jdkFile": "Path to where the compressed JDK is located. The path could be in your source repository or a local path on the agent.", + "loc.input.help.jdkFile": "Path to compressed JDK file. The path could be in your source repository or a local path on the agent.", "loc.input.label.azureResourceManagerEndpoint": "Azure subscription", "loc.input.help.azureResourceManagerEndpoint": "Choose the Azure Resource Manager subscription for the JDK.", "loc.input.label.azureStorageAccountName": "Storage account name", @@ -42,6 +42,8 @@ "loc.messages.CreateDestDir": "Creating destination folder: %s", "loc.messages.RetrievingJdkFromAzure": "Retrieving the JDK from Azure blob storage.", "loc.messages.RetrievingJdkFromLocalPath": "Retrieving the JDK from local path.", + "loc.messages.ArchiveWasExtractedEarlier": "This JDK file was extracted earlier. Use from cache.", + "loc.messages.ExtractingArchiveToPath": "Extracting JDK file to %s", "loc.messages.SucceedMsg": "Successfully extracted all files.", "loc.messages.SetJavaHome": "JAVA_HOME is being set to: %s", "loc.messages.SetExtendedJavaHome": "%s is being set to: %s", diff --git a/Tasks/JavaToolInstallerV0/javatoolinstaller.ts b/Tasks/JavaToolInstallerV0/javatoolinstaller.ts index 52365cefd465..0b81cbb26519 100644 --- a/Tasks/JavaToolInstallerV0/javatoolinstaller.ts +++ b/Tasks/JavaToolInstallerV0/javatoolinstaller.ts @@ -27,6 +27,7 @@ async function getJava(versionSpec: string) { const cleanDestinationDirectory: boolean = taskLib.getBoolInput('cleanDestinationDirectory', false); let compressedFileExtension: string; let jdkDirectory: string; + let extractionDirectory: string; const extendedJavaHome: string = `JAVA_HOME_${versionSpec}_${taskLib.getInput('jdkArchitectureOption', true)}`; toolLib.debug('Trying to get tool from local cache first'); @@ -36,7 +37,6 @@ async function getJava(versionSpec: string) { // Clean the destination folder before downloading and extracting? if (cleanDestinationDirectory && taskLib.exist(extractLocation) && taskLib.stats(extractLocation).isDirectory) { console.log(taskLib.loc('CleanDestDir', extractLocation)); - // delete the contents of the destination directory but leave the directory in place fs.readdirSync(extractLocation) .forEach((item: string) => { @@ -53,7 +53,7 @@ async function getJava(versionSpec: string) { throw new Error(taskLib.loc('JavaNotPreinstalled', versionSpec)); } console.log(taskLib.loc('UsePreinstalledJava', preInstalledJavaDirectory)); - jdkDirectory = preInstalledJavaDirectory; + jdkDirectory = JavaFilesExtractor.setJavaHome(preInstalledJavaDirectory, false); } else if (fromAzure) { //Download JDK from an Azure blob storage location and extract. console.log(taskLib.loc('RetrievingJdkFromAzure')); const fileNameAndPath: string = taskLib.getInput('azureCommonVirtualFile', false); @@ -66,17 +66,26 @@ async function getJava(versionSpec: string) { const extractSource = buildFilePath(extractLocation, compressedFileExtension, fileNameAndPath); const javaFilesExtractor = new JavaFilesExtractor(); - jdkDirectory = await javaFilesExtractor.unzipJavaDownload(extractSource, compressedFileExtension, extractLocation); + await javaFilesExtractor.unzipJavaDownload(extractSource, compressedFileExtension, extractLocation); + jdkDirectory = JavaFilesExtractor.setJavaHome(extractLocation); } else { //JDK is in a local directory. Extract to specified target directory. console.log(taskLib.loc('RetrievingJdkFromLocalPath')); - compressedFileExtension = getFileEnding(taskLib.getInput('jdkFile', true)); + const jdkFileName = taskLib.getInput('jdkFile', true); + compressedFileExtension = getFileEnding(jdkFileName); const javaFilesExtractor = new JavaFilesExtractor(); - jdkDirectory = await javaFilesExtractor.unzipJavaDownload(taskLib.getInput('jdkFile', true), compressedFileExtension, extractLocation); + const extractDirectoryName = `${extendedJavaHome}_${JavaFilesExtractor.getStrippedName(jdkFileName)}_${compressedFileExtension}`; + extractionDirectory = path.join(extractLocation, extractDirectoryName); + if (!cleanDestinationDirectory && taskLib.exist(extractionDirectory)){ + // do nothing since the files were extracted and ready for using + console.log(taskLib.loc('ArchiveWasExtractedEarlier')); + } else { + // unpack files to specified directory + console.log(taskLib.loc('ExtractingArchiveToPath', extractionDirectory)); + await javaFilesExtractor.unzipJavaDownload(jdkFileName, compressedFileExtension, extractionDirectory); + } + jdkDirectory = JavaFilesExtractor.setJavaHome(extractionDirectory); } - - console.log(taskLib.loc('SetJavaHome', jdkDirectory)); console.log(taskLib.loc('SetExtendedJavaHome', extendedJavaHome, jdkDirectory)); - taskLib.setVariable('JAVA_HOME', jdkDirectory); taskLib.setVariable(extendedJavaHome, jdkDirectory); toolLib.prependPath(path.join(jdkDirectory, BIN_FOLDER)); } @@ -89,12 +98,10 @@ function sleepFor(sleepDurationInMillisecondsSeconds): Promise { function buildFilePath(localPathRoot: string, fileEnding: string, fileNameAndPath: string): string { const fileName = fileNameAndPath.split(/[\\\/]/).pop(); - const extractSource = path.join(localPathRoot, fileName); - - return extractSource; + return path.join(localPathRoot, fileName); } -function getFileEnding(file: string): string { +export function getFileEnding(file: string): string { let fileEnding = ''; if (file.endsWith('.tar')) { diff --git a/Tasks/JavaToolInstallerV0/task.json b/Tasks/JavaToolInstallerV0/task.json index ee55b632b759..9f7ffc9317a1 100644 --- a/Tasks/JavaToolInstallerV0/task.json +++ b/Tasks/JavaToolInstallerV0/task.json @@ -13,7 +13,7 @@ "author": "Microsoft Corporation", "version": { "Major": 0, - "Minor": 171, + "Minor": 172, "Patch": 0 }, "satisfies": [ @@ -66,7 +66,7 @@ "label": "JDK file", "required": true, "visibleRule": "jdkSourceOption == LocalDirectory", - "helpMarkDown": "Path to where the compressed JDK is located. The path could be in your source repository or a local path on the agent." + "helpMarkDown": "Path to compressed JDK file. The path could be in your source repository or a local path on the agent." }, { "name": "azureResourceManagerEndpoint", @@ -166,6 +166,8 @@ "CreateDestDir": "Creating destination folder: %s", "RetrievingJdkFromAzure": "Retrieving the JDK from Azure blob storage.", "RetrievingJdkFromLocalPath": "Retrieving the JDK from local path.", + "ArchiveWasExtractedEarlier": "This JDK file was extracted earlier. Use from cache.", + "ExtractingArchiveToPath": "Extracting JDK file to %s", "SucceedMsg": "Successfully extracted all files.", "SetJavaHome": "JAVA_HOME is being set to: %s", "SetExtendedJavaHome": "%s is being set to: %s", diff --git a/Tasks/JavaToolInstallerV0/task.loc.json b/Tasks/JavaToolInstallerV0/task.loc.json index 2dbf1e1f0f81..01de1d78053a 100644 --- a/Tasks/JavaToolInstallerV0/task.loc.json +++ b/Tasks/JavaToolInstallerV0/task.loc.json @@ -13,7 +13,7 @@ "author": "Microsoft Corporation", "version": { "Major": 0, - "Minor": 171, + "Minor": 172, "Patch": 0 }, "satisfies": [ @@ -166,6 +166,8 @@ "CreateDestDir": "ms-resource:loc.messages.CreateDestDir", "RetrievingJdkFromAzure": "ms-resource:loc.messages.RetrievingJdkFromAzure", "RetrievingJdkFromLocalPath": "ms-resource:loc.messages.RetrievingJdkFromLocalPath", + "ArchiveWasExtractedEarlier": "ms-resource:loc.messages.ArchiveWasExtractedEarlier", + "ExtractingArchiveToPath": "ms-resource:loc.messages.ExtractingArchiveToPath", "SucceedMsg": "ms-resource:loc.messages.SucceedMsg", "SetJavaHome": "ms-resource:loc.messages.SetJavaHome", "SetExtendedJavaHome": "ms-resource:loc.messages.SetExtendedJavaHome", @@ -179,4 +181,4 @@ "UsePreinstalledJava": "ms-resource:loc.messages.UsePreinstalledJava", "WrongArchiveStructure": "ms-resource:loc.messages.WrongArchiveStructure" } -} +} \ No newline at end of file