Update build pipeline to support LocalPackage build config (parameter…

…ized; not enabled by default) (#20552)

BuildConfigGen/GitUtil.cs

@@ -95,30 +95,6 @@ public static void GetUntrackedFiles(string taskTarget, out IEnumerable<string>
  toAdd = untrackedOuput2;
  toRemove = untrackedOuputRemove;
  }
- internal static string GetGitRootPath(string currentDir)
- {
- const string args = "rev-parse --git-dir";
- string path = RunGitCommandScalar(currentDir, args);
-
- path = FixupPath(path);
-
- const string gitDir = ".git";
- if (path.EndsWith(gitDir))
- {
- path = path.Substring(0, path.Length - gitDir.Length);
-
- if (path == "")
- {
- return currentDir;
- }
-
- return path;
- }
- else
- {
- throw new Exception($"expected git {args} to return ");
- }
- }
 
  internal static IEnumerable<string> GetNonIgnoredFileListFromPath(string gitRoot, string taskTarget)
  {

BuildConfigGen/Program.cs

@@ -109,7 +109,7 @@ private static void MainInner(string? task, string? configs, int? currentSprintN
  }
 
  string currentDir = Environment.CurrentDirectory;
- string gitRootPath = GitUtil.GetGitRootPath(currentDir);
+ string gitRootPath = GetTasksRootPath(currentDir);
 
  string globalVersionPath = Path.Combine(gitRootPath, @"globalversion.txt");
  TaskVersion? globalVersion = GetGlobalVersion(gitRootPath, globalVersionPath);
@@ -208,7 +208,7 @@ private static void MainInner(string? task, string? configs, int? currentSprintN
  Console.WriteLine($"Global version: maxPatchForCurrentSprint = maxPatchForCurrentSprint + 1");
  }
 
- Console.WriteLine($"Global version update: globalVersion = {globalVersion} maxPatchForCurrentSprint={maxPatchForCurrentSprint}" );
+ Console.WriteLine($"Global version update: globalVersion = {globalVersion} maxPatchForCurrentSprint={maxPatchForCurrentSprint}");
  }
  else
  {
@@ -249,6 +249,14 @@ private static void MainInner(string? task, string? configs, int? currentSprintN
  }
  }
  }
+ else
+ {
+ // if we're not updating local packages, we need to ensure the global version is updated to the task major version. existing patch number is preserved
+ if (globalVersion is not null)
+ {
+ globalVersion = globalVersion.CloneWithMajor(taskMajorVersion);
+ }
+ }
 
  if (globalVersion is not null)
  {
@@ -265,7 +273,7 @@ private static void MainInner(string? task, string? configs, int? currentSprintN
  ensureUpdateModeVerifier!.WriteAllText(globalVersionPath, globalVersion!.MinorPatchToString(), false);
  }
 
- ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferError("(global)", skipContentCheck: false);
+ ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferError(null, skipContentCheck: false);
 
  foreach (var t in tasks)
  {
@@ -284,6 +292,37 @@ private static void MainInner(string? task, string? configs, int? currentSprintN
  }
  }
 
+ private static string GetTasksRootPath(string inputCurrentDir)
+ {
+ string? currentDir = inputCurrentDir;
+ string? tasksRootPath = null;
+
+ do
+ {
+ string currentDirGit = Path.Combine(currentDir, ".git");
+
+ if (Directory.Exists(currentDirGit))
+ {
+ tasksRootPath = currentDir;
+ }
+
+ currentDir = (new DirectoryInfo(currentDir)).Parent?.FullName;
+
+ } while (currentDir != null);
+
+ if (tasksRootPath == null)
+ {
+ throw new Exception($"could not find .git in {currentDir}");
+ }
+
+ if (!File.Exists(Path.Combine(tasksRootPath, "make-options.json")))
+ {
+ throw new Exception($"make-options.json not found in tasksRootPath={tasksRootPath}");
+ }
+
+ return tasksRootPath;
+ }
+
  private static IEnumerable<string> FilterConfigsForTask(string? configs, KeyValuePair<string, MakeOptionsReader.AgentTask> t)
  {
  var configsList = t.Value.Configs.AsEnumerable();
@@ -343,7 +382,7 @@ private static void GetVersions(string task, string configsString, out List<(str
 
  string currentDir = Environment.CurrentDirectory;
 
- string gitRootPath = GitUtil.GetGitRootPath(currentDir);
+ string gitRootPath = GetTasksRootPath(currentDir);
 
  string taskTargetPath = Path.Combine(gitRootPath, "Tasks", task);
  if (!Directory.Exists(taskTargetPath))
@@ -419,7 +458,7 @@ private static int GetCurrentSprint()
  return currentSprint;
  }
 
- private static void ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferError(string task, bool skipContentCheck)
+ private static void ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferError(string? task, bool skipContentCheck)
  {
  // if !writeUpdates, error if we have written any updates
  var verifyErrors = ensureUpdateModeVerifier!.GetVerifyErrors(skipContentCheck).ToList();
@@ -433,7 +472,14 @@ private static void ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferEr
  Console.WriteLine(s);
  }
 
- throw new Exception($"Updates needed, please run npm make.js --task {task} ");
+ if (task is null)
+ {
+ throw new Exception($"Updates needed, please run node make.js");
+ }
+ else
+ {
+ throw new Exception($"Updates needed, please run node make.js --task {task} ");
+ }
  }
  }
 
@@ -459,8 +505,8 @@ private static void MainUpdateTask(
  try
  {
  string currentDir = Environment.CurrentDirectory;
- string gitRootPath = GitUtil.GetGitRootPath(currentDir);
- string versionMapFile = GetVersionMapFile(task, gitRootPath, generatedFolder);
+ string gitRootPath = GetTasksRootPath(currentDir);
+ string versionMapFile = GetVersionMapFile(task, generatedFolder);
 
  string taskTargetPath = Path.Combine(gitRootPath, "Tasks", task);
  if (!Directory.Exists(taskTargetPath))
@@ -489,7 +535,11 @@ private static void MainUpdateTask(
 
  foreach (var config in targetConfigs)
  {
- if (config.useGlobalVersion && !hasGlobalVersion)
+ if (config.useGlobalVersion && !includeLocalPackagesBuildConfig)
+ {
+ Console.WriteLine($"Info: MainUpdateTask: Skipping useGlobalVersion config for task b/c --include-local-packages-build-config. not specified. hasGlobalVersion={hasGlobalVersion} config.useGlobalVersion={config.useGlobalVersion} includeLocalPackagesBuildConfig={includeLocalPackagesBuildConfig}");
+ }
+ else if (config.useGlobalVersion && !hasGlobalVersion)
  {
  Console.WriteLine($"Info: MainUpdateTask: Skipping useGlobalVersion config for task b/c GlobalVersion not initialized. (to opt-in and start producing LocalBuildConfig, run with --include-local-packages-build-config. hasGlobalVersion={hasGlobalVersion} config.useGlobalVersion={config.useGlobalVersion}). Note: this is not an error!");
  }
@@ -608,7 +658,7 @@ private static void MainUpdateTask(
  }
  }
 
- private static string GetVersionMapFile(string task, string gitRootPath, string generatedFolder)
+ private static string GetVersionMapFile(string task, string generatedFolder)
  {
  return Path.Combine(generatedFolder, @$"{task}.versionmap.txt");
  }
@@ -1078,7 +1128,7 @@ private static void CopyConfig(string gitRootPath, string taskTargetPathOrUnders
  private static void UpdateVersionsForTask(string task, TaskStateStruct taskState, HashSet<Config.ConfigRecord> targetConfigs, int currentSprint, string globalVersionPath, TaskVersion? globalVersion, string generatedFolder)
  {
  string currentDir = Environment.CurrentDirectory;
- string gitRootPath = GitUtil.GetGitRootPath(currentDir);
+ string gitRootPath = GetTasksRootPath(currentDir);
  string taskTargetPath = Path.Combine(gitRootPath, "Tasks", task);
 
  if (!Directory.Exists(taskTargetPath))
@@ -1093,7 +1143,7 @@ private static void UpdateVersionsForTask(string task, TaskStateStruct taskState
 
  bool defaultVersionMatchesSourceVersion;
 
- string versionMapFile = GetVersionMapFile(task, gitRootPath, generatedFolder);
+ string versionMapFile = GetVersionMapFile(task, generatedFolder);
 
  {
  TaskVersion? defaultVersion = null;

BuildConfigGen/dev.sh

@@ -36,17 +36,6 @@ function detect_platform_and_runtime_id ()
  fi
 }
 
-function cmd_build ()
-{
- heading "Building"
- dotnet build -o bin $SOLUTION_PATH || failed build
- #change execution flag to allow running with sudo
- if [[ ("$CURRENT_PLATFORM" == "linux") || ("$CURRENT_PLATFORM" == "darwin") ]]; then
- chmod +x "bin/BuildConfigGen"
- fi
-
-}
-
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 pushd "$SCRIPT_DIR"
 source "$SCRIPT_DIR/Misc/helpers.sh"
@@ -91,6 +80,3 @@ echo "Adding .NET to PATH ${DOTNETSDK_INSTALLDIR}"
 export PATH=${DOTNETSDK_INSTALLDIR}:$PATH
 echo "Path = $PATH"
 echo ".NET Version = $(dotnet --version)"
-
-SOLUTION_PATH="$SCRIPT_DIR/BuildConfigGen.sln"
-cmd_build

azure-pipelines.yml

@@ -29,6 +29,10 @@ parameters:
  displayName: Enable CodeQL for run
  type: boolean
  default: false
+- name: includeLocalPackagesBuildConfig
+ displayName: Flag to update LocalPackages buildconfig (for testing, this will be made default later)
+ type: boolean
+ default: false # note: keep in sync with ci\ci-test-tasks\canary-tests-v2.yml
 
 variables:
 - name: currentDate
@@ -41,7 +45,16 @@ variables:
  value: ${{ eq(parameters.enableCodeQL, true) }}
 - name: system.debug
  value: true
-
+- name: includeLocalPackagesBuildConfigParameter
+ ${{ if eq(parameters.includeLocalPackagesBuildConfig, true) }}:
+ value: '--includeLocalPackagesBuildConfig'
+ ${{ else }}:
+ value: ''
+- name: IncludeLocalPackagesBuildConfigTest
+ ${{ if eq(parameters.includeLocalPackagesBuildConfig, true) }}:
+ value: '1'
+ ${{ else }}:
+ value: ''
 
 extends:
  template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates

ci/build-all-steps.yml

@@ -106,7 +106,7 @@ steps:
  displayName: Clean tasks
 
 # Build Tasks
-- script: node make.js serverBuild --task "$(getTaskPattern.task_pattern)"
+- script: node make.js serverBuild --task "$(getTaskPattern.task_pattern)" $(includeLocalPackagesBuildConfigParameter)
  displayName: Build Tasks
  condition: and(succeeded(), ne(variables['numTasks'], 0))
 

ci/build-all-tasks.yml

@@ -30,12 +30,12 @@ steps:
  displayName: Clean tasks
 
 # Build tasks
-- script: node make.js serverBuild --task "$(task_pattern)"
+- script: node make.js serverBuild --task "$(task_pattern)" $(includeLocalPackagesBuildConfigParameter)
  displayName: Build tasks using pattern
  condition: and(succeeded(), eq('${{ parameters.deploy_all_tasks }}', false))
 
 # Build all tasks
-- script: node make.js serverBuild
+- script: node make.js serverBuild $(includeLocalPackagesBuildConfigParameter)
  displayName: Build all tasks
  condition: and(succeeded(), eq('${{ parameters.deploy_all_tasks }}', true))
 

ci/build-single-steps.yml

@@ -50,7 +50,7 @@ steps:
  displayName: Clean tasks
 
 # Build Tasks
-- script: node make.js serverBuild --task "$(task_pattern)"
+- script: node make.js serverBuild --task "$(task_pattern)" $(includeLocalPackagesBuildConfigParameter)
  displayName: Build Tasks
 
 # Check diff for task sources

ci/ci-test-tasks/build-task.yml

@@ -22,7 +22,7 @@ steps:
  displayName: Clean tasks
 
 # Build
-- script: node make.js serverBuild --task "${{ parameters.task }}"
+- script: node make.js serverBuild --task "${{ parameters.task }}" $(includeLocalPackagesBuildConfigParameter)
  displayName: ${{ parameters.task }} build
 
 # Check diff for task sources

ci/ci-test-tasks/canary-tests-v2.yml

@@ -1,3 +1,21 @@
+parameters:
+- name: includeLocalPackagesBuildConfig
+ displayName: Flag to update LocalPackages buildconfig (for testing, this will be made default later)
+ type: boolean
+ default: false # Note: keep in sync with /azure-pipelines.yml
+
+variables:
+- name: includeLocalPackagesBuildConfigParameter
+ ${{ if eq(parameters.includeLocalPackagesBuildConfig, true) }}: 
+ value: '--includeLocalPackagesBuildConfig'
+ ${{ else }}:
+ value: ''
+- name: IncludeLocalPackagesBuildConfigTest
+ ${{ if eq(parameters.includeLocalPackagesBuildConfig, true) }}:
+ value: '1'
+ ${{ else }}:
+ value: ''
+
 trigger:
  - master
  - releases/*

make-util.js

@@ -1747,28 +1747,22 @@ exports.renameCodeCoverageOutput = renameCodeCoverageOutput;
 //------------------------------------------------------------------------------
 
 /**
- * Returns path to BuldConfigGenerator, build it if needed. Fail on compilation failure
+ * Ensure Pre-reqs for buildConfigGen (e.g. dotnet)
  * @param {String} baseConfigToolPath base build config tool path
- * @returns {String} Path to the executed file
  */
-var getBuildConfigGenerator = function (baseConfigToolPath) {
- var programPath = "";
+var ensureBuildConfigGeneratorPrereqs = function (baseConfigToolPath) {
  var configToolBuildUtility = "";
 
  if (os.platform() === 'win32') {
- programPath = path.join(baseConfigToolPath, 'bin', 'BuildConfigGen.exe');
  configToolBuildUtility = path.join(baseConfigToolPath, "dev.cmd");
  } else {
- programPath = path.join(baseConfigToolPath, 'bin', 'BuildConfigGen');
  configToolBuildUtility = path.join(baseConfigToolPath, "dev.sh");
  }
 
  // build configToolBuildUtility if needed. (up-to-date check will skip build if not needed)
  run(configToolBuildUtility, true);
-
- return programPath;
 };
-exports.getBuildConfigGenerator = getBuildConfigGenerator;
+exports.ensureBuildConfigGeneratorPrereqs = ensureBuildConfigGeneratorPrereqs;
 
 /**
  * Function to validate or write generated tasks
@@ -1785,7 +1779,9 @@ var processGeneratedTasks = function(baseConfigToolPath, taskList, makeOptions,
  if (sprintNumber && !Number.isInteger(sprintNumber)) fail("Sprint is not a number");
 
  var tasks = taskList.join('|')
- const programPath = getBuildConfigGenerator(baseConfigToolPath);
+ ensureBuildConfigGeneratorPrereqs(baseConfigToolPath);
+ var programPath = `dotnet run --project "${baseConfigToolPath}/BuildConfigGen.csproj" -- `
+
  const args = [
  "--task",
  `"${tasks}"`