From 2ebeb2059dcf2fe95bd285ea8155173403294eff Mon Sep 17 00:00:00 2001 From: praveenkuttappan Date: Tue, 10 Nov 2020 17:25:16 -0800 Subject: [PATCH 1/7] Handle packages in Microsoft scopeto support renaming of opentelemetry --- eng/scripts/Language-Settings.ps1 | 4 ++-- eng/tools/dependency-testing/index.js | 8 +++++++- .../templates/matthews/styles/main.js | 2 +- eng/tools/versioning/increment.js | 3 +-- eng/tools/versioning/set-version.js | 3 +-- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 4413fbb55dc5..77d25aa34f15 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -11,8 +11,8 @@ function Get-javascript-PackageInfoFromRepo ($pkgPath, $serviceDirectory, $pkgNa if (Test-Path $projectPath) { $projectJson = Get-Content $projectPath | ConvertFrom-Json - $jsStylePkgName = $pkgName.replace("azure-", "@azure/") - if ($projectJson.name -eq "$jsStylePkgName") + $jsStylePkgName = $projectJson.name.Replace("@", "").Replace("/", "-") + if ($pkgName -eq "$jsStylePkgName") { return [PackageProps]::new($projectJson.name, $projectJson.version, $pkgPath, $serviceDirectory) } diff --git a/eng/tools/dependency-testing/index.js b/eng/tools/dependency-testing/index.js index 89ceb474d8b2..f2ecb8553c60 100644 --- a/eng/tools/dependency-testing/index.js +++ b/eng/tools/dependency-testing/index.js @@ -332,7 +332,13 @@ async function main(argv) { const testFolder = argv["test-folder"]; const dryRun = argv["dry-run"]; - const packageName = artifactName.replace("azure-", "@azure/"); + let packageName = ""; + if (artifactName.startsWith("microsoft-")) { + packageName = artifactName.replace("microsoft-", "@microsoft/"); + } + else { + packageName = artifactName.replace("azure-", "@azure/"); + } const targetPackage = await getPackageFromRush(repoRoot, packageName); const targetPackagePath = path.join(repoRoot, targetPackage.projectFolder); diff --git a/eng/tools/generate-static-index/templates/matthews/styles/main.js b/eng/tools/generate-static-index/templates/matthews/styles/main.js index 3520a4ef736a..5a500e096297 100644 --- a/eng/tools/generate-static-index/templates/matthews/styles/main.js +++ b/eng/tools/generate-static-index/templates/matthews/styles/main.js @@ -106,7 +106,7 @@ function getPackageUrl(language, package, version) { $(function () { $('h4').each(function () { var pkgName = $(this).text() - .replace("@azure/", "azure-"); + .replace("@", "").replace("/", "-"); populateIndexList($(this), pkgName) }); }) diff --git a/eng/tools/versioning/increment.js b/eng/tools/versioning/increment.js index b7b9a0a77941..7e4af1b38bd3 100644 --- a/eng/tools/versioning/increment.js +++ b/eng/tools/versioning/increment.js @@ -37,10 +37,9 @@ async function main(argv) { const repoRoot = argv["repo-root"]; const dryRun = argv["dry-run"]; - const packageName = artifactName.replace("azure-", "@azure/"); const rushSpec = await packageUtils.getRushSpec(repoRoot); const targetPackage = rushSpec.projects.find( - packageSpec => packageSpec.packageName == packageName + packageSpec => packageSpec.packageName.replace("@", "").replace("/", "-") == artifactName ); const targetPackagePath = path.join(repoRoot, targetPackage.projectFolder); diff --git a/eng/tools/versioning/set-version.js b/eng/tools/versioning/set-version.js index 504b037bc559..18f6bb21401d 100644 --- a/eng/tools/versioning/set-version.js +++ b/eng/tools/versioning/set-version.js @@ -33,11 +33,10 @@ async function main(argv) { const repoRoot = argv["repo-root"]; const dryRun = argv["dry-run"]; - const packageName = artifactName.replace("azure-", "@azure/"); const rushSpec = await packageUtils.getRushSpec(repoRoot); const targetPackage = rushSpec.projects.find( - packageSpec => packageSpec.packageName == packageName + packageSpec => packageSpec.packageName.replace("@", "").replace("/", "-") == packageName ); const targetPackagePath = path.join(repoRoot, targetPackage.projectFolder); From aedcffc2bee002c141b8f3c80cae46de9e3717d1 Mon Sep 17 00:00:00 2001 From: praveenkuttappan Date: Wed, 11 Nov 2020 12:47:42 -0800 Subject: [PATCH 2/7] Additional fixes --- eng/scripts/Language-Settings.ps1 | 8 +++++++- eng/tools/analyze-deps/index.js | 2 +- eng/tools/dependency-testing/index.js | 8 +------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 77d25aa34f15..1b2b9a4985c5 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -4,6 +4,7 @@ $PackageRepository = "NPM" $packagePattern = "*.tgz" $MetadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/js-packages.csv" $BlobStorageUrl = "https://azuresdkdocs.blob.core.windows.net/%24web?restype=container&comp=list&prefix=javascript%2F&delimiter=%2F" +$ArtifactToPackgeMap = @{'opentelemetry-exporter-azure-monitor'='microsoft-opentelemetry-exporter-azure-monitor'} function Get-javascript-PackageInfoFromRepo ($pkgPath, $serviceDirectory, $pkgName) { @@ -89,6 +90,11 @@ function Publish-javascript-GithubIODocs ($DocLocation, $PublicArtifactLocation) foreach ($Item in $PublishedDocs) { $PkgName = "azure-$($Item.BaseName)" + # Override package name from other scope. We don't have package name to parse this out here. + if ($ArtifactToPackgeMap.ContainsKey($($Item.BaseName))) + { + $PkgName = $ArtifactToPackgeMap[$($Item.BaseName)] + } Write-Host $PkgName Expand-Archive -Force -Path "$($DocLocation)/documentation/$($Item.Name)" -DestinationPath "$($DocLocation)/documentation/$($Item.BaseName)" $dirList = Get-ChildItem "$($DocLocation)/documentation/$($Item.BaseName)/$($Item.BaseName)" -Attributes Directory @@ -111,7 +117,7 @@ function Get-javascript-GithubIoDocIndex() { # Fetch out all package metadata from csv file. $metadata = Get-CSVMetadata -MetadataUri $MetadataUri # Get the artifacts name from blob storage - $artifacts = Get-BlobStorage-Artifacts -blobStorageUrl $BlobStorageUrl -blobDirectoryRegex "^javascript/azure-(.*)/$" -blobArtifactsReplacement "@azure/`${1}" + $artifacts = Get-BlobStorage-Artifacts -blobStorageUrl $BlobStorageUrl -blobDirectoryRegex "^javascript/([a-z]*)-(.*)/$" -blobArtifactsReplacement "@`${1}/`${2}" # Build up the artifact to service name mapping for GithubIo toc. $tocContent = Get-TocMapping -metadata $metadata -artifacts $artifacts # Generate yml/md toc files and build site. diff --git a/eng/tools/analyze-deps/index.js b/eng/tools/analyze-deps/index.js index 67830b11e8d3..b51404872028 100644 --- a/eng/tools/analyze-deps/index.js +++ b/eng/tools/analyze-deps/index.js @@ -186,7 +186,7 @@ const dumpRushPackages = (rushPackages, internalPackages, external) => { }; const resolveRushPackageDeps = (packages, internalPackages, pnpmLock, pkgId, external) => { - const yamlKey = `@rush-temp/${packages[pkgId].name.replace("@azure/", "")}`; + const yamlKey = `@rush-temp/${packages[pkgId].name.replace(/@[a-z]*\//i, "")}`; const packageKey = pnpmLock.dependencies[yamlKey]; const resolvedDeps = pnpmLock.packages[packageKey].dependencies; diff --git a/eng/tools/dependency-testing/index.js b/eng/tools/dependency-testing/index.js index f2ecb8553c60..e6e3613883a3 100644 --- a/eng/tools/dependency-testing/index.js +++ b/eng/tools/dependency-testing/index.js @@ -332,13 +332,7 @@ async function main(argv) { const testFolder = argv["test-folder"]; const dryRun = argv["dry-run"]; - let packageName = ""; - if (artifactName.startsWith("microsoft-")) { - packageName = artifactName.replace("microsoft-", "@microsoft/"); - } - else { - packageName = artifactName.replace("azure-", "@azure/"); - } + const packageName = artifactName.replace(/"?([a-z]*)"?-/i, "@$1/"); const targetPackage = await getPackageFromRush(repoRoot, packageName); const targetPackagePath = path.join(repoRoot, targetPackage.projectFolder); From 7512ef637bf837405bf1cb7f1562966642a8cbd9 Mon Sep 17 00:00:00 2001 From: praveenkuttappan Date: Wed, 11 Nov 2020 17:18:33 -0800 Subject: [PATCH 3/7] Retrieve package name from artifact --- .../scripts/artifact-metadata-parsing.ps1 | 33 ++++++++++++++++--- eng/scripts/Language-Settings.ps1 | 19 ++++++----- 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/eng/common/scripts/artifact-metadata-parsing.ps1 b/eng/common/scripts/artifact-metadata-parsing.ps1 index b2ca199d455e..9c740f89f82b 100644 --- a/eng/common/scripts/artifact-metadata-parsing.ps1 +++ b/eng/common/scripts/artifact-metadata-parsing.ps1 @@ -393,8 +393,8 @@ function GetExistingTags($apiUrl) { } } -# Retrieve release tag for artiface package. If multiple packages, then output the first one. -function RetrieveReleaseTag($pkgRepository, $artifactLocation, $continueOnError = $true) { +# Retrieve package info for artiface package. +function RetrieveReleasePackageInfo($pkgRepository, $artifactLocation, $continueOnError = $true) { if (!$artifactLocation) { return "" } @@ -402,17 +402,39 @@ function RetrieveReleaseTag($pkgRepository, $artifactLocation, $continueOnError $pkgs, $parsePkgInfoFn = RetrievePackages -pkgRepository $pkgRepository -artifactLocation $artifactLocation if (!$pkgs -or !$pkgs[0]) { Write-Host "No packages retrieved from artifact location." - return "" + return $null } if ($pkgs.Count -gt 1) { Write-Host "There are more than 1 packages retieved from artifact location." foreach ($pkg in $pkgs) { Write-Host "The package name is $($pkg.BaseName)" } - return "" + return $null } $parsedPackage = &$parsePkgInfoFn -pkg $pkgs[0] -workingDirectory $artifactLocation - return $parsedPackage.ReleaseTag + return $parsedPackage + } + catch { + if ($continueOnError) { + return $null + } + Write-Error "No release tag retrieved from $artifactLocation" + } +} + +# Retrieve release tag for artiface package. If multiple packages, then output the first one. +function RetrieveReleaseTag($pkgRepository, $artifactLocation, $continueOnError = $true) { + if (!$artifactLocation) { + return "" + } + try { + $parsedPackage = RetrieveReleasePackageInfo -pkgRepository $pkgRepository -artifactLocation $artifactLocation + $tag = "" + if ($parsedPackage -ne $null) + { + $tag = $parsedPackage.ReleaseTag + } + return $tag } catch { if ($continueOnError) { @@ -421,6 +443,7 @@ function RetrieveReleaseTag($pkgRepository, $artifactLocation, $continueOnError Write-Error "No release tag retrieved from $artifactLocation" } } + function RetrievePackages($pkgRepository, $artifactLocation) { $parsePkgInfoFn = "" $packagePattern = "" diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 1b2b9a4985c5..473876ea86b6 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -4,7 +4,6 @@ $PackageRepository = "NPM" $packagePattern = "*.tgz" $MetadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/js-packages.csv" $BlobStorageUrl = "https://azuresdkdocs.blob.core.windows.net/%24web?restype=container&comp=list&prefix=javascript%2F&delimiter=%2F" -$ArtifactToPackgeMap = @{'opentelemetry-exporter-azure-monitor'='microsoft-opentelemetry-exporter-azure-monitor'} function Get-javascript-PackageInfoFromRepo ($pkgPath, $serviceDirectory, $pkgName) { @@ -88,20 +87,22 @@ function Publish-javascript-GithubIODocs ($DocLocation, $PublicArtifactLocation) $PublishedDocs = Get-ChildItem "$($DocLocation)/documentation" | Where-Object -FilterScript { $_.Name.EndsWith(".zip") } foreach ($Item in $PublishedDocs) - { - $PkgName = "azure-$($Item.BaseName)" - # Override package name from other scope. We don't have package name to parse this out here. - if ($ArtifactToPackgeMap.ContainsKey($($Item.BaseName))) - { - $PkgName = $ArtifactToPackgeMap[$($Item.BaseName)] - } - Write-Host $PkgName + { Expand-Archive -Force -Path "$($DocLocation)/documentation/$($Item.Name)" -DestinationPath "$($DocLocation)/documentation/$($Item.BaseName)" $dirList = Get-ChildItem "$($DocLocation)/documentation/$($Item.BaseName)/$($Item.BaseName)" -Attributes Directory if ($dirList.Length -eq 1) { $DocVersion = $dirList[0].Name + $parsedPackage = RetrieveReleasePackageInfo "NPM" $PublicArtifactLocation + $tag = "" + # set default package name + $PkgName = "azure-$($Item.BaseName)" + if ($parsedPackage -ne $null) + { + $PkgName = $parsedPackage.PackageId.Replace("@", "").Replace("/", "-") + $tag = $parsedPackage.ReleaseTag + } Write-Host "Uploading Doc for $($PkgName) Version:- $($DocVersion)..." $releaseTag = RetrieveReleaseTag "NPM" $PublicArtifactLocation Upload-Blobs -DocDir "$($DocLocation)/documentation/$($Item.BaseName)/$($Item.BaseName)/$($DocVersion)" -PkgName $PkgName -DocVersion $DocVersion -ReleaseTag $releaseTag From e747db4bf0460cf7a3b72f17f0598b0bbb3ef739 Mon Sep 17 00:00:00 2001 From: praveenkuttappan Date: Thu, 12 Nov 2020 11:57:56 -0800 Subject: [PATCH 4/7] Revert changes in common and reuse existing function to get package name --- .../scripts/artifact-metadata-parsing.ps1 | 32 +++---------------- eng/scripts/Language-Settings.ps1 | 7 ++-- 2 files changed, 8 insertions(+), 31 deletions(-) diff --git a/eng/common/scripts/artifact-metadata-parsing.ps1 b/eng/common/scripts/artifact-metadata-parsing.ps1 index 9c740f89f82b..4aae1213ba83 100644 --- a/eng/common/scripts/artifact-metadata-parsing.ps1 +++ b/eng/common/scripts/artifact-metadata-parsing.ps1 @@ -393,8 +393,8 @@ function GetExistingTags($apiUrl) { } } -# Retrieve package info for artiface package. -function RetrieveReleasePackageInfo($pkgRepository, $artifactLocation, $continueOnError = $true) { +# Retrieve release tag for artiface package. If multiple packages, then output the first one. +function RetrieveReleaseTag($pkgRepository, $artifactLocation, $continueOnError = $true) { if (!$artifactLocation) { return "" } @@ -402,39 +402,17 @@ function RetrieveReleasePackageInfo($pkgRepository, $artifactLocation, $continue $pkgs, $parsePkgInfoFn = RetrievePackages -pkgRepository $pkgRepository -artifactLocation $artifactLocation if (!$pkgs -or !$pkgs[0]) { Write-Host "No packages retrieved from artifact location." - return $null + return "" } if ($pkgs.Count -gt 1) { Write-Host "There are more than 1 packages retieved from artifact location." foreach ($pkg in $pkgs) { Write-Host "The package name is $($pkg.BaseName)" } - return $null + return "" } $parsedPackage = &$parsePkgInfoFn -pkg $pkgs[0] -workingDirectory $artifactLocation - return $parsedPackage - } - catch { - if ($continueOnError) { - return $null - } - Write-Error "No release tag retrieved from $artifactLocation" - } -} - -# Retrieve release tag for artiface package. If multiple packages, then output the first one. -function RetrieveReleaseTag($pkgRepository, $artifactLocation, $continueOnError = $true) { - if (!$artifactLocation) { - return "" - } - try { - $parsedPackage = RetrieveReleasePackageInfo -pkgRepository $pkgRepository -artifactLocation $artifactLocation - $tag = "" - if ($parsedPackage -ne $null) - { - $tag = $parsedPackage.ReleaseTag - } - return $tag + return $parsedPackage.ReleaseTag } catch { if ($continueOnError) { diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 473876ea86b6..532159f8be29 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -94,14 +94,13 @@ function Publish-javascript-GithubIODocs ($DocLocation, $PublicArtifactLocation) if ($dirList.Length -eq 1) { $DocVersion = $dirList[0].Name - $parsedPackage = RetrieveReleasePackageInfo "NPM" $PublicArtifactLocation - $tag = "" + $pkgs, $parsePkgInfoFn = RetrievePackages "NPM" $PublicArtifactLocation # set default package name $PkgName = "azure-$($Item.BaseName)" - if ($parsedPackage -ne $null) + if ($pkgs -and $pkgs.Count -gt 0) { + $parsedPackage = Get-javascript-PackageInfoFromPackageFile $pkgs[0] $PublicArtifactLocation $PkgName = $parsedPackage.PackageId.Replace("@", "").Replace("/", "-") - $tag = $parsedPackage.ReleaseTag } Write-Host "Uploading Doc for $($PkgName) Version:- $($DocVersion)..." $releaseTag = RetrieveReleaseTag "NPM" $PublicArtifactLocation From dae004c781bade1812c090ccb74d6fc92ad2cbe1 Mon Sep 17 00:00:00 2001 From: praveenkuttappan Date: Thu, 12 Nov 2020 12:01:31 -0800 Subject: [PATCH 5/7] Revert changes in common script --- eng/common/scripts/artifact-metadata-parsing.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/common/scripts/artifact-metadata-parsing.ps1 b/eng/common/scripts/artifact-metadata-parsing.ps1 index 4aae1213ba83..b2ca199d455e 100644 --- a/eng/common/scripts/artifact-metadata-parsing.ps1 +++ b/eng/common/scripts/artifact-metadata-parsing.ps1 @@ -421,7 +421,6 @@ function RetrieveReleaseTag($pkgRepository, $artifactLocation, $continueOnError Write-Error "No release tag retrieved from $artifactLocation" } } - function RetrievePackages($pkgRepository, $artifactLocation) { $parsePkgInfoFn = "" $packagePattern = "" From 0faf6d9962ec024e680a0650d4f3dcb21c269818 Mon Sep 17 00:00:00 2001 From: praveenkuttappan Date: Mon, 16 Nov 2020 15:35:35 -0800 Subject: [PATCH 6/7] Fix as per review comments --- eng/scripts/Language-Settings.ps1 | 2 +- sdk/template/template/CHANGELOG.md | 3 +++ sdk/template/template/package.json | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 532159f8be29..5014760a6875 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -94,7 +94,7 @@ function Publish-javascript-GithubIODocs ($DocLocation, $PublicArtifactLocation) if ($dirList.Length -eq 1) { $DocVersion = $dirList[0].Name - $pkgs, $parsePkgInfoFn = RetrievePackages "NPM" $PublicArtifactLocation + $pkgs = Get-ChildItem -Path $PublicArtifactLocation -Include "*.tgz" -Recurse -File # set default package name $PkgName = "azure-$($Item.BaseName)" if ($pkgs -and $pkgs.Count -gt 0) diff --git a/sdk/template/template/CHANGELOG.md b/sdk/template/template/CHANGELOG.md index 4d84710b18c7..ccb34798ad39 100644 --- a/sdk/template/template/CHANGELOG.md +++ b/sdk/template/template/CHANGELOG.md @@ -1,4 +1,7 @@ # Release History +## 1.0.10-beta.1 (2020-11-16) +- Test Release Pipeline + ## 1.0.9-beta.13 (2020-10-13) - Test Release Pipeline diff --git a/sdk/template/template/package.json b/sdk/template/template/package.json index 4b2bb36452c6..cce5eb4603d5 100644 --- a/sdk/template/template/package.json +++ b/sdk/template/template/package.json @@ -1,6 +1,6 @@ { "name": "@azure/template", - "version": "1.0.9-beta.13", + "version": "1.0.10-beta.1", "description": "Template Library with typescript type definitions for node.js and browser.", "sdk-type": "client", "main": "dist/index.js", From c410f3db0a2adb58716eafc169b4678c9db4b204 Mon Sep 17 00:00:00 2001 From: praveenkuttappan Date: Tue, 17 Nov 2020 10:09:35 -0800 Subject: [PATCH 7/7] Added warning and check to ensure only one artifact is found in path --- eng/scripts/Language-Settings.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 5014760a6875..ade9993c6f47 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -97,11 +97,14 @@ function Publish-javascript-GithubIODocs ($DocLocation, $PublicArtifactLocation) $pkgs = Get-ChildItem -Path $PublicArtifactLocation -Include "*.tgz" -Recurse -File # set default package name $PkgName = "azure-$($Item.BaseName)" - if ($pkgs -and $pkgs.Count -gt 0) + if ($pkgs -and $pkgs.Count -eq 1) { $parsedPackage = Get-javascript-PackageInfoFromPackageFile $pkgs[0] $PublicArtifactLocation $PkgName = $parsedPackage.PackageId.Replace("@", "").Replace("/", "-") } + else{ + Write-Host "Package info is not available from artifact. Assuming package is in default scope @azure." + } Write-Host "Uploading Doc for $($PkgName) Version:- $($DocVersion)..." $releaseTag = RetrieveReleaseTag "NPM" $PublicArtifactLocation Upload-Blobs -DocDir "$($DocLocation)/documentation/$($Item.BaseName)/$($Item.BaseName)/$($DocVersion)" -PkgName $PkgName -DocVersion $DocVersion -ReleaseTag $releaseTag