Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/main' into lyshi/ga-changes
Browse files Browse the repository at this point in the history
  • Loading branch information
lynshi committed Jul 1, 2022
2 parents 5a8bdba + 6bea2f0 commit 44edea7
Show file tree
Hide file tree
Showing 2,149 changed files with 309,815 additions and 150,666 deletions.
5 changes: 4 additions & 1 deletion .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
################

# Git Hub integration and bot rules
/.github/ @jsquire
/.github/ @jsquire @ronniegeraghty

###########
# SDK
Expand Down Expand Up @@ -155,6 +155,9 @@
# PRLabel: %Cognitive - Translator
/sdk/translation/ @kristapratico @mohamedshabanofficial

# PRLabel: %Cognitive - Language
/sdk/cognitivelanguage/ @kristapratico @mohamedshabanofficial

# PRLabel: %Tables
/sdk/tables/ @annatisch @YalinLi0312

Expand Down
2 changes: 1 addition & 1 deletion common/smoketest/requirements_async.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
aiohttp==3.7.4
aiohttp==3.8.1
12 changes: 0 additions & 12 deletions eng/common/pipelines/templates/steps/cosmos-emulator.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,3 @@ steps:
-Stage "Launch"
pwsh: true
displayName: Launch Public Cosmos DB Emulator
continueOnError: true

- task: Powershell@2
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/scripts/Cosmos-Emulator.ps1
arguments: >
-EmulatorMsiUrl "${{ parameters.EmulatorMsiUrl }}"
-StartParameters "${{ parameters.StartParameters }}"
-Stage "Launch"
pwsh: true
displayName: Retry Launch of Public Cosmos DB Emulator
condition: failed()
24 changes: 18 additions & 6 deletions eng/common/pipelines/templates/steps/verify-readme.yml
Original file line number Diff line number Diff line change
@@ -1,17 +1,29 @@
parameters:
ScanPath: $(Build.SourcesDirectory)
RepoRoot: $(Build.SourcesDirectory)
SettingsPath: '$(Build.SourcesDirectory)/eng/.docsettings.yml'
DocWardenVersion : '0.7.2'
- name: ScanPath
type: string
default: ''
# Where ScanPath takes a single path, ScanPaths takes a comma separated list of paths to scan
- name: ScanPaths
type: string
default: ''
- name: RepoRoot
type: string
default: $(Build.SourcesDirectory)
- name: SettingsPath
type: string
default: '$(Build.SourcesDirectory)/eng/.docsettings.yml'
- name: DocWardenVersion
type: string
default: ''

steps:
- task: PowerShell@2
displayName: "Verify Readmes"
inputs:
filePath: "eng/common/scripts/Verify-Readme.ps1"
arguments: >
-DocWardenVersion ${{ parameters.DocWardenVersion }}
-ScanPath ${{ parameters.ScanPath }}
-DocWardenVersion '${{ parameters.DocWardenVersion }}'
-ScanPaths '${{ coalesce(parameters.ScanPath, parameters.ScanPaths) }}'
-RepoRoot ${{ parameters.RepoRoot }}
-SettingsPath ${{ parameters.SettingsPath }}
pwsh: true
11 changes: 9 additions & 2 deletions eng/common/scripts/Helpers/PSModule-Helpers.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,15 @@ function Update-PSModulePath()
}

# If we want to use another default repository other then PSGallery we can update the default parameters
function Install-ModuleIfNotInstalled($moduleName, $version, $repositoryUrl = $DefaultPSRepositoryUrl)
function Install-ModuleIfNotInstalled()
{
[CmdletBinding(SupportsShouldProcess = $true)]
param(
[string]$moduleName,
[string]$version,
[string]$repositoryUrl = $DefaultPSRepositoryUrl
)

# Check installed modules
$modules = (Get-Module -ListAvailable $moduleName)
if ($version -as [Version]) {
Expand Down Expand Up @@ -94,4 +101,4 @@ function Install-ModuleIfNotInstalled($moduleName, $version, $repositoryUrl = $D
return $modules[0]
}

Update-PSModulePath
Update-PSModulePath
101 changes: 91 additions & 10 deletions eng/common/scripts/Verify-Readme.ps1
Original file line number Diff line number Diff line change
@@ -1,24 +1,105 @@
# Wrapper Script for Readme Verification
[CmdletBinding()]
param (
[Parameter(Mandatory = $true)]
[Parameter(Mandatory = $false)]
[string]$DocWardenVersion,
[Parameter(Mandatory = $true)]
[string]$ScanPath,
[string]$RepoRoot,
[string]$ScanPaths,
[Parameter(Mandatory = $true)]
[string]$SettingsPath
)
. (Join-Path $PSScriptRoot common.ps1)
$DefaultDocWardenVersion = "0.7.2"
$script:FoundError = $false

function Test-Readme-Files {
param(
[string]$SettingsPath,
[string]$ScanPath,
[string]$RepoRoot)

Write-Host "Scanning..."

if ($RepoRoot)
{
Write-Host "ward scan -d $ScanPath -u $RepoRoot -c $SettingsPath"
ward scan -d $ScanPath -u $RepoRoot -c $SettingsPath
}
else
{
Write-Host "ward scan -d $ScanPath -c $SettingsPath"
ward scan -d $ScanPath -c $SettingsPath
}
# ward scan is what returns the non-zero exit code on failure.
# Since it's being called from a function, that error needs to
# be propagated back so the script can exit appropriately
if ($LASTEXITCODE -ne 0) {
$script:FoundError = $true
}
}

# Verify all of the inputs before running anything
if ([String]::IsNullOrWhiteSpace($DocWardenVersion)) {
$DocWardenVersion = $DefaultDocWardenVersion
}

# verify the doc settings file exists
if (!(Test-Path -Path $SettingsPath -PathType leaf)) {
LogError "Setting file, $SettingsPath, does not exist"
$script:FoundError = $true
}

$scanPathsArray = @()

# Verify that either ScanPath or ScanPaths were set but not both or neither
if ([String]::IsNullOrWhiteSpace($ScanPaths)) {
LogError "ScanPaths cannot be empty."
} else {
$scanPathsArray = $ScanPaths.Split(',')
foreach ($path in $scanPathsArray) {
if (!(Test-Path -Path $path -PathType Container)) {
LogError "path, $path, doesn't exist or isn't a directory"
$script:FoundError = $true
}
}
}

# Exit out now if there were any argument issues
if ($script:FoundError) {
LogError "There were argument failures, please see above for specifics"
exit 1
}

# Echo back the settings
Write-Host "DocWardenVersion=$DocWardenVersion"
Write-Host "SettingsPath=$SettingsPath"

if ($RepoRoot) {
Write-Host "RepoRoot=$RepoRoot"
}

Write-Host "ScanPath=$ScanPaths"

Write-Host "Installing setup tools and DocWarden"
Write-Host "pip install setuptools wheel --quiet"
pip install setuptools wheel --quiet
if ($LASTEXITCODE -ne 0) {
LogError "pip install setuptools wheel --quiet failed with exit code $LASTEXITCODE"
exit 1
}
Write-Host "pip install doc-warden==$DocWardenVersion --quiet"
pip install doc-warden==$DocWardenVersion --quiet

if ($RepoRoot)
{
ward scan -d $ScanPath -u $RepoRoot -c $SettingsPath
if ($LASTEXITCODE -ne 0) {
LogError "pip install doc-warden==$DocWardenVersion --quiet failed with exit code $LASTEXITCODE"
exit 1
}
else
{
ward scan -d $ScanPath -c $SettingsPath

# Finally, do the scanning
foreach ($path in $scanPathsArray) {
Test-Readme-Files $SettingsPath $path $RepoRoot
}

if ($script:FoundError) {
LogError "There were README verification failures, scroll up to see the issue(s)"
exit 1
}
2 changes: 1 addition & 1 deletion eng/common/scripts/common.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -54,4 +54,4 @@ $GetOnboardedDocsMsPackagesForMonikerFn = "Get-${Language}-OnboardedDocsMsPackag
$GetDocsMsTocDataFn = "Get-${Language}-DocsMsTocData"
$GetDocsMsTocChildrenForManagementPackagesFn = "Get-${Language}-DocsMsTocChildrenForManagementPackages"
$UpdateDocsMsTocFn = "Get-${Language}-UpdatedDocsMsToc"
$GetPackageLevelReadmeFn = "Get-${Language}-PackageLevelReadme"
$GetPackageLevelReadmeFn = "Get-${Language}-PackageLevelReadme"
5 changes: 4 additions & 1 deletion eng/common/scripts/stress-testing/deploy-stress-tests.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,10 @@ param(
[switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID),

# Optional namespace override, otherwise the shell user or chart annotation will be used
[string]$Namespace
[string]$Namespace,

# Override remote stress-test-addons with local on-disk addons for development
[System.IO.FileInfo]$LocalAddonsPath
)

. $PSScriptRoot/stress-test-deployment-lib.ps1
Expand Down
25 changes: 13 additions & 12 deletions eng/common/scripts/stress-testing/find-all-stress-packages.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ class StressTestPackageInfo {
[string]$ReleaseName
[string]$Dockerfile
[string]$DockerBuildDir
[string]$Deployer
}

function FindStressPackages(
Expand Down Expand Up @@ -50,6 +51,17 @@ function MatchesAnnotations([hashtable]$chart, [hashtable]$filters) {
return $true
}

function GetUsername() {
# Check GITHUB_USER for users in codespaces environments, since the default user is `codespaces` and
# we would like to avoid namespace overlaps for different codespaces users.
$stressUser = $env:GITHUB_USER ?? $env:USER ?? $env:USERNAME
# Remove spaces, underscores, etc. that may be in $namespace.
# Value must be a valid RFC 1123 DNS label: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-label-names
$stressUser = $stressUser -replace '_|\W', '-'

return $stressUser.ToLower()
}

function NewStressTestPackageInfo(
[hashtable]$chart,
[System.IO.FileInfo]$chartFile,
Expand All @@ -61,18 +73,7 @@ function NewStressTestPackageInfo(
} elseif ($CI) {
$chart.annotations.namespace
} else {
# Check GITHUB_USER for users in codespaces environments, since the default user is `codespaces` and
# we would like to avoid namespace overlaps for different codespaces users.
$namespace = if ($env:GITHUB_USER) {
$env:GITHUB_USER
} elseif ($env:USER) {
$env:USER
} else {
$env:USERNAME
}
# Remove spaces, underscores, etc. that may be in $namespace. Value must be a valid RFC 1123 DNS label:
# https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-label-names
$namespace -replace '_|\W', '-'
GetUsername
}

return [StressTestPackageInfo]@{
Expand Down
41 changes: 30 additions & 11 deletions eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -63,21 +63,29 @@ function Login([string]$subscription, [string]$clusterGroup, [switch]$pushImages
function DeployStressTests(
[string]$searchDirectory = '.',
[hashtable]$filters = @{},
[string]$environment = 'test',
# Default to playground environment
[string]$environment = 'pg',
[string]$repository = '',
[switch]$pushImages,
[string]$clusterGroup = '',
[string]$deployId = 'local',
[string]$deployId = '',
[switch]$login,
[string]$subscription = '',
[switch]$CI,
[string]$Namespace
[string]$Namespace,
[ValidateScript({
if (!(Test-Path $_)) {
throw "LocalAddonsPath $LocalAddonsPath does not exist"
}
return $true
})]
[System.IO.FileInfo]$LocalAddonsPath
) {
if ($environment -eq 'test') {
if ($environment -eq 'pg') {
if ($clusterGroup -or $subscription) {
Write-Warning "Overriding cluster group and subscription with defaults for 'test' environment."
Write-Warning "Overriding cluster group and subscription with defaults for 'pg' environment."
}
$clusterGroup = 'rg-stress-cluster-test'
$clusterGroup = 'rg-stress-cluster-pg'
$subscription = 'Azure SDK Developer Playground'
} elseif ($environment -eq 'prod') {
if ($clusterGroup -or $subscription) {
Expand All @@ -89,31 +97,42 @@ function DeployStressTests(

if ($login) {
if (!$clusterGroup -or !$subscription) {
throw "clusterGroup and subscription parameters must be specified when logging into an environment that is not test or prod."
throw "clusterGroup and subscription parameters must be specified when logging into an environment that is not pg or prod."
}
Login -subscription $subscription -clusterGroup $clusterGroup -pushImages:$pushImages
}

RunOrExitOnFailure helm repo add stress-test-charts https://stresstestcharts.blob.core.windows.net/helm/
$chartRepoName = 'stress-test-charts'
if ($LocalAddonsPath) {
$absAddonsPath = Resolve-Path $LocalAddonsPath
if (!(helm plugin list | Select-String 'file')) {
RunOrExitOnFailure helm plugin add (Join-Path $absAddonsPath file-plugin)
}
RunOrExitOnFailure helm repo add --force-update $chartRepoName file://$absAddonsPath
} else {
RunOrExitOnFailure helm repo add --force-update $chartRepoName https://stresstestcharts.blob.core.windows.net/helm/
}

Run helm repo update
if ($LASTEXITCODE) { return $LASTEXITCODE }

$deployer = if ($deployId) { $deployId } else { GetUsername }
$pkgs = FindStressPackages -directory $searchDirectory -filters $filters -CI:$CI -namespaceOverride $Namespace
Write-Host "" "Found $($pkgs.Length) stress test packages:"
Write-Host $pkgs.Directory ""
foreach ($pkg in $pkgs) {
Write-Host "Deploying stress test at '$($pkg.Directory)'"
DeployStressPackage `
-pkg $pkg `
-deployId $deployId `
-deployId $deployer `
-environment $environment `
-repositoryBase $repository `
-pushImages:$pushImages `
-login:$login
}

Write-Host "Releases deployed by $deployId"
Run helm list --all-namespaces -l deployId=$deployId
Write-Host "Releases deployed by $deployer"
Run helm list --all-namespaces -l deployId=$deployer

if ($FailedCommands) {
Write-Warning "The following commands failed:"
Expand Down
Loading

0 comments on commit 44edea7

Please sign in to comment.