From 399fde501d2e8f6431d352e1eadbafebfc76ceeb Mon Sep 17 00:00:00 2001 From: zzvswxy Date: Wed, 25 May 2022 18:18:16 +0800 Subject: [PATCH 01/30] new pipeline for codegen Signed-off-by: zzvswxy --- .../pipelines/steps/after-task.yml | 50 ------- .../pipelines/steps/environment-setup.yml | 54 -------- .../steps/generate-and-build-task.yml | 47 ------- .../pipelines/steps/generate-sdk.yml | 35 +++++ .../pipelines/steps/init-task.yml | 14 -- .../pipelines/steps/live-test-task.yml | 15 -- .../pipelines/steps/mock-test-task.yml | 17 --- .../steps/parse-and-publish-result.yml | 129 ++++++++++++++++++ .../pipelines/steps/pipeline-setup.yml | 56 ++++++++ .../pipelines/steps/precheck.yml | 53 +++++++ .../pipelines/steps/sdk-generation-tasks.yml | 94 ------------- .../sdk-generation-pipeline.yml | 116 +++++++--------- 12 files changed, 322 insertions(+), 358 deletions(-) delete mode 100644 tools/sdk-generation-pipeline/pipelines/steps/after-task.yml delete mode 100644 tools/sdk-generation-pipeline/pipelines/steps/environment-setup.yml delete mode 100644 tools/sdk-generation-pipeline/pipelines/steps/generate-and-build-task.yml create mode 100644 tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml delete mode 100644 tools/sdk-generation-pipeline/pipelines/steps/init-task.yml delete mode 100644 tools/sdk-generation-pipeline/pipelines/steps/live-test-task.yml delete mode 100644 tools/sdk-generation-pipeline/pipelines/steps/mock-test-task.yml create mode 100644 tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml create mode 100644 tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml create mode 100644 tools/sdk-generation-pipeline/pipelines/steps/precheck.yml delete mode 100644 tools/sdk-generation-pipeline/pipelines/steps/sdk-generation-tasks.yml diff --git a/tools/sdk-generation-pipeline/pipelines/steps/after-task.yml b/tools/sdk-generation-pipeline/pipelines/steps/after-task.yml deleted file mode 100644 index 670a6aed7e1..00000000000 --- a/tools/sdk-generation-pipeline/pipelines/steps/after-task.yml +++ /dev/null @@ -1,50 +0,0 @@ -parameters: - - name: sdkGenerationName - type: string - - - name: taskName - type: string - values: - - init - - generateAndBuild - - mockTest - - liveTest - - - name: displayTaskName - type: string - values: - - Init - - Generate - - Mock Test - - Live Test - -steps: - - bash: | - if [[ $PRE_TASK_RESULT == "failure" ]]; then - echo '##vso[task.setVariable variable=PreTaskResult]failure' - fi - if [[ -z "${TASK_NAME}" ]]; then - echo "Cannot Find TASK_NAME, exit" - exit 0 - fi - publishLog - retVal=$? - if [ $retVal -ne 0 ]; then - echo -e "\033[0;31mPublish Log failed" - curl --cert /tmp/sdk-generation.pem -X PATCH -H "Content-Type: application/json" https://$(CodegenApp_Server)/codegenerations/${SDK_GENERATION_NAME} -d '{"updateParameters": {"status": "completed"}}' - echo '##vso[task.setVariable variable=PreTaskResult]failure' - exit 1 - fi - - workingDirectory: $(SDK_REPO) - displayName: Publish ${{ parameters.displayTaskName }} Log - condition: and(contains(variables['TasksToRun'], '${{ parameters.taskName }}'), ne(variables['PreTaskResult'], 'failure')) - env: - TASK_NAME: ${{ parameters.taskName }} - AZURE_STORAGE_BLOB_SAS_URL: $(AZURE_STORAGE_BLOB_SAS_URL) - AZURE_BLOB_CONTAINER_NAME: logs - SDK_GENERATION_NAME: ${{ parameters.sdkGenerationName }} - BUILD_ID: $(Build.BuildId) - SDK_GENERATION_SERVICE_HOST: $(CodegenApp_Server) - PIPE_FULL_LOG: "/tmp/pipe.full.log" - PRE_TASK_RESULT: $(StepResult) diff --git a/tools/sdk-generation-pipeline/pipelines/steps/environment-setup.yml b/tools/sdk-generation-pipeline/pipelines/steps/environment-setup.yml deleted file mode 100644 index a21e5e012e4..00000000000 --- a/tools/sdk-generation-pipeline/pipelines/steps/environment-setup.yml +++ /dev/null @@ -1,54 +0,0 @@ -parameters: - - name: sdk - type: string - - - name: specRepoUrl - type: string - - - name: specRepoBaseBranch - type: string - - - name: sdkRepoUrl - type: string - - - name: sdkRepoBaseBranch - type: string - -steps: - - task: NodeTool@0 - displayName: Specify Node Version - inputs: - versionSpec: 14.x - - - task: AzureKeyVault@2 - displayName: Get Certificate from KeyVault - inputs: - azureSubscription: 'Azure Key Vault' - KeyVaultName: $(KeyVault) - SecretsFilter: 'sdk-generation' - RunAsPreJob: true - - - bash: | - echo "$(sdk-generation)" > sdk-generation.pem - openssl rsa -in sdk-generation.pem -out private.key - displayName: Store Cert and Get Private Key - workingDirectory: /tmp - - - bash: | - npm install -g @microsoft/rush - rush update - rush build - cd tools/sdk-generation-cli - rushx pack - PACKAGE_TGZ=`ls sdk-generation-cli-*.tgz` - npm install -g ${PACKAGE_TGZ} - npm install -g autorest - displayName: Setup Pipeline Runtime Environment - - - bash: | - git clone --branch ${{ parameters.specRepoBaseBranch }} ${{ parameters.specRepoUrl }} - git clone --branch ${{ parameters.sdkRepoBaseBranch }} ${{ parameters.sdkRepoUrl }} - getRepoName SPEC_REPO ${{ parameters.specRepoUrl }} - getRepoName SDK_REPO ${{ parameters.sdkRepoUrl }} - displayName: Clone Spec Repo and Sdk Repo - workingDirectory: ../ diff --git a/tools/sdk-generation-pipeline/pipelines/steps/generate-and-build-task.yml b/tools/sdk-generation-pipeline/pipelines/steps/generate-and-build-task.yml deleted file mode 100644 index 5f20b2eb1d6..00000000000 --- a/tools/sdk-generation-pipeline/pipelines/steps/generate-and-build-task.yml +++ /dev/null @@ -1,47 +0,0 @@ -parameters: - - name: sdkGenerationName - type: string - - - name: sdk - type: string - - - name: serviceType - type: string - - - name: resourceProvider - type: string - - - name: readmeFile - type: string - - - name: sdkRepoUrl - type: string - - - name: sdkRepoBaseBranch - type: string - -steps: - - bash: | - headSha=`git rev-parse HEAD` - export HEAD_SHA=${headSha} - runGenerateAndBuildTask - retVal=$? - if [ $retVal -ne 0 ]; then - echo -e "\033[0;31mGenerateAndBuild Task is failed" - exit 1 - fi - workingDirectory: $(SDK_REPO) - condition: and(contains(variables['TasksToRun'], 'generateAndBuild'), ne(variables['PreTaskResult'], 'failure')) - displayName: Generate - env: - TASKS_TO_RUN: $(TasksToRun) - SDK_REPO: $(SDK_REPO) - SPEC_FOLDER: $(SPEC_REPO) - RELATED_README_MD_FILE: ${{ parameters.readmeFile }} - LANGUAGE: ${{ parameters.sdk }} - RESOURCE_PROVIDER: ${{ parameters.resourceProvider }} - AZURE_STORAGE_BLOB_SAS_URL: $(AZURE_STORAGE_BLOB_SAS_URL) - SDK_GENERATION_NAME: ${{ parameters.sdkGenerationName }} - SERVICE_TYPE: ${{ parameters.serviceType }} - HEAD_REF: ${{ parameters.sdkRepoBaseBranch }} - REPO_HTTP_URL: ${{ parameters.sdkRepoUrl }} diff --git a/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml b/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml new file mode 100644 index 00000000000..a70232564dc --- /dev/null +++ b/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml @@ -0,0 +1,35 @@ +parameters: + - name: readmeFile + type: string + + - name: trigger + type: string + +steps: + - bash: | + publishResult \ + --storageType=eventhub \ + --pipelineStatus=in_progress \ + --buildId="$(Build.BuildId)" \ + --trigger='${{ parameters.trigger }}' + retVal=$? + if [ $retVal -ne 0 ]; then + echo "##vso[task.setvariable variable=pipelineResult;]failed" + exit 1 + fi + displayName: Publish status + env: + EVENTHUB_SAS_URL: $(EVENTHUB_SAS_URL) + + - bash: | + docker run \ + -v $(SPEC_REPO):/spec-repo \ + -v $(SDK_REPO):/sdk-repo \ + -v /tmp/output:/tmp/output \ + $(DOCKER_IMAGE) \ + --readme=${{ parameters.readmeFile }} + retVal=$? + if [ $retVal -ne 0 ]; then + echo "##vso[task.setvariable variable=pipelineResult;]failed" + fi + displayName: Generate sdk diff --git a/tools/sdk-generation-pipeline/pipelines/steps/init-task.yml b/tools/sdk-generation-pipeline/pipelines/steps/init-task.yml deleted file mode 100644 index 0269c38e4c6..00000000000 --- a/tools/sdk-generation-pipeline/pipelines/steps/init-task.yml +++ /dev/null @@ -1,14 +0,0 @@ -steps: - - bash: | - runInitTask - retVal=$? - if [ $retVal -ne 0 ]; then - echo -e "\033[0;31mInit Task is failed" - exit 1 - fi - displayName: Init - condition: contains(variables['TasksToRun'], 'init') - workingDirectory: $(SDK_REPO) - env: - SDK_REPO: $(SDK_REPO) - TASKS_TO_RUN: $(TasksToRun) diff --git a/tools/sdk-generation-pipeline/pipelines/steps/live-test-task.yml b/tools/sdk-generation-pipeline/pipelines/steps/live-test-task.yml deleted file mode 100644 index bbf93bdd37f..00000000000 --- a/tools/sdk-generation-pipeline/pipelines/steps/live-test-task.yml +++ /dev/null @@ -1,15 +0,0 @@ -steps: - - bash: | - runLiveTestTask - retVal=$? - if [ $retVal -ne 0 ]; then - echo -e "\033[0;31mLiveTest Task is failed" - exit 1 - fi - displayName: Live Test - condition: and(contains(variables['TasksToRun'], 'liveTest'), ne(variables['PreTaskResult'], 'failure')) - workingDirectory: $(SDK_REPO) - env: - SDK_REPO: $(SDK_REPO) - PACKAGE_FOLDERS: $(PackageFolders) - TASKS_TO_RUN: $(TasksToRun) diff --git a/tools/sdk-generation-pipeline/pipelines/steps/mock-test-task.yml b/tools/sdk-generation-pipeline/pipelines/steps/mock-test-task.yml deleted file mode 100644 index 1c7e19a1eb3..00000000000 --- a/tools/sdk-generation-pipeline/pipelines/steps/mock-test-task.yml +++ /dev/null @@ -1,17 +0,0 @@ -steps: - - bash: | - # start mock server - # TODO - runMockTestTask - retVal=$? - if [ $retVal -ne 0 ]; then - echo -e "\033[0;31mMockTest Task is failed" - exit 1 - fi - displayName: Mock Test - condition: and(contains(variables['TasksToRun'], 'mockTest'), ne(variables['PreTaskResult'], 'failure')) - workingDirectory: $(SDK_REPO) - env: - SDK_REPO: $(SDK_REPO) - PACKAGE_FOLDERS: $(PackageFolders) - TASKS_TO_RUN: $(TasksToRun) diff --git a/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml b/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml new file mode 100644 index 00000000000..4c23acb7e6d --- /dev/null +++ b/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml @@ -0,0 +1,129 @@ +parameters: + - name: sdk + type: string + + - name: service + type: string + + - name: serviceType + type: string + + - name: trigger + type: string + + - name: sdkGenerationName + type: string +steps: + - bash: | + echo "will parse task result" + LOG_Dir="/tmp/output/" + for TASK_NAME in "init" "generateAndBuild" "mockTest" + do + if [ ${TASK_NAME} = "mockTest" ]; then + TASK_OUTPUT_PATH="/tmp/output/mockTestOutput.json" + fi + + generateResult \ + --buildId=$(Build.BuildId) \ + --taskName=${TASK_NAME} \ + --logfile="/tmp/output/${TASK_NAME}-task.log" \ + --resultOutputPath="/tmp/output/${TASK_NAME}-task-result.json" \ + --dockerResultFile="/tmp/output/taskResults.json" \ + --taskOutputPath="${TASK_OUTPUT_PATH}" + retVal=$? + if [ $retVal -ne 0 ]; then + #TODO: when all language support mock test, set pipelineResult for parse task result + echo "wait all language support mock test" + #echo "##vso[task.setvariable variable=pipelineResult;]failed" + fi + done + displayName: Parse task result + condition: or(contains(variables['pipelineResult'], 'success'), contains(variables['pipelineResult'], 'failed')) + + - bash: | + prepareArtifactFiles \ + --artifactDir="$(System.ArtifactsDirectory)" \ + --generateAndBuildOutputFile="/tmp/output/generateAndBuildOutputJson.json" \ + --language="${{ parameters.sdk }}" + retVal=$? + if [ $retVal -ne 0 ]; then + echo "##vso[task.setvariable variable=pipelineResult;]failed" + exit 1 + fi + cp /tmp/output/docker.log $(System.ArtifactsDirectory)/logs/${{ parameters.sdkGenerationName }}-full.log + cp /tmp/output/init-task.log $(System.ArtifactsDirectory)/logs/${{ parameters.sdkGenerationName }}-init.log + cp /tmp/output/init-task-result.json $(System.ArtifactsDirectory)/logs/${{ parameters.sdkGenerationName }}-init-result.json + cp /tmp/output/generateAndBuild-task.log $(System.ArtifactsDirectory)/logs/${{ parameters.sdkGenerationName }}-generateAndBuild.log + cp /tmp/output/generateAndBuild-task-result.json $(System.ArtifactsDirectory)/logs/${{ parameters.sdkGenerationName }}-generateAndBuild-result.json + cp /tmp/output/mockTest-task.log $(System.ArtifactsDirectory)/logs/${{ parameters.sdkGenerationName }}-mockTest.log | echo "wait all language support mock test" + cp /tmp/output/mockTest-task-result.json $(System.ArtifactsDirectory)/logs/${{ parameters.sdkGenerationName }}-mockTest-result.json | echo "wait all language support mock test" + displayName: Prepare artifact + workingDirectory: $(SDK_REPO) + condition: contains(variables['pipelineResult'], 'success') + + - task: PublishPipelineArtifact@1 + displayName: Publish SDK Generation pipeline artifact + inputs: + targetPath: $(System.ArtifactsDirectory)/ + artifactName: sdk-generation + condition: contains(variables['pipelineResult'], 'success') + + - bash: | + publishResult \ + --storageType=eventhub \ + --pipelineStatus=completed \ + --buildId="$(Build.BuildId)" \ + --trigger='${{ parameters.trigger }}' \ + --logPath=$(Build.BuildId)/logs \ + --resultsPath='["/tmp/output/init-task-result.json", + "/tmp/output/generateAndBuild-task-result.json", + "/tmp/output/mockTest-task-result.json"]' + retVal=$? + if [ $retVal -ne 0 ]; then + echo "##vso[task.setvariable variable=pipelineResult;]failed" + exit 1 + fi + displayName: Publish status + env: + EVENTHUB_SAS_URL: $(EVENTHUB_SAS_URL) + condition: always() + + - bash: | + echo "will create PR" + git config --global user.email "no-reply@azure-sdk.com" + git config --global user.name "$(REPO_OWNER)" + + packages=`cat /tmp/output/generateAndBuildOutputJson.json | jq -c '.packages | .[]'` + for package in ${packages} + do + packageName=`echo ${package} | jq -r '.packageName'` + paths=`echo ${package} | jq -r '.path |.[]'` + for path in ${paths} + do + git add ${path} + done + git commit -m '[auto]create by sdk generation' + if [ "${{ parameters.serviceType }}" = "resource-manager" ];then + branchName=sdkGen/${{ parameters.sdkGenerationName }}/arm-${{ parameters.service }}-${packageName} + else + branchName=sdkGen/${{ parameters.sdkGenerationName }}/dpg-${{ parameters.service }}-${packageName} + fi + git checkout -b ${branchName} + pushSdkRepoUrl=$(echo $(sdkRepoUrl) | sed "s#https://#https://$(REPO_OWNER):$(REPO_OWNER_ACCESSKEY)@#") + pushSdkRepoUrl=$(echo ${pushSdkRepoUrl} | sed "s#Azure#$(REPO_OWNER)#") + git push ${pushSdkRepoUrl} + sdkRepoPullsApi=$(echo $(sdkRepoUrl) | sed "s#https://#https://api.#") + sdkRepoPullsApi=$(echo ${sdkRepoPullsApi} | sed "s#Azure#repos/$(REPO_OWNER)#") + sdkRepoPullsApi=$(echo ${sdkRepoPullsApi} | sed "s#.git\$#/pulls#") + curl \ + -u "$(REPO_OWNER)":"$(REPO_OWNER_ACCESSKEY)" \ + -H "Accept: application/vnd.github.v3+json" \ + -X POST \ + -d '{"title":"[AutoPR-${{ parameters.service }}]autogenerated","body":"","base":"main", "head":"'$(REPO_OWNER)':'${branchName}'"}' \ + ${sdkRepoPullsApi} + + git reset HEAD~ + done + workingDirectory: $(SDK_REPO) + displayName: Submit Pull Request + condition: contains(variables['pipelineResult'], 'success') \ No newline at end of file diff --git a/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml b/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml new file mode 100644 index 00000000000..1b1b28817f8 --- /dev/null +++ b/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml @@ -0,0 +1,56 @@ +parameters: + - name: sdk + type: string + + - name: sdkRepoBaseBranch + type: string + + - name: trigger + type: string + +steps: + - task: NodeTool@0 + displayName: Specify Node Version + inputs: + versionSpec: 14.x + + - bash: | + pwd + cd tools/sdk-generation-pipeline + npm install -g @microsoft/rush + rush update + rush build + cd packages/sdk-generation-cli + rushx pack + PACKAGE_TGZ=`ls azure-tools-sdk-generation-cli*.tgz` + npm install -g ${PACKAGE_TGZ} + npm install -g autorest + cd $(System.ArtifactsDirectory) + echo "mkdir for docker output" + mkdir -p /tmp/output + echo "mkdir for pipeline artifact" + mkdir -p logs + echo "##vso[task.setvariable variable=pipelineResult;]success" + publishResult \ + --storageType=eventhub \ + --pipelineStatus=queued \ + --buildId="$(Build.BuildId)" \ + --trigger='${{ parameters.trigger }}' + retVal=$? + if [ $retVal -ne 0 ]; then + echo "##vso[task.setvariable variable=pipelineResult;]failed" + exit 1 + fi + displayName: Setup Pipeline Runtime Environment + env: + EVENTHUB_SAS_URL: $(EVENTHUB_SAS_URL) + + - bash: | + #TODO: will support clone specific PR and commit + git clone --branch $(specRepoBranch) $(specRepoUrl) + downloadSdkRepoUrl=$(echo $(sdkRepoUrl) | sed "s#https://#https://$(REPO_OWNER):$(REPO_OWNER_ACCESSKEY)@#") + git clone --branch $(sdkRepoBaseBranch) ${downloadSdkRepoUrl} + getRepoName SPEC_REPO $(specRepoUrl) + getRepoName SDK_REPO $(sdkRepoUrl) + displayName: Repo pull + workingDirectory: ../ diff --git a/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml b/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml new file mode 100644 index 00000000000..c484fe45ea4 --- /dev/null +++ b/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml @@ -0,0 +1,53 @@ +parameters: + - name: swagger + type: object + + - name: sdk + type: string + values: + - js + - java + - go + - net + - python + + - name: service + type: string +steps: + - bash: | + if [ -z ${{ parameters.swagger.branch }} ];then + echo "swagger.branch is empty" + exit 1 + elif [ -z ${{ parameters.swagger.path }} ];then + echo "swagger.path is empty" + exit 1 + else + echo "##vso[task.setvariable variable=specRepoUrl;]${{ parameters.swagger.path }}" + echo "##vso[task.setvariable variable=specRepoBranch;]${{ parameters.swagger.branch }}" + fi + displayName: Check parameters + + - ${{ if eq(parameters.sdk, 'js') }}: + - bash: | + echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/Azure/azure-sdk-for-js.git" + displayName: Set js sdk repo + - ${{ elseif eq(parameters.sdk, 'java') }}: + - bash: | + echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/Azure/azure-sdk-for-java.git" + displayName: Set java sdk repo + - ${{ elseif eq(parameters.sdk, 'go') }}: + - bash: | + echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/Azure/azure-sdk-for-go.git" + displayName: Set go sdk repo + - ${{ elseif eq(parameters.sdk, 'net') }}: + - bash: | + echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/Azure/azure-sdk-for-net.git" + displayName: Set net sdk repo + - ${{ elseif eq(parameters.sdk, 'python') }}: + - bash: | + echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/Azure/azure-sdk-for-python.git" + displayName: Set python sdk repo + - ${{ else }}: + - bash: | + exit 1 + displayName: Set sdk repo diff --git a/tools/sdk-generation-pipeline/pipelines/steps/sdk-generation-tasks.yml b/tools/sdk-generation-pipeline/pipelines/steps/sdk-generation-tasks.yml deleted file mode 100644 index 4e995cc5771..00000000000 --- a/tools/sdk-generation-pipeline/pipelines/steps/sdk-generation-tasks.yml +++ /dev/null @@ -1,94 +0,0 @@ -parameters: - - name: sdkGenerationName - type: string - - - name: sdk - type: string - - - name: serviceType - type: string - - - name: resourceProvider - type: string - - - name: readmeFile - type: string - - - name: triggerType - type: string - - - name: specRepoType - type: string - - - name: specRepoUrl - type: string - - - name: specRepoBaseBranch - type: string - - - name: sdkRepoType - type: string - - - name: sdkRepoUrl - type: string - - - name: sdkRepoBaseBranch - type: string - - - name: skippedTask - type: string - -steps: - - bash: | - getTasksToRun - retVal=$? - if [ $retVal -ne 0 ]; then - echo -e "\033[0;31mGet tasks to run failed" - exit 1 - fi - displayName: Retrieve Tasks - name: retrieveTasks - workingDirectory: $(SDK_REPO) - env: - SDK_REPO: $(SDK_REPO) - SKIPPED_STEPS: ${{ parameters.skippedTask }} - - - template: init-task.yml - - - template: after-task.yml - parameters: - sdkGenerationName: ${{ parameters.sdkGenerationName }} - taskName: init - displayTaskName: Init - - - template: generate-and-build-task.yml - parameters: - readmeFile: ${{ parameters.readmeFile }} - sdk: ${{ parameters.sdk }} - resourceProvider: ${{ parameters.resourceProvider }} - sdkGenerationName: ${{ parameters.sdkGenerationName }} - serviceType: ${{ parameters.serviceType }} - sdkRepoBaseBranch: ${{ parameters.sdkRepoBaseBranch }} - sdkRepoUrl: ${{ parameters.sdkRepoUrl }} - - - template: after-task.yml - parameters: - sdkGenerationName: ${{ parameters.sdkGenerationName }} - taskName: generateAndBuild - displayTaskName: Generate - - - template: mock-test-task.yml - - - template: after-task.yml - parameters: - sdkGenerationName: ${{ parameters.sdkGenerationName }} - taskName: mockTest - displayTaskName: Mock Test - - - template: live-test-task.yml - - - template: after-task.yml - parameters: - sdkGenerationName: ${{ parameters.sdkGenerationName }} - taskName: liveTest - displayTaskName: Live Test diff --git a/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml b/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml index 4201e81eeab..3f153c10a0f 100644 --- a/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml +++ b/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml @@ -1,10 +1,19 @@ trigger: none -parameters: - - name: sdkGenerationName - type: string +variables: + - template: ../../eng/pipelines/templates/variables/globals.yml + + - name: codegenRepo + value: https://github.com/Azure/azure-sdk-tools.git + + - name: sdkRepoBaseBranch + value: main + + - name: sdkRepoType + value: github - - name: sdk +parameters: + - name: sdkToGernerate type: string values: - js @@ -13,19 +22,15 @@ parameters: - net - python - - name: resourceProvider + - name: service type: string - - name: readmeFile + - name: relativeReadmeFilePath type: string - - name: triggerType + - name: tag type: string - default: ad-hoc - values: - - ad-hoc - - ci - - release + default: "''" - name: serviceType type: string @@ -33,72 +38,49 @@ parameters: - resource-manager - data-plane - - name: specRepoType - type: string - default: github - values: - - github - - devops - - - name: specRepoUrl - type: string - - - name: specRepoBaseBranch + - name: swagger + type: object + default: + "type": "github" + "path": "https://github.com/azure/azure-rest-api-specs" + "baseBranch": "main" + "branch": "main" + "commit": "" + "pullRequest": "" + + - name: clientAgent type: string - - name: sdkRepoType - type: string - default: github - values: - - github - - devops - - - name: sdkRepoUrl - type: string - - - name: sdkRepoBaseBranch - type: string - - - name: skippedTask + - name: sdkGenerationName type: string stages: - stage: SdkGeneration jobs: - job: SdkGeneration - displayName: ${{ parameters.sdkGenerationName }}-${{ parameters.sdk }}-${{ parameters.serviceType }}-${{ parameters.resourceProvider }} + displayName: ${{ parameters.sdkToGernerate }}-${{ parameters.serviceType }}-${{ parameters.service }} steps: - - bash: | - curl -X PATCH -H "Content-Type: application/json" https://$(CodegenApp_Server)/codegenerations/${SDK_GENERATION_NAME} -d '{"updateParameters": {"status": "in_progress","lastPipelineBuildID": "$(Build.BuildId)"}}' - displayName: Notify Sdk Generation Server to Start - env: - SDK_GENERATION_NAME: ${{ parameters.sdkGenerationName }} + - template: ./pipelines/steps/precheck.yml + parameters: + swagger: ${{ parameters.swagger }} + sdk: ${{ parameters.sdkToGernerate }} + service: ${{ parameters.service }} - - template: ./pipelines/steps/environment-setup.yml + - template: ./pipelines/steps/pipeline-setup.yml parameters: - sdk: ${{ parameters.sdk }} - specRepoUrl: ${{ parameters.specRepoUrl }} - specRepoBaseBranch: ${{ parameters.specRepoBaseBranch }} - sdkRepoUrl: ${{ parameters.sdkRepoUrl }} - sdkRepoBaseBranch: ${{ parameters.sdkRepoBaseBranch }} + sdk: ${{ parameters.sdkToGernerate }} + sdkRepoBaseBranch: ${{ variables.sdkRepoBaseBranch }} + trigger: ${{ parameters.clientAgent }} - - template: ./pipelines/steps/sdk-generation-tasks.yml + - template: ./pipelines/steps/generate-sdk.yml parameters: - sdkGenerationName: ${{ parameters.sdkGenerationName }} - sdk: ${{ parameters.sdk }} - resourceProvider: ${{ parameters.resourceProvider }} - readmeFile: ${{ parameters.readmeFile }} - triggerType: ${{ parameters.triggerType }} - specRepoType: ${{ parameters.specRepoType }} - specRepoUrl: ${{ parameters.specRepoUrl }} - specRepoBaseBranch: ${{ parameters.specRepoBaseBranch }} - sdkRepoType: ${{ parameters.sdkRepoType }} - sdkRepoUrl: ${{ parameters.sdkRepoUrl }} - sdkRepoBaseBranch: ${{ parameters.sdkRepoBaseBranch }} - skippedTask: ${{ parameters.skippedTask }} + readmeFile: ${{ parameters.relativeReadmeFilePath }} + trigger: ${{ parameters.clientAgent }} + + - template: ./pipelines/steps/parse-and-publish-result.yml + parameters: + sdk: ${{ parameters.sdkToGernerate }} + service: ${{ parameters.service }} serviceType: ${{ parameters.serviceType }} - - bash: | - curl -X PATCH -H "Content-Type: application/json" https://$(CodegenApp_Server)/codegenerations/${SDK_GENERATION_NAME} -d '{"updateParameters": {"status": "completed"}}' - displayName: Set Complete - env: - SDK_GENERATION_NAME: ${{ parameters.sdkGenerationName }} + trigger: ${{ parameters.clientAgent }} + sdkGenerationName: ${{ parameters.sdkGenerationName }} From 9135cc59cbf247ed8ccfbeb267016a2897d68d2f Mon Sep 17 00:00:00 2001 From: zzvswxy Date: Mon, 30 May 2022 14:33:05 +0800 Subject: [PATCH 02/30] bugfix: taskresult is required, check exist before prepare sourcecode Signed-off-by: zzvswxy --- .../sdk-generation-cli/src/prepareArtifactFilesCli.ts | 4 ++++ .../packages/sdk-generation-lib/src/lib/generateResult.ts | 6 ++++-- .../packages/sdk-generation-lib/src/types/taskResult.ts | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/prepareArtifactFilesCli.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/prepareArtifactFilesCli.ts index 97e40d3f26e..22e40a77586 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/prepareArtifactFilesCli.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/prepareArtifactFilesCli.ts @@ -14,6 +14,10 @@ import { import { getFileListInPackageFolder } from './utils/git'; function copyFile(filePath: string, targetDir: string) { + if (!fs.existsSync(filePath)) { + logger.info(`${filePath} isn't exist, skipped it.`); + return; + } const fileDir = path.dirname(filePath); fs.mkdirSync(`${targetDir}/${fileDir}`, { recursive: true }); fs.copyFileSync(`${filePath}`, `${targetDir}/${filePath}`); diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/lib/generateResult.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/lib/generateResult.ts index c0714da920d..2a559c53ab6 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/lib/generateResult.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/lib/generateResult.ts @@ -41,7 +41,8 @@ export function parseGenerateLog( pipelineBuildId: string, taskName: string, logfile: string, - logFilter: LogFilter + logFilter: LogFilter, + taskExeResult: TaskResultStatus ): TaskResultCommon { let errorNum = 0; let warnNum = 0; @@ -86,6 +87,7 @@ export function parseGenerateLog( errorCount: errorNum, warningCount: warnNum, messages: messages, + result: taskExeResult, }; return result; @@ -109,7 +111,7 @@ export function createTaskResult( warningCount: 0, }; } else { - commonResult = parseGenerateLog(pipelineBuildId, taskname, logfile, logFilter); + commonResult = parseGenerateLog(pipelineBuildId, taskname, logfile, logFilter, taskExeResult); } if (taskname === AzureSDKTaskName.MockTest || taskname === AzureSDKTaskName.LiveTest) { if (taskOutput === undefined) { diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/types/taskResult.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/types/taskResult.ts index ab8ef7b2de5..ff03e2db41e 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/types/taskResult.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/types/taskResult.ts @@ -64,7 +64,7 @@ export type MessageRecord = ResultMessageRecord | RawMessageRecord | MarkdownMes export type TaskResultCommon = { name: string; pipelineBuildId: string; - result?: TaskResultStatus; + result: TaskResultStatus; errorCount?: number; warningCount?: number; logUrl?: string; From dd14435aa5e2e76d66668949ee59041d5a220569 Mon Sep 17 00:00:00 2001 From: zzvswxy Date: Mon, 6 Jun 2022 14:57:03 +0800 Subject: [PATCH 03/30] Simplify Judgment Conditions Signed-off-by: zzvswxy --- .../pipelines/steps/generate-sdk.yml | 3 +-- .../steps/parse-and-publish-result.yml | 15 +++++------ .../pipelines/steps/pipeline-setup.yml | 3 +-- .../pipelines/steps/precheck.yml | 27 +++---------------- 4 files changed, 12 insertions(+), 36 deletions(-) diff --git a/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml b/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml index a70232564dc..7feba363524 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml @@ -14,7 +14,6 @@ steps: --trigger='${{ parameters.trigger }}' retVal=$? if [ $retVal -ne 0 ]; then - echo "##vso[task.setvariable variable=pipelineResult;]failed" exit 1 fi displayName: Publish status @@ -30,6 +29,6 @@ steps: --readme=${{ parameters.readmeFile }} retVal=$? if [ $retVal -ne 0 ]; then - echo "##vso[task.setvariable variable=pipelineResult;]failed" + exit 1 fi displayName: Generate sdk diff --git a/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml b/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml index 4c23acb7e6d..6f2c38c63f8 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml @@ -21,6 +21,8 @@ steps: do if [ ${TASK_NAME} = "mockTest" ]; then TASK_OUTPUT_PATH="/tmp/output/mockTestOutput.json" + else + TASK_OUTPUT_PATH="" fi generateResult \ @@ -32,13 +34,13 @@ steps: --taskOutputPath="${TASK_OUTPUT_PATH}" retVal=$? if [ $retVal -ne 0 ]; then - #TODO: when all language support mock test, set pipelineResult for parse task result + #TODO: when all language support mock test, set exit 1 for parse task result failed echo "wait all language support mock test" - #echo "##vso[task.setvariable variable=pipelineResult;]failed" + #exit 1 fi done displayName: Parse task result - condition: or(contains(variables['pipelineResult'], 'success'), contains(variables['pipelineResult'], 'failed')) + condition: always() - bash: | prepareArtifactFiles \ @@ -59,14 +61,12 @@ steps: cp /tmp/output/mockTest-task-result.json $(System.ArtifactsDirectory)/logs/${{ parameters.sdkGenerationName }}-mockTest-result.json | echo "wait all language support mock test" displayName: Prepare artifact workingDirectory: $(SDK_REPO) - condition: contains(variables['pipelineResult'], 'success') - task: PublishPipelineArtifact@1 displayName: Publish SDK Generation pipeline artifact inputs: targetPath: $(System.ArtifactsDirectory)/ artifactName: sdk-generation - condition: contains(variables['pipelineResult'], 'success') - bash: | publishResult \ @@ -74,7 +74,7 @@ steps: --pipelineStatus=completed \ --buildId="$(Build.BuildId)" \ --trigger='${{ parameters.trigger }}' \ - --logPath=$(Build.BuildId)/logs \ + --logPath="www.demoLogUrl.com" \ --resultsPath='["/tmp/output/init-task-result.json", "/tmp/output/generateAndBuild-task-result.json", "/tmp/output/mockTest-task-result.json"]' @@ -125,5 +125,4 @@ steps: git reset HEAD~ done workingDirectory: $(SDK_REPO) - displayName: Submit Pull Request - condition: contains(variables['pipelineResult'], 'success') \ No newline at end of file + displayName: Submit Pull Request \ No newline at end of file diff --git a/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml b/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml index 1b1b28817f8..09df6a1ff9e 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml @@ -30,7 +30,7 @@ steps: mkdir -p /tmp/output echo "mkdir for pipeline artifact" mkdir -p logs - echo "##vso[task.setvariable variable=pipelineResult;]success" + publishResult \ --storageType=eventhub \ --pipelineStatus=queued \ @@ -38,7 +38,6 @@ steps: --trigger='${{ parameters.trigger }}' retVal=$? if [ $retVal -ne 0 ]; then - echo "##vso[task.setvariable variable=pipelineResult;]failed" exit 1 fi displayName: Setup Pipeline Runtime Environment diff --git a/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml b/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml index c484fe45ea4..bc3ec492189 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml @@ -27,27 +27,6 @@ steps: fi displayName: Check parameters - - ${{ if eq(parameters.sdk, 'js') }}: - - bash: | - echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/Azure/azure-sdk-for-js.git" - displayName: Set js sdk repo - - ${{ elseif eq(parameters.sdk, 'java') }}: - - bash: | - echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/Azure/azure-sdk-for-java.git" - displayName: Set java sdk repo - - ${{ elseif eq(parameters.sdk, 'go') }}: - - bash: | - echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/Azure/azure-sdk-for-go.git" - displayName: Set go sdk repo - - ${{ elseif eq(parameters.sdk, 'net') }}: - - bash: | - echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/Azure/azure-sdk-for-net.git" - displayName: Set net sdk repo - - ${{ elseif eq(parameters.sdk, 'python') }}: - - bash: | - echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/Azure/azure-sdk-for-python.git" - displayName: Set python sdk repo - - ${{ else }}: - - bash: | - exit 1 - displayName: Set sdk repo + - bash: | + echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/Azure/azure-sdk-for-${{parameters.sdk}}.git" + displayName: Set sdk repo From 701ede471e70b9ff65fe3ea4580b1deadcf95576 Mon Sep 17 00:00:00 2001 From: zzvswxy Date: Mon, 6 Jun 2022 15:03:36 +0800 Subject: [PATCH 04/30] delete unnecessary variable Signed-off-by: zzvswxy --- .../pipelines/steps/generate-sdk.yml | 6 ++---- .../pipelines/steps/parse-and-publish-result.yml | 9 +++------ .../pipelines/steps/pipeline-setup.yml | 3 +-- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml b/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml index 7feba363524..1f5ca0117c3 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml @@ -12,8 +12,7 @@ steps: --pipelineStatus=in_progress \ --buildId="$(Build.BuildId)" \ --trigger='${{ parameters.trigger }}' - retVal=$? - if [ $retVal -ne 0 ]; then + if [ $? -ne 0 ]; then exit 1 fi displayName: Publish status @@ -27,8 +26,7 @@ steps: -v /tmp/output:/tmp/output \ $(DOCKER_IMAGE) \ --readme=${{ parameters.readmeFile }} - retVal=$? - if [ $retVal -ne 0 ]; then + if [ $? -ne 0 ]; then exit 1 fi displayName: Generate sdk diff --git a/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml b/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml index 6f2c38c63f8..888e2fa6da9 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml @@ -32,8 +32,7 @@ steps: --resultOutputPath="/tmp/output/${TASK_NAME}-task-result.json" \ --dockerResultFile="/tmp/output/taskResults.json" \ --taskOutputPath="${TASK_OUTPUT_PATH}" - retVal=$? - if [ $retVal -ne 0 ]; then + if [ $? -ne 0 ]; then #TODO: when all language support mock test, set exit 1 for parse task result failed echo "wait all language support mock test" #exit 1 @@ -47,8 +46,7 @@ steps: --artifactDir="$(System.ArtifactsDirectory)" \ --generateAndBuildOutputFile="/tmp/output/generateAndBuildOutputJson.json" \ --language="${{ parameters.sdk }}" - retVal=$? - if [ $retVal -ne 0 ]; then + if [ $? -ne 0 ]; then echo "##vso[task.setvariable variable=pipelineResult;]failed" exit 1 fi @@ -78,8 +76,7 @@ steps: --resultsPath='["/tmp/output/init-task-result.json", "/tmp/output/generateAndBuild-task-result.json", "/tmp/output/mockTest-task-result.json"]' - retVal=$? - if [ $retVal -ne 0 ]; then + if [ $? -ne 0 ]; then echo "##vso[task.setvariable variable=pipelineResult;]failed" exit 1 fi diff --git a/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml b/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml index 09df6a1ff9e..5ce62ed9c7b 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml @@ -36,8 +36,7 @@ steps: --pipelineStatus=queued \ --buildId="$(Build.BuildId)" \ --trigger='${{ parameters.trigger }}' - retVal=$? - if [ $retVal -ne 0 ]; then + if [ $? -ne 0 ]; then exit 1 fi displayName: Setup Pipeline Runtime Environment From bca1a02b8fe4601a352ea8c78f98fa9c9a0b480c Mon Sep 17 00:00:00 2001 From: zzvswxy Date: Mon, 6 Jun 2022 16:56:21 +0800 Subject: [PATCH 05/30] separate artifacts from total one to three part Signed-off-by: zzvswxy --- .../cli/pipelineCli/prepareArtifactFilesCli.ts | 13 ++++++++----- .../pipelines/steps/parse-and-publish-result.yml | 16 ++++++++++++++-- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts index 79be814b628..7011b9e5f9c 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts @@ -5,7 +5,7 @@ import { logger, requireJsonc, SDK, - TaskResultStatus + TaskResultStatus, } from '@azure-tools/sdk-generation-lib'; import * as fs from 'fs'; import * as path from 'path'; @@ -45,10 +45,13 @@ async function prepareSourceCode( if (fs.lstatSync(packagePath).isDirectory()) { for (const filePath of getFileListInPackageFolder(packagePath)) { - copyFile(`${path.join(packagePath, filePath)}`, `${artifactDir}/${language}/${packageName}`); + copyFile( + `${path.join(packagePath, filePath)}`, + `${artifactDir}/${language}/sourceCode/${packageName}` + ); } } else { - copyFile(packagePath, `${artifactDir}/${language}/${packageName}`); + copyFile(packagePath, `${artifactDir}/${language}/sourceCode/${packageName}`); } } } @@ -69,8 +72,8 @@ async function prepareArtifacts(generateAndBuildOutput: GenerateAndBuildOutput, for (const artifact of artifacts) { const artifactName = path.basename(artifact); - fs.mkdirSync(`${artifactDir}/${language}`, { recursive: true }); - fs.copyFileSync(`${artifact}`, `${artifactDir}/${language}/${artifactName}`); + fs.mkdirSync(`${artifactDir}/${language}/artifact`, { recursive: true }); + fs.copyFileSync(`${artifact}`, `${artifactDir}/${language}/artifact/${artifactName}`); } } } diff --git a/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml b/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml index 888e2fa6da9..86df5dcc9b5 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml @@ -63,8 +63,20 @@ steps: - task: PublishPipelineArtifact@1 displayName: Publish SDK Generation pipeline artifact inputs: - targetPath: $(System.ArtifactsDirectory)/ - artifactName: sdk-generation + targetPath: $(System.ArtifactsDirectory)/${{ parameters.sdk }}/artifact/ + artifactName: sdk-generation-artifact + + - task: PublishPipelineArtifact@1 + displayName: Publish SDK Generation pipeline sourceCode + inputs: + targetPath: $(System.ArtifactsDirectory)/${{ parameters.sdk }}/sourceCode/ + artifactName: sdk-generation-sourceCode + + - task: PublishPipelineArtifact@1 + displayName: Publish SDK Generation pipeline logs + inputs: + targetPath: $(System.ArtifactsDirectory)/logs/ + artifactName: sdk-generation-logs - bash: | publishResult \ From 5c4c3dde44d70f5fef07059526493f5c3eabc9d2 Mon Sep 17 00:00:00 2001 From: zzvswxy Date: Mon, 6 Jun 2022 22:21:01 +0800 Subject: [PATCH 06/30] move mkdir out of loop for it has the same path Signed-off-by: zzvswxy --- .../src/cli/pipelineCli/prepareArtifactFilesCli.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts index 7011b9e5f9c..56ef1b40227 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts @@ -70,9 +70,9 @@ async function prepareArtifacts(generateAndBuildOutput: GenerateAndBuildOutput, continue; } + fs.mkdirSync(`${artifactDir}/${language}/artifact`, { recursive: true }); for (const artifact of artifacts) { const artifactName = path.basename(artifact); - fs.mkdirSync(`${artifactDir}/${language}/artifact`, { recursive: true }); fs.copyFileSync(`${artifact}`, `${artifactDir}/${language}/artifact/${artifactName}`); } } From d7d278de88a1b319ba488fa485b64ac5227be280 Mon Sep 17 00:00:00 2001 From: zzvswxy Date: Tue, 7 Jun 2022 17:29:19 +0800 Subject: [PATCH 07/30] may not generate artifact so event task failed, the pipeline should continue next task Signed-off-by: zzvswxy --- .../pipelines/steps/parse-and-publish-result.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml b/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml index 86df5dcc9b5..20c891bc1d2 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml @@ -65,6 +65,7 @@ steps: inputs: targetPath: $(System.ArtifactsDirectory)/${{ parameters.sdk }}/artifact/ artifactName: sdk-generation-artifact + continueOnError: true - task: PublishPipelineArtifact@1 displayName: Publish SDK Generation pipeline sourceCode From f63a7c1a07731399722f1ebc176f7611a7a7e24e Mon Sep 17 00:00:00 2001 From: zzvswxy Date: Wed, 8 Jun 2022 17:03:07 +0800 Subject: [PATCH 08/30] use pr number to checkout swagger repo Signed-off-by: zzvswxy --- .../steps/parse-and-publish-result.yml | 6 ++---- .../pipelines/steps/pipeline-setup.yml | 18 ++++++++++++++++-- .../pipelines/steps/precheck.yml | 8 +------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml b/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml index 20c891bc1d2..e87ba430a9e 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml @@ -47,7 +47,6 @@ steps: --generateAndBuildOutputFile="/tmp/output/generateAndBuildOutputJson.json" \ --language="${{ parameters.sdk }}" if [ $? -ne 0 ]; then - echo "##vso[task.setvariable variable=pipelineResult;]failed" exit 1 fi cp /tmp/output/docker.log $(System.ArtifactsDirectory)/logs/${{ parameters.sdkGenerationName }}-full.log @@ -90,7 +89,6 @@ steps: "/tmp/output/generateAndBuild-task-result.json", "/tmp/output/mockTest-task-result.json"]' if [ $? -ne 0 ]; then - echo "##vso[task.setvariable variable=pipelineResult;]failed" exit 1 fi displayName: Publish status @@ -100,8 +98,8 @@ steps: - bash: | echo "will create PR" - git config --global user.email "no-reply@azure-sdk.com" - git config --global user.name "$(REPO_OWNER)" + git config --global user.email "$(GIT_USER_EMAIL)" + git config --global user.name "$(GIT_USER_NAME)" packages=`cat /tmp/output/generateAndBuildOutputJson.json | jq -c '.packages | .[]'` for package in ${packages} diff --git a/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml b/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml index 5ce62ed9c7b..fca75e82fb0 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml @@ -8,6 +8,9 @@ parameters: - name: trigger type: string + - name: swagger + type: object + steps: - task: NodeTool@0 displayName: Specify Node Version @@ -45,10 +48,21 @@ steps: - bash: | #TODO: will support clone specific PR and commit - git clone --branch $(specRepoBranch) $(specRepoUrl) + git clone ${{ parameters.swagger.path }} downloadSdkRepoUrl=$(echo $(sdkRepoUrl) | sed "s#https://#https://$(REPO_OWNER):$(REPO_OWNER_ACCESSKEY)@#") git clone --branch $(sdkRepoBaseBranch) ${downloadSdkRepoUrl} - getRepoName SPEC_REPO $(specRepoUrl) + getRepoName SPEC_REPO ${{ parameters.swagger.path }} getRepoName SDK_REPO $(sdkRepoUrl) + cd ${SPEC_REPO} + GITHUB_TOKEN=$(REPO_OWNER_ACCESSKEY) + if [ -n "${{ parameters.swagger.branch }}" ];then + git checkout ${{ parameters.swagger.branch }} + elif [ -n "${{ parameters.swagger.pullRequest }}" ];then + gh pr checkout ${{ parameters.swagger.pullRequest }} + else + echo "invalid swagger parameters" + exit 1 + fi + displayName: Repo pull workingDirectory: ../ diff --git a/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml b/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml index bc3ec492189..9197cedb000 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml @@ -15,15 +15,9 @@ parameters: type: string steps: - bash: | - if [ -z ${{ parameters.swagger.branch }} ];then - echo "swagger.branch is empty" - exit 1 - elif [ -z ${{ parameters.swagger.path }} ];then + if [ -z ${{ parameters.swagger.path }} ];then echo "swagger.path is empty" exit 1 - else - echo "##vso[task.setvariable variable=specRepoUrl;]${{ parameters.swagger.path }}" - echo "##vso[task.setvariable variable=specRepoBranch;]${{ parameters.swagger.branch }}" fi displayName: Check parameters From 18dde524a46ed53539de730a9f5173ca00239d45 Mon Sep 17 00:00:00 2001 From: zzvswxy Date: Wed, 8 Jun 2022 17:09:35 +0800 Subject: [PATCH 09/30] [bugfix] fix missed parameter Signed-off-by: zzvswxy --- .../src/cliSchema/testConfig.ts | 28 +++++++ .../sdk-generation-cli/src/testCli.ts | 75 +++++++++++++++++++ .../sdk-generation-pipeline.yml | 1 + 3 files changed, 104 insertions(+) create mode 100644 tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/testConfig.ts create mode 100644 tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/testCli.ts diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/testConfig.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/testConfig.ts new file mode 100644 index 00000000000..ca4c5e69e81 --- /dev/null +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/testConfig.ts @@ -0,0 +1,28 @@ +import convict from 'convict'; + +const AssertNullOrEmpty = (value: string) => { + console.log(`get here`); + if (!value || value.length === 0) { + throw new Error(' must not be null or empty'); + } +}; + +const stringMustHaveLength = (value: string) => { + console.log(`get here`); + if (value.length === 0) { + throw new Error('must not be empty'); + } +}; + +export class ResultPublisherBlobConfig { + logsAndResultPath: string; +} + +export const resultPublisherBlobConfig = convict({ + logsAndResultPath: { + default: null, + format: AssertNullOrEmpty, + env: 'LOGS_AND_RESULT_PATH', + arg: 'logsAndResultPath', + }, +}); diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/testCli.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/testCli.ts new file mode 100644 index 00000000000..3e163626030 --- /dev/null +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/testCli.ts @@ -0,0 +1,75 @@ +#!/usr/bin/env node + +import * as fs from 'fs'; +// import { resultPublisherBlobConfig, ResultPublisherBlobConfig } from './cliSchema/testConfig'; +import { prepareArtifactFilesInput, PrepareArtifactFilesInput } from './cliSchema/prepareArtifactFilesCliConfig'; +import { GenerateAndBuildOutput, getGenerateAndBuildOutput, requireJsonc, SDK } from '@azure-tools/sdk-generation-lib'; + +function validateInput(config: PrepareArtifactFilesInput) { + if (!fs.existsSync(config.generateAndBuildOutputFile)) { + throw new Error(`generateAndBuildOutputFile:${config.generateAndBuildOutputFile} isn's exist!`); + } + if (!fs.existsSync(config.artifactDir)) { + throw new Error(`Invalid artifactDir:${config.artifactDir}!`); + } + if (!(Object).values(SDK).includes(config.language)) { + throw new Error(config.language + ` is not supported.`); + } +} + +// type AA = { +// name: string; +// }; + +// type BB = AA & { +// job: string; +// }; + +// type CC = { +// school: string; +// }; + +// enum etype { +// aa = 'aa', +// bb = 'bb', +// } + +async function main() { + console.log(`main`); + // resultPublisherBlobConfig.validate(); + // const jsonStr = '{"nam":"release-workflow"}'; + // // const jsonStr = '{"name":"zhou"}'; + // let jsonObj = JSON.parse(jsonStr); + // console.log(jsonStr); + // console.log(jsonObj); + // console.log(jsonObj as BB); + // console.log(jsonObj as AA); + // console.log(jsonObj as CC); + // type TYPEA = 'aa' | 'bb'; + // const t: TYPEA = 'bb'; + // throw new Error(`unsupported TYPEA` + (t as string)); + // const straa = 'aa'; + // if ((straa as etype) === etype.aa) { + // console.log('equal'); + // } + // resultPublisherBlobConfig.validate(); + // const config: ResultPublisherBlobConfig = resultPublisherBlobConfig.getProperties(); + // console.log(`${config.logsAndResultPath}`); + prepareArtifactFilesInput.validate(); + console.log('00'); + const config: PrepareArtifactFilesInput = prepareArtifactFilesInput.getProperties(); + console.log('11'); + validateInput(config); + console.log('22'); + const generateAndBuildOutput: GenerateAndBuildOutput = getGenerateAndBuildOutput( + requireJsonc(config.generateAndBuildOutputFile) + ); + + console.log(`main end`); +} + +main().catch((e) => { + console.error(`${e.message} + ${e.stack}`); + process.exit(1); +}); diff --git a/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml b/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml index 3f153c10a0f..140b9efbb51 100644 --- a/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml +++ b/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml @@ -71,6 +71,7 @@ stages: sdk: ${{ parameters.sdkToGernerate }} sdkRepoBaseBranch: ${{ variables.sdkRepoBaseBranch }} trigger: ${{ parameters.clientAgent }} + swagger: ${{ parameters.swagger }} - template: ./pipelines/steps/generate-sdk.yml parameters: From 338d5c3e2464c69436c735d0abc353524e12d6fc Mon Sep 17 00:00:00 2001 From: zzvswxy Date: Wed, 8 Jun 2022 17:12:16 +0800 Subject: [PATCH 10/30] delete test code Signed-off-by: zzvswxy --- .../src/cliSchema/testConfig.ts | 28 ------- .../sdk-generation-cli/src/testCli.ts | 75 ------------------- 2 files changed, 103 deletions(-) delete mode 100644 tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/testConfig.ts delete mode 100644 tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/testCli.ts diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/testConfig.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/testConfig.ts deleted file mode 100644 index ca4c5e69e81..00000000000 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/testConfig.ts +++ /dev/null @@ -1,28 +0,0 @@ -import convict from 'convict'; - -const AssertNullOrEmpty = (value: string) => { - console.log(`get here`); - if (!value || value.length === 0) { - throw new Error(' must not be null or empty'); - } -}; - -const stringMustHaveLength = (value: string) => { - console.log(`get here`); - if (value.length === 0) { - throw new Error('must not be empty'); - } -}; - -export class ResultPublisherBlobConfig { - logsAndResultPath: string; -} - -export const resultPublisherBlobConfig = convict({ - logsAndResultPath: { - default: null, - format: AssertNullOrEmpty, - env: 'LOGS_AND_RESULT_PATH', - arg: 'logsAndResultPath', - }, -}); diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/testCli.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/testCli.ts deleted file mode 100644 index 3e163626030..00000000000 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/testCli.ts +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env node - -import * as fs from 'fs'; -// import { resultPublisherBlobConfig, ResultPublisherBlobConfig } from './cliSchema/testConfig'; -import { prepareArtifactFilesInput, PrepareArtifactFilesInput } from './cliSchema/prepareArtifactFilesCliConfig'; -import { GenerateAndBuildOutput, getGenerateAndBuildOutput, requireJsonc, SDK } from '@azure-tools/sdk-generation-lib'; - -function validateInput(config: PrepareArtifactFilesInput) { - if (!fs.existsSync(config.generateAndBuildOutputFile)) { - throw new Error(`generateAndBuildOutputFile:${config.generateAndBuildOutputFile} isn's exist!`); - } - if (!fs.existsSync(config.artifactDir)) { - throw new Error(`Invalid artifactDir:${config.artifactDir}!`); - } - if (!(Object).values(SDK).includes(config.language)) { - throw new Error(config.language + ` is not supported.`); - } -} - -// type AA = { -// name: string; -// }; - -// type BB = AA & { -// job: string; -// }; - -// type CC = { -// school: string; -// }; - -// enum etype { -// aa = 'aa', -// bb = 'bb', -// } - -async function main() { - console.log(`main`); - // resultPublisherBlobConfig.validate(); - // const jsonStr = '{"nam":"release-workflow"}'; - // // const jsonStr = '{"name":"zhou"}'; - // let jsonObj = JSON.parse(jsonStr); - // console.log(jsonStr); - // console.log(jsonObj); - // console.log(jsonObj as BB); - // console.log(jsonObj as AA); - // console.log(jsonObj as CC); - // type TYPEA = 'aa' | 'bb'; - // const t: TYPEA = 'bb'; - // throw new Error(`unsupported TYPEA` + (t as string)); - // const straa = 'aa'; - // if ((straa as etype) === etype.aa) { - // console.log('equal'); - // } - // resultPublisherBlobConfig.validate(); - // const config: ResultPublisherBlobConfig = resultPublisherBlobConfig.getProperties(); - // console.log(`${config.logsAndResultPath}`); - prepareArtifactFilesInput.validate(); - console.log('00'); - const config: PrepareArtifactFilesInput = prepareArtifactFilesInput.getProperties(); - console.log('11'); - validateInput(config); - console.log('22'); - const generateAndBuildOutput: GenerateAndBuildOutput = getGenerateAndBuildOutput( - requireJsonc(config.generateAndBuildOutputFile) - ); - - console.log(`main end`); -} - -main().catch((e) => { - console.error(`${e.message} - ${e.stack}`); - process.exit(1); -}); From 7c26ca9f119783849ed77799d4cc7c2ab7441a00 Mon Sep 17 00:00:00 2001 From: zzvswxy Date: Wed, 8 Jun 2022 19:48:20 +0800 Subject: [PATCH 11/30] pipeline support use swagger pr to checkout spec repo Signed-off-by: zzvswxy --- .../pipelines/steps/pipeline-setup.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml b/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml index fca75e82fb0..1aa1b6aaab2 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml @@ -18,7 +18,6 @@ steps: versionSpec: 14.x - bash: | - pwd cd tools/sdk-generation-pipeline npm install -g @microsoft/rush rush update @@ -33,6 +32,9 @@ steps: mkdir -p /tmp/output echo "mkdir for pipeline artifact" mkdir -p logs + cd $(Agent.BuildDirectory) + getRepoName SPEC_REPO ${{ parameters.swagger.path }} + getRepoName SDK_REPO $(sdkRepoUrl) publishResult \ --storageType=eventhub \ @@ -47,14 +49,11 @@ steps: EVENTHUB_SAS_URL: $(EVENTHUB_SAS_URL) - bash: | - #TODO: will support clone specific PR and commit git clone ${{ parameters.swagger.path }} downloadSdkRepoUrl=$(echo $(sdkRepoUrl) | sed "s#https://#https://$(REPO_OWNER):$(REPO_OWNER_ACCESSKEY)@#") git clone --branch $(sdkRepoBaseBranch) ${downloadSdkRepoUrl} - getRepoName SPEC_REPO ${{ parameters.swagger.path }} - getRepoName SDK_REPO $(sdkRepoUrl) - cd ${SPEC_REPO} - GITHUB_TOKEN=$(REPO_OWNER_ACCESSKEY) + cd $(SPEC_REPO) + export GITHUB_TOKEN=$(REPO_OWNER_ACCESSKEY) if [ -n "${{ parameters.swagger.branch }}" ];then git checkout ${{ parameters.swagger.branch }} elif [ -n "${{ parameters.swagger.pullRequest }}" ];then From c3d79fc78fa7891e3f85e7b3dc3821f26ec1ccc4 Mon Sep 17 00:00:00 2001 From: zzvswxy Date: Wed, 8 Jun 2022 20:26:02 +0800 Subject: [PATCH 12/30] try upload logs file to artifact even sdk generation failed Signed-off-by: zzvswxy --- .../pipelines/steps/parse-and-publish-result.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml b/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml index e87ba430a9e..961afa8e6fa 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml @@ -42,13 +42,6 @@ steps: condition: always() - bash: | - prepareArtifactFiles \ - --artifactDir="$(System.ArtifactsDirectory)" \ - --generateAndBuildOutputFile="/tmp/output/generateAndBuildOutputJson.json" \ - --language="${{ parameters.sdk }}" - if [ $? -ne 0 ]; then - exit 1 - fi cp /tmp/output/docker.log $(System.ArtifactsDirectory)/logs/${{ parameters.sdkGenerationName }}-full.log cp /tmp/output/init-task.log $(System.ArtifactsDirectory)/logs/${{ parameters.sdkGenerationName }}-init.log cp /tmp/output/init-task-result.json $(System.ArtifactsDirectory)/logs/${{ parameters.sdkGenerationName }}-init-result.json @@ -56,8 +49,16 @@ steps: cp /tmp/output/generateAndBuild-task-result.json $(System.ArtifactsDirectory)/logs/${{ parameters.sdkGenerationName }}-generateAndBuild-result.json cp /tmp/output/mockTest-task.log $(System.ArtifactsDirectory)/logs/${{ parameters.sdkGenerationName }}-mockTest.log | echo "wait all language support mock test" cp /tmp/output/mockTest-task-result.json $(System.ArtifactsDirectory)/logs/${{ parameters.sdkGenerationName }}-mockTest-result.json | echo "wait all language support mock test" + prepareArtifactFiles \ + --artifactDir="$(System.ArtifactsDirectory)" \ + --generateAndBuildOutputFile="/tmp/output/generateAndBuildOutputJson.json" \ + --language="${{ parameters.sdk }}" + if [ $? -ne 0 ]; then + exit 1 + fi displayName: Prepare artifact workingDirectory: $(SDK_REPO) + condition: always() - task: PublishPipelineArtifact@1 displayName: Publish SDK Generation pipeline artifact @@ -77,6 +78,7 @@ steps: inputs: targetPath: $(System.ArtifactsDirectory)/logs/ artifactName: sdk-generation-logs + condition: always() - bash: | publishResult \ From 0d4719f8cdbfbde028d4153b35d96874b92b0bf7 Mon Sep 17 00:00:00 2001 From: zzvswxy Date: Mon, 30 May 2022 15:00:10 +0800 Subject: [PATCH 13/30] debug for pipeline Signed-off-by: Zhou Zheng --- .../pipelineCli/prepareArtifactFilesCli.ts | 110 ++++++++++++- .../src/cli/pipelineCli/publishResultCli.ts | 104 ++++++------- .../src/cliSchema/generateResultCliConfig.ts | 20 +-- .../prepareArtifactFilesCliConfig.ts | 48 +++++- .../src/lib/ResultPublisher.ts | 8 +- .../src/types/taskResult.ts | 27 +++- .../pipelines/steps/generate-sdk.yml | 43 ++++++ .../steps/parse-and-publish-result.yml | 144 ++++++++++++------ .../pipelines/steps/pipeline-setup.yml | 30 +++- .../sdk-generation-pipeline.yml | 10 ++ 10 files changed, 413 insertions(+), 131 deletions(-) diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts index 56ef1b40227..2044e29a082 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts @@ -1,16 +1,29 @@ #!/usr/bin/env node import { + CompletedEvent, GenerateAndBuildOutput, + generateTotalResult, getGenerateAndBuildOutput, + getTaskResults, + InProgressEvent, logger, + PipelineRunEvent, requireJsonc, SDK, + SDKPipelineStatus, + TaskResult, TaskResultStatus, + Trigger, } from '@azure-tools/sdk-generation-lib'; import * as fs from 'fs'; import * as path from 'path'; -import { PrepareArtifactFilesInput, prepareArtifactFilesInput } from '../../cliSchema/prepareArtifactFilesCliConfig'; +import { + PrepareArtifactFilesInput, + prepareArtifactFilesInput, + prepareResultArtifactInput, + PrepareResultArtifactInput, +} from '../../cliSchema/prepareArtifactFilesCliConfig'; import { getFileListInPackageFolder } from '../../utils/git'; function copyFile(filePath: string, targetDir: string) { @@ -80,7 +93,8 @@ async function prepareArtifacts(generateAndBuildOutput: GenerateAndBuildOutput, function validateInput(config: PrepareArtifactFilesInput) { if (!fs.existsSync(config.generateAndBuildOutputFile)) { - throw new Error(`generateAndBuildOutputFile:${config.generateAndBuildOutputFile} isn's exist!`); + logger.error(`generateAndBuildOutputFile:${config.generateAndBuildOutputFile} isn's exist!`); + process.exit(0); } if (!fs.existsSync(config.artifactDir)) { throw new Error(`Invalid artifactDir:${config.artifactDir}!`); @@ -90,7 +104,7 @@ function validateInput(config: PrepareArtifactFilesInput) { } } -async function main() { +async function prepareSourceCodeAndArtifacts() { prepareArtifactFilesInput.validate(); const config: PrepareArtifactFilesInput = prepareArtifactFilesInput.getProperties(); @@ -103,6 +117,96 @@ async function main() { await prepareArtifacts(generateAndBuildOutput, config.language, config.artifactDir); } +function validateResultInput(config: PrepareResultArtifactInput) { + if (!fs.existsSync(config.artifactDir)) { + throw new Error(`Invalid artifactDir:${config.artifactDir}!`); + } +} + +function getTrigger(config: PrepareResultArtifactInput): Trigger { + let trigger: Trigger; + try { + trigger = JSON.parse(config.trigger); + } catch (error) { + logger.error(`Wrong json format:` + config.trigger); + throw new Error(error); + } + + return trigger; +} + +function prepareResult(pipelineStatus: SDKPipelineStatus) { + prepareResultArtifactInput.validate(); + const config: PrepareResultArtifactInput = prepareResultArtifactInput.getProperties(); + + validateResultInput(config); + const trigger: Trigger = getTrigger(config); + let event: PipelineRunEvent = undefined; + + switch (pipelineStatus) { + case 'in_progress': + event = { + status: 'in_progress', + trigger: trigger, + pipelineBuildId: config.pipelineBuildId, + } as InProgressEvent; + break; + case 'completed': + if (!config.resultsPath || !config.logPath) { + throw new Error(`Invalid completed event parameter!`); + } + + const taskResults: TaskResult[] = getTaskResults(config.resultsPath); + const taskTotalResult: TaskResult = generateTotalResult(taskResults, config.pipelineBuildId); + event = { + status: 'completed', + trigger: trigger, + pipelineBuildId: config.pipelineBuildId, + logPath: config.logPath, + result: taskTotalResult, + } as CompletedEvent; + break; + default: + throw new Error(`Unsupported status: ` + (pipelineStatus as string)); + } + + fs.writeFileSync(config.artifactDir + `/` + pipelineStatus + `/result.json`, JSON.stringify(event, null, 2), { + encoding: 'utf-8', + }); +} + +async function main() { + const args = parseArgs(process.argv); + const pipelineStatus = args['pipelineStatus']; + + if (pipelineStatus === 'completed') { + prepareResult(pipelineStatus); + await prepareSourceCodeAndArtifacts(); + } else if (pipelineStatus === 'in_progress') { + prepareResult(pipelineStatus); + } else { + throw new Error(`Unknown pipelineStatus:${pipelineStatus}!`); + } +} + +/** + * Parse a list of command line arguments. + * @param argv List of cli args(process.argv) + */ +const flagRegex = /^--([^=:]+)([=:](.+))?$/; +export function parseArgs(argv: string[]) { + const result: any = {}; + for (const arg of argv) { + const match = flagRegex.exec(arg); + if (match) { + const key = match[1]; + const rawValue = match[3]; + result[key] = rawValue; + } + } + return result; +} + main().catch((e) => { logger.error(`${e.message} ${e.stack}`); diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/publishResultCli.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/publishResultCli.ts index 1e395fe778c..4d84f28f21b 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/publishResultCli.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/publishResultCli.ts @@ -5,6 +5,7 @@ import { CodeGeneration, CompletedEvent, generateTotalResult, + getTaskResults, InProgressEvent, logger, MongoConnectContext, @@ -17,7 +18,7 @@ import { SDKPipelineStatus, StorageType, TaskResult, - Trigger + Trigger, } from '@azure-tools/sdk-generation-lib'; import * as fs from 'fs'; @@ -29,7 +30,7 @@ import { ResultPublisherDBResultInput, resultPublisherDBResultInput, ResultPublisherEventHubInput, - resultPublisherEventHubInput + resultPublisherEventHubInput, } from '../../cliSchema/publishResultConfig'; async function publishBlob() { @@ -39,7 +40,7 @@ async function publishBlob() { pipelineBuildId: config.pipelineBuildId, sdkGenerationName: config.sdkGenerationName, azureStorageBlobSasUrl: config.azureStorageBlobSasUrl, - azureBlobContainerName: config.azureBlobContainerName + azureBlobContainerName: config.azureBlobContainerName, }; const resultBlobPublisher: ResultBlobPublisher = new ResultBlobPublisher(context); await resultBlobPublisher.uploadLogsAndResult(config.logsAndResultPath, config.taskName as AzureSDKTaskName); @@ -75,7 +76,7 @@ function initMongoConnectContext(config: ResultPublisherDBCodeGenerationInput): database: config.mongodb.database, ssl: config.mongodb.ssl, synchronize: true, - logging: true + logging: true, }; return mongoConnectContext; @@ -123,19 +124,6 @@ function getTrigger(config: ResultPublisherEventHubInput): Trigger { return trigger; } -function getTaskResults(taskResultsPath: string): TaskResult[] { - const taskResultsPathArray = JSON.parse(taskResultsPath); - const taskResults: TaskResult[] = []; - for (const taskResultPath of taskResultsPathArray) { - if (fs.existsSync(taskResultPath)) { - taskResults.push(requireJsonc(taskResultPath)); - } else { - logger.warn(`${taskResultPath} isn't exist, skip.`); - } - } - return taskResults; -} - async function publishEventhub(pipelineStatus: SDKPipelineStatus) { resultPublisherEventHubInput.validate(); const config: ResultPublisherEventHubInput = resultPublisherEventHubInput.getProperties(); @@ -144,37 +132,37 @@ async function publishEventhub(pipelineStatus: SDKPipelineStatus) { const publisher: ResultEventhubPublisher = new ResultEventhubPublisher(config.eventHubConnectionString); switch (pipelineStatus) { - case 'queued': - event = { - status: 'queued', - trigger: trigger, - pipelineBuildId: config.pipelineBuildId - } as QueuedEvent; - break; - case 'in_progress': - event = { - status: 'in_progress', - trigger: trigger, - pipelineBuildId: config.pipelineBuildId - } as InProgressEvent; - break; - case 'completed': - if (!config.resultsPath || !config.logPath) { - throw new Error(`Invalid completed event parameter!`); - } + case 'queued': + event = { + status: 'queued', + trigger: trigger, + pipelineBuildId: config.pipelineBuildId, + } as QueuedEvent; + break; + case 'in_progress': + event = { + status: 'in_progress', + trigger: trigger, + pipelineBuildId: config.pipelineBuildId, + } as InProgressEvent; + break; + case 'completed': + if (!config.resultsPath || !config.logPath) { + throw new Error(`Invalid completed event parameter!`); + } - const taskResults: TaskResult[] = getTaskResults(config.resultsPath); - const taskTotalResult: TaskResult = generateTotalResult(taskResults, config.pipelineBuildId); - event = { - status: 'completed', - trigger: trigger, - pipelineBuildId: config.pipelineBuildId, - logPath: config.logPath, - result: taskTotalResult - } as CompletedEvent; - break; - default: - throw new Error(`Unsupported status: ` + (pipelineStatus as string)); + const taskResults: TaskResult[] = getTaskResults(config.resultsPath); + const taskTotalResult: TaskResult = generateTotalResult(taskResults, config.pipelineBuildId); + event = { + status: 'completed', + trigger: trigger, + pipelineBuildId: config.pipelineBuildId, + logPath: config.logPath, + result: taskTotalResult, + } as CompletedEvent; + break; + default: + throw new Error(`Unsupported status: ` + (pipelineStatus as string)); } await publisher.publishEvent(event); await publisher.close(); @@ -186,17 +174,17 @@ async function main() { const pipelineStatus = args['pipelineStatus']; switch (storageType as StorageType) { - case StorageType.Blob: - await publishBlob(); - break; - case StorageType.Db: - await publishDB(pipelineStatus); - break; - case StorageType.EventHub: - await publishEventhub(pipelineStatus); - break; - default: - throw new Error(`Unknown storageType:${storageType}!`); + case StorageType.Blob: + await publishBlob(); + break; + case StorageType.Db: + await publishDB(pipelineStatus); + break; + case StorageType.EventHub: + await publishEventhub(pipelineStatus); + break; + default: + throw new Error(`Unknown storageType:${storageType}!`); } } diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/generateResultCliConfig.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/generateResultCliConfig.ts index adc0303e6b8..336721672d5 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/generateResultCliConfig.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/generateResultCliConfig.ts @@ -17,45 +17,45 @@ export const generateResultCliInput = convict({ pipelineBuildId: { default: null, format: assertNullOrEmpty, - arg: 'buildId' + arg: 'buildId', }, logfile: { default: null, format: assertNullOrEmpty, - arg: 'logfile' + arg: 'logfile', }, logFilterStr: { default: null, nullable: true, format: String, - arg: 'logFilterStr' + arg: 'logFilterStr', }, taskName: { default: null, format: ['init', 'generateAndBuild', 'mockTest', 'liveTest'], - arg: 'taskName' + arg: 'taskName', }, exeResult: { default: null, nullable: true, - format: ['success', 'failure'], - arg: 'exeResult' + format: ['succeeded', 'failed'], + arg: 'exeResult', }, taskOutputPath: { default: null, nullable: true, format: String, - arg: 'taskOutputPath' + arg: 'taskOutputPath', }, resultOutputPath: { default: null, format: assertNullOrEmpty, - arg: 'resultOutputPath' + arg: 'resultOutputPath', }, dockerResultFile: { default: null, nullable: true, format: String, - arg: 'dockerResultFile' - } + arg: 'dockerResultFile', + }, }); diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/prepareArtifactFilesCliConfig.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/prepareArtifactFilesCliConfig.ts index 8c59b602315..2ad5bff4353 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/prepareArtifactFilesCliConfig.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/prepareArtifactFilesCliConfig.ts @@ -12,17 +12,57 @@ export const prepareArtifactFilesInput = convict({ generateAndBuildOutputFile: { default: null, format: assertNullOrEmpty, - arg: 'generateAndBuildOutputFile' + arg: 'generateAndBuildOutputFile', }, artifactDir: { doc: 'The dir to publish artifact', default: null, format: assertNullOrEmpty, - arg: 'artifactDir' + arg: 'artifactDir', }, language: { default: null, format: ['js', 'python', 'go', 'net', 'java'], - arg: 'language' - } + arg: 'language', + }, +}); + +export class PrepareResultArtifactInput { + pipelineBuildId: string; + trigger: string; + artifactDir: string; + logPath?: string; + resultsPath?: string; +} + +export const prepareResultArtifactInput = convict({ + pipelineBuildId: { + default: null, + format: assertNullOrEmpty, + arg: 'buildId', + }, + trigger: { + default: null, + format: assertNullOrEmpty, + arg: 'trigger', + }, + artifactDir: { + doc: 'The dir to publish artifact', + default: null, + format: assertNullOrEmpty, + arg: 'artifactDir', + }, + logPath: { + default: null, + nullable: true, + format: String, + arg: 'logPath', + }, + resultsPath: { + doc: 'task result files array', + default: null, + nullable: true, + format: String, + arg: 'resultsPath', + }, }); diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/lib/ResultPublisher.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/lib/ResultPublisher.ts index c4403cb0f0e..99a9d0c8cff 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/lib/ResultPublisher.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/lib/ResultPublisher.ts @@ -50,7 +50,7 @@ export class ResultDBPublisher { ssl: this.context.ssl, synchronize: this.context.synchronize, logging: this.context.logging, - entities: [TaskResultEntity, CodeGeneration] + entities: [TaskResultEntity, CodeGeneration], }); } @@ -101,9 +101,9 @@ export class ResultBlobPublisher { const logsAndResultPathArray = JSON.parse(logsAndResultPath); for (const file of logsAndResultPathArray) { if (fs.existsSync(file)) { - const blobName: string = file.includes('.json') ? - `${this.pipelineBuildId}/logs/${this.sdkGenerationName}-${taskName}-result.json` : - `${this.pipelineBuildId}/logs/${this.sdkGenerationName}-${taskName}.log`; + const blobName: string = file.includes('.json') + ? `${this.pipelineBuildId}/logs/${this.sdkGenerationName}-${taskName}-result.json` + : `${this.pipelineBuildId}/logs/${this.sdkGenerationName}-${taskName}.log`; await this.azureBlobClient.publishBlob(file, blobName); logger.info(`Publish ${file} Success !!!`); } else { diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/types/taskResult.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/types/taskResult.ts index 5bb84f5612e..fd5f1f28917 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/types/taskResult.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/types/taskResult.ts @@ -4,18 +4,20 @@ import { Column, Entity, ObjectIdColumn } from 'typeorm'; import { getTaskBasicConfig, TaskBasicConfig } from './taskBasicConfig'; import { GenerateAndBuildOutput } from './taskInputAndOuputSchemaTypes/GenerateAndBuildOutput'; import { InitOutput } from './taskInputAndOuputSchemaTypes/InitOutput'; +import { logger } from '../utils/logger'; +import { requireJsonc } from '../utils/requireJsonc'; import { TestOutput } from './taskInputAndOuputSchemaTypes/TestOutput'; @Entity('sdkGenerationResults') export class TaskResultEntity { @ObjectIdColumn() - id: string; + id: string; @Column() - key: string; + key: string; @Column() - pipelineBuildId: string; + pipelineBuildId: string; @Column() - taskResult: TaskResult; + taskResult: TaskResult; } export enum TaskResultStatus { @@ -104,7 +106,7 @@ export function setTaskResult(config: TaskBasicConfig, taskName: string) { pipelineBuildId: '', result: TaskResultStatus.Success, errorCount: 0, - warningCount: 0 + warningCount: 0, }; } @@ -122,7 +124,7 @@ export function generateTotalResult(taskResults: TaskResult[], pipelineBuildId: pipelineBuildId: pipelineBuildId, result: TaskResultStatus.Success, errorCount: 0, - messages: [] + messages: [], }; if (taskResults.length === 0) { @@ -144,3 +146,16 @@ export function generateTotalResult(taskResults: TaskResult[], pipelineBuildId: return totalResult; } + +export function getTaskResults(taskResultsPath: string): TaskResult[] { + const taskResultsPathArray = JSON.parse(taskResultsPath); + const taskResults: TaskResult[] = []; + for (const taskResultPath of taskResultsPathArray) { + if (fs.existsSync(taskResultPath)) { + taskResults.push(requireJsonc(taskResultPath)); + } else { + logger.warn(`${taskResultPath} isn't exist, skip.`); + } + } + return taskResults; +} diff --git a/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml b/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml index 1f5ca0117c3..36e4cfa52c9 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml @@ -5,6 +5,9 @@ parameters: - name: trigger type: string + - name: triggerType + type: string + steps: - bash: | publishResult \ @@ -18,6 +21,8 @@ steps: displayName: Publish status env: EVENTHUB_SAS_URL: $(EVENTHUB_SAS_URL) + condition: ne('${{ parameters.triggerType }}', 'release') + - bash: | docker run \ @@ -30,3 +35,41 @@ steps: exit 1 fi displayName: Generate sdk + + - bash: | + #TODO: delete this bash + echo "check output dir" + ls /tmp/output + echo "show docker.log" + head -n 30 /tmp/output/docker.log + echo "*****" + tail -n 30 /tmp/output/docker.log + echo "show generateAndBuild-task.log" + head -n 30 /tmp/output/generateAndBuild-task.log + echo "*****" + tail -n 30 /tmp/output/generateAndBuild-task.log + echo "show generateAndBuildInput.json" + cat /tmp/output/generateAndBuildInput.json + echo "show generateAndBuildOutputJson.json" + cat /tmp/output/generateAndBuildOutputJson.json + echo "show init-task.log" + head -n 30 /tmp/output/init-task.log + echo "*****" + tail -n 30 /tmp/output/init-task.log + echo "show taskResults.json" + cat /tmp/output/taskResults.json + + echo "show mock-host.log" + head -n 30 /tmp/output/mock-host.log | echo "don't quite even failed" + echo "*****" + tail -n 30 /tmp/output/mock-host.log | echo "don't quite even failed" + + echo "show mockTest-task.log" + head -n 30 /tmp/output/mockTest-task.log | echo "don't quite even failed" + echo "*****" + tail -n 30 /tmp/output/mockTest-task.log | echo "don't quite even failed" + + echo "show mockTestOutput.json" + cat /tmp/output/mockTestOutput.json | echo "don't quite even failed" + displayName: Check output dir + condition: always() \ No newline at end of file diff --git a/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml b/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml index 961afa8e6fa..a06928e86bb 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml @@ -13,6 +13,10 @@ parameters: - name: sdkGenerationName type: string + + - name: triggerType + type: string + steps: - bash: | echo "will parse task result" @@ -41,6 +45,67 @@ steps: displayName: Parse task result condition: always() + - bash: | + #TODO: delete this bash + ls -al /tmp/output/ + echo "show generateAndBuild-task-result.json" + cat /tmp/output/generateAndBuild-task-result.json | echo "don't quite even failed" + displayName: Check output dir + condition: always() + + - bash: | + #TODO: delete this bash + echo "will show git info" + git status + displayName: Show git info + workingDirectory: $(SDK_REPO) + + - bash: | + echo "will create PR" + git config --global user.email "$(GIT_USER_EMAIL)" + git config --global user.name "$(GIT_USER_NAME)" + + packages=`cat /tmp/output/generateAndBuildOutputJson.json| jq -c '.packages|.[]'` + for package in ${packages} + do + packageName=`echo ${package} | jq -r '.packageName'` + paths=`echo ${package} | jq -r '.path|.[]'` + for path in ${paths} + do + git add ${path} + done + git commit -m '[auto]create by sdk generation' + if [ "${{ parameters.serviceType }}" = "resource-manager" ];then + branchName=sdkGen/${{ parameters.sdkGenerationName }}/arm-${{ parameters.service }}-${packageName} + else + branchName=sdkGen/${{ parameters.sdkGenerationName }}/dpg-${{ parameters.service }}-${packageName} + fi + git checkout -b ${branchName} + pushSdkRepoUrl=$(echo $(sdkRepoUrl) | sed "s#https://#https://$(REPO_OWNER):$(REPO_OWNER_ACCESSKEY)@#") + #TODO: push PR to Azure if is release to Azure no Azure SDK + pushSdkRepoUrl=$(echo ${pushSdkRepoUrl} | sed "s#Azure#$(REPO_OWNER)#") + git push ${pushSdkRepoUrl} + sdkRepoPullsApi=$(echo $(sdkRepoUrl) | sed "s#https://#https://api.#") + sdkRepoPullsApi=$(echo ${sdkRepoPullsApi} | sed "s#Azure#repos/$(REPO_OWNER)#") + sdkRepoPullsApi=$(echo ${sdkRepoPullsApi} | sed "s#.git\$#/pulls#") + RESULT=$(curl \ + -u "$(REPO_OWNER)":"$(REPO_OWNER_ACCESSKEY)" \ + -H "Accept: application/vnd.github.v3+json" \ + -X POST \ + -d '{"title":"[AutoPR-${{ parameters.service }}]autogenerated","body":"","base":"main", "head":"'$(REPO_OWNER)':'${branchName}'"}' \ + ${sdkRepoPullsApi}) + + prNumber=`echo ${RESULT} | jq -r '.number'` + echo "##vso[task.setvariable variable=prNumber;]${prNumber}" + prLink=`echo ${RESULT} | jq -r '.html_url'` + echo "##vso[task.setvariable variable=prLink;]${prLink}" + echo "##vso[task.setvariable variable=branchName;]${branchName}" + echo "##vso[task.setvariable variable=packageName;]${packageName}" + git reset HEAD~ + done + workingDirectory: $(SDK_REPO) + displayName: Submit Pull Request + - bash: | cp /tmp/output/docker.log $(System.ArtifactsDirectory)/logs/${{ parameters.sdkGenerationName }}-full.log cp /tmp/output/init-task.log $(System.ArtifactsDirectory)/logs/${{ parameters.sdkGenerationName }}-init.log @@ -50,31 +115,52 @@ steps: cp /tmp/output/mockTest-task.log $(System.ArtifactsDirectory)/logs/${{ parameters.sdkGenerationName }}-mockTest.log | echo "wait all language support mock test" cp /tmp/output/mockTest-task-result.json $(System.ArtifactsDirectory)/logs/${{ parameters.sdkGenerationName }}-mockTest-result.json | echo "wait all language support mock test" prepareArtifactFiles \ + --pipelineStatus=completed \ --artifactDir="$(System.ArtifactsDirectory)" \ + --buildId="$(Build.BuildId)" \ --generateAndBuildOutputFile="/tmp/output/generateAndBuildOutputJson.json" \ - --language="${{ parameters.sdk }}" - if [ $? -ne 0 ]; then - exit 1 + --trigger='${{ parameters.trigger }}' \ + --logPath="tmplogpath.com" \ + --language="${{ parameters.sdk }}" \ + --resultsPath='["/tmp/output/init-task-result.json", + "/tmp/output/generateAndBuild-task-result.json", + "/tmp/output/mockTest-task-result.json"]' + + #TODO: delete next line + cat /tmp/output/generateAndBuild-task-result.json + cat $(System.ArtifactsDirectory)/completed/result.json + + if [ -n "$(prLink)" ];then + newtaskresult=$(cat $(System.ArtifactsDirectory)/completed/result.json | \ + jq --arg prNumber $(prNumber) --arg prLink $(prLink) --arg branchName $(branchName) \ + --arg packageName $(packageName) \ + '.result += {prNumber:$prNumber, prLink:$prLink, branchName:$branchName} | + .package += [{packageName:$packageName}]' ) + echo ${newtaskresult} > $(System.ArtifactsDirectory)/completed/result.json fi + + #TODO: delete next line + cat $(System.ArtifactsDirectory)/completed/result.json displayName: Prepare artifact workingDirectory: $(SDK_REPO) condition: always() - task: PublishPipelineArtifact@1 - displayName: Publish SDK Generation pipeline artifact + displayName: Publish artifact inputs: targetPath: $(System.ArtifactsDirectory)/${{ parameters.sdk }}/artifact/ artifactName: sdk-generation-artifact continueOnError: true - task: PublishPipelineArtifact@1 - displayName: Publish SDK Generation pipeline sourceCode + displayName: Publish sourceCode inputs: targetPath: $(System.ArtifactsDirectory)/${{ parameters.sdk }}/sourceCode/ artifactName: sdk-generation-sourceCode + continueOnError: true - task: PublishPipelineArtifact@1 - displayName: Publish SDK Generation pipeline logs + displayName: Publish logs inputs: targetPath: $(System.ArtifactsDirectory)/logs/ artifactName: sdk-generation-logs @@ -96,43 +182,11 @@ steps: displayName: Publish status env: EVENTHUB_SAS_URL: $(EVENTHUB_SAS_URL) - condition: always() + condition: and(always(), ne('${{ parameters.triggerType }}', 'release')) - - bash: | - echo "will create PR" - git config --global user.email "$(GIT_USER_EMAIL)" - git config --global user.name "$(GIT_USER_NAME)" - - packages=`cat /tmp/output/generateAndBuildOutputJson.json | jq -c '.packages | .[]'` - for package in ${packages} - do - packageName=`echo ${package} | jq -r '.packageName'` - paths=`echo ${package} | jq -r '.path |.[]'` - for path in ${paths} - do - git add ${path} - done - git commit -m '[auto]create by sdk generation' - if [ "${{ parameters.serviceType }}" = "resource-manager" ];then - branchName=sdkGen/${{ parameters.sdkGenerationName }}/arm-${{ parameters.service }}-${packageName} - else - branchName=sdkGen/${{ parameters.sdkGenerationName }}/dpg-${{ parameters.service }}-${packageName} - fi - git checkout -b ${branchName} - pushSdkRepoUrl=$(echo $(sdkRepoUrl) | sed "s#https://#https://$(REPO_OWNER):$(REPO_OWNER_ACCESSKEY)@#") - pushSdkRepoUrl=$(echo ${pushSdkRepoUrl} | sed "s#Azure#$(REPO_OWNER)#") - git push ${pushSdkRepoUrl} - sdkRepoPullsApi=$(echo $(sdkRepoUrl) | sed "s#https://#https://api.#") - sdkRepoPullsApi=$(echo ${sdkRepoPullsApi} | sed "s#Azure#repos/$(REPO_OWNER)#") - sdkRepoPullsApi=$(echo ${sdkRepoPullsApi} | sed "s#.git\$#/pulls#") - curl \ - -u "$(REPO_OWNER)":"$(REPO_OWNER_ACCESSKEY)" \ - -H "Accept: application/vnd.github.v3+json" \ - -X POST \ - -d '{"title":"[AutoPR-${{ parameters.service }}]autogenerated","body":"","base":"main", "head":"'$(REPO_OWNER)':'${branchName}'"}' \ - ${sdkRepoPullsApi} - - git reset HEAD~ - done - workingDirectory: $(SDK_REPO) - displayName: Submit Pull Request \ No newline at end of file + - task: PublishPipelineArtifact@1 + displayName: Publish status artifact + inputs: + targetPath: $(System.ArtifactsDirectory)/completed/ + artifactName: sdk-generation-completed-result + condition: always() \ No newline at end of file diff --git a/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml b/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml index 1aa1b6aaab2..9be1e28d5ad 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml @@ -11,6 +11,9 @@ parameters: - name: swagger type: object + - name: triggerType + type: string + steps: - task: NodeTool@0 displayName: Specify Node Version @@ -18,6 +21,7 @@ steps: versionSpec: 14.x - bash: | + //install tools cd tools/sdk-generation-pipeline npm install -g @microsoft/rush rush update @@ -27,6 +31,8 @@ steps: PACKAGE_TGZ=`ls azure-tools-sdk-generation-cli*.tgz` npm install -g ${PACKAGE_TGZ} npm install -g autorest + + //mkdir for artifacts cd $(System.ArtifactsDirectory) echo "mkdir for docker output" mkdir -p /tmp/output @@ -35,7 +41,12 @@ steps: cd $(Agent.BuildDirectory) getRepoName SPEC_REPO ${{ parameters.swagger.path }} getRepoName SDK_REPO $(sdkRepoUrl) + //mkdir for result artifact + mkdir -p $(System.ArtifactsDirectory)/in_progress/ + mkdir -p $(System.ArtifactsDirectory)/completed/ + displayName: Setup Pipeline Runtime Environment + - bash: | publishResult \ --storageType=eventhub \ --pipelineStatus=queued \ @@ -44,9 +55,26 @@ steps: if [ $? -ne 0 ]; then exit 1 fi - displayName: Setup Pipeline Runtime Environment + displayName: Publish status env: EVENTHUB_SAS_URL: $(EVENTHUB_SAS_URL) + condition: ne('${{ parameters.triggerType }}', 'release') + + - bash: | + prepareArtifactFiles \ + --pipelineStatus=in_progress \ + --artifactDir="$(System.ArtifactsDirectory)" \ + --buildId="$(Build.BuildId)" \ + --trigger='${{ parameters.trigger }}' + displayName: Prepare in progress status + condition: always() + + - task: PublishPipelineArtifact@1 + displayName: Publish status artifact + inputs: + targetPath: $(System.ArtifactsDirectory)/in_progress/ + artifactName: sdk-generation-in_progress-result + continueOnError: true - bash: | git clone ${{ parameters.swagger.path }} diff --git a/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml b/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml index 140b9efbb51..dd1a2dd95fc 100644 --- a/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml +++ b/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml @@ -54,6 +54,13 @@ parameters: - name: sdkGenerationName type: string + - name: triggerType + type: string + values: + - release + - ad-hoc + - ci + stages: - stage: SdkGeneration jobs: @@ -72,11 +79,13 @@ stages: sdkRepoBaseBranch: ${{ variables.sdkRepoBaseBranch }} trigger: ${{ parameters.clientAgent }} swagger: ${{ parameters.swagger }} + triggerType: ${{ parameters.triggerType }} - template: ./pipelines/steps/generate-sdk.yml parameters: readmeFile: ${{ parameters.relativeReadmeFilePath }} trigger: ${{ parameters.clientAgent }} + triggerType: ${{ parameters.triggerType }} - template: ./pipelines/steps/parse-and-publish-result.yml parameters: @@ -85,3 +94,4 @@ stages: serviceType: ${{ parameters.serviceType }} trigger: ${{ parameters.clientAgent }} sdkGenerationName: ${{ parameters.sdkGenerationName }} + triggerType: ${{ parameters.triggerType }} From ed63dfe43623afa9b4f832c07c59d4e63b36b646 Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Mon, 11 Jul 2022 12:42:09 +0800 Subject: [PATCH 14/30] build dpg Signed-off-by: Zhou Zheng --- .../pipelines/steps/generate-sdk.yml | 46 ++++++++++++++++--- .../steps/parse-and-publish-result.yml | 20 ++++---- .../pipelines/steps/pipeline-setup.yml | 11 ++--- .../pipelines/steps/precheck.yml | 18 +++++++- .../sdk-generation-pipeline.yml | 9 +++- 5 files changed, 80 insertions(+), 24 deletions(-) diff --git a/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml b/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml index 36e4cfa52c9..5197bab8a19 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml @@ -8,6 +8,15 @@ parameters: - name: triggerType type: string + - name: serviceType + type: string + + - name: sdk + type: string + + - name: sdkConfig + type: string + steps: - bash: | publishResult \ @@ -23,14 +32,37 @@ steps: EVENTHUB_SAS_URL: $(EVENTHUB_SAS_URL) condition: ne('${{ parameters.triggerType }}', 'release') - - bash: | - docker run \ - -v $(SPEC_REPO):/spec-repo \ - -v $(SDK_REPO):/sdk-repo \ - -v /tmp/output:/tmp/output \ - $(DOCKER_IMAGE) \ - --readme=${{ parameters.readmeFile }} + if [ "${{ parameters.serviceType }}" = "resource-manager" ];then + docker run --privileged \ + -v $(SPEC_REPO):/spec-repo \ + -v $(SDK_REPO):/sdk-repo \ + -v /tmp/output:/tmp/output \ + $(DOCKER_IMAGE) \ + --readme=${{ parameters.readmeFile }} + else + echo '${{ parameters.sdkConfig }}' > /tmp/output/autorest.md + configLen=`cat /tmp/output/autorest.md | wc -L` + echo "configLen is :${configLen}" + #check if config is default '' + if [ 2 -ge ${configLen} ];then + docker run --privileged \ + -v $(SPEC_REPO):/spec-repo \ + -v $(SDK_REPO):/sdk-repo \ + -v /tmp/output:/tmp/output \ + $(DOCKER_IMAGE) \ + --readme=${{ parameters.readmeFile }} + else + docker run --privileged \ + -v $(SPEC_REPO):/spec-repo \ + -v $(SDK_REPO):/sdk-repo \ + -v /tmp/output/autorest.md:/autorest.md \ + -v /tmp/output:/tmp/output \ + $(DOCKER_IMAGE) \ + --readme=${{ parameters.readmeFile }} + fi + fi + if [ $? -ne 0 ]; then exit 1 fi diff --git a/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml b/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml index a06928e86bb..2c5c1b6866e 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml @@ -65,9 +65,10 @@ steps: git config --global user.email "$(GIT_USER_EMAIL)" git config --global user.name "$(GIT_USER_NAME)" - packages=`cat /tmp/output/generateAndBuildOutputJson.json| jq -c '.packages|.[]'` - for package in ${packages} - do + #packages=`cat /tmp/output/generateAndBuildOutputJson.json| jq -c '.packages|.[]'` + package=`cat /tmp/output/generateAndBuildOutputJson.json| jq -c '.packages|.[]'` + #for package in ${packages} + #do packageName=`echo ${package} | jq -r '.packageName'` paths=`echo ${package} | jq -r '.path|.[]'` for path in ${paths} @@ -83,17 +84,20 @@ steps: git checkout -b ${branchName} pushSdkRepoUrl=$(echo $(sdkRepoUrl) | sed "s#https://#https://$(REPO_OWNER):$(REPO_OWNER_ACCESSKEY)@#") #TODO: push PR to Azure if is release to Azure no Azure SDK - pushSdkRepoUrl=$(echo ${pushSdkRepoUrl} | sed "s#Azure#$(REPO_OWNER)#") - git push ${pushSdkRepoUrl} + pushSdkRepoUrl=$(echo ${pushSdkRepoUrl} | sed "s#$(SDK_OWNER)#$(SDK_PUSH_OWNER)#") + git push ${pushSdkRepoUrl} --force + echo "git push finished:$?" sdkRepoPullsApi=$(echo $(sdkRepoUrl) | sed "s#https://#https://api.#") - sdkRepoPullsApi=$(echo ${sdkRepoPullsApi} | sed "s#Azure#repos/$(REPO_OWNER)#") + sdkRepoPullsApi=$(echo ${sdkRepoPullsApi} | sed "s#$(SDK_OWNER)#repos/$(SDK_PUSH_OWNER)#") sdkRepoPullsApi=$(echo ${sdkRepoPullsApi} | sed "s#.git\$#/pulls#") + echo "will use api:${sdkRepoPullsApi}" RESULT=$(curl \ -u "$(REPO_OWNER)":"$(REPO_OWNER_ACCESSKEY)" \ -H "Accept: application/vnd.github.v3+json" \ -X POST \ - -d '{"title":"[AutoPR-${{ parameters.service }}]autogenerated","body":"","base":"main", "head":"'$(REPO_OWNER)':'${branchName}'"}' \ + -d '{"title":"[AutoPR-${{ parameters.service }}]autogenerated","body":"","base":"'$(sdkMainBranch)'", "head":"$(SDK_PUSH_OWNER):'${branchName}'"}' \ ${sdkRepoPullsApi}) + echo "creat pr finished:$?" prNumber=`echo ${RESULT} | jq -r '.number'` echo "##vso[task.setvariable variable=prNumber;]${prNumber}" @@ -102,7 +106,7 @@ steps: echo "##vso[task.setvariable variable=branchName;]${branchName}" echo "##vso[task.setvariable variable=packageName;]${packageName}" git reset HEAD~ - done + #done workingDirectory: $(SDK_REPO) displayName: Submit Pull Request diff --git a/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml b/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml index 9be1e28d5ad..cb587c1e62e 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml @@ -2,9 +2,6 @@ parameters: - name: sdk type: string - - name: sdkRepoBaseBranch - type: string - - name: trigger type: string @@ -21,7 +18,7 @@ steps: versionSpec: 14.x - bash: | - //install tools + #install tools cd tools/sdk-generation-pipeline npm install -g @microsoft/rush rush update @@ -32,7 +29,7 @@ steps: npm install -g ${PACKAGE_TGZ} npm install -g autorest - //mkdir for artifacts + #mkdir for artifacts cd $(System.ArtifactsDirectory) echo "mkdir for docker output" mkdir -p /tmp/output @@ -41,7 +38,7 @@ steps: cd $(Agent.BuildDirectory) getRepoName SPEC_REPO ${{ parameters.swagger.path }} getRepoName SDK_REPO $(sdkRepoUrl) - //mkdir for result artifact + #mkdir for result artifact mkdir -p $(System.ArtifactsDirectory)/in_progress/ mkdir -p $(System.ArtifactsDirectory)/completed/ displayName: Setup Pipeline Runtime Environment @@ -79,7 +76,7 @@ steps: - bash: | git clone ${{ parameters.swagger.path }} downloadSdkRepoUrl=$(echo $(sdkRepoUrl) | sed "s#https://#https://$(REPO_OWNER):$(REPO_OWNER_ACCESSKEY)@#") - git clone --branch $(sdkRepoBaseBranch) ${downloadSdkRepoUrl} + git clone --branch $(sdkMainBranch) ${downloadSdkRepoUrl} cd $(SPEC_REPO) export GITHUB_TOKEN=$(REPO_OWNER_ACCESSKEY) if [ -n "${{ parameters.swagger.branch }}" ];then diff --git a/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml b/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml index 9197cedb000..44175519ba7 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml @@ -13,6 +13,9 @@ parameters: - name: service type: string + + - name: sdkRepoBaseBranch + type: string steps: - bash: | if [ -z ${{ parameters.swagger.path }} ];then @@ -22,5 +25,18 @@ steps: displayName: Check parameters - bash: | - echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/Azure/azure-sdk-for-${{parameters.sdk}}.git" + #TODO remove hard code + if [ ${{parameters.sdk}} = "js" ]; then + echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/$(SDK_OWNER)/azure-sdk-for-js.git" + echo "##vso[task.setvariable variable=sdkMainBranch;]js-dpg" + elif [ ${{parameters.sdk}} = "net" ]; then + echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/$(SDK_OWNER)/azure-sdk-for-net.git" + echo "##vso[task.setvariable variable=sdkMainBranch;]yc/dpgauto" + elif [ ${{parameters.sdk}} = "python" ]; then + echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/$(SDK_OWNER)/azure-sdk-for-python.git" + echo "##vso[task.setvariable variable=sdkMainBranch;]dpg-readme-sdkGeneration" + else + echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/Azure/azure-sdk-for-${{parameters.sdk}}.git" + echo "##vso[task.setvariable variable=sdkMainBranch;]$(sdkRepoBaseBranch)" + fi displayName: Set sdk repo diff --git a/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml b/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml index dd1a2dd95fc..67d1ffa8842 100644 --- a/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml +++ b/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml @@ -61,6 +61,10 @@ parameters: - ad-hoc - ci + - name: sdkConfig + type: string + default: "''" + stages: - stage: SdkGeneration jobs: @@ -72,11 +76,11 @@ stages: swagger: ${{ parameters.swagger }} sdk: ${{ parameters.sdkToGernerate }} service: ${{ parameters.service }} + sdkRepoBaseBranch: ${{ variables.sdkRepoBaseBranch }} - template: ./pipelines/steps/pipeline-setup.yml parameters: sdk: ${{ parameters.sdkToGernerate }} - sdkRepoBaseBranch: ${{ variables.sdkRepoBaseBranch }} trigger: ${{ parameters.clientAgent }} swagger: ${{ parameters.swagger }} triggerType: ${{ parameters.triggerType }} @@ -86,6 +90,9 @@ stages: readmeFile: ${{ parameters.relativeReadmeFilePath }} trigger: ${{ parameters.clientAgent }} triggerType: ${{ parameters.triggerType }} + serviceType: ${{ parameters.serviceType }} + sdk: ${{ parameters.sdkToGernerate }} + sdkConfig: ${{ parameters.sdkConfig }} - template: ./pipelines/steps/parse-and-publish-result.yml parameters: From ab150728f5d239aa1887b1854a4e7e1a4d817eac Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Fri, 29 Jul 2022 12:24:18 +0800 Subject: [PATCH 15/30] codegen pipeline support java dpg Signed-off-by: Zhou Zheng --- .../sdk-generation-pipeline/pipelines/steps/precheck.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml b/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml index 44175519ba7..af59a62de94 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml @@ -27,14 +27,17 @@ steps: - bash: | #TODO remove hard code if [ ${{parameters.sdk}} = "js" ]; then - echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/$(SDK_OWNER)/azure-sdk-for-js.git" + echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/$(SDK_OWNER)/azure-sdk-for-${{parameters.sdk}}.git" echo "##vso[task.setvariable variable=sdkMainBranch;]js-dpg" elif [ ${{parameters.sdk}} = "net" ]; then - echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/$(SDK_OWNER)/azure-sdk-for-net.git" + echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/$(SDK_OWNER)/azure-sdk-for-${{parameters.sdk}}.git" echo "##vso[task.setvariable variable=sdkMainBranch;]yc/dpgauto" elif [ ${{parameters.sdk}} = "python" ]; then - echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/$(SDK_OWNER)/azure-sdk-for-python.git" + echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/$(SDK_OWNER)/azure-sdk-for-${{parameters.sdk}}.git" echo "##vso[task.setvariable variable=sdkMainBranch;]dpg-readme-sdkGeneration" + elif [ ${{parameters.sdk}} = "java" ]; then + echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/$(SDK_OWNER)/azure-sdk-for-${{parameters.sdk}}.git" + echo "##vso[task.setvariable variable=sdkMainBranch;]dpg-readme-codegen" else echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/Azure/azure-sdk-for-${{parameters.sdk}}.git" echo "##vso[task.setvariable variable=sdkMainBranch;]$(sdkRepoBaseBranch)" From cb67d269c369961bdde45ea033c4b9b564a2b981 Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Tue, 2 Aug 2022 21:44:12 +0800 Subject: [PATCH 16/30] add release pipeline name and build type in result Signed-off-by: Zhou Zheng --- .../pipelines/steps/parse-and-publish-result.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml b/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml index 2c5c1b6866e..ed99b154c92 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml @@ -105,6 +105,14 @@ steps: echo "##vso[task.setvariable variable=prLink;]${prLink}" echo "##vso[task.setvariable variable=branchName;]${branchName}" echo "##vso[task.setvariable variable=packageName;]${packageName}" + packageFolder=`echo ${package} | jq -r '.packageFolder'` + serviceDir=`echo $packageFolder | awk '{split($0,array,"/");print array[2]}' ` + if [ "${{ parameters.sdk }}" = "js" ] && [ "${{ parameters.serviceType }}" = "resource-manager" ];then + echo "##vso[task.setvariable variable=releasePipeline;]${{ parameters.sdk }} - ${serviceDir} - mgmt" + else + echo "##vso[task.setvariable variable=releasePipeline;]${{ parameters.sdk }} - ${serviceDir}" + fi + git reset HEAD~ #done workingDirectory: $(SDK_REPO) @@ -136,9 +144,10 @@ steps: if [ -n "$(prLink)" ];then newtaskresult=$(cat $(System.ArtifactsDirectory)/completed/result.json | \ - jq --arg prNumber $(prNumber) --arg prLink $(prLink) --arg branchName $(branchName) \ - --arg packageName $(packageName) \ - '.result += {prNumber:$prNumber, prLink:$prLink, branchName:$branchName} | + jq --arg prNumber "$(prNumber)" --arg prLink "$(prLink)" --arg branchName "$(branchName)" \ + --arg packageName "$(packageName)" --arg serviceType "${{ parameters.serviceType }}" \ + --arg releasePipeline "$(releasePipeline)" \ + '.result += {prNumber:$prNumber, prLink:$prLink, branchName:$branchName, serviceType:$serviceType, releasePipeline:$releasePipeline} | .package += [{packageName:$packageName}]' ) echo ${newtaskresult} > $(System.ArtifactsDirectory)/completed/result.json fi From 4d740f78d193d65028b4f07097e63aaf7d7346fd Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Wed, 3 Aug 2022 15:44:02 +0800 Subject: [PATCH 17/30] use azure-sdk account for create sdk pr Signed-off-by: Zhou Zheng --- .../pipelines/steps/parse-and-publish-result.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml b/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml index ed99b154c92..53ea20ae8bc 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml @@ -82,7 +82,7 @@ steps: branchName=sdkGen/${{ parameters.sdkGenerationName }}/dpg-${{ parameters.service }}-${packageName} fi git checkout -b ${branchName} - pushSdkRepoUrl=$(echo $(sdkRepoUrl) | sed "s#https://#https://$(REPO_OWNER):$(REPO_OWNER_ACCESSKEY)@#") + pushSdkRepoUrl=$(echo $(sdkRepoUrl) | sed "s#https://#https://$(REPO_OWNER):$(azuresdk-github-pat)@#") #TODO: push PR to Azure if is release to Azure no Azure SDK pushSdkRepoUrl=$(echo ${pushSdkRepoUrl} | sed "s#$(SDK_OWNER)#$(SDK_PUSH_OWNER)#") git push ${pushSdkRepoUrl} --force @@ -92,7 +92,7 @@ steps: sdkRepoPullsApi=$(echo ${sdkRepoPullsApi} | sed "s#.git\$#/pulls#") echo "will use api:${sdkRepoPullsApi}" RESULT=$(curl \ - -u "$(REPO_OWNER)":"$(REPO_OWNER_ACCESSKEY)" \ + -u "$(REPO_OWNER)":"$(azuresdk-github-pat)" \ -H "Accept: application/vnd.github.v3+json" \ -X POST \ -d '{"title":"[AutoPR-${{ parameters.service }}]autogenerated","body":"","base":"'$(sdkMainBranch)'", "head":"$(SDK_PUSH_OWNER):'${branchName}'"}' \ From ba8a468bdc0bcf5911d1cc4f96bbd22f20459ec8 Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Thu, 4 Aug 2022 11:19:11 +0800 Subject: [PATCH 18/30] support add ex cmd for docker Signed-off-by: Zhou Zheng --- .../pipelines/steps/generate-sdk.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml b/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml index 5197bab8a19..dab909bdcc1 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml @@ -39,7 +39,7 @@ steps: -v $(SDK_REPO):/sdk-repo \ -v /tmp/output:/tmp/output \ $(DOCKER_IMAGE) \ - --readme=${{ parameters.readmeFile }} + --readme=${{ parameters.readmeFile }} $(DOCKER_EX_CMD) else echo '${{ parameters.sdkConfig }}' > /tmp/output/autorest.md configLen=`cat /tmp/output/autorest.md | wc -L` @@ -51,7 +51,7 @@ steps: -v $(SDK_REPO):/sdk-repo \ -v /tmp/output:/tmp/output \ $(DOCKER_IMAGE) \ - --readme=${{ parameters.readmeFile }} + --readme=${{ parameters.readmeFile }} $(DOCKER_EX_CMD) else docker run --privileged \ -v $(SPEC_REPO):/spec-repo \ @@ -59,7 +59,7 @@ steps: -v /tmp/output/autorest.md:/autorest.md \ -v /tmp/output:/tmp/output \ $(DOCKER_IMAGE) \ - --readme=${{ parameters.readmeFile }} + --readme=${{ parameters.readmeFile }} $(DOCKER_EX_CMD) fi fi From a8881126d6e1979d4fe2fc202f2fc69d56cf1409 Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Fri, 5 Aug 2022 12:51:43 +0800 Subject: [PATCH 19/30] event generate and build stage failed, try to upload source code and artifact Signed-off-by: Zhou Zheng --- .../src/cli/pipelineCli/prepareArtifactFilesCli.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts index 2044e29a082..91c55d790b7 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts @@ -42,11 +42,6 @@ async function prepareSourceCode( artifactDir: string ) { for (const p of generateAndBuildOutput.packages) { - const result = p.result; - if (result === TaskResultStatus.Failure) { - logger.warn(`Build ${p.packageName} failed, skipped it`); - continue; - } const packageName = p.packageName; const packagePaths = p.path; @@ -72,11 +67,6 @@ async function prepareSourceCode( async function prepareArtifacts(generateAndBuildOutput: GenerateAndBuildOutput, language: string, artifactDir: string) { for (const p of generateAndBuildOutput.packages) { - const result = p.result; - if (result === TaskResultStatus.Failure) { - logger.warn(`Build ${p.packageName} failed, skipped it`); - continue; - } const artifacts = p.artifacts; if (!artifacts) { // artifacts is optional From 8eae16cb3b829609a1055991191223bbc9086ebb Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Fri, 5 Aug 2022 13:15:48 +0800 Subject: [PATCH 20/30] don't exit 1 event generate cmd return 1 Signed-off-by: Zhou Zheng --- .../sdk-generation-pipeline/pipelines/steps/generate-sdk.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml b/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml index dab909bdcc1..a68efcb5c0d 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml @@ -62,10 +62,6 @@ steps: --readme=${{ parameters.readmeFile }} $(DOCKER_EX_CMD) fi fi - - if [ $? -ne 0 ]; then - exit 1 - fi displayName: Generate sdk - bash: | From 045edca6226490cc714a887f811f1e1a2cbcc5fa Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Mon, 8 Aug 2022 11:14:35 +0800 Subject: [PATCH 21/30] use one token for all git cmd Signed-off-by: Zhou Zheng --- .../pipelines/steps/pipeline-setup.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml b/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml index cb587c1e62e..34f5aab3655 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/pipeline-setup.yml @@ -75,10 +75,10 @@ steps: - bash: | git clone ${{ parameters.swagger.path }} - downloadSdkRepoUrl=$(echo $(sdkRepoUrl) | sed "s#https://#https://$(REPO_OWNER):$(REPO_OWNER_ACCESSKEY)@#") + downloadSdkRepoUrl=$(echo $(sdkRepoUrl) | sed "s#https://#https://$(REPO_OWNER):$(azuresdk-github-pat)@#") git clone --branch $(sdkMainBranch) ${downloadSdkRepoUrl} cd $(SPEC_REPO) - export GITHUB_TOKEN=$(REPO_OWNER_ACCESSKEY) + export GITHUB_TOKEN=$(azuresdk-github-pat) if [ -n "${{ parameters.swagger.branch }}" ];then git checkout ${{ parameters.swagger.branch }} elif [ -n "${{ parameters.swagger.pullRequest }}" ];then From 114b72d08e28b62c75d4bb39770e0cf2527a120c Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Mon, 8 Aug 2022 17:59:42 +0800 Subject: [PATCH 22/30] set sdk url by variable Signed-off-by: Zhou Zheng --- tools/sdk-generation-pipeline/pipelines/steps/precheck.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml b/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml index af59a62de94..12908cd75ea 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml @@ -25,21 +25,16 @@ steps: displayName: Check parameters - bash: | - #TODO remove hard code + echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/$(SDK_OWNER)/azure-sdk-for-${{parameters.sdk}}.git" if [ ${{parameters.sdk}} = "js" ]; then - echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/$(SDK_OWNER)/azure-sdk-for-${{parameters.sdk}}.git" echo "##vso[task.setvariable variable=sdkMainBranch;]js-dpg" elif [ ${{parameters.sdk}} = "net" ]; then - echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/$(SDK_OWNER)/azure-sdk-for-${{parameters.sdk}}.git" echo "##vso[task.setvariable variable=sdkMainBranch;]yc/dpgauto" elif [ ${{parameters.sdk}} = "python" ]; then - echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/$(SDK_OWNER)/azure-sdk-for-${{parameters.sdk}}.git" echo "##vso[task.setvariable variable=sdkMainBranch;]dpg-readme-sdkGeneration" elif [ ${{parameters.sdk}} = "java" ]; then - echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/$(SDK_OWNER)/azure-sdk-for-${{parameters.sdk}}.git" echo "##vso[task.setvariable variable=sdkMainBranch;]dpg-readme-codegen" else - echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/Azure/azure-sdk-for-${{parameters.sdk}}.git" echo "##vso[task.setvariable variable=sdkMainBranch;]$(sdkRepoBaseBranch)" fi displayName: Set sdk repo From 10958d64db3b5cc5f5ad902d50e21fc2f545b22b Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Mon, 8 Aug 2022 18:11:14 +0800 Subject: [PATCH 23/30] use env variable to control whether to show debug info Signed-off-by: Zhou Zheng --- .../pipelines/steps/generate-sdk.yml | 3 +-- .../steps/parse-and-publish-result.yml | 20 ++----------------- 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml b/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml index a68efcb5c0d..4727461aae4 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/generate-sdk.yml @@ -65,7 +65,6 @@ steps: displayName: Generate sdk - bash: | - #TODO: delete this bash echo "check output dir" ls /tmp/output echo "show docker.log" @@ -100,4 +99,4 @@ steps: echo "show mockTestOutput.json" cat /tmp/output/mockTestOutput.json | echo "don't quite even failed" displayName: Check output dir - condition: always() \ No newline at end of file + condition: ne('$(DEBUGMODE)', 'true') \ No newline at end of file diff --git a/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml b/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml index 53ea20ae8bc..9a01e0217c8 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/parse-and-publish-result.yml @@ -45,21 +45,6 @@ steps: displayName: Parse task result condition: always() - - bash: | - #TODO: delete this bash - ls -al /tmp/output/ - echo "show generateAndBuild-task-result.json" - cat /tmp/output/generateAndBuild-task-result.json | echo "don't quite even failed" - displayName: Check output dir - condition: always() - - - bash: | - #TODO: delete this bash - echo "will show git info" - git status - displayName: Show git info - workingDirectory: $(SDK_REPO) - - bash: | echo "will create PR" git config --global user.email "$(GIT_USER_EMAIL)" @@ -83,7 +68,6 @@ steps: fi git checkout -b ${branchName} pushSdkRepoUrl=$(echo $(sdkRepoUrl) | sed "s#https://#https://$(REPO_OWNER):$(azuresdk-github-pat)@#") - #TODO: push PR to Azure if is release to Azure no Azure SDK pushSdkRepoUrl=$(echo ${pushSdkRepoUrl} | sed "s#$(SDK_OWNER)#$(SDK_PUSH_OWNER)#") git push ${pushSdkRepoUrl} --force echo "git push finished:$?" @@ -138,7 +122,7 @@ steps: "/tmp/output/generateAndBuild-task-result.json", "/tmp/output/mockTest-task-result.json"]' - #TODO: delete next line + #check parse result and completion status cat /tmp/output/generateAndBuild-task-result.json cat $(System.ArtifactsDirectory)/completed/result.json @@ -152,7 +136,7 @@ steps: echo ${newtaskresult} > $(System.ArtifactsDirectory)/completed/result.json fi - #TODO: delete next line + #recheck jq result cat $(System.ArtifactsDirectory)/completed/result.json displayName: Prepare artifact workingDirectory: $(SDK_REPO) From 1e6bd6b930d13bcc388a711b15198d2812e45772 Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Tue, 9 Aug 2022 15:21:42 +0800 Subject: [PATCH 24/30] remove hard code branch Signed-off-by: Zhou Zheng --- .../pipelines/steps/precheck.yml | 12 +----------- .../sdk-generation-pipeline.yml | 2 +- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml b/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml index 12908cd75ea..8f932cc9de2 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml @@ -26,15 +26,5 @@ steps: - bash: | echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/$(SDK_OWNER)/azure-sdk-for-${{parameters.sdk}}.git" - if [ ${{parameters.sdk}} = "js" ]; then - echo "##vso[task.setvariable variable=sdkMainBranch;]js-dpg" - elif [ ${{parameters.sdk}} = "net" ]; then - echo "##vso[task.setvariable variable=sdkMainBranch;]yc/dpgauto" - elif [ ${{parameters.sdk}} = "python" ]; then - echo "##vso[task.setvariable variable=sdkMainBranch;]dpg-readme-sdkGeneration" - elif [ ${{parameters.sdk}} = "java" ]; then - echo "##vso[task.setvariable variable=sdkMainBranch;]dpg-readme-codegen" - else - echo "##vso[task.setvariable variable=sdkMainBranch;]$(sdkRepoBaseBranch)" - fi + echo "##vso[task.setvariable variable=sdkMainBranch;]$(sdkRepoBaseBranch)" displayName: Set sdk repo diff --git a/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml b/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml index 67d1ffa8842..672916b17c7 100644 --- a/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml +++ b/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml @@ -7,7 +7,7 @@ variables: value: https://github.com/Azure/azure-sdk-tools.git - name: sdkRepoBaseBranch - value: main + value: master - name: sdkRepoType value: github From c2ffc613e8b8e920ef63566abc21296e0b47d4d8 Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Tue, 9 Aug 2022 16:08:01 +0800 Subject: [PATCH 25/30] roll back for debug Signed-off-by: Zhou Zheng --- .../pipelines/steps/precheck.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml b/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml index 8f932cc9de2..0b94cdd5737 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml @@ -26,5 +26,16 @@ steps: - bash: | echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/$(SDK_OWNER)/azure-sdk-for-${{parameters.sdk}}.git" - echo "##vso[task.setvariable variable=sdkMainBranch;]$(sdkRepoBaseBranch)" + #echo "##vso[task.setvariable variable=sdkMainBranch;]$(sdkRepoBaseBranch)" + if [ ${{parameters.sdk}} = "js" ]; then + echo "##vso[task.setvariable variable=sdkMainBranch;]js-dpg" + elif [ ${{parameters.sdk}} = "net" ]; then + echo "##vso[task.setvariable variable=sdkMainBranch;]yc/dpgauto" + elif [ ${{parameters.sdk}} = "python" ]; then + echo "##vso[task.setvariable variable=sdkMainBranch;]dpg-readme-sdkGeneration" + elif [ ${{parameters.sdk}} = "java" ]; then + echo "##vso[task.setvariable variable=sdkMainBranch;]dpg-readme-codegen" + else + echo "##vso[task.setvariable variable=sdkMainBranch;]$(sdkRepoBaseBranch)" + fi displayName: Set sdk repo From 9eda0677d5d3dc493faf3a80afcc9ac73fbd9dd5 Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Tue, 9 Aug 2022 17:00:31 +0800 Subject: [PATCH 26/30] test Signed-off-by: Zhou Zheng --- .../pipelines/steps/precheck.yml | 24 +++++++++---------- .../sdk-generation-pipeline.yml | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml b/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml index 0b94cdd5737..af96469103c 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml @@ -26,16 +26,16 @@ steps: - bash: | echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/$(SDK_OWNER)/azure-sdk-for-${{parameters.sdk}}.git" - #echo "##vso[task.setvariable variable=sdkMainBranch;]$(sdkRepoBaseBranch)" - if [ ${{parameters.sdk}} = "js" ]; then - echo "##vso[task.setvariable variable=sdkMainBranch;]js-dpg" - elif [ ${{parameters.sdk}} = "net" ]; then - echo "##vso[task.setvariable variable=sdkMainBranch;]yc/dpgauto" - elif [ ${{parameters.sdk}} = "python" ]; then - echo "##vso[task.setvariable variable=sdkMainBranch;]dpg-readme-sdkGeneration" - elif [ ${{parameters.sdk}} = "java" ]; then - echo "##vso[task.setvariable variable=sdkMainBranch;]dpg-readme-codegen" - else - echo "##vso[task.setvariable variable=sdkMainBranch;]$(sdkRepoBaseBranch)" - fi + echo "##vso[task.setvariable variable=sdkMainBranch;]$(sdkRepoBaseBranch)" + #if [ ${{parameters.sdk}} = "js" ]; then + # echo "##vso[task.setvariable variable=sdkMainBranch;]js-dpg" + #elif [ ${{parameters.sdk}} = "net" ]; then + # echo "##vso[task.setvariable variable=sdkMainBranch;]yc/dpgauto" + #elif [ ${{parameters.sdk}} = "python" ]; then + # echo "##vso[task.setvariable variable=sdkMainBranch;]dpg-readme-sdkGeneration" + #elif [ ${{parameters.sdk}} = "java" ]; then + # echo "##vso[task.setvariable variable=sdkMainBranch;]dpg-readme-codegen" + #else + # echo "##vso[task.setvariable variable=sdkMainBranch;]$(sdkRepoBaseBranch)" + #fi displayName: Set sdk repo diff --git a/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml b/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml index 672916b17c7..67d1ffa8842 100644 --- a/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml +++ b/tools/sdk-generation-pipeline/sdk-generation-pipeline.yml @@ -7,7 +7,7 @@ variables: value: https://github.com/Azure/azure-sdk-tools.git - name: sdkRepoBaseBranch - value: master + value: main - name: sdkRepoType value: github From 5b3fd875ac4e0e7e97cd25831053d4da7d556225 Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Wed, 10 Aug 2022 11:01:18 +0800 Subject: [PATCH 27/30] set basedir for git wapper Signed-off-by: Zhou Zheng --- .../src/cli/pipelineCli/prepareArtifactFilesCli.ts | 3 ++- .../pipelines/steps/precheck.yml | 11 ----------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts index 9bbaa71f159..980ab093072 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts @@ -41,7 +41,6 @@ async function prepareSourceCode( language: string, artifactDir: string ) { - const gitOperationWrapper: GitOperationWrapper = new GitOperationWrapper(); for (const p of generateAndBuildOutput.packages) { const packageName = p.packageName; const packagePaths = p.path; @@ -53,6 +52,8 @@ async function prepareSourceCode( } if (fs.lstatSync(packagePath).isDirectory()) { + const gitOperationWrapper: GitOperationWrapper = new GitOperationWrapper(packagePath); + for (const filePath of await gitOperationWrapper.getFileListInPackageFolder(packagePath)) { copyFile( `${path.join(packagePath, filePath)}`, diff --git a/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml b/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml index af96469103c..8f932cc9de2 100644 --- a/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml +++ b/tools/sdk-generation-pipeline/pipelines/steps/precheck.yml @@ -27,15 +27,4 @@ steps: - bash: | echo "##vso[task.setvariable variable=sdkRepoUrl;]https://github.com/$(SDK_OWNER)/azure-sdk-for-${{parameters.sdk}}.git" echo "##vso[task.setvariable variable=sdkMainBranch;]$(sdkRepoBaseBranch)" - #if [ ${{parameters.sdk}} = "js" ]; then - # echo "##vso[task.setvariable variable=sdkMainBranch;]js-dpg" - #elif [ ${{parameters.sdk}} = "net" ]; then - # echo "##vso[task.setvariable variable=sdkMainBranch;]yc/dpgauto" - #elif [ ${{parameters.sdk}} = "python" ]; then - # echo "##vso[task.setvariable variable=sdkMainBranch;]dpg-readme-sdkGeneration" - #elif [ ${{parameters.sdk}} = "java" ]; then - # echo "##vso[task.setvariable variable=sdkMainBranch;]dpg-readme-codegen" - #else - # echo "##vso[task.setvariable variable=sdkMainBranch;]$(sdkRepoBaseBranch)" - #fi displayName: Set sdk repo From a7a8ad85393f09ffa4f1e2255684cd7ed24c56fe Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Wed, 10 Aug 2022 14:26:25 +0800 Subject: [PATCH 28/30] lint fix Signed-off-by: Zhou Zheng --- .../pipelineCli/prepareArtifactFilesCli.ts | 53 ++++++----- .../src/cli/pipelineCli/publishResultCli.ts | 90 +++++++++---------- .../src/cliSchema/generateResultCliConfig.ts | 18 ++-- .../prepareArtifactFilesCliConfig.ts | 20 ++--- .../src/lib/ResultPublisher.ts | 8 +- .../src/lib/generateResult.ts | 2 +- .../src/types/taskResult.ts | 16 ++-- 7 files changed, 103 insertions(+), 104 deletions(-) diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts index 980ab093072..46b4b0a00e6 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/prepareArtifactFilesCli.ts @@ -12,8 +12,7 @@ import { SDK, SDKPipelineStatus, TaskResult, - TaskResultStatus, - Trigger, + Trigger } from '@azure-tools/sdk-generation-lib'; import * as fs from 'fs'; import * as path from 'path'; @@ -21,8 +20,8 @@ import * as path from 'path'; import { PrepareArtifactFilesInput, prepareArtifactFilesInput, - prepareResultArtifactInput, PrepareResultArtifactInput, + prepareResultArtifactInput } from '../../cliSchema/prepareArtifactFilesCliConfig'; import { GitOperationWrapper } from '../../utils/GitOperationWrapper'; @@ -136,34 +135,34 @@ function prepareResult(pipelineStatus: SDKPipelineStatus) { let event: PipelineRunEvent = undefined; switch (pipelineStatus) { - case 'in_progress': - event = { - status: 'in_progress', - trigger: trigger, - pipelineBuildId: config.pipelineBuildId, - } as InProgressEvent; - break; - case 'completed': - if (!config.resultsPath || !config.logPath) { - throw new Error(`Invalid completed event parameter!`); - } + case 'in_progress': + event = { + status: 'in_progress', + trigger: trigger, + pipelineBuildId: config.pipelineBuildId + } as InProgressEvent; + break; + case 'completed': + if (!config.resultsPath || !config.logPath) { + throw new Error(`Invalid completed event parameter!`); + } - const taskResults: TaskResult[] = getTaskResults(config.resultsPath); - const taskTotalResult: TaskResult = generateTotalResult(taskResults, config.pipelineBuildId); - event = { - status: 'completed', - trigger: trigger, - pipelineBuildId: config.pipelineBuildId, - logPath: config.logPath, - result: taskTotalResult, - } as CompletedEvent; - break; - default: - throw new Error(`Unsupported status: ` + (pipelineStatus as string)); + const taskResults: TaskResult[] = getTaskResults(config.resultsPath); + const taskTotalResult: TaskResult = generateTotalResult(taskResults, config.pipelineBuildId); + event = { + status: 'completed', + trigger: trigger, + pipelineBuildId: config.pipelineBuildId, + logPath: config.logPath, + result: taskTotalResult + } as CompletedEvent; + break; + default: + throw new Error(`Unsupported status: ` + (pipelineStatus as string)); } fs.writeFileSync(config.artifactDir + `/` + pipelineStatus + `/result.json`, JSON.stringify(event, null, 2), { - encoding: 'utf-8', + encoding: 'utf-8' }); } diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/publishResultCli.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/publishResultCli.ts index 4d84f28f21b..b9583a15788 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/publishResultCli.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cli/pipelineCli/publishResultCli.ts @@ -18,7 +18,7 @@ import { SDKPipelineStatus, StorageType, TaskResult, - Trigger, + Trigger } from '@azure-tools/sdk-generation-lib'; import * as fs from 'fs'; @@ -30,7 +30,7 @@ import { ResultPublisherDBResultInput, resultPublisherDBResultInput, ResultPublisherEventHubInput, - resultPublisherEventHubInput, + resultPublisherEventHubInput } from '../../cliSchema/publishResultConfig'; async function publishBlob() { @@ -40,7 +40,7 @@ async function publishBlob() { pipelineBuildId: config.pipelineBuildId, sdkGenerationName: config.sdkGenerationName, azureStorageBlobSasUrl: config.azureStorageBlobSasUrl, - azureBlobContainerName: config.azureBlobContainerName, + azureBlobContainerName: config.azureBlobContainerName }; const resultBlobPublisher: ResultBlobPublisher = new ResultBlobPublisher(context); await resultBlobPublisher.uploadLogsAndResult(config.logsAndResultPath, config.taskName as AzureSDKTaskName); @@ -76,7 +76,7 @@ function initMongoConnectContext(config: ResultPublisherDBCodeGenerationInput): database: config.mongodb.database, ssl: config.mongodb.ssl, synchronize: true, - logging: true, + logging: true }; return mongoConnectContext; @@ -132,37 +132,37 @@ async function publishEventhub(pipelineStatus: SDKPipelineStatus) { const publisher: ResultEventhubPublisher = new ResultEventhubPublisher(config.eventHubConnectionString); switch (pipelineStatus) { - case 'queued': - event = { - status: 'queued', - trigger: trigger, - pipelineBuildId: config.pipelineBuildId, - } as QueuedEvent; - break; - case 'in_progress': - event = { - status: 'in_progress', - trigger: trigger, - pipelineBuildId: config.pipelineBuildId, - } as InProgressEvent; - break; - case 'completed': - if (!config.resultsPath || !config.logPath) { - throw new Error(`Invalid completed event parameter!`); - } + case 'queued': + event = { + status: 'queued', + trigger: trigger, + pipelineBuildId: config.pipelineBuildId + } as QueuedEvent; + break; + case 'in_progress': + event = { + status: 'in_progress', + trigger: trigger, + pipelineBuildId: config.pipelineBuildId + } as InProgressEvent; + break; + case 'completed': + if (!config.resultsPath || !config.logPath) { + throw new Error(`Invalid completed event parameter!`); + } - const taskResults: TaskResult[] = getTaskResults(config.resultsPath); - const taskTotalResult: TaskResult = generateTotalResult(taskResults, config.pipelineBuildId); - event = { - status: 'completed', - trigger: trigger, - pipelineBuildId: config.pipelineBuildId, - logPath: config.logPath, - result: taskTotalResult, - } as CompletedEvent; - break; - default: - throw new Error(`Unsupported status: ` + (pipelineStatus as string)); + const taskResults: TaskResult[] = getTaskResults(config.resultsPath); + const taskTotalResult: TaskResult = generateTotalResult(taskResults, config.pipelineBuildId); + event = { + status: 'completed', + trigger: trigger, + pipelineBuildId: config.pipelineBuildId, + logPath: config.logPath, + result: taskTotalResult + } as CompletedEvent; + break; + default: + throw new Error(`Unsupported status: ` + (pipelineStatus as string)); } await publisher.publishEvent(event); await publisher.close(); @@ -174,17 +174,17 @@ async function main() { const pipelineStatus = args['pipelineStatus']; switch (storageType as StorageType) { - case StorageType.Blob: - await publishBlob(); - break; - case StorageType.Db: - await publishDB(pipelineStatus); - break; - case StorageType.EventHub: - await publishEventhub(pipelineStatus); - break; - default: - throw new Error(`Unknown storageType:${storageType}!`); + case StorageType.Blob: + await publishBlob(); + break; + case StorageType.Db: + await publishDB(pipelineStatus); + break; + case StorageType.EventHub: + await publishEventhub(pipelineStatus); + break; + default: + throw new Error(`Unknown storageType:${storageType}!`); } } diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/generateResultCliConfig.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/generateResultCliConfig.ts index 336721672d5..6cbb3ff901c 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/generateResultCliConfig.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/generateResultCliConfig.ts @@ -17,45 +17,45 @@ export const generateResultCliInput = convict({ pipelineBuildId: { default: null, format: assertNullOrEmpty, - arg: 'buildId', + arg: 'buildId' }, logfile: { default: null, format: assertNullOrEmpty, - arg: 'logfile', + arg: 'logfile' }, logFilterStr: { default: null, nullable: true, format: String, - arg: 'logFilterStr', + arg: 'logFilterStr' }, taskName: { default: null, format: ['init', 'generateAndBuild', 'mockTest', 'liveTest'], - arg: 'taskName', + arg: 'taskName' }, exeResult: { default: null, nullable: true, format: ['succeeded', 'failed'], - arg: 'exeResult', + arg: 'exeResult' }, taskOutputPath: { default: null, nullable: true, format: String, - arg: 'taskOutputPath', + arg: 'taskOutputPath' }, resultOutputPath: { default: null, format: assertNullOrEmpty, - arg: 'resultOutputPath', + arg: 'resultOutputPath' }, dockerResultFile: { default: null, nullable: true, format: String, - arg: 'dockerResultFile', - }, + arg: 'dockerResultFile' + } }); diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/prepareArtifactFilesCliConfig.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/prepareArtifactFilesCliConfig.ts index 2ad5bff4353..018fc478a0a 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/prepareArtifactFilesCliConfig.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/src/cliSchema/prepareArtifactFilesCliConfig.ts @@ -12,19 +12,19 @@ export const prepareArtifactFilesInput = convict({ generateAndBuildOutputFile: { default: null, format: assertNullOrEmpty, - arg: 'generateAndBuildOutputFile', + arg: 'generateAndBuildOutputFile' }, artifactDir: { doc: 'The dir to publish artifact', default: null, format: assertNullOrEmpty, - arg: 'artifactDir', + arg: 'artifactDir' }, language: { default: null, format: ['js', 'python', 'go', 'net', 'java'], - arg: 'language', - }, + arg: 'language' + } }); export class PrepareResultArtifactInput { @@ -39,30 +39,30 @@ export const prepareResultArtifactInput = convict({ pipelineBuildId: { default: null, format: assertNullOrEmpty, - arg: 'buildId', + arg: 'buildId' }, trigger: { default: null, format: assertNullOrEmpty, - arg: 'trigger', + arg: 'trigger' }, artifactDir: { doc: 'The dir to publish artifact', default: null, format: assertNullOrEmpty, - arg: 'artifactDir', + arg: 'artifactDir' }, logPath: { default: null, nullable: true, format: String, - arg: 'logPath', + arg: 'logPath' }, resultsPath: { doc: 'task result files array', default: null, nullable: true, format: String, - arg: 'resultsPath', - }, + arg: 'resultsPath' + } }); diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/lib/ResultPublisher.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/lib/ResultPublisher.ts index 99a9d0c8cff..c4403cb0f0e 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/lib/ResultPublisher.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/lib/ResultPublisher.ts @@ -50,7 +50,7 @@ export class ResultDBPublisher { ssl: this.context.ssl, synchronize: this.context.synchronize, logging: this.context.logging, - entities: [TaskResultEntity, CodeGeneration], + entities: [TaskResultEntity, CodeGeneration] }); } @@ -101,9 +101,9 @@ export class ResultBlobPublisher { const logsAndResultPathArray = JSON.parse(logsAndResultPath); for (const file of logsAndResultPathArray) { if (fs.existsSync(file)) { - const blobName: string = file.includes('.json') - ? `${this.pipelineBuildId}/logs/${this.sdkGenerationName}-${taskName}-result.json` - : `${this.pipelineBuildId}/logs/${this.sdkGenerationName}-${taskName}.log`; + const blobName: string = file.includes('.json') ? + `${this.pipelineBuildId}/logs/${this.sdkGenerationName}-${taskName}-result.json` : + `${this.pipelineBuildId}/logs/${this.sdkGenerationName}-${taskName}.log`; await this.azureBlobClient.publishBlob(file, blobName); logger.info(`Publish ${file} Success !!!`); } else { diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/lib/generateResult.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/lib/generateResult.ts index 2e6312aa498..6ce10c17beb 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/lib/generateResult.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/lib/generateResult.ts @@ -88,7 +88,7 @@ export function parseGenerateLog( errorCount: errorNum, warningCount: warnNum, messages: messages, - result: taskExeResult, + result: taskExeResult }; return result; diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/types/taskResult.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/types/taskResult.ts index fd5f1f28917..26ec9333e2e 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/types/taskResult.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-lib/src/types/taskResult.ts @@ -1,23 +1,23 @@ import * as fs from 'fs'; import { Column, Entity, ObjectIdColumn } from 'typeorm'; +import { logger } from '../utils/logger'; +import { requireJsonc } from '../utils/requireJsonc'; import { getTaskBasicConfig, TaskBasicConfig } from './taskBasicConfig'; import { GenerateAndBuildOutput } from './taskInputAndOuputSchemaTypes/GenerateAndBuildOutput'; import { InitOutput } from './taskInputAndOuputSchemaTypes/InitOutput'; -import { logger } from '../utils/logger'; -import { requireJsonc } from '../utils/requireJsonc'; import { TestOutput } from './taskInputAndOuputSchemaTypes/TestOutput'; @Entity('sdkGenerationResults') export class TaskResultEntity { @ObjectIdColumn() - id: string; + id: string; @Column() - key: string; + key: string; @Column() - pipelineBuildId: string; + pipelineBuildId: string; @Column() - taskResult: TaskResult; + taskResult: TaskResult; } export enum TaskResultStatus { @@ -106,7 +106,7 @@ export function setTaskResult(config: TaskBasicConfig, taskName: string) { pipelineBuildId: '', result: TaskResultStatus.Success, errorCount: 0, - warningCount: 0, + warningCount: 0 }; } @@ -124,7 +124,7 @@ export function generateTotalResult(taskResults: TaskResult[], pipelineBuildId: pipelineBuildId: pipelineBuildId, result: TaskResultStatus.Success, errorCount: 0, - messages: [], + messages: [] }; if (taskResults.length === 0) { From f8b958a5a03c98e6bccccd1f8b63e9ec954ccddd Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Wed, 10 Aug 2022 15:41:19 +0800 Subject: [PATCH 29/30] update js ci test repo commitid Signed-off-by: Zhou Zheng --- .../test/integration/integrationTest.ts | 55 ++++++++++++------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/test/integration/integrationTest.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/test/integration/integrationTest.ts index eb84c811556..160540e66eb 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/test/integration/integrationTest.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/test/integration/integrationTest.ts @@ -6,11 +6,11 @@ import * as process from 'process'; const repoCommitId = { 'azure-rest-api-specs': '0baca05c851c1749e92beb0d2134cd958827dd54', - 'azure-sdk-for-js': '67946c5b0ce135f58ecfeab1443e5be52604908e', + 'azure-sdk-for-js': '57382229a700e0e6f607d6ac0811379a6254f3d9', 'azure-sdk-for-java': '307df24267304fbf3947025bef7eaf9698410de8', 'azure-sdk-for-python': '53f66170cc47739204cedfe0a46989290c047c98', 'azure-sdk-for-go': '241bdb849ce431e1a5e398a5649cde93149ee374', - 'azure-sdk-for-net': 'e9db0733a642d50c34101339f74fdc487599d824' + 'azure-sdk-for-net': 'e9db0733a642d50c34101339f74fdc487599d824', }; const defaultImageName = 'sdkgeneration.azurecr.io/sdk-generation:v1.0'; @@ -25,31 +25,36 @@ async function prepareRepo(currentPath: string, repoName: string) { if (!existsSync(path.join(tmpFolder, repoName))) { execSync(`git clone https://github.com/Azure/${repoName}.git`, { cwd: tmpFolder, - stdio: 'inherit' + stdio: 'inherit', }); } execSync(`git restore --staged . && git restore . && git checkout . && git clean -fd`, { cwd: path.join(tmpFolder, repoName), - stdio: 'inherit' + stdio: 'inherit', }); - if (!!repoCommitId[repoName] && execSync(`git rev-parse HEAD`, { - encoding: 'utf-8', - cwd: path.join(tmpFolder, repoName) - }).trim() !== repoCommitId[repoName]) { + if ( + !!repoCommitId[repoName] && + execSync(`git rev-parse HEAD`, { + encoding: 'utf-8', + cwd: path.join(tmpFolder, repoName), + }).trim() !== repoCommitId[repoName] + ) { execSync(`git checkout ${repoCommitId[repoName]}`, { cwd: path.join(tmpFolder, repoName), - stdio: 'inherit' + stdio: 'inherit', }); } - if (execSync(`git rev-parse --abbrev-ref HEAD`, { - encoding: 'utf-8', - cwd: path.join(tmpFolder, repoName) - }).trim() !== integrationBranch) { + if ( + execSync(`git rev-parse --abbrev-ref HEAD`, { + encoding: 'utf-8', + cwd: path.join(tmpFolder, repoName), + }).trim() !== integrationBranch + ) { execSync(`git switch -c ${integrationBranch}`, { cwd: path.join(tmpFolder, repoName), - stdio: 'inherit' + stdio: 'inherit', }); } } @@ -57,19 +62,25 @@ async function prepareRepo(currentPath: string, repoName: string) { async function runDocker(currentPath: string, sdkRepoName: string, dockerImage: string) { const tmpFolder = path.join(currentPath, 'tmp'); // eslint-disable-next-line max-len - execSync(`docker run -v ${path.join(tmpFolder, 'azure-rest-api-specs')}:/spec-repo -v ${path.join(tmpFolder, sdkRepoName)}:/sdk-repo ${dockerImage} --readme=specification/agrifood/resource-manager/readme.md`, { - stdio: 'inherit' - }); + execSync( + `docker run -v ${path.join(tmpFolder, 'azure-rest-api-specs')}:/spec-repo -v ${path.join( + tmpFolder, + sdkRepoName + )}:/sdk-repo ${dockerImage} --readme=specification/agrifood/resource-manager/readme.md`, + { + stdio: 'inherit', + } + ); } async function buildDockImage(rushCwd: string, dockerCwd: string) { execSync(`rushx pack`, { cwd: rushCwd, - stdio: 'inherit' + stdio: 'inherit', }); execSync(`docker build -t ${defaultImageName} .`, { cwd: dockerCwd, - stdio: 'inherit' + stdio: 'inherit', }); } @@ -80,7 +91,9 @@ export async function main(options: any) { options['docker-image'] = defaultImageName; } if (!options['sdk-repo']) { - options['sdk-repo'] = Object.keys(repoCommitId).filter((ele) => ele !== 'azure-rest-api-specs').join(','); + options['sdk-repo'] = Object.keys(repoCommitId) + .filter((ele) => ele !== 'azure-rest-api-specs') + .join(','); } await prepareRepo(currentPath, 'azure-rest-api-specs'); for (const sdkRepo of options['sdk-repo'].split(',')) { @@ -91,7 +104,7 @@ export async function main(options: any) { const optionDefinitions = [ { name: 'docker-image', type: String }, - { name: 'sdk-repo', type: String } + { name: 'sdk-repo', type: String }, ]; const options = commandLineArgs(optionDefinitions); From ab7adde9bf6159cb77bdf11b4dd74b54c9800e57 Mon Sep 17 00:00:00 2001 From: Zhou Zheng Date: Wed, 10 Aug 2022 16:03:36 +0800 Subject: [PATCH 30/30] lint fix --- .../test/integration/integrationTest.ts | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/test/integration/integrationTest.ts b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/test/integration/integrationTest.ts index 160540e66eb..79cb3e652b1 100644 --- a/tools/sdk-generation-pipeline/packages/sdk-generation-cli/test/integration/integrationTest.ts +++ b/tools/sdk-generation-pipeline/packages/sdk-generation-cli/test/integration/integrationTest.ts @@ -10,7 +10,7 @@ const repoCommitId = { 'azure-sdk-for-java': '307df24267304fbf3947025bef7eaf9698410de8', 'azure-sdk-for-python': '53f66170cc47739204cedfe0a46989290c047c98', 'azure-sdk-for-go': '241bdb849ce431e1a5e398a5649cde93149ee374', - 'azure-sdk-for-net': 'e9db0733a642d50c34101339f74fdc487599d824', + 'azure-sdk-for-net': 'e9db0733a642d50c34101339f74fdc487599d824' }; const defaultImageName = 'sdkgeneration.azurecr.io/sdk-generation:v1.0'; @@ -25,36 +25,36 @@ async function prepareRepo(currentPath: string, repoName: string) { if (!existsSync(path.join(tmpFolder, repoName))) { execSync(`git clone https://github.com/Azure/${repoName}.git`, { cwd: tmpFolder, - stdio: 'inherit', + stdio: 'inherit' }); } execSync(`git restore --staged . && git restore . && git checkout . && git clean -fd`, { cwd: path.join(tmpFolder, repoName), - stdio: 'inherit', + stdio: 'inherit' }); if ( !!repoCommitId[repoName] && execSync(`git rev-parse HEAD`, { encoding: 'utf-8', - cwd: path.join(tmpFolder, repoName), + cwd: path.join(tmpFolder, repoName) }).trim() !== repoCommitId[repoName] ) { execSync(`git checkout ${repoCommitId[repoName]}`, { cwd: path.join(tmpFolder, repoName), - stdio: 'inherit', + stdio: 'inherit' }); } if ( execSync(`git rev-parse --abbrev-ref HEAD`, { encoding: 'utf-8', - cwd: path.join(tmpFolder, repoName), + cwd: path.join(tmpFolder, repoName) }).trim() !== integrationBranch ) { execSync(`git switch -c ${integrationBranch}`, { cwd: path.join(tmpFolder, repoName), - stdio: 'inherit', + stdio: 'inherit' }); } } @@ -68,7 +68,7 @@ async function runDocker(currentPath: string, sdkRepoName: string, dockerImage: sdkRepoName )}:/sdk-repo ${dockerImage} --readme=specification/agrifood/resource-manager/readme.md`, { - stdio: 'inherit', + stdio: 'inherit' } ); } @@ -76,11 +76,11 @@ async function runDocker(currentPath: string, sdkRepoName: string, dockerImage: async function buildDockImage(rushCwd: string, dockerCwd: string) { execSync(`rushx pack`, { cwd: rushCwd, - stdio: 'inherit', + stdio: 'inherit' }); execSync(`docker build -t ${defaultImageName} .`, { cwd: dockerCwd, - stdio: 'inherit', + stdio: 'inherit' }); } @@ -104,7 +104,7 @@ export async function main(options: any) { const optionDefinitions = [ { name: 'docker-image', type: String }, - { name: 'sdk-repo', type: String }, + { name: 'sdk-repo', type: String } ]; const options = commandLineArgs(optionDefinitions);