From 65bfd2dc717ab0c04d9664795fa3fe8c005c1cfb Mon Sep 17 00:00:00 2001 From: Brenna Hewer-Darroch <21015366+bcmdarroch@users.noreply.github.com> Date: Mon, 6 Feb 2023 18:08:23 -0500 Subject: [PATCH 01/10] Delete publish-sdk.yml --- .github/workflows/publish-sdk.yml | 85 ------------------------------- 1 file changed, 85 deletions(-) delete mode 100644 .github/workflows/publish-sdk.yml diff --git a/.github/workflows/publish-sdk.yml b/.github/workflows/publish-sdk.yml deleted file mode 100644 index 95106202..00000000 --- a/.github/workflows/publish-sdk.yml +++ /dev/null @@ -1,85 +0,0 @@ ---- -name: publish-sdk - -on: - workflow_dispatch: - inputs: - service: - description: 'The service whose SDK needs to be generated' - required: true - -jobs: - publish-sdk: - runs-on: ubuntu-latest - steps: - - name: Set env service branch - run: echo "SERVICE_BRANCH=auto-update-${{ github.event.inputs.service }}-sdk" >> $GITHUB_ENV - - - name: Checkout hcp-sdk-go - uses: actions/checkout@v2 - with: - ref: main - path: hcp-sdk-go - - - name: Checkout cloud-api - uses: actions/checkout@v2 - with: - repository: hashicorp/cloud-api - ref: auto-update-${{ github.event.inputs.service }}-specs - token: ${{ secrets.HCP_SDK_PIPELINE_TOKEN }} - path: cloud-api - - - name: Install go - uses: actions/setup-go@v2 - with: - go-version-file: 'go.mod' - - - name: Install dependencies - run: | - version=0.30.2 - goos=$(go env GOOS) - goarch="$(go env GOARCH)" - download_url="https://github.com/go-swagger/go-swagger/releases/download/v${version}/swagger_${goos}_${goarch}" - curl -o /usr/local/bin/swagger -L'#' "$download_url" - chmod +x /usr/local/bin/swagger - sudo apt-get update -y - sudo apt-get install -y rsync - - - name: Copy latest service specs - env: - SERVICE: ${{ github.event.inputs.service }} - run: | - rsync -a $GITHUB_WORKSPACE/cloud-api/specs/"$SERVICE" $GITHUB_WORKSPACE/hcp-sdk-go/temp - rsync -a $GITHUB_WORKSPACE/cloud-api/specs/cloud-shared $GITHUB_WORKSPACE/hcp-sdk-go/temp - rsync -a $GITHUB_WORKSPACE/cloud-api/specs/external $GITHUB_WORKSPACE/hcp-sdk-go/temp - - - name: Generate SDK for service - env: - SERVICE: ${{ github.event.inputs.service }} - run: | - cd $GITHUB_WORKSPACE/hcp-sdk-go - ./scripts/gen-go-service-sdk.sh $SERVICE - rm -rf $GITHUB_WORKSPACE/hcp-sdk-go/temp - - - name: Create branch with updated service SDK - run: | - cd $GITHUB_WORKSPACE/hcp-sdk-go - git config user.name "HashiCorp Cloud Services" - git config user.email "59096967+hashicorp-cloud@users.noreply.github.com" - service_branch_exists="$(git ls-remote --heads origin ${{ env.SERVICE_BRANCH }})" - [[ -n $service_branch_exists ]] && git push origin --delete ${{ env.SERVICE_BRANCH }} - git checkout -b ${{ env.SERVICE_BRANCH }} - git add clients/${{ github.event.inputs.service }}/* - git commit -m "Update ${{ github.event.inputs.service }} SDK" - git push --set-upstream origin ${{ env.SERVICE_BRANCH }} - - - name: Open PR - run: | - cd $GITHUB_WORKSPACE/hcp-sdk-go - gh pr create --title "$PR_TITLE" --body "$PR_BODY" -H "$PR_SOURCE" -B "$PR_TARGET" - env: - PR_TITLE: "[auto] Update ${{ github.event.inputs.service }} SDK" - PR_BODY: "This is an auto-generated PR created as part of the public SDK pipeline to update the SDK for ${{ github.event.inputs.service }}." - PR_SOURCE: "${{ env.SERVICE_BRANCH }}" - PR_TARGET: "main" - GITHUB_TOKEN: ${{ secrets.HCP_SDK_PIPELINE_TOKEN }} From 1dc2e715e421f7f8d02c4fb02a5c60060a48af9b Mon Sep 17 00:00:00 2001 From: Brenna Hewer-Darroch <21015366+bcmdarroch@users.noreply.github.com> Date: Mon, 6 Feb 2023 18:08:53 -0500 Subject: [PATCH 02/10] Delete publish-shared-sdk.yml --- .github/workflows/publish-shared-sdk.yml | 72 ------------------------ 1 file changed, 72 deletions(-) delete mode 100644 .github/workflows/publish-shared-sdk.yml diff --git a/.github/workflows/publish-shared-sdk.yml b/.github/workflows/publish-shared-sdk.yml deleted file mode 100644 index 6b79f4e9..00000000 --- a/.github/workflows/publish-shared-sdk.yml +++ /dev/null @@ -1,72 +0,0 @@ ---- -name: publish-shared-sdk - -on: workflow_dispatch - -jobs: - publish-shared-sdk: - runs-on: ubuntu-latest - steps: - - name: Checkout hcp-sdk-go - uses: actions/checkout@v2 - with: - ref: main - path: hcp-sdk-go - - - name: Checkout cloud-api - uses: actions/checkout@v2 - with: - repository: hashicorp/cloud-api - ref: master - token: ${{ secrets.HCP_SDK_PIPELINE_TOKEN }} - path: cloud-api - - - name: Install go - uses: actions/setup-go@v2 - with: - go-version: '1.16.0' - - - name: Install dependencies - run: | - version=0.30.2 - goos=$(go env GOOS) - goarch="$(go env GOARCH)" - download_url="https://github.com/go-swagger/go-swagger/releases/download/v${version}/swagger_${goos}_${goarch}" - curl -o /usr/local/bin/swagger -L'#' "$download_url" - chmod +x /usr/local/bin/swagger - sudo apt-get update -y - sudo apt-get install -y rsync - - - name: Copy latest cloud-shared specs - run: | - rsync -a $GITHUB_WORKSPACE/cloud-api/specs/cloud-shared $GITHUB_WORKSPACE/hcp-sdk-go/temp - rsync -a $GITHUB_WORKSPACE/cloud-api/specs/external $GITHUB_WORKSPACE/hcp-sdk-go/temp - - - name: Generate SDK for cloud-shared - run: | - cd $GITHUB_WORKSPACE/hcp-sdk-go - ./scripts/gen-go-shared-sdk.sh - rm -rf $GITHUB_WORKSPACE/hcp-sdk-go/temp - - - name: Create branch with updated service SDK - run: | - cd $GITHUB_WORKSPACE/hcp-sdk-go - git config user.name "HashiCorp Cloud Services" - git config user.email "${{ secrets.HCP_SERVICE_ACCOUNT_EMAIL }}" - service_branch_exists="$(git ls-remote --heads origin auto-update-cloud-shared-sdk)" - [[ -n $service_branch_exists ]] && git push origin --delete auto-update-cloud-shared-sdk - git checkout -b auto-update-cloud-shared-sdk - git add clients/cloud-shared/* - git commit -m "Update cloud-shared SDK" - git push --set-upstream origin auto-update-cloud-shared-sdk - - - name: Open PR - run: | - cd $GITHUB_WORKSPACE/hcp-sdk-go - gh pr create --title "$PR_TITLE" --body "$PR_BODY" -H "$PR_SOURCE" -B "$PR_TARGET" - env: - PR_TITLE: "[auto] Update Shared SDK" - PR_BODY: "This is an auto-generated PR created as part of the public SDK pipeline to update cloud-shared clients." - PR_SOURCE: "auto-update-cloud-shared-sdk" - PR_TARGET: "main" - GITHUB_TOKEN: ${{ secrets.HCP_SDK_PIPELINE_TOKEN }} From a37bd6a7f861810782fcbde21faf8c829c529a9f Mon Sep 17 00:00:00 2001 From: Brenna Hewer-Darroch <21015366+bcmdarroch@users.noreply.github.com> Date: Tue, 7 Feb 2023 18:16:15 -0500 Subject: [PATCH 03/10] Delete gen-go-service-sdk.sh --- scripts/gen-go-service-sdk.sh | 101 ---------------------------------- 1 file changed, 101 deletions(-) delete mode 100755 scripts/gen-go-service-sdk.sh diff --git a/scripts/gen-go-service-sdk.sh b/scripts/gen-go-service-sdk.sh deleted file mode 100755 index 705092ef..00000000 --- a/scripts/gen-go-service-sdk.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: MPL-2.0 - -set -euo pipefail - -# This script regenerates the Go SDK for a given HCP service ($1). -# It depends on pull-specs.sh to save specs in a temporary directory. - -# The steps are: -# 1. Remove the original SDK files for the service if they exist. -# 2. Run temporary transformations on those specs to prepare them for SDK generation. -# 3. Iterate over each stage and version of the service specs and generate the corresponding SDK. -# 4. Remove temporary directories. - -SCRIPTS_DIR=$(dirname "${BASH_SOURCE[0]}") - -BOLD='\033[1m' -GREEN='\033[32m' -NA='\033[0m' # no attributes (color or format) - -generate_sdk() { - service=$1 - stage=$2 - version=$3 - - echo -e "Creating target SDK directory: ${BOLD}hcp-sdk-go/clients/$service/$stage/$version${NA}" - mkdir -p "../../../clients/$service/$stage/$version" - - rel=("$version"/*.swagger.json) - spec="${rel[0]}" - target="../../../clients/$service/$stage/$version" - - echo -e "Generating SDK for ${BOLD}$service${NA} (stage: ${BOLD}$stage${NA}, version: ${BOLD}$version${NA})" - swagger generate client \ - -f "$spec" \ - -t "$target" \ - -q \ - -A "$service" -} - -# Beginning of generation script. -service=$1 - -if [ -d "clients/$service" ]; then \ - echo "Removing original SDK from clients/$service" && rm -rf "$SCRIPTS_DIR/../clients/$service"; \ -fi - -transformer=../../../cmd/transform-swagger -shared_specs=../../../temp/cloud-shared -external_spec=../../../temp/external/external.swagger.json - -cd temp/"$service" - -# Iterate over each stage directory. -for d in *; do - if [[ -d "$d" ]]; then - stage=$d - - cd "$stage" - fi - - # Iterate over each version directory. - for f in *; do - if [[ -d "$f" ]]; then - version=$f - service_spec=("$version"/*.swagger.json) - - # Transform the specs. - echo -e "Transforming specs for ${BOLD}$service${NA} (stage: ${BOLD}$stage${NA}, version: ${BOLD}$version${NA}) in preparation for SDK generation" - go run "$transformer" \ - -service="${service_spec[0]}" \ - -shared="$shared_specs" \ - -external="$external_spec" - - # Generate SDK from transformed specs. - generate_sdk "$service" "$stage" "$version" - fi - done - - cd .. -done - -# Navigate back to root. -cd ../.. - -echo -e "Regenerating shared ${BOLD}external${NA} SDK models" -swagger generate model \ - -f ./temp/external/external.swagger.json \ - -t ./clients/cloud-shared/v1 \ - -q - -echo -e "${GREEN}SDK for $service generated!${NA}" - -cleanup() { - # This is where hcloud clones cloud-api from which the specs are pulled. - rm -rf "$HOME/.local/share/hcp/repos/cloud-api" - rm -rf "$SCRIPTS_DIR"/../temp -} - -trap cleanup EXIT From f039dc356bee0b6c133403fb76bc2837c9fc6247 Mon Sep 17 00:00:00 2001 From: Brenna Hewer-Darroch <21015366+bcmdarroch@users.noreply.github.com> Date: Tue, 7 Feb 2023 18:16:24 -0500 Subject: [PATCH 04/10] Delete gen-go-shared-sdk.sh --- scripts/gen-go-shared-sdk.sh | 70 ------------------------------------ 1 file changed, 70 deletions(-) delete mode 100755 scripts/gen-go-shared-sdk.sh diff --git a/scripts/gen-go-shared-sdk.sh b/scripts/gen-go-shared-sdk.sh deleted file mode 100755 index ad8afc56..00000000 --- a/scripts/gen-go-shared-sdk.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: MPL-2.0 - -set -euo pipefail - -# This script regenerates the shared models for the HCP Go SDK. -# It depends on pull-specs.sh to save specs in a temporary directory. - -# The steps are: -# 1. Remove old shared SDK. -# 2. Iterate over each version and type of shared spec and generate the corresponding SDK. -# 3. Remove temporary directories. - -SCRIPTS_DIR=$(dirname "${BASH_SOURCE[0]}") - -BOLD='\033[1m' -GREEN='\033[32m' -NA='\033[0m' # no attributes (color or format) - -external_spec="$SCRIPTS_DIR"/../temp/external/external.swagger.json - -version="v1" -if [ -d "clients/cloud-shared/$version" ]; then \ - echo "Removing original SDK from cloud-shared/$version" && rm -rf "$SCRIPTS_DIR"/../clients/cloud-shared/"$version"; \ -fi - -echo -e "Creating target shared SDK directory: ${BOLD}hcp-sdk-go/clients/cloud-shared/$version${NA}" -mkdir -p "$SCRIPTS_DIR"/../clients/cloud-shared/"$version" - -cd temp/cloud-shared - -# Iterate over each shared type directory. -for d in *; do - type="$d" - if [[ -d "$type" ]]; then - cd "$type" - for f in *; do - spec=$f - echo -e "Generating shared SDK models (type: ${BOLD}$type${NA}, version: ${BOLD}$version${NA})" - swagger generate model \ - -f "$spec" \ - -t ../../../clients/cloud-shared/"$version" \ - -q - done - fi - - cd .. -done - -# Navigate back to root. -cd ../.. - -pwd - -echo -e "Regenerating shared ${BOLD}external${NA} SDK models" -swagger generate model \ - -f "$external_spec" \ - -t ./clients/cloud-shared/"$version" \ - -q - -echo -e "${GREEN}SDK for cloud-shared generated!${NA}" - -cleanup() { - # This is where hcloud clones cloud-api from which the specs are pulled. - rm -rf "$HOME/.local/share/hcp/repos/cloud-api" - rm -rf "$SCRIPTS_DIR"/../temp -} - -trap cleanup EXIT \ No newline at end of file From 3e39612db16b0658933474f46760fe6da274d8ad Mon Sep 17 00:00:00 2001 From: Brenna Hewer-Darroch <21015366+bcmdarroch@users.noreply.github.com> Date: Tue, 7 Feb 2023 18:16:39 -0500 Subject: [PATCH 05/10] Delete open-pr.sh --- scripts/open-pr.sh | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100755 scripts/open-pr.sh diff --git a/scripts/open-pr.sh b/scripts/open-pr.sh deleted file mode 100755 index d6db61d8..00000000 --- a/scripts/open-pr.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: MPL-2.0 - -set -euo pipefail - -CURRENT_BRANCH="$(git rev-parse --abbrev-ref HEAD)" - -if [ "$CURRENT_BRANCH" != "master" ] && [ "$CURRENT_BRANCH" != "main" ]; then - echo "ERROR: A PR can only be created on a branch of the 'master' or 'main' branch. Please checkout the 'master' or 'main' branch and try again." >&2 - exit -fi - -d=$(date +%Y-%m-%d-%H-%M) -fmtd=$(date +"%D") -service=$1 - -git checkout -b "update-$service-sdk-$d" -git add "clients/$service/*" -git commit -m "updates $service SDK on $fmtd" -git push -u - -gh pr create --title "Updates $service SDK" --body "Updates $service SDK as of $fmtd" - -git co - \ No newline at end of file From d587484992d9ba0ab99063cade0cf08b037419f1 Mon Sep 17 00:00:00 2001 From: Brenna Hewer-Darroch <21015366+bcmdarroch@users.noreply.github.com> Date: Tue, 7 Feb 2023 18:18:44 -0500 Subject: [PATCH 06/10] Delete pull-specs-local.sh --- scripts/pull-specs-local.sh | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100755 scripts/pull-specs-local.sh diff --git a/scripts/pull-specs-local.sh b/scripts/pull-specs-local.sh deleted file mode 100755 index edecb3ef..00000000 --- a/scripts/pull-specs-local.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: MPL-2.0 - -set -euo pipefail - -# This script fetches the latest public API specs for a given HCP service ($1) -# from the central spec repo, cloud-api. - -SCRIPTS_DIR=$(dirname "${BASH_SOURCE[0]}") - -service=$1 - -# Copy the latest service specs into a temporary directory in preparation for SDK generation. -mkdir -p "$SCRIPTS_DIR"/../temp/"$service" -cp -r "$GOPATH"/src/github.com/hashicorp/cloud-api/specs/"$service" "$SCRIPTS_DIR"/../temp -cp -r "$GOPATH"/src/github.com/hashicorp/cloud-api/specs/cloud-shared "$SCRIPTS_DIR"/../temp -cp -r "$GOPATH"/src/github.com/hashicorp/cloud-api/specs/external "$SCRIPTS_DIR"/../temp \ No newline at end of file From bd8bb75df98d02c0ed0f5becd8d4c726c899227b Mon Sep 17 00:00:00 2001 From: Brenna Hewer-Darroch <21015366+bcmdarroch@users.noreply.github.com> Date: Tue, 7 Feb 2023 18:19:22 -0500 Subject: [PATCH 07/10] Delete pull-specs.sh --- scripts/pull-specs.sh | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100755 scripts/pull-specs.sh diff --git a/scripts/pull-specs.sh b/scripts/pull-specs.sh deleted file mode 100755 index 422fc612..00000000 --- a/scripts/pull-specs.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: MPL-2.0 - -set -euo pipefail - -# This script fetches the latest public API specs for a given HCP service ($1) -# from the central spec repo, cloud-api. - -SCRIPTS_DIR=$(dirname "${BASH_SOURCE[0]}") - -BOLD='\033[1m' -NA='\033[0m' # no attributes (color or format) - -service=$1 - -echo -e "Fetching latest specs for ${BOLD}$service${NA}" -hcloud repo init \ - --refresh \ - --only=cloud-api - -# Copy the latest service specs into a temporary directory in preparation for SDK generation. -rsync -a "$HOME"/.local/share/hcp/repos/cloud-api/specs/"$service" "$SCRIPTS_DIR"/../temp -rsync -a "$HOME"/.local/share/hcp/repos/cloud-api/specs/cloud-shared "$SCRIPTS_DIR"/../temp -rsync -a "$HOME"/.local/share/hcp/repos/cloud-api/specs/external "$SCRIPTS_DIR"/../temp \ No newline at end of file From 493e3e6da8c47a70d4ef84ab5728f045f220d28b Mon Sep 17 00:00:00 2001 From: Brenna Hewer-Darroch <21015366+bcmdarroch@users.noreply.github.com> Date: Tue, 7 Feb 2023 18:19:56 -0500 Subject: [PATCH 08/10] remove generation make targets --- Makefile | 48 ------------------------------------------------ 1 file changed, 48 deletions(-) diff --git a/Makefile b/Makefile index d8bf54aa..df09bae8 100644 --- a/Makefile +++ b/Makefile @@ -31,51 +31,3 @@ go/lint: test-ci: go/lint go test -coverprofile=coverage.out -short ./... go tool cover -html=coverage.out -o coverage.html - -# args passed to sdk/update -commit=false - -# This recipe pulls the latest specs for the given service in cloud-api and re-generates the service's go clients. -.PHONY: sdk/update # service=cloud-foo-service commit=true/false -sdk/update: - @if [ -z $(GITHUB_TOKEN) ]; then \ - echo "ERROR: GITHUB_TOKEN is not set. Please ensure the token has 'repo' access and is SSO enabled." >&2; \ - exit 1; \ - fi - - @if [ -z $(service) ]; then \ - echo "ERROR: No service argument provided, please provide in the format 'service=...'" >&2; \ - exit 1; \ - fi - - bash ./scripts/pull-specs.sh $(service); - - @if [ $(service) = "cloud-shared" ]; then \ - echo "Generating latest SDK for cloud-shared"; \ - bash ./scripts/gen-go-shared-sdk.sh; \ - else \ - echo "Generating latest SDK for $(service)"; \ - bash ./scripts/gen-go-service-sdk.sh $(service); \ - fi - - @if [ $(commit) = true ]; then \ - ./scripts/open-pr.sh $(service); \ - fi - -# This recipe pulls the specs for the given service from locally cloned cloud-api and re-generates the service's go clients. -.PHONY: sdk/update-local # service=cloud-foo-service -sdk/update-local: - @if [ -z $(service) ]; then \ - echo "ERROR: No service argument provided, please provide in the format 'service=...'" >&2; \ - exit 1; \ - fi - - bash ./scripts/pull-specs-local.sh $(service); - - @if [ $(service) = "cloud-shared" ]; then \ - echo "Generating latest SDK for cloud-shared"; \ - bash ./scripts/gen-go-shared-sdk.sh; \ - else \ - echo "Generating latest SDK for $(service)"; \ - bash ./scripts/gen-go-service-sdk.sh $(service); \ - fi \ No newline at end of file From 7cfe6952bba0ffc24e18e31a53b89e68cf0680ab Mon Sep 17 00:00:00 2001 From: Brenna Hewer-Darroch <21015366+bcmdarroch@users.noreply.github.com> Date: Wed, 8 Feb 2023 16:21:59 -0500 Subject: [PATCH 09/10] Delete cmd/transform-swagger directory --- cmd/transform-swagger/README.md | 11 -- cmd/transform-swagger/main.go | 263 ----------------------------- cmd/transform-swagger/main_test.go | 152 ----------------- 3 files changed, 426 deletions(-) delete mode 100644 cmd/transform-swagger/README.md delete mode 100644 cmd/transform-swagger/main.go delete mode 100644 cmd/transform-swagger/main_test.go diff --git a/cmd/transform-swagger/README.md b/cmd/transform-swagger/README.md deleted file mode 100644 index 9db9ac86..00000000 --- a/cmd/transform-swagger/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# Transform Swagger - -The Transform Swagger tool allows for customization of the swagger files under `/specs`. - -The service swagger files are generated using the `protoc-gen-openapiv2` protoc plugin, part of the [grpc-gateway](https://github.com/grpc-ecosystem/grpc-gateway) set of tools. The Go SDK generator tool [go-swagger](https://github.com/go-swagger/go-swagger) uses those swagger files to generate SDK clients and models. - -In some cases, go-swagger needs additional data to generate a working SDK client. This is where this tool comes in to play. It first parses each swagger file into a mutable struct, applies any necessary changes, and then overwrites the json with the changes. - -## Transformations - -- **Add `x-go-type` to shared type definitions**: Type reusability across generated SDK clients requires the [x-go-type](https://goswagger.io/use/models/schemas.html#types-reusability) extension to be added to every single shared type definition. diff --git a/cmd/transform-swagger/main.go b/cmd/transform-swagger/main.go deleted file mode 100644 index 16406cc0..00000000 --- a/cmd/transform-swagger/main.go +++ /dev/null @@ -1,263 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 - -package main - -import ( - "encoding/json" - "flag" - "fmt" - "io/ioutil" - "log" - "os" - "path/filepath" - "regexp" - "strings" - - "github.com/go-openapi/loads" - "github.com/go-openapi/spec" - "github.com/iancoleman/strcase" -) - -// XGoType contains the fields of the go-swagger extension x-go-type -// In its final form, the extension looks like this on the generated type definition: -// -// "hashicorp.cloud.common.PaginationRequest": { -// "properties": { ... } -// ... -// "x-go-type": { -// "import": { -// "package": "github.com/hashicorp/hcp-sdk-go/clients/cloud-shared/v1/models", -// "alias": "cloud" -// }, -// "type": "HashicorpCloudCommonPaginationRequest" -// } -// } -// -// ref: https://goswagger.io/use/models/schemas.html#types-reusability -type XGoType struct { - Import Import `json:"import"` - Type string `json:"type"` -} - -// Import contains the import info used in the go-swagger extension x-go-type -type Import struct { - Package string `json:"package"` - Alias string `json:"alias"` -} - -func main() { - - // These flags collect: - // - the path of the service spec, i.e. specs/consul-foo-service/preview/2021-07-09/hcp.swagger.json - // - the path of the shared spec directory, should always end with 'specs/cloud-shared' - // - the path of the external spec, which should always end with 'specs/external/external.swagger.json' - - svcPathPtr := flag.String("service", "", "the path of the service specs") - sharedPathPtr := flag.String("shared", "", "the path of the shared specs") - externalSpecPathPtr := flag.String("external", "", "the path of the external types spec") - - flag.Parse() - - svcPath := *svcPathPtr - sharedPath := *sharedPathPtr - externalSpecPath := *externalSpecPathPtr - - log.Print("Creating map of shared type definitions") - sharedDefs, err := loadSharedDefinitions(sharedPath, svcPath) - if err != nil { - log.Fatalf("failed to add load shared type definitions: %v", err) - } - - // No changes are made to the original spec at this step. - log.Printf("Copying external type definitions from spec %q", svcPath) - err = copyExternalTypes(externalSpecPath, svcPath) - if err != nil { - log.Fatalf("failed to copy external type definitions: %v", err) - } - - svcDoc, err := loads.JSONSpec(svcPath) - if err != nil { - log.Fatalf("failed to load spec at path %q: %v", svcPath, err) - } - - sp := svcDoc.Spec() - - log.Printf("Adding shared type extension to spec %q", svcPath) - updatedSpec, err := addSharedExtension(sp, sharedDefs) - if err != nil { - log.Fatalf("failed to add shared extension: %v", err) - } - - json, err := json.MarshalIndent(updatedSpec, "", " ") - if err != nil { - log.Fatalf("failed to marshal json for path %q: %v", svcPath, err) - } - - // Overwrite original spec with the transformed spec. - log.Printf("Overwriting spec %q", svcPath) - err = os.WriteFile(svcPath, json, os.ModePerm) - if err != nil { - log.Fatalf("failed to overwrite spec at path %q: %v", svcPath, err) - } -} - -// loadSharedDefinitions parses both the service spec and shared specs and returns a map of every shared type name -func loadSharedDefinitions(sharedPath, svcPath string) (map[string]bool, error) { - sharedDefs := make(map[string]bool) - - // Walk the shared specs to find all internal shared type definitions. - err := filepath.Walk(sharedPath, func(path string, info os.FileInfo, err error) error { - if err != nil { - return fmt.Errorf("failed accessing path %q: %w", path, err) - } - - if info.IsDir() { - return nil - } - - doc, err := loads.JSONSpec(path) - if err != nil { - return fmt.Errorf("failed to load spec at path %q: %w", path, err) - } - - for _, def := range doc.Analyzer.AllDefinitions() { - if def.TopLevel { - sharedDefs[def.Name] = true - } - } - - return nil - }) - if err != nil { - log.Fatal(fmt.Errorf("error walking the path %q: %w", sharedPath, err)) - } - - doc, err := loads.JSONSpec(svcPath) - if err != nil { - return nil, fmt.Errorf("failed to load spec at path %q: %w", svcPath, err) - } - - re := regexp.MustCompile(`\w+\.\w+\.`) - if err != nil { - return nil, fmt.Errorf("failed to compile regex for shared definitions: %w", err) - } - - for _, def := range doc.Analyzer.AllDefinitions() { - if def.TopLevel { - // depending on which plugin used for swagger, some package info may not be available - // any types missing package information are skipped - fqnSwaggerFormat := re.MatchString(def.Name) - if fqnSwaggerFormat && !strings.HasPrefix(def.Name, "hashicorp.cloud") { - sharedDefs[def.Name] = true - } - } - } - - return sharedDefs, nil -} - -// sharedDefNameReplaces is used for replacing with specific values those that -// are incompatible with `strcase.ToCamel` -// -// For example, google.rpc.Status, when fed into ToCamel, yields GoogleRpcStatus, -// which is not the name of the structure, but GoogleRPCStatus -var sharedDefNameReplaces = map[string]string{ - "google.rpc.Status": "GoogleRPCStatus", -} - -// addSharedExtensions loops over each shared type definition in a service spec and adds the type that it should reuse. -// Without adding the type reuse extension, separate copies of each shared type definition are generated alongside the service-specific type definitions. -func addSharedExtension(apiSpec *spec.Swagger, sharedDefs map[string]bool) (*spec.Swagger, error) { - for sharedDefName := range sharedDefs { - - var def spec.Schema - var ok bool - if def, ok = apiSpec.SwaggerProps.Definitions[sharedDefName]; !ok { - continue - } - - genTypeName := sharedDefNameReplaces[sharedDefName] - if genTypeName == "" { - // Unless for some exceptions, the shared definition name gets transformed into a camelcased Go type. - // example: hashicorp.cloud.common.PaginationRequest -> HashicorpCloudCommonPaginationRequest - genTypeName = strcase.ToCamel(sharedDefName) - } - - // This struct contains all the data, like which package to import, needed for the SDK client generator to ensure the service reuses - // the generated shared type, rather than a service-specific duplicate of that shared type. - ext := XGoType{ - Import: Import{ - Package: "github.com/hashicorp/hcp-sdk-go/clients/cloud-shared/v1/models", - Alias: "cloud", - }, - Type: genTypeName, - } - - def.VendorExtensible.AddExtension("x-go-type", ext) - - // The value of 'def' is a copy of the definition's schema, so here the original must be overwritten by the copy with the extension. - apiSpec.SwaggerProps.Definitions[sharedDefName] = def - } - - return apiSpec, nil -} - -// copyExternalTypes reads a given service spec and finds any imported types that have been defined in external protos, -// then copies those type definitions into an 'external' spec file. Those external types can then be generated as shared models -// in the SDK for reuse across any service SDK that requires them. -func copyExternalTypes(externalSpecPath, svcPath string) error { - externalTypes := make(map[string]spec.Schema) - - // Create a map of existing external definitions. - externalDoc, err := loads.JSONSpec(externalSpecPath) - if err != nil { - return fmt.Errorf("failed to load spec at path %q: %w", externalSpecPath, err) - } - - for _, def := range externalDoc.Analyzer.AllDefinitions() { - if def.TopLevel { - externalTypes[def.Name] = *def.Schema - } - } - - // Load the service spec. - svcDoc, err := loads.JSONSpec(svcPath) - if err != nil { - return fmt.Errorf("failed to load spec at path %q: %w", svcPath, err) - } - - sp := svcDoc.Spec() - - // Read the service spec to find any external type definitions. - log.Printf("Copying external type definitions from %q", svcPath) - for name, def := range sp.SwaggerProps.Definitions { - - // The prefix "hashicorp.cloud" indicates a type that is either defined by a cloud service proto or one of the shared hashicorp/cloud protos. - // Any other type is deemed external, even if defined by another 'hashicorp' repo. - if !strings.HasPrefix(name, "hashicorp.cloud") { - // The type definition may include the XGoType extension from a prior transformation of the service spec. - // When the type definition is saved in the external spec, the extension must be removed to ensure the type can be generated as a shared model. - delete(def.VendorExtensible.Extensions, "x-go-type") - - externalTypes[name] = def - } - } - - log.Printf("Saving updated external type definitions to %q", externalSpecPath) - updatedExternalSpec := externalDoc.Spec() - updatedExternalSpec.SwaggerProps.Definitions = externalTypes - - json, err := json.MarshalIndent(updatedExternalSpec, "", " ") - if err != nil { - return fmt.Errorf("failed to marshal json for path %q: %v", externalSpecPath, err) - } - - // Overwrite existing external.swagger.json with the updated external definitions. - err = ioutil.WriteFile(externalSpecPath, json, os.ModePerm) - if err != nil { - log.Fatalf("failed to overwrite spec at path %q: %v", svcPath, err) - } - - return nil -} diff --git a/cmd/transform-swagger/main_test.go b/cmd/transform-swagger/main_test.go deleted file mode 100644 index 3f334efa..00000000 --- a/cmd/transform-swagger/main_test.go +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 - -package main - -import ( - "encoding/json" - "testing" - - "github.com/go-openapi/loads" - - "github.com/go-openapi/spec" - "github.com/stretchr/testify/require" -) - -var swagger = spec.Swagger{ - SwaggerProps: spec.SwaggerProps{ - ID: "http://localhost:3849/hcp-api", - Swagger: "2.0", - Host: "some.api.out.there", - BasePath: "/", - Definitions: map[string]spec.Schema{ - "hashicorp.cloud.common.PaginationRequest": {SchemaProps: spec.SchemaProps{Type: []string{"object"}}}, - "hashicorp.cloud.location.Link": {SchemaProps: spec.SchemaProps{Type: []string{"object"}}}, - "grpc.gateway.runtime.Error": {SchemaProps: spec.SchemaProps{Type: []string{"object"}}}, - }, - }, -} - -const specJSON = `{ - "id": "http://localhost:3849/hcp-api", - "swagger": "2.0", - "host": "some.api.out.there", - "basePath": "/", - "definitions": { - "hashicorp.cloud.common.PaginationRequest": { - "type": "object" - }, - "hashicorp.cloud.location.Link": { - "type": "object" - }, - "grpc.gateway.runtime.Error": { - "type": "object" - } - } -}` - -// For reference only, since we do more granular assertions in the test below. -//nolint -const transformedSpecJSON = `{ - "id": "http://localhost:3849/hcp-api", - "swagger": "2.0", - "host": "some.api.out.there", - "basePath": "/", - "definitions": { - "hashicorp.cloud.common.PaginationRequest": { - "type": "object", - "x-go-type": { - "import": { - "package": "github.com/hashicorp/hcp-sdk-go/clients/cloud-shared/v1/models", - "alias": "cloud" - }, - "type": "HashicorpCloudCommonPaginationRequest" - } - }, - "hashicorp.cloud.location.Link": { - "type": "object", - "x-go-type": { - "import": { - "package": "github.com/hashicorp/hcp-sdk-go/clients/cloud-shared/v1/models", - "alias": "cloud" - }, - "type": "HashicorpCloudLocationLink" - } - }, - "grpc.gateway.runtime.Error": { - "type": "object", - "x-go-type": { - "import": { - "package": "github.com/hashicorp/hcp-sdk-go/clients/cloud-shared/v1/models", - "alias": "cloud" - }, - "type": "GrpcGatewayRuntimeError" - } - } - } -}` - -func TestAddSharedExtensions(t *testing.T) { - - var actual spec.Swagger - err := json.Unmarshal([]byte(specJSON), &actual) - require.NoError(t, err) - require.EqualValues(t, actual, swagger) - - var raw json.RawMessage - err = json.Unmarshal([]byte(specJSON), &raw) - require.NoError(t, err) - - svcDoc, err := loads.Analyzed(raw, "") - require.NoError(t, err) - - svcDocs := map[string]*loads.Document{"temp": svcDoc} - - sharedDefs := map[string]bool{ - "hashicorp.cloud.common.PaginationRequest": true, - "hashicorp.cloud.location.Link": true, - "grpc.gateway.runtime.Error": true, - } - - sp := svcDocs["temp"].Spec() - - updatedSpec, err := addSharedExtension(sp, sharedDefs) - require.NoError(t, err) - - require.Equal(t, spec.VendorExtensible{ - Extensions: spec.Extensions{ - "x-go-type": XGoType{ - Import: Import{ - Package: "github.com/hashicorp/hcp-sdk-go/clients/cloud-shared/v1/models", - Alias: "cloud", - }, - Type: "HashicorpCloudCommonPaginationRequest", - }, - }, - }, updatedSpec.SwaggerProps.Definitions["hashicorp.cloud.common.PaginationRequest"].VendorExtensible) - - require.Equal(t, spec.VendorExtensible{ - Extensions: spec.Extensions{ - "x-go-type": XGoType{ - Import: Import{ - Package: "github.com/hashicorp/hcp-sdk-go/clients/cloud-shared/v1/models", - Alias: "cloud", - }, - Type: "HashicorpCloudLocationLink", - }, - }, - }, updatedSpec.SwaggerProps.Definitions["hashicorp.cloud.location.Link"].VendorExtensible) - - require.Equal(t, spec.VendorExtensible{ - Extensions: spec.Extensions{ - "x-go-type": XGoType{ - Import: Import{ - Package: "github.com/hashicorp/hcp-sdk-go/clients/cloud-shared/v1/models", - Alias: "cloud", - }, - Type: "GrpcGatewayRuntimeError", - }, - }, - }, updatedSpec.SwaggerProps.Definitions["grpc.gateway.runtime.Error"].VendorExtensible) - -} From fa540e39d1a7f417ee154b9d956347ebcd578ee4 Mon Sep 17 00:00:00 2001 From: Brenna Hewer-Darroch <21015366+bcmdarroch@users.noreply.github.com> Date: Thu, 9 Mar 2023 16:58:27 -0500 Subject: [PATCH 10/10] bonus: bump cloud-network to stable --- README.md | 2 +- cmd/hcp-sdk-go-client/main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 24fadfb3..058b7d23 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ HCP_ORGANIZATION_ID="22abc..." ```bash import ( - network "github.com/hashicorp/hcp-sdk-go/clients/cloud-network/preview/2020-09-07/client/network_service" + network "github.com/hashicorp/hcp-sdk-go/clients/cloud-network/stable/2020-09-07/client/network_service" ) ``` diff --git a/cmd/hcp-sdk-go-client/main.go b/cmd/hcp-sdk-go-client/main.go index 89576252..7f8e1eda 100644 --- a/cmd/hcp-sdk-go-client/main.go +++ b/cmd/hcp-sdk-go-client/main.go @@ -9,7 +9,7 @@ import ( "os" consul "github.com/hashicorp/hcp-sdk-go/clients/cloud-consul-service/stable/2021-02-04/client/consul_service" - network "github.com/hashicorp/hcp-sdk-go/clients/cloud-network/preview/2020-09-07/client/network_service" + network "github.com/hashicorp/hcp-sdk-go/clients/cloud-network/stable/2020-09-07/client/network_service" "github.com/hashicorp/hcp-sdk-go/httpclient" )