Commit the API schema to the repository (#5865)

cvat-ai · Mar 17, 2023 · 3e4fb8d · 3e4fb8d
1 parent 5af73a6
commit 3e4fb8d
Showing 10 changed files with 10,013 additions and 78 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -12,6 +12,7 @@
 
 # Component: Server
 /cvat/          @nmanovic
+/cvat/schema.yml @SpecLad
 
 # Component: CVAT UI
 /cvat-ui/       @bsekachev

diff --git a/.github/workflows/full.yml b/.github/workflows/full.yml
@@ -72,10 +72,6 @@ jobs:
 
       - name: CVAT SDK. Build
         run: |
-          docker load --input /tmp/cvat_server/image.tar
-          docker run --rm -v ${PWD}/cvat-sdk/schema/:/transfer \
-            --entrypoint /bin/bash -u root cvat/server \
-            -c 'python manage.py spectacular --file /transfer/schema.yml'
           pip3 install --user -r cvat-sdk/gen/requirements.txt
           ./cvat-sdk/gen/generate.sh
 
@@ -140,15 +136,38 @@ jobs:
           docker tag cvat/ui:latest cvat/ui:dev
           docker image ls -a
 
+      - name: Verify API schema
+        id: verify_schema
+        run: |
+          docker run --rm --entrypoint /bin/bash cvat/server:dev \
+            -c 'python manage.py spectacular' > cvat/schema-expected.yml
+
+          if ! git diff --no-index cvat/schema.yml cvat/schema-expected.yml; then
+            echo
+            echo 'API schema has changed! Please update cvat/schema.yml:'
+            echo
+            echo '  docker run --rm --entrypoint /bin/bash cvat/server:dev \'
+            echo "    -c 'python manage.py spectacular' > cvat/schema.yml"
+            exit 1
+          fi
+
+      - name: Upload expected schema as an artifact
+        if: failure() && steps.verify_schema.conclusion == 'failure'
+        uses: actions/upload-artifact@v3.1.2
+        with:
+          name: expected_schema
+          path: cvat/schema-expected.yml
+
       - name: Running REST API and SDK tests
+        id: run_tests
         run: |
           pip3 install --user '/tmp/cvat_sdk/[pytorch]'
           pip3 install --user cvat-cli/
           pip3 install --user -r tests/python/requirements.txt
           pytest tests/python -s -v
 
       - name: Creating a log file from cvat containers
-        if: failure()
+        if: failure() && steps.run_tests.conclusion == 'failure'
         env:
           LOGS_DIR: "${{ github.workspace }}/rest_api"
         run: |
@@ -159,8 +178,8 @@ jobs:
             docker logs test_cvat_opa_1 2> $LOGS_DIR/cvat_opa.log
 
       - name: Uploading "cvat" container logs as an artifact
-        if: failure()
-        uses: actions/upload-artifact@v3.1.1
+        if: failure() && steps.run_tests.conclusion == 'failure'
+        uses: actions/upload-artifact@v3.1.2
         with:
           name: rest_api_container_logs
           path: "${{ github.workspace }}/rest_api"

diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml
@@ -66,11 +66,8 @@ jobs:
         kubectl logs $(kubectl get pods -l component=server -o jsonpath='{.items[0].metadata.name}')
 
 
-    - name: Generate schema
+    - name: Generate SDK
       run: |
-        mkdir cvat-sdk/schema
-        kubectl exec $(kubectl get pods -l component=server -o jsonpath='{.items[0].metadata.name}') -- /bin/bash -c "python manage.py spectacular --file /tmp/schema.yml"
-        kubectl cp $(kubectl get pods -l component=server -o jsonpath='{.items[0].metadata.name}'):/tmp/schema.yml cvat-sdk/schema/schema.yml
         pip3 install --user -r cvat-sdk/gen/requirements.txt
         ./cvat-sdk/gen/generate.sh
 

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -69,10 +69,6 @@ jobs:
 
       - name: CVAT SDK. Build
         run: |
-          docker load --input /tmp/cvat_server/image.tar
-          docker run --rm -v ${PWD}/cvat-sdk/schema/:/transfer \
-            --entrypoint /bin/bash -u root cvat/server \
-            -c 'python manage.py spectacular --file /transfer/schema.yml'
           pip3 install --user -r cvat-sdk/gen/requirements.txt
           ./cvat-sdk/gen/generate.sh
 
@@ -132,15 +128,38 @@ jobs:
           docker tag cvat/ui:latest cvat/ui:${CVAT_VERSION}
           docker image ls -a
 
+      - name: Verify API schema
+        id: verify_schema
+        run: |
+          docker run --rm --entrypoint /bin/bash cvat/server:${CVAT_VERSION} \
+            -c 'python manage.py spectacular' > cvat/schema-expected.yml
+
+          if ! git diff --no-index cvat/schema.yml cvat/schema-expected.yml; then
+            echo
+            echo 'API schema has changed! Please update cvat/schema.yml:'
+            echo
+            echo '  docker run --rm --entrypoint /bin/bash cvat/server:dev \'
+            echo "    -c 'python manage.py spectacular' > cvat/schema.yml"
+            exit 1
+          fi
+
+      - name: Upload expected schema as an artifact
+        if: failure() && steps.verify_schema.conclusion == 'failure'
+        uses: actions/upload-artifact@v3.1.2
+        with:
+          name: expected_schema
+          path: cvat/schema-expected.yml
+
       - name: Running REST API and SDK tests
+        id: run_tests
         run: |
           pip3 install --user '/tmp/cvat_sdk/[pytorch]'
           pip3 install --user cvat-cli/
           pip3 install --user -r tests/python/requirements.txt
           pytest tests/python/ -s -v
 
       - name: Creating a log file from cvat containers
-        if: failure()
+        if: failure() && steps.run_tests.conclusion == 'failure'
         env:
           LOGS_DIR: "${{ github.workspace }}/rest_api_testing"
         run: |
@@ -151,8 +170,8 @@ jobs:
             docker logs test_cvat_opa_1 2> $LOGS_DIR/cvat_opa.log
 
       - name: Uploading "cvat" container logs as an artifact
-        if: failure()
-        uses: actions/upload-artifact@v3.1.1
+        if: failure() && steps.run_tests.conclusion == 'failure'
+        uses: actions/upload-artifact@v3.1.2
         with:
           name: rest_api_container_logs
           path: "${{ github.workspace }}/rest_api_testing"

diff --git a/.github/workflows/schedule.yml b/.github/workflows/schedule.yml
@@ -167,9 +167,6 @@ jobs:
 
       - name: REST API and SDK tests
         run: |
-          docker run --rm -v ${PWD}/cvat-sdk/schema/:/transfer \
-            --entrypoint /bin/bash -u root cvat/server \
-            -c 'python manage.py spectacular --file /transfer/schema.yml'
           pip3 install --user -r cvat-sdk/gen/requirements.txt
           ./cvat-sdk/gen/generate.sh
 

diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -357,7 +357,7 @@
             "python": "${command:python.interpreterPath}",
             "program": "${workspaceFolder}/cvat-sdk/gen/postprocess.py",
             "args": [
-                "--schema", "${workspaceFolder}/cvat-sdk/schema/schema.yml",
+                "--schema", "${workspaceFolder}/cvat/schema.yml",
                 "--input-path", "${workspaceFolder}/cvat-sdk/cvat_sdk/"
             ],
             "cwd": "${workspaceFolder}",
@@ -389,7 +389,7 @@
             "args": [
                 "spectacular",
                 "--file",
-                "schema.yml"
+                "${workspaceFolder}/cvat/schema.yml"
             ],
             "django": true,
             "cwd": "${workspaceFolder}",

diff --git a/cvat-sdk/.gitignore b/cvat-sdk/.gitignore
@@ -66,7 +66,6 @@ target/
 .ipynb_checkpoints
 
 # Workflow
-schema/
 .openapi-generator/
 
 # Generated code

diff --git a/cvat-sdk/gen/generate.sh b/cvat-sdk/gen/generate.sh
@@ -13,24 +13,27 @@ LIB_NAME="cvat_sdk"
 LAYER1_LIB_NAME="${LIB_NAME}/api_client"
 DST_DIR="$(cd "$(dirname -- "$0")/.." && pwd)"
 DOCS_DIR="$DST_DIR/docs"
-TEMPLATE_DIR_NAME="gen"
-TEMPLATE_DIR="$DST_DIR/$TEMPLATE_DIR_NAME"
-POST_PROCESS_SCRIPT="${TEMPLATE_DIR}/postprocess.py"
+GEN_DIR="${DST_DIR}/gen"
+POST_PROCESS_SCRIPT="${GEN_DIR}/postprocess.py"
+SCHEMA_PATH="${DST_DIR}/../cvat/schema.yml"
 
 rm -f -r "$DOCS_DIR" "${DST_DIR}/${LAYER1_LIB_NAME}" \
     "${DST_DIR}/requirements/api_client.txt"
 
 # Pass template dir here
 # https://github.com/OpenAPITools/openapi-generator/issues/8420
-docker run --rm -v "$DST_DIR:/local" -u "$(id -u)":"$(id -g)" \
+docker run --rm -u "$(id -u)":"$(id -g)" \
+    -v "${SCHEMA_PATH}:/mnt/schema.yml:ro" \
+    -v "${GEN_DIR}:/mnt/gen:ro" \
+    -v "${DST_DIR}:/mnt/dst" \
     openapitools/openapi-generator-cli:${GENERATOR_VERSION} generate \
-        -t "/local/${TEMPLATE_DIR_NAME}/templates/openapi-generator/" \
-        -i "/local/schema/schema.yml" \
-        --config "/local/${TEMPLATE_DIR_NAME}/generator-config.yml" \
+        -t "/mnt/gen/templates/openapi-generator/" \
+        -i "/mnt/schema.yml" \
+        --config "/mnt/gen/generator-config.yml" \
         -p "packageVersion=$VERSION" \
         -p "httpUserAgent=cvat_sdk/$VERSION" \
         -g python \
-        -o "/local/"
+        -o "/mnt/dst"
 
 echo "VERSION = \"$VERSION\"" > "${DST_DIR}/${LIB_NAME}/version.py"
 mv "${DST_DIR}/requirements.txt" "${DST_DIR}/requirements/api_client.txt"
@@ -44,10 +47,10 @@ mv "${DOCS_DIR}/"*.md "${MODEL_DOCS_DIR}"
 mv "${DST_DIR}/api_summary.md" "${DOCS_DIR}"
 
 # Do custom postprocessing for code files
-"${POST_PROCESS_SCRIPT}" --schema "${DST_DIR}/schema/schema.yml" \
+"${POST_PROCESS_SCRIPT}" --schema "${SCHEMA_PATH}" \
     --input-path "${DST_DIR}/${LIB_NAME}"
 
 # Do custom postprocessing for docs files
-"${POST_PROCESS_SCRIPT}" --schema "${DST_DIR}/schema/schema.yml" \
+"${POST_PROCESS_SCRIPT}" --schema "${SCHEMA_PATH}" \
     --input-path "$DOCS_DIR" --file-ext '.md'