actions change

.github/codecov.yml

@@ -16,16 +16,16 @@ component_management:
   - component_id: spanner-templates
     name: spanner-templates
     paths:
-    - "v1/src/main/java/com/google/cloud/teleport/spanner/**"
-    - "v1/src/main/java/com/google/cloud/teleport/templates/SpannerToText.java"
-    - "v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java"
-    - "v2/datastream-to-spanner/**"
-    - "v2/spanner-common/**"
-    - "v2/spanner-change-streams-to-sharded-file-sink/**"
-    - "v2/gcs-to-sourcedb/**"
-    - "v2/spanner-migrations-sdk/**"
-    - "v2/spanner-custom-shard/**"
-    - "v2/sourcedb-to-spanner/**"
+    # - "v1/src/main/java/com/google/cloud/teleport/spanner/**"
+    # - "v1/src/main/java/com/google/cloud/teleport/templates/SpannerToText.java"
+    # - "v1/src/main/java/com/google/cloud/teleport/templates/common/SpannerConverters.java"
+    # - "v2/datastream-to-spanner/**"
+    # - "v2/spanner-common/**"
+    # - "v2/spanner-change-streams-to-sharded-file-sink/**"
+    # - "v2/gcs-to-sourcedb/**"
+    # - "v2/spanner-migrations-sdk/**"
+    # - "v2/spanner-custom-shard/**"
+    # - "v2/sourcedb-to-spanner/**"
     - "v2/spanner-to-sourcedb/**"
     statuses:
     - type: project

.github/workflows/java-pr.yml

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 # Checks that are intended to run on PRs containing Java code.
 
 name: Java PR
@@ -21,23 +20,12 @@ on:
     branches:
       - 'main'
     paths:
-      - '**.java'
-      - '**.xml'
-      # Include python files and Dockerfiles used for YAML and xlang templates.
-      - '**.py'
-      - 'plugins/core-plugin/src/main/resources/**'
-      # Include relevant GitHub Action files for running these checks.
-      # This will make it easier to verify action changes don't break anything.
+      - 'v2/spanner-to-sourcedb/**.java'
+      - 'v2/spanner-to-sourcedb/**.xml'
+      - 'v2/spanner-to-sourcedb/**.py'
       - '.github/actions/setup-env/*'
       - '.github/workflows/java-pr.yml'
-      # Exclude spanner paths from global run (covered in https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/.github/workflows/spanner-pr.yml)
-      - '!v2/datastream-to-spanner/**'
-      - '!v2/spanner-common/**'
-      - '!v2/spanner-change-streams-to-sharded-file-sink/**'
-      - '!v2/gcs-to-sourcedb/**'
-      - '!v2/spanner-migrations-sdk/**'
-      - '!v2/spanner-custom-shard/**'
-      - '!v2/sourcedb-to-spanner/**'
+      - '!v1/**'      
   schedule:
     - cron: "0 */12 * * *"
   workflow_dispatch:
@@ -76,84 +64,84 @@ jobs:
         uses: ./.github/actions/setup-env
       - name: Run Checkstyle
         run: ./cicd/run-checkstyle
-  java_build:
-    name: Build
-    timeout-minutes: 60
-    runs-on: [self-hosted, it]
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
-      - name: Setup Environment
-        id: setup-env
-        uses: ./.github/actions/setup-env
-      - name: Run Build
-        run: ./cicd/run-build
-      - name: Cleanup Java Environment
-        uses: ./.github/actions/cleanup-java-env
-  java_unit_tests:
-    name: Unit Tests
-    needs: [java_build]
-    timeout-minutes: 60
-    runs-on: [self-hosted, it]
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
-      - name: Setup Environment
-        id: setup-env
-        uses: ./.github/actions/setup-env
-      - name: Run Unit Tests
-        run: ./cicd/run-unit-tests
-      - name: Upload Unit Tests Report
-        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
-        if: always() # always run even if the previous step fails
-        with:
-          name: surefire-test-results
-          path: '**/surefire-reports/TEST-*.xml'
-          retention-days: 1
-      - name: Upload coverage reports to Codecov
-        uses: codecov/codecov-action@v4.0.1
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          slug: GoogleCloudPlatform/DataflowTemplates
-          files: 'target/site/jacoco-aggregate/jacoco.xml'
-          # Temp fix for https://github.com/codecov/codecov-action/issues/1487
-          version: v0.6.0
-      - name: Cleanup Java Environment
-        uses: ./.github/actions/cleanup-java-env
-        if: always()
-  java_integration_smoke_tests_templates:
-    name: Dataflow Templates Integration Smoke Tests
-    needs: [spotless_check, checkstyle_check, java_build, java_unit_tests]
-    timeout-minutes: 60
-    # Run on any runner that matches all the specified runs-on values.
-    runs-on: [self-hosted, it]
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
-      - name: Setup Environment
-        id: setup-env
-        uses: ./.github/actions/setup-env
-      - name: Run Integration Smoke Tests
-        run: | 
-          ./cicd/run-it-smoke-tests \
-          --modules-to-build="DEFAULT" \
-          --it-region="us-central1" \
-          --it-project="cloud-teleport-testing" \
-          --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
-          --it-private-connectivity="datastream-private-connect-us-central1"
-      - name: Upload Smoke Tests Report
-        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
-        if: always() # always run even if the previous step fails
-        with:
-          name: surefire-test-results
-          path: '**/surefire-reports/TEST-*.xml'
-          retention-days: 1
-      - name: Cleanup Java Environment
-        uses: ./.github/actions/cleanup-java-env
-        if: always()
+  # java_build:
+  #   name: Build
+  #   timeout-minutes: 60
+  #   runs-on: [self-hosted, it]
+  #   steps:
+  #     - name: Checkout Code
+  #       uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+  #     - name: Setup Environment
+  #       id: setup-env
+  #       uses: ./.github/actions/setup-env
+  #     - name: Run Build
+  #       run: ./cicd/run-build
+  #     - name: Cleanup Java Environment
+  #       uses: ./.github/actions/cleanup-java-env
+  # java_unit_tests:
+  #   name: Unit Tests
+  #   # needs: [java_build]
+  #   timeout-minutes: 60
+  #   runs-on: [self-hosted, it]
+  #   steps:
+  #     - name: Checkout Code
+  #       uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+  #     - name: Setup Environment
+  #       id: setup-env
+  #       uses: ./.github/actions/setup-env
+  #     - name: Run Unit Tests
+  #       run: ./cicd/run-unit-tests
+  #     - name: Upload Unit Tests Report
+  #       uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+  #       if: always() # always run even if the previous step fails
+  #       with:
+  #         name: surefire-test-results
+  #         path: '**/surefire-reports/TEST-*.xml'
+  #         retention-days: 1
+  #     - name: Upload coverage reports to Codecov
+  #       uses: codecov/codecov-action@v4.0.1
+  #       with:
+  #         token: ${{ secrets.CODECOV_TOKEN }}
+  #         slug: ollionorg/DataflowTemplates-fork
+  #         files: 'target/site/jacoco-aggregate/jacoco.xml'
+  #         # Temp fix for https://github.com/codecov/codecov-action/issues/1487
+  #         version: v0.6.0
+  #     - name: Cleanup Java Environment
+  #       uses: ./.github/actions/cleanup-java-env
+  #       if: always()
+  # java_integration_smoke_tests_templates:
+  #   name: Dataflow Templates Integration Smoke Tests
+  #   needs: [spotless_check, checkstyle_check]
+  #   timeout-minutes: 60
+  #   # Run on any runner that matches all the specified runs-on values.
+  #   runs-on: [self-hosted, it]
+  #   steps:
+  #     - name: Checkout Code
+  #       uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+  #     - name: Setup Environment
+  #       id: setup-env
+  #       uses: ./.github/actions/setup-env
+  #     - name: Run Integration Smoke Tests
+  #       run: |     
+  #         ./cicd/run-it-smoke-tests \
+  #         --modules-to-build="v2/spanner-to-sourcedb" \
+  #         --it-region="us-central1" \
+  #         --it-project="cloud-teleport-testing" \
+  #         --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
+  #         --it-private-connectivity="datastream-private-connect-us-central1"
+  #     - name: Upload Smoke Tests Report
+  #       uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+  #       if: always() # always run even if the previous step fails
+  #       with:
+  #         name: surefire-test-results
+  #         path: '**/surefire-reports/TEST-*.xml'
+  #         retention-days: 1
+  #     - name: Cleanup Java Environment
+  #       uses: ./.github/actions/cleanup-java-env
+  #       if: always()
   java_integration_tests_templates:
     name: Dataflow Templates Integration Tests
-    needs: [java_integration_smoke_tests_templates]
+    # needs: [java_integration_smoke_tests_templates]
     timeout-minutes: 240
     # Run on any runner that matches all the specified runs-on values.
     runs-on: [self-hosted, it]
@@ -164,13 +152,14 @@ jobs:
         id: setup-env
         uses: ./.github/actions/setup-env
       - name: Run Integration Tests
-        run: | 
-          ./cicd/run-it-tests \
-          --modules-to-build="DEFAULT" \
-          --it-region="us-central1" \
-          --it-project="cloud-teleport-testing" \
-          --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
-          --it-private-connectivity="datastream-private-connect-us-central1"
+        run: mvn clean verify -PtemplatesIntegrationTests -pl v2/spanner-to-sourcedb -am -Dtest=SpannerToSourceDbCustomTransformationIT -Dcheckstyle.skip -Dspotless.check.skip=true -DartifactBucket="smt-df-templates-test" -Dproject="span-cloud-migrations-testing"  -Dregion=us-central1  -DspannerInstanceId="test-instance"
+          # |
+          # ./cicd/run-it-tests \
+          # --modules-to-build="v2/spanner-to-sourcedb" \
+          # --it-region="asia-south1" \
+          # --it-project="daring-fiber-439305-v4" \
+          # --it-artifact-bucket="ollion-teleport-testing" \
+          # --it-private-connectivity="datastream-private-connect-asia-south1"
       - name: Upload Integration Tests Report
         uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
         if: always() # always run even if the previous step fails
@@ -182,12 +171,12 @@ jobs:
         uses: ./.github/actions/cleanup-java-env
         if: always()
   java_load_tests_templates:
-    if: contains(github.event.pull_request.labels.*.name, 'run-load-tests')
+    # if: contains(github.event.pull_request.labels.*.name, 'run-load-tests')
     name: Dataflow Templates Load Tests
-    needs: [spotless_check, checkstyle_check, java_build, java_unit_tests, java_integration_tests_templates]
+    needs: [spotless_check, checkstyle_check, java_integration_tests_templates]
     timeout-minutes: 600
     # Run on any runner that matches all the specified runs-on values.
-    runs-on: [self-hosted, perf]
+    runs-on: [self-hosted]
     steps:
     - name: Checkout Code
       uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0