integration-tests.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: integration-tests

on:
  # Run every day at 03:00
  schedule:
    - cron: "0 3 * * *"
  push:
    paths:
      - '.github/workflows/integration-tests.yml'
      - '**/test/**/*IT.java'
      - '**/test/**/*ITCase.java'
      - '**/test/**/IT*.java'
  pull_request:
    paths:
      - '.github/workflows/integration-tests.yml'
      - '**/test/**/*IT.java'
      - '**/test/**/*ITCase.java'
      - '**/test/**/IT*.java'

env:
  DEFAULT_MAVEN_OPTS: >-
    -Xmx4g
    -XX:ReservedCodeCacheSize=1g
    -XX:+UseG1GC
    -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN
    -Daether.connector.http.retryHandler.count=5
    -Daether.connector.http.connectionMaxTtl=30
    -Duser.language=en
    -Duser.country=GB
    -Duser.timezone=UTC
  MAVEN_COMMAND: >-
    ./mvnw
    -V
    -nsu
    -ntp
    -ff

# skip assemblies and modules covered separately by the system-tests workflow
# skip unit tests (surefire), run integration tests (failsafe), python ITs included as part of the system-tests workflow
  MAVEN_BUILD_ARGUMENTS: >-
    -T 2.0C
    -D failsafe.failIfNoSpecifiedTests=false
    package
    -P skip-unit-tests
    -P integration-tests
    -P nifi-registry-integration-tests
    package
  MAVEN_BUILD_EXCLUDE_PROJECTS: >-
    -pl -:minifi-assembly
    -pl -:minifi-c2-assembly
    -pl -:nifi-assembly
    -pl -:nifi-toolkit-assembly
    -pl -:nifi-registry-assembly
    -pl -:nifi-registry-toolkit-assembly
    -pl -:nifi-runtime-manifest
    -pl -:nifi-runtime-manifest-test
    -pl -:nifi-stateless-assembly
    -pl -:nifi-stateless-processor-tests
    -pl -:nifi-stateless-system-test-suite
    -pl -:nifi-system-test-suite
    -pl -:nifi-nar-provider-assembly
    -pl -:nifi-py4j-integration-tests
# skip individual failing ITs (some won't work in a CI environment)
# always run the (failsafe default) IT*, *IT, *ITCase integration-tests
  MAVEN_FAILSAFE_SKIP_ITS: >-
    IT*, *IT, *ITCase,
    !AwsSecretsManagerSensitivePropertyProviderIT,
    !AwsKmsSensitivePropertyProviderIT,
    !AzureKeyVaultKeySensitivePropertyProviderIT,
    !GcpKmsSensitivePropertyProviderIT,
    !StandardHashiCorpVaultCommunicationServiceIT,
    !ConversionWithSchemaInferenceIT#testXmlToAvro,
    !ConversionWithSchemaInferenceIT#testXmlToJson,
    !ConversionWithSchemaInferenceIT#testJsonToCsv,
    !ConversionWithSchemaInferenceIT#testCsvToXml,
    !ConversionWithSchemaInferenceIT#testXmlToAvroToXml,
    !ConversionWithSchemaInferenceIT#testCsvToAvro,
    !ConversionWithSchemaInferenceIT#testCsvToJson,
    !ConversionWithSchemaInferenceIT#testJsonToAvroToJson,
    !ConversionWithSchemaInferenceIT#testCsvToAvroToCsv,
    !ConversionWithExplicitSchemaIT#testJsonToCsv,
    !ConversionWithExplicitSchemaIT#testJsonToAvro,
    !ConversionWithExplicitSchemaIT#testJsonToAvroToJson,
    !ITConsumeKinesisStreamEndpointOverride,
    !ITConsumeKinesisStreamConnectAWS,
    !ITPutKinesisStream,
    !ITPutKinesisStreamWithEndpointOverride#testIntegrationSuccess,
    !ITPutKinesisFirehoseWithEndpointOverride#testIntegrationSuccess,
    !ITPutKinesisFirehose,
    !ITTagS3Object,
    !ITFetchS3Object,
    !ITDeleteS3Object,
    !ITPutS3Object,
    !ITListS3,
    !ITPutSNS#testPublish,
    !ITPutSNS#testPublishWithCredentialsProviderService,
    !ITDeleteSQS,
    !ITGetSQS,
    !ITPutSQS,
    !ITPutLambda,
    !GetMongoIT#testDatabaseEL,
    !PutGridFSIT#testFileNameAndHashUniqueness,
    !PutGridFSIT#testHashUniqueness,
    !ITPutAzureCosmosDBRecord,
    !ITListAzureBlobStorage_v12,
    !ITPutAzureDataLakeStorage,
    !ITFetchAzureDataLakeStorage,
    !ITListAzureDataLakeStorage,
    !ITFetchAzureBlobStorage_v12,
    !ITPutAzureBlobStorage_v12,
    !ITDeleteAzureDataLakeStorage,
    !ITDeleteAzureBlobStorage_v12,
    !ITMoveAzureDataLakeStorage,
    !AzureGraphUserGroupProviderIT,
    !JMSPublisherConsumerIT#validateMessageRedeliveryWhenNotAcked,
    !ITPutInfluxDB,
    !ITExecuteInfluxDBQuery,
    !GremlinClientServiceYamlSettingsAndBytecodeIT,
    !GremlinClientServiceControllerSettingsIT,
    !ITestConsumeEmail#validateUrl,
    !PrometheusReportingTaskIT#testNullLabel,
    !SnowflakeConnectionPoolIT,
    !SalesforceRestServiceIT,
    !QuerySalesforceObjectIT,
    !PutSalesforceObjectIT,
    !FetchDropboxIT,
    !PutDropboxIT,
    !ListDropboxIT,
    !FetchBoxFileIT,
    !PutBoxFileIT,
    !ListBoxFileIT,
    !JsonConfigBasedBoxClientServiceIT,
    !QueryIoTDBIT,
    !PutIoTDBRecordIT,
    !StandardProcessSessionIT#testCloneThenWriteCountsClaimReferencesProperly,
    !StandardProcessSessionIT#testUpdateFlowFileRepoFailsOnSessionCommit,
    !JMSPublisherConsumerIT#testMultipleThreads,
    !ITRedisDistributedMapCacheClientService#testSentinelRedisWithAuthentication,
    !ITRedisDistributedMapCacheClientService#testSentinelRedis,
    !SnowflakePipeIT,
    !SecureFileIT,
    !SecureProxyIT#testAccessStatus,
    !SecureProxyIT#testAccessStatusAsProxiedAdmin,
    !SecureProxyIT#testAccessStatusUsingRegistryClient,
    !SecureNiFiRegistryClientIT#testGetAccessStatus,
    !SecureNiFiRegistryClientIT#testTenantsClientGroups,
    !SecureNiFiRegistryClientIT#testTenantsClientUsers,
    !SecureNiFiRegistryClientIT#testCrudOperations,
    !SecureDatabaseIT,
    !ITConnectionAccessControl,
    !ITLabelAccessControl,
    !ITOutputPortAccessControl,
    !ITProcessorAccessControl,
    !ITFlowAccessControl,
    !ITFunnelAccessControl,
    !ITInputPortAccessControl
    !ITCountersAccessControl,
    !ITAllowProxiedAnonymousAccess,
    !ITPreventProxiedAnonymousAccess,
    !ITPreventDirectAnonymousAccess,
    !ITAllowDirectAnonymousAccess,
    !ITProcessGroupAccessControl

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  build_and_test:
    strategy:
      fail-fast: false
      matrix:
        # TODO: macos-latest doesn't come with Docker and Colima seems to have problems with Test Containers - see NIFI-12191
        os: [ ubuntu-latest ]
        version: [ 21 ]
    timeout-minutes: 120
    runs-on: ${{ matrix.os }}
    name: ${{ matrix.os }} Java ${{ matrix.version }}
    env:
      # default to Amazon Corretto JDK
      JAVA_DISTRIBUTION: corretto
    steps:
      # Zulu JDK appears to be more reliable for MacOS
      - name: Use Java distribution Zulu
        if: ${{ runner.os == 'macos' }}
        run: echo "JAVA_DISTRIBUTION=zulu" >> "$GITHUB_ENV"
      - name: Checkout Code
        uses: actions/checkout@v4
      - name: Set up Java ${{ env.JAVA_DISTRIBUTION }} ${{ matrix.version }}
        uses: actions/setup-java@v3
        with:
          distribution: ${{ env.JAVA_DISTRIBUTION }}
          java-version: ${{ matrix.version }}
          cache: 'maven'

      # many integration-tests use Docker to run Testcontainers; MacOS doesn't come with Docker, so we need to install it
      - name: Set up Docker
        if: ${{ runner.os == 'macos' }}
        run: |-
          /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
          brew install docker
          colima start

          if [ ! -e /var/run/docker.sock ]; then sudo ln -s ~/.colima/docker.sock /var/run/docker.sock; fi
      - name: Confirm Docker
        run: id && docker info

      - name: Build and Run Tests
        env:
          MAVEN_OPTS: >-
            ${{ env.DEFAULT_MAVEN_OPTS }}
        run: >
          ${{ env.MAVEN_COMMAND }}
          ${{ env.MAVEN_BUILD_ARGUMENTS }}
          ${{ env.MAVEN_BUILD_EXCLUDE_PROJECTS }}
          '-Dit.test=${{ env.MAVEN_FAILSAFE_SKIP_ITS }}'
#      - name: Run Failsafe Tests
#        env:
#          MAVEN_OPTS: >-
#            ${{ env.DEFAULT_MAVEN_OPTS }}
#        run: >
#          ${{ env.MAVEN_COMMAND }}
#          ${{ env.MAVEN_FAILSAFE_RUN_ARGUMENTS }}
#          '-Dit.test=${{ env.MAVEN_FAILSAFE_SKIP_ITS }}'
      - name: Upload Troubleshooting Logs
        if: failure() || cancelled()
        uses: actions/upload-artifact@v3
        with:
          name: ${{ matrix.os }}-${{ matrix.version }}-failsafe-logs
          path: |
            **/target/failsafe-reports/**/*.txt
          retention-days: 7