From 2f18a8eaf1697c871c0d792f3a3dbdae970ac324 Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Fri, 13 Jun 2025 14:18:19 +0000 Subject: [PATCH 1/4] switch back to self hosted runner and comment out most of kafka test for now --- .../beam_PostCommit_Yaml_Xlang_Direct.yml | 2 +- .../yaml/extended_tests/messaging/kafka.yaml | 90 +++++++++---------- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml b/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml index 2139b1d88f2b..7af1e0ce88b7 100644 --- a/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml +++ b/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml @@ -54,7 +54,7 @@ jobs: github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request_target' || (github.event_name == 'schedule' && github.repository == 'apache/beam') - runs-on: ubuntu-latest + runs-on: [self-hosted, ubuntu-20.04, main] timeout-minutes: 100 name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) strategy: diff --git a/sdks/python/apache_beam/yaml/extended_tests/messaging/kafka.yaml b/sdks/python/apache_beam/yaml/extended_tests/messaging/kafka.yaml index c2e3c8657ee2..355f77556c8c 100644 --- a/sdks/python/apache_beam/yaml/extended_tests/messaging/kafka.yaml +++ b/sdks/python/apache_beam/yaml/extended_tests/messaging/kafka.yaml @@ -15,9 +15,9 @@ # limitations under the License. # -fixtures: - - name: TEMP_BOOTSTAP_SERVER - type: "apache_beam.yaml.integration_tests.temp_kafka_server" +# fixtures: +# - name: TEMP_BOOTSTAP_SERVER +# type: "apache_beam.yaml.integration_tests.temp_kafka_server" pipelines: # Kafka write pipeline @@ -30,51 +30,51 @@ pipelines: - {value: 123} - {value: 456} - {value: 789} - - type: MapToFields - config: - language: python - fields: - value: - callable: | - lambda row: str(row.value).encode('utf-8') - output_type: bytes - - type: WriteToKafka - config: - format: "RAW" - topic: "silly_topic" - bootstrap_servers: "{TEMP_BOOTSTAP_SERVER}" - producer_config_updates: - linger.ms: "0" + # - type: MapToFields + # config: + # language: python + # fields: + # value: + # callable: | + # lambda row: str(row.value).encode('utf-8') + # output_type: bytes + # - type: WriteToKafka + # config: + # format: "RAW" + # topic: "silly_topic" + # bootstrap_servers: "{TEMP_BOOTSTAP_SERVER}" + # producer_config_updates: + # linger.ms: "0" # Kafka read pipeline # Need a separate read pipeline to make sure the write pipeline is flushed - - pipeline: - type: chain - transforms: - - type: ReadFromKafka - config: - format: "RAW" - topic: "silly_topic" - bootstrap_servers: "{TEMP_BOOTSTAP_SERVER}" - consumer_config: - auto.offset.reset: "earliest" - group.id: "yaml-kafka-test-group" - max_read_time_seconds: 10 # will read forever if not set - - type: MapToFields - config: - language: python - fields: - value: - callable: | - # Kafka RAW format reads messages as bytes in the 'payload' field of a Row - lambda row: row.payload.decode('utf-8') - output_type: string - - type: AssertEqual - config: - elements: - - {value: "123"} - - {value: "456"} - - {value: "789"} + # - pipeline: + # type: chain + # transforms: + # - type: ReadFromKafka + # config: + # format: "RAW" + # topic: "silly_topic" + # bootstrap_servers: "{TEMP_BOOTSTAP_SERVER}" + # consumer_config: + # auto.offset.reset: "earliest" + # group.id: "yaml-kafka-test-group" + # max_read_time_seconds: 10 # will read forever if not set + # - type: MapToFields + # config: + # language: python + # fields: + # value: + # callable: | + # # Kafka RAW format reads messages as bytes in the 'payload' field of a Row + # lambda row: row.payload.decode('utf-8') + # output_type: string + # - type: AssertEqual + # config: + # elements: + # - {value: "123"} + # - {value: "456"} + # - {value: "789"} # TODO: Error handling hard to trigger upon initial investigations. Need to # investigate more. From 3a2f4c8ab95af91e6636d08e7d047f45dbb1398e Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Fri, 13 Jun 2025 14:20:13 +0000 Subject: [PATCH 2/4] add git issue to track --- sdks/python/apache_beam/yaml/extended_tests/messaging/kafka.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/python/apache_beam/yaml/extended_tests/messaging/kafka.yaml b/sdks/python/apache_beam/yaml/extended_tests/messaging/kafka.yaml index 355f77556c8c..790f7e70a5fc 100644 --- a/sdks/python/apache_beam/yaml/extended_tests/messaging/kafka.yaml +++ b/sdks/python/apache_beam/yaml/extended_tests/messaging/kafka.yaml @@ -30,6 +30,7 @@ pipelines: - {value: 123} - {value: 456} - {value: 789} +# TODO(#35272) - fix kafka test # - type: MapToFields # config: # language: python From f4e7cbfba84b72f56ebd77be1c871821c2dc811e Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Fri, 13 Jun 2025 14:27:05 +0000 Subject: [PATCH 3/4] revision --- .github/trigger_files/beam_PostCommit_Yaml_Xlang_Direct.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/trigger_files/beam_PostCommit_Yaml_Xlang_Direct.json b/.github/trigger_files/beam_PostCommit_Yaml_Xlang_Direct.json index 541dc4ea8e87..8ed972c9f579 100644 --- a/.github/trigger_files/beam_PostCommit_Yaml_Xlang_Direct.json +++ b/.github/trigger_files/beam_PostCommit_Yaml_Xlang_Direct.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run", - "revision": 2 + "revision": 3 } From 4a51e61e7ba8cca9410bee5ebb3909100628ca96 Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Fri, 13 Jun 2025 14:42:43 +0000 Subject: [PATCH 4/4] remove free space step - seems to be causing more issues than helping --- .github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml b/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml index 7af1e0ce88b7..9215aba0f1de 100644 --- a/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml +++ b/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml @@ -70,8 +70,6 @@ jobs: comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.test_set }} github_token: ${{ secrets.GITHUB_TOKEN }} github_job: ${{ matrix.job_name }} ${{ matrix.test_set }} (${{ matrix.job_phrase }} ${{ matrix.test_set }}) - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@v1.3.0 - name: Setup environment uses: ./.github/actions/setup-environment-action with: