From d6f21f9c1a260e2982f1ef985825171a2a99b869 Mon Sep 17 00:00:00 2001 From: Elena Khaustova Date: Wed, 4 Sep 2024 15:16:13 +0100 Subject: [PATCH 1/5] Added e2e test for packaging and running astro-airflow-iris Signed-off-by: Elena Khaustova --- features/package.feature | 10 ++++++++++ features/steps/run_steps.py | 26 ++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 features/package.feature diff --git a/features/package.feature b/features/package.feature new file mode 100644 index 00000000..8df126dc --- /dev/null +++ b/features/package.feature @@ -0,0 +1,10 @@ +Feature: Package and run all starters + + Scenario: Package astro-airflow-iris project and run packaged project + Given I have prepared a config file + And I have run a non-interactive kedro new with the starter astro-airflow-iris + When I execute the kedro command "package" + Then I should get a successful exit code + When I install the project's python package + And I execute the installed project + Then I should get a successful exit code \ No newline at end of file diff --git a/features/steps/run_steps.py b/features/steps/run_steps.py index c0853e48..90f7f269 100644 --- a/features/steps/run_steps.py +++ b/features/steps/run_steps.py @@ -169,3 +169,29 @@ def check_status_code(context): assert False, "Expected exit code {}" " but got {}".format( OK_EXIT_CODE, context.result.returncode ) + + +@when('I execute the kedro command "{command}"') +def exec_kedro_target(context, command): + """Execute Kedro target.""" + split_command = command.split() + cmd = [context.kedro, *split_command] + context.result = subprocess.run(cmd, env=context.env, cwd=str(context.root_project_dir)) + + +@when("I install the project's python package") +def install_project_package_via_pip(context): + """Install a python package using pip.""" + dist_dir = context.root_project_dir / "dist" + (whl_file,) = dist_dir.glob("*.whl") + subprocess.run([context.pip, "install", str(whl_file)], env=context.env) + + +@when("I execute the installed project") +def exec_project(context): + """Execute installed Kedro project target.""" + context.result = subprocess.run( + [context.python, "-m", context.project_name.replace("-", "_")], + cwd=context.root_project_dir, + env=context.env + ) From 2afcaef6c5232f79cb64ff40ab785065f5450513 Mon Sep 17 00:00:00 2001 From: Elena Khaustova Date: Wed, 4 Sep 2024 15:45:26 +0100 Subject: [PATCH 2/5] Added the rest e2e project packaging tests Signed-off-by: Elena Khaustova --- features/package.feature | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/features/package.feature b/features/package.feature index 8df126dc..76fbde36 100644 --- a/features/package.feature +++ b/features/package.feature @@ -7,4 +7,40 @@ Feature: Package and run all starters Then I should get a successful exit code When I install the project's python package And I execute the installed project + Then I should get a successful exit code + + Scenario: Package spaceflights-pandas project and run packaged project + Given I have prepared a config file + And I have run a non-interactive kedro new with the starter spaceflights-pandas + When I execute the kedro command "package" + Then I should get a successful exit code + When I install the project's python package + And I execute the installed project + Then I should get a successful exit code + + Scenario: Package spaceflights-pandas-viz project and run packaged project + Given I have prepared a config file + And I have run a non-interactive kedro new with the starter spaceflights-pandas-viz + When I execute the kedro command "package" + Then I should get a successful exit code + When I install the project's python package + And I execute the installed project + Then I should get a successful exit code + + Scenario: Package spaceflights-pyspark project and run packaged project + Given I have prepared a config file + And I have run a non-interactive kedro new with the starter spaceflights-pyspark + When I execute the kedro command "package" + Then I should get a successful exit code + When I install the project's python package + And I execute the installed project + Then I should get a successful exit code + + Scenario: Package spaceflights-pandas-viz project and run packaged project + Given I have prepared a config file + And I have run a non-interactive kedro new with the starter spaceflights-pyspark-viz + When I execute the kedro command "package" + Then I should get a successful exit code + When I install the project's python package + And I execute the installed project Then I should get a successful exit code \ No newline at end of file From b35b91eb610a7a26170ab08a51501da94d925e1c Mon Sep 17 00:00:00 2001 From: Elena Khaustova Date: Wed, 4 Sep 2024 17:23:15 +0100 Subject: [PATCH 3/5] Added github action to run added e2e tests at the CI Signed-off-by: Elena Khaustova --- .github/workflows/all-checks.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/all-checks.yml b/.github/workflows/all-checks.yml index a059e2e9..5ab0374f 100644 --- a/.github/workflows/all-checks.yml +++ b/.github/workflows/all-checks.yml @@ -78,3 +78,21 @@ jobs: run: make install-test-requirements - name: Run linter for all starters run: behave features/lint.feature + + package: + strategy: + matrix: + os: [ ubuntu-latest, windows-latest ] + python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Python ${{inputs.python-version}} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install test requirements + run: make install-test-requirements + - name: Package and run all starters + run: behave features/package.feature From ef209d250c30c970e871ff8c49505f1c1989b182 Mon Sep 17 00:00:00 2001 From: Elena Khaustova Date: Wed, 4 Sep 2024 19:11:04 +0100 Subject: [PATCH 4/5] Updated github action Signed-off-by: Elena Khaustova --- .github/workflows/all-checks.yml | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/.github/workflows/all-checks.yml b/.github/workflows/all-checks.yml index 5ab0374f..1178f469 100644 --- a/.github/workflows/all-checks.yml +++ b/.github/workflows/all-checks.yml @@ -82,17 +82,35 @@ jobs: package: strategy: matrix: - os: [ ubuntu-latest, windows-latest ] + os: [ windows-latest , ubuntu-latest ] python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ] runs-on: ${{ matrix.os }} steps: - name: Checkout code uses: actions/checkout@v3 - - name: Set up Python ${{inputs.python-version}} + - name: Set up Python ${{matrix.python-version}} uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Install test requirements - run: make install-test-requirements + run: | + make install-test-requirements + - name: Add MSBuild to PATH + if: matrix.os == 'windows-latest' + uses: microsoft/setup-msbuild@v1 + - name: Install Microsoft Visual C++ Redistributable + if: matrix.os == 'windows-latest' + run: | + choco install vcredist-all + - name: Setup Hadoop binary + if: matrix.os == 'windows-latest' + run: | + Invoke-WebRequest "https://github.com/steveloughran/winutils/blob/master/hadoop-2.7.1/bin/winutils.exe?raw=true" -OutFile winutils.exe + Invoke-WebRequest "https://github.com/steveloughran/winutils/blob/master/hadoop-2.7.1/bin/hadoop.dll?raw=true" -OutFile hadoop.dll + Move-Item .\hadoop.dll C:\Windows\System32 + New-Item -ItemType directory -Path C:\hadoop\bin + Move-Item .\winutils.exe C:\hadoop\bin + echo "HADOOP_HOME=C:\hadoop" | Out-File -Append -Encoding ascii -FilePath $env:GITHUB_ENV + echo "PATH=$env:HADOOP_HOME\bin;$env:PATH" | Out-File -Append -Encoding ascii -FilePath $env:GITHUB_ENV - name: Package and run all starters run: behave features/package.feature From 0f9b7ddbe969ae484e010f4ce2b63ac018ad3260 Mon Sep 17 00:00:00 2001 From: ElenaKhaustova <157851531+ElenaKhaustova@users.noreply.github.com> Date: Mon, 9 Sep 2024 12:11:04 +0100 Subject: [PATCH 5/5] Fix typo Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> --- features/package.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/package.feature b/features/package.feature index 76fbde36..0f67a2d9 100644 --- a/features/package.feature +++ b/features/package.feature @@ -36,7 +36,7 @@ Feature: Package and run all starters And I execute the installed project Then I should get a successful exit code - Scenario: Package spaceflights-pandas-viz project and run packaged project + Scenario: Package spaceflights-pyspark-viz project and run packaged project Given I have prepared a config file And I have run a non-interactive kedro new with the starter spaceflights-pyspark-viz When I execute the kedro command "package"