diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index cc8cb56..8710bc9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -45,19 +45,21 @@ jobs: environment-file: environment.yaml create-args: >- python=${{ matrix.python-version }} - conda cache-downloads: true cache-downloads-key: downloads-${{ needs.date.outputs.date }} cache-environment: true cache-environment-key: environment-${{ needs.date.outputs.date }} init-shell: none generate-run-shell: true - - run: conda env export --no-build | grep -v "name:" | grep -v "prefix:" - run: pytest -n auto tests/test_import.py - run: echo "filename=stenv-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python-version }}-${{ needs.date.outputs.date }}.yaml" >> $GITHUB_OUTPUT id: output - - run: conda env export --no-build | grep -v "name:" | grep -v "prefix:" > ${{ steps.output.outputs.filename }} - - run: cat ${{ steps.output.outputs.filename }} + - run: | + micromamba env export --no-build | grep -v "name:" | grep -v "prefix:" > ${{ steps.output.outputs.filename }} + pip freeze > pip_requirements.txt + python tools/mamba_export_pip.py ${{ steps.output.outputs.filename }} pip_requirements.txt > mamba_packages.yml + mv mamba_packages.yml ${{ steps.output.outputs.filename }} + cat ${{ steps.output.outputs.filename }} - uses: actions/upload-artifact@v4 with: name: ${{ steps.output.outputs.filename }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index abf3d95..bbcdcd5 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -85,18 +85,22 @@ jobs: environment-file: stenv/environment.yaml create-args: >- python=${{ matrix.python-version }} - conda cache-downloads: true cache-downloads-key: downloads-${{ needs.build.outputs.date }} cache-environment: true cache-environment-key: environment-${{ needs.build.outputs.date }} init-shell: none generate-run-shell: true - - run: echo "version=$(conda list | awk '$1 == "${{ matrix.package }}" {print $2}')" >> $GITHUB_OUTPUT + - run: echo "version=$(pip list | awk '$1 == "${{ matrix.package }}" {print $2}')" >> $GITHUB_OUTPUT id: package_version # TODO: figure out a better way to use package version when checking out a Git ref - run: micromamba install tox - - run: conda env export --no-build | grep -v "name:" | grep -v "prefix:" + - run: | + micromamba env export --no-build | grep -v "name:" | grep -v "prefix:" > ${{ steps.output.outputs.filename }} + pip freeze > pip_requirements.txt + python tools/mamba_export_pip.py ${{ steps.output.outputs.filename }} pip_requirements.txt > mamba_packages.yml + mv mamba_packages.yml ${{ steps.output.outputs.filename }} + cat ${{ steps.output.outputs.filename }} - if: matrix.env.CRDS_SERVER_URL != '' run: echo "context=$(crds list --operational-context)" >> $GITHUB_OUTPUT id: crds-context @@ -151,7 +155,6 @@ jobs: environment-file: environment.yaml create-args: >- python=${{ matrix.python-version }} - conda cache-downloads: true cache-downloads-key: downloads-${{ needs.build.outputs.date }} cache-environment: true @@ -160,7 +163,12 @@ jobs: generate-run-shell: true - if: matrix.extras != '' run: pip install "${{ matrix.package }}[${{ join(matrix.extras, ',') }}]" - - run: conda env export --no-build | grep -v "name:" | grep -v "prefix:" + - run: | + micromamba env export --no-build | grep -v "name:" | grep -v "prefix:" > ${{ steps.output.outputs.filename }} + pip freeze > pip_requirements.txt + python tools/mamba_export_pip.py ${{ steps.output.outputs.filename }} pip_requirements.txt > mamba_packages.yml + mv mamba_packages.yml ${{ steps.output.outputs.filename }} + cat ${{ steps.output.outputs.filename }} - if: matrix.env.CRDS_SERVER_URL != '' run: echo "context=$(crds list --operational-context)" >> $GITHUB_OUTPUT id: crds-context @@ -219,14 +227,13 @@ jobs: environment-file: stenv/environment.yaml create-args: >- python=${{ matrix.python-version }} - conda cache-downloads: true cache-downloads-key: downloads-${{ needs.build.outputs.date }} cache-environment: true cache-environment-key: environment-${{ needs.build.outputs.date }} init-shell: none generate-run-shell: true - - run: echo "version=$(conda list | awk '$1 == "${{ matrix.package }}" {print $2}')" >> $GITHUB_OUTPUT + - run: echo "version=$(pip list | awk '$1 == "${{ matrix.package }}" {print $2}')" >> $GITHUB_OUTPUT id: package_version # TODO: figure out a better way to use package version when checking out a Git ref - uses: actions/checkout@v4 @@ -238,7 +245,12 @@ jobs: - if: matrix.extras != '' run: pip install -e ".[${{ join(matrix.extras, ',') }}]" working-directory: ${{ matrix.package }} - - run: conda env export --no-build | grep -v "name:" | grep -v "prefix:" + - run: | + micromamba env export --no-build | grep -v "name:" | grep -v "prefix:" > ${{ steps.output.outputs.filename }} + pip freeze > pip_requirements.txt + python tools/mamba_export_pip.py ${{ steps.output.outputs.filename }} pip_requirements.txt > mamba_packages.yml + mv mamba_packages.yml ${{ steps.output.outputs.filename }} + cat ${{ steps.output.outputs.filename }} - if: matrix.env.CRDS_SERVER_URL != '' run: echo "context=$(crds list --operational-context)" >> $GITHUB_OUTPUT id: crds-context @@ -295,7 +307,6 @@ jobs: environment-file: stenv/environment.yaml create-args: >- python=${{ matrix.python-version }} - conda cache-downloads: true cache-downloads-key: downloads-${{ needs.build.outputs.date }} cache-environment: true @@ -304,7 +315,7 @@ jobs: generate-run-shell: true - run: micromamba install -y filelock requests - run: pip install parsely roman-datamodels git+https://github.com/spacetelescope/jwst - - run: echo "version=$(conda list | awk '$1 == "${{ matrix.package }}" {print $2}')" >> $GITHUB_OUTPUT + - run: echo "version=$(pip list | awk '$1 == "${{ matrix.package }}" {print $2}')" >> $GITHUB_OUTPUT id: package_version # TODO: figure out a better way to use package version when checking out a Git ref - uses: actions/checkout@v4 @@ -367,7 +378,6 @@ jobs: environment-file: environment.yaml create-args: >- python=${{ matrix.python-version }} - conda cache-downloads: true cache-downloads-key: downloads-${{ needs.build.outputs.date }} cache-environment: true @@ -376,7 +386,12 @@ jobs: generate-run-shell: true - if: matrix.extras != '' run: pip install ".[${{ join(matrix.extras, ',') }}]" - - run: conda env export --no-build | grep -v "name:" | grep -v "prefix:" + - run: | + micromamba env export --no-build | grep -v "name:" | grep -v "prefix:" > ${{ steps.output.outputs.filename }} + pip freeze > pip_requirements.txt + python tools/mamba_export_pip.py ${{ steps.output.outputs.filename }} pip_requirements.txt > mamba_packages.yml + mv mamba_packages.yml ${{ steps.output.outputs.filename }} + cat ${{ steps.output.outputs.filename }} - if: matrix.env.CRDS_SERVER_URL != '' run: echo "context=$(crds list --operational-context)" >> $GITHUB_OUTPUT id: crds-context diff --git a/tools/mamba_export_pip.py b/tools/mamba_export_pip.py new file mode 100644 index 0000000..8aed1f3 --- /dev/null +++ b/tools/mamba_export_pip.py @@ -0,0 +1,34 @@ +import argparse +import yaml +from pathlib import Path + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="mamba_export_pip", + description="appends the output of `pip freeze` to the end of an existing Mamba YAML exported with `mamba env export`, in the corresponding `pip` section", + epilog="this script replicates the functionality of https://github.com/mamba-org/mamba/pull/2104, and will be obsolete when it is merged", + ) + + parser.add_argument("mamba_export_filename") + parser.add_argument("pip_freeze_filename") + + arguments = parser.parse_args() + + mamba_export_filename = Path(arguments.mamba_export_filename) + pip_freeze_filename = Path(arguments.pip_freeze_filename) + + with open(mamba_export_filename) as mamba_export_file: + mamba_export = yaml.safe_load(mamba_export_file) + + with open(pip_freeze_filename) as pip_freeze_file: + pip_freeze = pip_freeze_file.readlines() + + pip_freeze = [ + requirement.strip() + for requirement in pip_freeze + if " @ file" not in requirement + ] + + mamba_export["dependencies"].append({"pip": pip_freeze}) + + print(yaml.dump(mamba_export))