Skip to content

Commit

Permalink
use script to insert pip freeze into mamba env export
Browse files Browse the repository at this point in the history
  • Loading branch information
zacharyburnett committed Jun 17, 2024
1 parent 339fee7 commit f5a308d
Show file tree
Hide file tree
Showing 3 changed files with 67 additions and 16 deletions.
10 changes: 6 additions & 4 deletions .github/workflows/build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -45,19 +45,21 @@ jobs:
environment-file: environment.yaml
create-args: >-
python=${{ matrix.python-version }}
conda
cache-downloads: true
cache-downloads-key: downloads-${{ needs.date.outputs.date }}
cache-environment: true
cache-environment-key: environment-${{ needs.date.outputs.date }}
init-shell: none
generate-run-shell: true
- run: conda env export --no-build | grep -v "name:" | grep -v "prefix:"
- run: pytest -n auto tests/test_import.py
- run: echo "filename=stenv-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python-version }}-${{ needs.date.outputs.date }}.yaml" >> $GITHUB_OUTPUT
id: output
- run: conda env export --no-build | grep -v "name:" | grep -v "prefix:" > ${{ steps.output.outputs.filename }}
- run: cat ${{ steps.output.outputs.filename }}
- run: |
micromamba env export --no-build | grep -v "name:" | grep -v "prefix:" > ${{ steps.output.outputs.filename }}
pip freeze > pip_requirements.txt
python tools/mamba_export_pip.py ${{ steps.output.outputs.filename }} pip_requirements.txt > mamba_packages.yml
mv mamba_packages.yml ${{ steps.output.outputs.filename }}
cat ${{ steps.output.outputs.filename }}
- uses: actions/upload-artifact@v4
with:
name: ${{ steps.output.outputs.filename }}
Expand Down
39 changes: 27 additions & 12 deletions .github/workflows/test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -85,18 +85,22 @@ jobs:
environment-file: stenv/environment.yaml
create-args: >-
python=${{ matrix.python-version }}
conda
cache-downloads: true
cache-downloads-key: downloads-${{ needs.build.outputs.date }}
cache-environment: true
cache-environment-key: environment-${{ needs.build.outputs.date }}
init-shell: none
generate-run-shell: true
- run: echo "version=$(conda list | awk '$1 == "${{ matrix.package }}" {print $2}')" >> $GITHUB_OUTPUT
- run: echo "version=$(pip list | awk '$1 == "${{ matrix.package }}" {print $2}')" >> $GITHUB_OUTPUT
id: package_version
# TODO: figure out a better way to use package version when checking out a Git ref
- run: micromamba install tox
- run: conda env export --no-build | grep -v "name:" | grep -v "prefix:"
- run: |
micromamba env export --no-build | grep -v "name:" | grep -v "prefix:" > ${{ steps.output.outputs.filename }}
pip freeze > pip_requirements.txt
python tools/mamba_export_pip.py ${{ steps.output.outputs.filename }} pip_requirements.txt > mamba_packages.yml
mv mamba_packages.yml ${{ steps.output.outputs.filename }}
cat ${{ steps.output.outputs.filename }}
- if: matrix.env.CRDS_SERVER_URL != ''
run: echo "context=$(crds list --operational-context)" >> $GITHUB_OUTPUT
id: crds-context
Expand Down Expand Up @@ -151,7 +155,6 @@ jobs:
environment-file: environment.yaml
create-args: >-
python=${{ matrix.python-version }}
conda
cache-downloads: true
cache-downloads-key: downloads-${{ needs.build.outputs.date }}
cache-environment: true
Expand All @@ -160,7 +163,12 @@ jobs:
generate-run-shell: true
- if: matrix.extras != ''
run: pip install "${{ matrix.package }}[${{ join(matrix.extras, ',') }}]"
- run: conda env export --no-build | grep -v "name:" | grep -v "prefix:"
- run: |
micromamba env export --no-build | grep -v "name:" | grep -v "prefix:" > ${{ steps.output.outputs.filename }}
pip freeze > pip_requirements.txt
python tools/mamba_export_pip.py ${{ steps.output.outputs.filename }} pip_requirements.txt > mamba_packages.yml
mv mamba_packages.yml ${{ steps.output.outputs.filename }}
cat ${{ steps.output.outputs.filename }}
- if: matrix.env.CRDS_SERVER_URL != ''
run: echo "context=$(crds list --operational-context)" >> $GITHUB_OUTPUT
id: crds-context
Expand Down Expand Up @@ -219,14 +227,13 @@ jobs:
environment-file: stenv/environment.yaml
create-args: >-
python=${{ matrix.python-version }}
conda
cache-downloads: true
cache-downloads-key: downloads-${{ needs.build.outputs.date }}
cache-environment: true
cache-environment-key: environment-${{ needs.build.outputs.date }}
init-shell: none
generate-run-shell: true
- run: echo "version=$(conda list | awk '$1 == "${{ matrix.package }}" {print $2}')" >> $GITHUB_OUTPUT
- run: echo "version=$(pip list | awk '$1 == "${{ matrix.package }}" {print $2}')" >> $GITHUB_OUTPUT
id: package_version
# TODO: figure out a better way to use package version when checking out a Git ref
- uses: actions/checkout@v4
Expand All @@ -238,7 +245,12 @@ jobs:
- if: matrix.extras != ''
run: pip install -e ".[${{ join(matrix.extras, ',') }}]"
working-directory: ${{ matrix.package }}
- run: conda env export --no-build | grep -v "name:" | grep -v "prefix:"
- run: |
micromamba env export --no-build | grep -v "name:" | grep -v "prefix:" > ${{ steps.output.outputs.filename }}
pip freeze > pip_requirements.txt
python tools/mamba_export_pip.py ${{ steps.output.outputs.filename }} pip_requirements.txt > mamba_packages.yml
mv mamba_packages.yml ${{ steps.output.outputs.filename }}
cat ${{ steps.output.outputs.filename }}
- if: matrix.env.CRDS_SERVER_URL != ''
run: echo "context=$(crds list --operational-context)" >> $GITHUB_OUTPUT
id: crds-context
Expand Down Expand Up @@ -295,7 +307,6 @@ jobs:
environment-file: stenv/environment.yaml
create-args: >-
python=${{ matrix.python-version }}
conda
cache-downloads: true
cache-downloads-key: downloads-${{ needs.build.outputs.date }}
cache-environment: true
Expand All @@ -304,7 +315,7 @@ jobs:
generate-run-shell: true
- run: micromamba install -y filelock requests
- run: pip install parsely roman-datamodels git+https://github.com/spacetelescope/jwst
- run: echo "version=$(conda list | awk '$1 == "${{ matrix.package }}" {print $2}')" >> $GITHUB_OUTPUT
- run: echo "version=$(pip list | awk '$1 == "${{ matrix.package }}" {print $2}')" >> $GITHUB_OUTPUT
id: package_version
# TODO: figure out a better way to use package version when checking out a Git ref
- uses: actions/checkout@v4
Expand Down Expand Up @@ -367,7 +378,6 @@ jobs:
environment-file: environment.yaml
create-args: >-
python=${{ matrix.python-version }}
conda
cache-downloads: true
cache-downloads-key: downloads-${{ needs.build.outputs.date }}
cache-environment: true
Expand All @@ -376,7 +386,12 @@ jobs:
generate-run-shell: true
- if: matrix.extras != ''
run: pip install ".[${{ join(matrix.extras, ',') }}]"
- run: conda env export --no-build | grep -v "name:" | grep -v "prefix:"
- run: |
micromamba env export --no-build | grep -v "name:" | grep -v "prefix:" > ${{ steps.output.outputs.filename }}
pip freeze > pip_requirements.txt
python tools/mamba_export_pip.py ${{ steps.output.outputs.filename }} pip_requirements.txt > mamba_packages.yml
mv mamba_packages.yml ${{ steps.output.outputs.filename }}
cat ${{ steps.output.outputs.filename }}
- if: matrix.env.CRDS_SERVER_URL != ''
run: echo "context=$(crds list --operational-context)" >> $GITHUB_OUTPUT
id: crds-context
Expand Down
34 changes: 34 additions & 0 deletions tools/mamba_export_pip.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
import argparse
import yaml
from pathlib import Path

if __name__ == "__main__":
parser = argparse.ArgumentParser(
prog="mamba_export_pip",
description="appends the output of `pip freeze` to the end of an existing Mamba YAML exported with `mamba env export`, in the corresponding `pip` section",
epilog="this script replicates the functionality of https://github.com/mamba-org/mamba/pull/2104, and will be obsolete when it is merged",
)

parser.add_argument("mamba_export_filename")
parser.add_argument("pip_freeze_filename")

arguments = parser.parse_args()

mamba_export_filename = Path(arguments.mamba_export_filename)
pip_freeze_filename = Path(arguments.pip_freeze_filename)

with open(mamba_export_filename) as mamba_export_file:
mamba_export = yaml.safe_load(mamba_export_file)

with open(pip_freeze_filename) as pip_freeze_file:
pip_freeze = pip_freeze_file.readlines()

pip_freeze = [
requirement.strip()
for requirement in pip_freeze
if " @ file" not in requirement
]

mamba_export["dependencies"].append({"pip": pip_freeze})

print(yaml.dump(mamba_export))

0 comments on commit f5a308d

Please sign in to comment.