use script to insert pip freeze into mamba env export

spacetelescope · Jun 17, 2024 · 49aa9d8 · 49aa9d8
1 parent 339fee7
commit 49aa9d8
Show file tree

Hide file tree

Showing 3 changed files with 62 additions and 16 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -45,19 +45,20 @@ jobs:
           environment-file: environment.yaml
           create-args: >-
             python=${{ matrix.python-version }}
-            conda
           cache-downloads: true
           cache-downloads-key: downloads-${{ needs.date.outputs.date }}
           cache-environment: true
           cache-environment-key: environment-${{ needs.date.outputs.date }}
           init-shell: none
           generate-run-shell: true
-      - run: conda env export --no-build | grep -v "name:" | grep -v "prefix:"
       - run: pytest -n auto tests/test_import.py
       - run: echo "filename=stenv-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python-version }}-${{ needs.date.outputs.date }}.yaml" >> $GITHUB_OUTPUT
         id: output
-      - run: conda env export --no-build | grep -v "name:" | grep -v "prefix:" > ${{ steps.output.outputs.filename }}
-      - run: cat ${{ steps.output.outputs.filename }}
+      - run: |
+          micromamba env export --no-build | grep -v "name:" | grep -v "prefix:" > ${{ steps.output.outputs.filename }}
+          pip freeze > pip_requirements.txt
+          python tools/mamba_export_pip.py ${{ steps.output.outputs.filename }} pip_requirements.txt > ${{ steps.output.outputs.filename }}
+          cat ${{ steps.output.outputs.filename }}
       - uses: actions/upload-artifact@v4
         with:
           name: ${{ steps.output.outputs.filename }}

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -85,18 +85,21 @@ jobs:
           environment-file: stenv/environment.yaml
           create-args: >-
             python=${{ matrix.python-version }}
-            conda
           cache-downloads: true
           cache-downloads-key: downloads-${{ needs.build.outputs.date }}
           cache-environment: true
           cache-environment-key: environment-${{ needs.build.outputs.date }}
           init-shell: none
           generate-run-shell: true
-      - run: echo "version=$(conda list | awk '$1 == "${{ matrix.package }}" {print $2}')" >> $GITHUB_OUTPUT
+      - run: echo "version=$(pip list | awk '$1 == "${{ matrix.package }}" {print $2}')" >> $GITHUB_OUTPUT
         id: package_version
         # TODO: figure out a better way to use package version when checking out a Git ref
       - run: micromamba install tox
-      - run: conda env export --no-build | grep -v "name:" | grep -v "prefix:"
+      - run: |
+          micromamba env export --no-build | grep -v "name:" | grep -v "prefix:" > ${{ steps.output.outputs.filename }}
+          pip freeze > pip_requirements.txt
+          python tools/mamba_export_pip.py ${{ steps.output.outputs.filename }} pip_requirements.txt > ${{ steps.output.outputs.filename }}
+          cat ${{ steps.output.outputs.filename }}
       - if: matrix.env.CRDS_SERVER_URL != ''
         run: echo "context=$(crds list --operational-context)" >> $GITHUB_OUTPUT
         id: crds-context
@@ -151,7 +154,6 @@ jobs:
           environment-file: environment.yaml
           create-args: >-
             python=${{ matrix.python-version }}
-            conda
           cache-downloads: true
           cache-downloads-key: downloads-${{ needs.build.outputs.date }}
           cache-environment: true
@@ -160,7 +162,11 @@ jobs:
           generate-run-shell: true
       - if: matrix.extras != ''
         run: pip install "${{ matrix.package }}[${{ join(matrix.extras, ',') }}]"
-      - run: conda env export --no-build | grep -v "name:" | grep -v "prefix:"
+      - run: |
+          micromamba env export --no-build | grep -v "name:" | grep -v "prefix:" > ${{ steps.output.outputs.filename }}
+          pip freeze > pip_requirements.txt
+          python tools/mamba_export_pip.py ${{ steps.output.outputs.filename }} pip_requirements.txt > ${{ steps.output.outputs.filename }}
+          cat ${{ steps.output.outputs.filename }}
       - if: matrix.env.CRDS_SERVER_URL != ''
         run: echo "context=$(crds list --operational-context)" >> $GITHUB_OUTPUT
         id: crds-context
@@ -219,14 +225,13 @@ jobs:
           environment-file: stenv/environment.yaml
           create-args: >-
             python=${{ matrix.python-version }}
-            conda
           cache-downloads: true
           cache-downloads-key: downloads-${{ needs.build.outputs.date }}
           cache-environment: true
           cache-environment-key: environment-${{ needs.build.outputs.date }}
           init-shell: none
           generate-run-shell: true
-      - run: echo "version=$(conda list | awk '$1 == "${{ matrix.package }}" {print $2}')" >> $GITHUB_OUTPUT
+      - run: echo "version=$(pip list | awk '$1 == "${{ matrix.package }}" {print $2}')" >> $GITHUB_OUTPUT
         id: package_version
         # TODO: figure out a better way to use package version when checking out a Git ref
       - uses: actions/checkout@v4
@@ -238,7 +243,11 @@ jobs:
       - if: matrix.extras != ''
         run: pip install -e ".[${{ join(matrix.extras, ',') }}]"
         working-directory: ${{ matrix.package }}
-      - run: conda env export --no-build | grep -v "name:" | grep -v "prefix:"
+      - run: |
+          micromamba env export --no-build | grep -v "name:" | grep -v "prefix:" > ${{ steps.output.outputs.filename }}
+          pip freeze > pip_requirements.txt
+          python tools/mamba_export_pip.py ${{ steps.output.outputs.filename }} pip_requirements.txt > ${{ steps.output.outputs.filename }}
+          cat ${{ steps.output.outputs.filename }}
       - if: matrix.env.CRDS_SERVER_URL != ''
         run: echo "context=$(crds list --operational-context)" >> $GITHUB_OUTPUT
         id: crds-context
@@ -295,7 +304,6 @@ jobs:
           environment-file: stenv/environment.yaml
           create-args: >-
             python=${{ matrix.python-version }}
-            conda
           cache-downloads: true
           cache-downloads-key: downloads-${{ needs.build.outputs.date }}
           cache-environment: true
@@ -304,7 +312,7 @@ jobs:
           generate-run-shell: true
       - run: micromamba install -y filelock requests
       - run: pip install parsely roman-datamodels git+https://github.com/spacetelescope/jwst
-      - run: echo "version=$(conda list | awk '$1 == "${{ matrix.package }}" {print $2}')" >> $GITHUB_OUTPUT
+      - run: echo "version=$(pip list | awk '$1 == "${{ matrix.package }}" {print $2}')" >> $GITHUB_OUTPUT
         id: package_version
         # TODO: figure out a better way to use package version when checking out a Git ref
       - uses: actions/checkout@v4
@@ -367,7 +375,6 @@ jobs:
           environment-file: environment.yaml
           create-args: >-
             python=${{ matrix.python-version }}
-            conda
           cache-downloads: true
           cache-downloads-key: downloads-${{ needs.build.outputs.date }}
           cache-environment: true
@@ -376,7 +383,11 @@ jobs:
           generate-run-shell: true
       - if: matrix.extras != ''
         run: pip install ".[${{ join(matrix.extras, ',') }}]"
-      - run: conda env export --no-build | grep -v "name:" | grep -v "prefix:"
+      - run: |
+          micromamba env export --no-build | grep -v "name:" | grep -v "prefix:" > ${{ steps.output.outputs.filename }}
+          pip freeze > pip_requirements.txt
+          python tools/mamba_export_pip.py ${{ steps.output.outputs.filename }} pip_requirements.txt > ${{ steps.output.outputs.filename }}
+          cat ${{ steps.output.outputs.filename }}
       - if: matrix.env.CRDS_SERVER_URL != ''
         run: echo "context=$(crds list --operational-context)" >> $GITHUB_OUTPUT
         id: crds-context

diff --git a/tools/mamba_export_pip.py b/tools/mamba_export_pip.py
@@ -0,0 +1,34 @@
+import argparse
+import yaml
+from pathlib import Path
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        prog="mamba_export_pip",
+        description="appends the output of `pip freeze` to the end of an existing Mamba YAML exported with `mamba env export`, in the corresponding `pip` section",
+        epilog="this script replicates the functionality of https://github.com/mamba-org/mamba/pull/2104, and will be obsolete when it is merged",
+    )
+
+    parser.add_argument("mamba_export_filename")
+    parser.add_argument("pip_freeze_filename")
+
+    arguments = parser.parse_args()
+
+    mamba_export_filename = Path(arguments.mamba_export_filename)
+    pip_freeze_filename = Path(arguments.pip_freeze_filename)
+
+    with open(mamba_export_filename) as mamba_export_file:
+        mamba_export = yaml.safe_load(mamba_export_file)
+
+    with open(pip_freeze_filename) as pip_freeze_file:
+        pip_freeze = pip_freeze_file.readlines()
+
+    pip_freeze = [
+        requirement.strip()
+        for requirement in pip_freeze
+        if " @ file" not in requirement
+    ]
+
+    mamba_export["dependencies"].append({"pip": pip_freeze})
+
+    print(yaml.dump(mamba_export))