Merge pull request #112 from naik-aakash/optimize_test_wf

Optimize test wf
autoatml · Sep 23, 2024 · f13ecc2 · f13ecc2
2 parents fc3eba4 + be025c6
commit f13ecc2
Show file tree

Hide file tree

Showing 2 changed files with 58 additions and 71 deletions.
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
@@ -4,8 +4,9 @@
 name: Testing Linux
 
 on:
+  workflow_dispatch:
   push:
-    branches: '*'
+    branches: ['*']
   pull_request:
     branches: [ main ]
 
@@ -16,55 +17,35 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
-        split: [1] #, 2, 3, 4]  # Number of splits
+        python-version: ["3.10", "3.11"]
+        split: [1, 2, 3, 4]  # Number of splits
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip cache purge
-        python -m pip install --upgrade pip
-        python -m pip install flake8 pytest pytest-mock pytest-split pytest-cov
-        python -m pip install types-setuptools
-        python -m pip cache purge
-        python -m pip install .[strict]
-        python -m pip cache purge
-        python -m pip install ase==3.23.0
-    - name: Print installed package versions
-      run: |
-        pip show torch torchdata
-    - name: Install Buildcell
-      run: |
-        curl -O https://www.mtg.msm.cam.ac.uk/files/airss-0.9.3.tgz; tar -xf airss-0.9.3.tgz; rm airss-0.9.3.tgz; cd airss; make ; make install ; make neat; cd ..
-    - name: Add Buildcell to PATH
-      run: echo "$GITHUB_WORKSPACE/airss/bin" >> $GITHUB_PATH
-    - name: Install Julia
-      run: curl -fsSL https://install.julialang.org | sh -s -- -y
-    - name: Set up Julia environment (Needed for ACEpotentials.jl interface)
-      run: |
-        julia -e 'using Pkg; Pkg.Registry.add("General"); Pkg.Registry.add(Pkg.Registry.RegistrySpec(url="https://github.com/ACEsuit/ACEregistry")); Pkg.add("ACEpotentials"); Pkg.add("DataFrames"); Pkg.add("CSV")'
-    - name: Linting with flake8
-      run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-    - name: Test with pytest and coverage
+    - name: Run tests using Docker image for Python ${{ matrix.python-version }}
       run: |
-        # run the line below locally to update tests durations file
-        # pytest --cov=autoplex --cov-append --splits 1 --group 1 --durations-path ./tests/test_data/.pytest-split-durations --store-durations
-        pytest --cov=autoplex --cov-report term-missing --cov-append --splits 1 --group ${{ matrix.split }} -vv --durations-path ./tests/test_data/.pytest-split-durations
+        docker pull ghcr.io/jageo/autoplex/autoplex:python-${{ matrix.python-version }}
+        docker run --rm \
+          -v ${{ github.workspace }}:/workspace \
+          -w /workspace \
+          ghcr.io/jageo/autoplex/autoplex:python-${{ matrix.python-version }} \
+          bash -c "
+          python -m pip install --upgrade pip && \
+          python -m uv pip install pytest pytest-mock pytest-split pytest-cov types-setuptools && \
+          python -m uv pip install .[strict] && \
+          python -m uv pip install ase==3.23.0 && \
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics && \ 
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics && \
+          pytest --cov=autoplex --cov-report term-missing --cov-append --splits 4 --group ${{ matrix.split }} -vv --durations-path ./tests/test_data/.pytest-split-durations
+        "
     - name: Upload coverage
       uses: actions/upload-artifact@v3
       with:
-        name: coverage-${{ matrix.split }}
-        path: .coverage
-
+        name: coverage-${{ matrix.python-version }}-${{ matrix.split }}
+        include-hidden-files: true
+        overwrite: false
+        path: ./.coverage
+
   coverage:
       needs: build
       runs-on: ubuntu-latest
@@ -88,20 +69,24 @@ jobs:
 
   docs:
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0} # enables conda/mamba env activation by reading bash profile
 
     steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.9"
-          cache: pip
-          cache-dependency-path: pyproject.toml
-
-      - name: Install dependencies
+      - name: Check out repo
+        uses: actions/checkout@v4
+      - name: Set up micromamba
+        uses: mamba-org/setup-micromamba@main
+      - name: Create mamba environment
         run: |
-          python -m pip install --upgrade pip
-          pip install .[docs,strict]
-
+            micromamba create -n autoplex_docs python=3.10 --yes
+      - name: Install uv
+        run: micromamba run -n autoplex_docs pip install uv
+      - name: Install autoplex and dependencies
+        run: |
+            micromamba activate autoplex_docs
+            uv pip install --upgrade pip
+            uv pip install .[docs,strict]
       - name: Build
-        run: sphinx-build -W docs _build
+        run: micromamba run -n autoplex_docs sphinx-build -W docs _build
diff --git a/Dockerfile b/Dockerfile
@@ -17,6 +17,7 @@ ARG PYTHON_VERSION
 # Install system dependencies
 RUN apt-get update && apt-get install -y \
     build-essential \
+    bash \
     bc \
     unzip \
     wget \
@@ -29,30 +30,31 @@ RUN apt-get update && apt-get install -y \
     git \
     && rm -rf /var/lib/apt/lists/*
 
+# Install Python
+RUN micromamba install -y -n base -c conda-forge \ python=${PYTHON_VERSION} && \
+    micromamba clean --all --yes
+
+# Install testing dependencies
+RUN python -m pip install --upgrade pip \
+    && pip install uv \
+    && uv pip install flake8 pre-commit pytest pytest-mock pytest-split pytest-cov types-setuptools
+
 # Install Julia
 RUN curl -fsSL https://julialang-s3.julialang.org/bin/linux/x64/1.9/julia-1.9.2-linux-x86_64.tar.gz | tar -xz -C /opt \
     && ln -s /opt/julia-1.9.2/bin/julia /usr/local/bin/julia
 
+
 # Set up Julia environment (ACEpotentials.jl interface)
 RUN julia -e 'using Pkg; Pkg.Registry.add("General"); Pkg.Registry.add(Pkg.Registry.RegistrySpec(url="https://github.com/ACEsuit/ACEregistry")); Pkg.add("ACEpotentials"); Pkg.add("DataFrames"); Pkg.add("CSV")'
 
-# Install Buildcell
-RUN curl -O https://www.mtg.msm.cam.ac.uk/files/airss-0.9.3.tgz \
-    && tar -xf airss-0.9.3.tgz \
-    && rm airss-0.9.3.tgz \
-    && cd airss \
+# Install Buildcell (airss)
+RUN curl -fsSL https://www.mtg.msm.cam.ac.uk/files/airss-0.9.3.tgz -o /opt/airss-0.9.3.tgz \
+    && tar -xf /opt/airss-0.9.3.tgz -C /opt \
+    && rm /opt/airss-0.9.3.tgz \
+    && cd /opt/airss \
     && make \
     && make install \
     && make neat
 
 # Add Buildcell to PATH
-ENV PATH="${PATH}:/airss/bin"
-
-RUN micromamba install -y -n base -c conda-forge \ python=${PYTHON_VERSION} && \
-    micromamba clean --all --yes
-
-
-# Install testing dependencies
-RUN python -m pip install --upgrade pip \
-    && pip install uv \
-    && uv pip install flake8 pre-commit pytest pytest-mock pytest-split pytest-cov types-setuptools
+ENV PATH="${PATH}:/opt/airss/bin"