diff --git a/.drone.yml b/.drone.yml deleted file mode 100644 index 5db4ddf1f8d..00000000000 --- a/.drone.yml +++ /dev/null @@ -1,91 +0,0 @@ -# Continuous integration - -kind: pipeline -name: skema-rs - -steps: -- name: test_rust_components - image: rust:1.67 - commands: - - apt update && apt install -y cmake openssl libclang-13-dev - - cd skema/skema-rs - - cargo test --verbose --all - when: - paths: - - .drone.yml - - skema/skema-rs/** - - data/gromet/** - ---- -kind: pipeline -name: skema_tr - -steps: -- name: test_scala - image: sbtscala/scala-sbt:openjdk-8u342_1.7.2_2.12.17 - commands: - - cd skema/text_reading/text_reading - - export _JAVA_OPTIONS=-Xmx16g - - sbt webapp/test - when: - paths: - - .drone.yml - - skema/text_reading/scala/** - ---- -kind: pipeline -name: skema-pa - -steps: -- name: test_program_analysis - image: python:3.8-bullseye - commands: - - apt-get update - - apt-get -y install build-essential graphviz libgraphviz-dev - - pip install ".[core,dev]" - - pytest --cov=skema --ignore=skema/img2mml skema/program_analysis/tests - when: - paths: - - pyproject.toml - - .drone.yml - - skema/program_analysis/** - - skema/gromet/** - - skema/model_assembly/** - - data/gromet/** - ---- -kind: pipeline -name: skema-py - -steps: -- name: test_skema_py - image: python:3.8-bullseye - commands: - - apt-get update - - apt-get -y install build-essential graphviz libgraphviz-dev - - pip install ".[core,dev]" - - pytest --cov=skema skema/skema_py/tests - when: - paths: - - pyproject.toml - - .drone.yml - - skema/skema_py/** - ---- -kind: pipeline -name: skema-er - -steps: -- name: test_equation_reading - image: python:3.8-bullseye - commands: - - apt-get update - - apt-get -y install build-essential graphviz libgraphviz-dev - - curl -L https://artifacts.askem.lum.ai/skema/img2mml/models/cnn_xfmer_arxiv_im2mml_with_fonts_boldface_best.pt > skema/img2mml/trained_models/cnn_xfmer_arxiv_im2mml_with_fonts_boldface_best.pt - - pip install ".[core,dev]" - - pytest -s --cov=skema skema/img2mml/tests - when: - paths: - - pyproject.toml - - .drone.yml - - skema/img2mml/** diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100644 index 1df1a6bba03..00000000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: SKEMA AWS -on: - workflow_run: - workflows: ["SKEMA docker"] - branches: [main] - types: ["completed"] -env: - SKEMA_ECS_CLUSTER_NAME_MAIN: ${{ secrets.SKEMA_ECS_CLUSTER_NAME_MAIN }} - SKEMA_ECS_SERVICE_NAME_MAIN: ${{ secrets.SKEMA_ECS_SERVICE_NAME_MAIN }} - SKEMA_AWS_REGION: ${{ secrets.SKEMA_AWS_REGION }} -jobs: - deploy: - if: ${{ github.event.workflow_run.conclusion == 'success' }} - name: "deploy skema system to AWS" - runs-on: ubuntu-latest - steps: - # main deployment - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.SKEMA_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SKEMA_AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.SKEMA_AWS_REGION }} - - name: Deploy - if: github.ref == 'refs/heads/main' - run: | - # update skema-py - aws ecs update-service --cluster $SKEMA_ECS_CLUSTER_NAME_MAIN --service darpa-askem-main-ecs-skema-py --force-new-deployment --region $SKEMA_AWS_REGION &>/dev/null - # update skema-rs - aws ecs update-service --cluster $SKEMA_ECS_CLUSTER_NAME_MAIN --service darpa-askem-main-ecs-skema-rs --force-new-deployment --region $SKEMA_AWS_REGION &>/dev/null \ No newline at end of file diff --git a/.github/workflows/tests-and-docs.yml b/.github/workflows/tests-and-docs.yml index 0b9b895fd01..0f3b4bb0386 100644 --- a/.github/workflows/tests-and-docs.yml +++ b/.github/workflows/tests-and-docs.yml @@ -81,7 +81,10 @@ jobs: working-directory: . run: | # retrieve latest model for img2mml component - curl -L https://artifacts.askem.lum.ai/skema/img2mml/models/cnn_xfmer_arxiv_im2mml_with_fonts_boldface_best.pt > skema/img2mml/trained_models/cnn_xfmer_arxiv_im2mml_with_fonts_boldface_best.pt + pip install huggingface_hub + python scripts/retrieve_model_ci.py + + # Install askem pip install ".[all]" # Install tree-sitter parser (for Python component unit tests) diff --git a/Dockerfile.skema-py b/Dockerfile.skema-py index eac477f26bc..169bdabea7c 100644 --- a/Dockerfile.skema-py +++ b/Dockerfile.skema-py @@ -53,7 +53,7 @@ ENV PATH="/root/.cargo/bin:${PATH}" RUN pip install wheel RUN pip install six # Download ML model (~150MB) -RUN curl -L https://artifacts.askem.lum.ai/skema/img2mml/models/cnn_xfmer_arxiv_im2mml_with_fonts_boldface_best.pt > skema/img2mml/trained_models/cnn_xfmer_arxiv_im2mml_with_fonts_boldface_best.pt +RUN pip install huggingface_hub && python scripts/retrieve_model_ci.py RUN tree /app #RUN pip install ".[all]" # exclude dependencies for docs diff --git a/docs/dev/env.md b/docs/dev/env.md index 3302c99eae8..2346c6cfd9d 100644 --- a/docs/dev/env.md +++ b/docs/dev/env.md @@ -9,7 +9,7 @@ conda activate skema # Install tree-sitter parsers python skema/program_analysis/tree_sitter_parsers/build_parsers.py --all # download the checkpoint for the img2mml service -curl -L https://artifacts.askem.lum.ai/skema/img2mml/models/cnn_xfmer_arxiv_im2mml_with_fonts_boldface_best.pt > skema/img2mml/trained_models/cnn_xfmer_arxiv_im2mml_with_fonts_boldface_best.pt +python scripts/retrieve_model.py # mathjax deps for img2mml (cd skema/img2mml/data_generation && npm install) ``` diff --git a/pyproject.toml b/pyproject.toml index 09c7a3b55de..a7a1698bae0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ isa = [ ] # shared ML dependencies -ml = ["torch==2.0.1", "torchvision==0.15.2", "beartype==0.15.0"] +ml = ["torch==2.0.1", "torchvision==0.15.2", "beartype==0.15.0", "huggingface_hub"] # Im2MML dependencies. The img2mml service converts equation images to MathML. # See the skema/img2mml directory. @@ -100,6 +100,7 @@ all = ["skema[core]", "skema[dev]", "skema[doc]", "skema[demo]", "skema[annotati "skema.rest" = "skema/rest" "skema.skema_py" = "skema/skema_py" "skema.utils" = "skema/utils" +"skema.data" = "skema/data" # re-map skema/text_reading/python to skema.text_reading #"skema.text_reading" = "skema/text_reading/python" @@ -110,7 +111,7 @@ all = ["skema[core]", "skema[dev]", "skema[doc]", "skema[demo]", "skema[annotati [tool.setuptools.package-data] # needed to ensure models are included in package/discoverable -"*" = ["*.json", "*vocab.txt", "*.pt", "*.png", "*.html", "*.yml", "*.yaml"] +"*" = ["*.json", "*vocab.txt", "*.pt", "*.png", "*.html", "*.yml", "*.yaml", "*.zip"] [tool.setuptools.dynamic] readme = {file = ["README.md"], content-type = "text/markdown"} diff --git a/scripts/retrieve_model_ci.py b/scripts/retrieve_model_ci.py new file mode 100644 index 00000000000..6726da52b4c --- /dev/null +++ b/scripts/retrieve_model_ci.py @@ -0,0 +1,31 @@ +import os +from pathlib import Path + +from huggingface_hub import hf_hub_download + +def retrieve_model(model_path=None) -> str: + """ + Retrieve the img2mml model from the specified path or download it if not found. + + Args: + model_path (str, optional): Path to the img2mml model file. Defaults to None. + + Returns: + str: Path to the loaded model file. + """ + cwd = Path(__file__).parents[0] + REPO_NAME = "lum-ai/img2mml" + MODEL_NAME = "cnn_xfmer_arxiv_im2mml_with_fonts_boldface_best.pt" + # If the model path is none or doesn't exist, the default model will be downloaded from server. + if model_path is None or not os.path.exists(model_path): + model_path = cwd / "trained_models" / MODEL_NAME + + # Check if the model file already exists + if not os.path.exists(model_path): + # If the file doesn't exist, download it from the specified URL + print(f"Downloading the model checkpoint from HuggingFace...") + hf_hub_download(repo_id=REPO_NAME, filename=MODEL_NAME, local_dir=model_path.parent, local_dir_use_symlinks=False) + + return str(model_path) + +retrieve_model() \ No newline at end of file diff --git a/skema/data/program_analysis/ABM-COVID-ABS.zip b/skema/data/program_analysis/ABM-COVID-ABS.zip new file mode 100644 index 00000000000..f906eddc34a Binary files /dev/null and b/skema/data/program_analysis/ABM-COVID-ABS.zip differ diff --git a/skema/data/program_analysis/ABM-COmplexVID-19.zip b/skema/data/program_analysis/ABM-COmplexVID-19.zip new file mode 100644 index 00000000000..160fdb0382e Binary files /dev/null and b/skema/data/program_analysis/ABM-COmplexVID-19.zip differ diff --git a/skema/data/program_analysis/ABM-Covasim.zip b/skema/data/program_analysis/ABM-Covasim.zip new file mode 100644 index 00000000000..c1a244bdec0 Binary files /dev/null and b/skema/data/program_analysis/ABM-Covasim.zip differ diff --git a/skema/data/program_analysis/ABM-REINA.zip b/skema/data/program_analysis/ABM-REINA.zip new file mode 100644 index 00000000000..4377c6f118a Binary files /dev/null and b/skema/data/program_analysis/ABM-REINA.zip differ diff --git a/skema/data/program_analysis/Bucky.zip b/skema/data/program_analysis/Bucky.zip new file mode 100644 index 00000000000..0d143a03989 Binary files /dev/null and b/skema/data/program_analysis/Bucky.zip differ diff --git a/skema/data/program_analysis/CHIME-SIR-model.zip b/skema/data/program_analysis/CHIME-SIR-model.zip new file mode 100644 index 00000000000..63a27b171fe Binary files /dev/null and b/skema/data/program_analysis/CHIME-SIR-model.zip differ diff --git a/skema/data/program_analysis/CHIME-SVIIvR-model.zip b/skema/data/program_analysis/CHIME-SVIIvR-model.zip new file mode 100644 index 00000000000..1f8f7913478 Binary files /dev/null and b/skema/data/program_analysis/CHIME-SVIIvR-model.zip differ diff --git a/skema/data/program_analysis/CHIME-penn-full-model.zip b/skema/data/program_analysis/CHIME-penn-full-model.zip new file mode 100644 index 00000000000..db3376fdb30 Binary files /dev/null and b/skema/data/program_analysis/CHIME-penn-full-model.zip differ diff --git a/skema/data/program_analysis/Climlab.zip b/skema/data/program_analysis/Climlab.zip new file mode 100644 index 00000000000..c8b94b28838 Binary files /dev/null and b/skema/data/program_analysis/Climlab.zip differ diff --git a/skema/data/program_analysis/Cornell-COVID19-sim-Frazier.zip b/skema/data/program_analysis/Cornell-COVID19-sim-Frazier.zip new file mode 100644 index 00000000000..007e1ef2b7e Binary files /dev/null and b/skema/data/program_analysis/Cornell-COVID19-sim-Frazier.zip differ diff --git a/skema/data/program_analysis/Generated-Halfar.zip b/skema/data/program_analysis/Generated-Halfar.zip new file mode 100644 index 00000000000..d37e3256f39 Binary files /dev/null and b/skema/data/program_analysis/Generated-Halfar.zip differ diff --git a/skema/data/program_analysis/MechBayes.zip b/skema/data/program_analysis/MechBayes.zip new file mode 100644 index 00000000000..15cb0ecb3e2 Binary files /dev/null and b/skema/data/program_analysis/MechBayes.zip differ diff --git a/skema/data/program_analysis/SIDARTHE.zip b/skema/data/program_analysis/SIDARTHE.zip new file mode 100644 index 00000000000..833a6f51c4f Binary files /dev/null and b/skema/data/program_analysis/SIDARTHE.zip differ diff --git a/skema/data/program_analysis/SV2AIR3-Waterloo-MATLAB.zip b/skema/data/program_analysis/SV2AIR3-Waterloo-MATLAB.zip new file mode 100644 index 00000000000..96ee3f54aaf Binary files /dev/null and b/skema/data/program_analysis/SV2AIR3-Waterloo-MATLAB.zip differ diff --git a/skema/data/program_analysis/Simple-SIR.zip b/skema/data/program_analysis/Simple-SIR.zip new file mode 100644 index 00000000000..856019c0e74 Binary files /dev/null and b/skema/data/program_analysis/Simple-SIR.zip differ diff --git a/skema/data/program_analysis/TIE-GCM.zip b/skema/data/program_analysis/TIE-GCM.zip new file mode 100644 index 00000000000..5c176cafe9d Binary files /dev/null and b/skema/data/program_analysis/TIE-GCM.zip differ diff --git a/skema/data/program_analysis/__init__.py b/skema/data/program_analysis/__init__.py new file mode 100644 index 00000000000..3a34362b019 --- /dev/null +++ b/skema/data/program_analysis/__init__.py @@ -0,0 +1,3 @@ +from pathlib import Path + +MODEL_ZIP_ROOT_PATH = Path(__file__).parent \ No newline at end of file diff --git a/skema/data/program_analysis/cism_v3.zip b/skema/data/program_analysis/cism_v3.zip new file mode 100644 index 00000000000..f91b294b18f Binary files /dev/null and b/skema/data/program_analysis/cism_v3.zip differ diff --git a/skema/data/program_analysis/climlab-v2.zip b/skema/data/program_analysis/climlab-v2.zip new file mode 100644 index 00000000000..e17db0abee6 Binary files /dev/null and b/skema/data/program_analysis/climlab-v2.zip differ diff --git a/skema/data/program_analysis/code_sir.zip b/skema/data/program_analysis/code_sir.zip new file mode 100644 index 00000000000..a64fefd55d5 Binary files /dev/null and b/skema/data/program_analysis/code_sir.zip differ diff --git a/skema/data/program_analysis/examples_python.zip b/skema/data/program_analysis/examples_python.zip new file mode 100644 index 00000000000..274a8914039 Binary files /dev/null and b/skema/data/program_analysis/examples_python.zip differ diff --git a/skema/img2mml/README.md b/skema/img2mml/README.md index 432b28e009d..d7da461433c 100644 --- a/skema/img2mml/README.md +++ b/skema/img2mml/README.md @@ -12,24 +12,23 @@ service was developed by Deepsana Shahi, Adarsh Pyarelal and Liang Zhang. The model itself is not checked into the repository, but you can get it from here: -https://artifacts.askem.lum.ai/skema/img2mml/models/cnn_xfmer_arxiv_im2mml_with_fonts_boldface_best.pt +https://huggingface.co/lum-ai/img2mml/blob/main/cnn_xfmer_arxiv_im2mml_with_fonts_boldface_best.pt Place the model file in the `trained_models` directory. -The curl command below should do the trick. +The Python command below should do the trick. ``` -curl -L https://artifacts.askem.lum.ai/skema/img2mml/models/cnn_xfmer_arxiv_im2mml_with_fonts_boldface_best.pt > trained_models/cnn_xfmer_arxiv_im2mml_with_fonts_boldface_best.pt +python ../../scripts/retrieve_model_ci.py ``` -If you have the checkpoint in the `trained_models` directory already and hope to update it, please run the above curl command that will replace the previous one. +If you have the checkpoint in the `trained_models` directory already and hope to update it, please run the above Python command that will replace the previous one. To update the model name or path, please make the following modifications to support updating the img2mml service and the corresponding Docker operations: 1. Modify the ENV variable of `SKEMA_IMG2MML_MODEL_PATH`. 2. Update the path settings in the "retrieve latest model for img2mml component" section of `skema/.github/workflows/tests-and-docs.yml`. -3. Adjust the curl command in the test_equation_reading section of `skema/.drone.yml` to download the checkpoint. -4. Update the download checkpoint path in `skema/img2mml/README.md`. +3. Update the download checkpoint path in `skema/img2mml/README.md`. These changes will ensure that the necessary files and paths are updated correctly. diff --git a/skema/img2mml/api.py b/skema/img2mml/api.py index 5f84c6462f1..10441f54218 100644 --- a/skema/img2mml/api.py +++ b/skema/img2mml/api.py @@ -13,6 +13,7 @@ from PIL import Image from io import BytesIO +from huggingface_hub import hf_hub_download def retrieve_model(model_path=None) -> str: """ @@ -25,7 +26,7 @@ def retrieve_model(model_path=None) -> str: str: Path to the loaded model file. """ cwd = Path(__file__).parents[0] - MODEL_BASE_ADDRESS = "https://artifacts.askem.lum.ai/skema/img2mml/models" + REPO_NAME = "lum-ai/img2mml" MODEL_NAME = "cnn_xfmer_arxiv_im2mml_with_fonts_boldface_best.pt" # If the model path is none or doesn't exist, the default model will be downloaded from server. if model_path is None or not os.path.exists(model_path): @@ -34,10 +35,9 @@ def retrieve_model(model_path=None) -> str: # Check if the model file already exists if not os.path.exists(model_path): # If the file doesn't exist, download it from the specified URL - url = f"{MODEL_BASE_ADDRESS}/{MODEL_NAME}" - print(f"Downloading the model checkpoint from {url}...") - urllib.request.urlretrieve(url, model_path) - + print(f"Downloading the model checkpoint from HuggingFace...") + hf_hub_download(repo_id=REPO_NAME, filename=MODEL_NAME, local_dir=model_path.parent, local_dir_use_symlinks=False) + return str(model_path) diff --git a/skema/program_analysis/model_coverage_report/model_coverage_report.py b/skema/program_analysis/model_coverage_report/model_coverage_report.py index 3320eba3031..ef8c899e04d 100644 --- a/skema/program_analysis/model_coverage_report/model_coverage_report.py +++ b/skema/program_analysis/model_coverage_report/model_coverage_report.py @@ -29,9 +29,11 @@ from skema.rest.utils import fn_preprocessor from skema.rest.workflows import code_snippets_to_pn_amr from skema.utils.fold import del_nulls, dictionary_to_gromet_json +from skema.data.program_analysis import MODEL_ZIP_ROOT_PATH from skema.utils.change_dir_back import change_dir_back from skema.skema_py.server import System + # Constants for file paths THIS_PATH = Path(__file__).parent.resolve() MODEL_YAML_PATH = THIS_PATH / "models.yaml" @@ -149,11 +151,12 @@ def process_single_model(html: HTML_Instance, output_dir: str, model_name: str): """Generate an HTML report for a single model""" html.add_model(model_name) - if model_name in MODEL_YAML: - model_url = MODEL_YAML[model_name]["zip_archive"] - response = requests.get(model_url) - - zip = ZipFile(BytesIO(response.content)) + if not model_name in MODEL_YAML: + return + + model_path = MODEL_ZIP_ROOT_PATH.resolve() / MODEL_YAML[model_name]["zip_archive"] + + zip = ZipFile(BytesIO(model_path.read_bytes())) with TemporaryDirectory() as temp: # We need to write all the files to the temporary directory before processing # This is because some steps may require additional files, such as include directories in Fortran @@ -297,7 +300,8 @@ def process_all_models(html: HTML_Instance, output_dir: str): try: supported, total = process_single_model(html, output_dir, model_name) model_line_coverage[model_name] = (supported, total) - except: + except Exception as e: + print(e) continue return model_line_coverage diff --git a/skema/program_analysis/model_coverage_report/models.yaml b/skema/program_analysis/model_coverage_report/models.yaml index 3d326f2db58..b77505c793d 100644 --- a/skema/program_analysis/model_coverage_report/models.yaml +++ b/skema/program_analysis/model_coverage_report/models.yaml @@ -1,57 +1,56 @@ ---- CHIME-penn-full: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/CHIME-penn-full-model.zip" + zip_archive: "CHIME-penn-full-model.zip" CHIME-SIR: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/CHIME-SIR-model.zip" + zip_archive: "CHIME-SIR-model.zip" CHIME-SVIIvR: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/CHIME-SVIIvR-model.zip" + zip_archive: "CHIME-SVIIvR-model.zip" ABM-COmplexVID-19: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/ABM-COmplexVID-19.zip" + zip_archive: "ABM-COmplexVID-19.zip" ABM-COVID-ABS: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/ABM-COVID-ABS.zip" + zip_archive: "ABM-COVID-ABS.zip" CHIME-penn-full: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/CHIME-penn-full-model.zip" + zip_archive: "CHIME-penn-full-model.zip" MechBayes: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/MechBayes.zip" + zip_archive: "MechBayes.zip" SIDARTHE: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/SIDARTHE.zip" + zip_archive: "SIDARTHE.zip" Simple-SIR: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/Simple-SIR.zip" + zip_archive: "Simple-SIR.zip" Climlab-v1: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/Climlab.zip" + zip_archive: "Climlab.zip" Climlab-v2: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/climlab-v2.zip" + zip_archive: "climlab-v2.zip" Examples-Python: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/examples_python.zip" + zip_archive: "examples_python.zip" Generated-Halfar: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/Generated-Halfar.zip" + zip_archive: "Generated-Halfar.zip" SV2AIR3-Waterloo-MATLAB: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/SV2AIR3-Waterloo-MATLAB.zip" + zip_archive: "SV2AIR3-Waterloo-MATLAB.zip" Bucky: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/Bucky.zip" + zip_archive: "Bucky.zip" ABM-REINA: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/ABM-REINA.zip" + zip_archive: "ABM-REINA.zip" Cornell-COVID19-sim-Frazier: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/Cornell-COVID19-sim-Frazier.zip" + zip_archive: "Cornell-COVID19-sim-Frazier.zip" ABM-Covasim: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/ABM-Covasim.zip" + zip_archive: "ABM-Covasim.zip" TIE-GCM: - zip_archive: "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/TIE-GCM.zip" + zip_archive: "TIE-GCM.zip" diff --git a/skema/program_analysis/tests/test_code2fn.py b/skema/program_analysis/tests/test_code2fn.py index 3b17ec03965..800eee8f900 100644 --- a/skema/program_analysis/tests/test_code2fn.py +++ b/skema/program_analysis/tests/test_code2fn.py @@ -7,8 +7,9 @@ from skema.program_analysis.multi_file_ingester import process_file_system from skema.gromet.fn import GrometFNModuleCollection +from skema.data.program_analysis import MODEL_ZIP_ROOT_PATH -BUCKY_ZIP_URL = "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/Bucky.zip" +BUCKY_ZIP_URL = MODEL_ZIP_ROOT_PATH.resolve() / "Bucky.zip" def test_code2fn(): """This is simply a smokescreen test to see if the PA pipeline runs to @@ -19,8 +20,7 @@ def test_code2fn(): flag or postprocessing function to be able to compare two GroMEts modulo their random components.""" - response = requests.get(BUCKY_ZIP_URL) - zip = zipfile.ZipFile(io.BytesIO(response.content)) + zip = zipfile.ZipFile(io.BytesIO(BUCKY_ZIP_URL.read_bytes())) with TemporaryDirectory() as temp: system_filepaths_path = Path(temp) / "system_filepaths.txt" diff --git a/skema/rest/tests/test_model_to_amr.py b/skema/rest/tests/test_model_to_amr.py index 5d168c37a57..6d3a533ad26 100644 --- a/skema/rest/tests/test_model_to_amr.py +++ b/skema/rest/tests/test_model_to_amr.py @@ -1,3 +1,6 @@ +import json +import httpx +import pytest import asyncio import requests from pathlib import Path @@ -16,17 +19,12 @@ from skema.rest.llm_proxy import Dynamics from skema.rest.proxies import SKEMA_RS_ADDESS from skema.skema_py import server as code2fn -import json -import httpx -import pytest +from skema.data.program_analysis import MODEL_ZIP_ROOT_PATH -CHIME_SIR_URL = ( - "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/CHIME-SIR-model.zip" -) -SIDARTHE_URL = ( - "https://artifacts.askem.lum.ai/askem/data/models/zip-archives/SIDARTHE.zip" -) +CHIME_SIR_PATH = MODEL_ZIP_ROOT_PATH.resolve() / "CHIME-SIR-model.zip" +SIDARTHE_PATH = MODEL_ZIP_ROOT_PATH.resolve() / "SIDARTHE.zip" + @pytest.mark.asyncio async def test_any_amr_chime_sir(): @@ -34,8 +32,8 @@ async def test_any_amr_chime_sir(): Unit test for checking that Chime-SIR model produces any AMR. This test zip contains 4 versions of CHIME SIR. This will test if just the core dynamics works, the whole script, and also rewritten scripts work. """ - response = requests.get(CHIME_SIR_URL) - zip_bytes = BytesIO(response.content) + + zip_bytes = BytesIO(CHIME_SIR_PATH.read_bytes()) # NOTE: For CI we are unable to use the LLM assisted functions due to API keys # So, we will instead mock the output for those functions instead @@ -136,8 +134,7 @@ async def test_any_amr_sidarthe(): Unit test for checking that Chime-SIR model produces any AMR. This test zip contains 4 versions of CHIME SIR. This will test if just the core dynamics works, the whole script, and also rewritten scripts work. """ - response = requests.get(SIDARTHE_URL) - zip_bytes = BytesIO(response.content) + zip_bytes = BytesIO(SIDARTHE_PATH.read_bytes()) # NOTE: For CI we are unable to use the LLM assisted functions due to API keys # So, we will instead mock the output for those functions instead