Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CI: Trigger the cache_data workflow in PRs if cache files are added/deleted/updated #2939

Merged
merged 3 commits into from
Jan 3, 2024
Merged

CI: Trigger the cache_data workflow in PRs if cache files are added/deleted/updated #2939

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
341e176
CI: Trigger the cache_data workflow in PRs if cache files are added/d…
seisman Jan 2, 2024
fdb76a9
Merge branch 'main' into cache-data
seisman Jan 2, 2024
24bb4fe
Apply suggestions from code review
seisman Jan 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 9 additions & 6 deletions .github/workflows/cache_data.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,21 @@
# Cache GMT remote data files and uploads as artifacts
# Cache GMT remote data files and upload as artifacts
#
# This workflow downloads data files needed by PyGMT tests/documentation from
# the GMT data server and uploads as workflow artifacts which can be accessed
# by other GitHub Actions workflows.
#
# It is scheduled to run every Sunday at 12:00 (UTC). If new remote files are
# needed urgently, maintainers can manually uncomment the 'pull_request:' line
# below to refresh the cache.
# needed urgently, maintainers can update the workflow file or
# 'pygmt/helpers/caching.py' file to refresh the cache.
#
name: Cache data

on:
# Uncomment the 'pull_request' line below to manually re-cache data artifacts
# pull_request:
pull_request:
# Make any changes to the following files to refresh the cache
paths:
- 'pygmt/helpers/caching.py'
- '.github/workflows/cache_data.yaml'
# Schedule runs on 12 noon every Sunday
schedule:
- cron: '0 12 * * 0'
Expand Down Expand Up @@ -61,7 +64,7 @@ jobs:
# Download remote files
- name: Download remote data
run: |
python -c "from pygmt.helpers.testing import download_test_data; download_test_data()"
python -c "from pygmt.helpers.caching import cache_data; cache_data()"

# Upload the downloaded files as artifacts to GitHub
- name: Upload artifacts to GitHub
Expand Down
87 changes: 87 additions & 0 deletions pygmt/helpers/caching.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
"""
Functions for download remote data files as cache.
"""
from pygmt.src import which


def cache_data():
"""
Download GMT remote data files used in PyGMT tests and docs as cache files.
"""
# List of datasets to download
datasets = [
# Earth relief grids
"@earth_gebco_01d_g",
"@earth_gebcosi_01d_g",
"@earth_gebcosi_15m_p",
"@earth_relief_01d_p",
"@earth_relief_01d_g",
"@earth_relief_30m_p",
"@earth_relief_30m_g",
"@earth_relief_10m_p",
"@earth_relief_10m_g",
"@earth_relief_05m_p",
"@earth_relief_05m_g",
"@earth_synbath_01d_g",
# List of tiles of 03s srtm data.
# Names like @N35E135.earth_relief_03s_g.nc are for internal use only.
# The naming scheme may change. DO NOT USE IT IN YOUR SCRIPTS.
"@N30W120.earth_relief_15s_p.nc",
"@N35E135.earth_relief_03s_g.nc",
"@N37W120.earth_relief_03s_g.nc",
"@N00W090.earth_relief_03m_p.nc",
"@N00E135.earth_relief_30s_g.nc",
"@N00W010.earth_relief_15s_p.nc", # Specific grid for 15s test
"@N04W010.earth_relief_03s_g.nc", # Specific grid for 03s test
# Earth synbath relief grid
"@S15W105.earth_synbath_30s_p.nc",
# Earth seafloor age grids
"@earth_age_01d_g",
"@N00W030.earth_age_01m_g.nc", # Specific grid for 01m test
# Earth geoid grids
"@earth_geoid_01d_g",
"@N00W030.earth_geoid_01m_g.nc", # Specific grid for 01m test
# Earth magnetic anomaly grids
"@earth_mag_01d_g",
"@S30W060.earth_mag_02m_p.nc", # Specific grid for 02m test
"@earth_mag4km_01d_g",
"@S30W120.earth_mag4km_02m_p.nc", # Specific grid for 02m test
# Earth mask grid
"@earth_mask_01d_g",
# Earth free-air anomaly grids
"@earth_faa_01d_g",
"@N00W030.earth_faa_01m_p.nc", # Specific grid for 01m test
# Earth vertical gravity gradient grids
"@earth_vgg_01d_g",
"@N00W030.earth_vgg_01m_p.nc", # Specific grid for 01m test
# Earth WDMAM grids
"@earth_wdmam_01d_g",
"@S90E000.earth_wdmam_03m_g.nc", # Specific grid for 03m test
# Earth day/night grids
"@earth_day_01d_p",
# Other cache files
"@capitals.gmt",
"@circuit.png",
"@earth_relief_20m_holes.grd",
"@EGM96_to_36.txt",
"@MaunaLoa_CO2.txt",
"@RidgeTest.shp",
"@RidgeTest.shx",
"@RidgeTest.dbf",
"@RidgeTest.prj",
"@Table_5_11.txt",
"@Table_5_11_mean.xyz",
"@fractures_06.txt",
"@hotspots.txt",
"@ridge.txt",
"@mars370d.txt",
"@srtm_tiles.nc", # needed for 03s and 01s relief data
"@static_earth_relief.nc",
"@ternary.txt",
"@test.dat.nc",
"@tut_bathy.nc",
"@tut_quakes.ngdc",
"@tut_ship.xyz",
"@usgs_quakes_22.txt",
]
which(fname=datasets, download="a")
83 changes: 0 additions & 83 deletions pygmt/helpers/testing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,89 +142,6 @@ def wrapper(*args, ext="png", request=None, **kwargs):
return decorator


def download_test_data():
"""
Convenience function to download remote data files used in PyGMT tests and docs.
"""
# List of datasets to download
datasets = [
# Earth relief grids
"@earth_gebco_01d_g",
"@earth_gebcosi_01d_g",
"@earth_gebcosi_15m_p",
"@earth_relief_01d_p",
"@earth_relief_01d_g",
"@earth_relief_30m_p",
"@earth_relief_30m_g",
"@earth_relief_10m_p",
"@earth_relief_10m_g",
"@earth_relief_05m_p",
"@earth_relief_05m_g",
"@earth_synbath_01d_g",
# List of tiles of 03s srtm data.
# Names like @N35E135.earth_relief_03s_g.nc is for internal use only.
# The naming scheme may change. DO NOT USE IT IN YOUR SCRIPTS.
"@N30W120.earth_relief_15s_p.nc",
"@N35E135.earth_relief_03s_g.nc",
"@N37W120.earth_relief_03s_g.nc",
"@N00W090.earth_relief_03m_p.nc",
"@N00E135.earth_relief_30s_g.nc",
"@N00W010.earth_relief_15s_p.nc", # Specific grid for 15s test
"@N04W010.earth_relief_03s_g.nc", # Specific grid for 03s test
# Earth synbath relief grids
"@S15W105.earth_synbath_30s_p.nc",
# Earth seafloor age grids
"@earth_age_01d_g",
"@N00W030.earth_age_01m_g.nc", # Specific grid for 01m test
# Earth geoid grids
"@earth_geoid_01d_g",
"@N00W030.earth_geoid_01m_g.nc", # Specific grid for 01m test
# Earth magnetic anomaly grids
"@earth_mag_01d_g",
"@S30W060.earth_mag_02m_p.nc", # Specific grid for 02m test
"@earth_mag4km_01d_g",
"@S30W120.earth_mag4km_02m_p.nc", # Specific grid for 02m test
# Earth mask grid
"@earth_mask_01d_g",
# Earth free-air anomaly grids
"@earth_faa_01d_g",
"@N00W030.earth_faa_01m_p.nc", # Specific grid for 01m test
# Earth vertical gravity gradient grids
"@earth_vgg_01d_g",
"@N00W030.earth_vgg_01m_p.nc", # Specific grid for 01m test
# Earth WDMAM grids
"@earth_wdmam_01d_g",
"@S90E000.earth_wdmam_03m_g.nc", # Specific grid for 03m test
# Earth day/night grids
"@earth_day_01d_p",
# Other cache files
"@capitals.gmt",
"@circuit.png",
"@earth_relief_20m_holes.grd",
"@EGM96_to_36.txt",
"@MaunaLoa_CO2.txt",
"@RidgeTest.shp",
"@RidgeTest.shx",
"@RidgeTest.dbf",
"@RidgeTest.prj",
"@Table_5_11.txt",
"@Table_5_11_mean.xyz",
"@fractures_06.txt",
"@hotspots.txt",
"@ridge.txt",
"@mars370d.txt",
"@srtm_tiles.nc", # needed for 03s and 01s relief data
"@static_earth_relief.nc",
"@ternary.txt",
"@test.dat.nc",
"@tut_bathy.nc",
"@tut_quakes.ngdc",
"@tut_ship.xyz",
"@usgs_quakes_22.txt",
]
which(fname=datasets, download="a")


def load_static_earth_relief():
"""
Load the static_earth_relief file for internal testing.
Expand Down
Loading