Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support for jura & kibo #31

Open
wants to merge 14 commits into
base: main
Choose a base branch
from
Open

Support for jura & kibo #31

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
4f249f9
add jura backups
weaverba137 Jul 24, 2024
32142a0
other backup configuration
weaverba137 Jul 24, 2024
71f92a5
fix typo in regex
weaverba137 Jul 29, 2024
47db372
don't exclude jura
weaverba137 Aug 5, 2024
ad179c7
add config for edr and dr1
weaverba137 Aug 5, 2024
2e1c39b
update vac configuration
weaverba137 Aug 6, 2024
19f216b
further jura tweaks
weaverba137 Aug 12, 2024
7cd3796
clean up other mock configuration
weaverba137 Sep 5, 2024
c2e5771
tweak readme configuration
weaverba137 Sep 18, 2024
e23230e
fix missing README
weaverba137 Sep 25, 2024
a178089
update job directory
weaverba137 Oct 8, 2024
9b4a2b8
also add kibo
weaverba137 Nov 4, 2024
6b1cb88
GFA.KPNO backup
weaverba137 Nov 4, 2024
4f5a806
daily/tiles/archive backups
weaverba137 Nov 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion bin/backupStatus.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ sections=$(grep -E '^ "[^"]+":\{' ${DESIBACKUP}/etc/desi.json | \
sed -r 's/^ "([^"]+)":\{/\1/' | \
grep -v config)
comments=$(cat <<COMMENTS
external:This directory provides links to non-DESI data sets. The actual data are stored elsewhere.
gsharing:Share data via Globus. The actual data are stored elsewhere.
software:Most DESI software is stored elsewhere, and the ultimate backups are the various git and svn repositories.
target:The most important targeting data is backed up with the <code>public/ets/</code> data.
Expand All @@ -128,7 +129,8 @@ COMMENTS
for d in ${sections}; do
c=$(grep "${d}:" <<<"${comments}" | cut -d: -f2)
j=0
if [[ "${d}" == "gsharing" || \
if [[ "${d}" == "external" || \
"${d}" == "gsharing" || \
"${d}" == "software" || \
"${d}" == "users" || \
"${d}" == "vac" || \
Expand Down
4 changes: 2 additions & 2 deletions bin/submit_backup_status.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ function usage() {
echo "Submit jobs to analyze backup status."
echo ""
echo " -h = Print help message and exit."
echo " -j DIR = Use DIR to stage jobs for submission (default '${HOME}/jobs')."
echo " -j DIR = Use DIR to stage jobs for submission (default '${DESI_ROOT}/users/${USER}/jobs')."
echo " -s RELEASE = Use DESI software RELEASE (default 'main')."
echo " -t = Test mode; do not actually submit jobs. Implies -v."
echo " -v = Verbose mode; print extra information."
Expand All @@ -16,7 +16,7 @@ function usage() {
testMode=/usr/bin/false
verbMode=/usr/bin/false
software=main
job_dir=${HOME}/jobs
job_dir=${DESI_ROOT}/users/${USER}/jobs
while getopts hj:s:tv argname; do
case ${argname} in
h) usage; exit 0 ;;
Expand Down
78 changes: 58 additions & 20 deletions etc/desi.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,12 +52,15 @@
"AbacusLensing":{},
"AbacusPotential":{},
"CoLoRe_lensing":{},
"DESI2_High_Density":{},
"ELG_HOD":{},
"EZmock_lightcones":{},
"FirstGenMocks":{},
"KP45":{},
"NeutrinoSims":{},
"NFW_Abacus":{},
"SecondGenMocks":{},
"ThirdGenMocks":{},
"TNG_Rockstar":{},
"Uchuu":{},
"UNIT-BAO-RSD-challenge":{},
Expand Down Expand Up @@ -190,6 +193,7 @@
"engineering":{
"__exclude__":[],
"2021_summer_illumination_checks":{},
"2023_summer_restart":{},
"donut":{},
"focalplane":{},
"fvc":{},
Expand All @@ -202,14 +206,18 @@
"svn_export_focalplane_12302018":{},
"umdata":{}
},
"external":{
"__exclude__":["README.txt"]
},
"gsharing":{
"__exclude__":[]
"__exclude__":["README"]
},
"metadata":{
"__exclude__":["README.html","scan_README.txt"],
"__top__":{
"([0-9][0-9][0-9][0-9])-([0-9][0-9])-[0-9][0-9]\\.(cfs|tlprojecta|tlproject2)\\.(desi|desicollab)\\.txt$":"EXCLUDE",
"([0-9][0-9][0-9][0-9])-([0-9][0-9])-[0-9][0-9]\\.(cfs|tlprojecta|tlproject2)\\.(desi|desicollab)\\.txt\\.gz$":"\\1_\\2_files.tar"
"([0-9][0-9][0-9][0-9])-([0-9][0-9])-[0-9][0-9]\\.(cfs|tlprojecta|tlproject2)\\.(desi|desicollab)\\.txt\\.gz$":"\\1_\\2_files.tar",
"[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]\\.json$":"EXCLUDE"
},
"backups":{
"backups/.*$":"EXCLUDE"
Expand All @@ -223,6 +231,7 @@
},
"mocks":{
"__exclude__":["lya_forest/README.html",
"lya_forest/london/README",
"lya_forest/london/v6.0/README",
"lya_forest/london/v6.0/lyasim.log",
"lya_forest/saclay/README",
Expand All @@ -238,7 +247,8 @@
},
"GLAM":{
"GLAM/ABACUS/.*$":"EXCLUDE",
"GLAM/GLAM_(BDM|LRG)_([^/]+)/.*$":"GLAM/GLAM_\\1_\\2.tar",
"GLAM/GLAM_BDM_MNRAS/.*$":"EXCLUDE",
"GLAM/GLAM_LRG_([^/]+)/.*$":"GLAM/GLAM_LRG_\\1.tar",
"GLAM/JFE_files/.*$":"EXCLUDE"
},
"Gadget":{
Expand Down Expand Up @@ -286,8 +296,11 @@
"lightcone_galform/.*$":"lightcone_galform.tar"
},
"lya_forest":{
"lya_forest/\\.ipynb_checkpoints/.*$":"EXCLUDE",
"lya_forest/develop/.*$":"EXCLUDE",
"lya_forest/dr11/.*$":"EXCLUDE",
"lya_forest/london/README$":"EXCLUDE",
"lya_forest/london/(colore_boxes|lyacolore_skewers|qq_desi_y1|qq_eboss)/.*$":"EXCLUDE",
"lya_forest/london/(v[23]\\.[0-9])/[^/]+$":"lya_forest/london/\\1/lya_forest_london_\\1_files.tar",
"lya_forest/london/(v[23]\\.[0-9])/([0-9]+)/.*$":"lya_forest/london/\\1/lya_forest_london_\\1_\\2.tar",
"lya_forest/london/(v[23]\\.[0-9])/quick-([0-9.]+)/.*$":"lya_forest/london/\\1/lya_forest_london_\\1_quick-\\2.tar",
Expand All @@ -301,11 +314,14 @@
"lya_forest/london/(v[45]\\.[0-9]\\.[0-9])/(logs|tests)/.*$":"lya_forest/london/\\1/lya_forest_london_\\1_\\2.tar",
"lya_forest/london/(v[6789]\\.[0-9])/README":"lya_forest/london/\\1/README",
"lya_forest/london/(v[6789]\\.[0-9])/.ipynb_checkpoints/.*$":"EXCLUDE",
"lya_forest/london/(v[6789]\\.[0-9])/colore_raw/.*$":"EXCLUDE",
"lya_forest/london/(v[6789]\\.[0-9])/(v[6789]\\.[0-9]\\.[0-9]+)/.ipynb_checkpoints/.*$":"EXCLUDE",
"lya_forest/london/(v[6789]\\.[0-9])/(v[6789]\\.[0-9]\\.[0-9]+)/[^/]+$":"lya_forest/london/\\1/\\2/lya_forest_london_\\1_\\2_files.tar",
"lya_forest/london/(v[6789]\\.[0-9])/(v[6789]\\.[0-9]\\.[0-9]+)/([0-9]+)/.*$":"lya_forest/london/\\1/\\2/lya_forest_london_\\1_\\2_\\3.tar",
"lya_forest/london/(v[6789]\\.[0-9])/(v[6789]\\.[0-9]\\.[0-9]+)/eboss-([0-9.]+)/.*$":"lya_forest/london/\\1/\\2/lya_forest_london_\\1_\\2_eboss-\\3.tar",
"lya_forest/london/(v[6789]\\.[0-9])/(v[6789]\\.[0-9]\\.[0-9]+)/logs/.*$":"lya_forest/london/\\1/\\2/lya_forest_london_\\1_\\2_logs.tar",
"lya_forest/picca/.*$":"EXCLUDE",
"lya_forest/public/.*$":"EXCLUDE",
"lya_forest/saclay/.*$":"EXCLUDE",
"lya_forest/uncorr/.*$":"EXCLUDE"
},
Expand Down Expand Up @@ -342,17 +358,29 @@
"doc/.*$":"EXCLUDE"
},
"dr1":{
"dr1/[^/]+$":"dr1_files.tar",
"dr1/spectro/data/([0-9]+)/.*$":"dr1/spectro/data/desi_spectro_data_\\1.tar",
"dr1/spectro/desi_spectro_calib/([v0-9.]+)/.*$":"dr1/spectro/desi_spectro_calib/dr1_spectro_desi_spectro_calib_\\1.tar",
"dr1/spectro/redux/(guadalupe|iron)/[^/]+$":"dr1/spectro/redux/\\1/redux_\\1_files.tar",
"dr1/spectro/redux/(guadalupe|iron)/(calibnight|dashboard|exposure_tables|processing_tables|run|zcatalog)/.*$":"dr1/spectro/redux/\\1/redux_\\1_\\2.tar",
"dr1/spectro/redux/(guadalupe|iron)/(calibnight|dashboard|exposure_tables|processing_tables|run)/.*$":"dr1/spectro/redux/\\1/redux_\\1_\\2.tar",
"dr1/spectro/redux/(guadalupe|iron)/(zcatalog)/(v[0-9])/.*$":"dr1/spectro/redux/\\1/redux_\\1_\\2_\\3.1.tar",
"dr1/spectro/redux/(guadalupe|iron)/(exposures|preproc)/([0-9]+)/.*$":"dr1/spectro/redux/\\1/\\2/redux_\\1_\\2_\\3.tar",
"dr1/spectro/redux/(guadalupe|iron)/healpix/[^/]+$":"dr1/spectro/redux/\\1/healpix/redux_\\1_healpix_files.tar",
"dr1/spectro/redux/(guadalupe|iron)/healpix/(cmx|main|special|sv1|sv2|sv3)/(backup|bright|dark|other)/([0-9]+)/.*$":"dr1/spectro/redux/\\1/healpix/\\2/\\3/redux_\\1_healpix_\\2_\\3_\\4.tar",
"dr1/spectro/redux/(guadalupe|iron)/tiles/(1x_depth|4x_depth|lowspeed|cumulative|perexp|pernight)/([0-9]+)/.*$":"dr1/spectro/redux/\\1/tiles/\\2/redux_\\1_tiles_\\2_\\3.tar",
"dr1/spectro/templates/basis_templates/([v0-9.]+)/.*$":"dr1/spectro/templates/dr1_spectro_templates_basis_templates_\\1.tar",
"dr1/survey/(GFA|ops)/.*$":"dr1/survey/dr1_survey_\\1.tar",
"dr1/target/fiberassign/tiles/tags/([0-9.]+)/[^/]+$":"dr1/fiberassign/tiles/tags/\\1/dr1_fiberassign_tiles_tags_\\1_files.tar",
"dr1/target/fiberassign/tiles/tags/([0-9.]+)/([0-9]{3})/.*$":"dr1/fiberassign/tiles/tags/\\1/dr1_fiberassign_tiles_tags_\\1_\\2.tar",
"dr1/vac/dr1/(fastspecfit|lsdr9-photometry)/(guadalupe|iron)/(v[0-9.]+)/.*$":"dr1/vac/dr1/\\1/\\2/dr1_vac_dr1_\\1_\\2_\\3.tar"
},
"dr1/vac/dr1/(fastspecfit|lsdr9-photometry)/(guadalupe)/(v[0-9.]+)/.*$":"dr1/vac/dr1/\\1/\\2/dr1_vac_dr1_\\1_\\2_\\3.tar",
"dr1/vac/dr1/(fastspecfit)/(iron)/(v[0-9.]+)/README\\.md$":"dr1/vac/dr1/\\1/\\2/\\3/README.md",
"dr1/vac/dr1/(fastspecfit)/(iron)/(v[0-9.]+)/catalogs/fastspec-iron\\.fits":"dr1/vac/dr1/\\1/\\2/\\3/catalogs/fastspec-iron.fits",
"dr1/vac/dr1/(fastspecfit)/(iron)/(v[0-9.]+)/catalogs/(?!fastspec-iron\\.fits)([^/]+)$":"dr1/vac/dr1/\\1/\\2/\\3/dr1_vac_dr1_\\1_\\2_\\3_catalogs.tar",
"dr1/vac/dr1/(fastspecfit)/(iron)/(v[0-9.]+)/logs/.*$":"dr1/vac/dr1/\\1/\\2/\\3/dr1_vac_dr1_\\1_\\2_\\3_logs.tar",
"dr1/vac/dr1/(fastspecfit)/(iron)/(v[0-9.]+)/healpix/(cmx|main|special|sv1|sv2|sv3)/.*$":"dr1/vac/dr1/\\1/\\2/\\3/healpix/dr1_vac_dr1_\\1_\\2_\\3_healpix_\\4.tar",
"dr1/vac/dr1/(lsdr9-photometry)/(iron)/(v[0-9.]+)/.*$":"dr1/vac/dr1/\\1/\\2/dr1_vac_dr1_\\1_\\2_\\3.tar"
},
"dr2":{},
"edr":{
"edr/[^/]+$":"edr_files.tar",
"edr/spectro/data/([0-9]+)/.*$":"edr/spectro/data/desi_spectro_data_\\1.tar",
Expand All @@ -369,8 +397,8 @@
"edr/target/fiberassign/tiles/tags/([0-9.]+)/[^/]+$":"edr/fiberassign/tiles/tags/\\1/edr_fiberassign_tiles_tags_\\1_files.tar",
"edr/target/fiberassign/tiles/tags/([0-9.]+)/([0-9]{3})/.*$":"edr/fiberassign/tiles/tags/\\1/edr_fiberassign_tiles_tags_\\1_\\2.tar",
"edr/target/(masks|skybricks|skyhealpixs)/([0-9dev.]+)/.*$":"edr/target/\\1/edr_target_\\1_\\2.tar",
"edr/vac/edr/(bal|lsdr9-photometry|lya|mws|zcat)/(fuji)/(v[0-9.]+)/.*$":"edr/vac/edr/edr_vac_edr_\\1_\\2_\\3.tar",
"edr/vac/edr/(lss|mgii-absorber|provabgs|vi)/(v[0-9.]+)/.*$":"edr/vac/edr/edr_vac_edr_\\1_\\2.tar"
"edr/vac/edr/(bal|cigale|dla|fastspecfit|lsdr9-photometry|lya|mws|zcat)/(fuji)/(v[0-9.]+)/.*$":"edr/vac/edr/edr_vac_edr_\\1_\\2_\\3.tar",
"edr/vac/edr/(epoviz|lss|mgii-absorber|provabgs|qso|stellar-mass-emline|uchuu|vi)/(v[0-9.]+)/.*$":"edr/vac/edr/edr_vac_edr_\\1_\\2.tar"
},
"epo":{
},
Expand All @@ -389,7 +417,7 @@
}
},
"science":{
"__exclude__":["README"],
"__exclude__":[],
"Y1KP":{},
"bgs":{},
"c3":{},
Expand Down Expand Up @@ -443,14 +471,19 @@
"quicklook":{
},
"redux":{
"redux/(?!andes|blanc|cascades|denali|everest|minisv2|month4|oak1)([^/]+)/.*$":"EXCLUDE",
"redux/(andes|blanc|cascades|denali|everest|month4)/[^/]+$":"redux/\\1/redux_\\1_files.tar",
"redux/(andes|blanc|cascades|denali|everest)/(calibnight|dashboard|exposure_tables|processing_tables|run|zcatalog)/.*$":"redux/\\1/redux_\\1_\\2.tar",
"redux/(?!andes|blanc|cascades|daily|denali|everest|jura|kibo|minisv2|month4|oak1)([^/]+)/.*$":"EXCLUDE",
"redux/daily/[^/]+$":"EXCLUDE",
"redux/daily/(?!tiles)([^/]+)/.*$":"EXCLUDE",
"redux/daily/tiles/archive/([0-9]+)/([0-9]+)/.*$":"redux/daily/tiles/archive/redux_daily_tiles_archive_\\1_\\2.tar",
"redux/(andes|blanc|cascades|denali|everest|jura|kibo|month4)/[^/]+$":"redux/\\1/redux_\\1_files.tar",
"redux/(andes|blanc|cascades|denali|everest|jura|kibo)/(calibnight|dashboard|exposure_tables|nightqa|processing_tables|run)/.*$":"redux/\\1/redux_\\1_\\2.tar",
"redux/(andes|blanc|cascades|denali|everest)/(zcatalog)/.*$":"redux/\\1/redux_\\1_\\2.tar",
"redux/(jura|kibo)/zcatalog/(v[0-9])/.*$":"redux/\\1/redux_\\1_zcatalog_\\2.tar",
"redux/(andes|blanc|cascades)/(exposures|preproc|tiles)/([0-9]+)/.*$":"redux/\\1/\\2/redux_\\1_\\2_\\3.tar",
"redux/(denali|everest)/(exposures|preproc)/([0-9]+)/.*$":"redux/\\1/\\2/redux_\\1_\\2_\\3.tar",
"redux/(everest)/healpix/[^/]+$":"redux/\\1/healpix/redux_\\1_healpix_files.tar",
"redux/(everest)/healpix/(cmx|main|special|sv1|sv2|sv3)/(backup|bright|dark|other)/([0-9]+)/.*$":"redux/\\1/healpix/\\2/\\3/redux_\\1_healpix_\\2_\\3_\\4.tar",
"redux/(denali|everest)/tiles/(1x_depth|4x_depth|lowspeed|cumulative|perexp|pernight)/([0-9]+)/.*$":"redux/\\1/tiles/\\2/redux_\\1_tiles_\\2_\\3.tar",
"redux/(denali|everest|jura|kibo)/(exposures|preproc)/([0-9]+)/.*$":"redux/\\1/\\2/redux_\\1_\\2_\\3.tar",
"redux/(everest|jura|kibo)/healpix/[^/]+$":"redux/\\1/healpix/redux_\\1_healpix_files.tar",
"redux/(everest|jura|kibo)/healpix/(cmx|main|special|sv1|sv2|sv3)/(backup|bright|dark|other)/([0-9]+)/.*$":"redux/\\1/healpix/\\2/\\3/redux_\\1_healpix_\\2_\\3_\\4.tar",
"redux/(denali|everest|jura|kibo)/tiles/(1x_depth|4x_depth|lowspeed|cumulative|perexp|pernight)/([0-9]+)/.*$":"redux/\\1/tiles/\\2/redux_\\1_tiles_\\2_\\3.tar",
"redux/(minisv2|month4)/(alt|calib2d|calibnight|plan|run|tiles)/.*$":"redux/\\1/redux_\\1_\\2.tar",
"redux/(minisv2|month4)/(exposures|preproc|spectra-64)/([0-9]+)/.*$":"redux/\\1/\\2/redux_\\1_\\2_\\3.tar",
"redux/oak1/.*$":"redux/oak1.tar"
Expand All @@ -469,16 +502,20 @@
}
},
"survey":{
"__exclude__":[],
"GFA":{},
"__exclude__":["GFA.KPNO.log"],
"GFA.KPNO":{
"GFA\\.KPNO/.*$": "GFA.KPNO.tar"
},
"GFA.NERSC":{},
"analysis":{},
"catalogs":{},
"fiberassign":{},
"fluxcalibration":{},
"observations":{},
"ops":{},
"planning":{},
"sims":{}
"sims":{},
"sv":{}
},
"sv":{
"__exclude__":[],
Expand Down Expand Up @@ -512,7 +549,8 @@
"catalogs/dr5/(pixweight|randoms)-dr5\\.fits$": "catalogs/dr5/\\1-dr5.fits",
"catalogs/dr7\\.1/0\\.(2[27])\\.0/[^/]+$": "catalogs/dr7.1/0.\\1.0/desi_target_catalogs_dr7.1_0.\\1.0_files.tar",
"catalogs/dr7\\.1/0\\.(2[27])\\.0/randoms-dr7\\.1-0\\.(2[27])\\.0\\.fits$": "catalogs/dr7.1/\\1/randoms-dr7.1-0.\\1.0.fits",
"catalogs/dr7\\.1/([0-9.]+|PR[0-9]+)/.*$": "catalogs/dr7.1/desi_target_catalogs_dr7.1_\\1.tar"
"catalogs/dr7\\.1/([0-9.]+|PR[0-9]+)/.*$": "catalogs/dr7.1/desi_target_catalogs_dr7.1_\\1.tar",
"catalogs/streamcache/.*$":"EXCLUDE"
},
"cmx_files":{
"cmx_files/.*$":"cmx_files.tar"
Expand Down
Loading