diff --git a/bin/backupStatus.sh b/bin/backupStatus.sh index f1885c1..bbcddeb 100755 --- a/bin/backupStatus.sh +++ b/bin/backupStatus.sh @@ -117,6 +117,7 @@ sections=$(grep -E '^ "[^"]+":\{' ${DESIBACKUP}/etc/desi.json | \ sed -r 's/^ "([^"]+)":\{/\1/' | \ grep -v config) comments=$(cat <public/ets/ data. @@ -128,7 +129,8 @@ COMMENTS for d in ${sections}; do c=$(grep "${d}:" <<<"${comments}" | cut -d: -f2) j=0 - if [[ "${d}" == "gsharing" || \ + if [[ "${d}" == "external" || \ + "${d}" == "gsharing" || \ "${d}" == "software" || \ "${d}" == "users" || \ "${d}" == "vac" || \ diff --git a/bin/submit_backup_status.sh b/bin/submit_backup_status.sh index daec87d..53fea2e 100755 --- a/bin/submit_backup_status.sh +++ b/bin/submit_backup_status.sh @@ -7,7 +7,7 @@ function usage() { echo "Submit jobs to analyze backup status." echo "" echo " -h = Print help message and exit." - echo " -j DIR = Use DIR to stage jobs for submission (default '${HOME}/jobs')." + echo " -j DIR = Use DIR to stage jobs for submission (default '${DESI_ROOT}/users/${USER}/jobs')." echo " -s RELEASE = Use DESI software RELEASE (default 'main')." echo " -t = Test mode; do not actually submit jobs. Implies -v." echo " -v = Verbose mode; print extra information." @@ -16,7 +16,7 @@ function usage() { testMode=/usr/bin/false verbMode=/usr/bin/false software=main -job_dir=${HOME}/jobs +job_dir=${DESI_ROOT}/users/${USER}/jobs while getopts hj:s:tv argname; do case ${argname} in h) usage; exit 0 ;; diff --git a/etc/desi.json b/etc/desi.json index 521fac5..e66ce4c 100644 --- a/etc/desi.json +++ b/etc/desi.json @@ -52,12 +52,15 @@ "AbacusLensing":{}, "AbacusPotential":{}, "CoLoRe_lensing":{}, + "DESI2_High_Density":{}, "ELG_HOD":{}, "EZmock_lightcones":{}, "FirstGenMocks":{}, "KP45":{}, + "NeutrinoSims":{}, "NFW_Abacus":{}, "SecondGenMocks":{}, + "ThirdGenMocks":{}, "TNG_Rockstar":{}, "Uchuu":{}, "UNIT-BAO-RSD-challenge":{}, @@ -190,6 +193,7 @@ "engineering":{ "__exclude__":[], "2021_summer_illumination_checks":{}, + "2023_summer_restart":{}, "donut":{}, "focalplane":{}, "fvc":{}, @@ -202,14 +206,18 @@ "svn_export_focalplane_12302018":{}, "umdata":{} }, + "external":{ + "__exclude__":["README.txt"] + }, "gsharing":{ - "__exclude__":[] + "__exclude__":["README"] }, "metadata":{ "__exclude__":["README.html","scan_README.txt"], "__top__":{ "([0-9][0-9][0-9][0-9])-([0-9][0-9])-[0-9][0-9]\\.(cfs|tlprojecta|tlproject2)\\.(desi|desicollab)\\.txt$":"EXCLUDE", - "([0-9][0-9][0-9][0-9])-([0-9][0-9])-[0-9][0-9]\\.(cfs|tlprojecta|tlproject2)\\.(desi|desicollab)\\.txt\\.gz$":"\\1_\\2_files.tar" + "([0-9][0-9][0-9][0-9])-([0-9][0-9])-[0-9][0-9]\\.(cfs|tlprojecta|tlproject2)\\.(desi|desicollab)\\.txt\\.gz$":"\\1_\\2_files.tar", + "[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]\\.json$":"EXCLUDE" }, "backups":{ "backups/.*$":"EXCLUDE" @@ -223,6 +231,7 @@ }, "mocks":{ "__exclude__":["lya_forest/README.html", + "lya_forest/london/README", "lya_forest/london/v6.0/README", "lya_forest/london/v6.0/lyasim.log", "lya_forest/saclay/README", @@ -238,7 +247,8 @@ }, "GLAM":{ "GLAM/ABACUS/.*$":"EXCLUDE", - "GLAM/GLAM_(BDM|LRG)_([^/]+)/.*$":"GLAM/GLAM_\\1_\\2.tar", + "GLAM/GLAM_BDM_MNRAS/.*$":"EXCLUDE", + "GLAM/GLAM_LRG_([^/]+)/.*$":"GLAM/GLAM_LRG_\\1.tar", "GLAM/JFE_files/.*$":"EXCLUDE" }, "Gadget":{ @@ -286,8 +296,11 @@ "lightcone_galform/.*$":"lightcone_galform.tar" }, "lya_forest":{ + "lya_forest/\\.ipynb_checkpoints/.*$":"EXCLUDE", "lya_forest/develop/.*$":"EXCLUDE", "lya_forest/dr11/.*$":"EXCLUDE", + "lya_forest/london/README$":"EXCLUDE", + "lya_forest/london/(colore_boxes|lyacolore_skewers|qq_desi_y1|qq_eboss)/.*$":"EXCLUDE", "lya_forest/london/(v[23]\\.[0-9])/[^/]+$":"lya_forest/london/\\1/lya_forest_london_\\1_files.tar", "lya_forest/london/(v[23]\\.[0-9])/([0-9]+)/.*$":"lya_forest/london/\\1/lya_forest_london_\\1_\\2.tar", "lya_forest/london/(v[23]\\.[0-9])/quick-([0-9.]+)/.*$":"lya_forest/london/\\1/lya_forest_london_\\1_quick-\\2.tar", @@ -301,11 +314,14 @@ "lya_forest/london/(v[45]\\.[0-9]\\.[0-9])/(logs|tests)/.*$":"lya_forest/london/\\1/lya_forest_london_\\1_\\2.tar", "lya_forest/london/(v[6789]\\.[0-9])/README":"lya_forest/london/\\1/README", "lya_forest/london/(v[6789]\\.[0-9])/.ipynb_checkpoints/.*$":"EXCLUDE", + "lya_forest/london/(v[6789]\\.[0-9])/colore_raw/.*$":"EXCLUDE", + "lya_forest/london/(v[6789]\\.[0-9])/(v[6789]\\.[0-9]\\.[0-9]+)/.ipynb_checkpoints/.*$":"EXCLUDE", "lya_forest/london/(v[6789]\\.[0-9])/(v[6789]\\.[0-9]\\.[0-9]+)/[^/]+$":"lya_forest/london/\\1/\\2/lya_forest_london_\\1_\\2_files.tar", "lya_forest/london/(v[6789]\\.[0-9])/(v[6789]\\.[0-9]\\.[0-9]+)/([0-9]+)/.*$":"lya_forest/london/\\1/\\2/lya_forest_london_\\1_\\2_\\3.tar", "lya_forest/london/(v[6789]\\.[0-9])/(v[6789]\\.[0-9]\\.[0-9]+)/eboss-([0-9.]+)/.*$":"lya_forest/london/\\1/\\2/lya_forest_london_\\1_\\2_eboss-\\3.tar", "lya_forest/london/(v[6789]\\.[0-9])/(v[6789]\\.[0-9]\\.[0-9]+)/logs/.*$":"lya_forest/london/\\1/\\2/lya_forest_london_\\1_\\2_logs.tar", "lya_forest/picca/.*$":"EXCLUDE", + "lya_forest/public/.*$":"EXCLUDE", "lya_forest/saclay/.*$":"EXCLUDE", "lya_forest/uncorr/.*$":"EXCLUDE" }, @@ -342,17 +358,29 @@ "doc/.*$":"EXCLUDE" }, "dr1":{ + "dr1/[^/]+$":"dr1_files.tar", "dr1/spectro/data/([0-9]+)/.*$":"dr1/spectro/data/desi_spectro_data_\\1.tar", + "dr1/spectro/desi_spectro_calib/([v0-9.]+)/.*$":"dr1/spectro/desi_spectro_calib/dr1_spectro_desi_spectro_calib_\\1.tar", "dr1/spectro/redux/(guadalupe|iron)/[^/]+$":"dr1/spectro/redux/\\1/redux_\\1_files.tar", - "dr1/spectro/redux/(guadalupe|iron)/(calibnight|dashboard|exposure_tables|processing_tables|run|zcatalog)/.*$":"dr1/spectro/redux/\\1/redux_\\1_\\2.tar", + "dr1/spectro/redux/(guadalupe|iron)/(calibnight|dashboard|exposure_tables|processing_tables|run)/.*$":"dr1/spectro/redux/\\1/redux_\\1_\\2.tar", + "dr1/spectro/redux/(guadalupe|iron)/(zcatalog)/(v[0-9])/.*$":"dr1/spectro/redux/\\1/redux_\\1_\\2_\\3.1.tar", "dr1/spectro/redux/(guadalupe|iron)/(exposures|preproc)/([0-9]+)/.*$":"dr1/spectro/redux/\\1/\\2/redux_\\1_\\2_\\3.tar", "dr1/spectro/redux/(guadalupe|iron)/healpix/[^/]+$":"dr1/spectro/redux/\\1/healpix/redux_\\1_healpix_files.tar", "dr1/spectro/redux/(guadalupe|iron)/healpix/(cmx|main|special|sv1|sv2|sv3)/(backup|bright|dark|other)/([0-9]+)/.*$":"dr1/spectro/redux/\\1/healpix/\\2/\\3/redux_\\1_healpix_\\2_\\3_\\4.tar", "dr1/spectro/redux/(guadalupe|iron)/tiles/(1x_depth|4x_depth|lowspeed|cumulative|perexp|pernight)/([0-9]+)/.*$":"dr1/spectro/redux/\\1/tiles/\\2/redux_\\1_tiles_\\2_\\3.tar", + "dr1/spectro/templates/basis_templates/([v0-9.]+)/.*$":"dr1/spectro/templates/dr1_spectro_templates_basis_templates_\\1.tar", + "dr1/survey/(GFA|ops)/.*$":"dr1/survey/dr1_survey_\\1.tar", "dr1/target/fiberassign/tiles/tags/([0-9.]+)/[^/]+$":"dr1/fiberassign/tiles/tags/\\1/dr1_fiberassign_tiles_tags_\\1_files.tar", "dr1/target/fiberassign/tiles/tags/([0-9.]+)/([0-9]{3})/.*$":"dr1/fiberassign/tiles/tags/\\1/dr1_fiberassign_tiles_tags_\\1_\\2.tar", - "dr1/vac/dr1/(fastspecfit|lsdr9-photometry)/(guadalupe|iron)/(v[0-9.]+)/.*$":"dr1/vac/dr1/\\1/\\2/dr1_vac_dr1_\\1_\\2_\\3.tar" - }, + "dr1/vac/dr1/(fastspecfit|lsdr9-photometry)/(guadalupe)/(v[0-9.]+)/.*$":"dr1/vac/dr1/\\1/\\2/dr1_vac_dr1_\\1_\\2_\\3.tar", + "dr1/vac/dr1/(fastspecfit)/(iron)/(v[0-9.]+)/README\\.md$":"dr1/vac/dr1/\\1/\\2/\\3/README.md", + "dr1/vac/dr1/(fastspecfit)/(iron)/(v[0-9.]+)/catalogs/fastspec-iron\\.fits":"dr1/vac/dr1/\\1/\\2/\\3/catalogs/fastspec-iron.fits", + "dr1/vac/dr1/(fastspecfit)/(iron)/(v[0-9.]+)/catalogs/(?!fastspec-iron\\.fits)([^/]+)$":"dr1/vac/dr1/\\1/\\2/\\3/dr1_vac_dr1_\\1_\\2_\\3_catalogs.tar", + "dr1/vac/dr1/(fastspecfit)/(iron)/(v[0-9.]+)/logs/.*$":"dr1/vac/dr1/\\1/\\2/\\3/dr1_vac_dr1_\\1_\\2_\\3_logs.tar", + "dr1/vac/dr1/(fastspecfit)/(iron)/(v[0-9.]+)/healpix/(cmx|main|special|sv1|sv2|sv3)/.*$":"dr1/vac/dr1/\\1/\\2/\\3/healpix/dr1_vac_dr1_\\1_\\2_\\3_healpix_\\4.tar", + "dr1/vac/dr1/(lsdr9-photometry)/(iron)/(v[0-9.]+)/.*$":"dr1/vac/dr1/\\1/\\2/dr1_vac_dr1_\\1_\\2_\\3.tar" + }, + "dr2":{}, "edr":{ "edr/[^/]+$":"edr_files.tar", "edr/spectro/data/([0-9]+)/.*$":"edr/spectro/data/desi_spectro_data_\\1.tar", @@ -369,8 +397,8 @@ "edr/target/fiberassign/tiles/tags/([0-9.]+)/[^/]+$":"edr/fiberassign/tiles/tags/\\1/edr_fiberassign_tiles_tags_\\1_files.tar", "edr/target/fiberassign/tiles/tags/([0-9.]+)/([0-9]{3})/.*$":"edr/fiberassign/tiles/tags/\\1/edr_fiberassign_tiles_tags_\\1_\\2.tar", "edr/target/(masks|skybricks|skyhealpixs)/([0-9dev.]+)/.*$":"edr/target/\\1/edr_target_\\1_\\2.tar", - "edr/vac/edr/(bal|lsdr9-photometry|lya|mws|zcat)/(fuji)/(v[0-9.]+)/.*$":"edr/vac/edr/edr_vac_edr_\\1_\\2_\\3.tar", - "edr/vac/edr/(lss|mgii-absorber|provabgs|vi)/(v[0-9.]+)/.*$":"edr/vac/edr/edr_vac_edr_\\1_\\2.tar" + "edr/vac/edr/(bal|cigale|dla|fastspecfit|lsdr9-photometry|lya|mws|zcat)/(fuji)/(v[0-9.]+)/.*$":"edr/vac/edr/edr_vac_edr_\\1_\\2_\\3.tar", + "edr/vac/edr/(epoviz|lss|mgii-absorber|provabgs|qso|stellar-mass-emline|uchuu|vi)/(v[0-9.]+)/.*$":"edr/vac/edr/edr_vac_edr_\\1_\\2.tar" }, "epo":{ }, @@ -389,7 +417,7 @@ } }, "science":{ - "__exclude__":["README"], + "__exclude__":[], "Y1KP":{}, "bgs":{}, "c3":{}, @@ -443,14 +471,19 @@ "quicklook":{ }, "redux":{ - "redux/(?!andes|blanc|cascades|denali|everest|minisv2|month4|oak1)([^/]+)/.*$":"EXCLUDE", - "redux/(andes|blanc|cascades|denali|everest|month4)/[^/]+$":"redux/\\1/redux_\\1_files.tar", - "redux/(andes|blanc|cascades|denali|everest)/(calibnight|dashboard|exposure_tables|processing_tables|run|zcatalog)/.*$":"redux/\\1/redux_\\1_\\2.tar", + "redux/(?!andes|blanc|cascades|daily|denali|everest|jura|kibo|minisv2|month4|oak1)([^/]+)/.*$":"EXCLUDE", + "redux/daily/[^/]+$":"EXCLUDE", + "redux/daily/(?!tiles)([^/]+)/.*$":"EXCLUDE", + "redux/daily/tiles/archive/([0-9]+)/([0-9]+)/.*$":"redux/daily/tiles/archive/redux_daily_tiles_archive_\\1_\\2.tar", + "redux/(andes|blanc|cascades|denali|everest|jura|kibo|month4)/[^/]+$":"redux/\\1/redux_\\1_files.tar", + "redux/(andes|blanc|cascades|denali|everest|jura|kibo)/(calibnight|dashboard|exposure_tables|nightqa|processing_tables|run)/.*$":"redux/\\1/redux_\\1_\\2.tar", + "redux/(andes|blanc|cascades|denali|everest)/(zcatalog)/.*$":"redux/\\1/redux_\\1_\\2.tar", + "redux/(jura|kibo)/zcatalog/(v[0-9])/.*$":"redux/\\1/redux_\\1_zcatalog_\\2.tar", "redux/(andes|blanc|cascades)/(exposures|preproc|tiles)/([0-9]+)/.*$":"redux/\\1/\\2/redux_\\1_\\2_\\3.tar", - "redux/(denali|everest)/(exposures|preproc)/([0-9]+)/.*$":"redux/\\1/\\2/redux_\\1_\\2_\\3.tar", - "redux/(everest)/healpix/[^/]+$":"redux/\\1/healpix/redux_\\1_healpix_files.tar", - "redux/(everest)/healpix/(cmx|main|special|sv1|sv2|sv3)/(backup|bright|dark|other)/([0-9]+)/.*$":"redux/\\1/healpix/\\2/\\3/redux_\\1_healpix_\\2_\\3_\\4.tar", - "redux/(denali|everest)/tiles/(1x_depth|4x_depth|lowspeed|cumulative|perexp|pernight)/([0-9]+)/.*$":"redux/\\1/tiles/\\2/redux_\\1_tiles_\\2_\\3.tar", + "redux/(denali|everest|jura|kibo)/(exposures|preproc)/([0-9]+)/.*$":"redux/\\1/\\2/redux_\\1_\\2_\\3.tar", + "redux/(everest|jura|kibo)/healpix/[^/]+$":"redux/\\1/healpix/redux_\\1_healpix_files.tar", + "redux/(everest|jura|kibo)/healpix/(cmx|main|special|sv1|sv2|sv3)/(backup|bright|dark|other)/([0-9]+)/.*$":"redux/\\1/healpix/\\2/\\3/redux_\\1_healpix_\\2_\\3_\\4.tar", + "redux/(denali|everest|jura|kibo)/tiles/(1x_depth|4x_depth|lowspeed|cumulative|perexp|pernight)/([0-9]+)/.*$":"redux/\\1/tiles/\\2/redux_\\1_tiles_\\2_\\3.tar", "redux/(minisv2|month4)/(alt|calib2d|calibnight|plan|run|tiles)/.*$":"redux/\\1/redux_\\1_\\2.tar", "redux/(minisv2|month4)/(exposures|preproc|spectra-64)/([0-9]+)/.*$":"redux/\\1/\\2/redux_\\1_\\2_\\3.tar", "redux/oak1/.*$":"redux/oak1.tar" @@ -469,8 +502,11 @@ } }, "survey":{ - "__exclude__":[], - "GFA":{}, + "__exclude__":["GFA.KPNO.log"], + "GFA.KPNO":{ + "GFA\\.KPNO/.*$": "GFA.KPNO.tar" + }, + "GFA.NERSC":{}, "analysis":{}, "catalogs":{}, "fiberassign":{}, @@ -478,7 +514,8 @@ "observations":{}, "ops":{}, "planning":{}, - "sims":{} + "sims":{}, + "sv":{} }, "sv":{ "__exclude__":[], @@ -512,7 +549,8 @@ "catalogs/dr5/(pixweight|randoms)-dr5\\.fits$": "catalogs/dr5/\\1-dr5.fits", "catalogs/dr7\\.1/0\\.(2[27])\\.0/[^/]+$": "catalogs/dr7.1/0.\\1.0/desi_target_catalogs_dr7.1_0.\\1.0_files.tar", "catalogs/dr7\\.1/0\\.(2[27])\\.0/randoms-dr7\\.1-0\\.(2[27])\\.0\\.fits$": "catalogs/dr7.1/\\1/randoms-dr7.1-0.\\1.0.fits", - "catalogs/dr7\\.1/([0-9.]+|PR[0-9]+)/.*$": "catalogs/dr7.1/desi_target_catalogs_dr7.1_\\1.tar" + "catalogs/dr7\\.1/([0-9.]+|PR[0-9]+)/.*$": "catalogs/dr7.1/desi_target_catalogs_dr7.1_\\1.tar", + "catalogs/streamcache/.*$":"EXCLUDE" }, "cmx_files":{ "cmx_files/.*$":"cmx_files.tar"