Skip to content

Commit

Permalink
Merge pull request #113 from SgmAstro/survey_nbs
Browse files Browse the repository at this point in the history
Survey nbs
  • Loading branch information
SgmAstro authored Feb 25, 2022
2 parents c985538 + 3e580d4 commit 7a067b8
Show file tree
Hide file tree
Showing 18 changed files with 5,110 additions and 1,957 deletions.
222 changes: 80 additions & 142 deletions docs/nb/d8LF_qa.ipynb

Large diffs are not rendered by default.

435 changes: 238 additions & 197 deletions docs/nb/ddp_QA.ipynb

Large diffs are not rendered by default.

477 changes: 259 additions & 218 deletions docs/nb/desi_qa.ipynb

Large diffs are not rendered by default.

266 changes: 158 additions & 108 deletions docs/nb/kE_catQA.ipynb

Large diffs are not rendered by default.

471 changes: 251 additions & 220 deletions docs/nb/lumfn.ipynb

Large diffs are not rendered by default.

938 changes: 287 additions & 651 deletions docs/nb/randoms_n8_qa.ipynb

Large diffs are not rendered by default.

6 changes: 6 additions & 0 deletions docs/nb/sandbox/.ipynb_checkpoints/Untitled-checkpoint.ipynb
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"cells": [],
"metadata": {},
"nbformat": 4,
"nbformat_minor": 5
}
6 changes: 6 additions & 0 deletions docs/nb/sandbox/.ipynb_checkpoints/Untitled1-checkpoint.ipynb
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"cells": [],
"metadata": {},
"nbformat": 4,
"nbformat_minor": 5
}
401 changes: 401 additions & 0 deletions docs/nb/sandbox/.ipynb_checkpoints/padded_hp-checkpoint.ipynb

Large diffs are not rendered by default.

2,025 changes: 2,025 additions & 0 deletions docs/nb/sandbox/.ipynb_checkpoints/sb_randoms_n8_qa-checkpoint.ipynb

Large diffs are not rendered by default.

242 changes: 242 additions & 0 deletions docs/nb/sandbox/Untitled.ipynb

Large diffs are not rendered by default.

232 changes: 232 additions & 0 deletions docs/nb/sandbox/Untitled1.ipynb

Large diffs are not rendered by default.

401 changes: 401 additions & 0 deletions docs/nb/sandbox/padded_hp.ipynb

Large diffs are not rendered by default.

738 changes: 401 additions & 337 deletions docs/nb/zmax_catQA.ipynb

Large diffs are not rendered by default.

123 changes: 79 additions & 44 deletions findfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
'zmax',\
'vmax',\
'lumfn',\
'lumfn_step',\
'ddp',\
'ddp_n8']

Expand Down Expand Up @@ -43,15 +44,18 @@ def fetch_fields(survey):

return fields

def release_dir(user=os.environ['USER'], versioned=True, survey='gama'):
assert survey == 'gama', 'TODO: Support DESI.'
def release_dir(user=os.environ['USER'], survey='gama', version=None):
#assert survey == 'gama', 'TODO: Support DESI.'

# E.g. /cosma/home/durham/dc-wils7/data/GAMA4/
if versioned:
if version == 'latest':
ff = glob.glob('/cosma/home/durham/{}/data/v*'.format(user))
ff.sort(key=os.path.getmtime)

return ff[-1]

elif version != None:
return '/cosma/home/durham/{}/data/{}/'.format(user, version)

else:
return '/cosma/home/durham/{}/data/GAMA4/'.format(user)
Expand All @@ -61,9 +65,9 @@ def overwrite_check(opath):
print('{} found on disk and overwrite forbidden (--nooverwrite).'.format(opath))
exit(0)

def findfile(ftype, dryrun=False, prefix=None, field=None, utier='{utier}', survey='gama', realz=0, debug=False):
def findfile(ftype, dryrun=False, prefix=None, field=None, utier='{utier}', survey='gama', realz=0, debug=False, version=None):
survey = survey.lower()

# Special case:
if (ftype == 'gold') & dryrun & (survey == 'gama'):
return os.environ['CODE_ROOT'] + '/data/gama_gold_dryrun.fits'
Expand All @@ -75,21 +79,47 @@ def findfile(ftype, dryrun=False, prefix=None, field=None, utier='{utier}', surv
fields = desi_fields

else:
raise NotImplementedError
raise NotImplementedError()

# TODO: Re-add assert and test
#if field != None:
# assert field in fields

if dryrun:
dryrun = '_dryrun'
debug = True

else:
dryrun = ''

realz = str(realz)

gold_dir = os.environ['GOLD_DIR']
rand_dir = os.environ['RANDOMS_DIR']
if version == None:
if 'GOLD_DIR' in os.environ:
gold_dir = os.environ['GOLD_DIR']

else:
gold_dir = os.environ['HOME'] + '/data/GAMA4/'

print('Warning: GOLD_DIR not defined in environment; assuming {gold_dir}')

if 'RANDOMS_DIR' in os.environ:
rand_dir = os.environ['RANDOMS_DIR']

else:
rand_dir = os.environ['HOME'] + '/data/GAMA4/randoms/'

print('Warning: RANDOMS_DIR not defined in environment; assuming {randoms_dir}')

else:
gold_dir = release_dir(version=version)
rand_dir = release_dir(version=version) + '/randoms/'

if isinstance(field, list):
return [findfile(ftype, dryrun=dryrun, prefix=prefix, field=ff, utier=utier) for ff in field]

if ftype == 'summary_log':
return gold_dir + 'summary.log'

if field == None:
file_types = {'gold': {'dir': gold_dir, 'id': f'{survey}', 'ftype': 'gold'},\
Expand All @@ -105,34 +135,36 @@ def findfile(ftype, dryrun=False, prefix=None, field=None, utier='{utier}', surv
fpath = parts['dir'] + '/{}_{}{}.fits'.format(parts['id'], parts['ftype'], dryrun)

else:
file_types = {'ddp_n8_d0': {'dir': gold_dir, 'id': f'{survey}_gold', 'ftype': 'ddp_n8_d0_{}'.format(utier)},\
'ddp_n8_d0_vmax': {'dir': gold_dir, 'id': f'{survey}_gold', 'ftype': 'ddp_n8_d0_{}_vmax'.format(utier)},\
'ddp_n8_d0_lumfn': {'dir': gold_dir, 'id': f'{survey}_gold', 'ftype': 'ddp_n8_d0_{}_lumfn'.format(utier)},\
'randoms': {'dir': rand_dir, 'id': 'randoms', 'ftype': realz},\
'randoms_n8': {'dir': rand_dir, 'id': 'randoms_N8', 'ftype': realz},\
'randoms_bd': {'dir': rand_dir, 'id': 'randoms_bd', 'ftype': realz},\
'randoms_bd_ddp_n8': {'dir': rand_dir, 'id': 'randoms_bd_ddp_n8', 'ftype': realz}
file_types = {'ddp_n8_d0': {'dir': gold_dir, 'id': f'{survey}_gold', 'ftype': 'ddp_n8_d0_{}'.format(utier)},\
'ddp_n8_d0_vmax': {'dir': gold_dir, 'id': f'{survey}_gold', 'ftype': 'ddp_n8_d0_{}_vmax'.format(utier)},\
'ddp_n8_d0_lumfn': {'dir': gold_dir, 'id': f'{survey}_gold', 'ftype': 'ddp_n8_d0_{}_lumfn'.format(utier)},\
'randoms': {'dir': rand_dir, 'id': 'randoms', 'ftype': realz},\
'randoms_n8': {'dir': rand_dir, 'id': 'randoms_N8', 'ftype': realz},\
'randoms_bd': {'dir': rand_dir, 'id': 'randoms_bd', 'ftype': realz},\
'randoms_bd_ddp_n8': {'dir': rand_dir, 'id': 'randoms_bd_ddp_n8', 'ftype': realz}
}

parts = file_types[ftype]
fpath = f'' + parts['dir'] + '/{}_{}_{}{}.fits'.format(parts['id'], field, parts['ftype'], dryrun)

if prefix != None:
assert 'randoms' in prefix;

dirname = os.path.dirname(fpath)
fpath = os.path.basename(fpath)
if prefix != None:
assert 'randoms' in prefix;
assert 'randoms' in fpath

dirname = os.path.dirname(fpath)
fpath = os.path.basename(fpath)

fpath = fpath.replace('randoms', prefix)
fpath = dirname + '/' + fpath

fpath = fpath.replace('randoms', prefix)
fpath = dirname + '/' + fpath

if debug:
print(f'DEBUG: findfile returns {fpath}')

fpath = fpath.replace('//', '/')

return fpath

def file_check(dryrun=None):
def file_check(dryrun=None):
try:
dryrun = os.environ['DRYRUN']

Expand All @@ -141,24 +173,26 @@ def file_check(dryrun=None):

dryrun = ''

fpaths = [findfile(xx, dryrun=False, prefix='', field=None) for xx in supported]

for field in fields:
fpaths.append(findfile('randoms', dryrun=False, prefix='', field=field))
fpaths.append(findfile('randoms_n8', dryrun=False, prefix='', field=field))
fpaths.append(findfile('randoms_bd', dryrun=False, prefix='', field=field))
fpaths.append(findfile('randoms_ddp1', dryrun=False, prefix='', field=field))
fpaths.append(findfile('randoms_ddp1_n8', dryrun=False, prefix='', field=field))
fpaths.append(findfile('randoms_ddp1_bd', dryrun=False, prefix='', field=field))
fpaths.append(findfile('randoms_ddp1_bd_n8', dryrun=False, prefix='', field=field))
fpaths.append(findfile('randoms_bd_ddp_n8', dryrun=False, prefix='', field=field))

for ii, _ in enumerate(d8_limits):
fpaths.append(findfile('ddp_n8_d0', dryrun=False, prefix='', field=field, utier=ii))
fpaths.append(findfile('ddp_n8_d0_vmax', dryrun=False, prefix='', field=field, utier=ii))
fpaths.append(findfile('ddp_n8_d0_lumfn', dryrun=False, prefix='', field=field, utier=ii))


fpaths = []

for survey in ['desi', 'gama']:
for xx in supported:
fpaths.append(findfile(xx, dryrun=False, survey=survey))

fields = fetch_fields(survey)

for field in fields:
for prefix in [None, 'randoms_ddp1']:
fpaths.append(findfile('randoms', dryrun=False, field=field, prefix=prefix))
fpaths.append(findfile('randoms_n8', dryrun=False, field=field, prefix=prefix))
fpaths.append(findfile('randoms_bd', dryrun=False, field=field, prefix=prefix))
fpaths.append(findfile('randoms_bd_ddp_n8', dryrun=False, field=field, prefix=prefix))

for ii, _ in enumerate(d8_limits):
fpaths.append(findfile('ddp_n8_d0', dryrun=False, field=field, utier=ii, survey=survey))
fpaths.append(findfile('ddp_n8_d0_vmax', dryrun=False, field=field, utier=ii, survey=survey))
fpaths.append(findfile('ddp_n8_d0_lumfn', dryrun=False, field=field, utier=ii, survey=survey))

print('\n\n---- SUPPORTED FPATHS ----\n')

for fp in fpaths:
Expand All @@ -175,7 +209,8 @@ def file_check(dryrun=None):
rand_paths = sorted(glob.glob(os.environ['RANDOMS_DIR'] + '/*.fits'))
all_paths = gold_paths + rand_paths

unsupported = [x for x in all_paths if (x not in fpaths and 'dryrun' not in x)]
unsupported = [x for x in all_paths if x not in fpaths]
unsupported = [x for x in unsupported if 'dryrun' not in x]

print('\n\n---- UNSUPPORTED FPATHS ----\n')

Expand All @@ -195,5 +230,5 @@ def file_check(dryrun=None):

if __name__ == '__main__':
failure = file_check()

print('\n\nSuccess: {}\n\n'.format(~failure))
3 changes: 3 additions & 0 deletions gama_gold.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,9 @@ def gama_gold(args):
idx = np.random.choice(np.arange(len(dat)), 5000, replace=False)
dat = dat[idx]

dat.meta = dat.meta = {'AREA': dat.meta['AREA'],\
'GOLD_NGAL': dat.meta['GOLD_NGAL']}

dat.write(os.environ['CODE_ROOT'] + '/data/gama_gold_dryrun.fits', format='fits', overwrite=True)


Expand Down
60 changes: 33 additions & 27 deletions test_allnb.py
Original file line number Diff line number Diff line change
@@ -1,41 +1,41 @@
import argparse
import papermill as pm

from tidyup import tidyup
from tidyup import tidyup
from findfile import fetch_fields

parser = argparse.ArgumentParser(description='Select field.')
parser.add_argument('-s', '--survey', help='Survey, e.g. GAMA, DESI, etc.', type=str, default='gama')

survey = args.survey.lower()

fields = fetchfields(survey)

# https://docs.pytest.org/en/6.2.x/
def test_allnbs():
def test_allnbs(survey='gama'):
print('Running all tests.')


if (survey != 'gama') and (survey != 'desi'):
raise NotImplementedError(f'No implementation for survey: {survey}')

tidyup()

run_randomqa()

run_goldqa()
run_randomqa(survey)

run_delta8qa()
run_goldqa(survey)

run_delta8qa(survey)

print('Done.')

def run_randomqa():
def run_randomqa(survey):
fields = fetch_fields(survey)

print(fields)

for field in fields:
print('Running random QA for field {}'.format(field))

pm.execute_notebook('docs/nb/randoms_n8_qa.ipynb',\
'test/pm_randoms_n8_{}_qa.ipynb'.format(field),\
parameters=dict(field=field),\
parameters=dict(field=field,survey=survey),\
kernel='lumfn',\
)

def run_goldqa():
def run_goldqa(survey):
print('Running gold QA')

tests = ['zmax_catQA', 'kE_catQA', 'ddp_QA', 'lumfn', 'delta8_qa']
Expand All @@ -44,12 +44,13 @@ def run_goldqa():
try:
pm.execute_notebook('docs/nb/{}.ipynb'.format(test),\
'test/pm_{}.ipynb'.format(test),\
parameters=dict(survey=survey),\
kernel='lumfn',\
)

except Error as E:
except:
print('Failed on {} test with error: '.format(test))
print(E)
#print(E)

'''
pm.execute_notebook('docs/nb/kE_catQA.ipynb',\
Expand All @@ -76,12 +77,17 @@ def run_goldqa():
# d8 LF.
# desi qa.

def run_delta8qa():
def run_delta8qa(survey):
fields = fetch_fields(survey)

for field in fields:
print('Running delta8 QA for field {}'.format(field))

pm.execute_notebook('docs/nb/delta8_qa.ipynb',\
'test/pm_delta8_qa_{}.ipynb'.format(field),\
parameters=dict(field=field),\
kernel='lumfn',\
)

pm.execute_notebook('docs/nb/d8LF_qa.ipynb',\
'test/pm_delta8_qa_{}.ipynb'.format(field),\
parameters=dict(field=field, survey=survey),\
kernel='lumfn',\
)

if __name__ == '__main__':
test_allnbs()
21 changes: 8 additions & 13 deletions tidyup.py
Original file line number Diff line number Diff line change
@@ -1,26 +1,21 @@
import os
import sys

from findfile import file_check, fields


DRYRUN = os.environ['DRYRUN']
GOLDDIR = os.environ['GOLD_DIR']
RANDDIR = os.environ['RANDOMS_DIR']
from findfile import file_check, findfile

def tidyup():
# File check summary.
sys.stdout = open(GOLDDIR + 'summary.log', 'w')
fpath = findfile('summary_log')

keep = sys.stdout

sys.stdout = open(fpath, 'w')

file_check()

sys.stdout.close()

# Gather Randoms and write to disk.
#
# fpaths = findfile('ddp_n8_d0', dryrun=False, prefix='', field=fields, utier=6)
# all_cats = gather_cat(fpaths)
# all_cats.pprint()

sys.stdout = keep


if __name__ == '__main__':
Expand Down

0 comments on commit 7a067b8

Please sign in to comment.