From d0f25c1d2cef4d30d224e1b922c3998e6d5914bc Mon Sep 17 00:00:00 2001 From: Patricia Larsen Date: Thu, 4 Jan 2018 13:28:41 -0800 Subject: [PATCH 01/64] first commit of shear test --- Shear_test.ipynb | 626 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 626 insertions(+) create mode 100644 Shear_test.ipynb diff --git a/Shear_test.ipynb b/Shear_test.ipynb new file mode 100644 index 00000000..5bcf0fe8 --- /dev/null +++ b/Shear_test.ipynb @@ -0,0 +1,626 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "#from __future__ import unicode_literals, absolute_import\n", + "import os\n", + "import sys\n", + "\n", + "sys.path.insert(0,'/global/common/software/lsst/common/miniconda/py3-4.2.12/lib/python3.6/site-packages')\n", + "sys.path.insert(0,'/global/common/software/lsst/common/miniconda/py3-4.2.12/bin/python')\n", + "\n", + "\n", + "from GCR import GCRQuery\n", + "import GCRCatalogs\n", + "import descqa \n", + "\n", + "from builtins import str\n", + "import yaml\n", + "import numpy as np\n", + "import healpy as hp\n", + "from descqa.plotting import plt\n", + "\n", + "from descqa import BaseValidationTest, TestResult\n", + "\n", + "\n", + "\n", + "import camb\n", + "import treecorr\n", + "#import treecorr\n", + "# neither of these will currently work\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "__all__ = ['ShearValidation']\n", + "\n", + "\n", + "\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class ExampleTest(BaseValidationTest):\n", + " def get_catalog_data(gc, quantities, filters=None):\n", + " data = {}\n", + " if not gc.has_quantities(quantities):\n", + " return TestResult(skipped=True, summary='Missing requested quantities')\n", + "\n", + " data = gc.get_quantities(quantities, filters=filters)\n", + " #make sure data entries are all finite\n", + " data = GCRQuery(*((np.isfinite, col) for col in data)).filter(data)\n", + "\n", + " return data\n", + " \n", + " def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):\n", + "\n", + " # check if needed quantities exist\n", + " if not catalog_instance.has_quantities(['ra', 'dec']):\n", + " return TestResult(skipped=True, summary='do not have needed quantities')\n", + " catalog_data = self.get_catalog_data(catalog_instance, [self.z, mag_field], filters=self.filters)\n", + "\n", + "\n", + " data = np.random.rand(10) #do your calculation with catalog_instance\n", + "\n", + " fig, ax = plt.subplots()\n", + "\n", + " for ax_this in (ax, self.summary_ax):\n", + " ax_this.plot(data, label=catalog_name)\n", + "\n", + " self.post_process_plot(ax)\n", + " fig.savefig(os.path.join(output_dir, 'plot.png'))\n", + " plt.close(fig)\n", + "\n", + " score = data[0] #calculate your summary statistics\n", + " return TestResult(score, passed=True)\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!ls /global/common/software/lsst/common/miniconda/py3-4.2.12/lib/python3.6/site-packages\n", + "!ls -l /global/common/software/lsst/common/miniconda/py3-4.2.12/lib/python3.6/site-packages\n", + "!groups" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#print(help('modules'))\n", + "#import treecorr\n", + "import treecorr\n", + "\n", + "cat = treecorr.Catalog('cat.fits', ra_col='RA', dec_col='DEC', ra_units='degrees', dec_units='degrees', g1_col='GAMMA1', g2_col='GAMMA2')\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#theory computation\n", + "# this is now fine!\n", + "\n", + "\n", + "#from astropy.cosmology import FlatLambdaCDM\n", + "#cosmo = FlatLambdaCDM(H0=70, Om0=0.3, Tcmb0=2.725) \n", + "from astropy.cosmology import WMAP9 as cosmo # want protoDC2 cosmology here\n", + "import camb\n", + "from scipy.integrate import quad\n", + "from scipy.interpolate import interp1d, InterpolatedUnivariateSpline\n", + "from astropy.cosmology import z_at_value\n", + "\n", + "import astropy.units as u\n", + "\n", + "#update cosmo to have the right parameters\n", + "pars = camb.CAMBparams()\n", + "h = 0.675\n", + "pars.set_cosmology(H0=h*100, ombh2=0.022, omch2=0.122)\n", + "pars.InitPower.set_params(ns=0.965)\n", + "\n", + "camb.set_halofit_version(version='takahashi')\n", + "p = camb.get_matter_power_interpolator(pars, nonlinear=True, k_hunit=False, hubble_units=False ,kmax=100., zmax=1100., k_per_logint=False).P\n", + "# no h corrections necessary for this. Everything in Mpc units\n", + "\n", + "import astropy.constants as const\n", + "from camb import model, initialpower\n", + "\n", + "chi_recomb = 14004.036207574154\n", + "\n", + "# p(z, k/h)\n", + "\n", + "def test_nz(n_z):\n", + " '''create mock n(z) distribution to test theory code'''\n", + " z_bins = np.linspace(0.0,2.0,101)\n", + " n = np.histogram(n_z,bins=z_bins)[0]\n", + " z = (z_bins[1:]-z_bins[:-1])/2. + z_bins[:-1]\n", + " #n2 = InterpolatedUnivariateSpline(z, n, w=None, bbox=[None, None], k=3, ext=0, check_finite=False)\n", + " n2 = interp1d(z,n,bounds_error=False,fill_value=0.0,kind='cubic')\n", + " n2_sum = quad(n2,0,2.0)[0]\n", + " #n2 = InterpolatedUnivariateSpline(z, n/n2_sum, w=None, bbox=[None, None], k=3, ext=0, check_finite=False)\n", + " n2 = interp1d(z,n/n2_sum,bounds_error=False,fill_value=0.0,kind='cubic')\n", + " return n2\n", + "\n", + "def integrand_w(x,n,chi,chi_int):\n", + " ''' This is the inner bit of GWL lensing kernel - z is related to x, not chi'''\n", + " z = chi_int(x)\n", + " H_z = cosmo.H(z).value*3.240779289469756e-20 #1/s units #.to(1./u.s) conversion\n", + " dchidz =9.715611890256315e-15 /H_z #const.c.to(u.Mpc/u.s).value / (H_z) # Mpc units\n", + " return n(z)/dchidz* (x - chi)/x\n", + "\n", + "# integrand_w * const.c.to(u.Mpc/u.s).value, divided by Hubble (km/Mpcs).to(1./u.s).value\n", + "def galaxy_W(z,n,chi_int):\n", + " ''' galaxy window function'''\n", + " chi = cosmo.comoving_distance(z).value # can be array\n", + " cst = 3./2. * cosmo.H(0).to(1./u.s)**2/const.c.to(u.Mpc / u.s)**2*cosmo.Om(0)# not sure about the z-dependence here\n", + " prefactor = cst * chi* (1.+z) *u.Mpc\n", + " val_array = []\n", + " for i in range(len(z)):\n", + " val_array.append(quad(integrand_w,chi[i],chi_recomb, args = (n,chi[i],chi_int))[0])\n", + " W = np.array(val_array)*prefactor *(u.Mpc) # now unitless\n", + " return W\n", + "\n", + "# not currently used\n", + "def cmb_W(z):\n", + " chi = cosmo.comoving_distance(z).value # can be array\n", + " cst = 3./2. * cosmo.H(0).to(1./u.s)**2/const.c.to(u.Mpc / u.s)**2*cosmo.Om(0)# not sure about the z-dependence here\n", + " prefactor = cst * chi* (1.+z) *u.Mpc**2\n", + " return (chi_recomb - chi)/chi_recomb * prefactor\n", + "\n", + "def integrand_lensing_limber(chi,l,galaxy_W_int,chi_int):\n", + " '''return overall integrand for one value of l'''\n", + " chi_unit = chi*u.Mpc\n", + " z = chi_int(chi)#z_at_value(cosmo.comoving_distance, chi_unit)\n", + " k = (l+0.5)/chi\n", + " integrand = p(z,k,grid=False) *galaxy_W_int(z)**2/chi**2 # check this divided by chi2\n", + " return integrand\n", + "\n", + "\n", + "\n", + "def phi(lmax,n_z):\n", + " z_array = np.logspace(-3,np.log10(10.),200)\n", + " chi_array = cosmo.comoving_distance(z_array).value\n", + " chi_int = interp1d(chi_array,z_array,bounds_error=False,fill_value=0.0)\n", + " n = compute_nz(n_z)\n", + " galaxy_W_int = interp1d(z_array,galaxy_W(z_array,n,chi_int),bounds_error=False,fill_value=0.0)\n", + " #cmb_W_int = interp1d(z_array,cmb_W(z_array),bounds_error=False,fill_value=0.0)\n", + " phi = []\n", + " l= range(0,lmax,1)#range(1,10,1) +range(10,100,5) #+range(100,1000,10)+ range(1000,10000,100)\n", + " l = np.array(l)\n", + " for i in l:\n", + " a = quad(integrand_lensing_limber,1.e-10,14004.036207574154,args=(i,galaxy_W_int,chi_int),epsrel = 1.e-6)[0]\n", + " phi.append(a)\n", + " phi = np.array(phi)\n", + " prefactor = 1.0#(l+2)*(l+1)*l*(l-1) / (l+0.5)**4\n", + " return l,phi*prefactor\n", + "\n", + "\n", + "def theory_corr(n_z,min_sep=2.5,max_sep=250.,nbins=20,lmax=20000):\n", + " '''The only thing you actually have to run'''\n", + " ll, pp = phi(lmax=lmax,n_z=n_z)\n", + " pp3_2 = np.zeros((lmax,4))\n", + " pp3_2[:,1]= pp[:]*(ll*(ll+1.))/(2.*np.pi)\n", + " xvals = np.logspace(np.log10(min_sep),np.log10(max_sep),nbins) #in arcminutes\n", + " cxvals = np.cos(xvals*(60.)*(180./np.pi))\n", + " vals = camb.correlations.cl2corr(pp3_2,cxvals)\n", + " return xvals, vals[:,1], vals[:,2]\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import treecorr\n", + "\n", + "infile = pf.open('protoDC2_WLPipe.fits')[1].data\n", + "print(len(infile))\n", + "mask = (infile['z_mean']>0.5)*(infile['z_mean']<0.6)\n", + "\n", + "cat_s = treecorr.Catalog(ra=infile['ra'][mask], dec=infile['dec'][mask], \n", + " g1=infile['e1'][mask]-np.mean(infile['e1'][mask]), g2=infile['e2'][mask]-np.mean(infile['e2'][mask]), ra_units='deg', dec_units='deg')\n", + "\n", + "gg = treecorr.GGCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin', bin_slop=0.1, verbose=True)\n", + "gg.process_cross(cat_s, cat_s)\n", + "\n", + "print(np.exp(gg.logr), gg.xip)\n", + "np.savez('output.npz', theta=np.exp(gg.logr), ggp=gg.xip, ggm=gg.xim, weight=gg.weight, npairs=gg.npairs)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "\n", + "__all__ = ['ShearTest']\n", + "\n", + "class ShearTest(BaseValidationTest):\n", + " \"\"\"\n", + " An example validation test\n", + " \"\"\"\n", + " def __init__(self, **kwargs):\n", + "\n", + " # load test config options\n", + " self.kwargs = kwargs\n", + " self.option1 = kwargs.get('option1', 'option1_default')\n", + " self.option2 = kwargs.get('option2', 'option2_default')\n", + " self.test_name = kwargs.get('test_name', 'example_test')\n", + "\n", + " # load validation data\n", + " with open(os.path.join(self.data_dir, 'README.md')) as f:\n", + " self.validation_data = f.readline().strip()\n", + "\n", + " # prepare summary plot\n", + " self.summary_fig, self.summary_ax = plt.subplots()\n", + "\n", + "\n", + " def post_process_plot(self, ax):\n", + " ax.text(0.05, 0.95, self.validation_data)\n", + " ax.legend()\n", + "\n", + "\n", + " def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):\n", + "\n", + " # check if needed quantities exist\n", + " if not catalog_instance.has_quantities(['ra', 'dec']):\n", + " return TestResult(skipped=True, summary='do not have needed quantities')\n", + "\n", + " data = np.random.rand(10) #do your calculation with catalog_instance\n", + "\n", + " fig, ax = plt.subplots()\n", + "\n", + " for ax_this in (ax, self.summary_ax):\n", + " ax_this.plot(data, label=catalog_name)\n", + "\n", + " self.post_process_plot(ax)\n", + " fig.savefig(os.path.join(output_dir, 'plot.png'))\n", + " plt.close(fig)\n", + "\n", + " score = data[0] #calculate your summary statistics\n", + " return TestResult(score, passed=True)\n", + "\n", + "\n", + " def conclude_test(self, output_dir):\n", + " self.post_process_plot(self.summary_ax)\n", + " self.summary_fig.savefig(os.path.join(output_dir, 'summary.png'))\n", + " plt.close(self.summary_fig)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "__all__ = ['ShearTest']\n", + "\n", + "class ShearTest(BaseValidationTest):\n", + " \"\"\"\n", + " Validation test for shear and convergence quantities\n", + " \"\"\"\n", + "\n", + " def __init__(self, z='redshift_true', ra = 'ra', dec = 'dec', e1 = 'shear_1', e2= 'shear_2', kappa ='kappa', nbins=20, min_sep=2.5, max_sep=250, \n", + " sep_units='arcmin', bin_slop=0.1, zlo = 0.5, zhi= 0.6, **kwargs):\n", + "\n", + " #catalog quantities\n", + " self.z = z\n", + "\n", + " #sep-bounds and binning\n", + " self.min_sep = min_sep\n", + " self.max_sep = max_sep\n", + " self.nbins = nbins\n", + " self.sep_bins = np.linspace(min_sep, max_sep, nbins+1)\n", + " self.sep_units = sep_units\n", + " self.bin_slop = bin_slop\n", + " \n", + " self.ra = ra\n", + " self.dec = dec\n", + " self.e1 = e1\n", + " self.e2 = e2\n", + " self.kappa = kappa\n", + " \n", + " \n", + " # cut in redshift\n", + " self.filters = [(lambda z: (z > zlo) & (z < zhi), self.z)]\n", + " \n", + "\n", + " self.summary_fig, self.summary_ax = plt.subplots()\n", + "\n", + " #validation data\n", + " # want this to change to theory... \n", + " #self.validation_data = {}\n", + " #self.observation = observation\n", + " #setup subplot configuration and get magnitude cuts for each plot\n", + " #self.mag_lo, self.mag_hi = self.init_plots(mag_lo, mag_hi)\n", + " #setup summary plot\n", + " #self.summary_fig, self.summary_ax = plt.subplots(self.nrows, self.ncolumns, figsize=(figx_p, figy_p), sharex='col')\n", + " #could plot summary validation data here if available but would need to evaluate labels, bin values etc.\n", + " #otherwise setup a check so that validation data is plotted only once on summary plot\n", + " #self.first_pass = True\n", + " #self._other_kwargs = kwargs\n", + " \n", + " \n", + " def compute_nz(n_z):\n", + " '''create interpolated n(z) distribution'''\n", + " z_bins = np.linspace(0.0,2.0,101)\n", + " n = np.histogram(n_z,bins=z_bins)[0]\n", + " z = (z_bins[1:]-z_bins[:-1])/2. + z_bins[:-1]\n", + " n2 = interp1d(z,n,bounds_error=False,fill_value=0.0,kind='cubic')\n", + " n2_sum = quad(n2,0,2.0)[0]\n", + " n2 = interp1d(z,n/n2_sum,bounds_error=False,fill_value=0.0,kind='cubic')\n", + " return n2\n", + "\n", + " def integrand_w(x,n,chi,chi_int):\n", + " ''' This is the inner bit of GWL lensing kernel - z is related to x, not chi'''\n", + " z = chi_int(x)\n", + " H_z = cosmo.H(z).value*3.240779289469756e-20 #1/s units #.to(1./u.s) conversion\n", + " dchidz = 9.715611890256315e-15 /H_z #const.c.to(u.Mpc/u.s).value / (H_z) # Mpc units\n", + " return n(z)/dchidz* (x - chi)/x\n", + "\n", + " def galaxy_W(z,n,chi_int):\n", + " ''' galaxy window function'''\n", + " chi = cosmo.comoving_distance(z).value # can be array\n", + " cst = 3./2. * cosmo.H(0).to(1./u.s)**2/const.c.to(u.Mpc / u.s)**2*cosmo.Om(0)# not sure about the z-dependence here\n", + " prefactor = cst * chi* (1.+z) *u.Mpc\n", + " val_array = []\n", + " for i in range(len(z)):\n", + " val_array.append(quad(integrand_w,chi[i],chi_recomb, args = (n,chi[i],chi_int))[0])\n", + " W = np.array(val_array)*prefactor *(u.Mpc) # now unitless\n", + " return W\n", + "\n", + " def integrand_lensing_limber(chi,l,galaxy_W_int,chi_int):\n", + " '''return overall integrand for one value of l'''\n", + " chi_unit = chi*u.Mpc\n", + " z = chi_int(chi)\n", + " k = (l+0.5)/chi\n", + " integrand = p(z,k,grid=False) *galaxy_W_int(z)**2/chi**2 \n", + " return integrand\n", + "\n", + " def phi(lmax,n_z):\n", + " z_array = np.logspace(-3,np.log10(10.),200)\n", + " chi_array = cosmo.comoving_distance(z_array).value\n", + " chi_int = interp1d(chi_array,z_array,bounds_error=False,fill_value=0.0)\n", + " n = self.compute_nz(n_z)\n", + " galaxy_W_int = interp1d(z_array,self.galaxy_W(z_array,n,chi_int),bounds_error=False,fill_value=0.0)\n", + " phi_array = []\n", + " l= range(0,lmax,1)\n", + " l = np.array(l)\n", + " for i in l:\n", + " a = quad(self.integrand_lensing_limber,1.e-10,chi_recomb,args=(i,galaxy_W_int,chi_int),epsrel = 1.e-6)[0]\n", + " phi_array.append(a)\n", + " phi_array = np.array(phi_array)\n", + " prefactor = 1.0#(l+2)*(l+1)*l*(l-1) / (l+0.5)**4\n", + " return l,phi_array*prefactor\n", + " \n", + "\n", + " def theory_corr(n_z,min_sep=2.5,max_sep=250.,nbins=20,lmax=20000):\n", + " ll, pp = self.phi(lmax=lmax,n_z=n_z)\n", + " pp3_2 = np.zeros((lmax,4))\n", + " pp3_2[:,1]= pp[:]*(ll*(ll+1.))/(2.*np.pi)\n", + " xvals = np.logspace(np.log10(min_sep),np.log10(max_sep),nbins) #in arcminutes\n", + " cxvals = np.cos(xvals*(60.)*(180./np.pi))\n", + " vals = camb.correlations.cl2corr(pp3_2,cxvals)\n", + " return xvals, vals[:,1], vals[:,2]\n", + "\n", + "\n", + " def get_chi2(measured,theory):\n", + " chi2 = (measured-theory)**2/theory\n", + " return chi2/float(len(measured))\n", + " \n", + " @staticmethod\n", + " def get_catalog_data(gc, quantities, filters=None):\n", + " data = {}\n", + " if not gc.has_quantities(quantities):\n", + " return TestResult(skipped=True, summary='Missing requested quantities')\n", + "\n", + " data = gc.get_quantities(quantities, filters=filters)\n", + " #make sure data entries are all finite\n", + " data = GCRQuery(*((np.isfinite, col) for col in data)).filter(data)\n", + "\n", + " return data\n", + "\n", + "\n", + " # define theory from within this class\n", + " \n", + " def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):\n", + "\n", + " # check if needed quantities exist\n", + " if not catalog_instance.has_quantities([self.z, self.ra, self.dec, self.e1, self.e2,self.kappa]):\n", + " #print(\"failed\")\n", + " return TestResult(skipped=True, summary='do not have needed quantities')\n", + " catalog_data = self.get_catalog_data(catalog_instance, [self.z, self.ra, self.dec, self.e1, self.e2, self.kappa], filters=self.filters)\n", + " # get required catalogue data \n", + " \n", + " e1 = catalog_data[self.e1]\n", + " max_e1 = np.max(e1)\n", + " min_e1 = np.min(e1)\n", + " e2 = catalog_data[self.e2]\n", + " max_e2 = np.max(e2)\n", + " min_e2 = np.min(e2)\n", + " if ((min_e1<(-1.)) or (max_e1>1.0)):\n", + " return TestResult(skipped=True, summary='e1 values out of range [-1,+1]')\n", + " if ((min_e2<(-1.)) or (max_e2>1.0)):\n", + " return TestResult(skipped=True, summary='e2 values out of range [-1,+1]')\n", + " \n", + " \n", + " n_z = catalog_data[self.z]\n", + " xvals,theory_plus, theory_minus = self.theory_corr(n_z,min_sep=2.5,max_sep=250.,nbins=20,lmax=20000)\n", + "\n", + " cat_s = treecorr.Catalog(ra=catalog_data[self.ra], dec=catalog_data[self.dec], \n", + " g1=catalog_data[self.e1]-np.mean(catalog_data[self.e1]), g2=catalog_data[self.e2]-np.mean(catalog_data[self.e2]), ra_units='deg', dec_units='deg')\n", + " \n", + " gg = treecorr.GGCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin', bin_slop=0.1, verbose=True)\n", + " gg.process(cat_s)\n", + " \n", + " chi2_dof_1= self.get_chi2(gg.xip,theory_plus) # correct this\n", + " \n", + " r = numpy.exp(gg.meanlogr)\n", + " xip = gg.xip\n", + " xim = gg.xim\n", + " \n", + " plt.figure()\n", + " plt.plot(xip)\n", + " plt.plot(theory_plus)\n", + " plt.show()\n", + " sig = numpy.sqrt(gg.varxi)\n", + " \n", + " # directly compare xip and xim to theory_plus and theory_minus. Take into account cosmic variance. \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " # further correlation functions \n", + " #dd = treecorr.NNCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin')\n", + " #dd.process(cat)\n", + " #treecorr.NGCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') # count-shear (i.e. (R))\n", + " #treecorr.NKCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') # count-kappa (i.e. (R))\n", + " #treecorr.KKCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') # count-kappa (i.e. (R))\n", + " # etc... \n", + "\n", + "\n", + "#print(np.exp(gg.logr), gg.xip)\n", + "#np.savez('output.npz', theta=np.exp(gg.logr), ggp=gg.xip, ggm=gg.xim, weight=gg.weight, npairs=gg.npairs)\n", + " \n", + " data = np.random.rand(10) #do your calculation with catalog_instance\n", + "\n", + " fig, ax = plt.subplots()\n", + "\n", + " for ax_this in (ax, self.summary_ax):\n", + " ax_this.plot(data, label=catalog_name)\n", + "\n", + " #self.post_process_plot(ax)\n", + " #fig.savefig(os.path.join(output_dir, 'plot.png'))\n", + " #plt.close(fig)\n", + "\n", + " score = data[0] #calculate your summary statistics\n", + " return TestResult(score, passed=True)\n", + "\n", + "\n", + "\n", + " def conclude_test(self, output_dir):\n", + " self.post_process_plot(self.summary_ax)\n", + " self.summary_fig.savefig(os.path.join(output_dir, 'summary.png'))\n", + " plt.close(self.summary_fig)\n", + " \n", + " \n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['halo_mass', 'shear_2', 'size_disk_true', 'size_bulge_true', 'mag_g_sdss', 'redshift_true', 'dec', 'stellar_mass', 'ellipticity_true', 'ellipticity_2_true', 'position_y', 'mag_r_lsst', 'Mag_true_i_lsst_z0', 'size_minor_bulge_true', 'ellipticity_2_disk_true', 'mag_i_sdss', 'position_angle_true', 'Mag_true_r_lsst_z0', 'ellipticity_disk_true', 'Mag_true_g_lsst_z0', 'mag_u_lsst', 'Mag_true_Y_lsst_z0', 'Mag_true_z_lsst_z0', 'Mag_true_i_sdss_z0', 'redshift', 'mag_i_lsst', 'mag_z_lsst', 'size_true', 'convergence', 'ra_true', 'sersic_bulge', 'galaxy_id', 'size_minor_disk_true', 'ra', 'position_z', 'Mag_true_r_sdss_z0', 'Mag_true_g_sdss_z0', 'mag_z_sdss', 'ellipticity_1_bulge_true', 'ellipticity_1_disk_true', 'is_central', 'mag_g_lsst', 'mag_r_sdss', 'velocity_y', 'sersic_disk', 'magnification', 'shear_1', 'ellipticity_bulge_true', 'velocity_z', 'Mag_true_u_lsst_z0', 'Mag_true_u_sdss_z0', 'ellipticity_2_bulge_true', 'position_x', 'velocity_x', 'halo_id', 'mag_u_sdss', 'dec_true', 'mag_Y_lsst', 'Mag_true_z_sdss_z0', 'ellipticity_1_true']\n" + ] + } + ], + "source": [ + "print(quantities)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD+CAYAAADWKtWTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADi1JREFUeJzt3WFoXed9x/Hvfxl169RSayaTMSw3qdYlS9O0TNvw0tIX\nzmBg/MYd5EXWprRwS1pa0gzcrE0DTprU8zKXmrE4It3CCC2MemM1DgWjF6UJpsQeNN2Lkqwwt3Rp\nleCucsimbut/L3QynSi2dSQd3Sv8/35AcJ/nPvfwv4/ln8997jmPIzORJNXwK6MuQJI0PIa+JBVi\n6EtSIYa+JBVi6EtSIYa+JBVi6EtSIYa+JBWyYuhHxDUR8VhEPHOJ598YEX8VEX8WEX8TEe/ov0xJ\nUh+6nOm/F/gnIC7x/F3ADzPzi8CXgK/0VJskqWcrhn5mfh24cJkhe4HTzdjvATdHxFg/5UmS+vSr\nPRxjB6/9R2G+6ZtfPjAiBsCg3Xf11Vf/zvXXX99DGZJUw9mzZ1/KzIm1vLaP0J8DtrXaY03f62Tm\nDDDT7puens4zZ870UIYk1RAR59b62jVdvRMR21tLOCeB3U3/TcB3M/N1Z/mSpNHrcvXO+4EPAr8e\nEfdGxJuAe4CPN0O+DOyKiHuBPwU+ulHFSpLWZ8Xlncz8FvCtZd0HWs//J/CJnuuSJG0Ab86SpEIM\nfUkqxNCXpEIMfUkqxNCXpEIMfUkqpI87clctIvYB+wCmpqZGUYIklTSSM/3MPJGZg8wcjI+Pj6IE\nSSrJ5R1JKsTQl6RCDH1JKsTQl6RCDH1JKsTQl6RCDH1JKsTQl6RCDH1JKsRtGCSpELdhkKRCXN6R\npEIMfUkqxNCXpEIMfUkqxNCXpEIMfUkqxNCXpEIMfUkqxDtyJakQ78iVpEJc3pGkQgx9SSrE0Jek\nQgx9SSrE0JekQgx9SSqk03X6EXErsB+YAzIzDy57/lrgYeAZ4N3AVzPzGz3XKklapxVDPyK2AseA\nGzNzISKOR8SezJxtDTsAPJWZX4qI9wB/Dxj6krTJdFne2Q2cy8yFpv00sHfZmJ8CE83jCeBsP+VJ\nkvrUZXlnB3Ch1Z5v+tqOAP8YEUeA3wMeuNiBImIADNp9k5OTnYuVJK1Pl9CfA7a12mNNX9vjwGOZ\n+bWImACej4jrMvN8e1BmzgAz7b7p6elcddWSpDXpsrxzGtgVEVua9i3AyYjYHhFjTd9O4IXm8c+A\nX3Y8tiRpiFY808/MVyLiTuBoRLwIPJuZsxFxGDgPHAI+DdwVEX8AXAt8NjNf2sjCJUmr1+mSzcw8\nBZxa1neg9fgp4Kl+S5Mk9c0lGEkqxNCXpEIMfUkqxNCXpEIMfUkqxNCXpEI6XbLZt4jYB+wDmJqa\nGkUJklTSSM70M/NEZg4yczA+Pj6KEiSpJJd3JKkQQ1+SCjH0JakQQ1+SCjH0JakQQ1+SCjH0JakQ\nQ1+SCjH0JakQt2GQpELchkGSCnF5R5IKMfQlqRBDX5IKMfQlqRBDX5IKMfQlqRBDX5IKMfQlqRBD\nX5IKcRsGSSrEbRgkqRCXdySpEENfkgox9CWpEENfkgox9CWpEENfkgrpdJ1+RNwK7AfmgMzMg8ue\nD+CTTfNtwFsy8yM91ilJ6sGKoR8RW4FjwI2ZuRARxyNiT2bOtob9CfAfmfl3zWvetTHlSpLWo8vy\nzm7gXGYuNO2ngb3LxtwObI+IT0XEQ8DLPdYoSepJl+WdHcCFVnu+6WvbBYxl5v0R8Q7gmxFxQ2b+\nb3tQRAyAQbtvcnJy9VVLktaky5n+HLCt1R5r+trmge8AZOZzzZidyw+UmTOZOd3+mZiYWFvlkqRV\n6xL6p4FdEbGlad8CnIyI7REx1vTNAtcBNH1XAT/pu1hJ0vqsuLyTma9ExJ3A0Yh4EXg2M2cj4jBw\nHjgE/DlwOCI+C7wduCMz/2sjC5ckrV6nSzYz8xRwalnfgdbjnwMf67c0SVLfvDlLkgox9CWpEENf\nkgox9CWpEENfkgox9CWpkE6XbPYtIvYB+wCmpqZGUYIklTSSM/3MPJGZg8wcjI+Pj6IESSrJ5R1J\nKsTQl6RCDH1JKsTQl6RCDH1JKsTQl6RCDH1JKsTQl6RCDH1JKsRtGCSpELdhkKRCXN6RpEIMfUkq\nxNCXpEIMfUkqxNCXpEIMfUkqxNCXpEIMfUkqxDtyJakQ78iVpEJc3pGkQgx9SSrE0JekQgx9SSrE\n0JekQgx9SSqk03X6EXErsB+YAzIzD15i3O3AE8C2zHy5tyolSb1YMfQjYitwDLgxMxci4nhE7MnM\n2WXjbgB+e4PqlCT1oMvyzm7gXGYuNO2ngb3tAc0/DAeAi34CkCRtDl2Wd3YAF1rt+aav7UHg/sz8\nRURc8kARMQAG7b7JyclulUqS1q1L6M8B21rtsaYPgIjYCbwVuK0V+HdHxJOZeaZ9oMycAWbafdPT\n07mGuiVJa9Al9E8DuyJiS7PEcwvw1xGxHfifzPwR8OFXB0fEF4EjfpErSZvPimv6mfkKcCdwNCK+\nADzbfIl7D/DxV8dFxERE3Ns0D0TEb2xEwZKktYvM0a6uTE9P55kzZ1YeKEkCICLOZub0Wl7rzVmS\nVIihL0mFGPqSVIihL0mFGPqSVIihL0mFdNpls28RsQ/YBzA1NTWKEiSppJGc6WfmicwcZOZgfHx8\nFCVIUkku70hSIYa+JBVi6EtSIYa+JBVi6EtSIYa+JBVi6EtSIYa+JBVi6EtSIW7DIEmFuA2DJBXi\n8o4kFWLoS1Ihhr4kFWLoS1Ihhr4kFWLoS1Ihhr4kFWLoS1Ihhr4kFeI2DJJUiNswSFIhLu9IUiGG\nviQVYuhLUiGGviQVYuhLUiGGviQV0uk6/Yi4FdgPzAGZmQeXPf8Z4BrgBWAauC8zv99zrZKkdVox\n9CNiK3AMuDEzFyLieETsyczZ1rA3A3dnZkbEbcBf0Nx8JUnaPLos7+wGzmXmQtN+GtjbHpCZn8/M\nbB3z5f5KlCT1pcvyzg7gQqs93/S9TkS8AbgD+MQlnh8Ag3bf5ORkp0IlSevXJfTngG2t9ljT9xpN\n4D8CfC4zf3CxA2XmDDDT7puens6LjZUk9a/L8s5pYFdEbGnatwAnI2J7RIzB/6/7PwocycyzEfGB\njSlXkrQeK57pZ+YrEXEncDQiXgSezczZiDgMnAcOAU8A7wSujQiAq4HjG1e2JGktOl2ymZmngFPL\n+g60Hu/vuS5J0gbw5ixJKsTQl6RCDH1JKsTQl6RCDH1JKsTQl6RCOl2y2beI2EezIdvU1NQoSpCk\nkkZypp+ZJzJzkJmD8fHxUZQgSSW5vCNJhRj6klSIoS9JhRj6klSIoS9JhRj6klSIoS9JhRj6klSI\noS9JhbgNgyQV4jYMklSIyzuSVIihL0mFGPqSVIihL0mFGPqSVIihL0mFGPqSVIihL0mFeEeuJBXi\nHbmSVIjLO5JUiKEvSYUY+pJUiKEvSYUY+pJUiKEvSYV0uk4/Im4F9gNzQGbmwWXPvxF4GPgx8JvA\nocx8rudaJUnrtGLoR8RW4BhwY2YuRMTxiNiTmbOtYXcBP8zMwxFxE/AV4H0bU7Ikaa26LO/sBs5l\n5kLTfhrYu2zMXuA0QGZ+D7g5IsZ6q1KS1Isuyzs7gAut9nzT12XMfHtQRAyAwbLXLkTEv3Sq9uLG\ngZ+v4/Wb6Ri/Brw04ho2wzHWOw991LBZjuFcLHEulvzWml+ZmZf9AfYAs6323cCRZWO+Dbyv1Z4H\nxlY6djP2TJdxl3n9zHpev8mO4Vz0MA+b5X04F87FZpyLLss7p4FdEbGlad8CnIyI7a0lnJMsLgPR\nrOl/NzPnX38oSdIorbi8k5mvRMSdwNGIeBF4NjNnI+IwcB44BHwZeDgi7gWmgI9uZNHLnLiCjrFe\nm+V9OBf9HmO9Nsv7cC76PcaaRPNRYWQi4kxmTo+0iE3CuVjkPCxxLpY4F0vWMxeb4easmVEXsIk4\nF4uchyXOxRLnYsma52LkZ/qSpOHZDGf6kqQhMfQlqZCh/B+57t2zpMNcfAa4BngBmAbuy8zvD73Q\nIVhpLlrjbgeeALZl5stDLHFoOvxeBPDJpvk24C2Z+ZGhFjkkHebiWhbz4hng3cBXM/MbQy90CCLi\nGuALwM2Z+bsXeX712bnemww63ESwFfhXYEvTPg7sWTbmHuBA8/gm4NsbXdcofjrOxQMsfddyG3Bi\n1HWPai6a/huAB4EE3jzqukf4e/FB4EOt9rtGXfcI5+IR4NPN4/cAz4+67g2cjz8G9nGJm7HWkp3D\nWN5x754lK85FZn4+mz9BFpffrsgzWzrMRbPZ3wHgop8AriBd/o7cDmyPiE9FxEMU/r0AfgpMNI8n\ngLNDqm3oMvPrvHaLm+VWnZ3DCP317N1zpen8PiPiDcAdwL1DqGsUuszFg8D9mfmLoVU1Gl3mYheL\nW5scBR4HvhkRVw2nvKHqMhdHgN+PiCPAfcDfDqm2zWjV2TmMNf05YFurPdb0rXbMlaDT+2wC/xHg\nc5n5gyHVNmyXnYuI2Am8FbhtcTkbgLsj4snMPDO0Koejy+/FPPAdgMx8rjmb2wn82zAKHKIuc/E4\n8Fhmfi0iJoDnI+K6zDw/pBo3k1Vn5zDO9N27Z8mKc9EsaTzK4qZ2ZyPiAyOqdaNddi4y80eZ+eHM\nPJSZh5oxR67AwIduf0dmgesAmr6rgJ8MvdKN12UudrJ4oQPAz4BfUuhKxPVm51BuzoqIP2TxC4kX\ngf/OzIOv7t2TmYci4k0sfgP9Aot79zyUV+7VOyvNxT8A7wT+vXnJ1XmRb+2vBCvNRTNmAvgYi19w\nPwA8mpk/HlXNG6XD78U4cBg4B7wdOJ6ZT46u4o3TYS7ey+J/3PTPwLXA2cw8NrqKN05EvB/4EPBH\nLH76/0sWv+Nac3Z6R64kFVLmI5EkydCXpFIMfUkqxNCXpEIMfUkqxNCXpEIMfUkqxNCXpEL+DzhE\naZUyoCgdAAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "gc = GCRCatalogs.load_catalog('protoDC2')\n", + "quantities = gc.list_all_quantities()\n", + "test = ShearTest()\n", + "test.run_on_single_catalog( gc, 'protoDC2', './')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "lsst", + "language": "python", + "name": "lsst" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 90fcdc4c9a2a7796a57caa313162e8cfc59fe596 Mon Sep 17 00:00:00 2001 From: Patricia Larsen Date: Thu, 4 Jan 2018 15:08:38 -0800 Subject: [PATCH 02/64] Commented out unnecessary code --- Shear_test.ipynb | 194 +++++++++-------------------------------------- 1 file changed, 36 insertions(+), 158 deletions(-) diff --git a/Shear_test.ipynb b/Shear_test.ipynb index 5bcf0fe8..8e359204 100644 --- a/Shear_test.ipynb +++ b/Shear_test.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -48,87 +48,36 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ - "class ExampleTest(BaseValidationTest):\n", - " def get_catalog_data(gc, quantities, filters=None):\n", - " data = {}\n", - " if not gc.has_quantities(quantities):\n", - " return TestResult(skipped=True, summary='Missing requested quantities')\n", - "\n", - " data = gc.get_quantities(quantities, filters=filters)\n", - " #make sure data entries are all finite\n", - " data = GCRQuery(*((np.isfinite, col) for col in data)).filter(data)\n", - "\n", - " return data\n", - " \n", - " def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):\n", - "\n", - " # check if needed quantities exist\n", - " if not catalog_instance.has_quantities(['ra', 'dec']):\n", - " return TestResult(skipped=True, summary='do not have needed quantities')\n", - " catalog_data = self.get_catalog_data(catalog_instance, [self.z, mag_field], filters=self.filters)\n", - "\n", - "\n", - " data = np.random.rand(10) #do your calculation with catalog_instance\n", - "\n", - " fig, ax = plt.subplots()\n", - "\n", - " for ax_this in (ax, self.summary_ax):\n", - " ax_this.plot(data, label=catalog_name)\n", - "\n", - " self.post_process_plot(ax)\n", - " fig.savefig(os.path.join(output_dir, 'plot.png'))\n", - " plt.close(fig)\n", - "\n", - " score = data[0] #calculate your summary statistics\n", - " return TestResult(score, passed=True)\n", - "\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!ls /global/common/software/lsst/common/miniconda/py3-4.2.12/lib/python3.6/site-packages\n", - "!ls -l /global/common/software/lsst/common/miniconda/py3-4.2.12/lib/python3.6/site-packages\n", - "!groups" + "#!ls /global/common/software/lsst/common/miniconda/py3-4.2.12/lib/python3.6/site-packages\n", + "#!ls -l /global/common/software/lsst/common/miniconda/py3-4.2.12/lib/python3.6/site-packages\n", + "#!groups" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ - "#print(help('modules'))\n", "#import treecorr\n", - "import treecorr\n", "\n", - "cat = treecorr.Catalog('cat.fits', ra_col='RA', dec_col='DEC', ra_units='degrees', dec_units='degrees', g1_col='GAMMA1', g2_col='GAMMA2')\n", + "#cat = treecorr.Catalog('cat.fits', ra_col='RA', dec_col='DEC', ra_units='degrees', dec_units='degrees', g1_col='GAMMA1', g2_col='GAMMA2')\n", "\n" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ + "\n", + "(markdown to comment out - I think this has all been added to the test but leaving here just in case)\n", + "\n", "#theory computation\n", - "# this is now fine!\n", + " this is now fine!\n", "\n", "\n", "#from astropy.cosmology import FlatLambdaCDM\n", @@ -149,14 +98,14 @@ "\n", "camb.set_halofit_version(version='takahashi')\n", "p = camb.get_matter_power_interpolator(pars, nonlinear=True, k_hunit=False, hubble_units=False ,kmax=100., zmax=1100., k_per_logint=False).P\n", - "# no h corrections necessary for this. Everything in Mpc units\n", + " no h corrections necessary for this. Everything in Mpc units\n", "\n", "import astropy.constants as const\n", "from camb import model, initialpower\n", "\n", "chi_recomb = 14004.036207574154\n", "\n", - "# p(z, k/h)\n", + " p(z, k/h)\n", "\n", "def test_nz(n_z):\n", " '''create mock n(z) distribution to test theory code'''\n", @@ -177,7 +126,7 @@ " dchidz =9.715611890256315e-15 /H_z #const.c.to(u.Mpc/u.s).value / (H_z) # Mpc units\n", " return n(z)/dchidz* (x - chi)/x\n", "\n", - "# integrand_w * const.c.to(u.Mpc/u.s).value, divided by Hubble (km/Mpcs).to(1./u.s).value\n", + " integrand_w * const.c.to(u.Mpc/u.s).value, divided by Hubble (km/Mpcs).to(1./u.s).value\n", "def galaxy_W(z,n,chi_int):\n", " ''' galaxy window function'''\n", " chi = cosmo.comoving_distance(z).value # can be array\n", @@ -189,7 +138,7 @@ " W = np.array(val_array)*prefactor *(u.Mpc) # now unitless\n", " return W\n", "\n", - "# not currently used\n", + " not currently used\n", "def cmb_W(z):\n", " chi = cosmo.comoving_distance(z).value # can be array\n", " cst = 3./2. * cosmo.H(0).to(1./u.s)**2/const.c.to(u.Mpc / u.s)**2*cosmo.Om(0)# not sure about the z-dependence here\n", @@ -239,91 +188,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ - "import treecorr\n", - "\n", - "infile = pf.open('protoDC2_WLPipe.fits')[1].data\n", - "print(len(infile))\n", - "mask = (infile['z_mean']>0.5)*(infile['z_mean']<0.6)\n", - "\n", - "cat_s = treecorr.Catalog(ra=infile['ra'][mask], dec=infile['dec'][mask], \n", - " g1=infile['e1'][mask]-np.mean(infile['e1'][mask]), g2=infile['e2'][mask]-np.mean(infile['e2'][mask]), ra_units='deg', dec_units='deg')\n", - "\n", - "gg = treecorr.GGCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin', bin_slop=0.1, verbose=True)\n", - "gg.process_cross(cat_s, cat_s)\n", - "\n", - "print(np.exp(gg.logr), gg.xip)\n", - "np.savez('output.npz', theta=np.exp(gg.logr), ggp=gg.xip, ggm=gg.xim, weight=gg.weight, npairs=gg.npairs)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "\n", - "__all__ = ['ShearTest']\n", - "\n", - "class ShearTest(BaseValidationTest):\n", - " \"\"\"\n", - " An example validation test\n", - " \"\"\"\n", - " def __init__(self, **kwargs):\n", - "\n", - " # load test config options\n", - " self.kwargs = kwargs\n", - " self.option1 = kwargs.get('option1', 'option1_default')\n", - " self.option2 = kwargs.get('option2', 'option2_default')\n", - " self.test_name = kwargs.get('test_name', 'example_test')\n", - "\n", - " # load validation data\n", - " with open(os.path.join(self.data_dir, 'README.md')) as f:\n", - " self.validation_data = f.readline().strip()\n", - "\n", - " # prepare summary plot\n", - " self.summary_fig, self.summary_ax = plt.subplots()\n", - "\n", - "\n", - " def post_process_plot(self, ax):\n", - " ax.text(0.05, 0.95, self.validation_data)\n", - " ax.legend()\n", + "#import treecorr#\n", + "#\n", + "#infile = pf.open('protoDC2_WLPipe.fits')[1].data\n", + "#print(len(infile))\n", + "#mask = (infile['z_mean']>0.5)*(infile['z_mean']<0.6)#\n", + "#\n", + "#cat_s = treecorr.Catalog(ra=infile['ra'][mask], dec=infile['dec'][mask], g1=infile['e1'][mask]-np.mean(infile['e1'][mask]), g2=infile['e2'][mask]-np.mean(infile['e2'][mask]), ra_units='deg', dec_units='deg')\n", "\n", + "#gg = treecorr.GGCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin', bin_slop=0.1, verbose=True)\n", + "#gg.process_cross(cat_s, cat_s)\n", "\n", - " def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):\n", - "\n", - " # check if needed quantities exist\n", - " if not catalog_instance.has_quantities(['ra', 'dec']):\n", - " return TestResult(skipped=True, summary='do not have needed quantities')\n", - "\n", - " data = np.random.rand(10) #do your calculation with catalog_instance\n", - "\n", - " fig, ax = plt.subplots()\n", - "\n", - " for ax_this in (ax, self.summary_ax):\n", - " ax_this.plot(data, label=catalog_name)\n", - "\n", - " self.post_process_plot(ax)\n", - " fig.savefig(os.path.join(output_dir, 'plot.png'))\n", - " plt.close(fig)\n", - "\n", - " score = data[0] #calculate your summary statistics\n", - " return TestResult(score, passed=True)\n", - "\n", - "\n", - " def conclude_test(self, output_dir):\n", - " self.post_process_plot(self.summary_ax)\n", - " self.summary_fig.savefig(os.path.join(output_dir, 'summary.png'))\n", - " plt.close(self.summary_fig)\n" + "#print(np.exp(gg.logr), gg.xip)\n", + "#np.savez('output.npz', theta=np.exp(gg.logr), ggp=gg.xip, ggm=gg.xim, weight=gg.weight, npairs=gg.npairs)" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -546,33 +432,25 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 16, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['halo_mass', 'shear_2', 'size_disk_true', 'size_bulge_true', 'mag_g_sdss', 'redshift_true', 'dec', 'stellar_mass', 'ellipticity_true', 'ellipticity_2_true', 'position_y', 'mag_r_lsst', 'Mag_true_i_lsst_z0', 'size_minor_bulge_true', 'ellipticity_2_disk_true', 'mag_i_sdss', 'position_angle_true', 'Mag_true_r_lsst_z0', 'ellipticity_disk_true', 'Mag_true_g_lsst_z0', 'mag_u_lsst', 'Mag_true_Y_lsst_z0', 'Mag_true_z_lsst_z0', 'Mag_true_i_sdss_z0', 'redshift', 'mag_i_lsst', 'mag_z_lsst', 'size_true', 'convergence', 'ra_true', 'sersic_bulge', 'galaxy_id', 'size_minor_disk_true', 'ra', 'position_z', 'Mag_true_r_sdss_z0', 'Mag_true_g_sdss_z0', 'mag_z_sdss', 'ellipticity_1_bulge_true', 'ellipticity_1_disk_true', 'is_central', 'mag_g_lsst', 'mag_r_sdss', 'velocity_y', 'sersic_disk', 'magnification', 'shear_1', 'ellipticity_bulge_true', 'velocity_z', 'Mag_true_u_lsst_z0', 'Mag_true_u_sdss_z0', 'ellipticity_2_bulge_true', 'position_x', 'velocity_x', 'halo_id', 'mag_u_sdss', 'dec_true', 'mag_Y_lsst', 'Mag_true_z_sdss_z0', 'ellipticity_1_true']\n" - ] - } - ], + "outputs": [], "source": [ - "print(quantities)" + "#print(quantities)" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 10, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" }, @@ -580,7 +458,7 @@ "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD+CAYAAADWKtWTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADi1JREFUeJzt3WFoXed9x/Hvfxl169RSayaTMSw3qdYlS9O0TNvw0tIX\nzmBg/MYd5EXWprRwS1pa0gzcrE0DTprU8zKXmrE4It3CCC2MemM1DgWjF6UJpsQeNN2Lkqwwt3Rp\nleCucsimbut/L3QynSi2dSQd3Sv8/35AcJ/nPvfwv4/ln8997jmPIzORJNXwK6MuQJI0PIa+JBVi\n6EtSIYa+JBVi6EtSIYa+JBVi6EtSIYa+JBWyYuhHxDUR8VhEPHOJ598YEX8VEX8WEX8TEe/ov0xJ\nUh+6nOm/F/gnIC7x/F3ADzPzi8CXgK/0VJskqWcrhn5mfh24cJkhe4HTzdjvATdHxFg/5UmS+vSr\nPRxjB6/9R2G+6ZtfPjAiBsCg3Xf11Vf/zvXXX99DGZJUw9mzZ1/KzIm1vLaP0J8DtrXaY03f62Tm\nDDDT7puens4zZ870UIYk1RAR59b62jVdvRMR21tLOCeB3U3/TcB3M/N1Z/mSpNHrcvXO+4EPAr8e\nEfdGxJuAe4CPN0O+DOyKiHuBPwU+ulHFSpLWZ8Xlncz8FvCtZd0HWs//J/CJnuuSJG0Ab86SpEIM\nfUkqxNCXpEIMfUkqxNCXpEIMfUkqpI87clctIvYB+wCmpqZGUYIklTSSM/3MPJGZg8wcjI+Pj6IE\nSSrJ5R1JKsTQl6RCDH1JKsTQl6RCDH1JKsTQl6RCDH1JKsTQl6RCDH1JKsRtGCSpELdhkKRCXN6R\npEIMfUkqxNCXpEIMfUkqxNCXpEIMfUkqxNCXpEIMfUkqxDtyJakQ78iVpEJc3pGkQgx9SSrE0Jek\nQgx9SSrE0JekQgx9SSqk03X6EXErsB+YAzIzDy57/lrgYeAZ4N3AVzPzGz3XKklapxVDPyK2AseA\nGzNzISKOR8SezJxtDTsAPJWZX4qI9wB/Dxj6krTJdFne2Q2cy8yFpv00sHfZmJ8CE83jCeBsP+VJ\nkvrUZXlnB3Ch1Z5v+tqOAP8YEUeA3wMeuNiBImIADNp9k5OTnYuVJK1Pl9CfA7a12mNNX9vjwGOZ\n+bWImACej4jrMvN8e1BmzgAz7b7p6elcddWSpDXpsrxzGtgVEVua9i3AyYjYHhFjTd9O4IXm8c+A\nX3Y8tiRpiFY808/MVyLiTuBoRLwIPJuZsxFxGDgPHAI+DdwVEX8AXAt8NjNf2sjCJUmr1+mSzcw8\nBZxa1neg9fgp4Kl+S5Mk9c0lGEkqxNCXpEIMfUkqxNCXpEIMfUkqxNCXpEI6XbLZt4jYB+wDmJqa\nGkUJklTSSM70M/NEZg4yczA+Pj6KEiSpJJd3JKkQQ1+SCjH0JakQQ1+SCjH0JakQQ1+SCjH0JakQ\nQ1+SCjH0JakQt2GQpELchkGSCnF5R5IKMfQlqRBDX5IKMfQlqRBDX5IKMfQlqRBDX5IKMfQlqRBD\nX5IKcRsGSSrEbRgkqRCXdySpEENfkgox9CWpEENfkgox9CWpEENfkgrpdJ1+RNwK7AfmgMzMg8ue\nD+CTTfNtwFsy8yM91ilJ6sGKoR8RW4FjwI2ZuRARxyNiT2bOtob9CfAfmfl3zWvetTHlSpLWo8vy\nzm7gXGYuNO2ngb3LxtwObI+IT0XEQ8DLPdYoSepJl+WdHcCFVnu+6WvbBYxl5v0R8Q7gmxFxQ2b+\nb3tQRAyAQbtvcnJy9VVLktaky5n+HLCt1R5r+trmge8AZOZzzZidyw+UmTOZOd3+mZiYWFvlkqRV\n6xL6p4FdEbGlad8CnIyI7REx1vTNAtcBNH1XAT/pu1hJ0vqsuLyTma9ExJ3A0Yh4EXg2M2cj4jBw\nHjgE/DlwOCI+C7wduCMz/2sjC5ckrV6nSzYz8xRwalnfgdbjnwMf67c0SVLfvDlLkgox9CWpEENf\nkgox9CWpEENfkgox9CWpkE6XbPYtIvYB+wCmpqZGUYIklTSSM/3MPJGZg8wcjI+Pj6IESSrJ5R1J\nKsTQl6RCDH1JKsTQl6RCDH1JKsTQl6RCDH1JKsTQl6RCDH1JKsRtGCSpELdhkKRCXN6RpEIMfUkq\nxNCXpEIMfUkqxNCXpEIMfUkqxNCXpEIMfUkqxDtyJakQ78iVpEJc3pGkQgx9SSrE0JekQgx9SSrE\n0JekQgx9SSqk03X6EXErsB+YAzIzD15i3O3AE8C2zHy5tyolSb1YMfQjYitwDLgxMxci4nhE7MnM\n2WXjbgB+e4PqlCT1oMvyzm7gXGYuNO2ngb3tAc0/DAeAi34CkCRtDl2Wd3YAF1rt+aav7UHg/sz8\nRURc8kARMQAG7b7JyclulUqS1q1L6M8B21rtsaYPgIjYCbwVuK0V+HdHxJOZeaZ9oMycAWbafdPT\n07mGuiVJa9Al9E8DuyJiS7PEcwvw1xGxHfifzPwR8OFXB0fEF4EjfpErSZvPimv6mfkKcCdwNCK+\nADzbfIl7D/DxV8dFxERE3Ns0D0TEb2xEwZKktYvM0a6uTE9P55kzZ1YeKEkCICLOZub0Wl7rzVmS\nVIihL0mFGPqSVIihL0mFGPqSVIihL0mFdNpls28RsQ/YBzA1NTWKEiSppJGc6WfmicwcZOZgfHx8\nFCVIUkku70hSIYa+JBVi6EtSIYa+JBVi6EtSIYa+JBVi6EtSIYa+JBVi6EtSIW7DIEmFuA2DJBXi\n8o4kFWLoS1Ihhr4kFWLoS1Ihhr4kFWLoS1Ihhr4kFWLoS1Ihhr4kFeI2DJJUiNswSFIhLu9IUiGG\nviQVYuhLUiGGviQVYuhLUiGGviQV0uk6/Yi4FdgPzAGZmQeXPf8Z4BrgBWAauC8zv99zrZKkdVox\n9CNiK3AMuDEzFyLieETsyczZ1rA3A3dnZkbEbcBf0Nx8JUnaPLos7+wGzmXmQtN+GtjbHpCZn8/M\nbB3z5f5KlCT1pcvyzg7gQqs93/S9TkS8AbgD+MQlnh8Ag3bf5ORkp0IlSevXJfTngG2t9ljT9xpN\n4D8CfC4zf3CxA2XmDDDT7puens6LjZUk9a/L8s5pYFdEbGnatwAnI2J7RIzB/6/7PwocycyzEfGB\njSlXkrQeK57pZ+YrEXEncDQiXgSezczZiDgMnAcOAU8A7wSujQiAq4HjG1e2JGktOl2ymZmngFPL\n+g60Hu/vuS5J0gbw5ixJKsTQl6RCDH1JKsTQl6RCDH1JKsTQl6RCOl2y2beI2EezIdvU1NQoSpCk\nkkZypp+ZJzJzkJmD8fHxUZQgSSW5vCNJhRj6klSIoS9JhRj6klSIoS9JhRj6klSIoS9JhRj6klSI\noS9JhbgNgyQV4jYMklSIyzuSVIihL0mFGPqSVIihL0mFGPqSVIihL0mFGPqSVIihL0mFeEeuJBXi\nHbmSVIjLO5JUiKEvSYUY+pJUiKEvSYUY+pJUiKEvSYV0uk4/Im4F9gNzQGbmwWXPvxF4GPgx8JvA\nocx8rudaJUnrtGLoR8RW4BhwY2YuRMTxiNiTmbOtYXcBP8zMwxFxE/AV4H0bU7Ikaa26LO/sBs5l\n5kLTfhrYu2zMXuA0QGZ+D7g5IsZ6q1KS1Isuyzs7gAut9nzT12XMfHtQRAyAwbLXLkTEv3Sq9uLG\ngZ+v4/Wb6Ri/Brw04ho2wzHWOw991LBZjuFcLHEulvzWml+ZmZf9AfYAs6323cCRZWO+Dbyv1Z4H\nxlY6djP2TJdxl3n9zHpev8mO4Vz0MA+b5X04F87FZpyLLss7p4FdEbGlad8CnIyI7a0lnJMsLgPR\nrOl/NzPnX38oSdIorbi8k5mvRMSdwNGIeBF4NjNnI+IwcB44BHwZeDgi7gWmgI9uZNHLnLiCjrFe\nm+V9OBf9HmO9Nsv7cC76PcaaRPNRYWQi4kxmTo+0iE3CuVjkPCxxLpY4F0vWMxeb4easmVEXsIk4\nF4uchyXOxRLnYsma52LkZ/qSpOHZDGf6kqQhMfQlqZCh/B+57t2zpMNcfAa4BngBmAbuy8zvD73Q\nIVhpLlrjbgeeALZl5stDLHFoOvxeBPDJpvk24C2Z+ZGhFjkkHebiWhbz4hng3cBXM/MbQy90CCLi\nGuALwM2Z+bsXeX712bnemww63ESwFfhXYEvTPg7sWTbmHuBA8/gm4NsbXdcofjrOxQMsfddyG3Bi\n1HWPai6a/huAB4EE3jzqukf4e/FB4EOt9rtGXfcI5+IR4NPN4/cAz4+67g2cjz8G9nGJm7HWkp3D\nWN5x754lK85FZn4+mz9BFpffrsgzWzrMRbPZ3wHgop8AriBd/o7cDmyPiE9FxEMU/r0AfgpMNI8n\ngLNDqm3oMvPrvHaLm+VWnZ3DCP317N1zpen8PiPiDcAdwL1DqGsUuszFg8D9mfmLoVU1Gl3mYheL\nW5scBR4HvhkRVw2nvKHqMhdHgN+PiCPAfcDfDqm2zWjV2TmMNf05YFurPdb0rXbMlaDT+2wC/xHg\nc5n5gyHVNmyXnYuI2Am8FbhtcTkbgLsj4snMPDO0Koejy+/FPPAdgMx8rjmb2wn82zAKHKIuc/E4\n8Fhmfi0iJoDnI+K6zDw/pBo3k1Vn5zDO9N27Z8mKc9EsaTzK4qZ2ZyPiAyOqdaNddi4y80eZ+eHM\nPJSZh5oxR67AwIduf0dmgesAmr6rgJ8MvdKN12UudrJ4oQPAz4BfUuhKxPVm51BuzoqIP2TxC4kX\ngf/OzIOv7t2TmYci4k0sfgP9Aot79zyUV+7VOyvNxT8A7wT+vXnJ1XmRb+2vBCvNRTNmAvgYi19w\nPwA8mpk/HlXNG6XD78U4cBg4B7wdOJ6ZT46u4o3TYS7ey+J/3PTPwLXA2cw8NrqKN05EvB/4EPBH\nLH76/0sWv+Nac3Z6R64kFVLmI5EkydCXpFIMfUkqxNCXpEIMfUkqxNCXpEIMfUkqxNCXpEL+DzhE\naZUyoCgdAAAAAElFTkSuQmCC\n", "text/plain": [ - "" + "" ] }, "metadata": {}, From fd1850162e3a8a503d33f489f8c5ba2dff4654e2 Mon Sep 17 00:00:00 2001 From: Patricia Larsen Date: Fri, 5 Jan 2018 07:56:15 -0800 Subject: [PATCH 03/64] added config file and python file --- descqa/configs/shear.yaml | 14 +++ descqa/shear_test.py | 242 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 256 insertions(+) create mode 100644 descqa/configs/shear.yaml create mode 100644 descqa/shear_test.py diff --git a/descqa/configs/shear.yaml b/descqa/configs/shear.yaml new file mode 100644 index 00000000..93eba19f --- /dev/null +++ b/descqa/configs/shear.yaml @@ -0,0 +1,14 @@ +subclass_name: shear_test.ShearTest +z: 'redshift_true' +ra: 'ra' +dec: 'dec' +e1: 'shear_1' +e2: 'shear_2' +kappa: 'kappa' +nbins: 20 +min_sep: 2.5 +max_sep: 250 +sep_units: 'arcmin' +bin_slop: 0.1 +zlo: 0.5 +zhi: 0.6 \ No newline at end of file diff --git a/descqa/shear_test.py b/descqa/shear_test.py new file mode 100644 index 00000000..4d6083d8 --- /dev/null +++ b/descqa/shear_test.py @@ -0,0 +1,242 @@ + +from __future__ import unicode_literals, absolute_import, division +import os +import numpy as np +from .base import BaseValidationTest, TestResult +from .plotting import plt + + +import sys +sys.path.insert(0,'/global/common/software/lsst/common/miniconda/py3-4.2.12/lib/python3.6/site-packages') +sys.path.insert(0,'/global/common/software/lsst/common/miniconda/py3-4.2.12/bin/python') + + +from GCR import GCRQuery +import GCRCatalogs +import descqa + +from builtins import str +import yaml +import numpy as np +import healpy as hp +from descqa.plotting import plt + +from descqa import BaseValidationTest, TestResult + +import camb +import treecorr + + + +__all__ = ['ShearTest'] + +class ShearTest(BaseValidationTest): + """ + Validation test for shear and convergence quantities + """ + + def __init__(self, z='redshift_true', ra = 'ra', dec = 'dec', e1 = 'shear_1', e2= 'shear_2', kappa ='kappa', nbins=20, min_sep=2.5, max_sep=250, + sep_units='arcmin', bin_slop=0.1, zlo = 0.5, zhi= 0.6, **kwargs): + + #catalog quantities + self.z = z + + #sep-bounds and binning + self.min_sep = min_sep + self.max_sep = max_sep + self.nbins = nbins + self.sep_bins = np.linspace(min_sep, max_sep, nbins+1) + self.sep_units = sep_units + self.bin_slop = bin_slop + + self.ra = ra + self.dec = dec + self.e1 = e1 + self.e2 = e2 + self.kappa = kappa + + + # cut in redshift + self.filters = [(lambda z: (z > zlo) & (z < zhi), self.z)] + + + self.summary_fig, self.summary_ax = plt.subplots() + + #validation data + # want this to change to theory... + #self.validation_data = {} + #self.observation = observation + #setup subplot configuration and get magnitude cuts for each plot + #self.mag_lo, self.mag_hi = self.init_plots(mag_lo, mag_hi) + #setup summary plot + #self.summary_fig, self.summary_ax = plt.subplots(self.nrows, self.ncolumns, figsize=(figx_p, figy_p), sharex='col') + #could plot summary validation data here if available but would need to evaluate labels, bin values etc. + #otherwise setup a check so that validation data is plotted only once on summary plot + #self.first_pass = True + #self._other_kwargs = kwargs + + + def compute_nz(n_z): + '''create interpolated n(z) distribution''' + z_bins = np.linspace(0.0,2.0,101) + n = np.histogram(n_z,bins=z_bins)[0] + z = (z_bins[1:]-z_bins[:-1])/2. + z_bins[:-1] + n2 = interp1d(z,n,bounds_error=False,fill_value=0.0,kind='cubic') + n2_sum = quad(n2,0,2.0)[0] + n2 = interp1d(z,n/n2_sum,bounds_error=False,fill_value=0.0,kind='cubic') + return n2 + + def integrand_w(x,n,chi,chi_int): + ''' This is the inner bit of GWL lensing kernel - z is related to x, not chi''' + z = chi_int(x) + H_z = cosmo.H(z).value*3.240779289469756e-20 #1/s units #.to(1./u.s) conversion + dchidz = 9.715611890256315e-15 /H_z #const.c.to(u.Mpc/u.s).value / (H_z) # Mpc units + return n(z)/dchidz* (x - chi)/x + + def galaxy_W(z,n,chi_int): + ''' galaxy window function''' + chi = cosmo.comoving_distance(z).value # can be array + cst = 3./2. * cosmo.H(0).to(1./u.s)**2/const.c.to(u.Mpc / u.s)**2*cosmo.Om(0)# not sure about the z-dependence here + prefactor = cst * chi* (1.+z) *u.Mpc + val_array = [] + for i in range(len(z)): + val_array.append(quad(integrand_w,chi[i],chi_recomb, args = (n,chi[i],chi_int))[0]) + W = np.array(val_array)*prefactor *(u.Mpc) # now unitless + return W + + def integrand_lensing_limber(chi,l,galaxy_W_int,chi_int): + '''return overall integrand for one value of l''' + chi_unit = chi*u.Mpc + z = chi_int(chi) + k = (l+0.5)/chi + integrand = p(z,k,grid=False) *galaxy_W_int(z)**2/chi**2 + return integrand + + def phi(lmax,n_z): + z_array = np.logspace(-3,np.log10(10.),200) + chi_array = cosmo.comoving_distance(z_array).value + chi_int = interp1d(chi_array,z_array,bounds_error=False,fill_value=0.0) + n = self.compute_nz(n_z) + galaxy_W_int = interp1d(z_array,self.galaxy_W(z_array,n,chi_int),bounds_error=False,fill_value=0.0) + phi_array = [] + l= range(0,lmax,1) + l = np.array(l) + for i in l: + a = quad(self.integrand_lensing_limber,1.e-10,chi_recomb,args=(i,galaxy_W_int,chi_int),epsrel = 1.e-6)[0] + phi_array.append(a) + phi_array = np.array(phi_array) + prefactor = 1.0#(l+2)*(l+1)*l*(l-1) / (l+0.5)**4 + return l,phi_array*prefactor + + + def theory_corr(n_z,min_sep=2.5,max_sep=250.,nbins=20,lmax=20000): + ll, pp = self.phi(lmax=lmax,n_z=n_z) + pp3_2 = np.zeros((lmax,4)) + pp3_2[:,1]= pp[:]*(ll*(ll+1.))/(2.*np.pi) + xvals = np.logspace(np.log10(min_sep),np.log10(max_sep),nbins) #in arcminutes + cxvals = np.cos(xvals*(60.)*(180./np.pi)) + vals = camb.correlations.cl2corr(pp3_2,cxvals) + return xvals, vals[:,1], vals[:,2] + + + def get_chi2(measured,theory): + chi2 = (measured-theory)**2/theory + return chi2/float(len(measured)) + + @staticmethod + def get_catalog_data(gc, quantities, filters=None): + data = {} + if not gc.has_quantities(quantities): + return TestResult(skipped=True, summary='Missing requested quantities') + + data = gc.get_quantities(quantities, filters=filters) + #make sure data entries are all finite + data = GCRQuery(*((np.isfinite, col) for col in data)).filter(data) + + return data + + + # define theory from within this class + + def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): + + # check if needed quantities exist + if not catalog_instance.has_quantities([self.z, self.ra, self.dec, self.e1, self.e2,self.kappa]): + #print("failed") + return TestResult(skipped=True, summary='do not have needed quantities') + catalog_data = self.get_catalog_data(catalog_instance, [self.z, self.ra, self.dec, self.e1, self.e2, self.kappa], filters=self.filters) + # get required catalogue data + + e1 = catalog_data[self.e1] + max_e1 = np.max(e1) + min_e1 = np.min(e1) + e2 = catalog_data[self.e2] + max_e2 = np.max(e2) + min_e2 = np.min(e2) + if ((min_e1<(-1.)) or (max_e1>1.0)): + return TestResult(skipped=True, summary='e1 values out of range [-1,+1]') + if ((min_e2<(-1.)) or (max_e2>1.0)): + return TestResult(skipped=True, summary='e2 values out of range [-1,+1]') + + + n_z = catalog_data[self.z] + xvals,theory_plus, theory_minus = self.theory_corr(n_z,min_sep=2.5,max_sep=250.,nbins=20,lmax=20000) + + cat_s = treecorr.Catalog(ra=catalog_data[self.ra], dec=catalog_data[self.dec], + g1=catalog_data[self.e1]-np.mean(catalog_data[self.e1]), g2=catalog_data[self.e2]-np.mean(catalog_data[self.e2]), ra_units='deg', dec_units='deg') + + gg = treecorr.GGCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin', bin_slop=0.1, verbose=True) + gg.process(cat_s) + + chi2_dof_1= self.get_chi2(gg.xip,theory_plus) # correct this + + r = numpy.exp(gg.meanlogr) + xip = gg.xip + xim = gg.xim + + plt.figure() + plt.plot(xip) + plt.plot(theory_plus) + plt.show() + sig = numpy.sqrt(gg.varxi) + + # directly compare xip and xim to theory_plus and theory_minus. Take into account cosmic variance. + + + + + + + # further correlation functions + #dd = treecorr.NNCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') + #dd.process(cat) + #treecorr.NGCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') # count-shear (i.e. (R)) + #treecorr.NKCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') # count-kappa (i.e. (R)) + #treecorr.KKCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') # count-kappa (i.e. (R)) + # etc... + + +#print(np.exp(gg.logr), gg.xip) +#np.savez('output.npz', theta=np.exp(gg.logr), ggp=gg.xip, ggm=gg.xim, weight=gg.weight, npairs=gg.npairs) + + data = np.random.rand(10) #do your calculation with catalog_instance + + fig, ax = plt.subplots() + + for ax_this in (ax, self.summary_ax): + ax_this.plot(data, label=catalog_name) + + #self.post_process_plot(ax) + #fig.savefig(os.path.join(output_dir, 'plot.png')) + #plt.close(fig) + + score = data[0] #calculate your summary statistics + return TestResult(score, passed=True) + + + + def conclude_test(self, output_dir): + self.post_process_plot(self.summary_ax) + self.summary_fig.savefig(os.path.join(output_dir, 'summary.png')) + plt.close(self.summary_fig) + \ No newline at end of file From f7260ddec695f8cdec0d3ebce2ca2f6c33d1e757 Mon Sep 17 00:00:00 2001 From: Patricia Larsen Date: Fri, 5 Jan 2018 08:09:22 -0800 Subject: [PATCH 04/64] fixed Jupyter to python copying issues --- Shear_test.ipynb | 2 +- descqa/shear_test.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Shear_test.ipynb b/Shear_test.ipynb index 8e359204..13082721 100644 --- a/Shear_test.ipynb +++ b/Shear_test.ipynb @@ -39,7 +39,7 @@ "\n", "\n", "\n", - "__all__ = ['ShearValidation']\n", + "#__all__ = ['ShearValidation']\n", "\n", "\n", "\n", diff --git a/descqa/shear_test.py b/descqa/shear_test.py index 4d6083d8..ab072d39 100644 --- a/descqa/shear_test.py +++ b/descqa/shear_test.py @@ -35,8 +35,7 @@ class ShearTest(BaseValidationTest): Validation test for shear and convergence quantities """ - def __init__(self, z='redshift_true', ra = 'ra', dec = 'dec', e1 = 'shear_1', e2= 'shear_2', kappa ='kappa', nbins=20, min_sep=2.5, max_sep=250, - sep_units='arcmin', bin_slop=0.1, zlo = 0.5, zhi= 0.6, **kwargs): + def __init__(self, z='redshift_true', ra = 'ra', dec = 'dec', e1 = 'shear_1', e2= 'shear_2', kappa ='kappa', nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin', bin_slop=0.1, zlo = 0.5, zhi= 0.6, **kwargs): #catalog quantities self.z = z From 6599e36cb3b82249d4174c3267a24068294a0fca Mon Sep 17 00:00:00 2001 From: Patricia Larsen Date: Fri, 5 Jan 2018 08:30:27 -0800 Subject: [PATCH 05/64] further minor fixes --- descqa/shear_test.py | 56 +++++++++++++++++++------------------------- 1 file changed, 24 insertions(+), 32 deletions(-) diff --git a/descqa/shear_test.py b/descqa/shear_test.py index ab072d39..a9bdd68d 100644 --- a/descqa/shear_test.py +++ b/descqa/shear_test.py @@ -1,5 +1,5 @@ -from __future__ import unicode_literals, absolute_import, division +#from __future__ import unicode_literals, absolute_import, division import os import numpy as np from .base import BaseValidationTest, TestResult @@ -7,8 +7,8 @@ import sys -sys.path.insert(0,'/global/common/software/lsst/common/miniconda/py3-4.2.12/lib/python3.6/site-packages') -sys.path.insert(0,'/global/common/software/lsst/common/miniconda/py3-4.2.12/bin/python') +sys.path.insert(0, '/global/common/software/lsst/common/miniconda/py3-4.2.12/lib/python3.6/site-packages') +sys.path.insert(0, '/global/common/software/lsst/common/miniconda/py3-4.2.12/bin/python') from GCR import GCRQuery @@ -35,11 +35,9 @@ class ShearTest(BaseValidationTest): Validation test for shear and convergence quantities """ - def __init__(self, z='redshift_true', ra = 'ra', dec = 'dec', e1 = 'shear_1', e2= 'shear_2', kappa ='kappa', nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin', bin_slop=0.1, zlo = 0.5, zhi= 0.6, **kwargs): - + def __init__(self, z='redshift_true', ra='ra', dec='dec', e1='shear_1', e2='shear_2', kappa='kappa', nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin', bin_slop=0.1, zlo=0.5, zhi=0.6, **kwargs): #catalog quantities self.z = z - #sep-bounds and binning self.min_sep = min_sep self.max_sep = max_sep @@ -47,20 +45,14 @@ def __init__(self, z='redshift_true', ra = 'ra', dec = 'dec', e1 = 'shear_1', e2 self.sep_bins = np.linspace(min_sep, max_sep, nbins+1) self.sep_units = sep_units self.bin_slop = bin_slop - self.ra = ra self.dec = dec self.e1 = e1 self.e2 = e2 self.kappa = kappa - - # cut in redshift self.filters = [(lambda z: (z > zlo) & (z < zhi), self.z)] - - self.summary_fig, self.summary_ax = plt.subplots() - #validation data # want this to change to theory... #self.validation_data = {} @@ -77,12 +69,12 @@ def __init__(self, z='redshift_true', ra = 'ra', dec = 'dec', e1 = 'shear_1', e2 def compute_nz(n_z): '''create interpolated n(z) distribution''' - z_bins = np.linspace(0.0,2.0,101) - n = np.histogram(n_z,bins=z_bins)[0] + z_bins = np.linspace(0.0, 2.0, 101) + n = np.histogram(n_z, bins=z_bins)[0] z = (z_bins[1:]-z_bins[:-1])/2. + z_bins[:-1] - n2 = interp1d(z,n,bounds_error=False,fill_value=0.0,kind='cubic') - n2_sum = quad(n2,0,2.0)[0] - n2 = interp1d(z,n/n2_sum,bounds_error=False,fill_value=0.0,kind='cubic') + n2 = interp1d(z, n, bounds_error=False, fill_value=0.0, kind='cubic') + n2_sum = quad(n2, 0, 2.0)[0] + n2 = interp1d(z, n/n2_sum, bounds_error=False, fill_value=0.0, kind='cubic') return n2 def integrand_w(x,n,chi,chi_int): @@ -99,46 +91,46 @@ def galaxy_W(z,n,chi_int): prefactor = cst * chi* (1.+z) *u.Mpc val_array = [] for i in range(len(z)): - val_array.append(quad(integrand_w,chi[i],chi_recomb, args = (n,chi[i],chi_int))[0]) + val_array.append(quad(integrand_w, chi[i], chi_recomb, args = (n, chi[i], chi_int))[0]) W = np.array(val_array)*prefactor *(u.Mpc) # now unitless return W - def integrand_lensing_limber(chi,l,galaxy_W_int,chi_int): + def integrand_lensing_limber(chi, l, galaxy_W_int, chi_int): '''return overall integrand for one value of l''' chi_unit = chi*u.Mpc z = chi_int(chi) k = (l+0.5)/chi - integrand = p(z,k,grid=False) *galaxy_W_int(z)**2/chi**2 + integrand = p(z, k, grid=False) *galaxy_W_int(z)**2/chi**2 return integrand def phi(lmax,n_z): z_array = np.logspace(-3,np.log10(10.),200) chi_array = cosmo.comoving_distance(z_array).value - chi_int = interp1d(chi_array,z_array,bounds_error=False,fill_value=0.0) + chi_int = interp1d(chi_array, z_array, bounds_error=False, fill_value=0.0) n = self.compute_nz(n_z) - galaxy_W_int = interp1d(z_array,self.galaxy_W(z_array,n,chi_int),bounds_error=False,fill_value=0.0) + galaxy_W_int = interp1d(z_array, self.galaxy_W(z_array, n, chi_int), bounds_error=False, fill_value=0.0) phi_array = [] - l= range(0,lmax,1) + l= range(0, lmax, 1) l = np.array(l) for i in l: - a = quad(self.integrand_lensing_limber,1.e-10,chi_recomb,args=(i,galaxy_W_int,chi_int),epsrel = 1.e-6)[0] + a = quad(self.integrand_lensing_limber, 1.e-10, chi_recomb, args=(i, galaxy_W_int, chi_int), epsrel = 1.e-6)[0] phi_array.append(a) phi_array = np.array(phi_array) prefactor = 1.0#(l+2)*(l+1)*l*(l-1) / (l+0.5)**4 - return l,phi_array*prefactor + return l, phi_array*prefactor - def theory_corr(n_z,min_sep=2.5,max_sep=250.,nbins=20,lmax=20000): - ll, pp = self.phi(lmax=lmax,n_z=n_z) + def theory_corr(n_z, min_sep=2.5, max_sep=250., nbins=20, lmax=20000): + ll, pp = self.phi(lmax=lmax, n_z=n_z) pp3_2 = np.zeros((lmax,4)) pp3_2[:,1]= pp[:]*(ll*(ll+1.))/(2.*np.pi) - xvals = np.logspace(np.log10(min_sep),np.log10(max_sep),nbins) #in arcminutes + xvals = np.logspace(np.log10(min_sep), np.log10(max_sep), nbins) #in arcminutes cxvals = np.cos(xvals*(60.)*(180./np.pi)) - vals = camb.correlations.cl2corr(pp3_2,cxvals) + vals = camb.correlations.cl2corr(pp3_2, cxvals) return xvals, vals[:,1], vals[:,2] - def get_chi2(measured,theory): + def get_chi2(measured, theory): chi2 = (measured-theory)**2/theory return chi2/float(len(measured)) @@ -179,7 +171,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): n_z = catalog_data[self.z] - xvals,theory_plus, theory_minus = self.theory_corr(n_z,min_sep=2.5,max_sep=250.,nbins=20,lmax=20000) + xvals, theory_plus, theory_minus = self.theory_corr(n_z, min_sep=2.5, max_sep=250., nbins=20, lmax=20000) cat_s = treecorr.Catalog(ra=catalog_data[self.ra], dec=catalog_data[self.dec], g1=catalog_data[self.e1]-np.mean(catalog_data[self.e1]), g2=catalog_data[self.e2]-np.mean(catalog_data[self.e2]), ra_units='deg', dec_units='deg') @@ -187,7 +179,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): gg = treecorr.GGCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin', bin_slop=0.1, verbose=True) gg.process(cat_s) - chi2_dof_1= self.get_chi2(gg.xip,theory_plus) # correct this + chi2_dof_1= self.get_chi2(gg.xip, theory_plus) # correct this r = numpy.exp(gg.meanlogr) xip = gg.xip From ee8974ad3ffca33c7b266388c5b40f5b356c5bfe Mon Sep 17 00:00:00 2001 From: Patricia Larsen Date: Fri, 5 Jan 2018 08:40:41 -0800 Subject: [PATCH 06/64] adding path - still tracking down build issue --- descqa/shear_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/descqa/shear_test.py b/descqa/shear_test.py index a9bdd68d..8e9fb1a4 100644 --- a/descqa/shear_test.py +++ b/descqa/shear_test.py @@ -9,6 +9,8 @@ import sys sys.path.insert(0, '/global/common/software/lsst/common/miniconda/py3-4.2.12/lib/python3.6/site-packages') sys.path.insert(0, '/global/common/software/lsst/common/miniconda/py3-4.2.12/bin/python') +sys.path.insert(0, '..') + from GCR import GCRQuery From b239e0f7774ce076a38c5fe308f1addab64c8af4 Mon Sep 17 00:00:00 2001 From: Patricia Larsen Date: Fri, 5 Jan 2018 09:32:26 -0800 Subject: [PATCH 07/64] minor fixes --- Shear_test.ipynb | 504 ------------------------------------------- descqa/shear_test.py | 32 ++- 2 files changed, 15 insertions(+), 521 deletions(-) delete mode 100644 Shear_test.ipynb diff --git a/Shear_test.ipynb b/Shear_test.ipynb deleted file mode 100644 index 13082721..00000000 --- a/Shear_test.ipynb +++ /dev/null @@ -1,504 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "#from __future__ import unicode_literals, absolute_import\n", - "import os\n", - "import sys\n", - "\n", - "sys.path.insert(0,'/global/common/software/lsst/common/miniconda/py3-4.2.12/lib/python3.6/site-packages')\n", - "sys.path.insert(0,'/global/common/software/lsst/common/miniconda/py3-4.2.12/bin/python')\n", - "\n", - "\n", - "from GCR import GCRQuery\n", - "import GCRCatalogs\n", - "import descqa \n", - "\n", - "from builtins import str\n", - "import yaml\n", - "import numpy as np\n", - "import healpy as hp\n", - "from descqa.plotting import plt\n", - "\n", - "from descqa import BaseValidationTest, TestResult\n", - "\n", - "\n", - "\n", - "import camb\n", - "import treecorr\n", - "#import treecorr\n", - "# neither of these will currently work\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "#__all__ = ['ShearValidation']\n", - "\n", - "\n", - "\n", - "%matplotlib inline" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "#!ls /global/common/software/lsst/common/miniconda/py3-4.2.12/lib/python3.6/site-packages\n", - "#!ls -l /global/common/software/lsst/common/miniconda/py3-4.2.12/lib/python3.6/site-packages\n", - "#!groups" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "#import treecorr\n", - "\n", - "#cat = treecorr.Catalog('cat.fits', ra_col='RA', dec_col='DEC', ra_units='degrees', dec_units='degrees', g1_col='GAMMA1', g2_col='GAMMA2')\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "(markdown to comment out - I think this has all been added to the test but leaving here just in case)\n", - "\n", - "#theory computation\n", - " this is now fine!\n", - "\n", - "\n", - "#from astropy.cosmology import FlatLambdaCDM\n", - "#cosmo = FlatLambdaCDM(H0=70, Om0=0.3, Tcmb0=2.725) \n", - "from astropy.cosmology import WMAP9 as cosmo # want protoDC2 cosmology here\n", - "import camb\n", - "from scipy.integrate import quad\n", - "from scipy.interpolate import interp1d, InterpolatedUnivariateSpline\n", - "from astropy.cosmology import z_at_value\n", - "\n", - "import astropy.units as u\n", - "\n", - "#update cosmo to have the right parameters\n", - "pars = camb.CAMBparams()\n", - "h = 0.675\n", - "pars.set_cosmology(H0=h*100, ombh2=0.022, omch2=0.122)\n", - "pars.InitPower.set_params(ns=0.965)\n", - "\n", - "camb.set_halofit_version(version='takahashi')\n", - "p = camb.get_matter_power_interpolator(pars, nonlinear=True, k_hunit=False, hubble_units=False ,kmax=100., zmax=1100., k_per_logint=False).P\n", - " no h corrections necessary for this. Everything in Mpc units\n", - "\n", - "import astropy.constants as const\n", - "from camb import model, initialpower\n", - "\n", - "chi_recomb = 14004.036207574154\n", - "\n", - " p(z, k/h)\n", - "\n", - "def test_nz(n_z):\n", - " '''create mock n(z) distribution to test theory code'''\n", - " z_bins = np.linspace(0.0,2.0,101)\n", - " n = np.histogram(n_z,bins=z_bins)[0]\n", - " z = (z_bins[1:]-z_bins[:-1])/2. + z_bins[:-1]\n", - " #n2 = InterpolatedUnivariateSpline(z, n, w=None, bbox=[None, None], k=3, ext=0, check_finite=False)\n", - " n2 = interp1d(z,n,bounds_error=False,fill_value=0.0,kind='cubic')\n", - " n2_sum = quad(n2,0,2.0)[0]\n", - " #n2 = InterpolatedUnivariateSpline(z, n/n2_sum, w=None, bbox=[None, None], k=3, ext=0, check_finite=False)\n", - " n2 = interp1d(z,n/n2_sum,bounds_error=False,fill_value=0.0,kind='cubic')\n", - " return n2\n", - "\n", - "def integrand_w(x,n,chi,chi_int):\n", - " ''' This is the inner bit of GWL lensing kernel - z is related to x, not chi'''\n", - " z = chi_int(x)\n", - " H_z = cosmo.H(z).value*3.240779289469756e-20 #1/s units #.to(1./u.s) conversion\n", - " dchidz =9.715611890256315e-15 /H_z #const.c.to(u.Mpc/u.s).value / (H_z) # Mpc units\n", - " return n(z)/dchidz* (x - chi)/x\n", - "\n", - " integrand_w * const.c.to(u.Mpc/u.s).value, divided by Hubble (km/Mpcs).to(1./u.s).value\n", - "def galaxy_W(z,n,chi_int):\n", - " ''' galaxy window function'''\n", - " chi = cosmo.comoving_distance(z).value # can be array\n", - " cst = 3./2. * cosmo.H(0).to(1./u.s)**2/const.c.to(u.Mpc / u.s)**2*cosmo.Om(0)# not sure about the z-dependence here\n", - " prefactor = cst * chi* (1.+z) *u.Mpc\n", - " val_array = []\n", - " for i in range(len(z)):\n", - " val_array.append(quad(integrand_w,chi[i],chi_recomb, args = (n,chi[i],chi_int))[0])\n", - " W = np.array(val_array)*prefactor *(u.Mpc) # now unitless\n", - " return W\n", - "\n", - " not currently used\n", - "def cmb_W(z):\n", - " chi = cosmo.comoving_distance(z).value # can be array\n", - " cst = 3./2. * cosmo.H(0).to(1./u.s)**2/const.c.to(u.Mpc / u.s)**2*cosmo.Om(0)# not sure about the z-dependence here\n", - " prefactor = cst * chi* (1.+z) *u.Mpc**2\n", - " return (chi_recomb - chi)/chi_recomb * prefactor\n", - "\n", - "def integrand_lensing_limber(chi,l,galaxy_W_int,chi_int):\n", - " '''return overall integrand for one value of l'''\n", - " chi_unit = chi*u.Mpc\n", - " z = chi_int(chi)#z_at_value(cosmo.comoving_distance, chi_unit)\n", - " k = (l+0.5)/chi\n", - " integrand = p(z,k,grid=False) *galaxy_W_int(z)**2/chi**2 # check this divided by chi2\n", - " return integrand\n", - "\n", - "\n", - "\n", - "def phi(lmax,n_z):\n", - " z_array = np.logspace(-3,np.log10(10.),200)\n", - " chi_array = cosmo.comoving_distance(z_array).value\n", - " chi_int = interp1d(chi_array,z_array,bounds_error=False,fill_value=0.0)\n", - " n = compute_nz(n_z)\n", - " galaxy_W_int = interp1d(z_array,galaxy_W(z_array,n,chi_int),bounds_error=False,fill_value=0.0)\n", - " #cmb_W_int = interp1d(z_array,cmb_W(z_array),bounds_error=False,fill_value=0.0)\n", - " phi = []\n", - " l= range(0,lmax,1)#range(1,10,1) +range(10,100,5) #+range(100,1000,10)+ range(1000,10000,100)\n", - " l = np.array(l)\n", - " for i in l:\n", - " a = quad(integrand_lensing_limber,1.e-10,14004.036207574154,args=(i,galaxy_W_int,chi_int),epsrel = 1.e-6)[0]\n", - " phi.append(a)\n", - " phi = np.array(phi)\n", - " prefactor = 1.0#(l+2)*(l+1)*l*(l-1) / (l+0.5)**4\n", - " return l,phi*prefactor\n", - "\n", - "\n", - "def theory_corr(n_z,min_sep=2.5,max_sep=250.,nbins=20,lmax=20000):\n", - " '''The only thing you actually have to run'''\n", - " ll, pp = phi(lmax=lmax,n_z=n_z)\n", - " pp3_2 = np.zeros((lmax,4))\n", - " pp3_2[:,1]= pp[:]*(ll*(ll+1.))/(2.*np.pi)\n", - " xvals = np.logspace(np.log10(min_sep),np.log10(max_sep),nbins) #in arcminutes\n", - " cxvals = np.cos(xvals*(60.)*(180./np.pi))\n", - " vals = camb.correlations.cl2corr(pp3_2,cxvals)\n", - " return xvals, vals[:,1], vals[:,2]\n", - "\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "#import treecorr#\n", - "#\n", - "#infile = pf.open('protoDC2_WLPipe.fits')[1].data\n", - "#print(len(infile))\n", - "#mask = (infile['z_mean']>0.5)*(infile['z_mean']<0.6)#\n", - "#\n", - "#cat_s = treecorr.Catalog(ra=infile['ra'][mask], dec=infile['dec'][mask], g1=infile['e1'][mask]-np.mean(infile['e1'][mask]), g2=infile['e2'][mask]-np.mean(infile['e2'][mask]), ra_units='deg', dec_units='deg')\n", - "\n", - "#gg = treecorr.GGCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin', bin_slop=0.1, verbose=True)\n", - "#gg.process_cross(cat_s, cat_s)\n", - "\n", - "#print(np.exp(gg.logr), gg.xip)\n", - "#np.savez('output.npz', theta=np.exp(gg.logr), ggp=gg.xip, ggm=gg.xim, weight=gg.weight, npairs=gg.npairs)" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "__all__ = ['ShearTest']\n", - "\n", - "class ShearTest(BaseValidationTest):\n", - " \"\"\"\n", - " Validation test for shear and convergence quantities\n", - " \"\"\"\n", - "\n", - " def __init__(self, z='redshift_true', ra = 'ra', dec = 'dec', e1 = 'shear_1', e2= 'shear_2', kappa ='kappa', nbins=20, min_sep=2.5, max_sep=250, \n", - " sep_units='arcmin', bin_slop=0.1, zlo = 0.5, zhi= 0.6, **kwargs):\n", - "\n", - " #catalog quantities\n", - " self.z = z\n", - "\n", - " #sep-bounds and binning\n", - " self.min_sep = min_sep\n", - " self.max_sep = max_sep\n", - " self.nbins = nbins\n", - " self.sep_bins = np.linspace(min_sep, max_sep, nbins+1)\n", - " self.sep_units = sep_units\n", - " self.bin_slop = bin_slop\n", - " \n", - " self.ra = ra\n", - " self.dec = dec\n", - " self.e1 = e1\n", - " self.e2 = e2\n", - " self.kappa = kappa\n", - " \n", - " \n", - " # cut in redshift\n", - " self.filters = [(lambda z: (z > zlo) & (z < zhi), self.z)]\n", - " \n", - "\n", - " self.summary_fig, self.summary_ax = plt.subplots()\n", - "\n", - " #validation data\n", - " # want this to change to theory... \n", - " #self.validation_data = {}\n", - " #self.observation = observation\n", - " #setup subplot configuration and get magnitude cuts for each plot\n", - " #self.mag_lo, self.mag_hi = self.init_plots(mag_lo, mag_hi)\n", - " #setup summary plot\n", - " #self.summary_fig, self.summary_ax = plt.subplots(self.nrows, self.ncolumns, figsize=(figx_p, figy_p), sharex='col')\n", - " #could plot summary validation data here if available but would need to evaluate labels, bin values etc.\n", - " #otherwise setup a check so that validation data is plotted only once on summary plot\n", - " #self.first_pass = True\n", - " #self._other_kwargs = kwargs\n", - " \n", - " \n", - " def compute_nz(n_z):\n", - " '''create interpolated n(z) distribution'''\n", - " z_bins = np.linspace(0.0,2.0,101)\n", - " n = np.histogram(n_z,bins=z_bins)[0]\n", - " z = (z_bins[1:]-z_bins[:-1])/2. + z_bins[:-1]\n", - " n2 = interp1d(z,n,bounds_error=False,fill_value=0.0,kind='cubic')\n", - " n2_sum = quad(n2,0,2.0)[0]\n", - " n2 = interp1d(z,n/n2_sum,bounds_error=False,fill_value=0.0,kind='cubic')\n", - " return n2\n", - "\n", - " def integrand_w(x,n,chi,chi_int):\n", - " ''' This is the inner bit of GWL lensing kernel - z is related to x, not chi'''\n", - " z = chi_int(x)\n", - " H_z = cosmo.H(z).value*3.240779289469756e-20 #1/s units #.to(1./u.s) conversion\n", - " dchidz = 9.715611890256315e-15 /H_z #const.c.to(u.Mpc/u.s).value / (H_z) # Mpc units\n", - " return n(z)/dchidz* (x - chi)/x\n", - "\n", - " def galaxy_W(z,n,chi_int):\n", - " ''' galaxy window function'''\n", - " chi = cosmo.comoving_distance(z).value # can be array\n", - " cst = 3./2. * cosmo.H(0).to(1./u.s)**2/const.c.to(u.Mpc / u.s)**2*cosmo.Om(0)# not sure about the z-dependence here\n", - " prefactor = cst * chi* (1.+z) *u.Mpc\n", - " val_array = []\n", - " for i in range(len(z)):\n", - " val_array.append(quad(integrand_w,chi[i],chi_recomb, args = (n,chi[i],chi_int))[0])\n", - " W = np.array(val_array)*prefactor *(u.Mpc) # now unitless\n", - " return W\n", - "\n", - " def integrand_lensing_limber(chi,l,galaxy_W_int,chi_int):\n", - " '''return overall integrand for one value of l'''\n", - " chi_unit = chi*u.Mpc\n", - " z = chi_int(chi)\n", - " k = (l+0.5)/chi\n", - " integrand = p(z,k,grid=False) *galaxy_W_int(z)**2/chi**2 \n", - " return integrand\n", - "\n", - " def phi(lmax,n_z):\n", - " z_array = np.logspace(-3,np.log10(10.),200)\n", - " chi_array = cosmo.comoving_distance(z_array).value\n", - " chi_int = interp1d(chi_array,z_array,bounds_error=False,fill_value=0.0)\n", - " n = self.compute_nz(n_z)\n", - " galaxy_W_int = interp1d(z_array,self.galaxy_W(z_array,n,chi_int),bounds_error=False,fill_value=0.0)\n", - " phi_array = []\n", - " l= range(0,lmax,1)\n", - " l = np.array(l)\n", - " for i in l:\n", - " a = quad(self.integrand_lensing_limber,1.e-10,chi_recomb,args=(i,galaxy_W_int,chi_int),epsrel = 1.e-6)[0]\n", - " phi_array.append(a)\n", - " phi_array = np.array(phi_array)\n", - " prefactor = 1.0#(l+2)*(l+1)*l*(l-1) / (l+0.5)**4\n", - " return l,phi_array*prefactor\n", - " \n", - "\n", - " def theory_corr(n_z,min_sep=2.5,max_sep=250.,nbins=20,lmax=20000):\n", - " ll, pp = self.phi(lmax=lmax,n_z=n_z)\n", - " pp3_2 = np.zeros((lmax,4))\n", - " pp3_2[:,1]= pp[:]*(ll*(ll+1.))/(2.*np.pi)\n", - " xvals = np.logspace(np.log10(min_sep),np.log10(max_sep),nbins) #in arcminutes\n", - " cxvals = np.cos(xvals*(60.)*(180./np.pi))\n", - " vals = camb.correlations.cl2corr(pp3_2,cxvals)\n", - " return xvals, vals[:,1], vals[:,2]\n", - "\n", - "\n", - " def get_chi2(measured,theory):\n", - " chi2 = (measured-theory)**2/theory\n", - " return chi2/float(len(measured))\n", - " \n", - " @staticmethod\n", - " def get_catalog_data(gc, quantities, filters=None):\n", - " data = {}\n", - " if not gc.has_quantities(quantities):\n", - " return TestResult(skipped=True, summary='Missing requested quantities')\n", - "\n", - " data = gc.get_quantities(quantities, filters=filters)\n", - " #make sure data entries are all finite\n", - " data = GCRQuery(*((np.isfinite, col) for col in data)).filter(data)\n", - "\n", - " return data\n", - "\n", - "\n", - " # define theory from within this class\n", - " \n", - " def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):\n", - "\n", - " # check if needed quantities exist\n", - " if not catalog_instance.has_quantities([self.z, self.ra, self.dec, self.e1, self.e2,self.kappa]):\n", - " #print(\"failed\")\n", - " return TestResult(skipped=True, summary='do not have needed quantities')\n", - " catalog_data = self.get_catalog_data(catalog_instance, [self.z, self.ra, self.dec, self.e1, self.e2, self.kappa], filters=self.filters)\n", - " # get required catalogue data \n", - " \n", - " e1 = catalog_data[self.e1]\n", - " max_e1 = np.max(e1)\n", - " min_e1 = np.min(e1)\n", - " e2 = catalog_data[self.e2]\n", - " max_e2 = np.max(e2)\n", - " min_e2 = np.min(e2)\n", - " if ((min_e1<(-1.)) or (max_e1>1.0)):\n", - " return TestResult(skipped=True, summary='e1 values out of range [-1,+1]')\n", - " if ((min_e2<(-1.)) or (max_e2>1.0)):\n", - " return TestResult(skipped=True, summary='e2 values out of range [-1,+1]')\n", - " \n", - " \n", - " n_z = catalog_data[self.z]\n", - " xvals,theory_plus, theory_minus = self.theory_corr(n_z,min_sep=2.5,max_sep=250.,nbins=20,lmax=20000)\n", - "\n", - " cat_s = treecorr.Catalog(ra=catalog_data[self.ra], dec=catalog_data[self.dec], \n", - " g1=catalog_data[self.e1]-np.mean(catalog_data[self.e1]), g2=catalog_data[self.e2]-np.mean(catalog_data[self.e2]), ra_units='deg', dec_units='deg')\n", - " \n", - " gg = treecorr.GGCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin', bin_slop=0.1, verbose=True)\n", - " gg.process(cat_s)\n", - " \n", - " chi2_dof_1= self.get_chi2(gg.xip,theory_plus) # correct this\n", - " \n", - " r = numpy.exp(gg.meanlogr)\n", - " xip = gg.xip\n", - " xim = gg.xim\n", - " \n", - " plt.figure()\n", - " plt.plot(xip)\n", - " plt.plot(theory_plus)\n", - " plt.show()\n", - " sig = numpy.sqrt(gg.varxi)\n", - " \n", - " # directly compare xip and xim to theory_plus and theory_minus. Take into account cosmic variance. \n", - " \n", - " \n", - "\n", - " \n", - " \n", - " \n", - " # further correlation functions \n", - " #dd = treecorr.NNCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin')\n", - " #dd.process(cat)\n", - " #treecorr.NGCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') # count-shear (i.e. (R))\n", - " #treecorr.NKCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') # count-kappa (i.e. (R))\n", - " #treecorr.KKCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') # count-kappa (i.e. (R))\n", - " # etc... \n", - "\n", - "\n", - "#print(np.exp(gg.logr), gg.xip)\n", - "#np.savez('output.npz', theta=np.exp(gg.logr), ggp=gg.xip, ggm=gg.xim, weight=gg.weight, npairs=gg.npairs)\n", - " \n", - " data = np.random.rand(10) #do your calculation with catalog_instance\n", - "\n", - " fig, ax = plt.subplots()\n", - "\n", - " for ax_this in (ax, self.summary_ax):\n", - " ax_this.plot(data, label=catalog_name)\n", - "\n", - " #self.post_process_plot(ax)\n", - " #fig.savefig(os.path.join(output_dir, 'plot.png'))\n", - " #plt.close(fig)\n", - "\n", - " score = data[0] #calculate your summary statistics\n", - " return TestResult(score, passed=True)\n", - "\n", - "\n", - "\n", - " def conclude_test(self, output_dir):\n", - " self.post_process_plot(self.summary_ax)\n", - " self.summary_fig.savefig(os.path.join(output_dir, 'summary.png'))\n", - " plt.close(self.summary_fig)\n", - " \n", - " \n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "#print(quantities)" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD+CAYAAADWKtWTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADi1JREFUeJzt3WFoXed9x/Hvfxl169RSayaTMSw3qdYlS9O0TNvw0tIX\nzmBg/MYd5EXWprRwS1pa0gzcrE0DTprU8zKXmrE4It3CCC2MemM1DgWjF6UJpsQeNN2Lkqwwt3Rp\nleCucsimbut/L3QynSi2dSQd3Sv8/35AcJ/nPvfwv4/ln8997jmPIzORJNXwK6MuQJI0PIa+JBVi\n6EtSIYa+JBVi6EtSIYa+JBVi6EtSIYa+JBWyYuhHxDUR8VhEPHOJ598YEX8VEX8WEX8TEe/ov0xJ\nUh+6nOm/F/gnIC7x/F3ADzPzi8CXgK/0VJskqWcrhn5mfh24cJkhe4HTzdjvATdHxFg/5UmS+vSr\nPRxjB6/9R2G+6ZtfPjAiBsCg3Xf11Vf/zvXXX99DGZJUw9mzZ1/KzIm1vLaP0J8DtrXaY03f62Tm\nDDDT7puens4zZ870UIYk1RAR59b62jVdvRMR21tLOCeB3U3/TcB3M/N1Z/mSpNHrcvXO+4EPAr8e\nEfdGxJuAe4CPN0O+DOyKiHuBPwU+ulHFSpLWZ8Xlncz8FvCtZd0HWs//J/CJnuuSJG0Ab86SpEIM\nfUkqxNCXpEIMfUkqxNCXpEIMfUkqpI87clctIvYB+wCmpqZGUYIklTSSM/3MPJGZg8wcjI+Pj6IE\nSSrJ5R1JKsTQl6RCDH1JKsTQl6RCDH1JKsTQl6RCDH1JKsTQl6RCDH1JKsRtGCSpELdhkKRCXN6R\npEIMfUkqxNCXpEIMfUkqxNCXpEIMfUkqxNCXpEIMfUkqxDtyJakQ78iVpEJc3pGkQgx9SSrE0Jek\nQgx9SSrE0JekQgx9SSqk03X6EXErsB+YAzIzDy57/lrgYeAZ4N3AVzPzGz3XKklapxVDPyK2AseA\nGzNzISKOR8SezJxtDTsAPJWZX4qI9wB/Dxj6krTJdFne2Q2cy8yFpv00sHfZmJ8CE83jCeBsP+VJ\nkvrUZXlnB3Ch1Z5v+tqOAP8YEUeA3wMeuNiBImIADNp9k5OTnYuVJK1Pl9CfA7a12mNNX9vjwGOZ\n+bWImACej4jrMvN8e1BmzgAz7b7p6elcddWSpDXpsrxzGtgVEVua9i3AyYjYHhFjTd9O4IXm8c+A\nX3Y8tiRpiFY808/MVyLiTuBoRLwIPJuZsxFxGDgPHAI+DdwVEX8AXAt8NjNf2sjCJUmr1+mSzcw8\nBZxa1neg9fgp4Kl+S5Mk9c0lGEkqxNCXpEIMfUkqxNCXpEIMfUkqxNCXpEI6XbLZt4jYB+wDmJqa\nGkUJklTSSM70M/NEZg4yczA+Pj6KEiSpJJd3JKkQQ1+SCjH0JakQQ1+SCjH0JakQQ1+SCjH0JakQ\nQ1+SCjH0JakQt2GQpELchkGSCnF5R5IKMfQlqRBDX5IKMfQlqRBDX5IKMfQlqRBDX5IKMfQlqRBD\nX5IKcRsGSSrEbRgkqRCXdySpEENfkgox9CWpEENfkgox9CWpEENfkgrpdJ1+RNwK7AfmgMzMg8ue\nD+CTTfNtwFsy8yM91ilJ6sGKoR8RW4FjwI2ZuRARxyNiT2bOtob9CfAfmfl3zWvetTHlSpLWo8vy\nzm7gXGYuNO2ngb3LxtwObI+IT0XEQ8DLPdYoSepJl+WdHcCFVnu+6WvbBYxl5v0R8Q7gmxFxQ2b+\nb3tQRAyAQbtvcnJy9VVLktaky5n+HLCt1R5r+trmge8AZOZzzZidyw+UmTOZOd3+mZiYWFvlkqRV\n6xL6p4FdEbGlad8CnIyI7REx1vTNAtcBNH1XAT/pu1hJ0vqsuLyTma9ExJ3A0Yh4EXg2M2cj4jBw\nHjgE/DlwOCI+C7wduCMz/2sjC5ckrV6nSzYz8xRwalnfgdbjnwMf67c0SVLfvDlLkgox9CWpEENf\nkgox9CWpEENfkgox9CWpkE6XbPYtIvYB+wCmpqZGUYIklTSSM/3MPJGZg8wcjI+Pj6IESSrJ5R1J\nKsTQl6RCDH1JKsTQl6RCDH1JKsTQl6RCDH1JKsTQl6RCDH1JKsRtGCSpELdhkKRCXN6RpEIMfUkq\nxNCXpEIMfUkqxNCXpEIMfUkqxNCXpEIMfUkqxDtyJakQ78iVpEJc3pGkQgx9SSrE0JekQgx9SSrE\n0JekQgx9SSqk03X6EXErsB+YAzIzD15i3O3AE8C2zHy5tyolSb1YMfQjYitwDLgxMxci4nhE7MnM\n2WXjbgB+e4PqlCT1oMvyzm7gXGYuNO2ngb3tAc0/DAeAi34CkCRtDl2Wd3YAF1rt+aav7UHg/sz8\nRURc8kARMQAG7b7JyclulUqS1q1L6M8B21rtsaYPgIjYCbwVuK0V+HdHxJOZeaZ9oMycAWbafdPT\n07mGuiVJa9Al9E8DuyJiS7PEcwvw1xGxHfifzPwR8OFXB0fEF4EjfpErSZvPimv6mfkKcCdwNCK+\nADzbfIl7D/DxV8dFxERE3Ns0D0TEb2xEwZKktYvM0a6uTE9P55kzZ1YeKEkCICLOZub0Wl7rzVmS\nVIihL0mFGPqSVIihL0mFGPqSVIihL0mFdNpls28RsQ/YBzA1NTWKEiSppJGc6WfmicwcZOZgfHx8\nFCVIUkku70hSIYa+JBVi6EtSIYa+JBVi6EtSIYa+JBVi6EtSIYa+JBVi6EtSIW7DIEmFuA2DJBXi\n8o4kFWLoS1Ihhr4kFWLoS1Ihhr4kFWLoS1Ihhr4kFWLoS1Ihhr4kFeI2DJJUiNswSFIhLu9IUiGG\nviQVYuhLUiGGviQVYuhLUiGGviQV0uk6/Yi4FdgPzAGZmQeXPf8Z4BrgBWAauC8zv99zrZKkdVox\n9CNiK3AMuDEzFyLieETsyczZ1rA3A3dnZkbEbcBf0Nx8JUnaPLos7+wGzmXmQtN+GtjbHpCZn8/M\nbB3z5f5KlCT1pcvyzg7gQqs93/S9TkS8AbgD+MQlnh8Ag3bf5ORkp0IlSevXJfTngG2t9ljT9xpN\n4D8CfC4zf3CxA2XmDDDT7puens6LjZUk9a/L8s5pYFdEbGnatwAnI2J7RIzB/6/7PwocycyzEfGB\njSlXkrQeK57pZ+YrEXEncDQiXgSezczZiDgMnAcOAU8A7wSujQiAq4HjG1e2JGktOl2ymZmngFPL\n+g60Hu/vuS5J0gbw5ixJKsTQl6RCDH1JKsTQl6RCDH1JKsTQl6RCOl2y2beI2EezIdvU1NQoSpCk\nkkZypp+ZJzJzkJmD8fHxUZQgSSW5vCNJhRj6klSIoS9JhRj6klSIoS9JhRj6klSIoS9JhRj6klSI\noS9JhbgNgyQV4jYMklSIyzuSVIihL0mFGPqSVIihL0mFGPqSVIihL0mFGPqSVIihL0mFeEeuJBXi\nHbmSVIjLO5JUiKEvSYUY+pJUiKEvSYUY+pJUiKEvSYV0uk4/Im4F9gNzQGbmwWXPvxF4GPgx8JvA\nocx8rudaJUnrtGLoR8RW4BhwY2YuRMTxiNiTmbOtYXcBP8zMwxFxE/AV4H0bU7Ikaa26LO/sBs5l\n5kLTfhrYu2zMXuA0QGZ+D7g5IsZ6q1KS1Isuyzs7gAut9nzT12XMfHtQRAyAwbLXLkTEv3Sq9uLG\ngZ+v4/Wb6Ri/Brw04ho2wzHWOw991LBZjuFcLHEulvzWml+ZmZf9AfYAs6323cCRZWO+Dbyv1Z4H\nxlY6djP2TJdxl3n9zHpev8mO4Vz0MA+b5X04F87FZpyLLss7p4FdEbGlad8CnIyI7a0lnJMsLgPR\nrOl/NzPnX38oSdIorbi8k5mvRMSdwNGIeBF4NjNnI+IwcB44BHwZeDgi7gWmgI9uZNHLnLiCjrFe\nm+V9OBf9HmO9Nsv7cC76PcaaRPNRYWQi4kxmTo+0iE3CuVjkPCxxLpY4F0vWMxeb4easmVEXsIk4\nF4uchyXOxRLnYsma52LkZ/qSpOHZDGf6kqQhMfQlqZCh/B+57t2zpMNcfAa4BngBmAbuy8zvD73Q\nIVhpLlrjbgeeALZl5stDLHFoOvxeBPDJpvk24C2Z+ZGhFjkkHebiWhbz4hng3cBXM/MbQy90CCLi\nGuALwM2Z+bsXeX712bnemww63ESwFfhXYEvTPg7sWTbmHuBA8/gm4NsbXdcofjrOxQMsfddyG3Bi\n1HWPai6a/huAB4EE3jzqukf4e/FB4EOt9rtGXfcI5+IR4NPN4/cAz4+67g2cjz8G9nGJm7HWkp3D\nWN5x754lK85FZn4+mz9BFpffrsgzWzrMRbPZ3wHgop8AriBd/o7cDmyPiE9FxEMU/r0AfgpMNI8n\ngLNDqm3oMvPrvHaLm+VWnZ3DCP317N1zpen8PiPiDcAdwL1DqGsUuszFg8D9mfmLoVU1Gl3mYheL\nW5scBR4HvhkRVw2nvKHqMhdHgN+PiCPAfcDfDqm2zWjV2TmMNf05YFurPdb0rXbMlaDT+2wC/xHg\nc5n5gyHVNmyXnYuI2Am8FbhtcTkbgLsj4snMPDO0Koejy+/FPPAdgMx8rjmb2wn82zAKHKIuc/E4\n8Fhmfi0iJoDnI+K6zDw/pBo3k1Vn5zDO9N27Z8mKc9EsaTzK4qZ2ZyPiAyOqdaNddi4y80eZ+eHM\nPJSZh5oxR67AwIduf0dmgesAmr6rgJ8MvdKN12UudrJ4oQPAz4BfUuhKxPVm51BuzoqIP2TxC4kX\ngf/OzIOv7t2TmYci4k0sfgP9Aot79zyUV+7VOyvNxT8A7wT+vXnJ1XmRb+2vBCvNRTNmAvgYi19w\nPwA8mpk/HlXNG6XD78U4cBg4B7wdOJ6ZT46u4o3TYS7ey+J/3PTPwLXA2cw8NrqKN05EvB/4EPBH\nLH76/0sWv+Nac3Z6R64kFVLmI5EkydCXpFIMfUkqxNCXpEIMfUkqxNCXpEIMfUkqxNCXpEL+DzhE\naZUyoCgdAAAAAElFTkSuQmCC\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "gc = GCRCatalogs.load_catalog('protoDC2')\n", - "quantities = gc.list_all_quantities()\n", - "test = ShearTest()\n", - "test.run_on_single_catalog( gc, 'protoDC2', './')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "lsst", - "language": "python", - "name": "lsst" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.2" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/descqa/shear_test.py b/descqa/shear_test.py index 8e9fb1a4..986571c3 100644 --- a/descqa/shear_test.py +++ b/descqa/shear_test.py @@ -1,33 +1,24 @@ #from __future__ import unicode_literals, absolute_import, division import os -import numpy as np -from .base import BaseValidationTest, TestResult -from .plotting import plt - - import sys -sys.path.insert(0, '/global/common/software/lsst/common/miniconda/py3-4.2.12/lib/python3.6/site-packages') -sys.path.insert(0, '/global/common/software/lsst/common/miniconda/py3-4.2.12/bin/python') -sys.path.insert(0, '..') - - +import numpy as np from GCR import GCRQuery import GCRCatalogs import descqa from builtins import str import yaml -import numpy as np import healpy as hp -from descqa.plotting import plt - -from descqa import BaseValidationTest, TestResult import camb import treecorr +from .base import BaseValidationTest, TestResult +from .plotting import plt + + __all__ = ['ShearTest'] @@ -151,6 +142,13 @@ def get_catalog_data(gc, quantities, filters=None): # define theory from within this class + + + def post_process_plot(self, ax): + ax.text(0.05, 0.95, "add text here") + ax.legend() + + def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): # check if needed quantities exist @@ -219,9 +217,9 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): for ax_this in (ax, self.summary_ax): ax_this.plot(data, label=catalog_name) - #self.post_process_plot(ax) - #fig.savefig(os.path.join(output_dir, 'plot.png')) - #plt.close(fig) + self.post_process_plot(ax) + fig.savefig(os.path.join(output_dir, 'plot.png')) + plt.close(fig) score = data[0] #calculate your summary statistics return TestResult(score, passed=True) From f032ddb46e975a8fdb715314c899ece20e29cf67 Mon Sep 17 00:00:00 2001 From: Patricia Larsen Date: Fri, 5 Jan 2018 11:15:12 -0800 Subject: [PATCH 08/64] commented out errant GCR load --- descqa/shear_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/descqa/shear_test.py b/descqa/shear_test.py index 986571c3..e3448508 100644 --- a/descqa/shear_test.py +++ b/descqa/shear_test.py @@ -4,8 +4,8 @@ import sys import numpy as np -from GCR import GCRQuery -import GCRCatalogs +#from GCR import GCRQuery +#import GCRCatalogs import descqa from builtins import str @@ -230,4 +230,4 @@ def conclude_test(self, output_dir): self.post_process_plot(self.summary_ax) self.summary_fig.savefig(os.path.join(output_dir, 'summary.png')) plt.close(self.summary_fig) - \ No newline at end of file + From 029be4971aebde06745cddcb50d3f4b70e4c2195 Mon Sep 17 00:00:00 2001 From: Patricia Larsen Date: Fri, 5 Jan 2018 14:58:37 -0800 Subject: [PATCH 09/64] added plots, fixed bug in theory separation values --- descqa/configs/shear.yaml | 2 +- descqa/shear_test.py | 83 +++++++++++++++++++++++++++------------ 2 files changed, 58 insertions(+), 27 deletions(-) diff --git a/descqa/configs/shear.yaml b/descqa/configs/shear.yaml index 93eba19f..648e3b7c 100644 --- a/descqa/configs/shear.yaml +++ b/descqa/configs/shear.yaml @@ -4,7 +4,7 @@ ra: 'ra' dec: 'dec' e1: 'shear_1' e2: 'shear_2' -kappa: 'kappa' +kappa: 'convergence' nbins: 20 min_sep: 2.5 max_sep: 250 diff --git a/descqa/shear_test.py b/descqa/shear_test.py index e3448508..8e1266c8 100644 --- a/descqa/shear_test.py +++ b/descqa/shear_test.py @@ -4,8 +4,8 @@ import sys import numpy as np -#from GCR import GCRQuery -#import GCRCatalogs +from GCR import GCRQuery +import GCRCatalogs import descqa from builtins import str @@ -15,12 +15,30 @@ import camb import treecorr +from astropy.cosmology import FlatLambdaCDM +cosmo = FlatLambdaCDM(H0=70, Om0=0.3, Tcmb0=2.725) +from scipy.interpolate import interp1d +from scipy.integrate import quad + +import astropy.units as u +import astropy.constants as const +import camb.correlations + +chi_recomb = 14004.036207574154 +pars = camb.CAMBparams() +h = 0.675 +pars.set_cosmology(H0=h*100, ombh2=0.022, omch2=0.122) +pars.InitPower.set_params(ns=0.965) +camb.set_halofit_version(version='takahashi') +p = camb.get_matter_power_interpolator(pars, nonlinear=True, k_hunit=False, hubble_units=False ,kmax=100., zmax=1100., k_per_logint=False).P + from .base import BaseValidationTest, TestResult from .plotting import plt + __all__ = ['ShearTest'] class ShearTest(BaseValidationTest): @@ -28,7 +46,7 @@ class ShearTest(BaseValidationTest): Validation test for shear and convergence quantities """ - def __init__(self, z='redshift_true', ra='ra', dec='dec', e1='shear_1', e2='shear_2', kappa='kappa', nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin', bin_slop=0.1, zlo=0.5, zhi=0.6, **kwargs): + def __init__(self, z='redshift_true', ra='ra', dec='dec', e1='shear_1', e2='shear_2', kappa='convergence', nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin', bin_slop=0.1, zlo=0.5, zhi=0.6, **kwargs): #catalog quantities self.z = z #sep-bounds and binning @@ -45,7 +63,8 @@ def __init__(self, z='redshift_true', ra='ra', dec='dec', e1='shear_1', e2='shea self.kappa = kappa # cut in redshift self.filters = [(lambda z: (z > zlo) & (z < zhi), self.z)] - self.summary_fig, self.summary_ax = plt.subplots() + self.summary_fig, self.summary_ax = plt.subplots(2, sharex=True) + #validation data # want this to change to theory... #self.validation_data = {} @@ -60,7 +79,7 @@ def __init__(self, z='redshift_true', ra='ra', dec='dec', e1='shear_1', e2='shea #self._other_kwargs = kwargs - def compute_nz(n_z): + def compute_nz(self,n_z): '''create interpolated n(z) distribution''' z_bins = np.linspace(0.0, 2.0, 101) n = np.histogram(n_z, bins=z_bins)[0] @@ -70,25 +89,25 @@ def compute_nz(n_z): n2 = interp1d(z, n/n2_sum, bounds_error=False, fill_value=0.0, kind='cubic') return n2 - def integrand_w(x,n,chi,chi_int): + def integrand_w(self,x,n,chi,chi_int): ''' This is the inner bit of GWL lensing kernel - z is related to x, not chi''' z = chi_int(x) H_z = cosmo.H(z).value*3.240779289469756e-20 #1/s units #.to(1./u.s) conversion dchidz = 9.715611890256315e-15 /H_z #const.c.to(u.Mpc/u.s).value / (H_z) # Mpc units return n(z)/dchidz* (x - chi)/x - def galaxy_W(z,n,chi_int): + def galaxy_W(self,z,n,chi_int): ''' galaxy window function''' chi = cosmo.comoving_distance(z).value # can be array cst = 3./2. * cosmo.H(0).to(1./u.s)**2/const.c.to(u.Mpc / u.s)**2*cosmo.Om(0)# not sure about the z-dependence here prefactor = cst * chi* (1.+z) *u.Mpc val_array = [] for i in range(len(z)): - val_array.append(quad(integrand_w, chi[i], chi_recomb, args = (n, chi[i], chi_int))[0]) + val_array.append(quad(self.integrand_w, chi[i], chi_recomb, args = (n, chi[i], chi_int))[0]) W = np.array(val_array)*prefactor *(u.Mpc) # now unitless return W - def integrand_lensing_limber(chi, l, galaxy_W_int, chi_int): + def integrand_lensing_limber(self,chi, l, galaxy_W_int, chi_int): '''return overall integrand for one value of l''' chi_unit = chi*u.Mpc z = chi_int(chi) @@ -96,7 +115,7 @@ def integrand_lensing_limber(chi, l, galaxy_W_int, chi_int): integrand = p(z, k, grid=False) *galaxy_W_int(z)**2/chi**2 return integrand - def phi(lmax,n_z): + def phi(self,lmax,n_z): z_array = np.logspace(-3,np.log10(10.),200) chi_array = cosmo.comoving_distance(z_array).value chi_int = interp1d(chi_array, z_array, bounds_error=False, fill_value=0.0) @@ -113,17 +132,17 @@ def phi(lmax,n_z): return l, phi_array*prefactor - def theory_corr(n_z, min_sep=2.5, max_sep=250., nbins=20, lmax=20000): - ll, pp = self.phi(lmax=lmax, n_z=n_z) - pp3_2 = np.zeros((lmax,4)) + def theory_corr(self,n_z2, min_sep2, max_sep2, nbins2, lmax2): + ll, pp = self.phi(lmax=lmax2, n_z=n_z2) + pp3_2 = np.zeros((lmax2,4)) pp3_2[:,1]= pp[:]*(ll*(ll+1.))/(2.*np.pi) - xvals = np.logspace(np.log10(min_sep), np.log10(max_sep), nbins) #in arcminutes - cxvals = np.cos(xvals*(60.)*(180./np.pi)) + xvals = np.logspace(np.log10(min_sep2), np.log10(max_sep2), nbins2) #in arcminutes + cxvals = np.cos(xvals/(60.)/(180./np.pi)) vals = camb.correlations.cl2corr(pp3_2, cxvals) return xvals, vals[:,1], vals[:,2] - def get_chi2(measured, theory): + def get_chi2(self,measured, theory): chi2 = (measured-theory)**2/theory return chi2/float(len(measured)) @@ -145,16 +164,23 @@ def get_catalog_data(gc, quantities, filters=None): def post_process_plot(self, ax): - ax.text(0.05, 0.95, "add text here") - ax.legend() + #ax.text(0.05, 0.95, "add text here") + plt.xscale('log') + ax[0].legend() + ax[1].legend() + def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): # check if needed quantities exist - if not catalog_instance.has_quantities([self.z, self.ra, self.dec, self.e1, self.e2,self.kappa]): + + if not catalog_instance.has_quantities([self.z, self.ra, self.dec]): + #print("failed") + return TestResult(skipped=True, summary='do not have needed location quantities') + if not catalog_instance.has_quantities([self.e1, self.e2,self.kappa]): #print("failed") - return TestResult(skipped=True, summary='do not have needed quantities') + return TestResult(skipped=True, summary='do not have needed shear quantities') catalog_data = self.get_catalog_data(catalog_instance, [self.z, self.ra, self.dec, self.e1, self.e2, self.kappa], filters=self.filters) # get required catalogue data @@ -171,17 +197,17 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): n_z = catalog_data[self.z] - xvals, theory_plus, theory_minus = self.theory_corr(n_z, min_sep=2.5, max_sep=250., nbins=20, lmax=20000) + xvals, theory_plus, theory_minus = self.theory_corr(n_z, self.min_sep, self.max_sep, self.nbins, 2000) cat_s = treecorr.Catalog(ra=catalog_data[self.ra], dec=catalog_data[self.dec], - g1=catalog_data[self.e1]-np.mean(catalog_data[self.e1]), g2=catalog_data[self.e2]-np.mean(catalog_data[self.e2]), ra_units='deg', dec_units='deg') + g1=catalog_data[self.e1]-np.mean(catalog_data[self.e1]), g2=-(catalog_data[self.e2]-np.mean(catalog_data[self.e2])), ra_units='deg', dec_units='deg') gg = treecorr.GGCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin', bin_slop=0.1, verbose=True) gg.process(cat_s) chi2_dof_1= self.get_chi2(gg.xip, theory_plus) # correct this - r = numpy.exp(gg.meanlogr) + r = np.exp(gg.meanlogr) xip = gg.xip xim = gg.xim @@ -189,7 +215,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): plt.plot(xip) plt.plot(theory_plus) plt.show() - sig = numpy.sqrt(gg.varxi) + sig = np.sqrt(gg.varxi) # directly compare xip and xim to theory_plus and theory_minus. Take into account cosmic variance. @@ -212,10 +238,15 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): data = np.random.rand(10) #do your calculation with catalog_instance - fig, ax = plt.subplots() + fig, ax = plt.subplots(2,sharex=True) for ax_this in (ax, self.summary_ax): - ax_this.plot(data, label=catalog_name) + ax_this[0].plot(xvals,xip,'x', label="w+ autocorrelation") + ax_this[0].plot(xvals,theory_plus,'x',color='g', label="w+ theory") + ax_this[1].plot(xvals,xim,'x',color='r', label="w- auto") + ax_this[1].plot(xvals,theory_minus,'x',color='y', label="w- theory") + + self.post_process_plot(ax) fig.savefig(os.path.join(output_dir, 'plot.png')) From 7bf1b16811f22c69749c83d2c54f5c045a06d860 Mon Sep 17 00:00:00 2001 From: Vinu Vikraman Date: Fri, 5 Jan 2018 21:47:00 -0800 Subject: [PATCH 10/64] Preliminary version of size-luminosity validation script and added Table 5 of van der Wal et al 2014 --- descqa/SizeStellarMassLuminosity.py | 135 ++++++++++++++++++ descqa/configs/size_vanderWal2014_SM_Lum.yaml | 35 +++++ ...Wel_2014_ApJ_788_28_Table5_StellarMass.txt | 27 ++++ ...Wel_2014_ApJ_788_28_Table5_VLuminosity.txt | 29 ++++ ...an_der_Wel_2014_ApJ_788_28_Table5_dict.txt | 12 ++ 5 files changed, 238 insertions(+) create mode 100644 descqa/SizeStellarMassLuminosity.py create mode 100644 descqa/configs/size_vanderWal2014_SM_Lum.yaml create mode 100644 descqa/data/size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_StellarMass.txt create mode 100644 descqa/data/size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_VLuminosity.txt create mode 100644 descqa/data/size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_dict.txt diff --git a/descqa/SizeStellarMassLuminosity.py b/descqa/SizeStellarMassLuminosity.py new file mode 100644 index 00000000..fe3d25aa --- /dev/null +++ b/descqa/SizeStellarMassLuminosity.py @@ -0,0 +1,135 @@ +from __future__ import print_function, division, unicode_literals, absolute_import +import os +import numpy as np +from GCR import GCRQuery +from scipy import interpolate + +from .base import BaseValidationTest, TestResult +from .plotting import plt +from .utils import generate_uniform_random_ra_dec_footprint, get_healpixel_footprint, generate_uniform_random_dist + + +__all__ = ['SizeStellarMassLuminosity'] + + +def redshift2dist(cosmology): + z = np.arange(0, 5.1, 0.5) + comov_d = cosmology.comoving_distance(z).to('kpc').value + spl = interpolate.splrep(z, comov_d) + return spl + + +class SizeStellarMassLuminosity(BaseValidationTest): + """ + Validation test of 2pt correlation function + """ + _C = 299792.458 + + def __init__(self, **kwargs): + self.possible_mag_fields = kwargs['possible_mag_fields'] + self.data_label = kwargs['data_label'] + self.z_bins = kwargs['z_bins'] + self.output_filename_template = kwargs['output_filename_template'] + self.label_template = kwargs['label_template'] + self.fig_xlabel = kwargs['fig_xlabel'] + self.fig_ylabel = kwargs['fig_ylabel'] + + + validation_filepath = os.path.join(self.data_dir, kwargs['data_filename']) + self.validation_data = np.genfromtxt(validation_filepath) + + @staticmethod + def ConvertAbsMagLuminosity(AbsM, band): + '''AbsM: absolute magnitude, band: filter''' + if type(AbsM) is list or type(AbsM) is np.ndarray: + AbsM = np.array(AbsM) + + bands = {'U':5.61, 'B':5.48, 'V':4.83, 'R':4.42, 'I':4.08, + 'J':3.64, 'H':3.32, 'K':3.28, 'g':5.33, 'r':4.67, + 'i':4.48, 'z':4.42, 'F300W':6.09, 'F450W':5.32, 'F555W':4.85, + 'F606W':4.66, 'F702W':4.32, 'F814W':4.15, 'CFHT_U':5.57, + 'CFHT_B':5.49, 'CFHT_V':4.81, 'CFHT_R':4.44, 'CFHT_I':4.06, + 'NIRI_J':3.64, 'NIRI_H':3.33, 'NIRI_K':3.29} + + if band in bands.keys(): + AbsSun = bands[band] + else: + raise ValueError('Filter not implemented') + + logL = (AbsSun - AbsM) / 2.5 #unit of sun + L = 10**logL + return L + + def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): + ''' + Loop over magnitude cuts and make plots + ''' + # load catalog data + spl = redshift2dist(catalog_instance.cosmology) + + colnames = dict() + #colnames['z'] = catalog_instance.first_available('redshift') + #colnames['size'] = catalog_instance.first_available('size') + colnames['z'] = 'redshift' + colnames['size'] = 'size' + colnames['mag'] = catalog_instance.first_available(*self.possible_mag_fields) + #print('colnames ', colnames) + if not all(v for v in colnames.values()): + return TestResult(skipped=True, summary='Missing requested quantities') + #Check whether the columns are finite or not + filters = [(np.isfinite, c) for c in colnames.values()] + #zz = [z_bin['z_max'] for z_bin in self.z_bins] + #print('z ', zz) + #Select objects within maximum and minimum redshift of the all the bins + filters.extend(( + '{} < {}'.format(colnames['z'], max(z_bin['z_max'] for z_bin in self.z_bins)), + '{} >= {}'.format(colnames['z'], min(z_bin['z_min'] for z_bin in self.z_bins)), + )) + catalog_data = catalog_instance.get_quantities(list(colnames.values()), filters=filters) + catalog_data = {k: catalog_data[v] for k, v in colnames.items()} + + fig, axes = plt.subplots(2,3, figsize=(6, 9), sharex=True, sharey=True) + try: + i = 0 + j = 0 + for z_bin in self.z_bins: + ax = axes[i,j] + + # filter catalog data for this bin + filters = [ + 'z < {}'.format(z_bin['z_max']), + 'z >= {}'.format(z_bin['z_min']), + ] + + catalog_data_this = GCRQuery(*filters).filter(catalog_data) + size_kpc = catalog_data_this['size'] * interpolate.splev(catalog_data_this['z'], spl) / (1 + catalog_data_this['z']) + + del catalog_data_this + + z_mean = (z_bin['z_max'] + z_bin['z_min']) / 2. + validation_this = self.validation_data[(self.validation_data[:,0] < z_mean + 0.02) & (self.validation_data[:,0] > z_mean - 0.02)] + + ax.plot(validation_this[:,1], validation_this[:, 2], label=self.label_template.format(z_bin['z_min'], z_bin['z_max'])) + ax.fill_between(validation_this[:,1], validation_this[:,3], validation_this[:,4], lw=0, alpha=0.2) + #ax.errorbar(xi_rad, xi*scale_wp, xi_sig*scale_wp, marker='o', ls='', c=color) + j += 1 + if j > 2: + j = 0 + i += 1 + + ax.legend(loc='best') + fig.add_subplot(111, frameon=False) + # hide tick and tick label of the big axes + plt.tick_params(labelcolor='none', top='off', bottom='off', left='off', right='off') + plt.grid(False) + plt.xlabel(self.fig_xlabel) + plt.ylabel(self.fig_ylabel) + fig.subplots_adjust(hspace=0, wspace=0) + fig.suptitle('{} vs. {}'.format(catalog_name, self.data_label), fontsize='medium') + finally: + #fig.savefig(os.path.join(output_dir, '{:s}.png'.format('t')), bbox_inches='tight') + fig.savefig(os.path.join('./', '{:s}.png'.format('t')), bbox_inches='tight') + plt.close(fig) + + #TODO: calculate summary statistics + return TestResult(0, passed=True) diff --git a/descqa/configs/size_vanderWal2014_SM_Lum.yaml b/descqa/configs/size_vanderWal2014_SM_Lum.yaml new file mode 100644 index 00000000..0f2fcf88 --- /dev/null +++ b/descqa/configs/size_vanderWal2014_SM_Lum.yaml @@ -0,0 +1,35 @@ +subclass_name: SizeStellarMassLuminosity.SizeStellarMassLuminosity + +possible_mag_fields: + - Mag_true_g_sdss_z0 + - Mag_true_r_sdss_z0 + - Mag_true_g_des_z0 + - Mag_true_r_des_z0 + - Mag_true_g_lsst_z0 + - Mag_true_r_lsst_z0 + - Mag_true_g_sdss_z01 + - Mag_true_r_sdss_z01 + - Mag_true_g_des_z01 + - Mag_true_r_des_z01 + - Mag_true_g_lsst_z01 + - Mag_true_r_lsst_z01 + +output_filename_template: 'size_lum.dat' +label_template: '${} < z < {}$' + +data_filename: 'size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_VLuminosity.txt' +data_label: 'van der Wal+2014' + +z_bins: + - {z_min: 0.0, z_max: 0.5} + - {z_min: 0.5, z_max: 1.0} + - {z_min: 1.0, z_max: 1.5} + - {z_min: 1.5, z_max: 2.0} + - {z_min: 2.0, z_max: 2.5} +# - {z_min: 2.5, z_max: 3.0} + +fig_xlabel: '$\log_{10}(L/L_{\odot}) [{\rm V-band Luminosity}]$' +fig_ylabel: '$\log_{10}(R_e) [{\rm kpc}]$' + +description: | + Compare evolution of size as a function of luminosity and redshift and van der Wal et al (2014) HST-3D and CANDELS observations diff --git a/descqa/data/size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_StellarMass.txt b/descqa/data/size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_StellarMass.txt new file mode 100644 index 00000000..a8741272 --- /dev/null +++ b/descqa/data/size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_StellarMass.txt @@ -0,0 +1,27 @@ +#Table 5 +#Logarithmic Size Distributions (16%-84% Range) for the Full Population +#(Early- and Late-type Galaxies Combined) as a Function of Mass and Redshift, +#Early+Late Type Galaxies +#z log10M* log10Size_M* log10SizeM*_high log10SizeM*_low +0.25 9.25 0.46 0.69 0.21 +0.25 9.75 0.54 0.77 0.28 +0.25 10.25 0.54 0.82 0.27 +0.25 10.75 0.75 0.99 0.49 +0.75 9.25 0.41 0.64 0.16 +0.75 9.75 0.52 0.74 0.25 +0.75 10.25 0.52 0.78 0.18 +0.75 10.75 0.59 0.84 0.35 +0.75 11.25 0.85 1.04 0.66 +1.25 9.75 0.47 0.68 0.21 +1.25 10.25 0.52 0.74 0.15 +1.25 10.75 0.57 0.80 0.23 +1.25 11.25 0.74 0.94 0.49 +1.75 9.75 0.42 0.65 0.15 +1.75 10.25 0.48 0.69 0.12 +1.75 10.75 0.48 0.74 0.09 +1.75 11.25 0.64 0.83 0.34 +2.25 10.25 0.41 0.63 0.10 +2.25 10.75 0.45 0.68 0.03 +2.25 11.25 0.59 0.83 0.28 +2.75 10.75 0.43 0.65 0.01 +2.75 11.25 0.52 0.75 0.27 diff --git a/descqa/data/size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_VLuminosity.txt b/descqa/data/size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_VLuminosity.txt new file mode 100644 index 00000000..c032f517 --- /dev/null +++ b/descqa/data/size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_VLuminosity.txt @@ -0,0 +1,29 @@ +#Table 5 +#Logarithmic Size Distributions (16%-84% Range) for the Full Population +#(Early- and Late-type Galaxies Combined) as a Function of rest frame V-band luminosity and Redshift, +#Early+Late Type Galaxies +#2.75 10.75 0.41 0.62 0.11 +#2.75 11.25 0.52 0.74 0.12 +#z log10L* log10Size_L* log10Size_L*_high log10Size_L*_low +0.25 9.25 0.37 0.62 0.14 +0.25 9.75 0.53 0.74 0.26 +0.25 10.25 0.62 0.85 0.37 +0.25 10.75 0.79 1.05 0.62 +0.75 9.25 0.29 0.54 0.05 +0.75 9.75 0.44 0.66 0.18 +0.75 10.25 0.54 0.76 0.28 +0.75 10.75 0.68 0.90 0.40 +0.75 11.25 0.83 1.04 0.62 +1.25 9.25 0.30 0.64 0.02 +1.25 9.75 0.36 0.60 0.10 +1.25 10.25 0.47 0.69 0.19 +1.25 10.75 0.58 0.79 0.29 +1.25 11.25 0.67 0.91 0.36 +1.75 9.75 0.32 0.58 0.05 +1.75 10.25 0.39 0.61 0.12 +1.75 10.75 0.51 0.73 0.16 +1.75 11.25 0.51 0.83 0.22 +2.25 10.25 0.33 0.55 0.07 +2.25 10.75 0.43 0.64 0.15 +2.25 11.25 0.45 0.74 0.10 + diff --git a/descqa/data/size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_dict.txt b/descqa/data/size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_dict.txt new file mode 100644 index 00000000..b70fdfff --- /dev/null +++ b/descqa/data/size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_dict.txt @@ -0,0 +1,12 @@ +#Table 5 +#Logarithmic Size Distributions (16%-84% Range) for the Full Population +#(Early- and Late-type Galaxies Combined) as a Function of Mass and Redshift, +#and Rest-frame V-band Luminosity and Redshift +#Early+Late Type Galaxies + +Table5 = {0.25:{'log10M*':[9.25, 9.75, 10.25, 10.75], 'log10Size_M*':[0.46, 0.54, 0.54, 0.75], 'log10SizeM*_low':[0.21, 0.28, 0.27, 0.49], 'log10SizeM*_high':[0.69, 0.77, 0.82, 0.99], 'log10L*':[9.25, 9.75, 10.25, 10.75], 'log10Size_L*':[0.37, 0.53, 0.62, 0.79], 'log10Size_L*_low':[0.14, 0.26, 0.37, 0.62], 'log10Size_L*_high':[0.62, 0.74, 0.85, 1.05]}, + 0.75:{'log10M*':[9.25, 9.75, 10.25, 10.75, 11.25], 'log10Size_M*':[0.41, 0.52, 0.52, 0.59, 0.85], 'log10Size_M*_low':[0.16, 0.25, 0.18, 0.35, 0.66], 'log10SizeM*_high':[0.64, 0.74, 0.78, 0.84, 1.04], 'log10L*':[9.25, 9.75, 10.25, 10.75, 11.25], 'log10Size_L*':[0.29, 0.44, 0.54, 0.68, 0.83], 'log10Size_L*_low':[0.05, 0.18, 0.28, 0.40, 0.62], 'log10Size_L*_high':[0.54, 0.66, 0.76, 0.90, 1.04]}, + 1.25:{'log10M*':[9.75, 10.25, 10.75, 11.25], 'log10Size_M*':[0.47, 0.52, 0.57, 0.74], 'log10Size_M*_low':[0.21, 0.15, 0.23, 0.49], 'log10SizeM*_high':[0.68, 0.74, 0.80, 0.94], 'log10L*':[9.25, 9.75, 10.25, 10.75, 11.25], 'log10Size_L*':[0.30, 0.36, 0.47, 0.58, 0.67], 'log10Size_L*_low':[0.02, 0.10, 0.19, 0.29, 0.36], 'log10Size_L*_high':[0.64, 0.60, 0.69, 0.79, 0.91]}, + 1.75:{'log10M*':[9.75, 10.25, 10.75, 11.25], 'log10Size_M*':[0.42, 0.48, 0.48, 0.64], 'log10Size_M*_low':[0.15, 0.12, 0.09, 0.34], 'log10SizeM*_high':[0.65, 0.69, 0.74, 0.83], 'log10L*':[9.75, 10.25, 10.75, 11.25], 'log10Size_L*':[0.32, 0.39, 0.51, 0.51], 'log10Size_L*_low':[0.05, 0.12, 0.16, 0.22], 'log10Size_L*_high':[0.58, 0.61, 0.73, 0.83]}, + 2.25:{'log10M*':[10.25, 10.75, 11.25], 'log10Size_M*':[0.41, 0.45, 0.59], 'log10Size_M*_low':[0.10, 0.03, 0.28], 'log10SizeM*_high':[0.63, 0.68, 0.83], 'log10L*':[10.25, 10.75, 11.25], 'log10Size_L*':[0.33, 0.43, 0.45], 'log10Size_L*_low':[0.07, 0.15, 0.10], 'log10Size_L*_high':[0.55, 0.64, 0.74]}, + 2.75:{{'log10M*':[10.75, 11.25], 'log10Size_M*':[0.43, 0.52], 'log10Size_M*_low':[0.01, 0.27], 'log10SizeM*_high':[0.65, 0.75], 'log10L*':[10.75, 11.25], 'log10Size_L*':[0.41, 0.52], 'log10Size_L*_low':[0.11, 0.12], 'log10Size_L*_high':[0.62, 0.74]} From 12c7d0aa742292872d004f4fe8dbf93c095ee375 Mon Sep 17 00:00:00 2001 From: Vinu Vikraman Date: Sat, 6 Jan 2018 10:37:24 -0800 Subject: [PATCH 11/64] Working version of size luminosity relation. Using absolute DES g-band from Buzzard but van der Wel et al using rest frame V-band luminosity --- descqa/SizeStellarMassLuminosity.py | 55 +++++++++++-------- descqa/configs/size_vanderWal2014_SM_Lum.yaml | 6 +- 2 files changed, 35 insertions(+), 26 deletions(-) diff --git a/descqa/SizeStellarMassLuminosity.py b/descqa/SizeStellarMassLuminosity.py index fe3d25aa..6b87d99d 100644 --- a/descqa/SizeStellarMassLuminosity.py +++ b/descqa/SizeStellarMassLuminosity.py @@ -3,6 +3,7 @@ import numpy as np from GCR import GCRQuery from scipy import interpolate +from scipy.stats import binned_statistic from .base import BaseValidationTest, TestResult from .plotting import plt @@ -23,10 +24,11 @@ class SizeStellarMassLuminosity(BaseValidationTest): """ Validation test of 2pt correlation function """ - _C = 299792.458 + _ARCSEC_TO_RADIAN = np.pi / 180. / 3600. def __init__(self, **kwargs): self.possible_mag_fields = kwargs['possible_mag_fields'] + self.test_name = kwargs['test_name'] self.data_label = kwargs['data_label'] self.z_bins = kwargs['z_bins'] self.output_filename_template = kwargs['output_filename_template'] @@ -58,29 +60,29 @@ def ConvertAbsMagLuminosity(AbsM, band): logL = (AbsSun - AbsM) / 2.5 #unit of sun L = 10**logL - return L + return L, logL def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): ''' Loop over magnitude cuts and make plots ''' # load catalog data + default_L_bin_edges = np.array([9, 9.5, 10, 10.5, 11, 11.5]) + default_L_bins = (default_L_bin_edges[1:] + default_L_bin_edges[:-1]) / 2. spl = redshift2dist(catalog_instance.cosmology) colnames = dict() - #colnames['z'] = catalog_instance.first_available('redshift') + #colnames['z'] = catalog_instance.first_available('redshift', 'redshift_true') #colnames['size'] = catalog_instance.first_available('size') colnames['z'] = 'redshift' colnames['size'] = 'size' colnames['mag'] = catalog_instance.first_available(*self.possible_mag_fields) - #print('colnames ', colnames) if not all(v for v in colnames.values()): return TestResult(skipped=True, summary='Missing requested quantities') #Check whether the columns are finite or not filters = [(np.isfinite, c) for c in colnames.values()] - #zz = [z_bin['z_max'] for z_bin in self.z_bins] - #print('z ', zz) - #Select objects within maximum and minimum redshift of the all the bins + + #Select objects within maximum and minimum redshift of all the bins filters.extend(( '{} < {}'.format(colnames['z'], max(z_bin['z_max'] for z_bin in self.z_bins)), '{} >= {}'.format(colnames['z'], min(z_bin['z_min'] for z_bin in self.z_bins)), @@ -88,13 +90,12 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): catalog_data = catalog_instance.get_quantities(list(colnames.values()), filters=filters) catalog_data = {k: catalog_data[v] for k, v in colnames.items()} - fig, axes = plt.subplots(2,3, figsize=(6, 9), sharex=True, sharey=True) + fig, axes = plt.subplots(2,3, figsize=(9, 6), sharex=True, sharey=True) try: - i = 0 - j = 0 + col = 0 + row = 0 for z_bin in self.z_bins: - ax = axes[i,j] - + ax = axes[row, col] # filter catalog data for this bin filters = [ 'z < {}'.format(z_bin['z_max']), @@ -102,22 +103,31 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): ] catalog_data_this = GCRQuery(*filters).filter(catalog_data) - size_kpc = catalog_data_this['size'] * interpolate.splev(catalog_data_this['z'], spl) / (1 + catalog_data_this['z']) + if len(catalog_data_this['z']) == 0: + continue + size_kpc = catalog_data_this['size'] * self._ARCSEC_TO_RADIAN * interpolate.splev(catalog_data_this['z'], spl) / (1 + catalog_data_this['z']) + L_G, logL_G = self.ConvertAbsMagLuminosity(catalog_data_this['mag'], 'g') + binned_size_kpc, tmp_1, tmp_2 = binned_statistic(logL_G, size_kpc, bins=default_L_bin_edges, statistic='mean') + binned_size_kpc_err, tmp_1, tmp_2 = binned_statistic(logL_G, size_kpc, bins=default_L_bin_edges, statistic='std') + + output_filepath = os.path.join(output_dir, self.output_filename_template.format(z_bin['z_min'], z_bin['z_max'])) + np.savetxt(output_filepath, np.transpose((default_L_bins, binned_size_kpc, binned_size_kpc_err))) del catalog_data_this z_mean = (z_bin['z_max'] + z_bin['z_min']) / 2. validation_this = self.validation_data[(self.validation_data[:,0] < z_mean + 0.02) & (self.validation_data[:,0] > z_mean - 0.02)] - ax.plot(validation_this[:,1], validation_this[:, 2], label=self.label_template.format(z_bin['z_min'], z_bin['z_max'])) - ax.fill_between(validation_this[:,1], validation_this[:,3], validation_this[:,4], lw=0, alpha=0.2) - #ax.errorbar(xi_rad, xi*scale_wp, xi_sig*scale_wp, marker='o', ls='', c=color) - j += 1 - if j > 2: - j = 0 - i += 1 + ax.semilogy(validation_this[:,1], 10**validation_this[:, 2], label=self.label_template.format(z_bin['z_min'], z_bin['z_max'])) + ax.fill_between(validation_this[:,1], 10**validation_this[:,3], 10**validation_this[:,4], lw=0, alpha=0.2) + ax.errorbar(default_L_bins, binned_size_kpc, binned_size_kpc_err, marker='o', ls='') + col += 1 + if col > 2: + col = 0 + row += 1 ax.legend(loc='best') + fig.add_subplot(111, frameon=False) # hide tick and tick label of the big axes plt.tick_params(labelcolor='none', top='off', bottom='off', left='off', right='off') @@ -125,10 +135,9 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): plt.xlabel(self.fig_xlabel) plt.ylabel(self.fig_ylabel) fig.subplots_adjust(hspace=0, wspace=0) - fig.suptitle('{} vs. {}'.format(catalog_name, self.data_label), fontsize='medium') + fig.suptitle('{} vs. {}'.format(catalog_name, self.data_label), fontsize='medium', y=0.93) finally: - #fig.savefig(os.path.join(output_dir, '{:s}.png'.format('t')), bbox_inches='tight') - fig.savefig(os.path.join('./', '{:s}.png'.format('t')), bbox_inches='tight') + fig.savefig(os.path.join(output_dir, '{:s}.png'.format(self.test_name)), bbox_inches='tight') plt.close(fig) #TODO: calculate summary statistics diff --git a/descqa/configs/size_vanderWal2014_SM_Lum.yaml b/descqa/configs/size_vanderWal2014_SM_Lum.yaml index 0f2fcf88..28de8ef2 100644 --- a/descqa/configs/size_vanderWal2014_SM_Lum.yaml +++ b/descqa/configs/size_vanderWal2014_SM_Lum.yaml @@ -14,7 +14,7 @@ possible_mag_fields: - Mag_true_g_lsst_z01 - Mag_true_r_lsst_z01 -output_filename_template: 'size_lum.dat' +output_filename_template: 'size_lum_z_{}_{}.dat' label_template: '${} < z < {}$' data_filename: 'size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_VLuminosity.txt' @@ -26,9 +26,9 @@ z_bins: - {z_min: 1.0, z_max: 1.5} - {z_min: 1.5, z_max: 2.0} - {z_min: 2.0, z_max: 2.5} -# - {z_min: 2.5, z_max: 3.0} + - {z_min: 2.5, z_max: 3.0} -fig_xlabel: '$\log_{10}(L/L_{\odot}) [{\rm V-band Luminosity}]$' +fig_xlabel: '$\log_{10}(L/L_{\odot}) [{\rm V-band}]$' fig_ylabel: '$\log_{10}(R_e) [{\rm kpc}]$' description: | From 57c61d37e64f0872461a994bb3b9f4ad7762f721 Mon Sep 17 00:00:00 2001 From: Vinu Vikraman Date: Sat, 6 Jan 2018 10:39:00 -0800 Subject: [PATCH 12/64] Changed Wal to Wel in the author name --- ...nderWal2014_SM_Lum.yaml => size_vanderWel2014_SM_Lum.yaml} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename descqa/configs/{size_vanderWal2014_SM_Lum.yaml => size_vanderWel2014_SM_Lum.yaml} (90%) diff --git a/descqa/configs/size_vanderWal2014_SM_Lum.yaml b/descqa/configs/size_vanderWel2014_SM_Lum.yaml similarity index 90% rename from descqa/configs/size_vanderWal2014_SM_Lum.yaml rename to descqa/configs/size_vanderWel2014_SM_Lum.yaml index 28de8ef2..2898d236 100644 --- a/descqa/configs/size_vanderWal2014_SM_Lum.yaml +++ b/descqa/configs/size_vanderWel2014_SM_Lum.yaml @@ -18,7 +18,7 @@ output_filename_template: 'size_lum_z_{}_{}.dat' label_template: '${} < z < {}$' data_filename: 'size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_VLuminosity.txt' -data_label: 'van der Wal+2014' +data_label: 'van der Wel+2014' z_bins: - {z_min: 0.0, z_max: 0.5} @@ -32,4 +32,4 @@ fig_xlabel: '$\log_{10}(L/L_{\odot}) [{\rm V-band}]$' fig_ylabel: '$\log_{10}(R_e) [{\rm kpc}]$' description: | - Compare evolution of size as a function of luminosity and redshift and van der Wal et al (2014) HST-3D and CANDELS observations + Compare evolution of size as a function of luminosity and redshift and van der Wel et al (2014) HST-3D and CANDELS observations From 36f595936edd715001ec68e1d74ce8458200d165 Mon Sep 17 00:00:00 2001 From: Vinu Vikraman Date: Mon, 8 Jan 2018 14:04:13 -0800 Subject: [PATCH 13/64] Minor change to remove tick labels in the subplots --- descqa/SizeStellarMassLuminosity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/descqa/SizeStellarMassLuminosity.py b/descqa/SizeStellarMassLuminosity.py index 6b87d99d..cab12a26 100644 --- a/descqa/SizeStellarMassLuminosity.py +++ b/descqa/SizeStellarMassLuminosity.py @@ -130,7 +130,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): fig.add_subplot(111, frameon=False) # hide tick and tick label of the big axes - plt.tick_params(labelcolor='none', top='off', bottom='off', left='off', right='off') + plt.tick_params(labelcolor='none', which='both', top='off', bottom='off', left='off', right='off') plt.grid(False) plt.xlabel(self.fig_xlabel) plt.ylabel(self.fig_ylabel) From 55a81c8ef8401fdfca0104ed8c82eba8b54c5800 Mon Sep 17 00:00:00 2001 From: Vinu Vikraman Date: Mon, 8 Jan 2018 21:06:04 -0800 Subject: [PATCH 14/64] Changed figure title in the yaml file for size-luminosity test --- descqa/configs/size_vanderWel2014_SM_Lum.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/descqa/configs/size_vanderWel2014_SM_Lum.yaml b/descqa/configs/size_vanderWel2014_SM_Lum.yaml index 2898d236..b69548eb 100644 --- a/descqa/configs/size_vanderWel2014_SM_Lum.yaml +++ b/descqa/configs/size_vanderWel2014_SM_Lum.yaml @@ -18,7 +18,7 @@ output_filename_template: 'size_lum_z_{}_{}.dat' label_template: '${} < z < {}$' data_filename: 'size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_VLuminosity.txt' -data_label: 'van der Wel+2014' +data_label: 'van der Wel+2014: comparing mean and scatter' z_bins: - {z_min: 0.0, z_max: 0.5} From 0c191eb8354fb59f900fca5ffbd024c95264c4c6 Mon Sep 17 00:00:00 2001 From: Patricia Larsen Date: Wed, 10 Jan 2018 11:49:34 -0800 Subject: [PATCH 15/64] added jack-knife errors, tidied up plots --- descqa/configs/shear.yaml | 6 +-- descqa/shear_test.py | 101 +++++++++++++++++++++++++++++--------- 2 files changed, 82 insertions(+), 25 deletions(-) diff --git a/descqa/configs/shear.yaml b/descqa/configs/shear.yaml index 648e3b7c..0c7d9779 100644 --- a/descqa/configs/shear.yaml +++ b/descqa/configs/shear.yaml @@ -7,8 +7,8 @@ e2: 'shear_2' kappa: 'convergence' nbins: 20 min_sep: 2.5 -max_sep: 250 +max_sep: 200 sep_units: 'arcmin' bin_slop: 0.1 -zlo: 0.5 -zhi: 0.6 \ No newline at end of file +zlo: 0.3 +zhi: 0.9 diff --git a/descqa/shear_test.py b/descqa/shear_test.py index 8e1266c8..aac83e26 100644 --- a/descqa/shear_test.py +++ b/descqa/shear_test.py @@ -142,9 +142,9 @@ def theory_corr(self,n_z2, min_sep2, max_sep2, nbins2, lmax2): return xvals, vals[:,1], vals[:,2] - def get_chi2(self,measured, theory): - chi2 = (measured-theory)**2/theory - return chi2/float(len(measured)) + def get_score(self,measured, theory): + diff = (measured-theory)**2/theory**2 + return np.sqrt(np.sum(diff))/float(len(measured)) @staticmethod def get_catalog_data(gc, quantities, filters=None): @@ -168,8 +168,22 @@ def post_process_plot(self, ax): plt.xscale('log') ax[0].legend() ax[1].legend() - - + max_height1 = 80 + max_height2 = 15 + min_height1 = -5 + min_height2 = -1 + + #very rough DES-like limits (maximum and minimum scales) + ax[0].vlines(2.5,min_height1,max_height1,linestyles='--') + ax[0].vlines(200.,min_height1,max_height1,linestyles='--') + ax[1].vlines(35.,min_height2,max_height2,linestyles='--') + ax[1].vlines(200.,min_height2,max_height2,linestyles='--') + + plt.xlabel(r'$\theta \rm{(arcmin)}$') + ax[0].set_ylabel(r'$\chi_{+} (10^{-6}) $') + ax[1].set_ylabel(r'$\chi_{-} (10^{-6}) $') + ax[0].set_ylim([min_height1,max_height1]) + ax[1].set_ylim([min_height2,max_height2]) def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): @@ -202,26 +216,66 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): cat_s = treecorr.Catalog(ra=catalog_data[self.ra], dec=catalog_data[self.dec], g1=catalog_data[self.e1]-np.mean(catalog_data[self.e1]), g2=-(catalog_data[self.e2]-np.mean(catalog_data[self.e2])), ra_units='deg', dec_units='deg') - gg = treecorr.GGCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin', bin_slop=0.1, verbose=True) + gg = treecorr.GGCorrelation(nbins=self.nbins, min_sep=2.5, max_sep=250, sep_units='arcmin', bin_slop=0.1, verbose=True) gg.process(cat_s) - - chi2_dof_1= self.get_chi2(gg.xip, theory_plus) # correct this - + #base result + r = np.exp(gg.meanlogr) xip = gg.xip xim = gg.xim - - plt.figure() - plt.plot(xip) - plt.plot(theory_plus) - plt.show() + sig = np.sqrt(gg.varxi) + + + ####### code to do jack-knifing ################ + + #k-means clustering to define areas + from sklearn.cluster import k_means + import time + + a = time.time() + N_clust = 4 + nn = np.stack((catalog_data[self.ra],catalog_data[self.dec]),axis=1) + cents, labs, inert = k_means(n_clusters=N_clust,random_state=0,X=nn,n_jobs=-1) + print(time.time()-a) + print("done with k-means clustering") + + + # jack-knife code + xip_jack=[] + xim_jack=[] + gg = treecorr.GGCorrelation(nbins=self.nbins, min_sep=2.5, max_sep=250, sep_units='arcmin', bin_slop=0.1, verbose=True) + for i in range(N_clust): + cat_s = treecorr.Catalog(ra=catalog_data[self.ra][labs!=i], dec=catalog_data[self.dec][labs!=i], g1=catalog_data[self.e1][labs!=i]-np.mean(catalog_data[self.e1][labs!=i]), g2=-(catalog_data[self.e2][labs!=i]-np.mean(catalog_data[self.e2][labs!=i])), ra_units='deg', dec_units='deg') + gg.process(cat_s) + xip_jack.append(gg.xip) + xim_jack.append(gg.xim) + + - # directly compare xip and xim to theory_plus and theory_minus. Take into account cosmic variance. + ### assign covariance matrix - loop is poor python syntax but compared to the time taken for the rest of the test doesn't really matter + cp_xip = np.zeros((self.nbins,self.nbins)) + for i in range(self.nbins): + for j in range(self.nbins): + for k in range(N_clust): + cp_xip[i][j] += (N_clust-1.)/N_clust * (xip[i]-xip_jack[k][i])*(xip[j]-xip_jack[k][j]) + + + # Diagonal covariances for error bars on the plots. Use full covariance matrix for chi2 testing. + sig_jack = np.zeros((self.nbins)) + for i in range(self.nbins): + sig_jack[i] = np.sqrt(cp_xip[i][i]) + + ################################################# + + + + - + + chi2_dof_1= self.get_score(gg.xip, theory_plus) # correct this # further correlation functions @@ -231,7 +285,8 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): #treecorr.NKCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') # count-kappa (i.e. (R)) #treecorr.KKCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') # count-kappa (i.e. (R)) # etc... - + print(xip*1.e6) + print(sig*1.e6) #print(np.exp(gg.logr), gg.xip) #np.savez('output.npz', theta=np.exp(gg.logr), ggp=gg.xip, ggm=gg.xim, weight=gg.weight, npairs=gg.npairs) @@ -241,10 +296,12 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): fig, ax = plt.subplots(2,sharex=True) for ax_this in (ax, self.summary_ax): - ax_this[0].plot(xvals,xip,'x', label="w+ autocorrelation") - ax_this[0].plot(xvals,theory_plus,'x',color='g', label="w+ theory") - ax_this[1].plot(xvals,xim,'x',color='r', label="w- auto") - ax_this[1].plot(xvals,theory_minus,'x',color='y', label="w- theory") + #ax_this[0].plot(xvals,xip*1.e6,'o', color = "#3f9b0b",label=r'$\chi_{+}$') + ax_this[0].errorbar(xvals,xip*1.e6,sig_jack*1.e6,lw=0.3,marker='o',ls='',color = "#3f9b0b",label=r'$\chi_{+}$') + ax_this[0].plot(xvals,theory_plus*1.e6,'o',color="#9a0eea", label=r'$\chi_{+}$'+" theory") + #ax_this[1].plot(xvals,xim*1.e6,'o',color="#3f9b0b", label=r'$\chi_{-}$') + ax_this[1].errorbar(xvals,xim*1.e6,sig_jack*1.e6,lw=0.3,marker='o',ls='',color = "#3f9b0b",label=r'$\chi_{-}$') + ax_this[1].plot(xvals,theory_minus*1.e6,'o',color="#9a0eea", label=r'$\chi_{-}$'+" theory") @@ -252,7 +309,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): fig.savefig(os.path.join(output_dir, 'plot.png')) plt.close(fig) - score = data[0] #calculate your summary statistics + score = chi2_dof_1 #calculate your summary statistics return TestResult(score, passed=True) From 03f8e3f81d23ec4535f441dc342bef81b7ae8d0e Mon Sep 17 00:00:00 2001 From: Patricia Larsen Date: Wed, 10 Jan 2018 15:53:12 -0800 Subject: [PATCH 16/64] added (diagonal) chi2 based on jack-knife errors, and fail criteria --- descqa/configs/shear.yaml | 4 +-- descqa/shear_test.py | 74 ++++++++++++++++++++++++++------------- 2 files changed, 52 insertions(+), 26 deletions(-) diff --git a/descqa/configs/shear.yaml b/descqa/configs/shear.yaml index 0c7d9779..eb63ca9b 100644 --- a/descqa/configs/shear.yaml +++ b/descqa/configs/shear.yaml @@ -5,10 +5,10 @@ dec: 'dec' e1: 'shear_1' e2: 'shear_2' kappa: 'convergence' -nbins: 20 +nbins: 6 min_sep: 2.5 max_sep: 200 sep_units: 'arcmin' -bin_slop: 0.1 +bin_slop: 0.07 zlo: 0.3 zhi: 0.9 diff --git a/descqa/shear_test.py b/descqa/shear_test.py index aac83e26..98321b8a 100644 --- a/descqa/shear_test.py +++ b/descqa/shear_test.py @@ -132,19 +132,26 @@ def phi(self,lmax,n_z): return l, phi_array*prefactor - def theory_corr(self,n_z2, min_sep2, max_sep2, nbins2, lmax2): + def theory_corr(self,n_z2, xvals, lmax2): ll, pp = self.phi(lmax=lmax2, n_z=n_z2) pp3_2 = np.zeros((lmax2,4)) pp3_2[:,1]= pp[:]*(ll*(ll+1.))/(2.*np.pi) - xvals = np.logspace(np.log10(min_sep2), np.log10(max_sep2), nbins2) #in arcminutes + #xvals = np.logspace(np.log10(min_sep2), np.log10(max_sep2), nbins2) #in arcminutes cxvals = np.cos(xvals/(60.)/(180./np.pi)) vals = camb.correlations.cl2corr(pp3_2, cxvals) return xvals, vals[:,1], vals[:,2] - def get_score(self,measured, theory): - diff = (measured-theory)**2/theory**2 - return np.sqrt(np.sum(diff))/float(len(measured)) + def get_score(self,measured,theory,cov, opt='diagonal'): + if (opt=='cov'): + cov = np.matrix(cov).I + print("inverse covariance matrix") + print(cov) + chi2 = np.matrix(measured-theory)*cov*np.matrix(measured-theory).T + else: + chi2 =np.sum( [(measured[i]-theory[i])**2/cov[i][i] for i in range(len(measured))]) + diff = chi2/float(len(measured)) + return diff#np.sqrt(np.sum(diff))/float(len(measured)) @staticmethod def get_catalog_data(gc, quantities, filters=None): @@ -210,13 +217,11 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): return TestResult(skipped=True, summary='e2 values out of range [-1,+1]') - n_z = catalog_data[self.z] - xvals, theory_plus, theory_minus = self.theory_corr(n_z, self.min_sep, self.max_sep, self.nbins, 2000) cat_s = treecorr.Catalog(ra=catalog_data[self.ra], dec=catalog_data[self.dec], g1=catalog_data[self.e1]-np.mean(catalog_data[self.e1]), g2=-(catalog_data[self.e2]-np.mean(catalog_data[self.e2])), ra_units='deg', dec_units='deg') - gg = treecorr.GGCorrelation(nbins=self.nbins, min_sep=2.5, max_sep=250, sep_units='arcmin', bin_slop=0.1, verbose=True) + gg = treecorr.GGCorrelation(nbins=self.nbins, min_sep=self.min_sep, max_sep=self.max_sep, sep_units='arcmin', bin_slop=self.bin_slop, verbose=True) gg.process(cat_s) #base result @@ -234,7 +239,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): import time a = time.time() - N_clust = 4 + N_clust = 7 nn = np.stack((catalog_data[self.ra],catalog_data[self.dec]),axis=1) cents, labs, inert = k_means(n_clusters=N_clust,random_state=0,X=nn,n_jobs=-1) print(time.time()-a) @@ -244,15 +249,17 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): # jack-knife code xip_jack=[] xim_jack=[] - gg = treecorr.GGCorrelation(nbins=self.nbins, min_sep=2.5, max_sep=250, sep_units='arcmin', bin_slop=0.1, verbose=True) + gg = treecorr.GGCorrelation(nbins=self.nbins, min_sep=self.min_sep, max_sep=self.max_sep, sep_units='arcmin', bin_slop=self.bin_slop, verbose=True) for i in range(N_clust): cat_s = treecorr.Catalog(ra=catalog_data[self.ra][labs!=i], dec=catalog_data[self.dec][labs!=i], g1=catalog_data[self.e1][labs!=i]-np.mean(catalog_data[self.e1][labs!=i]), g2=-(catalog_data[self.e2][labs!=i]-np.mean(catalog_data[self.e2][labs!=i])), ra_units='deg', dec_units='deg') gg.process(cat_s) xip_jack.append(gg.xip) xim_jack.append(gg.xim) + print("xip_jack") + print(i) + print(gg.xip) - - + ### assign covariance matrix - loop is poor python syntax but compared to the time taken for the rest of the test doesn't really matter cp_xip = np.zeros((self.nbins,self.nbins)) for i in range(self.nbins): @@ -260,23 +267,39 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): for k in range(N_clust): cp_xip[i][j] += (N_clust-1.)/N_clust * (xip[i]-xip_jack[k][i])*(xip[j]-xip_jack[k][j]) + cp_xim = np.zeros((self.nbins,self.nbins)) + for i in range(self.nbins): + for j in range(self.nbins): + for k in range(N_clust): + cp_xim[i][j] += (N_clust-1.)/N_clust * (xim[i]-xim_jack[k][i])*(xim[j]-xim_jack[k][j]) + + # Diagonal covariances for error bars on the plots. Use full covariance matrix for chi2 testing. sig_jack = np.zeros((self.nbins)) + sigm_jack = np.zeros((self.nbins)) for i in range(self.nbins): sig_jack[i] = np.sqrt(cp_xip[i][i]) + sigm_jack[i] = np.sqrt(cp_xim[i][i]) ################################################# - - - + + n_z = catalog_data[self.z] + xvals, theory_plus, theory_minus = self.theory_corr(n_z, r , 10000) - - - - chi2_dof_1= self.get_score(gg.xip, theory_plus) # correct this - + #assuming r == xvals + chi2_dof_1= self.get_score(xip, theory_plus,cp_xip) # correct this + chi2_dof_2 = self.get_score(xim, theory_minus, cp_xim) + + print(xvals) + print(r) + print(chi2_dof_1) + print(chi2_dof_2) + print("chi2 values") + + print("CP_XIP:") + print(cp_xip) # further correlation functions #dd = treecorr.NNCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') @@ -286,7 +309,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): #treecorr.KKCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') # count-kappa (i.e. (R)) # etc... print(xip*1.e6) - print(sig*1.e6) + print(sig_jack*1.e6) #print(np.exp(gg.logr), gg.xip) #np.savez('output.npz', theta=np.exp(gg.logr), ggp=gg.xip, ggm=gg.xim, weight=gg.weight, npairs=gg.npairs) @@ -297,10 +320,10 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): for ax_this in (ax, self.summary_ax): #ax_this[0].plot(xvals,xip*1.e6,'o', color = "#3f9b0b",label=r'$\chi_{+}$') - ax_this[0].errorbar(xvals,xip*1.e6,sig_jack*1.e6,lw=0.3,marker='o',ls='',color = "#3f9b0b",label=r'$\chi_{+}$') + ax_this[0].errorbar(r,xip*1.e6,sig_jack*1.e6,lw=0.6,marker='o',ls='',color = "#3f9b0b",label=r'$\chi_{+}$') ax_this[0].plot(xvals,theory_plus*1.e6,'o',color="#9a0eea", label=r'$\chi_{+}$'+" theory") #ax_this[1].plot(xvals,xim*1.e6,'o',color="#3f9b0b", label=r'$\chi_{-}$') - ax_this[1].errorbar(xvals,xim*1.e6,sig_jack*1.e6,lw=0.3,marker='o',ls='',color = "#3f9b0b",label=r'$\chi_{-}$') + ax_this[1].errorbar(r,xim*1.e6,sigm_jack*1.e6,lw=0.6,marker='o',ls='',color = "#3f9b0b",label=r'$\chi_{-}$') ax_this[1].plot(xvals,theory_minus*1.e6,'o',color="#9a0eea", label=r'$\chi_{-}$'+" theory") @@ -310,7 +333,10 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): plt.close(fig) score = chi2_dof_1 #calculate your summary statistics - return TestResult(score, passed=True) + if (score<2): + return TestResult(score, passed=True) + else: + return TestResult(score, passed=False) From 5114898c9336bb00368fa89611853edeba90b07d Mon Sep 17 00:00:00 2001 From: Vinu Vikraman Date: Tue, 20 Feb 2018 16:58:40 -0800 Subject: [PATCH 17/64] Added SizeStellarMassLuminosityBulgeDisk.py and corresponding config, data files --- descqa/SizeStellarMassLuminosityBulgeDisk.py | 207 ++++++++++++++++++ descqa/configs/size_COSMOS.yaml | 52 +++++ .../sizes_appmag_Mandelbaum2014.txt | 67 ++++++ 3 files changed, 326 insertions(+) create mode 100644 descqa/SizeStellarMassLuminosityBulgeDisk.py create mode 100644 descqa/configs/size_COSMOS.yaml create mode 100644 descqa/data/size_sm_lum/sizes_appmag_Mandelbaum2014.txt diff --git a/descqa/SizeStellarMassLuminosityBulgeDisk.py b/descqa/SizeStellarMassLuminosityBulgeDisk.py new file mode 100644 index 00000000..b7ed7184 --- /dev/null +++ b/descqa/SizeStellarMassLuminosityBulgeDisk.py @@ -0,0 +1,207 @@ +from __future__ import print_function, division, unicode_literals, absolute_import +import os +import numpy as np +from GCR import GCRQuery +from scipy import interpolate +from scipy.stats import binned_statistic + +from .base import BaseValidationTest, TestResult +from .plotting import plt +from .utils import generate_uniform_random_ra_dec_footprint, get_healpixel_footprint, generate_uniform_random_dist + + +__all__ = ['SizeStellarMassLuminosityBulgeDisk'] + + +def redshift2dist(cosmology): + z = np.arange(0, 5.1, 0.5) + comov_d = cosmology.comoving_distance(z).to('kpc').value + spl = interpolate.splrep(z, comov_d) + return spl + + +class SizeStellarMassLuminosityBulgeDisk(BaseValidationTest): + """ + Validation test of 2pt correlation function + """ + _ARCSEC_TO_RADIAN = np.pi / 180. / 3600. + + def __init__(self, **kwargs): + self.simulated_catalog = kwargs['simulated_catalog'] + if self.simulated_catalog == 'protodc': + self.possible_mag_fields = kwargs['possible_appmag_fields'] + self.data_label = kwargs['protodc_data_label'] + self.output_filename_template = kwargs['protodc_output_filename_template'] + self.mag_bin_separation = kwargs['mag_bin_separation'] + self.fig_xlabel = kwargs['protodc_fig_xlabel'] + self.fig_ylabel = kwargs['protodc_fig_ylabel'] + + validation_filepath = os.path.join(self.data_dir, kwargs['protodc_data_filename']) + self.validation_data = np.genfromtxt(validation_filepath) + elif self.simulated_catalog == 'buzzard': + self.possible_mag_fields = kwargs['possible_AbsMag_fields'] + self.data_label = kwargs['buzzard_data_label'] + self.output_filename_template = kwargs['buzzard_output_filename_template'] + self.fig_xlabel = kwargs['buzzard_fig_xlabel'] + self.fig_ylabel = kwargs['buzzard_fig_ylabel'] + + validation_filepath = os.path.join(self.data_dir, kwargs['buzzard_data_filename']) + self.validation_data = np.genfromtxt(validation_filepath) + self.test_name = kwargs['test_name'] + self.z_bins = kwargs['z_bins'] + self.label_template = kwargs['label_template'] + + @staticmethod + def ConvertAbsMagLuminosity(AbsM, band): + '''AbsM: absolute magnitude, band: filter''' + if type(AbsM) is list or type(AbsM) is np.ndarray: + AbsM = np.array(AbsM) + + bands = {'U':5.61, 'B':5.48, 'V':4.83, 'R':4.42, 'I':4.08, + 'J':3.64, 'H':3.32, 'K':3.28, 'g':5.33, 'r':4.67, + 'i':4.48, 'z':4.42, 'F300W':6.09, 'F450W':5.32, 'F555W':4.85, + 'F606W':4.66, 'F702W':4.32, 'F814W':4.15, 'CFHT_U':5.57, + 'CFHT_B':5.49, 'CFHT_V':4.81, 'CFHT_R':4.44, 'CFHT_I':4.06, + 'NIRI_J':3.64, 'NIRI_H':3.33, 'NIRI_K':3.29} + + if band in bands.keys(): + AbsSun = bands[band] + else: + raise ValueError('Filter not implemented') + + logL = (AbsSun - AbsM) / 2.5 #unit of sun + L = 10**logL + return L, logL + + @staticmethod + def ConvertLuminosityAppMag(L, dl): + '''L: luminosity in a band, dl: luminosity distance''' + + AbsM = -2.5 * np.log10(L) + AppM = AbsM - 5 + 5 * np.log10(dl * 1e6) + return AbsM, AppM + + def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): + ''' + Loop over magnitude cuts and make plots + ''' + if self.simulated_catalog == 'protodc': + # load catalog data + default_mag_bin_edges = np.arange(17, 32, self.mag_bin_separation) + default_mag_bins = (default_mag_bin_edges[1:] + default_mag_bin_edges[:-1]) / 2. + colnames = dict() + colnames['z'] = 'redshift' + colnames['size_bulge'] = 'size_bulge_true' + colnames['size_disk'] = 'size_disk_true' + colnames['mag'] = catalog_instance.first_available(*self.possible_mag_fields) + if not all(v for v in colnames.values()): + return TestResult(skipped=True, summary='Missing requested quantities') + #Check whether the columns are finite or not + filters = [(np.isfinite, c) for c in colnames.values()] + + #Select objects within maximum and minimum redshift of all the bins + filters.extend(( + '{} < {}'.format(colnames['z'], max(z_bin['z_max'] for z_bin in self.z_bins)), + '{} >= {}'.format(colnames['z'], min(z_bin['z_min'] for z_bin in self.z_bins)), + )) + catalog_data = catalog_instance.get_quantities(list(colnames.values()), filters=filters) + catalog_data = {k: catalog_data[v] for k, v in colnames.items()} + elif self.simulated_catalog == 'buzzard': + # load catalog data + default_L_bin_edges = np.array([9, 9.5, 10, 10.5, 11, 11.5]) + default_L_bins = (default_L_bin_edges[1:] + default_L_bin_edges[:-1]) / 2. + spl = redshift2dist(catalog_instance.cosmology) + + colnames = dict() + #colnames['z'] = catalog_instance.first_available('redshift', 'redshift_true') + #colnames['size'] = catalog_instance.first_available('size') + colnames['z'] = 'redshift' + colnames['size'] = 'size' + colnames['mag'] = catalog_instance.first_available(*self.possible_mag_fields) + if not all(v for v in colnames.values()): + return TestResult(skipped=True, summary='Missing requested quantities') + #Check whether the columns are finite or not + filters = [(np.isfinite, c) for c in colnames.values()] + + #Select objects within maximum and minimum redshift of all the bins + filters.extend(( + '{} < {}'.format(colnames['z'], max(z_bin['z_max'] for z_bin in self.z_bins)), + '{} >= {}'.format(colnames['z'], min(z_bin['z_min'] for z_bin in self.z_bins)), + )) + catalog_data = catalog_instance.get_quantities(list(colnames.values()), filters=filters) + catalog_data = {k: catalog_data[v] for k, v in colnames.items()} + + fig, axes = plt.subplots(1, 3, figsize=(9, 3), sharex=True, sharey=True) + colors = ['r', 'b'] + try: + col = 0 + for z_bin in self.z_bins: + ax = axes[col] + # filter catalog data for this bin + filters = [ + 'z < {}'.format(z_bin['z_max']), + 'z >= {}'.format(z_bin['z_min']), + ] + + z_mean = (z_bin['z_max'] + z_bin['z_min']) / 2. + output_filepath = os.path.join(output_dir, self.output_filename_template.format(z_bin['z_min'], z_bin['z_max'])) + catalog_data_this = GCRQuery(*filters).filter(catalog_data) + if len(catalog_data_this['z']) == 0: + continue + if self.simulated_catalog == 'protodc': + binned_bulgesize_arcsec, tmp_1, tmp_2 = binned_statistic(catalog_data_this['mag'], catalog_data_this['size_bulge'], bins=default_mag_bin_edges, statistic='mean') + binned_bulgesize_arcsec_err, tmp_1, tmp_2 = binned_statistic(catalog_data_this['mag'], catalog_data_this['size_bulge'], bins=default_mag_bin_edges, statistic='std') + binned_disksize_arcsec, tmp_1, tmp_2 = binned_statistic(catalog_data_this['mag'], catalog_data_this['size_disk'], bins=default_mag_bin_edges, statistic='mean') + binned_disksize_arcsec_err, tmp_1, tmp_2 = binned_statistic(catalog_data_this['mag'], catalog_data_this['size_disk'], bins=default_mag_bin_edges, statistic='std') + binned_bulgesize_arcsec = np.nan_to_num(binned_bulgesize_arcsec) + binned_bulgesize_arcsec_err = np.nan_to_num(binned_bulgesize_arcsec_err) + binned_disksize_arcsec = np.nan_to_num(binned_disksize_arcsec) + binned_disksize_arcsec_err = np.nan_to_num(binned_disksize_arcsec_err) + np.savetxt(output_filepath, np.transpose((default_mag_bins, binned_bulgesize_arcsec, binned_bulgesize_arcsec_err, binned_disksize_arcsec, binned_disksize_arcsec_err))) + + validation_this = self.validation_data[(self.validation_data[:,0] < z_mean + 0.25) & (self.validation_data[:,0] > z_mean - 0.25)] + + #ax.set_title(self.label_template.format(z_bin['z_min'], z_bin['z_max'])) + ax.text(23, 1, self.label_template.format(z_bin['z_min'], z_bin['z_max'])) + ax.semilogy(validation_this[:,1], validation_this[:, 2], label='Bulge', color=colors[0]) + ax.fill_between(validation_this[:,1], validation_this[:, 2] + validation_this[:,4], validation_this[:, 2] - validation_this[:,4], lw=0, alpha=0.2, facecolor=colors[0]) + ax.semilogy(validation_this[:,1] + 0.5, validation_this[:, 3], label='Disk', color=colors[1]) + ax.fill_between(validation_this[:,1] + 0.5, validation_this[:, 3] + validation_this[:,5], validation_this[:, 3] - validation_this[:,5], lw=0, alpha=0.2, facecolor=colors[1]) + + ax.errorbar(default_mag_bins, binned_bulgesize_arcsec, binned_bulgesize_arcsec_err, marker='o', ls='', c=colors[0]) + ax.errorbar(default_mag_bins+0.5, binned_disksize_arcsec, binned_disksize_arcsec_err, marker='o', ls='', c=colors[1]) + ax.set_xlim([17, 29]) + ax.set_ylim([1e-2, 1e1]) + ax.set_yscale('log', nonposy='clip') + elif self.simulated_catalog == 'buzzard': + size_kpc = catalog_data_this['size'] * self._ARCSEC_TO_RADIAN * interpolate.splev(catalog_data_this['z'], spl) / (1 + catalog_data_this['z']) + L_G, logL_G = self.ConvertAbsMagLuminosity(catalog_data_this['mag'], 'g') + binned_size_kpc, tmp_1, tmp_2 = binned_statistic(logL_G, size_kpc, bins=default_L_bin_edges, statistic='mean') + binned_size_kpc_err, tmp_1, tmp_2 = binned_statistic(logL_G, size_kpc, bins=default_L_bin_edges, statistic='std') + np.savetxt(output_filepath, np.transpose((default_L_bins, binned_size_kpc, binned_size_kpc_err))) + + validation_this = self.validation_data[(self.validation_data[:,0] < z_mean + 0.02) & (self.validation_data[:,0] > z_mean - 0.02)] + + ax.semilogy(validation_this[:,1], 10**validation_this[:, 2], label=self.label_template.format(z_bin['z_min'], z_bin['z_max'])) + ax.fill_between(validation_this[:,1], 10**validation_this[:,3], 10**validation_this[:,4], lw=0, alpha=0.2) + ax.errorbar(default_L_bins, binned_size_kpc, binned_size_kpc_err, marker='o', ls='') + ax.set_yscale('log', nonposy='clip') + del catalog_data_this + col += 1 + ax.legend(loc='best') + + fig.add_subplot(111, frameon=False) + # hide tick and tick label of the big axes + plt.tick_params(labelcolor='none', which='both', top='off', bottom='off', left='off', right='off') + plt.grid(False) + plt.xlabel(self.fig_xlabel) + plt.ylabel(self.fig_ylabel) + plt.ylabel(self.fig_ylabel) + fig.subplots_adjust(hspace=0, wspace=0) + fig.suptitle('{} vs. {}'.format(catalog_name, self.data_label), fontsize='medium', y=0.93) + finally: + fig.savefig(os.path.join(output_dir, '{:s}.png'.format(self.test_name)), bbox_inches='tight') + plt.close(fig) + + #TODO: calculate summary statistics + return TestResult(0, passed=True) diff --git a/descqa/configs/size_COSMOS.yaml b/descqa/configs/size_COSMOS.yaml new file mode 100644 index 00000000..30f450e5 --- /dev/null +++ b/descqa/configs/size_COSMOS.yaml @@ -0,0 +1,52 @@ +subclass_name: SizeStellarMassLuminosityBulgeDisk.SizeStellarMassLuminosityBulgeDisk + +#observation: either protodc or buzzard +#simulated_catalog: protodc +simulated_catalog: buzzard + +possible_appmag_fields: + - mag_i_lsst + - mag_i_sdss + - mag_i_des + +possible_AbsMag_fields: + - Mag_true_i_lsst_z01 + - Mag_true_i_sdss_z01 + - Mag_true_i_des_z01 + +size_fields: + - size_bulge_true + - size_disk_true + +size_field: size_true + +mag_bin_separation: 1 + +protodc_output_filename_template: 'size_lum_COSMOS_z_{}_{}.dat' +buzzard_output_filename_template: 'size_lum_van_z_{}_{}.dat' + +protodc_output_figname: 'size_lum_protodc' +buzzrad_output_figname: 'size_lum_buzzard' + +label_template: '${} < z < {}$' + +protodc_data_filename: 'size_sm_lum/sizes_appmag_Mandelbaum2014.txt' +buzzard_data_filename: 'size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_VLuminosity.txt' +protodc_data_label: 'Mandelbaum+2014: COSMOSIS comparing mean and scatter' +buzzard_data_label: 'van der Wel+2014: comparing mean and scatter' + +z_bins: + - {z_min: 0.0, z_max: 0.5} + - {z_min: 0.5, z_max: 1.0} + - {z_min: 1.0, z_max: 1.5} + +protodc_fig_xlabel: '$i$ mag' +protodc_fig_ylabel: '$\log_{10}(R_e)$ (arcsec)' +buzzard_fig_xlabel: '$\log_{10}(L/L_{\odot})$ (V-band)' +buzzard_fig_ylabel: '$\log_{10}(R_e)$ (kpc)' + + +protodc_description: | + Compare evolution of bulge and disk sizes as a function of i-band magnitude of LSST and redshift and comparing to Mandelbaum et al (2015) HST COSMOS observations +buzzard_description: | + Compare evolution of size as a function of luminosity and redshift and van der Wel et al (2014) HST-3D and CANDELS observations diff --git a/descqa/data/size_sm_lum/sizes_appmag_Mandelbaum2014.txt b/descqa/data/size_sm_lum/sizes_appmag_Mandelbaum2014.txt new file mode 100644 index 00000000..287f64d6 --- /dev/null +++ b/descqa/data/size_sm_lum/sizes_appmag_Mandelbaum2014.txt @@ -0,0 +1,67 @@ +# zavg avg_mag avg_bhlr avg_dhlr std_bhlr std_dhlr +0.250 19.166 0.470 1.415 0.784 2.360 +0.250 20.527 0.331 0.830 0.719 1.383 +0.250 21.525 0.253 0.674 0.437 1.107 +0.250 22.525 0.197 0.480 0.377 0.680 +0.250 23.535 0.131 0.316 0.184 0.231 +0.250 24.544 0.086 0.218 0.087 0.127 +0.250 25.184 0.062 0.163 0.045 0.083 +0.250 26.326 0.051 0.121 0.029 0.087 +0.250 27.184 0.021 0.034 0.012 0.012 +0.250 28.425 0.015 0.017 0.000 0.000 +0.250 0.000 0.000 0.000 0.000 0.000 +0.750 19.578 0.803 5.471 1.451 7.125 +0.750 20.622 0.494 1.460 1.404 2.589 +0.750 21.571 0.239 0.717 0.451 1.099 +0.750 22.539 0.163 0.459 0.266 0.621 +0.750 23.529 0.117 0.311 0.148 0.356 +0.750 24.513 0.082 0.211 0.086 0.131 +0.750 25.162 0.062 0.160 0.045 0.080 +0.750 26.354 0.049 0.112 0.030 0.087 +0.750 27.278 0.038 0.058 0.024 0.027 +0.750 28.599 0.017 0.033 0.008 0.010 +0.750 0.000 0.000 0.000 0.000 0.000 +1.250 19.331 2.454 16.261 3.245 4.160 +1.250 20.600 3.066 8.753 5.486 4.035 +1.250 21.642 0.781 2.754 1.375 3.514 +1.250 22.634 0.279 0.800 0.475 1.186 +1.250 23.578 0.154 0.399 0.202 0.352 +1.250 24.522 0.092 0.233 0.094 0.149 +1.250 25.187 0.066 0.162 0.051 0.086 +1.250 26.375 0.054 0.101 0.036 0.048 +1.250 27.173 0.068 0.111 0.038 0.032 +1.250 28.386 0.033 0.036 0.019 0.014 +1.250 0.000 0.000 0.000 0.000 0.000 +1.750 19.279 1.885 17.609 1.406 3.657 +1.750 20.565 1.967 9.276 3.468 3.589 +1.750 21.559 1.922 6.244 2.735 4.261 +1.750 22.597 0.786 2.129 1.179 2.057 +1.750 23.634 0.208 0.559 0.297 0.843 +1.750 24.578 0.098 0.237 0.099 0.174 +1.750 25.199 0.066 0.156 0.049 0.087 +1.750 26.281 0.050 0.096 0.034 0.052 +1.750 27.165 0.016 0.020 0.005 0.002 +1.750 0.000 0.000 0.000 0.000 0.000 +1.750 0.000 0.000 0.000 0.000 0.000 +2.250 19.611 1.906 16.778 1.410 3.865 +2.250 20.512 4.666 11.320 5.912 5.261 +2.250 21.462 3.037 6.158 4.928 3.367 +2.250 22.598 0.746 1.954 1.198 1.531 +2.250 23.676 0.160 0.450 0.206 0.534 +2.250 24.582 0.094 0.223 0.096 0.166 +2.250 25.193 0.065 0.156 0.047 0.095 +2.250 26.347 0.060 0.102 0.041 0.044 +2.250 27.574 0.013 0.021 0.002 0.001 +2.250 28.228 0.013 0.014 0.000 0.000 +2.250 0.000 0.000 0.000 0.000 0.000 +2.750 19.119 5.411 12.492 6.256 1.315 +2.750 20.510 3.143 8.794 4.153 2.749 +2.750 21.558 1.926 6.074 2.749 4.172 +2.750 22.584 1.304 2.870 1.834 2.626 +2.750 23.690 0.201 0.494 0.310 0.546 +2.750 24.604 0.093 0.214 0.101 0.162 +2.750 25.195 0.065 0.148 0.053 0.087 +2.750 26.382 0.046 0.082 0.036 0.043 +2.750 27.245 0.029 0.119 0.011 0.135 +2.750 0.000 0.000 0.000 0.000 0.000 +2.750 0.000 0.000 0.000 0.000 0.000 From 2617aaa423b9935ede54ebb8babdea7a3f398444 Mon Sep 17 00:00:00 2001 From: Vinu Vikraman Date: Tue, 20 Feb 2018 17:59:06 -0800 Subject: [PATCH 18/64] Minor changes to SizeStellarMassLuminosityBulgeDisk.py configs/size_COSMOS.yaml --- descqa/SizeStellarMassLuminosityBulgeDisk.py | 58 ++++---------------- descqa/configs/size_COSMOS.yaml | 2 +- 2 files changed, 11 insertions(+), 49 deletions(-) diff --git a/descqa/SizeStellarMassLuminosityBulgeDisk.py b/descqa/SizeStellarMassLuminosityBulgeDisk.py index b7ed7184..63f75930 100644 --- a/descqa/SizeStellarMassLuminosityBulgeDisk.py +++ b/descqa/SizeStellarMassLuminosityBulgeDisk.py @@ -13,13 +13,6 @@ __all__ = ['SizeStellarMassLuminosityBulgeDisk'] -def redshift2dist(cosmology): - z = np.arange(0, 5.1, 0.5) - comov_d = cosmology.comoving_distance(z).to('kpc').value - spl = interpolate.splrep(z, comov_d) - return spl - - class SizeStellarMassLuminosityBulgeDisk(BaseValidationTest): """ Validation test of 2pt correlation function @@ -49,37 +42,8 @@ def __init__(self, **kwargs): self.validation_data = np.genfromtxt(validation_filepath) self.test_name = kwargs['test_name'] self.z_bins = kwargs['z_bins'] - self.label_template = kwargs['label_template'] - - @staticmethod - def ConvertAbsMagLuminosity(AbsM, band): - '''AbsM: absolute magnitude, band: filter''' - if type(AbsM) is list or type(AbsM) is np.ndarray: - AbsM = np.array(AbsM) - - bands = {'U':5.61, 'B':5.48, 'V':4.83, 'R':4.42, 'I':4.08, - 'J':3.64, 'H':3.32, 'K':3.28, 'g':5.33, 'r':4.67, - 'i':4.48, 'z':4.42, 'F300W':6.09, 'F450W':5.32, 'F555W':4.85, - 'F606W':4.66, 'F702W':4.32, 'F814W':4.15, 'CFHT_U':5.57, - 'CFHT_B':5.49, 'CFHT_V':4.81, 'CFHT_R':4.44, 'CFHT_I':4.06, - 'NIRI_J':3.64, 'NIRI_H':3.33, 'NIRI_K':3.29} - - if band in bands.keys(): - AbsSun = bands[band] - else: - raise ValueError('Filter not implemented') - - logL = (AbsSun - AbsM) / 2.5 #unit of sun - L = 10**logL - return L, logL - - @staticmethod - def ConvertLuminosityAppMag(L, dl): - '''L: luminosity in a band, dl: luminosity distance''' - - AbsM = -2.5 * np.log10(L) - AppM = AbsM - 5 + 5 * np.log10(dl * 1e6) - return AbsM, AppM + self.text_template = kwargs['text_template'] + self.colors = ['r', 'b'] def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): ''' @@ -132,7 +96,6 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): catalog_data = {k: catalog_data[v] for k, v in colnames.items()} fig, axes = plt.subplots(1, 3, figsize=(9, 3), sharex=True, sharey=True) - colors = ['r', 'b'] try: col = 0 for z_bin in self.z_bins: @@ -161,15 +124,14 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): validation_this = self.validation_data[(self.validation_data[:,0] < z_mean + 0.25) & (self.validation_data[:,0] > z_mean - 0.25)] - #ax.set_title(self.label_template.format(z_bin['z_min'], z_bin['z_max'])) - ax.text(23, 1, self.label_template.format(z_bin['z_min'], z_bin['z_max'])) - ax.semilogy(validation_this[:,1], validation_this[:, 2], label='Bulge', color=colors[0]) - ax.fill_between(validation_this[:,1], validation_this[:, 2] + validation_this[:,4], validation_this[:, 2] - validation_this[:,4], lw=0, alpha=0.2, facecolor=colors[0]) - ax.semilogy(validation_this[:,1] + 0.5, validation_this[:, 3], label='Disk', color=colors[1]) - ax.fill_between(validation_this[:,1] + 0.5, validation_this[:, 3] + validation_this[:,5], validation_this[:, 3] - validation_this[:,5], lw=0, alpha=0.2, facecolor=colors[1]) + ax.text(23, 1, self.text_template.format(z_bin['z_min'], z_bin['z_max'])) + ax.semilogy(validation_this[:,1], validation_this[:, 2], label='Bulge', color=self.colors[0]) + ax.fill_between(validation_this[:,1], validation_this[:, 2] + validation_this[:,4], validation_this[:, 2] - validation_this[:,4], lw=0, alpha=0.2, facecolor=self.colors[0]) + ax.semilogy(validation_this[:,1] + 0.5, validation_this[:, 3], label='Disk', color=self.colors[1]) + ax.fill_between(validation_this[:,1] + 0.5, validation_this[:, 3] + validation_this[:,5], validation_this[:, 3] - validation_this[:,5], lw=0, alpha=0.2, facecolor=self.colors[1]) - ax.errorbar(default_mag_bins, binned_bulgesize_arcsec, binned_bulgesize_arcsec_err, marker='o', ls='', c=colors[0]) - ax.errorbar(default_mag_bins+0.5, binned_disksize_arcsec, binned_disksize_arcsec_err, marker='o', ls='', c=colors[1]) + ax.errorbar(default_mag_bins, binned_bulgesize_arcsec, binned_bulgesize_arcsec_err, marker='o', ls='', c=self.colors[0]) + ax.errorbar(default_mag_bins+0.5, binned_disksize_arcsec, binned_disksize_arcsec_err, marker='o', ls='', c=self.colors[1]) ax.set_xlim([17, 29]) ax.set_ylim([1e-2, 1e1]) ax.set_yscale('log', nonposy='clip') @@ -182,7 +144,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): validation_this = self.validation_data[(self.validation_data[:,0] < z_mean + 0.02) & (self.validation_data[:,0] > z_mean - 0.02)] - ax.semilogy(validation_this[:,1], 10**validation_this[:, 2], label=self.label_template.format(z_bin['z_min'], z_bin['z_max'])) + ax.semilogy(validation_this[:,1], 10**validation_this[:, 2], label=self.text_template.format(z_bin['z_min'], z_bin['z_max'])) ax.fill_between(validation_this[:,1], 10**validation_this[:,3], 10**validation_this[:,4], lw=0, alpha=0.2) ax.errorbar(default_L_bins, binned_size_kpc, binned_size_kpc_err, marker='o', ls='') ax.set_yscale('log', nonposy='clip') diff --git a/descqa/configs/size_COSMOS.yaml b/descqa/configs/size_COSMOS.yaml index 30f450e5..b76a6e74 100644 --- a/descqa/configs/size_COSMOS.yaml +++ b/descqa/configs/size_COSMOS.yaml @@ -28,7 +28,7 @@ buzzard_output_filename_template: 'size_lum_van_z_{}_{}.dat' protodc_output_figname: 'size_lum_protodc' buzzrad_output_figname: 'size_lum_buzzard' -label_template: '${} < z < {}$' +text_template: '${} < z < {}$' protodc_data_filename: 'size_sm_lum/sizes_appmag_Mandelbaum2014.txt' buzzard_data_filename: 'size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_VLuminosity.txt' From 41af3dac039b3ab5a57c16f61aaebb46d4b9157e Mon Sep 17 00:00:00 2001 From: Vinu Vikraman Date: Tue, 20 Feb 2018 20:56:17 -0800 Subject: [PATCH 19/64] Renamed SizeStellarMassLuminosityBulgeDisk.py to SizeAppmagBulgeDisk.py and configs/size_COSMOS.yaml to configs/size_bulge_disk_COSMOS.yaml to distingush from the weighted size to the bulge, disk sizes --- ...zeStellarMassLuminosityBulgeDisk.py => SizeAppmagBulgeDisk.py} | 0 descqa/configs/{size_COSMOS.yaml => size_bulge_disk_COSMOS.yaml} | 0 ...delbaum2014.txt => sizes_appmag_bulge_disk_Mandelbaum2014.txt} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename descqa/{SizeStellarMassLuminosityBulgeDisk.py => SizeAppmagBulgeDisk.py} (100%) rename descqa/configs/{size_COSMOS.yaml => size_bulge_disk_COSMOS.yaml} (100%) rename descqa/data/size_sm_lum/{sizes_appmag_Mandelbaum2014.txt => sizes_appmag_bulge_disk_Mandelbaum2014.txt} (100%) diff --git a/descqa/SizeStellarMassLuminosityBulgeDisk.py b/descqa/SizeAppmagBulgeDisk.py similarity index 100% rename from descqa/SizeStellarMassLuminosityBulgeDisk.py rename to descqa/SizeAppmagBulgeDisk.py diff --git a/descqa/configs/size_COSMOS.yaml b/descqa/configs/size_bulge_disk_COSMOS.yaml similarity index 100% rename from descqa/configs/size_COSMOS.yaml rename to descqa/configs/size_bulge_disk_COSMOS.yaml diff --git a/descqa/data/size_sm_lum/sizes_appmag_Mandelbaum2014.txt b/descqa/data/size_sm_lum/sizes_appmag_bulge_disk_Mandelbaum2014.txt similarity index 100% rename from descqa/data/size_sm_lum/sizes_appmag_Mandelbaum2014.txt rename to descqa/data/size_sm_lum/sizes_appmag_bulge_disk_Mandelbaum2014.txt From 8d2068228c245288b5a3c82a1666a58c5e100073 Mon Sep 17 00:00:00 2001 From: Vinu Vikraman Date: Tue, 20 Feb 2018 20:57:47 -0800 Subject: [PATCH 20/64] Changed the function accordingly in SizeAppmagBulgeDisk.py and configs/size_bulge_disk_COSMOS.yaml --- descqa/SizeAppmagBulgeDisk.py | 4 ++-- descqa/configs/size_bulge_disk_COSMOS.yaml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/descqa/SizeAppmagBulgeDisk.py b/descqa/SizeAppmagBulgeDisk.py index 63f75930..5db3c6c5 100644 --- a/descqa/SizeAppmagBulgeDisk.py +++ b/descqa/SizeAppmagBulgeDisk.py @@ -10,10 +10,10 @@ from .utils import generate_uniform_random_ra_dec_footprint, get_healpixel_footprint, generate_uniform_random_dist -__all__ = ['SizeStellarMassLuminosityBulgeDisk'] +__all__ = ['SizeAppmagBulgeDisk'] -class SizeStellarMassLuminosityBulgeDisk(BaseValidationTest): +class SizeAppmagBulgeDisk(BaseValidationTest): """ Validation test of 2pt correlation function """ diff --git a/descqa/configs/size_bulge_disk_COSMOS.yaml b/descqa/configs/size_bulge_disk_COSMOS.yaml index b76a6e74..7a7b36b8 100644 --- a/descqa/configs/size_bulge_disk_COSMOS.yaml +++ b/descqa/configs/size_bulge_disk_COSMOS.yaml @@ -1,8 +1,8 @@ -subclass_name: SizeStellarMassLuminosityBulgeDisk.SizeStellarMassLuminosityBulgeDisk +subclass_name: SizeAppmagBulgeDisk.SizeAppmagBulgeDisk #observation: either protodc or buzzard -#simulated_catalog: protodc -simulated_catalog: buzzard +simulated_catalog: protodc +#simulated_catalog: buzzard possible_appmag_fields: - mag_i_lsst @@ -30,7 +30,7 @@ buzzrad_output_figname: 'size_lum_buzzard' text_template: '${} < z < {}$' -protodc_data_filename: 'size_sm_lum/sizes_appmag_Mandelbaum2014.txt' +protodc_data_filename: 'size_sm_lum/sizes_appmag_bulge_disk_Mandelbaum2014.txt' buzzard_data_filename: 'size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_VLuminosity.txt' protodc_data_label: 'Mandelbaum+2014: COSMOSIS comparing mean and scatter' buzzard_data_label: 'van der Wel+2014: comparing mean and scatter' From 40933b17a581b3ebb10cf44e286f7c338670a609 Mon Sep 17 00:00:00 2001 From: Vinu Vikraman Date: Tue, 20 Feb 2018 21:16:06 -0800 Subject: [PATCH 21/64] Changed data/size_sm_lum to data/size to generalize size of many observation --- .../sizes_appmag_bulge_disk_Mandelbaum2014.txt | 0 .../van_der_Wel_2014_ApJ_788_28_Table5_StellarMass.txt | 0 .../van_der_Wel_2014_ApJ_788_28_Table5_VLuminosity.txt | 0 .../van_der_Wel_2014_ApJ_788_28_Table5_dict.txt | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename descqa/data/{size_sm_lum => size}/sizes_appmag_bulge_disk_Mandelbaum2014.txt (100%) rename descqa/data/{size_sm_lum => size}/van_der_Wel_2014_ApJ_788_28_Table5_StellarMass.txt (100%) rename descqa/data/{size_sm_lum => size}/van_der_Wel_2014_ApJ_788_28_Table5_VLuminosity.txt (100%) rename descqa/data/{size_sm_lum => size}/van_der_Wel_2014_ApJ_788_28_Table5_dict.txt (100%) diff --git a/descqa/data/size_sm_lum/sizes_appmag_bulge_disk_Mandelbaum2014.txt b/descqa/data/size/sizes_appmag_bulge_disk_Mandelbaum2014.txt similarity index 100% rename from descqa/data/size_sm_lum/sizes_appmag_bulge_disk_Mandelbaum2014.txt rename to descqa/data/size/sizes_appmag_bulge_disk_Mandelbaum2014.txt diff --git a/descqa/data/size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_StellarMass.txt b/descqa/data/size/van_der_Wel_2014_ApJ_788_28_Table5_StellarMass.txt similarity index 100% rename from descqa/data/size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_StellarMass.txt rename to descqa/data/size/van_der_Wel_2014_ApJ_788_28_Table5_StellarMass.txt diff --git a/descqa/data/size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_VLuminosity.txt b/descqa/data/size/van_der_Wel_2014_ApJ_788_28_Table5_VLuminosity.txt similarity index 100% rename from descqa/data/size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_VLuminosity.txt rename to descqa/data/size/van_der_Wel_2014_ApJ_788_28_Table5_VLuminosity.txt diff --git a/descqa/data/size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_dict.txt b/descqa/data/size/van_der_Wel_2014_ApJ_788_28_Table5_dict.txt similarity index 100% rename from descqa/data/size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_dict.txt rename to descqa/data/size/van_der_Wel_2014_ApJ_788_28_Table5_dict.txt From bbe26c0c466d423691883eabd8d30651735bb5ee Mon Sep 17 00:00:00 2001 From: Vinu Vikraman Date: Wed, 21 Feb 2018 13:27:23 -0800 Subject: [PATCH 22/64] Adding SizeAppmagBulgeDiskCOSMOS.py which will remove later. Minor changes to config size file. Data file remove shortly --- ...geDisk.py => SizeAppmagBulgeDiskCOSMOS.py} | 4 +- ...ize_Mandelbaum2014_bulge_disk_COSMOS.yaml} | 16 ++-- ...Mandelbaum2014_sizes_appmag_bulge_disk.txt | 79 +++++++++++++++++++ ...sizes_appmag_bulge_disk_Mandelbaum2014.txt | 67 ---------------- 4 files changed, 89 insertions(+), 77 deletions(-) rename descqa/{SizeAppmagBulgeDisk.py => SizeAppmagBulgeDiskCOSMOS.py} (99%) rename descqa/configs/{size_bulge_disk_COSMOS.yaml => size_Mandelbaum2014_bulge_disk_COSMOS.yaml} (70%) create mode 100644 descqa/data/size/Mandelbaum2014_sizes_appmag_bulge_disk.txt delete mode 100644 descqa/data/size/sizes_appmag_bulge_disk_Mandelbaum2014.txt diff --git a/descqa/SizeAppmagBulgeDisk.py b/descqa/SizeAppmagBulgeDiskCOSMOS.py similarity index 99% rename from descqa/SizeAppmagBulgeDisk.py rename to descqa/SizeAppmagBulgeDiskCOSMOS.py index 5db3c6c5..5b43f67b 100644 --- a/descqa/SizeAppmagBulgeDisk.py +++ b/descqa/SizeAppmagBulgeDiskCOSMOS.py @@ -10,10 +10,10 @@ from .utils import generate_uniform_random_ra_dec_footprint, get_healpixel_footprint, generate_uniform_random_dist -__all__ = ['SizeAppmagBulgeDisk'] +__all__ = ['SizeAppmagBulgeDiskCOSMOS'] -class SizeAppmagBulgeDisk(BaseValidationTest): +class SizeAppmagBulgeDiskCOSMOS(BaseValidationTest): """ Validation test of 2pt correlation function """ diff --git a/descqa/configs/size_bulge_disk_COSMOS.yaml b/descqa/configs/size_Mandelbaum2014_bulge_disk_COSMOS.yaml similarity index 70% rename from descqa/configs/size_bulge_disk_COSMOS.yaml rename to descqa/configs/size_Mandelbaum2014_bulge_disk_COSMOS.yaml index 7a7b36b8..69a23ed9 100644 --- a/descqa/configs/size_bulge_disk_COSMOS.yaml +++ b/descqa/configs/size_Mandelbaum2014_bulge_disk_COSMOS.yaml @@ -1,4 +1,4 @@ -subclass_name: SizeAppmagBulgeDisk.SizeAppmagBulgeDisk +subclass_name: SizeAppmagBulgeDiskCOSMOS.SizeAppmagBulgeDiskCOSMOS #observation: either protodc or buzzard simulated_catalog: protodc @@ -22,16 +22,16 @@ size_field: size_true mag_bin_separation: 1 -protodc_output_filename_template: 'size_lum_COSMOS_z_{}_{}.dat' -buzzard_output_filename_template: 'size_lum_van_z_{}_{}.dat' +protodc_output_filename_template: 'size_COSMOS_z_{}_{}.dat' +buzzard_output_filename_template: 'size_van_z_{}_{}.dat' -protodc_output_figname: 'size_lum_protodc' -buzzrad_output_figname: 'size_lum_buzzard' +protodc_output_figname: 'size_protodc' +buzzrad_output_figname: 'size_buzzard' text_template: '${} < z < {}$' -protodc_data_filename: 'size_sm_lum/sizes_appmag_bulge_disk_Mandelbaum2014.txt' -buzzard_data_filename: 'size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_VLuminosity.txt' +protodc_data_filename: 'size/Mandelbaum2014_sizes_appmag_bulge_disk.txt' +buzzard_data_filename: 'size/van_der_Wel_2014_ApJ_788_28_Table5_VLuminosity.txt' protodc_data_label: 'Mandelbaum+2014: COSMOSIS comparing mean and scatter' buzzard_data_label: 'van der Wel+2014: comparing mean and scatter' @@ -47,6 +47,6 @@ buzzard_fig_ylabel: '$\log_{10}(R_e)$ (kpc)' protodc_description: | - Compare evolution of bulge and disk sizes as a function of i-band magnitude of LSST and redshift and comparing to Mandelbaum et al (2015) HST COSMOS observations + Compare evolution of bulge and disk sizes as a function of i-band magnitude of LSST and redshift and comparing to Mandelbaum et al (2015) HST COSMOS F814W observations buzzard_description: | Compare evolution of size as a function of luminosity and redshift and van der Wel et al (2014) HST-3D and CANDELS observations diff --git a/descqa/data/size/Mandelbaum2014_sizes_appmag_bulge_disk.txt b/descqa/data/size/Mandelbaum2014_sizes_appmag_bulge_disk.txt new file mode 100644 index 00000000..64fd942e --- /dev/null +++ b/descqa/data/size/Mandelbaum2014_sizes_appmag_bulge_disk.txt @@ -0,0 +1,79 @@ +# zavg mag_avg hlr_avg hlr_std +0.250 16.571 19.292 50.922 +0.250 17.625 2.028 4.338 +0.250 18.596 2.252 11.190 +0.250 19.568 1.466 7.550 +0.250 20.518 0.715 2.142 +0.250 21.532 0.580 1.486 +0.250 22.536 0.391 0.699 +0.250 23.509 0.272 0.316 +0.250 24.476 0.193 0.141 +0.250 25.172 0.143 0.089 +0.250 26.293 0.070 0.038 +0.250 27.267 0.028 0.026 +0.250 28.483 0.019 0.000 +0.750 16.687 106.844 0.000 +0.750 17.170 167.093 68.505 +0.750 18.677 4.798 9.607 +0.750 19.663 1.819 9.508 +0.750 20.604 0.868 3.281 +0.750 21.551 0.510 1.292 +0.750 22.523 0.348 0.624 +0.750 23.506 0.257 0.298 +0.750 24.458 0.188 0.150 +0.750 25.145 0.143 0.084 +0.750 26.400 0.072 0.051 +0.750 27.181 0.051 0.034 +0.750 28.639 0.112 0.094 +1.250 0.000 0.000 0.000 +1.250 17.527 270.914 47.990 +1.250 18.691 86.808 82.841 +1.250 19.625 8.726 18.980 +1.250 20.661 1.932 6.614 +1.250 21.595 0.789 2.343 +1.250 22.549 0.470 0.993 +1.250 23.528 0.328 0.432 +1.250 24.473 0.207 0.169 +1.250 25.165 0.152 0.095 +1.250 26.395 0.075 0.047 +1.250 27.306 0.025 0.007 +1.250 28.061 0.002 0.000 +1.750 0.000 0.000 0.000 +1.750 17.271 267.923 144.816 +1.750 18.604 136.937 10.654 +1.750 19.573 33.373 29.027 +1.750 20.626 15.986 21.252 +1.750 21.655 1.579 4.032 +1.750 22.541 0.610 1.515 +1.750 23.565 0.408 0.604 +1.750 24.520 0.216 0.195 +1.750 25.185 0.139 0.086 +1.750 26.462 0.069 0.042 +1.750 27.301 0.020 0.009 +1.750 0.000 0.000 0.000 +2.250 0.000 0.000 0.000 +2.250 0.000 0.000 0.000 +2.250 18.358 202.837 0.000 +2.250 19.900 22.267 21.398 +2.250 20.696 6.685 11.271 +2.250 21.701 1.460 3.799 +2.250 22.542 0.522 1.298 +2.250 23.564 0.352 0.588 +2.250 24.519 0.205 0.189 +2.250 25.176 0.145 0.100 +2.250 26.294 0.079 0.054 +2.250 27.233 0.016 0.000 +2.250 0.000 0.000 0.000 +2.750 0.000 0.000 0.000 +2.750 0.000 0.000 0.000 +2.750 18.969 2.252 0.000 +2.750 19.616 1.527 0.250 +2.750 20.657 16.304 17.977 +2.750 21.681 0.906 3.155 +2.750 22.524 0.374 1.223 +2.750 23.584 0.336 0.530 +2.750 24.533 0.198 0.180 +2.750 25.168 0.139 0.100 +2.750 26.499 0.088 0.059 +2.750 0.000 0.000 0.000 +2.750 0.000 0.000 0.000 diff --git a/descqa/data/size/sizes_appmag_bulge_disk_Mandelbaum2014.txt b/descqa/data/size/sizes_appmag_bulge_disk_Mandelbaum2014.txt deleted file mode 100644 index 287f64d6..00000000 --- a/descqa/data/size/sizes_appmag_bulge_disk_Mandelbaum2014.txt +++ /dev/null @@ -1,67 +0,0 @@ -# zavg avg_mag avg_bhlr avg_dhlr std_bhlr std_dhlr -0.250 19.166 0.470 1.415 0.784 2.360 -0.250 20.527 0.331 0.830 0.719 1.383 -0.250 21.525 0.253 0.674 0.437 1.107 -0.250 22.525 0.197 0.480 0.377 0.680 -0.250 23.535 0.131 0.316 0.184 0.231 -0.250 24.544 0.086 0.218 0.087 0.127 -0.250 25.184 0.062 0.163 0.045 0.083 -0.250 26.326 0.051 0.121 0.029 0.087 -0.250 27.184 0.021 0.034 0.012 0.012 -0.250 28.425 0.015 0.017 0.000 0.000 -0.250 0.000 0.000 0.000 0.000 0.000 -0.750 19.578 0.803 5.471 1.451 7.125 -0.750 20.622 0.494 1.460 1.404 2.589 -0.750 21.571 0.239 0.717 0.451 1.099 -0.750 22.539 0.163 0.459 0.266 0.621 -0.750 23.529 0.117 0.311 0.148 0.356 -0.750 24.513 0.082 0.211 0.086 0.131 -0.750 25.162 0.062 0.160 0.045 0.080 -0.750 26.354 0.049 0.112 0.030 0.087 -0.750 27.278 0.038 0.058 0.024 0.027 -0.750 28.599 0.017 0.033 0.008 0.010 -0.750 0.000 0.000 0.000 0.000 0.000 -1.250 19.331 2.454 16.261 3.245 4.160 -1.250 20.600 3.066 8.753 5.486 4.035 -1.250 21.642 0.781 2.754 1.375 3.514 -1.250 22.634 0.279 0.800 0.475 1.186 -1.250 23.578 0.154 0.399 0.202 0.352 -1.250 24.522 0.092 0.233 0.094 0.149 -1.250 25.187 0.066 0.162 0.051 0.086 -1.250 26.375 0.054 0.101 0.036 0.048 -1.250 27.173 0.068 0.111 0.038 0.032 -1.250 28.386 0.033 0.036 0.019 0.014 -1.250 0.000 0.000 0.000 0.000 0.000 -1.750 19.279 1.885 17.609 1.406 3.657 -1.750 20.565 1.967 9.276 3.468 3.589 -1.750 21.559 1.922 6.244 2.735 4.261 -1.750 22.597 0.786 2.129 1.179 2.057 -1.750 23.634 0.208 0.559 0.297 0.843 -1.750 24.578 0.098 0.237 0.099 0.174 -1.750 25.199 0.066 0.156 0.049 0.087 -1.750 26.281 0.050 0.096 0.034 0.052 -1.750 27.165 0.016 0.020 0.005 0.002 -1.750 0.000 0.000 0.000 0.000 0.000 -1.750 0.000 0.000 0.000 0.000 0.000 -2.250 19.611 1.906 16.778 1.410 3.865 -2.250 20.512 4.666 11.320 5.912 5.261 -2.250 21.462 3.037 6.158 4.928 3.367 -2.250 22.598 0.746 1.954 1.198 1.531 -2.250 23.676 0.160 0.450 0.206 0.534 -2.250 24.582 0.094 0.223 0.096 0.166 -2.250 25.193 0.065 0.156 0.047 0.095 -2.250 26.347 0.060 0.102 0.041 0.044 -2.250 27.574 0.013 0.021 0.002 0.001 -2.250 28.228 0.013 0.014 0.000 0.000 -2.250 0.000 0.000 0.000 0.000 0.000 -2.750 19.119 5.411 12.492 6.256 1.315 -2.750 20.510 3.143 8.794 4.153 2.749 -2.750 21.558 1.926 6.074 2.749 4.172 -2.750 22.584 1.304 2.870 1.834 2.626 -2.750 23.690 0.201 0.494 0.310 0.546 -2.750 24.604 0.093 0.214 0.101 0.162 -2.750 25.195 0.065 0.148 0.053 0.087 -2.750 26.382 0.046 0.082 0.036 0.043 -2.750 27.245 0.029 0.119 0.011 0.135 -2.750 0.000 0.000 0.000 0.000 0.000 -2.750 0.000 0.000 0.000 0.000 0.000 From 0d08abaa8bee92e1ef549864ddb9ff6fe1365ea1 Mon Sep 17 00:00:00 2001 From: Vinu Vikraman Date: Wed, 21 Feb 2018 13:29:34 -0800 Subject: [PATCH 23/64] Added a new validation data from Mandelbaum et al 2014 for HST COSMOS F814W galaxies --- ...isk.txt => Mandelbaum2014_appmag_size.txt} | 0 .../Mandelbaum2014_appmag_size_bulge_disk.txt | 37 +++++++++++++++++++ 2 files changed, 37 insertions(+) rename descqa/data/size/{Mandelbaum2014_sizes_appmag_bulge_disk.txt => Mandelbaum2014_appmag_size.txt} (100%) create mode 100644 descqa/data/size/Mandelbaum2014_appmag_size_bulge_disk.txt diff --git a/descqa/data/size/Mandelbaum2014_sizes_appmag_bulge_disk.txt b/descqa/data/size/Mandelbaum2014_appmag_size.txt similarity index 100% rename from descqa/data/size/Mandelbaum2014_sizes_appmag_bulge_disk.txt rename to descqa/data/size/Mandelbaum2014_appmag_size.txt diff --git a/descqa/data/size/Mandelbaum2014_appmag_size_bulge_disk.txt b/descqa/data/size/Mandelbaum2014_appmag_size_bulge_disk.txt new file mode 100644 index 00000000..f93cb1a4 --- /dev/null +++ b/descqa/data/size/Mandelbaum2014_appmag_size_bulge_disk.txt @@ -0,0 +1,37 @@ +# zavg avg_mag avg_bhlr avg_dhlr std_bhlr std_dhlr +0.250 17.543 0.688 3.114 0.396 4.177 +0.250 19.321 0.419 1.273 0.848 1.724 +0.250 21.105 0.263 0.679 0.471 0.895 +0.250 23.082 0.150 0.374 0.271 0.360 +0.250 24.686 0.079 0.202 0.077 0.123 +0.250 26.426 0.047 0.134 0.028 0.098 +0.750 0.000 0.000 0.000 0.000 0.000 +0.750 19.659 0.762 3.855 1.616 5.902 +0.750 21.365 0.243 0.773 0.513 1.121 +0.750 23.099 0.126 0.358 0.159 0.318 +0.750 24.617 0.076 0.198 0.075 0.123 +0.750 26.697 0.040 0.105 0.024 0.093 +1.250 0.000 0.000 0.000 0.000 0.000 +1.250 19.264 4.182 16.471 5.221 3.193 +1.250 21.494 1.146 2.993 2.610 3.904 +1.250 23.332 0.166 0.461 0.238 0.532 +1.250 24.656 0.084 0.216 0.084 0.140 +1.250 26.408 0.052 0.101 0.030 0.042 +1.750 0.000 0.000 0.000 0.000 0.000 +1.750 19.072 0.904 18.019 0.583 4.456 +1.750 21.265 1.890 7.087 2.505 4.280 +1.750 23.506 0.248 0.689 0.450 0.924 +1.750 24.753 0.086 0.213 0.085 0.159 +1.750 26.318 0.046 0.097 0.032 0.054 +2.250 0.000 0.000 0.000 0.000 0.000 +2.250 19.539 1.336 14.911 0.456 0.268 +2.250 21.262 5.059 7.022 6.502 5.365 +2.250 23.558 0.222 0.590 0.537 0.854 +2.250 24.737 0.085 0.207 0.083 0.159 +2.250 26.491 0.055 0.089 0.039 0.037 +2.750 0.000 0.000 0.000 0.000 0.000 +2.750 19.603 1.922 12.283 0.000 0.000 +2.750 21.180 2.872 6.999 3.832 4.379 +2.750 23.602 0.274 0.625 0.789 1.324 +2.750 24.782 0.081 0.192 0.087 0.142 +2.750 26.699 0.030 0.070 0.011 0.042 From 86e2175e69aefc81132d269c6abaa93e68e79f5c Mon Sep 17 00:00:00 2001 From: Melanie Simet Date: Wed, 21 Feb 2018 18:27:58 -0800 Subject: [PATCH 24/64] Add draft of size distribution test --- descqa/SizeDistribution.py | 107 +++ descqa/configs/size_dist_COSMOS.yaml | 12 + ...COSMOS_Great3_sample_size_distribution.txt | 714 ++++++++++++++++++ 3 files changed, 833 insertions(+) create mode 100644 descqa/SizeDistribution.py create mode 100644 descqa/configs/size_dist_COSMOS.yaml create mode 100644 descqa/data/size_dist/COSMOS_Great3_sample_size_distribution.txt diff --git a/descqa/SizeDistribution.py b/descqa/SizeDistribution.py new file mode 100644 index 00000000..c4df16ae --- /dev/null +++ b/descqa/SizeDistribution.py @@ -0,0 +1,107 @@ +from GCR import GCRQuery +from .base import BaseValidationTest, TestResult +from .plotting import plt +from .utils import get_opt_binpoints + +__all__ = ['SizeDistribution'] + +class SizeDistribuction(BaseValidationTest): + """ + validation test to check the slope of the size distribution at small sizes. + """ + + #plotting constants + validation_color = 'black' + validation_marker = 'o' + default_markers = ['v', 's', 'd', 'H', '^', 'D', 'h', '<', '>', '.'] + msize = 4 #marker-size + yaxis_xoffset = 0.02 + yaxis_yoffset = 0.5 + + def __init__(self, observation='', **kwargs): + + #validation data + validation_filepath = os.path.join(self.data_dir, kwargs['data_filename']) + self.validation_data = np.loadtxt(validation_filepath) + + self.acceptable_keys = kwargs['possible_size_fields'] + + #check for valid observations + if not observation: + print('Warning: no data file supplied, no observation requested; only catalog data will be shown') + elif observation not in self.possible_observations: + raise ValueError('Observation {} not available'.format(observation)) + else: + self.validation_data = self.get_validation_data(observation) + + self._color_iterator = ('C{}'.format(i) for i in count()) + self._other_kwargs = kwargs + + def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): + # update color and marker to preserve catalog colors and markers across tests + catalog_color = next(self._color_iterator) + + # check catalog data for required quantities + key = catalog_instance.first_available(*self.acceptable_keys) + if not key: + summary = 'Missing required quantity' + ' or '.join(['{}']*len(self.acceptable_keys) + return TestResult(skipped=True, summary=summary.format(*self.acceptable_keys)) + + # get data + catalog_data = catalog_instance.get_quantities(key) + sizes = catalog_data[key] + min_sizes = np.min(sizes) + max_sizes = np.max(sizes) + + # Compute N(size) and its slope at the small end. + # Things seem to be roughly linear where N(size)>0.5*Ntot so use those points. + # Get ~20 points for the line fit, but compute the whole graph + median = np.median(sizes) + n_bins = int(20*(max_sizes-min_sizes)/(median-min_sizes)) + N, bin_edges = np.histogram(sizes, bins=n_bins) + sumM = np.histogram(sizes, weights=sizes, bins=bin_edges)[0] + sumM2 = np.histogram(sizes, weights=sizes**2, bins=bin_edges)[0] + size_pts = get_opt_binpoints(N, sumM, sumM2, bin_edges) + + mask = size_pts min_sizes) & (validation_data[:,0] Date: Thu, 22 Feb 2018 08:04:54 -0800 Subject: [PATCH 25/64] recent updates --- descqa/configs/shear.yaml | 6 +- descqa/shear_test.py | 178 ++++++++++++++++++++------------------ 2 files changed, 97 insertions(+), 87 deletions(-) diff --git a/descqa/configs/shear.yaml b/descqa/configs/shear.yaml index eb63ca9b..a18d8290 100644 --- a/descqa/configs/shear.yaml +++ b/descqa/configs/shear.yaml @@ -10,5 +10,7 @@ min_sep: 2.5 max_sep: 200 sep_units: 'arcmin' bin_slop: 0.07 -zlo: 0.3 -zhi: 0.9 +zlo: 0.2 +zhi: 1.0 +do_jackknife: True +N_clust: 10 diff --git a/descqa/shear_test.py b/descqa/shear_test.py index 98321b8a..846fed1c 100644 --- a/descqa/shear_test.py +++ b/descqa/shear_test.py @@ -46,7 +46,7 @@ class ShearTest(BaseValidationTest): Validation test for shear and convergence quantities """ - def __init__(self, z='redshift_true', ra='ra', dec='dec', e1='shear_1', e2='shear_2', kappa='convergence', nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin', bin_slop=0.1, zlo=0.5, zhi=0.6, **kwargs): + def __init__(self, z='redshift_true', ra='ra', dec='dec', e1='shear_1', e2='shear_2', kappa='convergence', nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin', bin_slop=0.1, zlo=0.5, zhi=0.6,do_jackknife=False,N_clust=10, **kwargs): #catalog quantities self.z = z #sep-bounds and binning @@ -61,9 +61,12 @@ def __init__(self, z='redshift_true', ra='ra', dec='dec', e1='shear_1', e2='shea self.e1 = e1 self.e2 = e2 self.kappa = kappa + self.N_clust = N_clust + self.do_jackknife=do_jackknife # cut in redshift self.filters = [(lambda z: (z > zlo) & (z < zhi), self.z)] self.summary_fig, self.summary_ax = plt.subplots(2, sharex=True) + #validation data # want this to change to theory... @@ -152,6 +155,53 @@ def get_score(self,measured,theory,cov, opt='diagonal'): chi2 =np.sum( [(measured[i]-theory[i])**2/cov[i][i] for i in range(len(measured))]) diff = chi2/float(len(measured)) return diff#np.sqrt(np.sum(diff))/float(len(measured)) + + + def jackknife(self,catalog_data,xip,xim): + " computing jack-knife covariance matrix using K-means clustering" + #k-means clustering to define areas + from sklearn.cluster import k_means + N_clust = self.N_clust + nn = np.stack((catalog_data[self.ra],catalog_data[self.dec]),axis=1) + cents, labs, inert = k_means(n_clusters=N_clust,random_state=0,X=nn,n_jobs=-1) # check random state, n_jobs is in debugging mode + print("computing jack-knife errors") + import time + time_jack = time.time() + # jack-knife code + xip_jack=[] + xim_jack=[] + gg = treecorr.GGCorrelation(nbins=self.nbins, min_sep=self.min_sep, max_sep=self.max_sep, sep_units='arcmin', bin_slop=self.bin_slop, verbose=True) + for i in range(N_clust): + ##### shear computation excluding each jack-knife region + cat_s = treecorr.Catalog(ra=catalog_data[self.ra][labs!=i], dec=catalog_data[self.dec][labs!=i], g1=catalog_data[self.e1][labs!=i]-np.mean(catalog_data[self.e1][labs!=i]), g2=-(catalog_data[self.e2][labs!=i]-np.mean(catalog_data[self.e2][labs!=i])), ra_units='deg', dec_units='deg') + gg.process(cat_s) + + xip_jack.append(gg.xip) + xim_jack.append(gg.xim) + ## debugging outputs + print("xip_jack") + print(i) + print(gg.xip) + print("time = "+str(time.time()-time_jack)) + + ### assign covariance matrix - loop is poor python syntax but compared to the time taken for the rest of the test doesn't really matter + cp_xip = np.zeros((self.nbins,self.nbins)) + for i in range(self.nbins): + for j in range(self.nbins): + for k in range(N_clust): + cp_xip[i][j] += (N_clust-1.)/N_clust * (xip[i]-xip_jack[k][i]*1.e6)*(xip[j]-xip_jack[k][j]*1.e6) + + cp_xim = np.zeros((self.nbins,self.nbins)) + for i in range(self.nbins): + for j in range(self.nbins): + for k in range(N_clust): + cp_xim[i][j] += (N_clust-1.)/N_clust * (xim[i]-xim_jack[k][i]*1.e6)*(xim[j]-xim_jack[k][j]*1.e6) + return cp_xip,cp_xim + + + + + @staticmethod def get_catalog_data(gc, quantities, filters=None): @@ -195,16 +245,13 @@ def post_process_plot(self, ax): def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): # check if needed quantities exist - if not catalog_instance.has_quantities([self.z, self.ra, self.dec]): - #print("failed") return TestResult(skipped=True, summary='do not have needed location quantities') if not catalog_instance.has_quantities([self.e1, self.e2,self.kappa]): - #print("failed") return TestResult(skipped=True, summary='do not have needed shear quantities') catalog_data = self.get_catalog_data(catalog_instance, [self.z, self.ra, self.dec, self.e1, self.e2, self.kappa], filters=self.filters) - # get required catalogue data + # read in shear values and check limits e1 = catalog_data[self.e1] max_e1 = np.max(e1) min_e1 = np.min(e1) @@ -216,64 +263,36 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): if ((min_e2<(-1.)) or (max_e2>1.0)): return TestResult(skipped=True, summary='e2 values out of range [-1,+1]') - - + import h5py + f = h5py.File("/global/homes/p/plarsen/protoDC2_values.hdf5", "w") + dset = f.create_dataset("shear_1", np.shape(e1),dtype='f') + dset[...] = e1 + dset2 = f.create_dataset("shear_2", np.shape(e1),dtype='f') + dset3 = f.create_dataset("ra", np.shape(e1),dtype='f') + dset4 = f.create_dataset("dec", np.shape(e1),dtype='f') + dset5 = f.create_dataset("z", np.shape(e1),dtype='f') + dset6 = f.create_dataset("kappa", np.shape(e1),dtype='f') + dset2[...] = e2 + dset3[...] = catalog_data[self.ra] + dset4[...] = catalog_data[self.dec] + dset5[...] = catalog_data[self.z] + dset6[...] = catalog_data[self.kappa] + f.close() + + + # compute shear auto-correlation cat_s = treecorr.Catalog(ra=catalog_data[self.ra], dec=catalog_data[self.dec], g1=catalog_data[self.e1]-np.mean(catalog_data[self.e1]), g2=-(catalog_data[self.e2]-np.mean(catalog_data[self.e2])), ra_units='deg', dec_units='deg') - - gg = treecorr.GGCorrelation(nbins=self.nbins, min_sep=self.min_sep, max_sep=self.max_sep, sep_units='arcmin', bin_slop=self.bin_slop, verbose=True) - gg.process(cat_s) - #base result - - r = np.exp(gg.meanlogr) - xip = gg.xip - xim = gg.xim - - sig = np.sqrt(gg.varxi) - - - ####### code to do jack-knifing ################ - - #k-means clustering to define areas - from sklearn.cluster import k_means - import time - - a = time.time() - N_clust = 7 - nn = np.stack((catalog_data[self.ra],catalog_data[self.dec]),axis=1) - cents, labs, inert = k_means(n_clusters=N_clust,random_state=0,X=nn,n_jobs=-1) - print(time.time()-a) - print("done with k-means clustering") - - - # jack-knife code - xip_jack=[] - xim_jack=[] gg = treecorr.GGCorrelation(nbins=self.nbins, min_sep=self.min_sep, max_sep=self.max_sep, sep_units='arcmin', bin_slop=self.bin_slop, verbose=True) - for i in range(N_clust): - cat_s = treecorr.Catalog(ra=catalog_data[self.ra][labs!=i], dec=catalog_data[self.dec][labs!=i], g1=catalog_data[self.e1][labs!=i]-np.mean(catalog_data[self.e1][labs!=i]), g2=-(catalog_data[self.e2][labs!=i]-np.mean(catalog_data[self.e2][labs!=i])), ra_units='deg', dec_units='deg') - gg.process(cat_s) - xip_jack.append(gg.xip) - xim_jack.append(gg.xim) - print("xip_jack") - print(i) - print(gg.xip) + #gg.process(cat_s) - - ### assign covariance matrix - loop is poor python syntax but compared to the time taken for the rest of the test doesn't really matter - cp_xip = np.zeros((self.nbins,self.nbins)) - for i in range(self.nbins): - for j in range(self.nbins): - for k in range(N_clust): - cp_xip[i][j] += (N_clust-1.)/N_clust * (xip[i]-xip_jack[k][i])*(xip[j]-xip_jack[k][j]) - - cp_xim = np.zeros((self.nbins,self.nbins)) - for i in range(self.nbins): - for j in range(self.nbins): - for k in range(N_clust): - cp_xim[i][j] += (N_clust-1.)/N_clust * (xim[i]-xim_jack[k][i])*(xim[j]-xim_jack[k][j]) - + #r = np.exp(gg.meanlogr) + #xip = gg.xip*1.e6 + #xim = gg.xim*1.e6 + #sig = np.sqrt(gg.varxi) # this is shape noise only - should be very low for simulation data + # get jackknife covariances. + #cp_xip,cp_xim = self.jackknife(catalog_data,xip,xim) # Diagonal covariances for error bars on the plots. Use full covariance matrix for chi2 testing. sig_jack = np.zeros((self.nbins)) @@ -282,51 +301,40 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): sig_jack[i] = np.sqrt(cp_xip[i][i]) sigm_jack[i] = np.sqrt(cp_xim[i][i]) - ################################################# n_z = catalog_data[self.z] - xvals, theory_plus, theory_minus = self.theory_corr(n_z, r , 10000) - + #xvals, theory_plus, theory_minus = self.theory_corr(n_z, r , 10000) - #assuming r == xvals - chi2_dof_1= self.get_score(xip, theory_plus,cp_xip) # correct this - chi2_dof_2 = self.get_score(xim, theory_minus, cp_xim) + #determine chi2/dof values + ##chi2_dof_1= self.get_score(xip, theory_plus,cp_xip,opt='cov') # correct this + #chi2_dof_2 = self.get_score(xim, theory_minus, cp_xim,opt='cov') - print(xvals) - print(r) - print(chi2_dof_1) - print(chi2_dof_2) - print("chi2 values") + #print(chi2_dof_1) + #print(chi2_dof_2) + #print("chi2 values") + #debugging output print("CP_XIP:") print(cp_xip) + print(xip) + print(sig_jack) + - # further correlation functions + # further correlation functions - can add in later #dd = treecorr.NNCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') #dd.process(cat) #treecorr.NGCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') # count-shear (i.e. (R)) #treecorr.NKCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') # count-kappa (i.e. (R)) #treecorr.KKCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') # count-kappa (i.e. (R)) # etc... - print(xip*1.e6) - print(sig_jack*1.e6) -#print(np.exp(gg.logr), gg.xip) -#np.savez('output.npz', theta=np.exp(gg.logr), ggp=gg.xip, ggm=gg.xim, weight=gg.weight, npairs=gg.npairs) - - data = np.random.rand(10) #do your calculation with catalog_instance fig, ax = plt.subplots(2,sharex=True) - for ax_this in (ax, self.summary_ax): - #ax_this[0].plot(xvals,xip*1.e6,'o', color = "#3f9b0b",label=r'$\chi_{+}$') - ax_this[0].errorbar(r,xip*1.e6,sig_jack*1.e6,lw=0.6,marker='o',ls='',color = "#3f9b0b",label=r'$\chi_{+}$') - ax_this[0].plot(xvals,theory_plus*1.e6,'o',color="#9a0eea", label=r'$\chi_{+}$'+" theory") - #ax_this[1].plot(xvals,xim*1.e6,'o',color="#3f9b0b", label=r'$\chi_{-}$') - ax_this[1].errorbar(r,xim*1.e6,sigm_jack*1.e6,lw=0.6,marker='o',ls='',color = "#3f9b0b",label=r'$\chi_{-}$') - ax_this[1].plot(xvals,theory_minus*1.e6,'o',color="#9a0eea", label=r'$\chi_{-}$'+" theory") - - + ax_this[0].errorbar(r,xip,sig_jack,lw=0.6,marker='o',ls='',color = "#3f9b0b",label=r'$\chi_{+}$') + ax_this[0].plot(xvals,theory_plus,'o',color="#9a0eea", label=r'$\chi_{+}$'+" theory") + ax_this[1].errorbar(r,xim,sigm_jack,lw=0.6,marker='o',ls='',color = "#3f9b0b",label=r'$\chi_{-}$') + ax_this[1].plot(xvals,theory_minus,'o',color="#9a0eea", label=r'$\chi_{-}$'+" theory") self.post_process_plot(ax) fig.savefig(os.path.join(output_dir, 'plot.png')) From f621acc3dda06e440da6465682a7cafc8e646ecd Mon Sep 17 00:00:00 2001 From: Patricia Larsen Date: Thu, 22 Feb 2018 09:03:48 -0800 Subject: [PATCH 26/64] minor fixes --- descqa/configs/shear.yaml | 6 +-- descqa/shear_test.py | 94 +++++++++++++++++++++++++-------------- 2 files changed, 64 insertions(+), 36 deletions(-) diff --git a/descqa/configs/shear.yaml b/descqa/configs/shear.yaml index a18d8290..5d6d2195 100644 --- a/descqa/configs/shear.yaml +++ b/descqa/configs/shear.yaml @@ -10,7 +10,7 @@ min_sep: 2.5 max_sep: 200 sep_units: 'arcmin' bin_slop: 0.07 -zlo: 0.2 -zhi: 1.0 -do_jackknife: True +zlo: 0.7 +zhi: 0.8 +do_jackknife: False N_clust: 10 diff --git a/descqa/shear_test.py b/descqa/shear_test.py index 846fed1c..53685a12 100644 --- a/descqa/shear_test.py +++ b/descqa/shear_test.py @@ -151,8 +151,10 @@ def get_score(self,measured,theory,cov, opt='diagonal'): print("inverse covariance matrix") print(cov) chi2 = np.matrix(measured-theory)*cov*np.matrix(measured-theory).T - else: + elif (opt=='diagonal'): chi2 =np.sum( [(measured[i]-theory[i])**2/cov[i][i] for i in range(len(measured))]) + else: + chi2 = np.sum( [(measured[i]-theory[i])**2/theory[i]**2 for i in range(len(measured))]) diff = chi2/float(len(measured)) return diff#np.sqrt(np.sum(diff))/float(len(measured)) @@ -263,48 +265,74 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): if ((min_e2<(-1.)) or (max_e2>1.0)): return TestResult(skipped=True, summary='e2 values out of range [-1,+1]') - import h5py - f = h5py.File("/global/homes/p/plarsen/protoDC2_values.hdf5", "w") - dset = f.create_dataset("shear_1", np.shape(e1),dtype='f') - dset[...] = e1 - dset2 = f.create_dataset("shear_2", np.shape(e1),dtype='f') - dset3 = f.create_dataset("ra", np.shape(e1),dtype='f') - dset4 = f.create_dataset("dec", np.shape(e1),dtype='f') - dset5 = f.create_dataset("z", np.shape(e1),dtype='f') - dset6 = f.create_dataset("kappa", np.shape(e1),dtype='f') - dset2[...] = e2 - dset3[...] = catalog_data[self.ra] - dset4[...] = catalog_data[self.dec] - dset5[...] = catalog_data[self.z] - dset6[...] = catalog_data[self.kappa] - f.close() + + # outputting values for easy testing on home computer + #import h5py + #f = h5py.File("/global/homes/p/plarsen/protoDC2_values.hdf5", "w") + #dset = f.create_dataset("shear_1", np.shape(e1),dtype='f') + #dset[...] = e1 + #dset2 = f.create_dataset("shear_2", np.shape(e1),dtype='f') + #dset3 = f.create_dataset("ra", np.shape(e1),dtype='f') + #dset4 = f.create_dataset("dec", np.shape(e1),dtype='f') + #dset5 = f.create_dataset("z", np.shape(e1),dtype='f') + #dset6 = f.create_dataset("kappa", np.shape(e1),dtype='f') + #dset2[...] = e2 + #dset3[...] = catalog_data[self.ra] + #dset4[...] = catalog_data[self.dec] + #dset5[...] = catalog_data[self.z] + #dset6[...] = catalog_data[self.kappa] + #f.close() # compute shear auto-correlation cat_s = treecorr.Catalog(ra=catalog_data[self.ra], dec=catalog_data[self.dec], g1=catalog_data[self.e1]-np.mean(catalog_data[self.e1]), g2=-(catalog_data[self.e2]-np.mean(catalog_data[self.e2])), ra_units='deg', dec_units='deg') gg = treecorr.GGCorrelation(nbins=self.nbins, min_sep=self.min_sep, max_sep=self.max_sep, sep_units='arcmin', bin_slop=self.bin_slop, verbose=True) - #gg.process(cat_s) + gg.process(cat_s) - #r = np.exp(gg.meanlogr) - #xip = gg.xip*1.e6 - #xim = gg.xim*1.e6 - #sig = np.sqrt(gg.varxi) # this is shape noise only - should be very low for simulation data + r = np.exp(gg.meanlogr) + xip = gg.xip*1.e6 + xim = gg.xim*1.e6 + sig = np.sqrt(gg.varxi) # this is shape noise only - should be very low for simulation data + do_jackknife=self.do_jackknife # get jackknife covariances. #cp_xip,cp_xim = self.jackknife(catalog_data,xip,xim) # Diagonal covariances for error bars on the plots. Use full covariance matrix for chi2 testing. - sig_jack = np.zeros((self.nbins)) - sigm_jack = np.zeros((self.nbins)) - for i in range(self.nbins): - sig_jack[i] = np.sqrt(cp_xip[i][i]) - sigm_jack[i] = np.sqrt(cp_xim[i][i]) - + + if (do_jackknife==True): + cp_xip,cp_xim = self.jackknife(catalog_data,xip,xim) + sig_jack = np.zeros((self.nbins)) + sigm_jack = np.zeros((self.nbins)) + for i in range(self.nbins): + sig_jack[i] = np.sqrt(cp_xip[i][i]) + sigm_jack[i] = np.sqrt(cp_xim[i][i]) + else: + sig_jack = np.zeros((self.nbins)) + sigm_jack = np.zeros((self.nbins)) + for i in range(self.nbins): + sig_jack[i] = np.sqrt(gg.varxi[i]) + sigm_jack[i] = np.sqrt(gg.varxi[i]) + n_z = catalog_data[self.z] - #xvals, theory_plus, theory_minus = self.theory_corr(n_z, r , 10000) - + xvals, theory_plus, theory_minus = self.theory_corr(n_z, r , 10000) + + theory_plus = theory_plus*1.e6 + theory_minus = theory_minus*1.e6 + + + if (do_jackknife==True): + chi2_dof_1= self.get_score(xip, theory_plus,cp_xip,opt='diagonal') # correct this + else: + chi2_dof_1= self.get_score(xip, theory_plus,0,opt='nojack') # correct this + + + print(theory_plus) + print(theory_minus) + print(xip) + print(xim) #determine chi2/dof values ##chi2_dof_1= self.get_score(xip, theory_plus,cp_xip,opt='cov') # correct this #chi2_dof_2 = self.get_score(xim, theory_minus, cp_xim,opt='cov') @@ -314,10 +342,10 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): #print("chi2 values") #debugging output - print("CP_XIP:") - print(cp_xip) - print(xip) - print(sig_jack) + #print("CP_XIP:") + #print(cp_xip) + #print(xip) + #print(sig_jack) # further correlation functions - can add in later From 50b9b36f2b98fa2c2e20992f6c865fcfd93d94dc Mon Sep 17 00:00:00 2001 From: Patricia Larsen Date: Thu, 22 Feb 2018 09:23:24 -0800 Subject: [PATCH 27/64] travis build fixes --- descqa/shear_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/descqa/shear_test.py b/descqa/shear_test.py index 53685a12..01d77565 100644 --- a/descqa/shear_test.py +++ b/descqa/shear_test.py @@ -5,11 +5,11 @@ import numpy as np from GCR import GCRQuery -import GCRCatalogs -import descqa +#import GCRCatalogs +#import descqa from builtins import str -import yaml +#import yaml import healpy as hp import camb From 44b48269c99d86f8916885fffa69ced9c586f086 Mon Sep 17 00:00:00 2001 From: Yao-Yuan Mao Date: Thu, 22 Feb 2018 14:04:00 -0500 Subject: [PATCH 28/64] add camb --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index f88d8d65..60f68cc3 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ from setuptools import setup with open(os.path.join(os.path.dirname(__file__), 'descqa', 'version.py')) as f: - exec(f.read()) + exec(f.read()) #pylint: disable=W0122 setup( name='descqa', @@ -31,7 +31,7 @@ packages=['descqa'], install_requires=['future', 'pyyaml', 'jinja2'], extras_require={ - 'full': ['numpy', 'scipy', 'matplotlib', 'healpy', 'GCR>=0.6.2', 'treecorr'], + 'full': ['numpy', 'scipy', 'matplotlib', 'healpy', 'GCR>=0.6.2', 'treecorr', 'camb'], }, package_data={'descqa': ['configs/*.yaml', 'data/*']}, ) From 25af573a0beb485057e28e7dbb9b32a30f304b4a Mon Sep 17 00:00:00 2001 From: Yao-Yuan Mao Date: Thu, 22 Feb 2018 14:04:44 -0500 Subject: [PATCH 29/64] fix imports; fix pylint; reformat --- descqa/shear_test.py | 313 ++++++++++++++++++++++--------------------- 1 file changed, 162 insertions(+), 151 deletions(-) diff --git a/descqa/shear_test.py b/descqa/shear_test.py index 01d77565..14e1769f 100644 --- a/descqa/shear_test.py +++ b/descqa/shear_test.py @@ -1,59 +1,66 @@ - -#from __future__ import unicode_literals, absolute_import, division +from __future__ import unicode_literals, absolute_import, division import os -import sys +import time import numpy as np -from GCR import GCRQuery -#import GCRCatalogs -#import descqa - -from builtins import str -#import yaml -import healpy as hp - -import camb -import treecorr - -from astropy.cosmology import FlatLambdaCDM -cosmo = FlatLambdaCDM(H0=70, Om0=0.3, Tcmb0=2.725) from scipy.interpolate import interp1d from scipy.integrate import quad - +from sklearn.cluster import k_means +import treecorr +import camb +import camb.correlations +from astropy.cosmology import FlatLambdaCDM import astropy.units as u import astropy.constants as const -import camb.correlations +from GCR import GCRQuery + +from .base import BaseValidationTest, TestResult +from .plotting import plt + +cosmo = FlatLambdaCDM(H0=70, Om0=0.3, Tcmb0=2.725) chi_recomb = 14004.036207574154 pars = camb.CAMBparams() h = 0.675 -pars.set_cosmology(H0=h*100, ombh2=0.022, omch2=0.122) +pars.set_cosmology(H0=h * 100, ombh2=0.022, omch2=0.122) pars.InitPower.set_params(ns=0.965) camb.set_halofit_version(version='takahashi') -p = camb.get_matter_power_interpolator(pars, nonlinear=True, k_hunit=False, hubble_units=False ,kmax=100., zmax=1100., k_per_logint=False).P - -from .base import BaseValidationTest, TestResult -from .plotting import plt - - - - +p = camb.get_matter_power_interpolator( + pars, nonlinear=True, k_hunit=False, hubble_units=False, kmax=100., zmax=1100., k_per_logint=False).P __all__ = ['ShearTest'] + class ShearTest(BaseValidationTest): """ Validation test for shear and convergence quantities """ - def __init__(self, z='redshift_true', ra='ra', dec='dec', e1='shear_1', e2='shear_2', kappa='convergence', nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin', bin_slop=0.1, zlo=0.5, zhi=0.6,do_jackknife=False,N_clust=10, **kwargs): + def __init__(self, + z='redshift_true', + ra='ra', + dec='dec', + e1='shear_1', + e2='shear_2', + kappa='convergence', + nbins=20, + min_sep=2.5, + max_sep=250, + sep_units='arcmin', + bin_slop=0.1, + zlo=0.5, + zhi=0.6, + do_jackknife=False, + N_clust=10, + **kwargs): + #pylint: disable=W0231 #catalog quantities self.z = z #sep-bounds and binning self.min_sep = min_sep self.max_sep = max_sep self.nbins = nbins - self.sep_bins = np.linspace(min_sep, max_sep, nbins+1) + self.sep_bins = np.linspace(min_sep, max_sep, nbins + 1) self.sep_units = sep_units self.bin_slop = bin_slop self.ra = ra @@ -62,14 +69,13 @@ def __init__(self, z='redshift_true', ra='ra', dec='dec', e1='shear_1', e2='shea self.e2 = e2 self.kappa = kappa self.N_clust = N_clust - self.do_jackknife=do_jackknife + self.do_jackknife = do_jackknife # cut in redshift self.filters = [(lambda z: (z > zlo) & (z < zhi), self.z)] self.summary_fig, self.summary_ax = plt.subplots(2, sharex=True) - - + #validation data - # want this to change to theory... + # want this to change to theory... #self.validation_data = {} #self.observation = observation #setup subplot configuration and get magnitude cuts for each plot @@ -80,131 +86,138 @@ def __init__(self, z='redshift_true', ra='ra', dec='dec', e1='shear_1', e2='shea #otherwise setup a check so that validation data is plotted only once on summary plot #self.first_pass = True #self._other_kwargs = kwargs - - - def compute_nz(self,n_z): + + def compute_nz(self, n_z): '''create interpolated n(z) distribution''' z_bins = np.linspace(0.0, 2.0, 101) n = np.histogram(n_z, bins=z_bins)[0] - z = (z_bins[1:]-z_bins[:-1])/2. + z_bins[:-1] + z = (z_bins[1:] - z_bins[:-1]) / 2. + z_bins[:-1] n2 = interp1d(z, n, bounds_error=False, fill_value=0.0, kind='cubic') n2_sum = quad(n2, 0, 2.0)[0] - n2 = interp1d(z, n/n2_sum, bounds_error=False, fill_value=0.0, kind='cubic') + n2 = interp1d(z, n / n2_sum, bounds_error=False, fill_value=0.0, kind='cubic') return n2 - def integrand_w(self,x,n,chi,chi_int): + def integrand_w(self, x, n, chi, chi_int): ''' This is the inner bit of GWL lensing kernel - z is related to x, not chi''' z = chi_int(x) - H_z = cosmo.H(z).value*3.240779289469756e-20 #1/s units #.to(1./u.s) conversion - dchidz = 9.715611890256315e-15 /H_z #const.c.to(u.Mpc/u.s).value / (H_z) # Mpc units - return n(z)/dchidz* (x - chi)/x + H_z = cosmo.H(z).value * 3.240779289469756e-20 #1/s units #.to(1./u.s) conversion + dchidz = 9.715611890256315e-15 / H_z #const.c.to(u.Mpc/u.s).value / (H_z) # Mpc units + return n(z) / dchidz * (x - chi) / x - def galaxy_W(self,z,n,chi_int): + def galaxy_W(self, z, n, chi_int): ''' galaxy window function''' - chi = cosmo.comoving_distance(z).value # can be array - cst = 3./2. * cosmo.H(0).to(1./u.s)**2/const.c.to(u.Mpc / u.s)**2*cosmo.Om(0)# not sure about the z-dependence here - prefactor = cst * chi* (1.+z) *u.Mpc + #pylint: disable=E1101 + chi = cosmo.comoving_distance(z).value # can be array + cst = 3. / 2. * cosmo.H(0).to(1. / u.s)**2 / const.c.to(u.Mpc / u.s)**2 * cosmo.Om( + 0) # not sure about the z-dependence here + prefactor = cst * chi * (1. + z) * u.Mpc val_array = [] for i in range(len(z)): - val_array.append(quad(self.integrand_w, chi[i], chi_recomb, args = (n, chi[i], chi_int))[0]) - W = np.array(val_array)*prefactor *(u.Mpc) # now unitless + val_array.append(quad(self.integrand_w, chi[i], chi_recomb, args=(n, chi[i], chi_int))[0]) + W = np.array(val_array) * prefactor * (u.Mpc) # now unitless return W - def integrand_lensing_limber(self,chi, l, galaxy_W_int, chi_int): + def integrand_lensing_limber(self, chi, l, galaxy_W_int, chi_int): '''return overall integrand for one value of l''' - chi_unit = chi*u.Mpc + #chi_unit = chi * u.Mpc z = chi_int(chi) - k = (l+0.5)/chi - integrand = p(z, k, grid=False) *galaxy_W_int(z)**2/chi**2 + k = (l + 0.5) / chi + integrand = p(z, k, grid=False) * galaxy_W_int(z)**2 / chi**2 return integrand - def phi(self,lmax,n_z): - z_array = np.logspace(-3,np.log10(10.),200) + def phi(self, lmax, n_z): + z_array = np.logspace(-3, np.log10(10.), 200) chi_array = cosmo.comoving_distance(z_array).value chi_int = interp1d(chi_array, z_array, bounds_error=False, fill_value=0.0) n = self.compute_nz(n_z) galaxy_W_int = interp1d(z_array, self.galaxy_W(z_array, n, chi_int), bounds_error=False, fill_value=0.0) phi_array = [] - l= range(0, lmax, 1) + l = range(0, lmax, 1) l = np.array(l) for i in l: - a = quad(self.integrand_lensing_limber, 1.e-10, chi_recomb, args=(i, galaxy_W_int, chi_int), epsrel = 1.e-6)[0] + a = quad( + self.integrand_lensing_limber, 1.e-10, chi_recomb, args=(i, galaxy_W_int, chi_int), epsrel=1.e-6)[0] phi_array.append(a) phi_array = np.array(phi_array) - prefactor = 1.0#(l+2)*(l+1)*l*(l-1) / (l+0.5)**4 - return l, phi_array*prefactor - + prefactor = 1.0 #(l+2)*(l+1)*l*(l-1) / (l+0.5)**4 + return l, phi_array * prefactor - def theory_corr(self,n_z2, xvals, lmax2): + def theory_corr(self, n_z2, xvals, lmax2): ll, pp = self.phi(lmax=lmax2, n_z=n_z2) - pp3_2 = np.zeros((lmax2,4)) - pp3_2[:,1]= pp[:]*(ll*(ll+1.))/(2.*np.pi) + pp3_2 = np.zeros((lmax2, 4)) + pp3_2[:, 1] = pp[:] * (ll * (ll + 1.)) / (2. * np.pi) #xvals = np.logspace(np.log10(min_sep2), np.log10(max_sep2), nbins2) #in arcminutes - cxvals = np.cos(xvals/(60.)/(180./np.pi)) + cxvals = np.cos(xvals / (60.) / (180. / np.pi)) vals = camb.correlations.cl2corr(pp3_2, cxvals) - return xvals, vals[:,1], vals[:,2] - + return xvals, vals[:, 1], vals[:, 2] - def get_score(self,measured,theory,cov, opt='diagonal'): - if (opt=='cov'): + def get_score(self, measured, theory, cov, opt='diagonal'): + if (opt == 'cov'): cov = np.matrix(cov).I print("inverse covariance matrix") print(cov) - chi2 = np.matrix(measured-theory)*cov*np.matrix(measured-theory).T - elif (opt=='diagonal'): - chi2 =np.sum( [(measured[i]-theory[i])**2/cov[i][i] for i in range(len(measured))]) + chi2 = np.matrix(measured - theory) * cov * np.matrix(measured - theory).T + elif (opt == 'diagonal'): + chi2 = np.sum([(measured[i] - theory[i])**2 / cov[i][i] for i in range(len(measured))]) else: - chi2 = np.sum( [(measured[i]-theory[i])**2/theory[i]**2 for i in range(len(measured))]) - diff = chi2/float(len(measured)) - return diff#np.sqrt(np.sum(diff))/float(len(measured)) - - - def jackknife(self,catalog_data,xip,xim): + chi2 = np.sum([(measured[i] - theory[i])**2 / theory[i]**2 for i in range(len(measured))]) + diff = chi2 / float(len(measured)) + return diff #np.sqrt(np.sum(diff))/float(len(measured)) + + def jackknife(self, catalog_data, xip, xim): " computing jack-knife covariance matrix using K-means clustering" #k-means clustering to define areas - from sklearn.cluster import k_means N_clust = self.N_clust - nn = np.stack((catalog_data[self.ra],catalog_data[self.dec]),axis=1) - cents, labs, inert = k_means(n_clusters=N_clust,random_state=0,X=nn,n_jobs=-1) # check random state, n_jobs is in debugging mode + nn = np.stack((catalog_data[self.ra], catalog_data[self.dec]), axis=1) + _, labs, _ = k_means( + n_clusters=N_clust, random_state=0, X=nn, n_jobs=-1) # check random state, n_jobs is in debugging mode print("computing jack-knife errors") - import time time_jack = time.time() # jack-knife code - xip_jack=[] - xim_jack=[] - gg = treecorr.GGCorrelation(nbins=self.nbins, min_sep=self.min_sep, max_sep=self.max_sep, sep_units='arcmin', bin_slop=self.bin_slop, verbose=True) + xip_jack = [] + xim_jack = [] + gg = treecorr.GGCorrelation( + nbins=self.nbins, + min_sep=self.min_sep, + max_sep=self.max_sep, + sep_units='arcmin', + bin_slop=self.bin_slop, + verbose=True) for i in range(N_clust): ##### shear computation excluding each jack-knife region - cat_s = treecorr.Catalog(ra=catalog_data[self.ra][labs!=i], dec=catalog_data[self.dec][labs!=i], g1=catalog_data[self.e1][labs!=i]-np.mean(catalog_data[self.e1][labs!=i]), g2=-(catalog_data[self.e2][labs!=i]-np.mean(catalog_data[self.e2][labs!=i])), ra_units='deg', dec_units='deg') + cat_s = treecorr.Catalog( + ra=catalog_data[self.ra][labs != i], + dec=catalog_data[self.dec][labs != i], + g1=catalog_data[self.e1][labs != i] - np.mean(catalog_data[self.e1][labs != i]), + g2=-(catalog_data[self.e2][labs != i] - np.mean(catalog_data[self.e2][labs != i])), + ra_units='deg', + dec_units='deg') gg.process(cat_s) - + xip_jack.append(gg.xip) xim_jack.append(gg.xim) ## debugging outputs print("xip_jack") print(i) print(gg.xip) - print("time = "+str(time.time()-time_jack)) - + print("time = " + str(time.time() - time_jack)) + ### assign covariance matrix - loop is poor python syntax but compared to the time taken for the rest of the test doesn't really matter - cp_xip = np.zeros((self.nbins,self.nbins)) + cp_xip = np.zeros((self.nbins, self.nbins)) for i in range(self.nbins): for j in range(self.nbins): for k in range(N_clust): - cp_xip[i][j] += (N_clust-1.)/N_clust * (xip[i]-xip_jack[k][i]*1.e6)*(xip[j]-xip_jack[k][j]*1.e6) + cp_xip[i][j] += (N_clust - 1.) / N_clust * (xip[i] - xip_jack[k][i] * 1.e6) * ( + xip[j] - xip_jack[k][j] * 1.e6) - cp_xim = np.zeros((self.nbins,self.nbins)) + cp_xim = np.zeros((self.nbins, self.nbins)) for i in range(self.nbins): for j in range(self.nbins): for k in range(N_clust): - cp_xim[i][j] += (N_clust-1.)/N_clust * (xim[i]-xim_jack[k][i]*1.e6)*(xim[j]-xim_jack[k][j]*1.e6) - return cp_xip,cp_xim - - - - - - + cp_xim[i][j] += (N_clust - 1.) / N_clust * (xim[i] - xim_jack[k][i] * 1.e6) * ( + xim[j] - xim_jack[k][j] * 1.e6) + return cp_xip, cp_xim + @staticmethod def get_catalog_data(gc, quantities, filters=None): data = {} @@ -217,11 +230,8 @@ def get_catalog_data(gc, quantities, filters=None): return data - # define theory from within this class - - - + def post_process_plot(self, ax): #ax.text(0.05, 0.95, "add text here") plt.xscale('log') @@ -233,26 +243,26 @@ def post_process_plot(self, ax): min_height2 = -1 #very rough DES-like limits (maximum and minimum scales) - ax[0].vlines(2.5,min_height1,max_height1,linestyles='--') - ax[0].vlines(200.,min_height1,max_height1,linestyles='--') - ax[1].vlines(35.,min_height2,max_height2,linestyles='--') - ax[1].vlines(200.,min_height2,max_height2,linestyles='--') + ax[0].vlines(2.5, min_height1, max_height1, linestyles='--') + ax[0].vlines(200., min_height1, max_height1, linestyles='--') + ax[1].vlines(35., min_height2, max_height2, linestyles='--') + ax[1].vlines(200., min_height2, max_height2, linestyles='--') plt.xlabel(r'$\theta \rm{(arcmin)}$') ax[0].set_ylabel(r'$\chi_{+} (10^{-6}) $') ax[1].set_ylabel(r'$\chi_{-} (10^{-6}) $') - ax[0].set_ylim([min_height1,max_height1]) - ax[1].set_ylim([min_height2,max_height2]) - - def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): + ax[0].set_ylim([min_height1, max_height1]) + ax[1].set_ylim([min_height2, max_height2]) + def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): # check if needed quantities exist if not catalog_instance.has_quantities([self.z, self.ra, self.dec]): return TestResult(skipped=True, summary='do not have needed location quantities') - if not catalog_instance.has_quantities([self.e1, self.e2,self.kappa]): + if not catalog_instance.has_quantities([self.e1, self.e2, self.kappa]): return TestResult(skipped=True, summary='do not have needed shear quantities') - catalog_data = self.get_catalog_data(catalog_instance, [self.z, self.ra, self.dec, self.e1, self.e2, self.kappa], filters=self.filters) - + catalog_data = self.get_catalog_data( + catalog_instance, [self.z, self.ra, self.dec, self.e1, self.e2, self.kappa], filters=self.filters) + # read in shear values and check limits e1 = catalog_data[self.e1] max_e1 = np.max(e1) @@ -260,14 +270,13 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): e2 = catalog_data[self.e2] max_e2 = np.max(e2) min_e2 = np.min(e2) - if ((min_e1<(-1.)) or (max_e1>1.0)): + if ((min_e1 < (-1.)) or (max_e1 > 1.0)): return TestResult(skipped=True, summary='e1 values out of range [-1,+1]') - if ((min_e2<(-1.)) or (max_e2>1.0)): + if ((min_e2 < (-1.)) or (max_e2 > 1.0)): return TestResult(skipped=True, summary='e2 values out of range [-1,+1]') - # outputting values for easy testing on home computer - #import h5py + #import h5py #f = h5py.File("/global/homes/p/plarsen/protoDC2_values.hdf5", "w") #dset = f.create_dataset("shear_1", np.shape(e1),dtype='f') #dset[...] = e1 @@ -283,26 +292,36 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): #dset6[...] = catalog_data[self.kappa] #f.close() - # compute shear auto-correlation - cat_s = treecorr.Catalog(ra=catalog_data[self.ra], dec=catalog_data[self.dec], - g1=catalog_data[self.e1]-np.mean(catalog_data[self.e1]), g2=-(catalog_data[self.e2]-np.mean(catalog_data[self.e2])), ra_units='deg', dec_units='deg') - gg = treecorr.GGCorrelation(nbins=self.nbins, min_sep=self.min_sep, max_sep=self.max_sep, sep_units='arcmin', bin_slop=self.bin_slop, verbose=True) + cat_s = treecorr.Catalog( + ra=catalog_data[self.ra], + dec=catalog_data[self.dec], + g1=catalog_data[self.e1] - np.mean(catalog_data[self.e1]), + g2=-(catalog_data[self.e2] - np.mean(catalog_data[self.e2])), + ra_units='deg', + dec_units='deg') + gg = treecorr.GGCorrelation( + nbins=self.nbins, + min_sep=self.min_sep, + max_sep=self.max_sep, + sep_units='arcmin', + bin_slop=self.bin_slop, + verbose=True) gg.process(cat_s) r = np.exp(gg.meanlogr) - xip = gg.xip*1.e6 - xim = gg.xim*1.e6 - sig = np.sqrt(gg.varxi) # this is shape noise only - should be very low for simulation data + xip = gg.xip * 1.e6 + xim = gg.xim * 1.e6 + #sig = np.sqrt(gg.varxi) # this is shape noise only - should be very low for simulation data - do_jackknife=self.do_jackknife + do_jackknife = self.do_jackknife # get jackknife covariances. #cp_xip,cp_xim = self.jackknife(catalog_data,xip,xim) - # Diagonal covariances for error bars on the plots. Use full covariance matrix for chi2 testing. + # Diagonal covariances for error bars on the plots. Use full covariance matrix for chi2 testing. - if (do_jackknife==True): - cp_xip,cp_xim = self.jackknife(catalog_data,xip,xim) + if (do_jackknife == True): + cp_xip, cp_xim = self.jackknife(catalog_data, xip, xim) sig_jack = np.zeros((self.nbins)) sigm_jack = np.zeros((self.nbins)) for i in range(self.nbins): @@ -315,26 +334,23 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): sig_jack[i] = np.sqrt(gg.varxi[i]) sigm_jack[i] = np.sqrt(gg.varxi[i]) - n_z = catalog_data[self.z] - xvals, theory_plus, theory_minus = self.theory_corr(n_z, r , 10000) + xvals, theory_plus, theory_minus = self.theory_corr(n_z, r, 10000) - theory_plus = theory_plus*1.e6 - theory_minus = theory_minus*1.e6 + theory_plus = theory_plus * 1.e6 + theory_minus = theory_minus * 1.e6 - - if (do_jackknife==True): - chi2_dof_1= self.get_score(xip, theory_plus,cp_xip,opt='diagonal') # correct this + if (do_jackknife == True): + chi2_dof_1 = self.get_score(xip, theory_plus, cp_xip, opt='diagonal') # correct this else: - chi2_dof_1= self.get_score(xip, theory_plus,0,opt='nojack') # correct this - + chi2_dof_1 = self.get_score(xip, theory_plus, 0, opt='nojack') # correct this print(theory_plus) print(theory_minus) print(xip) print(xim) #determine chi2/dof values - ##chi2_dof_1= self.get_score(xip, theory_plus,cp_xip,opt='cov') # correct this + ##chi2_dof_1= self.get_score(xip, theory_plus,cp_xip,opt='cov') # correct this #chi2_dof_2 = self.get_score(xim, theory_minus, cp_xim,opt='cov') #print(chi2_dof_1) @@ -347,37 +363,32 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): #print(xip) #print(sig_jack) - # further correlation functions - can add in later #dd = treecorr.NNCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') #dd.process(cat) #treecorr.NGCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') # count-shear (i.e. (R)) #treecorr.NKCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') # count-kappa (i.e. (R)) #treecorr.KKCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') # count-kappa (i.e. (R)) - # etc... - + # etc... - fig, ax = plt.subplots(2,sharex=True) + fig, ax = plt.subplots(2, sharex=True) for ax_this in (ax, self.summary_ax): - ax_this[0].errorbar(r,xip,sig_jack,lw=0.6,marker='o',ls='',color = "#3f9b0b",label=r'$\chi_{+}$') - ax_this[0].plot(xvals,theory_plus,'o',color="#9a0eea", label=r'$\chi_{+}$'+" theory") - ax_this[1].errorbar(r,xim,sigm_jack,lw=0.6,marker='o',ls='',color = "#3f9b0b",label=r'$\chi_{-}$') - ax_this[1].plot(xvals,theory_minus,'o',color="#9a0eea", label=r'$\chi_{-}$'+" theory") + ax_this[0].errorbar(r, xip, sig_jack, lw=0.6, marker='o', ls='', color="#3f9b0b", label=r'$\chi_{+}$') + ax_this[0].plot(xvals, theory_plus, 'o', color="#9a0eea", label=r'$\chi_{+}$' + " theory") + ax_this[1].errorbar(r, xim, sigm_jack, lw=0.6, marker='o', ls='', color="#3f9b0b", label=r'$\chi_{-}$') + ax_this[1].plot(xvals, theory_minus, 'o', color="#9a0eea", label=r'$\chi_{-}$' + " theory") self.post_process_plot(ax) fig.savefig(os.path.join(output_dir, 'plot.png')) plt.close(fig) - score = chi2_dof_1 #calculate your summary statistics - if (score<2): + score = chi2_dof_1 #calculate your summary statistics + if (score < 2): return TestResult(score, passed=True) else: return TestResult(score, passed=False) - - def conclude_test(self, output_dir): self.post_process_plot(self.summary_ax) self.summary_fig.savefig(os.path.join(output_dir, 'summary.png')) plt.close(self.summary_fig) - From a8e7ab097056b6eb68a9db065147514804db5746 Mon Sep 17 00:00:00 2001 From: Yao-Yuan Mao Date: Thu, 22 Feb 2018 14:31:12 -0500 Subject: [PATCH 30/64] install gfortran for camb in travis --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 5283fd79..21f9db68 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,10 @@ python: - "2.7" - "3.6" +before_install: +- sudo apt-get -qq update +- sudo apt-get -y install gfortran + install: - pip install .[full] - pip install pylint nose From dd07485452fcf5003500ff831f6d1aaf5bea7643 Mon Sep 17 00:00:00 2001 From: Yao-Yuan Mao Date: Thu, 22 Feb 2018 14:42:56 -0500 Subject: [PATCH 31/64] need gfortran 4.9 in travis --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 21f9db68..52dfc65a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,8 +7,9 @@ python: - "3.6" before_install: -- sudo apt-get -qq update -- sudo apt-get -y install gfortran +- sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y +- sudo apt-get update -qq +- sudo apt-get install gfortran-4.9 -y install: - pip install .[full] From e704639402eef24df2f680e45fdbdbcc3a915995 Mon Sep 17 00:00:00 2001 From: Yao-Yuan Mao Date: Thu, 22 Feb 2018 14:59:40 -0500 Subject: [PATCH 32/64] i don't know why gfortran is not found... --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 52dfc65a..05456a29 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ python: before_install: - sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y - sudo apt-get update -qq -- sudo apt-get install gfortran-4.9 -y +- sudo apt-get install gfortran -y install: - pip install .[full] From cf962268841378ab68295ba8770510b9e74d6788 Mon Sep 17 00:00:00 2001 From: Yao-Yuan Mao Date: Thu, 22 Feb 2018 15:18:40 -0500 Subject: [PATCH 33/64] another try to get gfortran 4.9 --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 05456a29..093e078c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,10 +6,11 @@ python: - "2.7" - "3.6" -before_install: +before_install: # camb requires gfortran to be at least gfortran-4.9 - sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y - sudo apt-get update -qq -- sudo apt-get install gfortran -y +- sudo apt-get install gfortran-4.9 -y +- alias gfortran=gfortran-4.9 install: - pip install .[full] From a4d0c1a0fddb73870920aa85945e11d48a8b74f8 Mon Sep 17 00:00:00 2001 From: Yao-Yuan Mao Date: Thu, 22 Feb 2018 15:32:59 -0500 Subject: [PATCH 34/64] alias is not enough... --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 093e078c..57b652b7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ before_install: # camb requires gfortran to be at least gfortran-4.9 - sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y - sudo apt-get update -qq - sudo apt-get install gfortran-4.9 -y -- alias gfortran=gfortran-4.9 +- sudo ln -s /usr/bin/gfortran-4.9 /usr/bin/gfortran install: - pip install .[full] From 16b4550ce77051022b8a2cdcd9fa386fd9d07174 Mon Sep 17 00:00:00 2001 From: Yao-Yuan Mao Date: Thu, 22 Feb 2018 15:42:10 -0500 Subject: [PATCH 35/64] also needs scikit-learn --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 60f68cc3..26fbf690 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ packages=['descqa'], install_requires=['future', 'pyyaml', 'jinja2'], extras_require={ - 'full': ['numpy', 'scipy', 'matplotlib', 'healpy', 'GCR>=0.6.2', 'treecorr', 'camb'], + 'full': ['numpy', 'scipy', 'matplotlib', 'GCR>=0.6.2', 'healpy', 'treecorr', 'camb', 'scikit-learn'], }, package_data={'descqa': ['configs/*.yaml', 'data/*']}, ) From f30b3a5524a1a086b539f9c7760114e17221d7b9 Mon Sep 17 00:00:00 2001 From: Patricia Larsen Date: Thu, 22 Feb 2018 15:35:50 -0800 Subject: [PATCH 36/64] updated cosmology --- descqa/shear_test.py | 104 ++++++++++++++----------------------------- 1 file changed, 33 insertions(+), 71 deletions(-) diff --git a/descqa/shear_test.py b/descqa/shear_test.py index 14e1769f..aa9f166e 100644 --- a/descqa/shear_test.py +++ b/descqa/shear_test.py @@ -16,17 +16,8 @@ from .base import BaseValidationTest, TestResult from .plotting import plt - -cosmo = FlatLambdaCDM(H0=70, Om0=0.3, Tcmb0=2.725) - -chi_recomb = 14004.036207574154 pars = camb.CAMBparams() -h = 0.675 -pars.set_cosmology(H0=h * 100, ombh2=0.022, omch2=0.122) -pars.InitPower.set_params(ns=0.965) -camb.set_halofit_version(version='takahashi') -p = camb.get_matter_power_interpolator( - pars, nonlinear=True, k_hunit=False, hubble_units=False, kmax=100., zmax=1100., k_per_logint=False).P +# note that chi_recomb should be a function of cosmology but this shouldn't have a major impact __all__ = ['ShearTest'] @@ -55,6 +46,8 @@ def __init__(self, **kwargs): #pylint: disable=W0231 #catalog quantities + + self.z = z #sep-bounds and binning self.min_sep = min_sep @@ -74,18 +67,6 @@ def __init__(self, self.filters = [(lambda z: (z > zlo) & (z < zhi), self.z)] self.summary_fig, self.summary_ax = plt.subplots(2, sharex=True) - #validation data - # want this to change to theory... - #self.validation_data = {} - #self.observation = observation - #setup subplot configuration and get magnitude cuts for each plot - #self.mag_lo, self.mag_hi = self.init_plots(mag_lo, mag_hi) - #setup summary plot - #self.summary_fig, self.summary_ax = plt.subplots(self.nrows, self.ncolumns, figsize=(figx_p, figy_p), sharex='col') - #could plot summary validation data here if available but would need to evaluate labels, bin values etc. - #otherwise setup a check so that validation data is plotted only once on summary plot - #self.first_pass = True - #self._other_kwargs = kwargs def compute_nz(self, n_z): '''create interpolated n(z) distribution''' @@ -97,27 +78,27 @@ def compute_nz(self, n_z): n2 = interp1d(z, n / n2_sum, bounds_error=False, fill_value=0.0, kind='cubic') return n2 - def integrand_w(self, x, n, chi, chi_int): + def integrand_w(self, x, n, chi, chi_int, cosmo): ''' This is the inner bit of GWL lensing kernel - z is related to x, not chi''' z = chi_int(x) H_z = cosmo.H(z).value * 3.240779289469756e-20 #1/s units #.to(1./u.s) conversion dchidz = 9.715611890256315e-15 / H_z #const.c.to(u.Mpc/u.s).value / (H_z) # Mpc units return n(z) / dchidz * (x - chi) / x - def galaxy_W(self, z, n, chi_int): + def galaxy_W(self, z, n, chi_int, cosmo, chi_recomb): ''' galaxy window function''' #pylint: disable=E1101 chi = cosmo.comoving_distance(z).value # can be array cst = 3. / 2. * cosmo.H(0).to(1. / u.s)**2 / const.c.to(u.Mpc / u.s)**2 * cosmo.Om( - 0) # not sure about the z-dependence here + 0) prefactor = cst * chi * (1. + z) * u.Mpc val_array = [] for i in range(len(z)): - val_array.append(quad(self.integrand_w, chi[i], chi_recomb, args=(n, chi[i], chi_int))[0]) + val_array.append(quad(self.integrand_w, chi[i], chi_recomb, args=(n, chi[i], chi_int, cosmo))[0]) W = np.array(val_array) * prefactor * (u.Mpc) # now unitless return W - def integrand_lensing_limber(self, chi, l, galaxy_W_int, chi_int): + def integrand_lensing_limber(self, chi, l, galaxy_W_int, chi_int, p): '''return overall integrand for one value of l''' #chi_unit = chi * u.Mpc z = chi_int(chi) @@ -125,25 +106,25 @@ def integrand_lensing_limber(self, chi, l, galaxy_W_int, chi_int): integrand = p(z, k, grid=False) * galaxy_W_int(z)**2 / chi**2 return integrand - def phi(self, lmax, n_z): + def phi(self, lmax, n_z, cosmo, p, chi_recomb): z_array = np.logspace(-3, np.log10(10.), 200) chi_array = cosmo.comoving_distance(z_array).value chi_int = interp1d(chi_array, z_array, bounds_error=False, fill_value=0.0) n = self.compute_nz(n_z) - galaxy_W_int = interp1d(z_array, self.galaxy_W(z_array, n, chi_int), bounds_error=False, fill_value=0.0) + galaxy_W_int = interp1d(z_array, self.galaxy_W(z_array, n, chi_int, cosmo, chi_recomb), bounds_error=False, fill_value=0.0) phi_array = [] l = range(0, lmax, 1) l = np.array(l) for i in l: a = quad( - self.integrand_lensing_limber, 1.e-10, chi_recomb, args=(i, galaxy_W_int, chi_int), epsrel=1.e-6)[0] + self.integrand_lensing_limber, 1.e-10, chi_recomb, args=(i, galaxy_W_int, chi_int, p), epsrel=1.e-6)[0] phi_array.append(a) phi_array = np.array(phi_array) prefactor = 1.0 #(l+2)*(l+1)*l*(l-1) / (l+0.5)**4 return l, phi_array * prefactor - def theory_corr(self, n_z2, xvals, lmax2): - ll, pp = self.phi(lmax=lmax2, n_z=n_z2) + def theory_corr(self, n_z2, xvals, lmax2, cosmo, p, chi_recomb): + ll, pp = self.phi(lmax=lmax2, n_z=n_z2, cosmo=cosmo, p=p, chi_recomb=chi_recomb) pp3_2 = np.zeros((lmax2, 4)) pp3_2[:, 1] = pp[:] * (ll * (ll + 1.)) / (2. * np.pi) #xvals = np.logspace(np.log10(min_sep2), np.log10(max_sep2), nbins2) #in arcminutes @@ -162,7 +143,7 @@ def get_score(self, measured, theory, cov, opt='diagonal'): else: chi2 = np.sum([(measured[i] - theory[i])**2 / theory[i]**2 for i in range(len(measured))]) diff = chi2 / float(len(measured)) - return diff #np.sqrt(np.sum(diff))/float(len(measured)) + return diff def jackknife(self, catalog_data, xip, xim): " computing jack-knife covariance matrix using K-means clustering" @@ -263,6 +244,18 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): catalog_data = self.get_catalog_data( catalog_instance, [self.z, self.ra, self.dec, self.e1, self.e2, self.kappa], filters=self.filters) + + #TODO: ns set to 0.963 for now, as this isn't within astropy's cosmology dictionaries. + cosmo = catalog_instance.cosmology + pars.set_cosmology(H0 = cosmo.H0.value , ombh2 = cosmo.Ob0 * (cosmo.H0.value /100.)**2, omch2 = (cosmo.Om0 -cosmo.Ob0 )* (cosmo.H0.value /100.)**2) + pars.InitPower.set_params(ns = 0.963) + camb.set_halofit_version(version='takahashi') + p = camb.get_matter_power_interpolator( pars, nonlinear=True, k_hunit=False, hubble_units=False, kmax=100., zmax=1100., k_per_logint=False).P + chi_recomb = cosmo.comoving_distance(1100.).value + + + + print(cosmo) # read in shear values and check limits e1 = catalog_data[self.e1] max_e1 = np.max(e1) @@ -275,22 +268,6 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): if ((min_e2 < (-1.)) or (max_e2 > 1.0)): return TestResult(skipped=True, summary='e2 values out of range [-1,+1]') - # outputting values for easy testing on home computer - #import h5py - #f = h5py.File("/global/homes/p/plarsen/protoDC2_values.hdf5", "w") - #dset = f.create_dataset("shear_1", np.shape(e1),dtype='f') - #dset[...] = e1 - #dset2 = f.create_dataset("shear_2", np.shape(e1),dtype='f') - #dset3 = f.create_dataset("ra", np.shape(e1),dtype='f') - #dset4 = f.create_dataset("dec", np.shape(e1),dtype='f') - #dset5 = f.create_dataset("z", np.shape(e1),dtype='f') - #dset6 = f.create_dataset("kappa", np.shape(e1),dtype='f') - #dset2[...] = e2 - #dset3[...] = catalog_data[self.ra] - #dset4[...] = catalog_data[self.dec] - #dset5[...] = catalog_data[self.z] - #dset6[...] = catalog_data[self.kappa] - #f.close() # compute shear auto-correlation cat_s = treecorr.Catalog( @@ -315,9 +292,6 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): #sig = np.sqrt(gg.varxi) # this is shape noise only - should be very low for simulation data do_jackknife = self.do_jackknife - # get jackknife covariances. - #cp_xip,cp_xim = self.jackknife(catalog_data,xip,xim) - # Diagonal covariances for error bars on the plots. Use full covariance matrix for chi2 testing. if (do_jackknife == True): @@ -335,7 +309,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): sigm_jack[i] = np.sqrt(gg.varxi[i]) n_z = catalog_data[self.z] - xvals, theory_plus, theory_minus = self.theory_corr(n_z, r, 10000) + xvals, theory_plus, theory_minus = self.theory_corr(n_z, r, 10000, cosmo, p, chi_recomb) theory_plus = theory_plus * 1.e6 theory_minus = theory_minus * 1.e6 @@ -349,27 +323,12 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): print(theory_minus) print(xip) print(xim) - #determine chi2/dof values - ##chi2_dof_1= self.get_score(xip, theory_plus,cp_xip,opt='cov') # correct this - #chi2_dof_2 = self.get_score(xim, theory_minus, cp_xim,opt='cov') - - #print(chi2_dof_1) - #print(chi2_dof_2) - #print("chi2 values") - - #debugging output - #print("CP_XIP:") - #print(cp_xip) - #print(xip) - #print(sig_jack) - - # further correlation functions - can add in later - #dd = treecorr.NNCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') - #dd.process(cat) + + #The following are further treecorr correlation functions that could be added in later to extend the test + #treecorr.NNCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') #treecorr.NGCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') # count-shear (i.e. (R)) #treecorr.NKCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') # count-kappa (i.e. (R)) #treecorr.KKCorrelation(nbins=20, min_sep=2.5, max_sep=250, sep_units='arcmin') # count-kappa (i.e. (R)) - # etc... fig, ax = plt.subplots(2, sharex=True) for ax_this in (ax, self.summary_ax): @@ -383,6 +342,9 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): plt.close(fig) score = chi2_dof_1 #calculate your summary statistics + + #TODO: This criteria for the score is effectively a placeholder if jackknifing isn't used and assumes a diagonal covariance if it is + # Proper validation criteria need to be assigned to this test if (score < 2): return TestResult(score, passed=True) else: From 140a34ba6f0552419f27d93609366f33893e75df Mon Sep 17 00:00:00 2001 From: Vinu Vikraman Date: Thu, 22 Feb 2018 16:16:45 -0800 Subject: [PATCH 37/64] Modified SizeStellarMassLuminosity.py script to have single size validation and bulge and disk validation. --- descqa/SizeStellarMassLuminosity.py | 75 +++++++++++++------ ...size_Mandelbaum2014_bulge_disk_COSMOS.yaml | 48 ++++-------- descqa/configs/size_vanderWel2014_SM_Lum.yaml | 4 +- .../Mandelbaum2014_appmag_size_bulge_disk.txt | 2 +- 4 files changed, 71 insertions(+), 58 deletions(-) diff --git a/descqa/SizeStellarMassLuminosity.py b/descqa/SizeStellarMassLuminosity.py index cab12a26..7da5ec02 100644 --- a/descqa/SizeStellarMassLuminosity.py +++ b/descqa/SizeStellarMassLuminosity.py @@ -27,6 +27,8 @@ class SizeStellarMassLuminosity(BaseValidationTest): _ARCSEC_TO_RADIAN = np.pi / 180. / 3600. def __init__(self, **kwargs): + self.kwargs = kwargs + self.observation = kwargs['observation'] self.possible_mag_fields = kwargs['possible_mag_fields'] self.test_name = kwargs['test_name'] self.data_label = kwargs['data_label'] @@ -36,7 +38,6 @@ def __init__(self, **kwargs): self.fig_xlabel = kwargs['fig_xlabel'] self.fig_ylabel = kwargs['fig_ylabel'] - validation_filepath = os.path.join(self.data_dir, kwargs['data_filename']) self.validation_data = np.genfromtxt(validation_filepath) @@ -67,16 +68,17 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): Loop over magnitude cuts and make plots ''' # load catalog data - default_L_bin_edges = np.array([9, 9.5, 10, 10.5, 11, 11.5]) - default_L_bins = (default_L_bin_edges[1:] + default_L_bin_edges[:-1]) / 2. spl = redshift2dist(catalog_instance.cosmology) colnames = dict() - #colnames['z'] = catalog_instance.first_available('redshift', 'redshift_true') - #colnames['size'] = catalog_instance.first_available('size') colnames['z'] = 'redshift' - colnames['size'] = 'size' colnames['mag'] = catalog_instance.first_available(*self.possible_mag_fields) + if self.observation == 'onecomp': + colnames['size'] = 'size_true' + elif self.observation == 'twocomp': + colnames['size_bulge'] = 'size_bulge_true' + colnames['size_disk'] = 'size_disk_true' + if not all(v for v in colnames.values()): return TestResult(skipped=True, summary='Missing requested quantities') #Check whether the columns are finite or not @@ -105,22 +107,53 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): catalog_data_this = GCRQuery(*filters).filter(catalog_data) if len(catalog_data_this['z']) == 0: continue - size_kpc = catalog_data_this['size'] * self._ARCSEC_TO_RADIAN * interpolate.splev(catalog_data_this['z'], spl) / (1 + catalog_data_this['z']) - L_G, logL_G = self.ConvertAbsMagLuminosity(catalog_data_this['mag'], 'g') - binned_size_kpc, tmp_1, tmp_2 = binned_statistic(logL_G, size_kpc, bins=default_L_bin_edges, statistic='mean') - binned_size_kpc_err, tmp_1, tmp_2 = binned_statistic(logL_G, size_kpc, bins=default_L_bin_edges, statistic='std') - + z_mean = (z_bin['z_max'] + z_bin['z_min']) / 2. output_filepath = os.path.join(output_dir, self.output_filename_template.format(z_bin['z_min'], z_bin['z_max'])) - np.savetxt(output_filepath, np.transpose((default_L_bins, binned_size_kpc, binned_size_kpc_err))) - + colors = ['r', 'b'] + default_L_bin_edges = np.array([9, 9.5, 10, 10.5, 11, 11.5]) + default_L_bins = (default_L_bin_edges[1:] + default_L_bin_edges[:-1]) / 2. + if self.observation == 'onecomp': + L_G, logL_G = self.ConvertAbsMagLuminosity(catalog_data_this['mag'], 'g') + size_kpc = catalog_data_this['size'] * self._ARCSEC_TO_RADIAN * interpolate.splev(catalog_data_this['z'], spl) / (1 + catalog_data_this['z']) + binned_size_kpc, tmp_1, tmp_2 = binned_statistic(logL_G, size_kpc, bins=default_L_bin_edges, statistic='mean') + binned_size_kpc_err, tmp_1, tmp_2 = binned_statistic(logL_G, size_kpc, bins=default_L_bin_edges, statistic='std') + + np.savetxt(output_filepath, np.transpose((default_L_bins, binned_size_kpc, binned_size_kpc_err))) + + validation_this = self.validation_data[(self.validation_data[:,0] < z_mean + 0.25) & (self.validation_data[:,0] > z_mean - 0.25)] + + ax.semilogy(validation_this[:,1], 10**validation_this[:, 2], label=self.label_template.format(z_bin['z_min'], z_bin['z_max'])) + ax.fill_between(validation_this[:,1], 10**validation_this[:,3], 10**validation_this[:,4], lw=0, alpha=0.2) + ax.errorbar(default_L_bins, binned_size_kpc, binned_size_kpc_err, marker='o', ls='') + elif self.observation == 'twocomp': + L_I, logL_I = self.ConvertAbsMagLuminosity(catalog_data_this['mag'], 'i') + arcsec_to_kpc = self._ARCSEC_TO_RADIAN * interpolate.splev(catalog_data_this['z'], spl) / (1 + catalog_data_this['z']) + + binned_bulgesize_kpc, tmp_1, tmp_2 = binned_statistic(logL_I, catalog_data_this['size_bulge'] * arcsec_to_kpc, bins=default_L_bin_edges, statistic='mean') + binned_bulgesize_kpc_err, tmp_1, tmp_2 = binned_statistic(logL_I, catalog_data_this['size_bulge'] * arcsec_to_kpc, bins=default_L_bin_edges, statistic='std') + binned_disksize_kpc, tmp_1, tmp_2 = binned_statistic(logL_I, catalog_data_this['size_disk'] * arcsec_to_kpc, bins=default_L_bin_edges, statistic='mean') + binned_disksize_kpc_err, tmp_1, tmp_2 = binned_statistic(logL_I, catalog_data_this['size_disk'] * arcsec_to_kpc, bins=default_L_bin_edges, statistic='std') + binned_bulgesize_kpc = np.nan_to_num(binned_bulgesize_kpc) + binned_bulgesize_kpc_err = np.nan_to_num(binned_bulgesize_kpc_err) + binned_disksize_kpc = np.nan_to_num(binned_disksize_kpc) + binned_disksize_kpc_err = np.nan_to_num(binned_disksize_kpc_err) + np.savetxt(output_filepath, np.transpose((default_L_bins, binned_bulgesize_kpc, binned_bulgesize_kpc_err, binned_disksize_kpc, binned_disksize_kpc_err))) + + validation_this = self.validation_data[(self.validation_data[:,0] < z_mean + 0.25) & (self.validation_data[:,0] > z_mean - 0.25)] + + ax.text(11, 0.3, self.label_template.format(z_bin['z_min'], z_bin['z_max'])) + ax.semilogy(validation_this[:,1], validation_this[:, 2], label='Bulge', color=colors[0]) + ax.fill_between(validation_this[:,1], validation_this[:, 2] + validation_this[:,4], validation_this[:, 2] - validation_this[:,4], lw=0, alpha=0.2, facecolor=colors[0]) + ax.semilogy(validation_this[:,1] + 0.2, validation_this[:, 3], label='Disk', color=colors[1]) + ax.fill_between(validation_this[:,1] + 0.2, validation_this[:, 3] + validation_this[:,5], validation_this[:, 3] - validation_this[:,5], lw=0, alpha=0.2, facecolor=colors[1]) + + ax.errorbar(default_L_bins, binned_bulgesize_kpc, binned_bulgesize_kpc_err, marker='o', ls='', c=colors[0]) + ax.errorbar(default_L_bins+0.2, binned_disksize_kpc, binned_disksize_kpc_err, marker='o', ls='', c=colors[1]) + ax.set_xlim([9, 13]) + ax.set_ylim([1e-1, 25]) + ax.set_yscale('log', nonposy='clip') del catalog_data_this - z_mean = (z_bin['z_max'] + z_bin['z_min']) / 2. - validation_this = self.validation_data[(self.validation_data[:,0] < z_mean + 0.02) & (self.validation_data[:,0] > z_mean - 0.02)] - - ax.semilogy(validation_this[:,1], 10**validation_this[:, 2], label=self.label_template.format(z_bin['z_min'], z_bin['z_max'])) - ax.fill_between(validation_this[:,1], 10**validation_this[:,3], 10**validation_this[:,4], lw=0, alpha=0.2) - ax.errorbar(default_L_bins, binned_size_kpc, binned_size_kpc_err, marker='o', ls='') col += 1 if col > 2: col = 0 @@ -134,8 +167,8 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): plt.grid(False) plt.xlabel(self.fig_xlabel) plt.ylabel(self.fig_ylabel) - fig.subplots_adjust(hspace=0, wspace=0) - fig.suptitle('{} vs. {}'.format(catalog_name, self.data_label), fontsize='medium', y=0.93) + fig.subplots_adjust(hspace=0, wspace=0.2) + fig.suptitle('{} ($M_V$) vs. {}'.format(catalog_name, self.data_label), fontsize='medium', y=0.93) finally: fig.savefig(os.path.join(output_dir, '{:s}.png'.format(self.test_name)), bbox_inches='tight') plt.close(fig) diff --git a/descqa/configs/size_Mandelbaum2014_bulge_disk_COSMOS.yaml b/descqa/configs/size_Mandelbaum2014_bulge_disk_COSMOS.yaml index 69a23ed9..fe29b360 100644 --- a/descqa/configs/size_Mandelbaum2014_bulge_disk_COSMOS.yaml +++ b/descqa/configs/size_Mandelbaum2014_bulge_disk_COSMOS.yaml @@ -1,52 +1,30 @@ -subclass_name: SizeAppmagBulgeDiskCOSMOS.SizeAppmagBulgeDiskCOSMOS +subclass_name: SizeStellarMassLuminosity.SizeStellarMassLuminosity #observation: either protodc or buzzard -simulated_catalog: protodc -#simulated_catalog: buzzard +observation: twocomp -possible_appmag_fields: - - mag_i_lsst - - mag_i_sdss - - mag_i_des - -possible_AbsMag_fields: - - Mag_true_i_lsst_z01 - - Mag_true_i_sdss_z01 - - Mag_true_i_des_z01 - -size_fields: - - size_bulge_true - - size_disk_true - -size_field: size_true +possible_mag_fields: + - Mag_true_i_lsst_z0 + - Mag_true_i_sdss_z0 + - Mag_true_i_des_z0 mag_bin_separation: 1 -protodc_output_filename_template: 'size_COSMOS_z_{}_{}.dat' -buzzard_output_filename_template: 'size_van_z_{}_{}.dat' - -protodc_output_figname: 'size_protodc' -buzzrad_output_figname: 'size_buzzard' +output_filename_template: 'size_COSMOS_z_{}_{}.dat' -text_template: '${} < z < {}$' +label_template: '${} < z < {}$' -protodc_data_filename: 'size/Mandelbaum2014_sizes_appmag_bulge_disk.txt' -buzzard_data_filename: 'size/van_der_Wel_2014_ApJ_788_28_Table5_VLuminosity.txt' -protodc_data_label: 'Mandelbaum+2014: COSMOSIS comparing mean and scatter' -buzzard_data_label: 'van der Wel+2014: comparing mean and scatter' +data_filename: 'size/Mandelbaum2014_LumF814W_size_bulge_disk.txt' +data_label: 'Mandelbaum+2014: F814W comparing mean and scatter' z_bins: - {z_min: 0.0, z_max: 0.5} - {z_min: 0.5, z_max: 1.0} - {z_min: 1.0, z_max: 1.5} -protodc_fig_xlabel: '$i$ mag' -protodc_fig_ylabel: '$\log_{10}(R_e)$ (arcsec)' -buzzard_fig_xlabel: '$\log_{10}(L/L_{\odot})$ (V-band)' -buzzard_fig_ylabel: '$\log_{10}(R_e)$ (kpc)' +fig_xlabel: '$(L/L_{\odot})_{I}$' +fig_ylabel: '$\log_{10}(R_e)$ (kpc)' -protodc_description: | +description: | Compare evolution of bulge and disk sizes as a function of i-band magnitude of LSST and redshift and comparing to Mandelbaum et al (2015) HST COSMOS F814W observations -buzzard_description: | - Compare evolution of size as a function of luminosity and redshift and van der Wel et al (2014) HST-3D and CANDELS observations diff --git a/descqa/configs/size_vanderWel2014_SM_Lum.yaml b/descqa/configs/size_vanderWel2014_SM_Lum.yaml index b69548eb..ea815115 100644 --- a/descqa/configs/size_vanderWel2014_SM_Lum.yaml +++ b/descqa/configs/size_vanderWel2014_SM_Lum.yaml @@ -1,5 +1,7 @@ subclass_name: SizeStellarMassLuminosity.SizeStellarMassLuminosity +observation: onecomp + possible_mag_fields: - Mag_true_g_sdss_z0 - Mag_true_r_sdss_z0 @@ -17,7 +19,7 @@ possible_mag_fields: output_filename_template: 'size_lum_z_{}_{}.dat' label_template: '${} < z < {}$' -data_filename: 'size_sm_lum/van_der_Wel_2014_ApJ_788_28_Table5_VLuminosity.txt' +data_filename: 'size/van_der_Wel_2014_ApJ_788_28_Table5_VLuminosity.txt' data_label: 'van der Wel+2014: comparing mean and scatter' z_bins: diff --git a/descqa/data/size/Mandelbaum2014_appmag_size_bulge_disk.txt b/descqa/data/size/Mandelbaum2014_appmag_size_bulge_disk.txt index f93cb1a4..a32becec 100644 --- a/descqa/data/size/Mandelbaum2014_appmag_size_bulge_disk.txt +++ b/descqa/data/size/Mandelbaum2014_appmag_size_bulge_disk.txt @@ -1,4 +1,4 @@ -# zavg avg_mag avg_bhlr avg_dhlr std_bhlr std_dhlr +# zavg avg_mag_F814W avg_bhlr_F814W avg_dhlr_F814W std_bhlr_F814W std_dhlr_F814W 0.250 17.543 0.688 3.114 0.396 4.177 0.250 19.321 0.419 1.273 0.848 1.724 0.250 21.105 0.263 0.679 0.471 0.895 From 07e2968c55a4260c7c0d6e4e67de4cf99dd22161 Mon Sep 17 00:00:00 2001 From: Vinu Vikraman Date: Thu, 22 Feb 2018 16:27:18 -0800 Subject: [PATCH 38/64] Found the luminosity of COSMOS sample in F814W using photoz as Mandelbaum2014_LumF814W_size.txt --- ...OSMOS.yaml => size_Mandelbaum2014_BD.yaml} | 0 .../size/Mandelbaum2014_LumF814W_size.txt | 79 +++++++++++++++++++ ...andelbaum2014_LumF814W_size_bulge_disk.txt | 37 +++++++++ .../data/size/Mandelbaum2014_appmag_size.txt | 79 ------------------- .../Mandelbaum2014_appmag_size_bulge_disk.txt | 37 --------- 5 files changed, 116 insertions(+), 116 deletions(-) rename descqa/configs/{size_Mandelbaum2014_bulge_disk_COSMOS.yaml => size_Mandelbaum2014_BD.yaml} (100%) create mode 100644 descqa/data/size/Mandelbaum2014_LumF814W_size.txt create mode 100644 descqa/data/size/Mandelbaum2014_LumF814W_size_bulge_disk.txt delete mode 100644 descqa/data/size/Mandelbaum2014_appmag_size.txt delete mode 100644 descqa/data/size/Mandelbaum2014_appmag_size_bulge_disk.txt diff --git a/descqa/configs/size_Mandelbaum2014_bulge_disk_COSMOS.yaml b/descqa/configs/size_Mandelbaum2014_BD.yaml similarity index 100% rename from descqa/configs/size_Mandelbaum2014_bulge_disk_COSMOS.yaml rename to descqa/configs/size_Mandelbaum2014_BD.yaml diff --git a/descqa/data/size/Mandelbaum2014_LumF814W_size.txt b/descqa/data/size/Mandelbaum2014_LumF814W_size.txt new file mode 100644 index 00000000..e37d9e9a --- /dev/null +++ b/descqa/data/size/Mandelbaum2014_LumF814W_size.txt @@ -0,0 +1,79 @@ +# zavg LumF814W hlr_kpc hlr_kpc_std +0.250 11.232 75.433 199.107 +0.250 10.811 7.928 16.962 +0.250 10.422 8.806 43.753 +0.250 10.034 5.731 29.521 +0.250 9.654 2.795 8.377 +0.250 9.248 2.266 5.809 +0.250 8.846 1.530 2.734 +0.250 8.457 1.063 1.234 +0.250 8.070 0.753 0.550 +0.250 7.792 0.560 0.348 +0.250 7.344 0.273 0.149 +0.250 6.954 0.108 0.101 +0.250 6.468 0.074 0.000 +0.750 12.317 784.034 0.000 +0.750 12.124 1226.148 502.701 +0.750 11.521 35.209 70.495 +0.750 11.127 13.348 69.769 +0.750 10.751 6.372 24.079 +0.750 10.372 3.742 9.480 +0.750 9.983 2.554 4.579 +0.750 9.590 1.888 2.186 +0.750 9.209 1.380 1.099 +0.750 8.934 1.052 0.618 +0.750 8.432 0.531 0.373 +0.750 8.120 0.376 0.251 +0.750 7.536 0.822 0.690 +1.250 0.000 0.000 0.000 +1.250 12.529 2259.360 400.221 +1.250 12.064 723.960 690.871 +1.250 11.690 72.775 158.292 +1.250 11.276 16.112 55.159 +1.250 10.902 6.577 19.536 +1.250 10.520 3.923 8.279 +1.250 10.129 2.732 3.603 +1.250 9.751 1.724 1.411 +1.250 9.474 1.266 0.790 +1.250 8.982 0.628 0.392 +1.250 8.618 0.208 0.056 +1.250 8.315 0.017 0.000 +1.750 0.000 0.000 0.000 +1.750 12.992 2265.736 1224.665 +1.750 12.459 1158.031 90.097 +1.750 12.071 282.221 245.469 +1.750 11.650 135.186 179.722 +1.750 11.238 13.354 34.097 +1.750 10.884 5.158 12.815 +1.750 10.474 3.450 5.106 +1.750 10.093 1.823 1.652 +1.750 9.826 1.175 0.723 +1.750 9.316 0.584 0.354 +1.750 8.980 0.173 0.080 +1.750 0.000 0.000 0.000 +2.250 0.000 0.000 0.000 +2.250 0.000 0.000 0.000 +2.250 12.825 1670.598 0.000 +2.250 12.208 183.390 176.241 +2.250 11.889 55.061 92.828 +2.250 11.488 12.025 31.293 +2.250 11.151 4.299 10.688 +2.250 10.742 2.898 4.843 +2.250 10.360 1.687 1.560 +2.250 10.097 1.191 0.826 +2.250 9.650 0.648 0.449 +2.250 9.275 0.132 0.000 +2.250 0.000 0.000 0.000 +2.750 0.000 0.000 0.000 +2.750 0.000 0.000 0.000 +2.750 12.792 17.772 0.000 +2.750 12.533 12.048 1.975 +2.750 12.117 128.668 141.874 +2.750 11.707 7.150 24.896 +2.750 11.370 2.952 9.651 +2.750 10.946 2.652 4.181 +2.750 10.566 1.560 1.422 +2.750 10.312 1.099 0.788 +2.750 9.780 0.693 0.465 +2.750 0.000 0.000 0.000 +2.750 0.000 0.000 0.000 diff --git a/descqa/data/size/Mandelbaum2014_LumF814W_size_bulge_disk.txt b/descqa/data/size/Mandelbaum2014_LumF814W_size_bulge_disk.txt new file mode 100644 index 00000000..64b0ab71 --- /dev/null +++ b/descqa/data/size/Mandelbaum2014_LumF814W_size_bulge_disk.txt @@ -0,0 +1,37 @@ +# zavg LumF814W bhlr_kpc dhlr_kpc bhlr_kpc_std dhlr_kpc_std +0.250 10.844 2.690 12.175 1.547 16.333 +0.250 10.132 1.638 4.976 3.314 6.743 +0.250 9.419 1.030 2.654 1.842 3.501 +0.250 8.628 0.587 1.461 1.058 1.407 +0.250 7.986 0.308 0.788 0.299 0.481 +0.250 7.290 0.184 0.525 0.108 0.381 +0.750 0.000 0.000 0.000 0.000 0.000 +0.750 11.129 5.590 28.291 11.862 43.308 +0.750 10.446 1.786 5.670 3.766 8.225 +0.750 9.752 0.924 2.629 1.164 2.335 +0.750 9.145 0.558 1.451 0.553 0.902 +0.750 8.313 0.296 0.767 0.173 0.681 +1.250 0.000 0.000 0.000 0.000 0.000 +1.250 11.834 34.877 137.362 43.542 26.629 +1.250 10.942 9.560 24.965 21.768 32.557 +1.250 10.207 1.388 3.842 1.987 4.434 +1.250 9.677 0.697 1.804 0.701 1.164 +1.250 8.977 0.434 0.842 0.252 0.352 +1.750 0.000 0.000 0.000 0.000 0.000 +1.750 12.272 7.643 152.384 4.927 37.685 +1.750 11.395 15.984 59.931 21.188 36.195 +1.750 10.498 2.097 5.826 3.801 7.811 +1.750 9.999 0.727 1.801 0.716 1.344 +1.750 9.373 0.391 0.817 0.272 0.453 +2.250 0.000 0.000 0.000 0.000 0.000 +2.250 12.352 11.004 122.805 3.756 2.203 +2.250 11.663 41.667 57.836 53.552 44.190 +2.250 10.745 1.832 4.862 4.420 7.030 +2.250 10.273 0.702 1.703 0.686 1.312 +2.250 9.571 0.451 0.733 0.319 0.304 +2.750 0.000 0.000 0.000 0.000 0.000 +2.750 12.538 15.168 96.935 0.000 0.000 +2.750 11.907 22.664 55.237 30.239 34.557 +2.750 10.938 2.165 4.932 6.231 10.449 +2.750 10.467 0.637 1.517 0.689 1.123 +2.750 9.700 0.237 0.554 0.085 0.331 diff --git a/descqa/data/size/Mandelbaum2014_appmag_size.txt b/descqa/data/size/Mandelbaum2014_appmag_size.txt deleted file mode 100644 index 64fd942e..00000000 --- a/descqa/data/size/Mandelbaum2014_appmag_size.txt +++ /dev/null @@ -1,79 +0,0 @@ -# zavg mag_avg hlr_avg hlr_std -0.250 16.571 19.292 50.922 -0.250 17.625 2.028 4.338 -0.250 18.596 2.252 11.190 -0.250 19.568 1.466 7.550 -0.250 20.518 0.715 2.142 -0.250 21.532 0.580 1.486 -0.250 22.536 0.391 0.699 -0.250 23.509 0.272 0.316 -0.250 24.476 0.193 0.141 -0.250 25.172 0.143 0.089 -0.250 26.293 0.070 0.038 -0.250 27.267 0.028 0.026 -0.250 28.483 0.019 0.000 -0.750 16.687 106.844 0.000 -0.750 17.170 167.093 68.505 -0.750 18.677 4.798 9.607 -0.750 19.663 1.819 9.508 -0.750 20.604 0.868 3.281 -0.750 21.551 0.510 1.292 -0.750 22.523 0.348 0.624 -0.750 23.506 0.257 0.298 -0.750 24.458 0.188 0.150 -0.750 25.145 0.143 0.084 -0.750 26.400 0.072 0.051 -0.750 27.181 0.051 0.034 -0.750 28.639 0.112 0.094 -1.250 0.000 0.000 0.000 -1.250 17.527 270.914 47.990 -1.250 18.691 86.808 82.841 -1.250 19.625 8.726 18.980 -1.250 20.661 1.932 6.614 -1.250 21.595 0.789 2.343 -1.250 22.549 0.470 0.993 -1.250 23.528 0.328 0.432 -1.250 24.473 0.207 0.169 -1.250 25.165 0.152 0.095 -1.250 26.395 0.075 0.047 -1.250 27.306 0.025 0.007 -1.250 28.061 0.002 0.000 -1.750 0.000 0.000 0.000 -1.750 17.271 267.923 144.816 -1.750 18.604 136.937 10.654 -1.750 19.573 33.373 29.027 -1.750 20.626 15.986 21.252 -1.750 21.655 1.579 4.032 -1.750 22.541 0.610 1.515 -1.750 23.565 0.408 0.604 -1.750 24.520 0.216 0.195 -1.750 25.185 0.139 0.086 -1.750 26.462 0.069 0.042 -1.750 27.301 0.020 0.009 -1.750 0.000 0.000 0.000 -2.250 0.000 0.000 0.000 -2.250 0.000 0.000 0.000 -2.250 18.358 202.837 0.000 -2.250 19.900 22.267 21.398 -2.250 20.696 6.685 11.271 -2.250 21.701 1.460 3.799 -2.250 22.542 0.522 1.298 -2.250 23.564 0.352 0.588 -2.250 24.519 0.205 0.189 -2.250 25.176 0.145 0.100 -2.250 26.294 0.079 0.054 -2.250 27.233 0.016 0.000 -2.250 0.000 0.000 0.000 -2.750 0.000 0.000 0.000 -2.750 0.000 0.000 0.000 -2.750 18.969 2.252 0.000 -2.750 19.616 1.527 0.250 -2.750 20.657 16.304 17.977 -2.750 21.681 0.906 3.155 -2.750 22.524 0.374 1.223 -2.750 23.584 0.336 0.530 -2.750 24.533 0.198 0.180 -2.750 25.168 0.139 0.100 -2.750 26.499 0.088 0.059 -2.750 0.000 0.000 0.000 -2.750 0.000 0.000 0.000 diff --git a/descqa/data/size/Mandelbaum2014_appmag_size_bulge_disk.txt b/descqa/data/size/Mandelbaum2014_appmag_size_bulge_disk.txt deleted file mode 100644 index a32becec..00000000 --- a/descqa/data/size/Mandelbaum2014_appmag_size_bulge_disk.txt +++ /dev/null @@ -1,37 +0,0 @@ -# zavg avg_mag_F814W avg_bhlr_F814W avg_dhlr_F814W std_bhlr_F814W std_dhlr_F814W -0.250 17.543 0.688 3.114 0.396 4.177 -0.250 19.321 0.419 1.273 0.848 1.724 -0.250 21.105 0.263 0.679 0.471 0.895 -0.250 23.082 0.150 0.374 0.271 0.360 -0.250 24.686 0.079 0.202 0.077 0.123 -0.250 26.426 0.047 0.134 0.028 0.098 -0.750 0.000 0.000 0.000 0.000 0.000 -0.750 19.659 0.762 3.855 1.616 5.902 -0.750 21.365 0.243 0.773 0.513 1.121 -0.750 23.099 0.126 0.358 0.159 0.318 -0.750 24.617 0.076 0.198 0.075 0.123 -0.750 26.697 0.040 0.105 0.024 0.093 -1.250 0.000 0.000 0.000 0.000 0.000 -1.250 19.264 4.182 16.471 5.221 3.193 -1.250 21.494 1.146 2.993 2.610 3.904 -1.250 23.332 0.166 0.461 0.238 0.532 -1.250 24.656 0.084 0.216 0.084 0.140 -1.250 26.408 0.052 0.101 0.030 0.042 -1.750 0.000 0.000 0.000 0.000 0.000 -1.750 19.072 0.904 18.019 0.583 4.456 -1.750 21.265 1.890 7.087 2.505 4.280 -1.750 23.506 0.248 0.689 0.450 0.924 -1.750 24.753 0.086 0.213 0.085 0.159 -1.750 26.318 0.046 0.097 0.032 0.054 -2.250 0.000 0.000 0.000 0.000 0.000 -2.250 19.539 1.336 14.911 0.456 0.268 -2.250 21.262 5.059 7.022 6.502 5.365 -2.250 23.558 0.222 0.590 0.537 0.854 -2.250 24.737 0.085 0.207 0.083 0.159 -2.250 26.491 0.055 0.089 0.039 0.037 -2.750 0.000 0.000 0.000 0.000 0.000 -2.750 19.603 1.922 12.283 0.000 0.000 -2.750 21.180 2.872 6.999 3.832 4.379 -2.750 23.602 0.274 0.625 0.789 1.324 -2.750 24.782 0.081 0.192 0.087 0.142 -2.750 26.699 0.030 0.070 0.011 0.042 From 429ff2899868387a8c121c1168ee51d685e1e699 Mon Sep 17 00:00:00 2001 From: Melanie Simet Date: Fri, 23 Feb 2018 00:21:09 -0800 Subject: [PATCH 39/64] Fix bugs in SizeDistribution test, plus catch nans, infs, negative values, and anomalous optimal bin points --- descqa/SizeDistribution.py | 43 +++++++++++++------ ...COSMOS_Great3_sample_size_distribution.txt | 2 +- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/descqa/SizeDistribution.py b/descqa/SizeDistribution.py index c4df16ae..02875176 100644 --- a/descqa/SizeDistribution.py +++ b/descqa/SizeDistribution.py @@ -2,10 +2,13 @@ from .base import BaseValidationTest, TestResult from .plotting import plt from .utils import get_opt_binpoints +import os +import numpy as np +from itertools import count __all__ = ['SizeDistribution'] -class SizeDistribuction(BaseValidationTest): +class SizeDistribution(BaseValidationTest): """ validation test to check the slope of the size distribution at small sizes. """ @@ -44,12 +47,18 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): # check catalog data for required quantities key = catalog_instance.first_available(*self.acceptable_keys) if not key: - summary = 'Missing required quantity' + ' or '.join(['{}']*len(self.acceptable_keys) + summary = 'Missing required quantity' + ' or '.join(['{}']*len(self.acceptable_keys)) return TestResult(skipped=True, summary=summary.format(*self.acceptable_keys)) # get data catalog_data = catalog_instance.get_quantities(key) sizes = catalog_data[key] + good_data_mask = np.logical_not(np.logical_or(np.isinf(sizes), np.isnan(sizes))) + sizes = sizes[good_data_mask] + non_neg_mask = sizes>0 + if np.sum(non_neg_mask)>0: + print('Warning: some sizes were negative or zero; these are being masked') + sizes = sizes[non_neg_mask] min_sizes = np.min(sizes) max_sizes = np.max(sizes) @@ -62,6 +71,11 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): sumM = np.histogram(sizes, weights=sizes, bins=bin_edges)[0] sumM2 = np.histogram(sizes, weights=sizes**2, bins=bin_edges)[0] size_pts = get_opt_binpoints(N, sumM, sumM2, bin_edges) + diff = size_pts[1:] - size_pts[:-1] + if not np.all(diff>=0): + # Sparsely populated bins sometimes cause problems for + # get_opt_binpoints; replace with the dumb solution + size_pts = 0.5*(bin_edges[:-1]+bin_edges[1:]) mask = size_pts min_sizes) & (validation_data[:,0] min_sizes) & (validation_data[:,0] Date: Fri, 23 Feb 2018 00:24:06 -0800 Subject: [PATCH 40/64] Remove extra lines from copied file in size_dist config file --- descqa/configs/size_dist_COSMOS.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/descqa/configs/size_dist_COSMOS.yaml b/descqa/configs/size_dist_COSMOS.yaml index 02e2b2bd..9aed647b 100644 --- a/descqa/configs/size_dist_COSMOS.yaml +++ b/descqa/configs/size_dist_COSMOS.yaml @@ -4,9 +4,6 @@ possible_size_fields: - size - size_true -output_filename_template: 'w_theta_r_{}_{}.dat' -label_template: '${} < r < {}$' - data_filename: 'size_dist/COSMOS_Great3_sample_size_distribution.txt' description: Compare slope of N(size) to the same slope in the Great3 COSMOS postage stamps From 98c42f411d965cb56567121d526d0ebdeb96578d Mon Sep 17 00:00:00 2001 From: Patricia Larsen Date: Fri, 23 Feb 2018 08:28:22 -0800 Subject: [PATCH 41/64] syntax and removal of old cosmology --- descqa/shear_test.py | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/descqa/shear_test.py b/descqa/shear_test.py index aa9f166e..be553bf3 100644 --- a/descqa/shear_test.py +++ b/descqa/shear_test.py @@ -9,7 +9,6 @@ import treecorr import camb import camb.correlations -from astropy.cosmology import FlatLambdaCDM import astropy.units as u import astropy.constants as const from GCR import GCRQuery @@ -90,7 +89,7 @@ def galaxy_W(self, z, n, chi_int, cosmo, chi_recomb): #pylint: disable=E1101 chi = cosmo.comoving_distance(z).value # can be array cst = 3. / 2. * cosmo.H(0).to(1. / u.s)**2 / const.c.to(u.Mpc / u.s)**2 * cosmo.Om( - 0) + 0) prefactor = cst * chi * (1. + z) * u.Mpc val_array = [] for i in range(len(z)): @@ -133,17 +132,17 @@ def theory_corr(self, n_z2, xvals, lmax2, cosmo, p, chi_recomb): return xvals, vals[:, 1], vals[:, 2] def get_score(self, measured, theory, cov, opt='diagonal'): - if (opt == 'cov'): + if opt == 'cov': cov = np.matrix(cov).I print("inverse covariance matrix") print(cov) chi2 = np.matrix(measured - theory) * cov * np.matrix(measured - theory).T - elif (opt == 'diagonal'): + elif opt == 'diagonal': chi2 = np.sum([(measured[i] - theory[i])**2 / cov[i][i] for i in range(len(measured))]) else: chi2 = np.sum([(measured[i] - theory[i])**2 / theory[i]**2 for i in range(len(measured))]) diff = chi2 / float(len(measured)) - return diff + return diff def jackknife(self, catalog_data, xip, xim): " computing jack-knife covariance matrix using K-means clustering" @@ -201,6 +200,9 @@ def jackknife(self, catalog_data, xip, xim): @staticmethod def get_catalog_data(gc, quantities, filters=None): + ''' + Get quantities from catalog + ''' data = {} if not gc.has_quantities(quantities): return TestResult(skipped=True, summary='Missing requested quantities') @@ -214,6 +216,9 @@ def get_catalog_data(gc, quantities, filters=None): # define theory from within this class def post_process_plot(self, ax): + ''' + Post-processing routines on plot + ''' #ax.text(0.05, 0.95, "add text here") plt.xscale('log') ax[0].legend() @@ -236,6 +241,9 @@ def post_process_plot(self, ax): ax[1].set_ylim([min_height2, max_height2]) def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): + ''' + run test on a single catalog + ''' # check if needed quantities exist if not catalog_instance.has_quantities([self.z, self.ra, self.dec]): return TestResult(skipped=True, summary='do not have needed location quantities') @@ -245,12 +253,12 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): catalog_instance, [self.z, self.ra, self.dec, self.e1, self.e2, self.kappa], filters=self.filters) - #TODO: ns set to 0.963 for now, as this isn't within astropy's cosmology dictionaries. + #TODO: ns set to 0.963 for now, as this isn't within astropy's cosmology dictionaries. cosmo = catalog_instance.cosmology - pars.set_cosmology(H0 = cosmo.H0.value , ombh2 = cosmo.Ob0 * (cosmo.H0.value /100.)**2, omch2 = (cosmo.Om0 -cosmo.Ob0 )* (cosmo.H0.value /100.)**2) - pars.InitPower.set_params(ns = 0.963) + pars.set_cosmology(H0=cosmo.H0.value, ombh2=cosmo.Ob0*(cosmo.H0.value /100.)**2, omch2=(cosmo.Om0-cosmo.Ob0)*(cosmo.H0.value /100.)**2) + pars.InitPower.set_params(ns=0.963) camb.set_halofit_version(version='takahashi') - p = camb.get_matter_power_interpolator( pars, nonlinear=True, k_hunit=False, hubble_units=False, kmax=100., zmax=1100., k_per_logint=False).P + p = camb.get_matter_power_interpolator(pars, nonlinear=True, k_hunit=False, hubble_units=False, kmax=100., zmax=1100., k_per_logint=False).P chi_recomb = cosmo.comoving_distance(1100.).value @@ -294,7 +302,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): do_jackknife = self.do_jackknife # Diagonal covariances for error bars on the plots. Use full covariance matrix for chi2 testing. - if (do_jackknife == True): + if do_jackknife: cp_xip, cp_xim = self.jackknife(catalog_data, xip, xim) sig_jack = np.zeros((self.nbins)) sigm_jack = np.zeros((self.nbins)) @@ -314,7 +322,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): theory_plus = theory_plus * 1.e6 theory_minus = theory_minus * 1.e6 - if (do_jackknife == True): + if do_jackknife: chi2_dof_1 = self.get_score(xip, theory_plus, cp_xip, opt='diagonal') # correct this else: chi2_dof_1 = self.get_score(xip, theory_plus, 0, opt='nojack') # correct this @@ -344,8 +352,8 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): score = chi2_dof_1 #calculate your summary statistics #TODO: This criteria for the score is effectively a placeholder if jackknifing isn't used and assumes a diagonal covariance if it is - # Proper validation criteria need to be assigned to this test - if (score < 2): + # Proper validation criteria need to be assigned to this test + if score < 2: return TestResult(score, passed=True) else: return TestResult(score, passed=False) From 401940cd0767443d4750f45c0171dba0064b1ffc Mon Sep 17 00:00:00 2001 From: Vinu Vikraman Date: Fri, 23 Feb 2018 09:13:18 -0800 Subject: [PATCH 42/64] Removed SizeAppmagBulgeDiskCOSMOS.py from size-luminosity-test branch --- descqa/SizeAppmagBulgeDiskCOSMOS.py | 169 ---------------------------- 1 file changed, 169 deletions(-) delete mode 100644 descqa/SizeAppmagBulgeDiskCOSMOS.py diff --git a/descqa/SizeAppmagBulgeDiskCOSMOS.py b/descqa/SizeAppmagBulgeDiskCOSMOS.py deleted file mode 100644 index 5b43f67b..00000000 --- a/descqa/SizeAppmagBulgeDiskCOSMOS.py +++ /dev/null @@ -1,169 +0,0 @@ -from __future__ import print_function, division, unicode_literals, absolute_import -import os -import numpy as np -from GCR import GCRQuery -from scipy import interpolate -from scipy.stats import binned_statistic - -from .base import BaseValidationTest, TestResult -from .plotting import plt -from .utils import generate_uniform_random_ra_dec_footprint, get_healpixel_footprint, generate_uniform_random_dist - - -__all__ = ['SizeAppmagBulgeDiskCOSMOS'] - - -class SizeAppmagBulgeDiskCOSMOS(BaseValidationTest): - """ - Validation test of 2pt correlation function - """ - _ARCSEC_TO_RADIAN = np.pi / 180. / 3600. - - def __init__(self, **kwargs): - self.simulated_catalog = kwargs['simulated_catalog'] - if self.simulated_catalog == 'protodc': - self.possible_mag_fields = kwargs['possible_appmag_fields'] - self.data_label = kwargs['protodc_data_label'] - self.output_filename_template = kwargs['protodc_output_filename_template'] - self.mag_bin_separation = kwargs['mag_bin_separation'] - self.fig_xlabel = kwargs['protodc_fig_xlabel'] - self.fig_ylabel = kwargs['protodc_fig_ylabel'] - - validation_filepath = os.path.join(self.data_dir, kwargs['protodc_data_filename']) - self.validation_data = np.genfromtxt(validation_filepath) - elif self.simulated_catalog == 'buzzard': - self.possible_mag_fields = kwargs['possible_AbsMag_fields'] - self.data_label = kwargs['buzzard_data_label'] - self.output_filename_template = kwargs['buzzard_output_filename_template'] - self.fig_xlabel = kwargs['buzzard_fig_xlabel'] - self.fig_ylabel = kwargs['buzzard_fig_ylabel'] - - validation_filepath = os.path.join(self.data_dir, kwargs['buzzard_data_filename']) - self.validation_data = np.genfromtxt(validation_filepath) - self.test_name = kwargs['test_name'] - self.z_bins = kwargs['z_bins'] - self.text_template = kwargs['text_template'] - self.colors = ['r', 'b'] - - def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): - ''' - Loop over magnitude cuts and make plots - ''' - if self.simulated_catalog == 'protodc': - # load catalog data - default_mag_bin_edges = np.arange(17, 32, self.mag_bin_separation) - default_mag_bins = (default_mag_bin_edges[1:] + default_mag_bin_edges[:-1]) / 2. - colnames = dict() - colnames['z'] = 'redshift' - colnames['size_bulge'] = 'size_bulge_true' - colnames['size_disk'] = 'size_disk_true' - colnames['mag'] = catalog_instance.first_available(*self.possible_mag_fields) - if not all(v for v in colnames.values()): - return TestResult(skipped=True, summary='Missing requested quantities') - #Check whether the columns are finite or not - filters = [(np.isfinite, c) for c in colnames.values()] - - #Select objects within maximum and minimum redshift of all the bins - filters.extend(( - '{} < {}'.format(colnames['z'], max(z_bin['z_max'] for z_bin in self.z_bins)), - '{} >= {}'.format(colnames['z'], min(z_bin['z_min'] for z_bin in self.z_bins)), - )) - catalog_data = catalog_instance.get_quantities(list(colnames.values()), filters=filters) - catalog_data = {k: catalog_data[v] for k, v in colnames.items()} - elif self.simulated_catalog == 'buzzard': - # load catalog data - default_L_bin_edges = np.array([9, 9.5, 10, 10.5, 11, 11.5]) - default_L_bins = (default_L_bin_edges[1:] + default_L_bin_edges[:-1]) / 2. - spl = redshift2dist(catalog_instance.cosmology) - - colnames = dict() - #colnames['z'] = catalog_instance.first_available('redshift', 'redshift_true') - #colnames['size'] = catalog_instance.first_available('size') - colnames['z'] = 'redshift' - colnames['size'] = 'size' - colnames['mag'] = catalog_instance.first_available(*self.possible_mag_fields) - if not all(v for v in colnames.values()): - return TestResult(skipped=True, summary='Missing requested quantities') - #Check whether the columns are finite or not - filters = [(np.isfinite, c) for c in colnames.values()] - - #Select objects within maximum and minimum redshift of all the bins - filters.extend(( - '{} < {}'.format(colnames['z'], max(z_bin['z_max'] for z_bin in self.z_bins)), - '{} >= {}'.format(colnames['z'], min(z_bin['z_min'] for z_bin in self.z_bins)), - )) - catalog_data = catalog_instance.get_quantities(list(colnames.values()), filters=filters) - catalog_data = {k: catalog_data[v] for k, v in colnames.items()} - - fig, axes = plt.subplots(1, 3, figsize=(9, 3), sharex=True, sharey=True) - try: - col = 0 - for z_bin in self.z_bins: - ax = axes[col] - # filter catalog data for this bin - filters = [ - 'z < {}'.format(z_bin['z_max']), - 'z >= {}'.format(z_bin['z_min']), - ] - - z_mean = (z_bin['z_max'] + z_bin['z_min']) / 2. - output_filepath = os.path.join(output_dir, self.output_filename_template.format(z_bin['z_min'], z_bin['z_max'])) - catalog_data_this = GCRQuery(*filters).filter(catalog_data) - if len(catalog_data_this['z']) == 0: - continue - if self.simulated_catalog == 'protodc': - binned_bulgesize_arcsec, tmp_1, tmp_2 = binned_statistic(catalog_data_this['mag'], catalog_data_this['size_bulge'], bins=default_mag_bin_edges, statistic='mean') - binned_bulgesize_arcsec_err, tmp_1, tmp_2 = binned_statistic(catalog_data_this['mag'], catalog_data_this['size_bulge'], bins=default_mag_bin_edges, statistic='std') - binned_disksize_arcsec, tmp_1, tmp_2 = binned_statistic(catalog_data_this['mag'], catalog_data_this['size_disk'], bins=default_mag_bin_edges, statistic='mean') - binned_disksize_arcsec_err, tmp_1, tmp_2 = binned_statistic(catalog_data_this['mag'], catalog_data_this['size_disk'], bins=default_mag_bin_edges, statistic='std') - binned_bulgesize_arcsec = np.nan_to_num(binned_bulgesize_arcsec) - binned_bulgesize_arcsec_err = np.nan_to_num(binned_bulgesize_arcsec_err) - binned_disksize_arcsec = np.nan_to_num(binned_disksize_arcsec) - binned_disksize_arcsec_err = np.nan_to_num(binned_disksize_arcsec_err) - np.savetxt(output_filepath, np.transpose((default_mag_bins, binned_bulgesize_arcsec, binned_bulgesize_arcsec_err, binned_disksize_arcsec, binned_disksize_arcsec_err))) - - validation_this = self.validation_data[(self.validation_data[:,0] < z_mean + 0.25) & (self.validation_data[:,0] > z_mean - 0.25)] - - ax.text(23, 1, self.text_template.format(z_bin['z_min'], z_bin['z_max'])) - ax.semilogy(validation_this[:,1], validation_this[:, 2], label='Bulge', color=self.colors[0]) - ax.fill_between(validation_this[:,1], validation_this[:, 2] + validation_this[:,4], validation_this[:, 2] - validation_this[:,4], lw=0, alpha=0.2, facecolor=self.colors[0]) - ax.semilogy(validation_this[:,1] + 0.5, validation_this[:, 3], label='Disk', color=self.colors[1]) - ax.fill_between(validation_this[:,1] + 0.5, validation_this[:, 3] + validation_this[:,5], validation_this[:, 3] - validation_this[:,5], lw=0, alpha=0.2, facecolor=self.colors[1]) - - ax.errorbar(default_mag_bins, binned_bulgesize_arcsec, binned_bulgesize_arcsec_err, marker='o', ls='', c=self.colors[0]) - ax.errorbar(default_mag_bins+0.5, binned_disksize_arcsec, binned_disksize_arcsec_err, marker='o', ls='', c=self.colors[1]) - ax.set_xlim([17, 29]) - ax.set_ylim([1e-2, 1e1]) - ax.set_yscale('log', nonposy='clip') - elif self.simulated_catalog == 'buzzard': - size_kpc = catalog_data_this['size'] * self._ARCSEC_TO_RADIAN * interpolate.splev(catalog_data_this['z'], spl) / (1 + catalog_data_this['z']) - L_G, logL_G = self.ConvertAbsMagLuminosity(catalog_data_this['mag'], 'g') - binned_size_kpc, tmp_1, tmp_2 = binned_statistic(logL_G, size_kpc, bins=default_L_bin_edges, statistic='mean') - binned_size_kpc_err, tmp_1, tmp_2 = binned_statistic(logL_G, size_kpc, bins=default_L_bin_edges, statistic='std') - np.savetxt(output_filepath, np.transpose((default_L_bins, binned_size_kpc, binned_size_kpc_err))) - - validation_this = self.validation_data[(self.validation_data[:,0] < z_mean + 0.02) & (self.validation_data[:,0] > z_mean - 0.02)] - - ax.semilogy(validation_this[:,1], 10**validation_this[:, 2], label=self.text_template.format(z_bin['z_min'], z_bin['z_max'])) - ax.fill_between(validation_this[:,1], 10**validation_this[:,3], 10**validation_this[:,4], lw=0, alpha=0.2) - ax.errorbar(default_L_bins, binned_size_kpc, binned_size_kpc_err, marker='o', ls='') - ax.set_yscale('log', nonposy='clip') - del catalog_data_this - col += 1 - ax.legend(loc='best') - - fig.add_subplot(111, frameon=False) - # hide tick and tick label of the big axes - plt.tick_params(labelcolor='none', which='both', top='off', bottom='off', left='off', right='off') - plt.grid(False) - plt.xlabel(self.fig_xlabel) - plt.ylabel(self.fig_ylabel) - plt.ylabel(self.fig_ylabel) - fig.subplots_adjust(hspace=0, wspace=0) - fig.suptitle('{} vs. {}'.format(catalog_name, self.data_label), fontsize='medium', y=0.93) - finally: - fig.savefig(os.path.join(output_dir, '{:s}.png'.format(self.test_name)), bbox_inches='tight') - plt.close(fig) - - #TODO: calculate summary statistics - return TestResult(0, passed=True) From 6ddb848c40132e5c615b69cff35d61da99018030 Mon Sep 17 00:00:00 2001 From: Vinu Vikraman Date: Fri, 23 Feb 2018 12:57:27 -0800 Subject: [PATCH 43/64] Removed unused variables in SizeStellarMassLuminosity.py --- descqa/SizeStellarMassLuminosity.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/descqa/SizeStellarMassLuminosity.py b/descqa/SizeStellarMassLuminosity.py index 7da5ec02..b9028b78 100644 --- a/descqa/SizeStellarMassLuminosity.py +++ b/descqa/SizeStellarMassLuminosity.py @@ -7,7 +7,6 @@ from .base import BaseValidationTest, TestResult from .plotting import plt -from .utils import generate_uniform_random_ra_dec_footprint, get_healpixel_footprint, generate_uniform_random_dist __all__ = ['SizeStellarMassLuminosity'] @@ -61,7 +60,7 @@ def ConvertAbsMagLuminosity(AbsM, band): logL = (AbsSun - AbsM) / 2.5 #unit of sun L = 10**logL - return L, logL + return logL def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): ''' @@ -113,10 +112,10 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): default_L_bin_edges = np.array([9, 9.5, 10, 10.5, 11, 11.5]) default_L_bins = (default_L_bin_edges[1:] + default_L_bin_edges[:-1]) / 2. if self.observation == 'onecomp': - L_G, logL_G = self.ConvertAbsMagLuminosity(catalog_data_this['mag'], 'g') + logL_G = self.ConvertAbsMagLuminosity(catalog_data_this['mag'], 'g') size_kpc = catalog_data_this['size'] * self._ARCSEC_TO_RADIAN * interpolate.splev(catalog_data_this['z'], spl) / (1 + catalog_data_this['z']) - binned_size_kpc, tmp_1, tmp_2 = binned_statistic(logL_G, size_kpc, bins=default_L_bin_edges, statistic='mean') - binned_size_kpc_err, tmp_1, tmp_2 = binned_statistic(logL_G, size_kpc, bins=default_L_bin_edges, statistic='std') + binned_size_kpc = binned_statistic(logL_G, size_kpc, bins=default_L_bin_edges, statistic='mean')[0] + binned_size_kpc_err = binned_statistic(logL_G, size_kpc, bins=default_L_bin_edges, statistic='std')[0] np.savetxt(output_filepath, np.transpose((default_L_bins, binned_size_kpc, binned_size_kpc_err))) @@ -126,13 +125,13 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): ax.fill_between(validation_this[:,1], 10**validation_this[:,3], 10**validation_this[:,4], lw=0, alpha=0.2) ax.errorbar(default_L_bins, binned_size_kpc, binned_size_kpc_err, marker='o', ls='') elif self.observation == 'twocomp': - L_I, logL_I = self.ConvertAbsMagLuminosity(catalog_data_this['mag'], 'i') + logL_I = self.ConvertAbsMagLuminosity(catalog_data_this['mag'], 'i') arcsec_to_kpc = self._ARCSEC_TO_RADIAN * interpolate.splev(catalog_data_this['z'], spl) / (1 + catalog_data_this['z']) - binned_bulgesize_kpc, tmp_1, tmp_2 = binned_statistic(logL_I, catalog_data_this['size_bulge'] * arcsec_to_kpc, bins=default_L_bin_edges, statistic='mean') - binned_bulgesize_kpc_err, tmp_1, tmp_2 = binned_statistic(logL_I, catalog_data_this['size_bulge'] * arcsec_to_kpc, bins=default_L_bin_edges, statistic='std') - binned_disksize_kpc, tmp_1, tmp_2 = binned_statistic(logL_I, catalog_data_this['size_disk'] * arcsec_to_kpc, bins=default_L_bin_edges, statistic='mean') - binned_disksize_kpc_err, tmp_1, tmp_2 = binned_statistic(logL_I, catalog_data_this['size_disk'] * arcsec_to_kpc, bins=default_L_bin_edges, statistic='std') + binned_bulgesize_kpc = binned_statistic(logL_I, catalog_data_this['size_bulge'] * arcsec_to_kpc, bins=default_L_bin_edges, statistic='mean')[0] + binned_bulgesize_kpc_err = binned_statistic(logL_I, catalog_data_this['size_bulge'] * arcsec_to_kpc, bins=default_L_bin_edges, statistic='std')[0] + binned_disksize_kpc = binned_statistic(logL_I, catalog_data_this['size_disk'] * arcsec_to_kpc, bins=default_L_bin_edges, statistic='mean')[0] + binned_disksize_kpc_err = binned_statistic(logL_I, catalog_data_this['size_disk'] * arcsec_to_kpc, bins=default_L_bin_edges, statistic='std')[0] binned_bulgesize_kpc = np.nan_to_num(binned_bulgesize_kpc) binned_bulgesize_kpc_err = np.nan_to_num(binned_bulgesize_kpc_err) binned_disksize_kpc = np.nan_to_num(binned_disksize_kpc) From eaf8d0c2e048a1b54afcc2d93a155cb0de40267c Mon Sep 17 00:00:00 2001 From: Patricia Larsen Date: Fri, 23 Feb 2018 13:16:54 -0800 Subject: [PATCH 44/64] note on scaling factor --- descqa/shear_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/descqa/shear_test.py b/descqa/shear_test.py index be553bf3..6f3aa00b 100644 --- a/descqa/shear_test.py +++ b/descqa/shear_test.py @@ -295,6 +295,8 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): gg.process(cat_s) r = np.exp(gg.meanlogr) + + #NOTE: We are computing 10^6 x correlation function for easier comparison xip = gg.xip * 1.e6 xim = gg.xim * 1.e6 #sig = np.sqrt(gg.varxi) # this is shape noise only - should be very low for simulation data @@ -313,8 +315,8 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): sig_jack = np.zeros((self.nbins)) sigm_jack = np.zeros((self.nbins)) for i in range(self.nbins): - sig_jack[i] = np.sqrt(gg.varxi[i]) - sigm_jack[i] = np.sqrt(gg.varxi[i]) + sig_jack[i] = np.sqrt(gg.varxi[i])*1.e6 + sigm_jack[i] = np.sqrt(gg.varxi[i])*1.e6 n_z = catalog_data[self.z] xvals, theory_plus, theory_minus = self.theory_corr(n_z, r, 10000, cosmo, p, chi_recomb) From 41730b53f21fccc15d2d7182c0c61652d4cac95b Mon Sep 17 00:00:00 2001 From: Melanie Simet Date: Fri, 23 Feb 2018 13:55:15 -0800 Subject: [PATCH 45/64] Address changes requested by reviewer --- descqa/SizeDistribution.py | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/descqa/SizeDistribution.py b/descqa/SizeDistribution.py index 02875176..2f789162 100644 --- a/descqa/SizeDistribution.py +++ b/descqa/SizeDistribution.py @@ -1,10 +1,9 @@ -from GCR import GCRQuery -from .base import BaseValidationTest, TestResult -from .plotting import plt -from .utils import get_opt_binpoints import os import numpy as np from itertools import count +from .base import BaseValidationTest, TestResult +from .plotting import plt +from .utils import get_opt_binpoints __all__ = ['SizeDistribution'] @@ -21,24 +20,15 @@ class SizeDistribution(BaseValidationTest): yaxis_xoffset = 0.02 yaxis_yoffset = 0.5 - def __init__(self, observation='', **kwargs): - + def __init__(self, **kwargs): + #pylint: disable=W0231 #validation data validation_filepath = os.path.join(self.data_dir, kwargs['data_filename']) self.validation_data = np.loadtxt(validation_filepath) self.acceptable_keys = kwargs['possible_size_fields'] - #check for valid observations - if not observation: - print('Warning: no data file supplied, no observation requested; only catalog data will be shown') - elif observation not in self.possible_observations: - raise ValueError('Observation {} not available'.format(observation)) - else: - self.validation_data = self.get_validation_data(observation) - self._color_iterator = ('C{}'.format(i) for i in count()) - self._other_kwargs = kwargs def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): # update color and marker to preserve catalog colors and markers across tests @@ -90,8 +80,8 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): validation_data = self.validation_data.copy() validation_mask = (validation_data[:,0] > min_sizes) & (validation_data[:,0] Date: Fri, 23 Feb 2018 13:58:09 -0800 Subject: [PATCH 46/64] Fix whitespace errors --- descqa/SizeDistribution.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/descqa/SizeDistribution.py b/descqa/SizeDistribution.py index 2f789162..4a473065 100644 --- a/descqa/SizeDistribution.py +++ b/descqa/SizeDistribution.py @@ -45,8 +45,8 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): sizes = catalog_data[key] good_data_mask = np.logical_not(np.logical_or(np.isinf(sizes), np.isnan(sizes))) sizes = sizes[good_data_mask] - non_neg_mask = sizes>0 - if np.sum(non_neg_mask)>0: + non_neg_mask = sizes > 0 + if np.sum(non_neg_mask) > 0: print('Warning: some sizes were negative or zero; these are being masked') sizes = sizes[non_neg_mask] min_sizes = np.min(sizes) @@ -56,18 +56,18 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): # Things seem to be roughly linear where N(size)>0.5*Ntot so use those points. # Get ~20 points for the line fit, but compute the whole graph median = np.median(sizes) - n_bins = int(20*(max_sizes-min_sizes)/(median-min_sizes)) + n_bins = int(20*(max_sizes-min_sizes)/(median-min_sizes)) N, bin_edges = np.histogram(sizes, bins=n_bins) sumM = np.histogram(sizes, weights=sizes, bins=bin_edges)[0] sumM2 = np.histogram(sizes, weights=sizes**2, bins=bin_edges)[0] size_pts = get_opt_binpoints(N, sumM, sumM2, bin_edges) diff = size_pts[1:] - size_pts[:-1] - if not np.all(diff>=0): + if not np.all(diff >= 0): # Sparsely populated bins sometimes cause problems for # get_opt_binpoints; replace with the dumb solution size_pts = 0.5*(bin_edges[:-1]+bin_edges[1:]) - mask = size_pts min_sizes) & (validation_data[:,0] min_sizes) & (validation_data[:, 0] < median) + validation_data[:, 1] /= validation_data[validation_mask, 1][0] validation_slope, _ = np.polyfit(validation_data[validation_mask, 0], - validation_data[validation_mask, 1], 1) + validation_data[validation_mask, 1], 1) # plot a histogram of sizes. This is easier to see as log(sizes) so do that. fig, (hist_ax, cumul_ax) = plt.subplots(1, 2) fig.subplots_adjust(wspace=0.4) - hist_ax.hist(np.log10(sizes), color=catalog_color, edgecolor='black', alpha=0.75, + hist_ax.hist(np.log10(sizes), color=catalog_color, edgecolor='black', alpha=0.75, normed=True, bins=20) hist_ax.set_xlabel("Log10({})".format(key)) hist_ax.set_ylabel("dN/d log({})".format(key)) @@ -96,7 +96,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): cumul_ax.plot(size_pts[mask], (data_intercept+data_slope*size_pts[mask]), color='gray') cumul_ax.set_xscale('log') cumul_ax.text(0.95, 0.96, - 'validation=${:.3f}$\nslope=${:.3f}$'.format(validation_slope, data_slope), + 'validation=${:.3f}$\nslope=${:.3f}$'.format(validation_slope, data_slope), horizontalalignment='right', verticalalignment='top', transform=cumul_ax.transAxes) cumul_ax.set_xlabel("{}".format(key)) @@ -105,7 +105,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): with open(os.path.join(output_dir, 'size_distribution_{}.txt'.format(catalog_name)), 'w' ) as f: f.write("# Slope, intercept\n") - f.write("%7f %9f\n"%(data_slope, data_intercept)) + f.write("%7f %9f\n"%(data_slope, data_intercept)) fig.savefig(os.path.join(output_dir, 'size_distribution_{}.png'.format(catalog_name))) plt.close(fig) From a8b9f16baeac930a6f61b90e6cb5eb70522021b2 Mon Sep 17 00:00:00 2001 From: Yao-Yuan Mao Date: Fri, 23 Feb 2018 21:28:25 -0500 Subject: [PATCH 47/64] fix linting errors --- descqa/SizeStellarMassLuminosity.py | 45 ++++++++++++++--------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/descqa/SizeStellarMassLuminosity.py b/descqa/SizeStellarMassLuminosity.py index b9028b78..980957d4 100644 --- a/descqa/SizeStellarMassLuminosity.py +++ b/descqa/SizeStellarMassLuminosity.py @@ -2,7 +2,7 @@ import os import numpy as np from GCR import GCRQuery -from scipy import interpolate +from scipy import interpolate from scipy.stats import binned_statistic from .base import BaseValidationTest, TestResult @@ -15,7 +15,7 @@ def redshift2dist(cosmology): z = np.arange(0, 5.1, 0.5) comov_d = cosmology.comoving_distance(z).to('kpc').value - spl = interpolate.splrep(z, comov_d) + spl = interpolate.splrep(z, comov_d) return spl @@ -26,6 +26,7 @@ class SizeStellarMassLuminosity(BaseValidationTest): _ARCSEC_TO_RADIAN = np.pi / 180. / 3600. def __init__(self, **kwargs): + #pylint: disable=W0231 self.kwargs = kwargs self.observation = kwargs['observation'] self.possible_mag_fields = kwargs['possible_mag_fields'] @@ -39,18 +40,17 @@ def __init__(self, **kwargs): validation_filepath = os.path.join(self.data_dir, kwargs['data_filename']) self.validation_data = np.genfromtxt(validation_filepath) - + @staticmethod def ConvertAbsMagLuminosity(AbsM, band): '''AbsM: absolute magnitude, band: filter''' - if type(AbsM) is list or type(AbsM) is np.ndarray: - AbsM = np.array(AbsM) - - bands = {'U':5.61, 'B':5.48, 'V':4.83, 'R':4.42, 'I':4.08, - 'J':3.64, 'H':3.32, 'K':3.28, 'g':5.33, 'r':4.67, - 'i':4.48, 'z':4.42, 'F300W':6.09, 'F450W':5.32, 'F555W':4.85, - 'F606W':4.66, 'F702W':4.32, 'F814W':4.15, 'CFHT_U':5.57, - 'CFHT_B':5.49, 'CFHT_V':4.81, 'CFHT_R':4.44, 'CFHT_I':4.06, + AbsM = np.asarray(AbsM) + + bands = {'U':5.61, 'B':5.48, 'V':4.83, 'R':4.42, 'I':4.08, + 'J':3.64, 'H':3.32, 'K':3.28, 'g':5.33, 'r':4.67, + 'i':4.48, 'z':4.42, 'F300W':6.09, 'F450W':5.32, 'F555W':4.85, + 'F606W':4.66, 'F702W':4.32, 'F814W':4.15, 'CFHT_U':5.57, + 'CFHT_B':5.49, 'CFHT_V':4.81, 'CFHT_R':4.44, 'CFHT_I':4.06, 'NIRI_J':3.64, 'NIRI_H':3.33, 'NIRI_K':3.29} if band in bands.keys(): @@ -59,7 +59,6 @@ def ConvertAbsMagLuminosity(AbsM, band): raise ValueError('Filter not implemented') logL = (AbsSun - AbsM) / 2.5 #unit of sun - L = 10**logL return logL def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): @@ -68,7 +67,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): ''' # load catalog data spl = redshift2dist(catalog_instance.cosmology) - + colnames = dict() colnames['z'] = 'redshift' colnames['mag'] = catalog_instance.first_available(*self.possible_mag_fields) @@ -77,7 +76,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): elif self.observation == 'twocomp': colnames['size_bulge'] = 'size_bulge_true' colnames['size_disk'] = 'size_disk_true' - + if not all(v for v in colnames.values()): return TestResult(skipped=True, summary='Missing requested quantities') #Check whether the columns are finite or not @@ -91,7 +90,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): catalog_data = catalog_instance.get_quantities(list(colnames.values()), filters=filters) catalog_data = {k: catalog_data[v] for k, v in colnames.items()} - fig, axes = plt.subplots(2,3, figsize=(9, 6), sharex=True, sharey=True) + fig, axes = plt.subplots(2, 3, figsize=(9, 6), sharex=True, sharey=True) try: col = 0 row = 0 @@ -105,7 +104,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): catalog_data_this = GCRQuery(*filters).filter(catalog_data) if len(catalog_data_this['z']) == 0: - continue + continue z_mean = (z_bin['z_max'] + z_bin['z_min']) / 2. output_filepath = os.path.join(output_dir, self.output_filename_template.format(z_bin['z_min'], z_bin['z_max'])) colors = ['r', 'b'] @@ -121,7 +120,7 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): validation_this = self.validation_data[(self.validation_data[:,0] < z_mean + 0.25) & (self.validation_data[:,0] > z_mean - 0.25)] - ax.semilogy(validation_this[:,1], 10**validation_this[:, 2], label=self.label_template.format(z_bin['z_min'], z_bin['z_max'])) + ax.semilogy(validation_this[:,1], 10**validation_this[:,2], label=self.label_template.format(z_bin['z_min'], z_bin['z_max'])) ax.fill_between(validation_this[:,1], 10**validation_this[:,3], 10**validation_this[:,4], lw=0, alpha=0.2) ax.errorbar(default_L_bins, binned_size_kpc, binned_size_kpc_err, marker='o', ls='') elif self.observation == 'twocomp': @@ -141,10 +140,10 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): validation_this = self.validation_data[(self.validation_data[:,0] < z_mean + 0.25) & (self.validation_data[:,0] > z_mean - 0.25)] ax.text(11, 0.3, self.label_template.format(z_bin['z_min'], z_bin['z_max'])) - ax.semilogy(validation_this[:,1], validation_this[:, 2], label='Bulge', color=colors[0]) - ax.fill_between(validation_this[:,1], validation_this[:, 2] + validation_this[:,4], validation_this[:, 2] - validation_this[:,4], lw=0, alpha=0.2, facecolor=colors[0]) - ax.semilogy(validation_this[:,1] + 0.2, validation_this[:, 3], label='Disk', color=colors[1]) - ax.fill_between(validation_this[:,1] + 0.2, validation_this[:, 3] + validation_this[:,5], validation_this[:, 3] - validation_this[:,5], lw=0, alpha=0.2, facecolor=colors[1]) + ax.semilogy(validation_this[:,1], validation_this[:,2], label='Bulge', color=colors[0]) + ax.fill_between(validation_this[:,1], validation_this[:,2] + validation_this[:,4], validation_this[:,2] - validation_this[:,4], lw=0, alpha=0.2, facecolor=colors[0]) + ax.semilogy(validation_this[:,1] + 0.2, validation_this[:,3], label='Disk', color=colors[1]) + ax.fill_between(validation_this[:,1] + 0.2, validation_this[:,3] + validation_this[:,5], validation_this[:,3] - validation_this[:,5], lw=0, alpha=0.2, facecolor=colors[1]) ax.errorbar(default_L_bins, binned_bulgesize_kpc, binned_bulgesize_kpc_err, marker='o', ls='', c=colors[0]) ax.errorbar(default_L_bins+0.2, binned_disksize_kpc, binned_disksize_kpc_err, marker='o', ls='', c=colors[1]) @@ -171,6 +170,6 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): finally: fig.savefig(os.path.join(output_dir, '{:s}.png'.format(self.test_name)), bbox_inches='tight') plt.close(fig) - + #TODO: calculate summary statistics - return TestResult(0, passed=True) + return TestResult(inspect_only=True) From 7e47f4730b1c88c77da510040ad782b6dd15c5bb Mon Sep 17 00:00:00 2001 From: Yao-Yuan Mao Date: Fri, 23 Feb 2018 21:40:36 -0500 Subject: [PATCH 48/64] Delete van_der_Wel_2014_ApJ_788_28_Table5_dict.txt --- .../size/van_der_Wel_2014_ApJ_788_28_Table5_dict.txt | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 descqa/data/size/van_der_Wel_2014_ApJ_788_28_Table5_dict.txt diff --git a/descqa/data/size/van_der_Wel_2014_ApJ_788_28_Table5_dict.txt b/descqa/data/size/van_der_Wel_2014_ApJ_788_28_Table5_dict.txt deleted file mode 100644 index b70fdfff..00000000 --- a/descqa/data/size/van_der_Wel_2014_ApJ_788_28_Table5_dict.txt +++ /dev/null @@ -1,12 +0,0 @@ -#Table 5 -#Logarithmic Size Distributions (16%-84% Range) for the Full Population -#(Early- and Late-type Galaxies Combined) as a Function of Mass and Redshift, -#and Rest-frame V-band Luminosity and Redshift -#Early+Late Type Galaxies - -Table5 = {0.25:{'log10M*':[9.25, 9.75, 10.25, 10.75], 'log10Size_M*':[0.46, 0.54, 0.54, 0.75], 'log10SizeM*_low':[0.21, 0.28, 0.27, 0.49], 'log10SizeM*_high':[0.69, 0.77, 0.82, 0.99], 'log10L*':[9.25, 9.75, 10.25, 10.75], 'log10Size_L*':[0.37, 0.53, 0.62, 0.79], 'log10Size_L*_low':[0.14, 0.26, 0.37, 0.62], 'log10Size_L*_high':[0.62, 0.74, 0.85, 1.05]}, - 0.75:{'log10M*':[9.25, 9.75, 10.25, 10.75, 11.25], 'log10Size_M*':[0.41, 0.52, 0.52, 0.59, 0.85], 'log10Size_M*_low':[0.16, 0.25, 0.18, 0.35, 0.66], 'log10SizeM*_high':[0.64, 0.74, 0.78, 0.84, 1.04], 'log10L*':[9.25, 9.75, 10.25, 10.75, 11.25], 'log10Size_L*':[0.29, 0.44, 0.54, 0.68, 0.83], 'log10Size_L*_low':[0.05, 0.18, 0.28, 0.40, 0.62], 'log10Size_L*_high':[0.54, 0.66, 0.76, 0.90, 1.04]}, - 1.25:{'log10M*':[9.75, 10.25, 10.75, 11.25], 'log10Size_M*':[0.47, 0.52, 0.57, 0.74], 'log10Size_M*_low':[0.21, 0.15, 0.23, 0.49], 'log10SizeM*_high':[0.68, 0.74, 0.80, 0.94], 'log10L*':[9.25, 9.75, 10.25, 10.75, 11.25], 'log10Size_L*':[0.30, 0.36, 0.47, 0.58, 0.67], 'log10Size_L*_low':[0.02, 0.10, 0.19, 0.29, 0.36], 'log10Size_L*_high':[0.64, 0.60, 0.69, 0.79, 0.91]}, - 1.75:{'log10M*':[9.75, 10.25, 10.75, 11.25], 'log10Size_M*':[0.42, 0.48, 0.48, 0.64], 'log10Size_M*_low':[0.15, 0.12, 0.09, 0.34], 'log10SizeM*_high':[0.65, 0.69, 0.74, 0.83], 'log10L*':[9.75, 10.25, 10.75, 11.25], 'log10Size_L*':[0.32, 0.39, 0.51, 0.51], 'log10Size_L*_low':[0.05, 0.12, 0.16, 0.22], 'log10Size_L*_high':[0.58, 0.61, 0.73, 0.83]}, - 2.25:{'log10M*':[10.25, 10.75, 11.25], 'log10Size_M*':[0.41, 0.45, 0.59], 'log10Size_M*_low':[0.10, 0.03, 0.28], 'log10SizeM*_high':[0.63, 0.68, 0.83], 'log10L*':[10.25, 10.75, 11.25], 'log10Size_L*':[0.33, 0.43, 0.45], 'log10Size_L*_low':[0.07, 0.15, 0.10], 'log10Size_L*_high':[0.55, 0.64, 0.74]}, - 2.75:{{'log10M*':[10.75, 11.25], 'log10Size_M*':[0.43, 0.52], 'log10Size_M*_low':[0.01, 0.27], 'log10SizeM*_high':[0.65, 0.75], 'log10L*':[10.75, 11.25], 'log10Size_L*':[0.41, 0.52], 'log10Size_L*_low':[0.11, 0.12], 'log10Size_L*_high':[0.62, 0.74]} From 4ac9f64128131fc1b804882f3b384ef430d6144c Mon Sep 17 00:00:00 2001 From: Yao-Yuan Mao Date: Fri, 23 Feb 2018 19:33:13 -0800 Subject: [PATCH 49/64] adjust column width in matrix view --- descqaweb/matrix.py | 2 +- web-static/style.css | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/descqaweb/matrix.py b/descqaweb/matrix.py index c7a79efb..a5266917 100644 --- a/descqaweb/matrix.py +++ b/descqaweb/matrix.py @@ -80,7 +80,7 @@ def prepare_matrix(run=None, catalog_prefix=None, test_prefix=None): catalogs_this = descqa_run.get_catalogs(catalog_prefix) - table_width = (len(catalogs_this) + 1)*130 + table_width = len(catalogs_this)*120 + 200 if table_width > 1280: data['table_width'] = "100%" else: diff --git a/web-static/style.css b/web-static/style.css index 5bd67531..d6ce7f99 100644 --- a/web-static/style.css +++ b/web-static/style.css @@ -54,7 +54,7 @@ h3 {font-size: 110%; line-height: 1.25;} } .matrix td:first-child { text-align: right; - width: 130px; + width: 200px; font-weight: bold; padding-right: 0.4em; } From 5e116762714887b8325bf8d79f6a497068b7381f Mon Sep 17 00:00:00 2001 From: Yao-Yuan Mao Date: Fri, 23 Feb 2018 20:44:20 -0800 Subject: [PATCH 50/64] fix skip color --- web-static/style.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-static/style.css b/web-static/style.css index d6ce7f99..e1225e08 100644 --- a/web-static/style.css +++ b/web-static/style.css @@ -115,7 +115,7 @@ span.orangered { color:#D55E00; } span.gold { - color:#56B4E9; + color:#F0E442; } span.green { color:#009E73; From ff3e035b220f16c253aa4dbbb5c73b2f6400f6b5 Mon Sep 17 00:00:00 2001 From: Yao-Yuan Mao Date: Sat, 24 Feb 2018 11:34:33 -0500 Subject: [PATCH 51/64] Update SizeStellarMassLuminosity.py --- descqa/SizeStellarMassLuminosity.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/descqa/SizeStellarMassLuminosity.py b/descqa/SizeStellarMassLuminosity.py index 980957d4..991c47d9 100644 --- a/descqa/SizeStellarMassLuminosity.py +++ b/descqa/SizeStellarMassLuminosity.py @@ -69,13 +69,13 @@ def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): spl = redshift2dist(catalog_instance.cosmology) colnames = dict() - colnames['z'] = 'redshift' + colnames['z'] = catalog_instance.first_available('redshift', 'redshift_true') colnames['mag'] = catalog_instance.first_available(*self.possible_mag_fields) if self.observation == 'onecomp': - colnames['size'] = 'size_true' + colnames['size'] = catalog_instance.first_available('size', 'size_true') elif self.observation == 'twocomp': - colnames['size_bulge'] = 'size_bulge_true' - colnames['size_disk'] = 'size_disk_true' + colnames['size_bulge'] = catalog_instance.first_available('size_bulge', 'size_bulge_true') + colnames['size_disk'] = catalog_instance.first_available('size_disk', 'size_disk_true') if not all(v for v in colnames.values()): return TestResult(skipped=True, summary='Missing requested quantities') From 0784fdb5a45f49613ec9e0ca27f08baee3b1a955 Mon Sep 17 00:00:00 2001 From: Yao-Yuan Mao Date: Sat, 24 Feb 2018 12:30:02 -0500 Subject: [PATCH 52/64] add month dir; add search --- descqarun/master.py | 29 ++++++++++++++-------------- descqaweb/bigtable.py | 44 ++++++++++++++++++++++++++++-------------- descqaweb/interface.py | 29 ++++++++++++++-------------- descqaweb/main.py | 23 ++++++++++++++-------- descqaweb/matrix.py | 2 +- 5 files changed, 75 insertions(+), 52 deletions(-) diff --git a/descqarun/master.py b/descqarun/master.py index f337a3fd..48821645 100644 --- a/descqarun/master.py +++ b/descqarun/master.py @@ -116,21 +116,22 @@ def check_copy(src, dst): return dst -def make_output_dir(root_output_dir, create_subdir=True): +def make_output_dir(root_output_dir): root_output_dir = make_path_absolute(root_output_dir) - if create_subdir: - if not os.path.isdir(root_output_dir): - raise OSError('{} does not exist'.format(root_output_dir)) - new_dir_name = time.strftime('%Y-%m-%d') - output_dir = pjoin(root_output_dir, new_dir_name) - if os.path.exists(output_dir): - i = max((int(s.partition('_')[-1] or 0) for s in os.listdir(root_output_dir) if s.startswith(new_dir_name))) - output_dir += '_{}'.format(i+1) - else: - if os.path.exists(root_output_dir): - raise OSError('{} already exists'.format(root_output_dir)) - output_dir = root_output_dir - os.mkdir(output_dir) + if not os.path.isdir(root_output_dir): + raise OSError('{} does not exist'.format(root_output_dir)) + + new_dir_name = time.strftime('%Y-%m-%d') + parent_dir_name = new_dir_name.rpartition('-')[0] + output_dir = pjoin(root_output_dir, parent_dir_name, new_dir_name) + + if os.path.exists(output_dir): + i = max((int(s.partition('_')[-1] or 0) + for s in os.listdir(pjoin(root_output_dir, parent_dir_name)) + if s.startswith(new_dir_name))) + output_dir += '_{}'.format(i+1) + + os.makedirs(output_dir) return output_dir diff --git a/descqaweb/bigtable.py b/descqaweb/bigtable.py index c09e6a15..768969fd 100644 --- a/descqaweb/bigtable.py +++ b/descqaweb/bigtable.py @@ -1,5 +1,5 @@ from __future__ import unicode_literals -import cgi +import html from .interface import iter_all_runs, DescqaRun from . import config @@ -16,17 +16,14 @@ def format_status_count(status_count): try: for name, d in status_count.items(): total = sum(d.values()) - output.append(name + ' - ' + '; '.join(('{}/{} {}'.format(d[k], total, cgi.escape(k)) for k in d))) + output.append(name + ' - ' + '; '.join(('{}/{} {}'.format(d[k], total, html.escape(k)) for k in d))) except AttributeError: if isinstance(status_count, unicode): - output = [cgi.escape(l) for l in status_count.splitlines()] + output = [html.escape(l) for l in status_count.splitlines()] return '
'.join(output) -def format_bigtable_row(run): - - descqa_run = DescqaRun(run, config.root_dir, validated=True) - +def format_bigtable_row(descqa_run): user = descqa_run.status.get('user', '') user = ' ({})'.format(user) if user else '' @@ -58,14 +55,31 @@ def format_bigtable_row(run): return '\n'.join(output) -def prepare_bigtable(page=1): - all_runs = list(iter_all_runs(config.root_dir)) +def filter_search_results(descqa_run, search): + if 'user' in search and descqa_run.status.get('user') != search['user']: + return False + if 'test' in search and not any(t.startswith(search['test']) for t in descqa_run.tests): + return False + if 'catalog' in search and not any(t.startswith(search['catalog']) for t in descqa_run.catalogs): + return False + return True + + +def prepare_bigtable(page=1, months=3, search=None): + all_runs = list(iter_all_runs(config.root_dir, months_to_search=months)) + if search: + all_runs = [descqa_run + for descqa_run in (DescqaRun(run, config.root_dir, validated=True) for run in all_runs) + if filter_search_results(descqa_run, search)] + n_per_page = config.run_per_page - if all_runs: - npages = ((len(all_runs) - 1) // n_per_page) + 1 - if page > npages: - page = npages - all_runs = all_runs[n_per_page*(page-1):n_per_page*page] + npages = ((len(all_runs) - 1) // n_per_page) + 1 + if page > npages: + page = npages + all_runs = all_runs[n_per_page*(page-1):n_per_page*page] + + if not search: + all_runs = [DescqaRun(run, config.root_dir, validated=True) for run in all_runs] table_out = [] table_out.append('') @@ -73,4 +87,4 @@ def prepare_bigtable(page=1): table_out.append('{}'.format(format_bigtable_row(run))) table_out.append('
') - return dict(table='\n'.join(table_out), page=page, npages=npages, static_dir=config.static_dir) + return dict(table='\n'.join(table_out), page=page, npages=npages, static_dir=config.static_dir, search=search) diff --git a/descqaweb/interface.py b/descqaweb/interface.py index c1b29474..cc924e16 100644 --- a/descqaweb/interface.py +++ b/descqaweb/interface.py @@ -39,11 +39,12 @@ def data(self): class DescqaItem(object): def __init__(self, test, catalog, run, base_dir): + run_parent = run.rpartition('-')[0] if catalog is None: - self.relpath = os.path.join(run, test) + self.relpath = os.path.join(run_parent, run, test) self.is_test_summary = True else: - self.relpath = os.path.join(run, test, catalog) + self.relpath = os.path.join(run_parent, run, test, catalog) self.is_test_summary = False self.path = os.path.join(base_dir, self.relpath) self.test = test @@ -118,8 +119,9 @@ def files(self): return self._files -def validate_descqa_run_name(run_name, base_dir, earliest_datetime=None): - path = os.path.join(base_dir, run_name) +def validate_descqa_run_name(run_name, base_dir): + run_parent = run_name.rpartition('-')[0] + path = os.path.join(base_dir, run_parent, run_name) if not os.path.isdir(path): return if not os.access(path, os.R_OK + os.X_OK): @@ -131,8 +133,6 @@ def validate_descqa_run_name(run_name, base_dir, earliest_datetime=None): return m = m.groups() t = datetime.datetime(*(int(i) for i in m[0].split('-')), microsecond=int(m[1] or 0)) - if earliest_datetime and t < earliest_datetime: - return return t @@ -228,16 +228,17 @@ def status(self): return self._status -def iter_all_runs_unsorted(base_dir, days_to_search=None): - if days_to_search: - earliest_datetime = datetime.datetime.today() - datetime.timedelta(days=days_to_search) - else: - earliest_datetime = None +def iter_all_runs_unsorted(base_dir): for run_name in os.listdir(base_dir): - run_key = validate_descqa_run_name(run_name, base_dir, earliest_datetime) + run_key = validate_descqa_run_name(run_name, base_dir) if run_key: yield (run_name, run_key) -def iter_all_runs(base_dir, days_to_search=None): - return (r[0] for r in sorted(iter_all_runs_unsorted(base_dir, days_to_search), key=lambda r: r[1], reverse=True)) +def iter_all_runs(base_dir, months_to_search=None): + for i, month_dir in enumerate(sorted(os.listdir(base_dir), reverse=True)): + if months_to_search is not None and i >= int(months_to_search): + break + sub_base_dir = os.path.join(base_dir, month_dir) + for r in sorted(iter_all_runs_unsorted(sub_base_dir), key=lambda r: r[1], reverse=True): + yield r[0] diff --git a/descqaweb/main.py b/descqaweb/main.py index c56988ac..045febd2 100644 --- a/descqaweb/main.py +++ b/descqaweb/main.py @@ -4,14 +4,22 @@ from jinja2 import Environment, PackageLoader from . import config -from .bigtable import * -from .twopanels import * -from .matrix import * +from .bigtable import prepare_bigtable +from .twopanels import prepare_leftpanel, print_file +from .matrix import prepare_matrix __all__ = ['run'] env = Environment(loader=PackageLoader('descqaweb', 'templates')) + +def _convert_to_integer(value, default=0): + try: + return int(value) + except (ValueError, TypeError): + return default + + def run(): form = cgi.FieldStorage() @@ -30,13 +38,12 @@ def run(): _run = form.getfirst('run', '') if _run.lower() == 'all': - try: - page = int(form.getfirst('page', 1)) - except (ValueError, TypeError): - page = 1 + page = _convert_to_integer(form.getfirst('page'), 1) + months = _convert_to_integer(form.getfirst('months'), 3) + search = {item: form.getfirst(item) for item in ('user', 'test', 'catalog') if form.getfirst(item)} print(env.get_template('header.html').render(full_header=True, please_wait=True, config=config)) sys.stdout.flush() - print(env.get_template('bigtable.html').render(**prepare_bigtable(page))) + print(env.get_template('bigtable.html').render(**prepare_bigtable(page, months, search))) return elif _run: diff --git a/descqaweb/matrix.py b/descqaweb/matrix.py index c7a79efb..17b67e81 100644 --- a/descqaweb/matrix.py +++ b/descqaweb/matrix.py @@ -9,7 +9,7 @@ def find_last_descqa_run(): last_run = None - for run in iter_all_runs(config.root_dir, 180): + for run in iter_all_runs(config.root_dir): descqa_run = DescqaRun(run, config.root_dir, validated=True) if last_run is None: last_run = descqa_run From b944a7fea98c750da2fd9423b4eb4596683115b2 Mon Sep 17 00:00:00 2001 From: Yao-Yuan Mao Date: Sat, 24 Feb 2018 12:44:20 -0500 Subject: [PATCH 53/64] bug fix --- descqaweb/interface.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/descqaweb/interface.py b/descqaweb/interface.py index cc924e16..e206d616 100644 --- a/descqaweb/interface.py +++ b/descqaweb/interface.py @@ -39,12 +39,11 @@ def data(self): class DescqaItem(object): def __init__(self, test, catalog, run, base_dir): - run_parent = run.rpartition('-')[0] if catalog is None: - self.relpath = os.path.join(run_parent, run, test) + self.relpath = os.path.join(run, test) self.is_test_summary = True else: - self.relpath = os.path.join(run_parent, run, test, catalog) + self.relpath = os.path.join(run, test, catalog) self.is_test_summary = False self.path = os.path.join(base_dir, self.relpath) self.test = test @@ -119,9 +118,8 @@ def files(self): return self._files -def validate_descqa_run_name(run_name, base_dir): - run_parent = run_name.rpartition('-')[0] - path = os.path.join(base_dir, run_parent, run_name) +def validate_descqa_run_name(run_name, sub_base_dir): + path = os.path.join(sub_base_dir, run_name) if not os.path.isdir(path): return if not os.access(path, os.R_OK + os.X_OK): @@ -138,6 +136,9 @@ def validate_descqa_run_name(run_name, base_dir): class DescqaRun(object): def __init__(self, run_name, base_dir, validated=False): + run_parent = run_name.rpartition('-')[0] + if os.path.basename(os.path.normpath(base_dir)) != run_parent: + base_dir = os.path.join(base_dir, run_parent) if not validated: assert validate_descqa_run_name(run_name, base_dir) is not None self.base_dir = base_dir @@ -240,5 +241,5 @@ def iter_all_runs(base_dir, months_to_search=None): if months_to_search is not None and i >= int(months_to_search): break sub_base_dir = os.path.join(base_dir, month_dir) - for r in sorted(iter_all_runs_unsorted(sub_base_dir), key=lambda r: r[1], reverse=True): - yield r[0] + for run_name, _ in sorted(iter_all_runs_unsorted(sub_base_dir), key=lambda r: r[1], reverse=True): + yield run_name From f4ee5cc1dbe135eb5d4bd96c999b5751b836b04f Mon Sep 17 00:00:00 2001 From: Yao-Yuan Mao Date: Sat, 24 Feb 2018 12:58:22 -0500 Subject: [PATCH 54/64] fix file path --- descqaweb/interface.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/descqaweb/interface.py b/descqaweb/interface.py index e206d616..4dcb9da1 100644 --- a/descqaweb/interface.py +++ b/descqaweb/interface.py @@ -39,13 +39,15 @@ def data(self): class DescqaItem(object): def __init__(self, test, catalog, run, base_dir): - if catalog is None: - self.relpath = os.path.join(run, test) - self.is_test_summary = True - else: - self.relpath = os.path.join(run, test, catalog) + self.path = os.path.join(base_dir, run, test) + self.relpath = os.path.join(os.path.basename(os.path.normpath(base_dir)), run, test) + self.is_test_summary = True + + if catalog is not None: + self.path = os.path.join(self.path, catalog) + self.relpath = os.path.join(self.relpath, catalog) self.is_test_summary = False - self.path = os.path.join(base_dir, self.relpath) + self.test = test self.catalog = catalog self.run = run @@ -136,9 +138,8 @@ def validate_descqa_run_name(run_name, sub_base_dir): class DescqaRun(object): def __init__(self, run_name, base_dir, validated=False): - run_parent = run_name.rpartition('-')[0] - if os.path.basename(os.path.normpath(base_dir)) != run_parent: - base_dir = os.path.join(base_dir, run_parent) + if not run_name.startswith(os.path.basename(os.path.normpath(base_dir))): + base_dir = os.path.join(base_dir, run_name.rpartition('-')[0]) if not validated: assert validate_descqa_run_name(run_name, base_dir) is not None self.base_dir = base_dir From efb434aa8f27abd0d09709431b992631f8a3f1ac Mon Sep 17 00:00:00 2001 From: Yao-Yuan Mao Date: Sat, 24 Feb 2018 10:17:29 -0800 Subject: [PATCH 55/64] improve search function --- descqaweb/bigtable.py | 6 +++--- descqaweb/config.py | 2 +- descqaweb/main.py | 2 +- descqaweb/templates/bigtable.html | 10 ++++++++++ index.cgi | 4 ++-- 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/descqaweb/bigtable.py b/descqaweb/bigtable.py index 768969fd..efe4a575 100644 --- a/descqaweb/bigtable.py +++ b/descqaweb/bigtable.py @@ -56,11 +56,11 @@ def format_bigtable_row(descqa_run): def filter_search_results(descqa_run, search): - if 'user' in search and descqa_run.status.get('user') != search['user']: + if 'users' in search and descqa_run.status.get('users') not in search['users'].split(): return False - if 'test' in search and not any(t.startswith(search['test']) for t in descqa_run.tests): + if 'tests' in search and not all(any(t.startswith(ts) for t in descqa_run.tests) for ts in search['tests'].split()): return False - if 'catalog' in search and not any(t.startswith(search['catalog']) for t in descqa_run.catalogs): + if 'catalogs' in search and not all(any(c.startswith(cs) for c in descqa_run.catalogs) for cs in search['catalogs'].split()): return False return True diff --git a/descqaweb/config.py b/descqaweb/config.py index 0cfb56db..1f037e08 100644 --- a/descqaweb/config.py +++ b/descqaweb/config.py @@ -2,7 +2,7 @@ __all__ = ['site_title', 'root_dir', 'general_info', 'static_dir', 'run_per_page', 'logo_filename', 'github_url'] -root_dir = '/global/projecta/projectdirs/lsst/groups/CS/descqa/run/v2' +root_dir = '/global/projecta/projectdirs/lsst/groups/CS/descqa/run/v2-new' site_title = 'DESCQA (v2): LSST DESC Quality Assurance for Galaxy Catalogs' run_per_page = 20 diff --git a/descqaweb/main.py b/descqaweb/main.py index 045febd2..e48f46d9 100644 --- a/descqaweb/main.py +++ b/descqaweb/main.py @@ -40,7 +40,7 @@ def run(): if _run.lower() == 'all': page = _convert_to_integer(form.getfirst('page'), 1) months = _convert_to_integer(form.getfirst('months'), 3) - search = {item: form.getfirst(item) for item in ('user', 'test', 'catalog') if form.getfirst(item)} + search = {item: form.getfirst(item) for item in ('users', 'tests', 'catalogs') if form.getfirst(item)} print(env.get_template('header.html').render(full_header=True, please_wait=True, config=config)) sys.stdout.flush() print(env.get_template('bigtable.html').render(**prepare_bigtable(page, months, search))) diff --git a/descqaweb/templates/bigtable.html b/descqaweb/templates/bigtable.html index 47fda2df..dc3c6a3c 100644 --- a/descqaweb/templates/bigtable.html +++ b/descqaweb/templates/bigtable.html @@ -19,8 +19,18 @@ {{ i }} {% endif %} {% endfor %} + +
+
+ Users: + Tests: + Catalogs: + + +
+
Legend: Run did not successfully complete Run completed but some tests have execution errors diff --git a/index.cgi b/index.cgi index 3399a472..b0083d00 100755 --- a/index.cgi +++ b/index.cgi @@ -1,7 +1,7 @@ #!/global/common/cori/contrib/lsst/apps/anaconda/py3-envs/DESCQA/bin/python -#import cgitb -#cgitb.enable() +import cgitb +cgitb.enable() import descqaweb descqaweb.run() From f369a1cff9362ffeab088603a4a204556e450974 Mon Sep 17 00:00:00 2001 From: Yao-Yuan Mao Date: Sat, 24 Feb 2018 10:22:04 -0800 Subject: [PATCH 56/64] bug fix --- descqaweb/bigtable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/descqaweb/bigtable.py b/descqaweb/bigtable.py index efe4a575..f9129c02 100644 --- a/descqaweb/bigtable.py +++ b/descqaweb/bigtable.py @@ -56,7 +56,7 @@ def format_bigtable_row(descqa_run): def filter_search_results(descqa_run, search): - if 'users' in search and descqa_run.status.get('users') not in search['users'].split(): + if 'users' in search and descqa_run.status.get('user') not in search['users'].split(): return False if 'tests' in search and not all(any(t.startswith(ts) for t in descqa_run.tests) for ts in search['tests'].split()): return False From 60058e2313f5aefe104a196b70fd6a9b73f6ec83 Mon Sep 17 00:00:00 2001 From: Yao-Yuan Mao Date: Sat, 24 Feb 2018 10:23:55 -0800 Subject: [PATCH 57/64] pass pylint for py27 --- descqaweb/bigtable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/descqaweb/bigtable.py b/descqaweb/bigtable.py index f9129c02..3242796b 100644 --- a/descqaweb/bigtable.py +++ b/descqaweb/bigtable.py @@ -8,7 +8,7 @@ try: unicode except NameError: - unicode = str + unicode = str #pylint: disable=redefined-builtin def format_status_count(status_count): From d71245c4cc5c69c0184cca3739b92e134cd6e854 Mon Sep 17 00:00:00 2001 From: Yao-Yuan Mao Date: Sat, 24 Feb 2018 10:39:49 -0800 Subject: [PATCH 58/64] disable cgitb --- index.cgi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/index.cgi b/index.cgi index b0083d00..3399a472 100755 --- a/index.cgi +++ b/index.cgi @@ -1,7 +1,7 @@ #!/global/common/cori/contrib/lsst/apps/anaconda/py3-envs/DESCQA/bin/python -import cgitb -cgitb.enable() +#import cgitb +#cgitb.enable() import descqaweb descqaweb.run() From 363c1ac877cfed2109b85a8db79ec3bc01702ac3 Mon Sep 17 00:00:00 2001 From: Yao-Yuan Mao Date: Sat, 24 Feb 2018 10:41:37 -0800 Subject: [PATCH 59/64] version increment --- descqarun/__init__.py | 4 ++-- descqaweb/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/descqarun/__init__.py b/descqarun/__init__.py index bb208af3..a8c4e762 100644 --- a/descqarun/__init__.py +++ b/descqarun/__init__.py @@ -1,5 +1,5 @@ """ DESCQA Execution Script """ -from .master import * #pylint: disable=W0401 -__version__ = '2.0.1' +from .master import main +__version__ = '2.1.0' diff --git a/descqaweb/__init__.py b/descqaweb/__init__.py index a845df0c..d8e12253 100644 --- a/descqaweb/__init__.py +++ b/descqaweb/__init__.py @@ -2,4 +2,4 @@ DESCQA Web Interface """ from .main import run -__version__ = '2.0.1' +__version__ = '2.1.0' From 36a3a8ef9f00d8dbc6223f3fa7579e12f1a60460 Mon Sep 17 00:00:00 2001 From: Yao-Yuan Mao Date: Sat, 24 Feb 2018 10:42:21 -0800 Subject: [PATCH 60/64] remove outdated archiver --- descqarun/archiver.py | 51 ------------------------------------------- 1 file changed, 51 deletions(-) delete mode 100644 descqarun/archiver.py diff --git a/descqarun/archiver.py b/descqarun/archiver.py deleted file mode 100644 index 936d36e5..00000000 --- a/descqarun/archiver.py +++ /dev/null @@ -1,51 +0,0 @@ -from __future__ import print_function -import os -import re -import time -import subprocess -import argparse - - -__all__ = ['archive', 'main'] - - -def archive(src, dst): - if os.path.exists(dst): - raise ValueError('{} already exists'.format(dst)) - subprocess.check_call(['tar', '-czf', dst, src]) - subprocess.check_call(['rm', '-rf', src]) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('source_dir') - parser.add_argument('dest_dir') - parser.add_argument('archive_to_date', help='in the format of YYYY-MM-DD') - args = parser.parse_args() - - if re.match(r'20\d{2}-[01]\d-[0123]\d', args.archive_to_date) is None: - parser.error('`archive_to_date` must be in the format of YYYY-MM-DD') - - items = os.listdir(args.source_dir) - for d in items: - src = os.path.join(args.source_dir, d) - dst = os.path.join(args.dest_dir, d + '.tar.gz') - - if not os.path.isdir(src): - continue - m = re.match(r'(20\d{2}-[01]\d-[0123]\d)(?:_\d+)?', d) - if m is None: - continue - if m.groups()[0] > args.archive_to_date: - continue - - try: - archive(src, dst) - except: - print('Failed to archive {}'.format(d)) - else: - print('{} archived'.format(d)) - - -if __name__ == '__main__': - main() From b3d3b56443222b229a9d034efdc757865d66f556 Mon Sep 17 00:00:00 2001 From: Yao-Yuan Mao Date: Sat, 24 Feb 2018 13:54:13 -0500 Subject: [PATCH 61/64] Update .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5283fd79..e65c5e1a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ install: - pip install pylint nose before_script: -- export CHANGED=$(git diff --name-only $TRAVIS_COMMIT_RANGE | grep '^\(descqa.*\|tests\)/.\+\.py$') +- export CHANGED=$(git diff --diff-filter=d --name-only $TRAVIS_COMMIT_RANGE | grep '^\(descqa.*\|tests\)/.\+\.py$') script: - nosetests From c0eceebb21cdb0ef1737b0a5652491cbe23d5ffa Mon Sep 17 00:00:00 2001 From: Yao-Yuan Mao Date: Sat, 24 Feb 2018 11:05:00 -0800 Subject: [PATCH 62/64] bug fix in pagination --- descqaweb/templates/bigtable.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/descqaweb/templates/bigtable.html b/descqaweb/templates/bigtable.html index dc3c6a3c..f12c89bc 100644 --- a/descqaweb/templates/bigtable.html +++ b/descqaweb/templates/bigtable.html @@ -1,13 +1,13 @@
{% if page > 1 %} -<< +<< {% else %} << {% endif %}  |  {% if page < npages %} ->> +>> {% else %} >> {% endif %} @@ -16,7 +16,7 @@ {% if i==page %} {{ i }} {% else %} - {{ i }} + {{ i }} {% endif %} {% endfor %}
From 49de0b539a1f2573b1401f665a54e48757c72571 Mon Sep 17 00:00:00 2001 From: Yao-Yuan Mao Date: Sat, 24 Feb 2018 14:07:01 -0500 Subject: [PATCH 63/64] remove temporary location --- descqaweb/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/descqaweb/config.py b/descqaweb/config.py index 1f037e08..0cfb56db 100644 --- a/descqaweb/config.py +++ b/descqaweb/config.py @@ -2,7 +2,7 @@ __all__ = ['site_title', 'root_dir', 'general_info', 'static_dir', 'run_per_page', 'logo_filename', 'github_url'] -root_dir = '/global/projecta/projectdirs/lsst/groups/CS/descqa/run/v2-new' +root_dir = '/global/projecta/projectdirs/lsst/groups/CS/descqa/run/v2' site_title = 'DESCQA (v2): LSST DESC Quality Assurance for Galaxy Catalogs' run_per_page = 20 From bcc649c507952682461089f827f2f483fff303dc Mon Sep 17 00:00:00 2001 From: Yao-Yuan Mao Date: Sat, 24 Feb 2018 11:49:54 -0800 Subject: [PATCH 64/64] add months_to_search option to config --- descqaweb/config.py | 3 ++- descqaweb/main.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/descqaweb/config.py b/descqaweb/config.py index 0cfb56db..64c15ffe 100644 --- a/descqaweb/config.py +++ b/descqaweb/config.py @@ -1,11 +1,12 @@ from __future__ import unicode_literals -__all__ = ['site_title', 'root_dir', 'general_info', 'static_dir', 'run_per_page', 'logo_filename', 'github_url'] +__all__ = ['site_title', 'root_dir', 'general_info', 'static_dir', 'run_per_page', 'logo_filename', 'github_url', 'months_to_search'] root_dir = '/global/projecta/projectdirs/lsst/groups/CS/descqa/run/v2' site_title = 'DESCQA (v2): LSST DESC Quality Assurance for Galaxy Catalogs' run_per_page = 20 +months_to_search = 3 static_dir = 'web-static' logo_filename = 'desc-logo-small.png' diff --git a/descqaweb/main.py b/descqaweb/main.py index e48f46d9..2f8b06c2 100644 --- a/descqaweb/main.py +++ b/descqaweb/main.py @@ -39,7 +39,7 @@ def run(): if _run.lower() == 'all': page = _convert_to_integer(form.getfirst('page'), 1) - months = _convert_to_integer(form.getfirst('months'), 3) + months = _convert_to_integer(form.getfirst('months'), config.months_to_search) search = {item: form.getfirst(item) for item in ('users', 'tests', 'catalogs') if form.getfirst(item)} print(env.get_template('header.html').render(full_header=True, please_wait=True, config=config)) sys.stdout.flush()