From d9332b6dc0b85a5f87fc05a3f7115298dccfd625 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Tue, 1 Oct 2024 16:27:28 -0700 Subject: [PATCH 1/8] update notebook with new type of patching --- doc/nb/TestPatchDailyWithJura.ipynb | 589 +++++++--------------------- 1 file changed, 131 insertions(+), 458 deletions(-) diff --git a/doc/nb/TestPatchDailyWithJura.ipynb b/doc/nb/TestPatchDailyWithJura.ipynb index 7e0bd20..ac51779 100644 --- a/doc/nb/TestPatchDailyWithJura.ipynb +++ b/doc/nb/TestPatchDailyWithJura.ipynb @@ -5,7 +5,7 @@ "id": "31725415-8b51-49d7-a2d6-7100227808d5", "metadata": {}, "source": [ - "# Patch daily with jura" + "# Patch daily with jura or kibo" ] }, { @@ -18,37 +18,42 @@ "outputs": [], "source": [ "import os\n", + "import datetime\n", + "import pytz\n", "import numpy as np\n", + "from unittest.mock import patch\n", "from astropy.table import Table, join\n", "from astropy.io import fits\n", "# from desispec.io import read_table\n", + "from desiutil.log import get_logger, DEBUG\n", "from desispec.io.meta import faflavor2program\n", - "from specprodDB.util import cameraid" + "from specprodDB.util import cameraid\n", + "from specprodDB.patch import get_options, get_data, patch_frames, patch_exposures, patch_tiles, patch_missing_frames_mjd" ] }, { "cell_type": "code", "execution_count": null, - "id": "c2571c97-816c-4b12-a572-8e987c88b654", + "id": "3d9291ce-ae1b-4196-8d9b-f6069f353828", "metadata": { "tags": [] }, "outputs": [], "source": [ - "daily_tiles_file = os.path.join(os.environ['DESI_SPECTRO_REDUX'], 'daily', 'tiles-daily.csv')\n", - "daily_exposures_file = os.path.join(os.environ['DESI_SPECTRO_REDUX'], 'daily', 'exposures-daily.fits')\n", - "jura_tiles_file = os.path.join(os.environ['DESI_SPECTRO_REDUX'], 'jura', 'tiles-jura.csv')\n", - "jura_exposures_file = os.path.join(os.environ['DESI_SPECTRO_REDUX'], 'jura', 'exposures-jura.fits')\n", - "daily_tiles = Table.read(daily_tiles_file, format='ascii.csv')\n", - "daily_exposures = Table.read(daily_exposures_file, format='fits', hdu='EXPOSURES')\n", - "# daily_exposures = read_table(daily_exposures_file, ext='EXPOSURES')\n", - "daily_frames = Table.read(daily_exposures_file, format='fits', hdu='FRAMES')\n", - "# daily_frames = read_table(daily_exposures_file, ext='FRAMES')\n", - "jura_tiles = Table.read(jura_tiles_file, format='ascii.csv')\n", - "jura_exposures = Table.read(jura_exposures_file, format='fits', hdu='EXPOSURES')\n", - "# jura_exposures = read_table(jura_exposures_file, ext='EXPOSURES')\n", - "jura_frames = Table.read(jura_exposures_file, format='fits', hdu='FRAMES')\n", - "# jura_frames = read_table(jura_exposures_file, ext='FRAMES')" + "with patch('sys.argv', ['patch_specprod', '--source', 'kibo', '--destination', 'daily', '--overwrite', os.environ['SCRATCH']]):\n", + " options = get_options()\n", + "log = get_logger(DEBUG)\n", + "src, dst = get_data(options)" + ] + }, + { + "cell_type": "markdown", + "id": "1c9487e7-9b9d-4790-80ac-912104091180", + "metadata": { + "tags": [] + }, + "source": [ + "## QA on SURVEY, PROGRAM" ] }, { @@ -56,7 +61,7 @@ "id": "522008bd-0ee9-4d41-93bc-acc5e25d0e29", "metadata": {}, "source": [ - "## Do some QA on jura" + "## Do some QA on patch specprod" ] }, { @@ -68,8 +73,8 @@ }, "outputs": [], "source": [ - "for c in jura_frames.colnames:\n", - " if hasattr(jura_frames[c], 'mask'):\n", + "for c in src['frames'].colnames:\n", + " if hasattr(src['frames'][c], 'mask'):\n", " print(c)" ] }, @@ -82,7 +87,7 @@ }, "outputs": [], "source": [ - "jura_exposures['MJD'].min()" + "src['frames']['MJD'].min()" ] }, { @@ -94,21 +99,22 @@ }, "outputs": [], "source": [ - "jura_exposures_bad_rows = list()\n", + "src_exposures_bad_rows = list()\n", "bad_columns = list()\n", - "for c in jura_exposures.colnames:\n", - " if hasattr(jura_exposures[c], 'mask'):\n", + "for c in src['exposures'].colnames:\n", + " if hasattr(src['exposures'][c], 'mask'):\n", " print(c)\n", " bad_columns.append(c)\n", - " jura_exposures_bad_rows.append(np.where(jura_exposures[c].mask)[0])\n", - " elif jura_exposures[c].dtype.kind == 'f' and not np.isfinite(jura_exposures[c]).all():\n", + " src_exposures_bad_rows.append(np.where(src['exposures'][c].mask)[0])\n", + " elif src['exposures'][c].dtype.kind == 'f' and not np.isfinite(src['exposures'][c]).all():\n", " print(c)\n", " bad_columns.append(c)\n", - " jura_exposures_bad_rows.append(np.where(~np.isfinite(jura_exposures[c]))[0])\n", + " src_exposures_bad_rows.append(np.where(~np.isfinite(src['exposures'][c]))[0])\n", " else:\n", " pass\n", - "jura_exposures_bad_rows = np.unique(np.hstack(jura_exposures_bad_rows))\n", - "jura_exposures[jura_exposures_bad_rows]" + "src_exposures_bad_rows\n", + "# src_exposures_bad_rows = np.unique(np.hstack(src_exposures_bad_rows))\n", + "# src['exposures'][src_exposures_bad_rows]" ] }, { @@ -128,8 +134,8 @@ }, "outputs": [], "source": [ - "jura_exposures_bad_tiles = np.in1d(jura_exposures['TILEID'], jura_exposures[jura_exposures_bad_rows]['TILEID'])\n", - "jura_exposures[jura_exposures_bad_tiles]" + "src_exposures_bad_tiles = np.in1d(src['exposures']['TILEID'], src['exposures']['TILEID'][src_exposures_bad_rows])\n", + "src['exposures'][src_exposures_bad_tiles]" ] }, { @@ -141,609 +147,276 @@ }, "outputs": [], "source": [ - "jura_tiles_bad_rows = np.in1d(jura_tiles['TILEID'], jura_exposures[jura_exposures_bad_rows]['TILEID'])\n", - "jura_tiles[jura_tiles_bad_rows]" - ] - }, - { - "cell_type": "markdown", - "id": "777a4cde-ab8e-4f24-aa31-323dbf819321", - "metadata": { - "tags": [] - }, - "source": [ - "## Find jura exposures not in daily, daily exposures not in jura" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "74d311b4-d697-4944-945a-d56a68d1a39e", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "assert (np.unique(jura_exposures['EXPID']) == sorted(jura_exposures['EXPID'])).all()\n", - "assert (np.unique(daily_exposures['EXPID']) == sorted(daily_exposures['EXPID'])).all()\n", - "assert (np.unique(jura_frames['EXPID']) == sorted(jura_exposures['EXPID'])).all()\n", - "assert (np.unique(daily_frames['EXPID']) == sorted(daily_exposures['EXPID'])).all()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ca1d655b-e315-4e14-9fdc-5d31f5dadcb1", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "first_jura_exposure, last_jura_exposure = jura_exposures['EXPID'].min(), jura_exposures['EXPID'].max()\n", - "first_jura_exposure, last_jura_exposure" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "29c2d15b-3247-4320-bba7-a9b4e9fcc398", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "first_jura_night = jura_exposures['NIGHT'][jura_exposures['EXPID'] == first_jura_exposure].min()\n", - "last_jura_night = jura_exposures['NIGHT'][jura_exposures['EXPID'] == last_jura_exposure].max()\n", - "first_jura_night, last_jura_night" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "51366113-4bfe-46e4-be86-dd4699cd2218", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "jura_tiles['LASTNIGHT'].min(), jura_tiles['LASTNIGHT'].max()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ded2208b-f17e-4c7f-aeff-3e7cca0f77a4", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "jura_expid_set = frozenset(jura_exposures['EXPID'].tolist())\n", - "daily_expid_set = frozenset(daily_exposures['EXPID'].tolist())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "473307c7-30f6-40ca-b297-3b927577d5b1", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "jura_not_in_daily = jura_expid_set - daily_expid_set\n", - "jura_not_in_daily" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "879e049d-3aca-484b-80e3-5b06344077c5", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "daily_not_in_jura = daily_expid_set - jura_expid_set\n", - "# daily_not_in_jura" + "src_tiles_bad_rows = np.in1d(src['tiles']['TILEID'], src['exposures']['TILEID'][src_exposures_bad_rows])\n", + "src['tiles'][src_tiles_bad_rows]" ] }, { "cell_type": "markdown", - "id": "96e7d429-3edc-4f7e-9e66-0ce8e3c8dcec", + "id": "1b969ba8-784e-40b2-98c5-440a24633c90", "metadata": {}, "source": [ - "## Patch frames\n", - "\n", - "We don't necessarily want to change the values in the `FRAMES` table, just make sure it is consistent with the `EXPOSURES` table." + "### Consistency of SURVEY, PROGRAM, etc." ] }, { "cell_type": "code", "execution_count": null, - "id": "11f82181-9647-42e4-b05d-bd8427d20c9a", + "id": "fec068e2-85d6-412c-8557-206f38a9a40d", "metadata": { "tags": [] }, "outputs": [], "source": [ - "daily_frames_join = Table()\n", - "daily_frames_join['FRAMEID'] = np.array([100*row['EXPID'] + cameraid(row['CAMERA']) for row in daily_frames])\n", - "daily_frames_join['DAILY_INDEX'] = np.arange(len(daily_frames))" + "for row in src['tiles']:\n", + " w = np.where(src['exposures']['TILEID'] == row['TILEID'])[0]\n", + " for c in ('SURVEY', 'PROGRAM', 'FAPRGRM', 'FAFLAVOR', 'GOALTYPE'):\n", + " assert (src['exposures'][c][w] == row[c]).all()" ] }, { "cell_type": "code", "execution_count": null, - "id": "333dbff1-65b2-4e66-a35f-b1d2b34a05c4", + "id": "b17dd4e1-5d9f-4f29-b71f-c968050f8ed4", "metadata": { "tags": [] }, "outputs": [], "source": [ - "jura_frames_join = Table()\n", - "jura_frames_join['FRAMEID'] = np.array([100*row['EXPID'] + cameraid(row['CAMERA']) for row in jura_frames])\n", - "jura_frames_join['JURA_INDEX'] = np.arange(len(jura_frames))" + "for row in src['exposures']:\n", + " w = np.where(src['frames']['EXPID'] == row['EXPID'])[0]\n", + " for c in ('SURVEY', 'PROGRAM', 'FAPRGRM', 'FAFLAVOR', 'GOALTYPE'):\n", + " if c in src['frames'].colnames:\n", + " assert (src['frames'][c][w] == row[c]).all()" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "8d6ca37d-2350-47b6-8e90-5ab8b0cc01ad", + "cell_type": "markdown", + "id": "777a4cde-ab8e-4f24-aa31-323dbf819321", "metadata": { "tags": [] }, - "outputs": [], "source": [ - "j = join(daily_frames_join, jura_frames_join, join_type='outer', keys='FRAMEID')\n", - "# j" + "## Find patch exposures not in daily, daily exposures not in patch" ] }, { "cell_type": "code", "execution_count": null, - "id": "32431f66-d899-490e-98ad-4d9c299f74d7", + "id": "74d311b4-d697-4944-945a-d56a68d1a39e", "metadata": { "tags": [] }, "outputs": [], "source": [ - "daily_frames_index = j[(~j['JURA_INDEX'].mask) & (~j['DAILY_INDEX'].mask)]['DAILY_INDEX']\n", - "jura_frames_index = j[(~j['JURA_INDEX'].mask) & (~j['DAILY_INDEX'].mask)]['JURA_INDEX']" + "assert (np.unique(src['exposures']['EXPID']) == sorted(src['exposures']['EXPID'])).all()\n", + "assert (np.unique(dst['exposures']['EXPID']) == sorted(dst['exposures']['EXPID'])).all()\n", + "assert (np.unique(src['frames']['EXPID']) == sorted(src['exposures']['EXPID'])).all()\n", + "assert (np.unique(dst['frames']['EXPID']) == sorted(dst['exposures']['EXPID'])).all()" ] }, { "cell_type": "code", "execution_count": null, - "id": "31e94315-0a93-454f-b228-7c10e14c22b1", + "id": "ca1d655b-e315-4e14-9fdc-5d31f5dadcb1", "metadata": { "tags": [] }, "outputs": [], "source": [ - "daily_frames_patched = daily_frames.copy()\n", - "for column in daily_frames_patched.colnames:\n", - " if hasattr(daily_frames_patched[column], 'mask') and not column.startswith('TSNR2_'):\n", - " if np.any(daily_frames_patched[column].mask[daily_frames_index]):\n", - " print(\"Patching {0:d} rows in dst_frames column {1}.\".format(np.sum(daily_frames_patched[column].mask[daily_frames_index]), column))\n", - " jura_frames_matched = jura_frames[column][jura_frames_index]\n", - " daily_frames_matched = daily_frames_patched[column][daily_frames_index]\n", - " daily_frames_mask_matched = daily_frames_patched[column].mask[daily_frames_index]\n", - " assert np.sum(daily_frames_mask_matched) == np.sum(daily_frames_patched[column].mask[daily_frames_index])\n", - " daily_frames_matched[daily_frames_mask_matched] = jura_frames_matched[daily_frames_mask_matched]\n", - " daily_frames_matched.mask[daily_frames_mask_matched] = False\n", - " daily_frames_patched[column][daily_frames_index] = daily_frames_matched\n", - " daily_frames_patched[column].mask[daily_frames_index] = daily_frames_matched.mask\n", - " # print(type(daily_frames_patched[column].data.data))\n", - " assert not (daily_frames_patched[column].data.data == daily_frames[column].data.data).all()\n", - "daily_frames_patched[daily_frames['SEEING_ETC'].mask]" - ] - }, - { - "cell_type": "markdown", - "id": "ca7c8da8-2c0b-45df-ab15-6f8aac0c9c6a", - "metadata": {}, - "source": [ - "## Patch exposures\n", - "\n", - "We want to only *patch* and rows that:\n", - "\n", - "* Appear in `jura`.\n", - "* Have `NIGHT >= first_jura_night`.\n", - "* Have `EFFTIME_SPEC > 0`.\n", - "\n", - "We want to only *load* rows that:\n", - "\n", - "* Have `NIGHT >= first_jura_night`.\n", - "* Have `EFFTIME_SPEC > 0`.\n", - "\n", - "which is slightly different. However we don't want to *remove* rows that *don't* satisfy these criteria." + "first_src_exposure, last_src_exposure = src['exposures']['EXPID'].min(), src['exposures']['EXPID'].max()\n", + "first_src_exposure, last_src_exposure" ] }, { "cell_type": "code", "execution_count": null, - "id": "e800b0bf-3824-4b78-94e5-e1f3fdb1e9ef", + "id": "29c2d15b-3247-4320-bba7-a9b4e9fcc398", "metadata": { "tags": [] }, "outputs": [], "source": [ - "daily_exposures_join = Table()\n", - "daily_exposures_join['EXPID'] = daily_exposures['EXPID']\n", - "# daily_exposures_join['NIGHT'] = daily_exposures['NIGHT']\n", - "# daily_exposures_join['EFFTIME_SPEC'] = daily_exposures['EFFTIME_SPEC']\n", - "daily_exposures_join['DAILY_INDEX'] = np.arange(len(daily_exposures))" + "first_src_night = src['exposures']['NIGHT'][src['exposures']['EXPID'] == first_src_exposure].min()\n", + "last_src_night = src['exposures']['NIGHT'][src['exposures']['EXPID'] == last_src_exposure].max()\n", + "first_src_night, last_src_night" ] }, { "cell_type": "code", "execution_count": null, - "id": "933274c3-bc01-4287-ba1f-bf138d22c901", + "id": "51366113-4bfe-46e4-be86-dd4699cd2218", "metadata": { "tags": [] }, "outputs": [], "source": [ - "jura_exposures_join = Table()\n", - "jura_exposures_join['EXPID'] = jura_exposures['EXPID']\n", - "jura_exposures_join['JURA_INDEX'] = np.arange(len(jura_exposures))" + "src['tiles']['LASTNIGHT'].min(), src['tiles']['LASTNIGHT'].max()" ] }, { "cell_type": "code", "execution_count": null, - "id": "6fec01f6-f4b2-443b-9763-c663375bae25", + "id": "ded2208b-f17e-4c7f-aeff-3e7cca0f77a4", "metadata": { "tags": [] }, "outputs": [], "source": [ - "j = join(daily_exposures_join, jura_exposures_join, join_type='outer', keys='EXPID')\n", - "j" + "src_expid_set = frozenset(src['exposures']['EXPID'].tolist())\n", + "dst_expid_set = frozenset(dst['exposures']['EXPID'].tolist())" ] }, { "cell_type": "code", "execution_count": null, - "id": "83004567-d880-4ba6-968d-0555662ef9a2", + "id": "473307c7-30f6-40ca-b297-3b927577d5b1", "metadata": { "tags": [] }, "outputs": [], "source": [ - "daily_exposures_index = j[(~j['JURA_INDEX'].mask) & (~j['DAILY_INDEX'].mask)]['DAILY_INDEX']\n", - "jura_exposures_index = j[(~j['JURA_INDEX'].mask) & (~j['DAILY_INDEX'].mask)]['JURA_INDEX']" + "src_not_in_dst = src_expid_set - dst_expid_set\n", + "src_not_in_dst" ] }, { "cell_type": "code", "execution_count": null, - "id": "a63345ad-5660-4f46-9940-60f13370312a", + "id": "879e049d-3aca-484b-80e3-5b06344077c5", "metadata": { "tags": [] }, "outputs": [], "source": [ - "daily_exposures_patched = daily_exposures.copy()\n", - "can_patch = ('NIGHT', 'EXPID', 'TILEID', 'TILERA', 'TILEDEC', 'MJD', 'SURVEY', 'PROGRAM', 'FAPRGRM', 'FAFLAVOR', 'EXPTIME', 'GOALTIME', 'GOALTYPE', 'MINTFRAC', 'AIRMASS', 'EBV', 'SEEING_ETC', 'EFFTIME_ETC',\n", - " 'TRANSPARENCY_GFA', 'SEEING_GFA', 'FIBER_FRACFLUX_GFA', 'FIBER_FRACFLUX_ELG_GFA', 'FIBER_FRACFLUX_BGS_GFA', 'FIBERFAC_GFA', 'FIBERFAC_ELG_GFA', 'FIBERFAC_BGS_GFA', 'AIRMASS_GFA', 'SKY_MAG_AB_GFA',\n", - " 'EFFTIME_GFA', 'EFFTIME_DARK_GFA', 'EFFTIME_BRIGHT_GFA', 'EFFTIME_BACKUP_GFA')\n", - "for column in ['TILERA', 'TILEDEC', 'MJD', 'SURVEY'] + [c for c in daily_exposures_patched.colnames if hasattr(daily_exposures_patched[c], 'mask') and c in can_patch]:\n", - " print(f\"Patching {column}...\")\n", - " if hasattr(jura_exposures[column], 'mask'):\n", - " if np.any(jura_exposures[column].mask[jura_exposures_index]):\n", - " jura_exposures[column][jura_exposures[column].mask] = 0\n", - " jura_exposures[column].mask[jura_exposures[column].mask] = False\n", - " daily_exposures_patched[column][daily_exposures_index] = jura_exposures[column][jura_exposures_index]\n", - " if hasattr(daily_exposures_patched[column], 'mask'):\n", - " daily_exposures_patched[column].mask[daily_exposures_index] = False\n", - "\n", - "known_bad_exposures = np.in1d(daily_exposures_patched['EXPID'], np.array([74307, 79769, 83420, 110852, 190752]))\n", - "daily_exposures_patched[known_bad_exposures]" + "dst_not_in_src = dst_expid_set - src_expid_set\n", + "# dst_not_in_src" ] }, { "cell_type": "markdown", - "id": "2cad2268-9557-444e-897c-e04bc63b42c6", + "id": "96e7d429-3edc-4f7e-9e66-0ce8e3c8dcec", "metadata": {}, "source": [ - "### After patching are there still missing data?" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "00e2e401-93f6-4d87-8307-9c1f0db767e2", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "assert not (daily_exposures_patched['TILERA'] == daily_exposures['TILERA']).all()\n", - "assert not (daily_exposures_patched['TILEDEC'] == daily_exposures['TILEDEC']).all()\n", - "assert not (daily_exposures_patched['MJD'] == daily_exposures['MJD']).all()\n", - "assert not (daily_exposures_patched['SURVEY'] == daily_exposures['SURVEY']).all()\n", - "assert (daily_exposures_patched['PROGRAM'] == daily_exposures['PROGRAM']).all()\n", - "assert (daily_exposures_patched['FAPRGRM'] == daily_exposures['FAPRGRM']).all()\n", - "assert (daily_exposures_patched['FAFLAVOR'] == daily_exposures['FAFLAVOR']).all()" + "## Perform initial patching and QA" ] }, { "cell_type": "code", "execution_count": null, - "id": "60427b0b-913d-456b-b45e-6ccc3a46866b", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "missing_mjd = np.where((daily_exposures_patched['NIGHT'] >= first_jura_night) & (daily_exposures_patched['EFFTIME_SPEC'] > 0) & (daily_exposures_patched['MJD'] < 50000))[0]\n", - "for row in daily_exposures_patched[missing_mjd]:\n", - " raw_data_file = os.path.join(os.environ['DESI_SPECTRO_DATA'], \"{0:08d}\".format(row['NIGHT']), \"{0:08d}\".format(row['EXPID']), \"desi-{0:08d}.fits.fz\".format(row['EXPID']))\n", - " with fits.open(raw_data_file, mode='readonly') as hdulist:\n", - " mjd_obs = hdulist['SPEC'].header['MJD-OBS']\n", - " print(\"INFO: tile {0:d} exposure {1:d} has MJD-OBS = {2:f} in {3}!\".format(row['TILEID'], row['EXPID'], mjd_obs, raw_data_file))\n", - " w = np.where(daily_exposures_patched['EXPID'] == row['EXPID'])[0]\n", - " assert len(w) == 1\n", - " daily_exposures_patched['MJD'][w] = mjd_obs\n", - " print(daily_exposures_patched[w][['NIGHT', 'EXPID', 'TILEID', 'MJD']])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fdc2baa0-8cd9-4fbf-8c5f-ec83a931a72b", + "id": "11f82181-9647-42e4-b05d-bd8427d20c9a", "metadata": { "tags": [] }, "outputs": [], "source": [ - "still_missing_mjd = np.where((daily_exposures_patched['NIGHT'] >= first_jura_night) & (daily_exposures_patched['EFFTIME_SPEC'] > 0) & (daily_exposures_patched['MJD'] < 50000))[0]\n", - "daily_exposures_patched[still_missing_mjd]" + "timestamp = datetime.datetime.now(tz=pytz.timezone('US/Pacific'))\n", + "ymd = timestamp.strftime('%Y%m%d')\n", + "patched = dict()\n", + "patched['tiles_file'] = os.path.join(options.output, f'tiles-{options.dst}-patched-with-{options.src}-{ymd}.csv')\n", + "patched['exposures_file'] = os.path.join(options.output, f'exposures-{options.dst}-patched-with-{options.src}-{ymd}.fits')\n", + "patched['frames'] = patch_frames(src['frames'], dst['frames'])\n", + "patched['exposures'] = patch_exposures(src['exposures'], dst['exposures'])\n", + "patched['frames'] = patch_missing_frames_mjd(patched['exposures'], patched['frames'])\n", + "patched['tiles'] = patch_tiles(src['tiles'], dst['tiles'], timestamp)" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "be0fd08b-0f73-4bf5-a1ee-a955ace6c3ae", + "cell_type": "markdown", + "id": "6bd36677-8e86-4fbc-9894-be20b89defa1", "metadata": { "tags": [] }, - "outputs": [], "source": [ - "possible_expid_still_patchable = list()\n", - "for c in daily_exposures_patched.colnames:\n", - " if hasattr(daily_exposures_patched[c], 'mask'):\n", - " if daily_exposures_patched[c].mask.any():\n", - " n_masked = daily_exposures_patched[c].mask.sum()\n", - " not_in_jura = [e in daily_not_in_jura for e in daily_exposures_patched['EXPID'][daily_exposures_patched[c].mask]]\n", - " if all(not_in_jura):\n", - " print(f\"Column {c} still has {n_masked:d} masked values, but the exposures are not in jura and cannot be patched.\")\n", - " else:\n", - " if c in can_patch:\n", - " for e in daily_exposures_patched['EXPID'][daily_exposures_patched[c].mask]:\n", - " if e not in daily_not_in_jura:\n", - " possible_expid_still_patchable.append(int(e))\n", - " print(f\"Column {c} still has {n_masked:d} masked values, and some exposures can still be patched.\")\n", - " else:\n", - " print(f\"Column {c} still has {n_masked:d} masked values, and some exposures could still be patched, but they do not meet the patchable column guidelines.\")\n", + "### Are the values of `FAFLAVOR` and similar columns consistent among all tables?\n", "\n", - "possible_expid_still_patchable = np.unique(np.array(possible_expid_still_patchable))\n", - "daily_exposures_patched[np.in1d(daily_exposures_patched['EXPID'], possible_expid_still_patchable)]" - ] - }, - { - "cell_type": "markdown", - "id": "ed22c786-df96-47f7-b167-fac29e15d8c4", - "metadata": {}, - "source": [ - "### Fill remaining masked values with zero." + "For \"historical\" reasons the frames table does not contain `PROGRAM`." ] }, { "cell_type": "code", "execution_count": null, - "id": "9b8e3874-fbf7-4fce-b31f-0c31218dc6c5", + "id": "9ac3d66d-ca4f-4dc7-a863-abb730e19cf3", "metadata": { "tags": [] }, "outputs": [], "source": [ - "for c in daily_exposures_patched.colnames:\n", - " if hasattr(daily_exposures_patched[c], 'mask'):\n", - " if daily_exposures_patched[c].mask.any():\n", - " daily_exposures_patched[c][daily_exposures_patched[c].mask] = 0\n", - " daily_exposures_patched[c].mask[daily_exposures_patched[c].mask] = False" - ] - }, - { - "cell_type": "markdown", - "id": "f708faa2-36c4-4b2d-9a93-1c5b76b415d1", - "metadata": {}, - "source": [ - "## Patch tiles\n", - "\n", - "Similar to the discussion above, we want `LASTNIGHT >= first_jura_night`." + "for row in patched['tiles']:\n", + " w = np.where(patched['exposures']['TILEID'] == row['TILEID'])[0]\n", + " for c in ('SURVEY', 'PROGRAM', 'FAPRGRM', 'FAFLAVOR', 'GOALTYPE'):\n", + " if (patched['exposures'][c][w] != row[c]).any() and not (patched['exposures'][c][w] == 'unknown').all():\n", + " print(c, row['TILEID'], row[c], patched['exposures'][c][w].tolist())" ] }, { "cell_type": "code", "execution_count": null, - "id": "76fd512d-ee77-4e0f-b6c1-3ba5eb5d77c0", + "id": "8b4f9da5-b520-40d7-b595-d902b61fa50b", "metadata": { "tags": [] }, "outputs": [], "source": [ - "assert (np.unique(jura_exposures['TILEID']) == sorted(jura_tiles['TILEID'])).all()\n", - "assert (np.unique(daily_exposures['TILEID']) == sorted(daily_tiles['TILEID'])).all()" + "for row in patched['exposures']:\n", + " w = np.where(patched['frames']['EXPID'] == row['EXPID'])[0]\n", + " for c in ('SURVEY', 'PROGRAM', 'FAPRGRM', 'FAFLAVOR', 'GOALTYPE'):\n", + " if c in patched['frames'].colnames:\n", + " if (patched['frames'][c][w] != row[c]).any():\n", + " print(c, row['EXPID'], row[c], patched['frames'][c][w].tolist())" ] }, { "cell_type": "markdown", - "id": "17112d1a-97dc-4dec-9b71-605c6c381896", + "id": "a068bafa-1d32-4cf5-9cb0-fc943634686b", "metadata": {}, "source": [ - "### First patch PROGRAM with faflavor2program()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "de4595e0-6549-44aa-a2b2-52462f6ec28d", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "daily_tiles_patched = daily_tiles.copy()\n", - "daily_tiles_patched['PROGRAM'] = faflavor2program(daily_tiles_patched['FAFLAVOR'])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "71a87a7e-10b9-4287-8cdd-9d65578471af", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "oddball_survey = np.where((daily_tiles_patched['SURVEY'] != 'cmx') & (daily_tiles_patched['SURVEY'] != 'sv1') & (daily_tiles_patched['SURVEY'] != 'sv2') & (daily_tiles_patched['SURVEY'] != 'sv3') & (daily_tiles_patched['SURVEY'] != 'main') & (daily_tiles_patched['SURVEY'] != 'special'))[0]\n", - "oddball_program = np.where((daily_tiles_patched['PROGRAM'] != 'backup') & (daily_tiles_patched['PROGRAM'] != 'bright') & (daily_tiles_patched['PROGRAM'] != 'dark') & (daily_tiles_patched['PROGRAM'] != 'other'))[0]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fb685210-58eb-47d6-a17b-6fda9c9a9739", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "assert (daily_tiles['SURVEY'][oddball_survey] == 'unknown').all()\n", - "assert len(oddball_program) == 0" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3d9f64c2-d3a9-48f1-a4c9-70577f354c1c", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "unknown_cmx = np.where((daily_tiles['SURVEY'] == 'unknown') & (daily_tiles['FAFLAVOR'] != 'unknown'))[0]\n", - "print(daily_tiles[['TILEID', 'SURVEY', 'PROGRAM', 'FAPRGRM', 'FAFLAVOR', 'GOALTYPE', 'LASTNIGHT']][unknown_cmx])" + "## \"Back\" patch exposures and frames with values of SURVEY, PROGRAM, ... from tiles." ] }, { "cell_type": "code", "execution_count": null, - "id": "eb5eaf4f-a295-47b0-bfe5-e388fecedda5", + "id": "b3620eee-c28a-4356-8c79-e3de47a2fbbe", "metadata": { "tags": [] }, "outputs": [], "source": [ - "unknown_unknown = np.where((daily_tiles['SURVEY'] == 'unknown') & (daily_tiles['FAFLAVOR'] == 'unknown'))[0]\n", - "daily_tiles['LASTNIGHT'][unknown_unknown].max()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4aafa0e4-64c5-4344-99e5-7b87fb008612", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "print(daily_tiles[['TILEID', 'SURVEY', 'PROGRAM', 'FAPRGRM', 'FAFLAVOR', 'GOALTYPE', 'LASTNIGHT']][unknown_unknown])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a3029d17-9611-4afc-8346-0404be3d2954", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# For now just patch with 'cmx'\n", - "daily_tiles['SURVEY'][oddball_survey] = 'cmx'" + "back_patch = {'tiles': 'exposures', 'exposures': 'frames'}\n", + "for s, d in back_patch.items():\n", + " for row in patched[s]:\n", + " key = 'TILEID' if s == 'tiles' else 'EXPID'\n", + " w = np.where(patched[d][key] == row[key])[0]\n", + " for column in ('SURVEY', 'PROGRAM', 'FAPRGRM', 'FAFLAVOR', 'GOALTYPE'):\n", + " if column in patched[d].colnames:\n", + " if (patched[d][column][w] != row[column]).any():\n", + " log.info(\"Patching %s associated with %s %d with %s = '%s'.\", d, ('tile' if s == 'tiles' else 'exposure'), row[key], column, row[column])\n", + " patched[d][column][w] = row[column]" ] }, { "cell_type": "markdown", - "id": "27c9cc2d-c9fc-4393-964b-ce7296814590", + "id": "70842f2d-5d9b-4d4a-b388-6dafc24dbe2d", "metadata": {}, "source": [ - "## Write out the patched files" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ac5fdfb7-4635-4e5e-ba2d-b77dd9603554", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "daily_tiles_patched.write(os.path.join(os.environ['DESI_ROOT'], 'users', os.environ['USER'], 'tiles-daily-patched-with-jura.csv'), format='ascii.csv', overwrite=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "384e6030-fcc1-453e-b990-a34688f12494", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "daily_exposures_patched.write(os.path.join(os.environ['DESI_ROOT'], 'users', os.environ['USER'], 'exposures-daily-patched-with-jura.csv'), format='ascii.csv', overwrite=True)" + "### Re-run QA" ] }, { "cell_type": "code", "execution_count": null, - "id": "26d7bad1-f89f-4cde-bfa1-a0cbcabcf866", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "daily_exposures_fits = fits.HDUList([fits.PrimaryHDU(), fits.table_to_hdu(daily_exposures_patched), fits.table_to_hdu(daily_frames_patched)])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d06fcfd7-c5d9-4d9c-9989-eb313c70bc07", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "daily_exposures_fits.info()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d4e5751a-895b-4ca7-8c44-994a4d7d7524", - "metadata": { - "tags": [] - }, + "id": "ac798c48-0563-4f80-b202-66b01cf6d415", + "metadata": {}, "outputs": [], "source": [ - "daily_exposures_fits.writeto(os.path.join(os.environ['DESI_ROOT'], 'users', os.environ['USER'], 'exposures-daily-patched-with-jura.fits'), overwrite=True)" + "for s, d in back_patch.items():\n", + " for row in patched[s]:\n", + " key = 'TILEID' if s == 'tiles' else 'EXPID'\n", + " w = np.where(patched[d][key] == row[key])[0]\n", + " for column in ('SURVEY', 'PROGRAM', 'FAPRGRM', 'FAFLAVOR', 'GOALTYPE'):\n", + " if column in patched[d].colnames:\n", + " assert (patched[d][column][w] == row[column]).all()" ] }, { From ed165577be253d616eef05d312582a574858da47 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Tue, 1 Oct 2024 16:27:58 -0700 Subject: [PATCH 2/8] rename patching notebook --- doc/nb/{TestPatchDailyWithJura.ipynb => TestPatchDaily.ipynb} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename doc/nb/{TestPatchDailyWithJura.ipynb => TestPatchDaily.ipynb} (100%) diff --git a/doc/nb/TestPatchDailyWithJura.ipynb b/doc/nb/TestPatchDaily.ipynb similarity index 100% rename from doc/nb/TestPatchDailyWithJura.ipynb rename to doc/nb/TestPatchDaily.ipynb From 2cca3e4be99b1aff0050e0bc092591c32914bc02 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Tue, 1 Oct 2024 16:36:38 -0700 Subject: [PATCH 3/8] copy back-patch code into patch.py --- doc/changes.rst | 5 +++++ py/specprodDB/patch.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/doc/changes.rst b/doc/changes.rst index 73d496e..897a0d6 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -8,6 +8,11 @@ Change Log *Planned*: Support loading ongoing daily reductions, in particular, updates to tiles already in the database. +1.2.1 (unreleased) +------------------ + +* Finalize patches of ``tiles-daily`` and ``exposures-daily`` files. + 1.2.0 (2024-09-26) ------------------ diff --git a/py/specprodDB/patch.py b/py/specprodDB/patch.py index 9ccdd0a..ce88629 100644 --- a/py/specprodDB/patch.py +++ b/py/specprodDB/patch.py @@ -367,6 +367,48 @@ def patch_tiles(src_tiles, dst_tiles, timestamp): return dst_tiles_patched +def back_patch_inconsistent_values(patched): + """When the primary round of patching is done, copy some values back + into the exposures and frames files. + + Parameters + ---------- + patched : :class:`dict` + A dictionary containing tables for further patching. + + Returns + ------- + :class:`tuple` + A tuple containing the back-patched exposures and frames tables. + Not strictly necessary as this function will modify the tables in + `patched` in-place. + """ + log = get_logger() + back_patch = {'tiles': 'exposures', 'exposures': 'frames'} + for s, d in back_patch.items(): + for row in patched[s]: + key = 'TILEID' if s == 'tiles' else 'EXPID' + w = np.where(patched[d][key] == row[key])[0] + for column in ('SURVEY', 'PROGRAM', 'FAPRGRM', 'FAFLAVOR', 'GOALTYPE'): + if column in patched[d].colnames: + if (patched[d][column][w] != row[column]).any(): + log.info("Patching %s associated with %s %d with %s = '%s'.", + d, ('tile' if s == 'tiles' else 'exposure'), row[key], + column, row[column]) + patched[d][column][w] = row[column] + # + # Run a QA step. + # + for s, d in back_patch.items(): + for row in patched[s]: + key = 'TILEID' if s == 'tiles' else 'EXPID' + w = np.where(patched[d][key] == row[key])[0] + for column in ('SURVEY', 'PROGRAM', 'FAPRGRM', 'FAFLAVOR', 'GOALTYPE'): + if column in patched[d].colnames: + assert (patched[d][column][w] == row[column]).all() + return (patched['exposures'], patched['frames']) + + def get_data(options): """Read in source and destination data. From d373b29529d611caf90175a46c4c1c4bd5be25b0 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Tue, 1 Oct 2024 16:37:37 -0700 Subject: [PATCH 4/8] add back-patch step to main --- py/specprodDB/patch.py | 1 + 1 file changed, 1 insertion(+) diff --git a/py/specprodDB/patch.py b/py/specprodDB/patch.py index ce88629..c18d20e 100644 --- a/py/specprodDB/patch.py +++ b/py/specprodDB/patch.py @@ -500,6 +500,7 @@ def main(): patched['exposures'] = patch_exposures(src['exposures'], dst['exposures']) patched['frames'] = patch_missing_frames_mjd(patched['exposures'], patched['frames']) patched['tiles'] = patch_tiles(src['tiles'], dst['tiles'], timestamp) + back_exposures, back_tiles = back_patch_inconsistent_values(patched) # # Write out data. # From 7fae9c2cb6ae548af5959719d42aa563ffc7410e Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Fri, 4 Oct 2024 10:54:30 -0700 Subject: [PATCH 5/8] patches for goaltype --- py/specprodDB/patch.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/py/specprodDB/patch.py b/py/specprodDB/patch.py index c18d20e..1b5e073 100644 --- a/py/specprodDB/patch.py +++ b/py/specprodDB/patch.py @@ -355,6 +355,18 @@ def patch_tiles(src_tiles, dst_tiles, timestamp): assert len(oddball_program) == 0 dst_tiles_patched['SURVEY'][oddball_survey] = 'cmx' # + # Patch GOALTYPE. + # + dst_tiles_patched['GOALTYPE'][dst_tiles_patched['TILEID'] == 83004] = 'dark' + dst_tiles_patched['GOALTYPE'][dst_tiles_patched['TILEID'] == 83024] = 'bright' + oddball_goaltype = np.where(dst_tiles_patched['GOALTYPE'] == 'unknown')[0] + dst_tiles_patched['GOALTYPE'][oddball_goaltype] = 'other' + # + # Patch EFFTIME_SPEC. + # + dst_tiles_patched['EFFTIME_SPEC'][dst_tiles_patched['TILEID'] == 1825] = 0 + dst_tiles_patched['EFFTIME_SPEC'][dst_tiles_patched['TILEID'] == 21273] = 0 + # # Add UPDATED. # dst_tiles_patched['UPDATED'] = np.array([timestamp.strftime("%Y-%m-%dT%H:%M:%S%z")]*len(dst_tiles_patched)) From e6ab507bfde699538e494934687fe811e1877060 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Fri, 4 Oct 2024 11:39:43 -0700 Subject: [PATCH 6/8] patch EFFTIME_SPEC --- py/specprodDB/patch.py | 52 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/py/specprodDB/patch.py b/py/specprodDB/patch.py index 1b5e073..c94e472 100644 --- a/py/specprodDB/patch.py +++ b/py/specprodDB/patch.py @@ -133,7 +133,7 @@ def patch_exposures(src_exposures, dst_exposures, first_night=None): Parameters ---------- src_exposures : :class:`~astropy.table.Table` - Source of tiles data. + Source of exposures data. dst_exposures : :class:`~astropy.table.Table` Data to be patched. first_night : :class:`int`, optional @@ -421,6 +421,55 @@ def back_patch_inconsistent_values(patched): return (patched['exposures'], patched['frames']) +def patch_exposures_efftime_spec(src_exposures, dst_exposures, dst_tiles): + """Patch exposures that have ``EFFTIME_SPEC == 0`` where the corresponding + tile has ``EFFTIME_SPEC > 0``. + + Parameters + ---------- + src_exposures : :class:`~astropy.table.Table` + Source of exposures data. + dst_exposures : :class:`~astropy.table.Table` + Data to be patched. + dst_tiles : :class:`~astropy.table.Table` + Tiles data for comparison. Should not be modified. + + Returns + ------- + :class:`~astropy.table.Table` + A *copy* of `dst_exposures` with data replaced from `src_exposures`. + """ + log = get_logger() + dst_exposures_patched = dst_exposures.copy() + candidate_tiles = dst_tiles[(dst_tiles['LASTNIGHT'] >= 20201214) & + (dst_tiles['EFFTIME_SPEC'] > 0)] + for t in candidate_tiles: + row_index = np.where((dst_exposures_patched['TILEID'] == t['TILEID']) & + (dst_exposures_patched['EFFTIME_SPEC'] > 0))[0] + if len(row_index) == 0: + log.error("No valid exposures found for tile %d, even though EFFTIME_SPEC == %.2f!", + t['TILEID'], t['EFFTIME_SPEC']) + bad_index = np.where((dst_exposures_patched['TILEID'] == t['TILEID']))[0] + w = np.in1d(src_exposures['EXPID'], dst_exposures_patched['EXPID'][bad_index]) + n_src = w.sum() + can_patch = False + if n_src == 0: + log.error("Tile %d cannot be patched with upstream data.", t['TILEID']) + elif n_src == len(bad_index): + log.info("Tile %d can be fully patched with upstream data.", t['TILEID']) + can_patch = True + elif n_src < len(bad_index): + log.warning("Tile %d can be partially patched with upstream data.", t['TILEID']) + can_patch = True + else: + log.critical("Should not actually reach this point. This is weird.") + if can_patch: + for row in src_exposures[w]: + ww = dst_exposures_patched['EXPID'] == row['EXPID'] + dst_exposures_patched['EFFTIME_SPEC'][ww] = row['EFFTIME_SPEC'] + return dst_exposures_patched + + def get_data(options): """Read in source and destination data. @@ -513,6 +562,7 @@ def main(): patched['frames'] = patch_missing_frames_mjd(patched['exposures'], patched['frames']) patched['tiles'] = patch_tiles(src['tiles'], dst['tiles'], timestamp) back_exposures, back_tiles = back_patch_inconsistent_values(patched) + patched['exposures'] = patch_exposures_efftime_spec(src['exposures'], patched['exposures'], patched['tiles']) # # Write out data. # From 1bc230b1fac46b209c5e24293f0cf71457c67f28 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Fri, 4 Oct 2024 13:54:43 -0700 Subject: [PATCH 7/8] update patch notebook --- doc/nb/TestPatchDaily.ipynb | 115 +++++++++++------------------------- 1 file changed, 35 insertions(+), 80 deletions(-) diff --git a/doc/nb/TestPatchDaily.ipynb b/doc/nb/TestPatchDaily.ipynb index ac51779..3999801 100644 --- a/doc/nb/TestPatchDaily.ipynb +++ b/doc/nb/TestPatchDaily.ipynb @@ -28,7 +28,7 @@ "from desiutil.log import get_logger, DEBUG\n", "from desispec.io.meta import faflavor2program\n", "from specprodDB.util import cameraid\n", - "from specprodDB.patch import get_options, get_data, patch_frames, patch_exposures, patch_tiles, patch_missing_frames_mjd" + "import specprodDB.patch as p" ] }, { @@ -41,9 +41,9 @@ "outputs": [], "source": [ "with patch('sys.argv', ['patch_specprod', '--source', 'kibo', '--destination', 'daily', '--overwrite', os.environ['SCRATCH']]):\n", - " options = get_options()\n", + " options = p.get_options()\n", "log = get_logger(DEBUG)\n", - "src, dst = get_data(options)" + "src, dst = p.get_data(options)" ] }, { @@ -315,114 +315,69 @@ "patched = dict()\n", "patched['tiles_file'] = os.path.join(options.output, f'tiles-{options.dst}-patched-with-{options.src}-{ymd}.csv')\n", "patched['exposures_file'] = os.path.join(options.output, f'exposures-{options.dst}-patched-with-{options.src}-{ymd}.fits')\n", - "patched['frames'] = patch_frames(src['frames'], dst['frames'])\n", - "patched['exposures'] = patch_exposures(src['exposures'], dst['exposures'])\n", - "patched['frames'] = patch_missing_frames_mjd(patched['exposures'], patched['frames'])\n", - "patched['tiles'] = patch_tiles(src['tiles'], dst['tiles'], timestamp)" + "patched['frames'] = p.patch_frames(src['frames'], dst['frames'])\n", + "patched['exposures'] = p.patch_exposures(src['exposures'], dst['exposures'])\n", + "patched['frames'] = p.patch_missing_frames_mjd(patched['exposures'], patched['frames'])\n", + "patched['tiles'] = p.patch_tiles(src['tiles'], dst['tiles'], timestamp)\n", + "back_exposures, back_tiles = p.back_patch_inconsistent_values(patched)\n", + "patched['exposures'] = p.patch_exposures_efftime_spec(src['exposures'], patched['exposures'], patched['tiles'])" ] }, { "cell_type": "markdown", - "id": "6bd36677-8e86-4fbc-9894-be20b89defa1", - "metadata": { - "tags": [] - }, + "id": "7f126e49-1ebc-40a6-a9c7-2d340783047c", + "metadata": {}, "source": [ - "### Are the values of `FAFLAVOR` and similar columns consistent among all tables?\n", + "## Analyze tiles that are candidates for loading\n", "\n", - "For \"historical\" reasons the frames table does not contain `PROGRAM`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9ac3d66d-ca4f-4dc7-a863-abb730e19cf3", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "for row in patched['tiles']:\n", - " w = np.where(patched['exposures']['TILEID'] == row['TILEID'])[0]\n", - " for c in ('SURVEY', 'PROGRAM', 'FAPRGRM', 'FAFLAVOR', 'GOALTYPE'):\n", - " if (patched['exposures'][c][w] != row[c]).any() and not (patched['exposures'][c][w] == 'unknown').all():\n", - " print(c, row['TILEID'], row[c], patched['exposures'][c][w].tolist())" + "Note, 20201214: Earliest night in `jura`/`kibo`." ] }, { "cell_type": "code", "execution_count": null, - "id": "8b4f9da5-b520-40d7-b595-d902b61fa50b", + "id": "12b33463-0eaf-4531-b96d-3e8e2d18682c", "metadata": { "tags": [] }, "outputs": [], "source": [ - "for row in patched['exposures']:\n", - " w = np.where(patched['frames']['EXPID'] == row['EXPID'])[0]\n", - " for c in ('SURVEY', 'PROGRAM', 'FAPRGRM', 'FAFLAVOR', 'GOALTYPE'):\n", - " if c in patched['frames'].colnames:\n", - " if (patched['frames'][c][w] != row[c]).any():\n", - " print(c, row['EXPID'], row[c], patched['frames'][c][w].tolist())" - ] - }, - { - "cell_type": "markdown", - "id": "a068bafa-1d32-4cf5-9cb0-fc943634686b", - "metadata": {}, - "source": [ - "## \"Back\" patch exposures and frames with values of SURVEY, PROGRAM, ... from tiles." + "candidate_tiles = patched['tiles'][(patched['tiles']['LASTNIGHT'] >= 20201214) & (patched['tiles']['EFFTIME_SPEC'] > 0)]" ] }, { "cell_type": "code", "execution_count": null, - "id": "b3620eee-c28a-4356-8c79-e3de47a2fbbe", + "id": "36d1c46a-5d10-45a1-96ee-698030dff465", "metadata": { "tags": [] }, "outputs": [], "source": [ - "back_patch = {'tiles': 'exposures', 'exposures': 'frames'}\n", - "for s, d in back_patch.items():\n", - " for row in patched[s]:\n", - " key = 'TILEID' if s == 'tiles' else 'EXPID'\n", - " w = np.where(patched[d][key] == row[key])[0]\n", - " for column in ('SURVEY', 'PROGRAM', 'FAPRGRM', 'FAFLAVOR', 'GOALTYPE'):\n", - " if column in patched[d].colnames:\n", - " if (patched[d][column][w] != row[column]).any():\n", - " log.info(\"Patching %s associated with %s %d with %s = '%s'.\", d, ('tile' if s == 'tiles' else 'exposure'), row[key], column, row[column])\n", - " patched[d][column][w] = row[column]" - ] - }, - { - "cell_type": "markdown", - "id": "70842f2d-5d9b-4d4a-b388-6dafc24dbe2d", - "metadata": {}, - "source": [ - "### Re-run QA" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ac798c48-0563-4f80-b202-66b01cf6d415", - "metadata": {}, - "outputs": [], - "source": [ - "for s, d in back_patch.items():\n", - " for row in patched[s]:\n", - " key = 'TILEID' if s == 'tiles' else 'EXPID'\n", - " w = np.where(patched[d][key] == row[key])[0]\n", - " for column in ('SURVEY', 'PROGRAM', 'FAPRGRM', 'FAFLAVOR', 'GOALTYPE'):\n", - " if column in patched[d].colnames:\n", - " assert (patched[d][column][w] == row[column]).all()" + "for new_tile in candidate_tiles:\n", + " row_index = np.where((patched['exposures']['TILEID'] == new_tile['TILEID']) & (patched['exposures']['EFFTIME_SPEC'] > 0))[0]\n", + " if len(row_index) == 0:\n", + " print(\"ERROR: No valid exposures found for tile {0:d}, even though EFFTIME_SPEC == {1:f}!\".format(new_tile['TILEID'], new_tile['EFFTIME_SPEC']))\n", + " bad_index = np.where((patched['exposures']['TILEID'] == new_tile['TILEID']))[0]\n", + " print(patched['exposures'][['TILEID', 'EXPID', 'NIGHT', 'SURVEY', 'PROGRAM', 'FAPRGRM', 'FAFLAVOR', 'EFFTIME_SPEC']][bad_index])\n", + " w = np.in1d(src['exposures']['EXPID'], patched['exposures']['EXPID'][bad_index])\n", + " n_src = w.sum()\n", + " print(src['exposures'][['TILEID', 'EXPID', 'NIGHT', 'SURVEY', 'PROGRAM', 'FAPRGRM', 'FAFLAVOR', 'EFFTIME_SPEC']][w])\n", + " if n_src == 0:\n", + " print(\"ERROR: Tile {0:d} cannot be patched with upstream data.\".format(new_tile['TILEID']))\n", + " elif n_src == len(bad_index):\n", + " print(\"INFO: Tile {0:d} can be fully patched with upstream data.\".format(new_tile['TILEID']))\n", + " elif n_src < len(bad_index):\n", + " print(\"WARNING: Tile {0:d} can be partially patched with upstream data.\".format(new_tile['TILEID']))\n", + " else:\n", + " print(\"CRITICAL: This is weird.\")\n", + " print('+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++')" ] }, { "cell_type": "code", "execution_count": null, - "id": "1f2a17e0-074c-433d-9b45-a35d91cc6a93", + "id": "e6c7934f-f87e-4567-adb2-4e8f7c1197f6", "metadata": {}, "outputs": [], "source": [] From 2a4442ccef42b9027cb2888d70293f52b76bab1a Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Mon, 7 Oct 2024 13:01:04 -0700 Subject: [PATCH 8/8] update Sphinx config --- doc/conf.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index f4bf071..d2d0aa7 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -20,7 +20,7 @@ project = 'specprod-db' python_project = 'specprodDB' -copyright = '2023, DESI Collaboration' +copyright = '2023-2024, DESI Collaboration' author = 'DESI Collaboration' @@ -35,7 +35,8 @@ 'sphinx.ext.todo', 'sphinx.ext.mathjax', 'sphinx.ext.viewcode', - 'sphinx.ext.napoleon' + 'sphinx.ext.napoleon', + 'sphinx_rtd_theme' ] # Configuration for intersphinx, copied from astropy. @@ -46,9 +47,10 @@ 'matplotlib': ('https://matplotlib.org/stable/', None), 'astropy': ('https://docs.astropy.org/en/stable/', None), 'h5py': ('https://docs.h5py.org/en/latest/', None), - 'sqlalchemy': ('https://docs.sqlalchemy.org/en/14/', None), + 'sqlalchemy': ('https://docs.sqlalchemy.org/en/20/', None), 'desispec': ('https://desispec.readthedocs.io/en/latest/', None), 'desiutil': ('https://desiutil.readthedocs.io/en/latest/', None), + 'desitarget': ('https://desitarget.readthedocs.io/en/latest/', None), } # Add any paths that contain templates here, relative to this directory. @@ -88,13 +90,9 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -# -try: - import sphinx_rtd_theme - html_theme = 'sphinx_rtd_theme' - html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] -except ImportError: - html_theme = 'alabaster' +#html_theme = 'default' +#html_theme = 'haiku' +html_theme = 'sphinx_rtd_theme' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files,