From dbea5c779bb430df94ce7268c1f91b7306f8b121 Mon Sep 17 00:00:00 2001 From: Ben Sunnquist Date: Wed, 28 Feb 2024 10:48:34 -0500 Subject: [PATCH 1/4] updated bias monitor to work with new django databases --- .../common_monitors/bias_monitor.py | 54 ++++++++++--------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/bias_monitor.py b/jwql/instrument_monitors/common_monitors/bias_monitor.py index f47bc6213..3dde3b69b 100755 --- a/jwql/instrument_monitors/common_monitors/bias_monitor.py +++ b/jwql/instrument_monitors/common_monitors/bias_monitor.py @@ -35,7 +35,6 @@ import datetime import logging import os -from time import sleep from astropy.io import fits from astropy.stats import sigma_clip, sigma_clipped_stats @@ -47,21 +46,26 @@ from mpl_toolkits.axes_grid1 import make_axes_locatable # noqa: E402 (module import not at top) import numpy as np # noqa: E402 (module import not at top) from pysiaf import Siaf # noqa: E402 (module import not at top) -from sqlalchemy.sql.expression import and_ # noqa: E402 (module import not at top) -from jwql.database.database_interface import session, engine # noqa: E402 (module import not at top) -from jwql.database.database_interface import NIRCamBiasQueryHistory, NIRCamBiasStats, NIRISSBiasQueryHistory # noqa: E402 (module import not at top) -from jwql.database.database_interface import NIRISSBiasStats, NIRSpecBiasQueryHistory, NIRSpecBiasStats # noqa: E402 (module import not at top) from jwql.instrument_monitors import pipeline_tools # noqa: E402 (module import not at top) -from jwql.shared_tasks.shared_tasks import only_one, run_pipeline, run_parallel_pipeline # noqa: E402 (module import not at top) +from jwql.shared_tasks.shared_tasks import only_one, run_parallel_pipeline # noqa: E402 (module import not at top) from jwql.utils import instrument_properties, monitor_utils # noqa: E402 (module import not at top) -from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE # noqa: E402 (module import not at top) +from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, ON_GITHUB_ACTIONS, ON_READTHEDOCS # noqa: E402 (module import not at top) from jwql.utils.logging_functions import log_info, log_fail # noqa: E402 (module import not at top) -from jwql.utils.monitor_utils import update_monitor_table # noqa: E402 (module import not at top) from jwql.utils.permissions import set_permissions # noqa: E402 (module import not at top) -from jwql.utils.utils import copy_files, ensure_dir_exists, filesystem_path, get_config # noqa: E402 (module import not at top) +from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config # noqa: E402 (module import not at top) from jwql.website.apps.jwql.monitor_pages.monitor_bias_bokeh import BiasMonitorPlots # noqa: E402 (module import not at top) +if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS: + # Need to set up django apps before we can access the models + import django # noqa: E402 (module level import not at top of file) + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings") + django.setup() + + # Import * is okay here because this module specifically only contains database models + # for this monitor + from jwql.website.apps.jwql.monitor_models.bias import * # noqa: E402 (module level import not at top of file) + class Bias(): """Class for executing the bias monitor. @@ -201,15 +205,13 @@ def file_exists_in_database(self, filename): ``True`` if filename exists in the bias stats database. """ - query = session.query(self.stats_table) - results = query.filter(self.stats_table.uncal_filename == filename).all() + records = self.stats_table.objects.filter(uncal_filename__iexact=filename).all() - if len(results) != 0: + if len(records) != 0: file_exists = True else: file_exists = False - session.close() return file_exists def get_amp_medians(self, image, amps): @@ -346,16 +348,16 @@ def most_recent_search(self): where the bias monitor was run. """ - query = session.query(self.query_table).filter(and_(self.query_table.aperture == self.aperture, - self.query_table.run_monitor == True)).order_by(self.query_table.end_time_mjd).all() # noqa: E348 (comparison to true) + filters = {'aperture__iexact': self.aperture, + 'run_monitor': True} + record = self.query_table.objects.filter(**filters).order_by('-end_time_mjd').first() - if len(query) == 0: + if record is None: query_result = 59607.0 # a.k.a. Jan 28, 2022 == First JWST images (MIRI) logging.info(('\tNo query history for {}. Beginning search date will be set to {}.'.format(self.aperture, query_result))) else: - query_result = query[-1].end_time_mjd + query_result = record.end_time_mjd - session.close() return query_result def process(self, file_list): @@ -420,18 +422,18 @@ def process(self, file_list): 'mean': float(mean), 'median': float(median), 'stddev': float(stddev), - 'collapsed_rows': collapsed_rows.astype(float), - 'collapsed_columns': collapsed_columns.astype(float), - 'counts': counts.astype(float), - 'bin_centers': bin_centers.astype(float), + 'collapsed_rows': list(collapsed_rows.astype(float)), + 'collapsed_columns': list(collapsed_columns.astype(float)), + 'counts': list(counts.astype(float)), + 'bin_centers': list(bin_centers.astype(float)), 'entry_date': datetime.datetime.now() } for key in amp_medians.keys(): bias_db_entry[key] = float(amp_medians[key]) # Add this new entry to the bias database table - with engine.begin() as connection: - connection.execute(self.stats_table.__table__.insert(), bias_db_entry) + entry = self.stats_table(**bias_db_entry) + entry.save() # Don't print long arrays of numbers to the log file log_dict = {} @@ -545,8 +547,8 @@ def run(self): 'files_found': len(new_files), 'run_monitor': monitor_run, 'entry_date': datetime.datetime.now()} - with engine.begin() as connection: - connection.execute(self.query_table.__table__.insert(), new_entry) + entry = self.query_table(**new_entry) + entry.save() logging.info('\tUpdated the query history table') # Update the bias monitor plots From 6740da658fd5a01849b8a0516bc8de37052f2d95 Mon Sep 17 00:00:00 2001 From: Ben Sunnquist Date: Wed, 28 Feb 2024 16:55:20 -0500 Subject: [PATCH 2/4] bias monitor bokeh plots now working with django databases --- .../common_monitors/bias_monitor.py | 2 +- .../jwql/monitor_pages/monitor_bias_bokeh.py | 117 +++++++----------- 2 files changed, 49 insertions(+), 70 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/bias_monitor.py b/jwql/instrument_monitors/common_monitors/bias_monitor.py index 3dde3b69b..b1247ac58 100755 --- a/jwql/instrument_monitors/common_monitors/bias_monitor.py +++ b/jwql/instrument_monitors/common_monitors/bias_monitor.py @@ -348,7 +348,7 @@ def most_recent_search(self): where the bias monitor was run. """ - filters = {'aperture__iexact': self.aperture, + filters = {'aperture__iexact': self.aperture, 'run_monitor': True} record = self.query_table.objects.filter(**filters).order_by('-end_time_mjd').first() diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py index 0cd7d31e2..8ed29980b 100644 --- a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py @@ -23,11 +23,9 @@ from datetime import datetime, timedelta import os -from astropy.stats import sigma_clip - from bokeh.embed import components, file_html from bokeh.layouts import layout -from bokeh.models import ColorBar, ColumnDataSource, DatetimeTickFormatter, HoverTool, Legend, LinearAxis +from bokeh.models import ColumnDataSource, DatetimeTickFormatter, HoverTool from bokeh.models.layouts import Tabs, TabPanel from bokeh.plotting import figure, output_file, save from bokeh.resources import CDN @@ -37,13 +35,22 @@ from PIL import Image from sqlalchemy import func -from jwql.bokeh_templating import BokehTemplate from jwql.database.database_interface import get_unique_values_per_column, NIRCamBiasStats, NIRISSBiasStats, NIRSpecBiasStats, session -from jwql.utils.constants import FULL_FRAME_APERTURES, JWST_INSTRUMENT_NAMES_MIXEDCASE +from jwql.utils.constants import FULL_FRAME_APERTURES, JWST_INSTRUMENT_NAMES_MIXEDCASE, ON_GITHUB_ACTIONS, ON_READTHEDOCS from jwql.utils.permissions import set_permissions from jwql.utils.utils import read_png from jwql.website.apps.jwql.bokeh_utils import PlaceholderPlot +if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS: + # Need to set up django apps before we can access the models + import django # noqa: E402 (module level import not at top of file) + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings") + django.setup() + + # Import * is okay here because this module specifically only contains database models + # for this monitor + from jwql.website.apps.jwql.monitor_models.bias import * # noqa: E402 (module level import not at top of file) + SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) TEMPLATE_DIR = os.path.join(SCRIPT_DIR, '../templates') @@ -67,8 +74,8 @@ class BiasMonitorData(): Latest bias data for a particular aperture, from the stats_table - stats_table : sqlalchemy.orm.decl_api.DeclarativeMeta - Bias stats sqlalchemy table + stats_table : jwql.website.apps.jwql.monitor_models.bias.NIRCamBiasStats + Bias stats django database trending_data : pandas.DataFrame Data from the stats table to be used for the trending plot @@ -93,35 +100,24 @@ def retrieve_trending_data(self, aperture): """ # Query database for all data in bias stats with a matching aperture, # and sort the data by exposure start time. - tmp_trending_data = session.query(self.stats_table.amp1_even_med, - self.stats_table.amp1_odd_med, - self.stats_table.amp2_even_med, - self.stats_table.amp2_odd_med, - self.stats_table.amp3_even_med, - self.stats_table.amp3_odd_med, - self.stats_table.amp4_even_med, - self.stats_table.amp4_odd_med, - self.stats_table.expstart, - self.stats_table.uncal_filename) \ - .filter(self.stats_table.aperture == aperture) \ - .order_by(self.stats_table.expstart) \ - .all() - - session.close() + columns = ['amp1_even_med', 'amp1_odd_med', 'amp2_even_med', 'amp2_odd_med', + 'amp3_even_med', 'amp3_odd_med', 'amp4_even_med', 'amp4_odd_med', + 'expstart', 'uncal_filename'] + tmp_trending_data = self.stats_table.objects.filter(aperture__iexact=aperture).order_by('expstart').all().values(*columns) # Convert the query results to a pandas dataframe - self.trending_data = pd.DataFrame(tmp_trending_data, columns=['amp1_even_med', 'amp1_odd_med', - 'amp2_even_med', 'amp2_odd_med', - 'amp3_even_med', 'amp3_odd_med', - 'amp4_even_med', 'amp4_odd_med', - 'expstart_str', 'uncal_filename']) - uncal_basename = [os.path.basename(e) for e in self.trending_data['uncal_filename']] - self.trending_data['uncal_filename'] = uncal_basename - - # Add a column of expstart values that are datetime objects - format_data = "%Y-%m-%dT%H:%M:%S.%f" - datetimes = [datetime.strptime(entry, format_data) for entry in self.trending_data['expstart_str']] - self.trending_data['expstart'] = datetimes + if len(tmp_trending_data) != 0: + self.trending_data = pd.DataFrame.from_records(tmp_trending_data) + uncal_basename = [os.path.basename(e) for e in self.trending_data['uncal_filename']] + self.trending_data['uncal_filename'] = uncal_basename + + # Add a column of expstart values that are datetime objects + format_data = "%Y-%m-%dT%H:%M:%S.%f" + datetimes = [datetime.strptime(entry, format_data) for entry in self.trending_data['expstart']] + self.trending_data['expstart_str'] = self.trending_data['expstart'] + self.trending_data['expstart'] = datetimes + else: + self.trending_data = pd.DataFrame(None, columns=columns + ['uncal_filename', 'expstart_str']) def retrieve_latest_data(self, aperture): """Query the database table to get the data needed for the non-trending @@ -132,40 +128,23 @@ def retrieve_latest_data(self, aperture): aperture : str Aperture name (e.g. NRCA1_FULL) """ - subq = (session.query(self.stats_table.aperture, func.max(self.stats_table.expstart).label("max_created")) - .group_by(self.stats_table.aperture) - .subquery() - ) - - query = (session.query(self.stats_table.aperture, - self.stats_table.uncal_filename, - self.stats_table.cal_filename, - self.stats_table.cal_image, - self.stats_table.expstart, - self.stats_table.collapsed_rows, - self.stats_table.collapsed_columns, - self.stats_table.counts, - self.stats_table.bin_centers, - self.stats_table.entry_date) - .filter(self.stats_table.aperture == aperture) - .order_by(self.stats_table.entry_date) \ - .join(subq, self.stats_table.expstart == subq.c.max_created) - ) - - latest_data = query.all() - session.close() - - # Put the returned data in a dataframe. Include only the most recent entry. - # The query has already filtered to include only entries using the latest - # expstart value. - self.latest_data = pd.DataFrame(latest_data[-1:], columns=['aperture', 'uncal_filename', 'cal_filename', - 'cal_image', 'expstart_str', 'collapsed_rows', - 'collapsed_columns', 'counts', 'bin_centers', - 'entry_date']) - # Add a column of expstart values that are datetime objects - format_data = "%Y-%m-%dT%H:%M:%S.%f" - datetimes = [datetime.strptime(entry, format_data) for entry in self.latest_data['expstart_str']] - self.latest_data['expstart'] = datetimes + # Query database for the most recent bias stats entry with a matching aperture + columns = ['aperture', 'uncal_filename', 'cal_filename', 'cal_image', 'expstart', + 'collapsed_rows', 'collapsed_columns', 'counts', 'bin_centers', 'entry_date'] + tmp_data = self.stats_table.objects.filter(aperture__iexact=aperture).order_by('-expstart').all().values(*columns).first() + + # Put the returned data in a dataframe + if tmp_data is not None: + # Orient and transpose needed due to list column entries e.g. counts + self.latest_data = pd.DataFrame.from_dict(tmp_data, orient='index').transpose() + + # Add a column of expstart values that are datetime objects + format_data = "%Y-%m-%dT%H:%M:%S.%f" + datetimes = [datetime.strptime(entry, format_data) for entry in self.latest_data['expstart']] + self.latest_data['expstart_str'] = self.latest_data['expstart'] + self.latest_data['expstart'] = datetimes + else: + self.latest_data = pd.DataFrame(None, columns=columns + ['expstart_str']) class BiasMonitorPlots(): @@ -232,7 +211,7 @@ def __init__(self, instrument): self.db = BiasMonitorData(self.instrument) # Now we need to loop over the available apertures and create plots for each - self.available_apertures = get_unique_values_per_column(self.db.stats_table, 'aperture') + self.available_apertures = sorted(self.db.stats_table.objects.values_list('aperture', flat=True).distinct()) # Make sure all full frame apertures are present. If there are no data for a # particular full frame entry, then produce an empty plot, in order to From a8a8311deff6c9ea6995cc57d53e7a1f88709604 Mon Sep 17 00:00:00 2001 From: Ben Sunnquist Date: Wed, 28 Feb 2024 17:02:48 -0500 Subject: [PATCH 3/4] removing unnecessary imports --- .../apps/jwql/monitor_pages/monitor_bias_bokeh.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py index 8ed29980b..685392906 100644 --- a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py @@ -20,7 +20,7 @@ monitor_template.input_parameters = ('NIRCam', 'NRCA1_FULL') """ -from datetime import datetime, timedelta +from datetime import datetime import os from bokeh.embed import components, file_html @@ -29,13 +29,10 @@ from bokeh.models.layouts import Tabs, TabPanel from bokeh.plotting import figure, output_file, save from bokeh.resources import CDN -from datetime import datetime, timedelta +from datetime import datetime import numpy as np import pandas as pd -from PIL import Image -from sqlalchemy import func -from jwql.database.database_interface import get_unique_values_per_column, NIRCamBiasStats, NIRISSBiasStats, NIRSpecBiasStats, session from jwql.utils.constants import FULL_FRAME_APERTURES, JWST_INSTRUMENT_NAMES_MIXEDCASE, ON_GITHUB_ACTIONS, ON_READTHEDOCS from jwql.utils.permissions import set_permissions from jwql.utils.utils import read_png @@ -277,7 +274,7 @@ def ensure_all_full_frame_apertures(self): self.available_apertures.append(ap) def modify_bokeh_saved_html(self): - """Given an html string produced by Bokeh when saving bad pixel monitor plots, + """Given an html string produced by Bokeh when saving bias monitor plots, make tweaks such that the page follows the general JWQL page formatting. """ # Insert into our html template and save From 3435309cd64dcde90334abe56d36498c6dcca3d3 Mon Sep 17 00:00:00 2001 From: Ben Sunnquist Date: Thu, 21 Nov 2024 16:05:00 -0500 Subject: [PATCH 4/4] remove duplicate datetime import --- jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py index 4c3526ebd..9fe7fcd49 100644 --- a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py @@ -29,7 +29,6 @@ from bokeh.models.layouts import Tabs, TabPanel from bokeh.plotting import figure, output_file, save from bokeh.resources import CDN -from datetime import datetime import numpy as np import pandas as pd