From 4142c8093914e6e2819f7e4258000f027f3089c4 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Tue, 7 May 2024 11:32:58 -0400 Subject: [PATCH 1/3] Updating max lengths in model and migrations. Updating string parsing for monitors as well. --- .../ta_monitors/msata_monitor.py | 3 ++- .../ta_monitors/wata_monitor.py | 3 ++- jwql/utils/constants.py | 1 + ...pecmsatastats_nirspecwatastats_and_more.py | 18 ++++++++-------- jwql/website/apps/jwql/monitor_models/ta.py | 21 +++++++++---------- 5 files changed, 24 insertions(+), 22 deletions(-) diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py index 33d2358ff..c0ea6201c 100755 --- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py +++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py @@ -38,6 +38,7 @@ import os import logging from datetime import datetime, timezone, timedelta +from dateutil import parser from random import randint import numpy as np @@ -467,7 +468,7 @@ def add_time_column(self): def add_timezone(self, date_str): """Method to bypass timezone warning from Django""" - dt_timezone = datetime.fromisoformat(date_str).replace(tzinfo=timezone.utc) + dt_timezone = parser.parse(date_str).replace(tzinfo=timezone.utc) return dt_timezone def plt_status(self): diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py index 0adcc983a..e11955420 100755 --- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py +++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py @@ -34,6 +34,7 @@ import os import logging from datetime import datetime, timezone, timedelta +from dateutil import parser import numpy as np import pandas as pd @@ -988,7 +989,7 @@ def update_ta_success_txtfile(self): def add_timezone(self, date_str): """Method to bypass timezone warning from Django""" - dt_timezone = datetime.fromisoformat(date_str).replace(tzinfo=timezone.utc) + dt_timezone = parser.parse(date_str).replace(tzinfo=timezone.utc) return dt_timezone def add_wata_data(self): diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index c1d9b00b1..a5884a7e4 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -1059,3 +1059,4 @@ class QueryConfigKeys: MAX_LEN_TIME = 50 MAX_LEN_TYPE = 40 MAX_LEN_USER = 50 +MAX_LEN_VISIT = 30 diff --git a/jwql/website/apps/jwql/migrations/0024_nirspecmsatastats_nirspecwatastats_and_more.py b/jwql/website/apps/jwql/migrations/0024_nirspecmsatastats_nirspecwatastats_and_more.py index d56f844b0..938d4e443 100644 --- a/jwql/website/apps/jwql/migrations/0024_nirspecmsatastats_nirspecwatastats_and_more.py +++ b/jwql/website/apps/jwql/migrations/0024_nirspecmsatastats_nirspecwatastats_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.6 on 2024-05-02 16:16 +# Generated by Django 4.2.6 on 2024-05-07 15:15 import django.contrib.postgres.fields from django.db import migrations, models @@ -17,8 +17,8 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('filename', models.CharField(blank=True, max_length=1000, null=True)), ('date_obs', models.DateTimeField(blank=True, null=True)), - ('visit_id', models.CharField(blank=True, null=True)), - ('tafilter', models.CharField(blank=True, null=True)), + ('visit_id', models.CharField(blank=True, max_length=30, null=True)), + ('tafilter', models.CharField(blank=True, max_length=7, null=True)), ('detector', models.CharField(blank=True, max_length=40, null=True)), ('readout', models.CharField(blank=True, max_length=40, null=True)), ('subarray', models.CharField(blank=True, max_length=40, null=True)), @@ -44,10 +44,10 @@ class Migration(migrations.Migration): ('samroll', models.FloatField(blank=True, null=True)), ('box_peak_value', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), ('reference_star_mag', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), - ('convergence_status', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(), size=None)), + ('convergence_status', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(blank=True, max_length=100, null=True), size=None)), ('reference_star_number', django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(), size=None)), - ('lsf_removed_status', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(), size=None)), - ('lsf_removed_reason', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(), size=None)), + ('lsf_removed_status', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(blank=True, max_length=100, null=True), size=None)), + ('lsf_removed_reason', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(blank=True, max_length=100, null=True), size=None)), ('lsf_removed_x', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), ('lsf_removed_y', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), ('planned_v2', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), @@ -67,10 +67,10 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('filename', models.CharField(blank=True, max_length=1000, null=True)), ('date_obs', models.DateTimeField(blank=True, null=True)), - ('visit_id', models.CharField(blank=True, null=True)), - ('tafilter', models.CharField(blank=True, null=True)), + ('visit_id', models.CharField(blank=True, max_length=30, null=True)), + ('tafilter', models.CharField(blank=True, max_length=7, null=True)), ('readout', models.CharField(blank=True, max_length=40, null=True)), - ('ta_status', models.CharField(blank=True, null=True)), + ('ta_status', models.CharField(blank=True, max_length=100, null=True)), ('star_name', models.IntegerField(blank=True, null=True)), ('star_ra', models.FloatField(blank=True, null=True)), ('star_dec', models.FloatField(blank=True, null=True)), diff --git a/jwql/website/apps/jwql/monitor_models/ta.py b/jwql/website/apps/jwql/monitor_models/ta.py index e97533417..3c49f8913 100644 --- a/jwql/website/apps/jwql/monitor_models/ta.py +++ b/jwql/website/apps/jwql/monitor_models/ta.py @@ -33,14 +33,13 @@ from jwql.utils.constants import ( MAX_LEN_APERTURE, MAX_LEN_DETECTOR, - MAX_LEN_TIME, MAX_LEN_FILENAME, + MAX_LEN_FILTER, MAX_LEN_INSTRUMENT, - MAX_LEN_NGROUPS, - MAX_LEN_NINTS, MAX_LEN_PATH, MAX_LEN_READPATTERN, MAX_LEN_SUBARRAY, + MAX_LEN_VISIT, ) @@ -97,10 +96,10 @@ class Meta: class NIRSpecWataStats(models.Model): filename = models.CharField(max_length=MAX_LEN_FILENAME, blank=True, null=True) date_obs = models.DateTimeField(blank=True, null=True) - visit_id = models.CharField(blank=True, null=True) - tafilter = models.CharField(blank=True, null=True) + visit_id = models.CharField(max_length=MAX_LEN_VISIT, blank=True, null=True) + tafilter = models.CharField(max_length=MAX_LEN_FILTER, blank=True, null=True) readout = models.CharField(max_length=MAX_LEN_READPATTERN, blank=True, null=True) - ta_status = models.CharField(blank=True, null=True) + ta_status = models.CharField(max_length=100,blank=True, null=True) star_name = models.IntegerField(blank=True, null=True) star_ra = models.FloatField(blank=True, null=True) star_dec = models.FloatField(blank=True, null=True) @@ -142,8 +141,8 @@ class Meta: class NIRSpecMsataStats(models.Model): filename = models.CharField(max_length=MAX_LEN_FILENAME, blank=True, null=True) date_obs = models.DateTimeField(blank=True, null=True) - visit_id = models.CharField(blank=True, null=True) - tafilter = models.CharField(blank=True, null=True) + visit_id = models.CharField(max_length=MAX_LEN_VISIT, blank=True, null=True) + tafilter = models.CharField(max_length=MAX_LEN_FILTER, blank=True, null=True) detector = models.CharField(max_length=MAX_LEN_DETECTOR, blank=True, null=True) readout = models.CharField(max_length=MAX_LEN_READPATTERN, blank=True, null=True) subarray = models.CharField(max_length=MAX_LEN_SUBARRAY, blank=True, null=True) @@ -169,10 +168,10 @@ class NIRSpecMsataStats(models.Model): samroll = models.FloatField(blank=True, null=True) box_peak_value = ArrayField(models.FloatField()) reference_star_mag = ArrayField(models.FloatField()) - convergence_status = ArrayField(models.CharField()) + convergence_status = ArrayField(models.CharField(max_length=100,blank=True, null=True)) reference_star_number = ArrayField(models.IntegerField()) - lsf_removed_status = ArrayField(models.CharField()) - lsf_removed_reason = ArrayField(models.CharField()) + lsf_removed_status = ArrayField(models.CharField(max_length=100,blank=True, null=True)) + lsf_removed_reason = ArrayField(models.CharField(max_length=100,blank=True, null=True)) lsf_removed_x = ArrayField(models.FloatField()) lsf_removed_y = ArrayField(models.FloatField()) planned_v2 = ArrayField(models.FloatField()) From 8ad1bde39a54029a7f8f766a10a17954feffa878 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Tue, 7 May 2024 11:34:43 -0400 Subject: [PATCH 2/3] pep8 --- jwql/website/apps/jwql/monitor_models/ta.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_models/ta.py b/jwql/website/apps/jwql/monitor_models/ta.py index 3c49f8913..290bd03f2 100644 --- a/jwql/website/apps/jwql/monitor_models/ta.py +++ b/jwql/website/apps/jwql/monitor_models/ta.py @@ -99,7 +99,7 @@ class NIRSpecWataStats(models.Model): visit_id = models.CharField(max_length=MAX_LEN_VISIT, blank=True, null=True) tafilter = models.CharField(max_length=MAX_LEN_FILTER, blank=True, null=True) readout = models.CharField(max_length=MAX_LEN_READPATTERN, blank=True, null=True) - ta_status = models.CharField(max_length=100,blank=True, null=True) + ta_status = models.CharField(max_length=100, blank=True, null=True) star_name = models.IntegerField(blank=True, null=True) star_ra = models.FloatField(blank=True, null=True) star_dec = models.FloatField(blank=True, null=True) @@ -168,10 +168,10 @@ class NIRSpecMsataStats(models.Model): samroll = models.FloatField(blank=True, null=True) box_peak_value = ArrayField(models.FloatField()) reference_star_mag = ArrayField(models.FloatField()) - convergence_status = ArrayField(models.CharField(max_length=100,blank=True, null=True)) + convergence_status = ArrayField(models.CharField(max_length=100, blank=True, null=True)) reference_star_number = ArrayField(models.IntegerField()) - lsf_removed_status = ArrayField(models.CharField(max_length=100,blank=True, null=True)) - lsf_removed_reason = ArrayField(models.CharField(max_length=100,blank=True, null=True)) + lsf_removed_status = ArrayField(models.CharField(max_length=100, blank=True, null=True)) + lsf_removed_reason = ArrayField(models.CharField(max_length=100, blank=True, null=True)) lsf_removed_x = ArrayField(models.FloatField()) lsf_removed_y = ArrayField(models.FloatField()) planned_v2 = ArrayField(models.FloatField()) From d0dfba3ee84d0cdb9f97bb87e4e3711acfc6134e Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Tue, 7 May 2024 11:57:51 -0400 Subject: [PATCH 3/3] adding generic text constant --- jwql/utils/constants.py | 1 + jwql/website/apps/jwql/monitor_models/ta.py | 15 +++++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index a5884a7e4..3691d0cd8 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -1045,6 +1045,7 @@ class QueryConfigKeys: MAX_LEN_DIFF_IMAGE = 1000 MAX_LEN_FILENAME = 1000 MAX_LEN_FILTER = 7 +MAX_LEN_GENERIC_TEXT = 100 MAX_LEN_GRATING = 40 MAX_LEN_INSTRUMENT = 7 MAX_LEN_MNEMONIC = 40 diff --git a/jwql/website/apps/jwql/monitor_models/ta.py b/jwql/website/apps/jwql/monitor_models/ta.py index 290bd03f2..34ef20988 100644 --- a/jwql/website/apps/jwql/monitor_models/ta.py +++ b/jwql/website/apps/jwql/monitor_models/ta.py @@ -35,6 +35,7 @@ MAX_LEN_DETECTOR, MAX_LEN_FILENAME, MAX_LEN_FILTER, + MAX_LEN_GENERIC_TEXT, MAX_LEN_INSTRUMENT, MAX_LEN_PATH, MAX_LEN_READPATTERN, @@ -99,7 +100,7 @@ class NIRSpecWataStats(models.Model): visit_id = models.CharField(max_length=MAX_LEN_VISIT, blank=True, null=True) tafilter = models.CharField(max_length=MAX_LEN_FILTER, blank=True, null=True) readout = models.CharField(max_length=MAX_LEN_READPATTERN, blank=True, null=True) - ta_status = models.CharField(max_length=100, blank=True, null=True) + ta_status = models.CharField(max_length=MAX_LEN_GENERIC_TEXT, blank=True, null=True) star_name = models.IntegerField(blank=True, null=True) star_ra = models.FloatField(blank=True, null=True) star_dec = models.FloatField(blank=True, null=True) @@ -168,10 +169,16 @@ class NIRSpecMsataStats(models.Model): samroll = models.FloatField(blank=True, null=True) box_peak_value = ArrayField(models.FloatField()) reference_star_mag = ArrayField(models.FloatField()) - convergence_status = ArrayField(models.CharField(max_length=100, blank=True, null=True)) + convergence_status = ArrayField( + models.CharField(max_length=MAX_LEN_GENERIC_TEXT, blank=True, null=True) + ) reference_star_number = ArrayField(models.IntegerField()) - lsf_removed_status = ArrayField(models.CharField(max_length=100, blank=True, null=True)) - lsf_removed_reason = ArrayField(models.CharField(max_length=100, blank=True, null=True)) + lsf_removed_status = ArrayField( + models.CharField(max_length=MAX_LEN_GENERIC_TEXT, blank=True, null=True) + ) + lsf_removed_reason = ArrayField( + models.CharField(max_length=MAX_LEN_GENERIC_TEXT, blank=True, null=True) + ) lsf_removed_x = ArrayField(models.FloatField()) lsf_removed_y = ArrayField(models.FloatField()) planned_v2 = ArrayField(models.FloatField())