Skip to content

Commit

Permalink
Merge pull request #1498 from mfixstsci/filesystem-mon-use-django-db
Browse files Browse the repository at this point in the history 
Update Filesystem Monitor to use Django DB Models
  • Loading branch information
@mfixstsci
mfixstsci authored Jan 17, 2025
2 parents e9669fd + d77a9fc commit 3c7e1a4
Show file tree
Hide file tree
Showing 3 changed files with 114 additions and 46 deletions.
50 changes: 25 additions & 25 deletions jwql/jwql_monitors/monitor_filesystem.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,22 +49,27 @@
import numpy as np
from sqlalchemy.exc import DataError

from jwql.database.database_interface import engine
from jwql.database.database_interface import session
from jwql.database.database_interface import FilesystemCharacteristics
from jwql.database.database_interface import FilesystemGeneral
from jwql.database.database_interface import FilesystemInstrument
from jwql.database.database_interface import CentralStore
from jwql.utils.logging_functions import log_info, log_fail
from jwql.utils.permissions import set_permissions
from jwql.utils.constants import FILESYSTEM_MONITOR_SUBDIRS, FILE_SUFFIX_TYPES, FILTERS_PER_INSTRUMENT, INSTRUMENT_SERVICE_MATCH
from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES_MIXEDCASE
from jwql.utils.constants import ON_GITHUB_ACTIONS, ON_READTHEDOCS
from jwql.utils.utils import filename_parser
from jwql.utils.utils import get_config
from jwql.utils.monitor_utils import initialize_instrument_monitor, update_monitor_table
from jwql.utils.protect_module import lock_module
from jwql.website.apps.jwql.data_containers import get_instrument_proposals

if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS:
# Need to set up django apps before we can access the models
import django # noqa: E402 (module level import not at top of file)
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings")
django.setup()

# Import * is okay here because this module specifically only contains database models
# for this monitor
from jwql.website.apps.jwql.monitor_models.common import * # noqa: E402 (module level import not at top of file)

SETTINGS = get_config()
FILESYSTEM = SETTINGS['filesystem']
PROPRIETARY_FILESYSTEM = os.path.join(FILESYSTEM, 'proprietary')
Expand All @@ -74,6 +79,7 @@
PREVIEW_IMAGES = SETTINGS['preview_image_filesystem']
THUMBNAILS = SETTINGS['thumbnail_filesystem']
LOGS = SETTINGS['log_dir']
WORKING = SETTINGS['working']


def files_per_filter():
Expand Down Expand Up @@ -232,7 +238,8 @@ def get_area_stats(central_storage_dict):
'logs': LOGS,
'preview_images': PREVIEW_IMAGES,
'thumbnails': THUMBNAILS,
'all': CENTRAL}
'all': CENTRAL,
'working':WORKING}

counteddirs = []

Expand Down Expand Up @@ -368,7 +375,7 @@ def initialize_results_dicts():
A dictionary for the ``central_storage`` database table
"""

now = datetime.datetime.now()
now = datetime.datetime.now(datetime.timezone.utc)

general_results_dict = {}
general_results_dict['date'] = now
Expand Down Expand Up @@ -430,9 +437,9 @@ def update_central_store_database(central_storage_dict):
new_record['size'] = central_storage_dict[area]['size']
new_record['used'] = central_storage_dict[area]['used']
new_record['available'] = central_storage_dict[area]['available']
with engine.begin() as connection:
connection.execute(CentralStore.__table__.insert(), new_record)
session.close()

entry = CentralStorage(**new_record)
entry.save()


def update_characteristics_database(char_info):
Expand All @@ -447,7 +454,7 @@ def update_characteristics_database(char_info):
using that filter/pupil.
"""
logging.info('\tUpdating the characteristics database')
now = datetime.datetime.now()
now = datetime.datetime.now(datetime.timezone.utc)

# Add data to filesystem_instrument table
for instrument in ['nircam', 'niriss', 'nirspec', 'miri']:
Expand All @@ -458,11 +465,9 @@ def update_characteristics_database(char_info):
new_record['instrument'] = instrument
new_record['filter_pupil'] = optics
new_record['obs_per_filter_pupil'] = values
with engine.begin() as connection:
connection.execute(
FilesystemCharacteristics.__table__.insert(), new_record)

session.close()
entry = FilesystemCharacteristics(**new_record)
entry.save()


def update_database(general_results_dict, instrument_results_dict, central_storage_dict):
Expand All @@ -478,8 +483,8 @@ def update_database(general_results_dict, instrument_results_dict, central_stora
"""
logging.info('\tUpdating the database')

with engine.begin() as connection:
connection.execute(FilesystemGeneral.__table__.insert(), general_results_dict)
fs_general_entry = FilesystemGeneral(**general_results_dict)
fs_general_entry.save()

# Add data to filesystem_instrument table
for instrument in JWST_INSTRUMENT_NAMES:
Expand All @@ -493,13 +498,8 @@ def update_database(general_results_dict, instrument_results_dict, central_stora

# Protect against updated enum options that have not been propagated to
# the table definition
try:
with engine.begin() as connection:
connection.execute(FilesystemInstrument.__table__.insert(), new_record)
except DataError as e:
logging.error(e)

session.close()
fs_instrument_entry = FilesystemInstrument(**new_record)
fs_instrument_entry.save()


@lock_module
Expand Down
39 changes: 39 additions & 0 deletions ...ebsite/apps/jwql/migrations/0028_alter_filesystemcharacteristics_filter_pupil_and_more.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
# Generated by Django 5.1.4 on 2025-01-16 21:35

import django.contrib.postgres.fields
from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
('jwql', '0027_alter_fgsbadpixelstats_source_files_and_more'),
]

operations = [
migrations.AlterField(
model_name='filesystemcharacteristics',
name='filter_pupil',
field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(default='empty', help_text='filter and/or pupil name', max_length=7), blank=True, null=True, size=None),
),
migrations.AlterField(
model_name='filesystemcharacteristics',
name='instrument',
field=models.CharField(),
),
migrations.AlterField(
model_name='filesystemcharacteristics',
name='obs_per_filter_pupil',
field=django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(), blank=True, null=True, size=None),
),
migrations.AlterField(
model_name='filesysteminstrument',
name='filetype',
field=models.CharField(),
),
migrations.AlterField(
model_name='filesysteminstrument',
name='instrument',
field=models.CharField(default='empty', help_text='JWST instrument name', max_length=7),
),
]
71 changes: 50 additions & 21 deletions jwql/website/apps/jwql/monitor_models/common.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,22 +172,29 @@ class Meta:
For more information please see:
```https://docs.djangoproject.com/en/2.0/topics/db/models/```
"""

# This is an auto-generated Django model module.
# Feel free to rename the models, but don't rename db_table values or field names.
from django.db import models
from django.contrib.postgres.fields import ArrayField

from jwql.utils.constants import (
DEFAULT_MODEL_CHARFIELD,
MAX_LEN_FILTER,
MAX_LEN_INSTRUMENT,
)


class Monitor(models.Model):
monitor_name = models.CharField()
start_time = models.DateTimeField()
end_time = models.DateTimeField(blank=True, null=True)
status = models.TextField(blank=True, null=True) # This field type is a guess.
status = models.TextField(blank=True, null=True)
log_file = models.CharField()

class Meta:
managed = True
db_table = 'monitor'
db_table = "monitor"


class CentralStorage(models.Model):
Expand All @@ -199,18 +206,30 @@ class CentralStorage(models.Model):

class Meta:
managed = True
db_table = 'central_storage'
db_table = "central_storage"


class FilesystemCharacteristics(models.Model):
date = models.DateTimeField()
instrument = models.TextField() # This field type is a guess.
filter_pupil = models.TextField(blank=True, null=True) # This field type is a guess.
obs_per_filter_pupil = models.TextField(blank=True, null=True) # This field type is a guess.
instrument = models.CharField()
filter_pupil = ArrayField(
models.CharField(
max_length=MAX_LEN_FILTER,
help_text="filter and/or pupil name",
default=DEFAULT_MODEL_CHARFIELD,
),
blank=True,
null=True,
)
obs_per_filter_pupil = ArrayField(
models.IntegerField(),
blank=True,
null=True,
)

class Meta:
managed = True
db_table = 'filesystem_characteristics'
db_table = "filesystem_characteristics"


class FilesystemGeneral(models.Model):
Expand All @@ -224,20 +243,24 @@ class FilesystemGeneral(models.Model):

class Meta:
managed = True
db_table = 'filesystem_general'
db_table = "filesystem_general"


class FilesystemInstrument(models.Model):
date = models.DateTimeField()
instrument = models.TextField() # This field type is a guess.
filetype = models.TextField() # This field type is a guess.
instrument = models.CharField(
max_length=MAX_LEN_INSTRUMENT,
help_text="JWST instrument name",
default=DEFAULT_MODEL_CHARFIELD,
)
filetype = models.CharField()
count = models.IntegerField()
size = models.FloatField()

class Meta:
managed = True
db_table = 'filesystem_instrument'
unique_together = (('date', 'instrument', 'filetype'),)
db_table = "filesystem_instrument"
unique_together = (("date", "instrument", "filetype"),)


class FgsAnomaly(models.Model):
Expand All @@ -257,7 +280,7 @@ class FgsAnomaly(models.Model):

class Meta:
managed = True
db_table = 'fgs_anomaly'
db_table = "fgs_anomaly"


class MiriAnomaly(models.Model):
Expand All @@ -274,15 +297,19 @@ class MiriAnomaly(models.Model):
row_pull_down = models.BooleanField()
other = models.BooleanField()
column_pull_down = models.BooleanField()
mrs_glow = models.BooleanField(db_column='MRS_Glow') # Field name made lowercase.
mrs_zipper = models.BooleanField(db_column='MRS_Zipper') # Field name made lowercase.
mrs_glow = models.BooleanField(db_column="MRS_Glow") # Field name made lowercase.
mrs_zipper = models.BooleanField(
db_column="MRS_Zipper"
) # Field name made lowercase.
row_pull_up = models.BooleanField()
lrs_contamination = models.BooleanField(db_column='LRS_Contamination') # Field name made lowercase.
lrs_contamination = models.BooleanField(
db_column="LRS_Contamination"
) # Field name made lowercase.
tree_rings = models.BooleanField()

class Meta:
managed = True
db_table = 'miri_anomaly'
db_table = "miri_anomaly"


class NircamAnomaly(models.Model):
Expand All @@ -307,7 +334,7 @@ class NircamAnomaly(models.Model):

class Meta:
managed = True
db_table = 'nircam_anomaly'
db_table = "nircam_anomaly"


class NirissAnomaly(models.Model):
Expand All @@ -329,7 +356,7 @@ class NirissAnomaly(models.Model):

class Meta:
managed = True
db_table = 'niriss_anomaly'
db_table = "niriss_anomaly"


class NirspecAnomaly(models.Model):
Expand All @@ -345,10 +372,12 @@ class NirspecAnomaly(models.Model):
data_transfer_error = models.BooleanField()
ghost = models.BooleanField()
snowball = models.BooleanField()
dominant_msa_leakage = models.BooleanField(db_column='Dominant_MSA_Leakage') # Field name made lowercase.
dominant_msa_leakage = models.BooleanField(
db_column="Dominant_MSA_Leakage"
) # Field name made lowercase.
optical_short = models.BooleanField()
other = models.BooleanField()

class Meta:
managed = True
db_table = 'nirspec_anomaly'
db_table = "nirspec_anomaly"

0 comments on commit 3c7e1a4

Please sign in to comment.