From a7266fe67df69ee74a3d338704e386c6f089077e Mon Sep 17 00:00:00 2001 From: Lukasz Wyrzykowski Date: Fri, 22 Nov 2024 10:33:24 +0100 Subject: [PATCH 01/11] fix to Gaia Alerts description in import, issue bhtom2/224 --- bhtom2/bhtom_targets/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bhtom2/bhtom_targets/utils.py b/bhtom2/bhtom_targets/utils.py index 0ac414a..8231cc5 100644 --- a/bhtom2/bhtom_targets/utils.py +++ b/bhtom2/bhtom_targets/utils.py @@ -132,8 +132,8 @@ def import_targets(targets, group_name=None, user=None): ra = catalog_data["ra"] dec = catalog_data["dec"] disc = catalog_data["discovery_date"] - - description = target_fields.get('description', '') + description = catalog_data["description"] + #description = target_fields.get('description', '') importance = target_fields.get('importance', str(9.99)) cadence = target_fields.get('cadence', str(1.0)) targetType = target_fields.get('type', Target.SIDEREAL) From 9bb23ebf0a6be93e958adce6c1ed452150e383de Mon Sep 17 00:00:00 2001 From: Lukasz Wyrzykowski Date: Fri, 22 Nov 2024 12:11:46 +0100 Subject: [PATCH 02/11] now fixing import to allow description field in Gaia Alerts to be used instead of the catalog description field. Related to issue #224. --- bhtom2/bhtom_targets/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bhtom2/bhtom_targets/utils.py b/bhtom2/bhtom_targets/utils.py index 8866d96..f53b610 100644 --- a/bhtom2/bhtom_targets/utils.py +++ b/bhtom2/bhtom_targets/utils.py @@ -132,8 +132,10 @@ def import_targets(targets, group_name=None, user=None): ra = catalog_data["ra"] dec = catalog_data["dec"] disc = catalog_data["discovery_date"] - description = catalog_data["description"] - #description = target_fields.get('description', '') + # description = catalog_data["description"] + # description = target_fields.get('description', '') + # Use catalog_data's description as the default + description = target_fields.get('description', catalog_data.get("description", '')) importance = target_fields.get('importance', str(9.99)) cadence = target_fields.get('cadence', str(1.0)) targetType = target_fields.get('type', Target.SIDEREAL) From 6c7400fe6708bac201fa111f917faa43240d4bfa Mon Sep 17 00:00:00 2001 From: Yurii Purdenko Date: Tue, 26 Nov 2024 10:13:07 +0100 Subject: [PATCH 03/11] fix: #137 --- bhtom2/bhtom_targets/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bhtom2/bhtom_targets/utils.py b/bhtom2/bhtom_targets/utils.py index f53b610..cfbd9b9 100644 --- a/bhtom2/bhtom_targets/utils.py +++ b/bhtom2/bhtom_targets/utils.py @@ -90,6 +90,8 @@ def import_targets(targets, group_name=None, user=None): elif k_source_name == 'GAIA_ALERT': target_names['GAIA_ALERTS'] = row_k_value elif k == 'classification': + target_fields['classification'] = row_k_value + elif k == 'description': target_fields['description'] = row_k_value elif k == 'priority': target_fields['importance'] = row_k_value From 700f5ff27876333d548600bd8b8edf45b68caffe Mon Sep 17 00:00:00 2001 From: Yurii Purdenko Date: Wed, 27 Nov 2024 12:27:46 +0100 Subject: [PATCH 04/11] fix: import targets, delete migrations --- .gitignore | 3 +- .../migrations/0001_initial.py | 59 ------------ ...libration_data_calibration_log_and_more.py | 24 ----- .../migrations/0001_initial.py | 94 ------------------- ...servatory_comment_alter_observatory_lon.py | 24 ----- 5 files changed, 2 insertions(+), 202 deletions(-) delete mode 100644 bhtom2/bhtom_calibration/migrations/0001_initial.py delete mode 100644 bhtom2/bhtom_calibration/migrations/0002_calibration_data_calibration_log_and_more.py delete mode 100644 bhtom2/bhtom_observatory/migrations/0001_initial.py delete mode 100644 bhtom2/bhtom_observatory/migrations/0002_alter_observatory_comment_alter_observatory_lon.py diff --git a/.gitignore b/.gitignore index d63b483..563e1c7 100644 --- a/.gitignore +++ b/.gitignore @@ -174,4 +174,5 @@ __pycache__/ bhtom2/.DS_Store *.env -/data \ No newline at end of file +/data +.gitattributes \ No newline at end of file diff --git a/bhtom2/bhtom_calibration/migrations/0001_initial.py b/bhtom2/bhtom_calibration/migrations/0001_initial.py deleted file mode 100644 index 8418dd0..0000000 --- a/bhtom2/bhtom_calibration/migrations/0001_initial.py +++ /dev/null @@ -1,59 +0,0 @@ -# Generated by Django 4.0.4 on 2024-03-08 15:49 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ('bhtom_dataproducts', '0001_initial'), - ] - - operations = [ - migrations.CreateModel( - name='Catalogs', - fields=[ - ('id', models.IntegerField(primary_key=True, serialize=False)), - ('survey', models.TextField(editable=False)), - ('filters', models.TextField(editable=False)), - ('isActive', models.BooleanField(default=True)), - ], - ), - migrations.CreateModel( - name='Calibration_data', - fields=[ - ('id', models.AutoField(db_index=True, primary_key=True, serialize=False)), - ('status', models.CharField(choices=[('C', 'TO DO'), ('P', 'IN PROGRESS'), ('S', 'SUCCESS'), ('E', 'ERROR')], db_index=True, default='C', max_length=1)), - ('status_message', models.TextField(blank=True, null=True)), - ('mjd', models.FloatField()), - ('exp_time', models.FloatField(blank=True, null=True)), - ('mag', models.FloatField(blank=True, null=True)), - ('mag_error', models.FloatField(blank=True, null=True)), - ('ra', models.FloatField(blank=True, null=True)), - ('dec', models.FloatField(blank=True, null=True)), - ('zeropoint', models.FloatField(blank=True, null=True)), - ('outlier_fraction', models.FloatField(blank=True, null=True)), - ('scatter', models.FloatField(blank=True, null=True)), - ('npoints', models.IntegerField(blank=True, null=True)), - ('processing_time', models.FloatField(blank=True, null=True)), - ('created', models.DateTimeField(blank=True, editable=False, null=True)), - ('modified', models.DateTimeField(blank=True, null=True)), - ('start_processing', models.DateTimeField(blank=True, null=True)), - ('best_filter', models.CharField(blank=True, max_length=5, null=True)), - ('survey', models.CharField(blank=True, max_length=32, null=True)), - ('match_distans', models.FloatField(default=0.5)), - ('no_plot', models.BooleanField(default=True)), - ('calibration_plot', models.TextField(blank=True, null=True)), - ('number_tries', models.IntegerField(default=0)), - ('dataproduct', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='bhtom_dataproducts.dataproduct')), - ('use_catalog', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='bhtom_calibration.catalogs')), - ], - options={ - 'verbose_name': 'cpcs processing file', - 'verbose_name_plural': 'calibration_data', - }, - ), - ] \ No newline at end of file diff --git a/bhtom2/bhtom_calibration/migrations/0002_calibration_data_calibration_log_and_more.py b/bhtom2/bhtom_calibration/migrations/0002_calibration_data_calibration_log_and_more.py deleted file mode 100644 index 0763725..0000000 --- a/bhtom2/bhtom_calibration/migrations/0002_calibration_data_calibration_log_and_more.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 4.0.4 on 2024-06-11 08:27 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('bhtom_calibration', '0001_initial'), - ] - - operations = [ - migrations.AddField( - model_name='calibration_data', - name='calibration_log', - field=models.URLField(blank=True, default=None, null=True), - ), - migrations.AlterField( - model_name='calibration_data', - name='use_catalog', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='bhtom_calibration.catalogs'), - ), - ] \ No newline at end of file diff --git a/bhtom2/bhtom_observatory/migrations/0001_initial.py b/bhtom2/bhtom_observatory/migrations/0001_initial.py deleted file mode 100644 index 018bded..0000000 --- a/bhtom2/bhtom_observatory/migrations/0001_initial.py +++ /dev/null @@ -1,94 +0,0 @@ -# Generated by Django 4.0.4 on 2024-03-08 15:49 - -import bhtom2.bhtom_observatory.models -from django.conf import settings -import django.core.validators -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ] - - operations = [ - migrations.CreateModel( - name='Observatory', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=255, unique=True, verbose_name='Observatory name')), - ('lon', models.FloatField(db_index=True, validators=[django.core.validators.MinValueValidator(-180.0, message='longitude must be greater than -180.'), django.core.validators.MaxValueValidator(180.0, message='longitude must be less than 180.')], verbose_name='Longitude (West is positive) [deg]')), - ('lat', models.FloatField(db_index=True, validators=[django.core.validators.MinValueValidator(-180.0, message='latitude must be greater than -90.'), django.core.validators.MaxValueValidator(180.0, message='latitude must be less than 90.')], verbose_name='Latitude (North is positive) [deg]')), - ('altitude', models.FloatField(default=0.0, null=True, verbose_name='Altitude [m]')), - ('calibration_flg', models.BooleanField(db_index=True, default='False', verbose_name='Only instrumental photometry file')), - ('comment', models.TextField(blank=True, null=True, verbose_name='Comments (e.g. hyperlink to the observatory website, camera specifications, telescope info)')), - ('approx_lim_mag', models.FloatField(default=18.0, null=True, verbose_name='Approximate limit magnitude [mag]')), - ('filters', models.CharField(blank=True, default='V,R,I', max_length=100, null=True, verbose_name='Filters (comma-separated list, as they are visible in FITS)')), - ('origin', models.CharField(blank=True, max_length=255, null=True, verbose_name='Origin')), - ('telescope', models.CharField(blank=True, max_length=255, null=True, verbose_name='Telescope name')), - ('aperture', models.FloatField(blank=True, default=0.0, null=True, verbose_name='Aperture [m]')), - ('focal_length', models.FloatField(default=0.0, null=True, verbose_name='Focal length [mm]')), - ('seeing', models.FloatField(blank=True, default=0.0, null=True)), - ('created', models.DateTimeField(auto_now_add=True, db_index=True, null=True)), - ('modified', models.DateTimeField(auto_now_add=True, null=True)), - ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), - ], - options={ - 'verbose_name_plural': 'observatories', - 'unique_together': {('name', 'lon', 'lat')}, - }, - ), - migrations.CreateModel( - name='Camera', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('camera_name', models.CharField(max_length=255, verbose_name='Camera name')), - ('active_flg', models.BooleanField(db_index=True, default='False')), - ('prefix', models.CharField(blank=True, max_length=100, null=True, unique=True)), - ('gain', models.FloatField(default=2, null=True, verbose_name='Gain [e/ADU]')), - ('example_file', models.FileField(null=True, upload_to=bhtom2.bhtom_observatory.models.example_file_path, verbose_name='Sample fits')), - ('readout_noise', models.FloatField(default=2, null=True, verbose_name='Readout Noise [e]')), - ('binning', models.IntegerField(default=1, null=True)), - ('saturation_level', models.FloatField(default=63000, null=True, verbose_name='Saturation Level [ADU]')), - ('pixel_scale', models.FloatField(default=0.8, null=True, verbose_name='Pixel Scale [arcseconds/pixel]')), - ('readout_speed', models.FloatField(default=9999.0, null=True, verbose_name='Readout Speed [microseconds/pixel]')), - ('pixel_size', models.FloatField(default=13.5, null=True, verbose_name='Pixel size [micrometers]')), - ('date_time_keyword', models.CharField(default='DATE-OBS', max_length=255, verbose_name='Date & Time keyword')), - ('time_keyword', models.CharField(default='TIME-OBS', max_length=255, verbose_name='Time keyword')), - ('exposure_time_keyword', models.CharField(default='EXPTIME', max_length=255, verbose_name='Exposure time keyword')), - ('mode_recognition_keyword', models.CharField(blank=True, default='', max_length=255, null=True, verbose_name='Mode recognition keyword name')), - ('additional_info', models.TextField(blank=True, default='', null=True, verbose_name='Additional info')), - ('created', models.DateTimeField(auto_now_add=True, db_index=True, null=True)), - ('modified', models.DateTimeField(auto_now=True, null=True)), - ('observatory', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='bhtom_observatory.observatory')), - ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), - ], - options={ - 'verbose_name_plural': 'cameras', - 'unique_together': {('observatory', 'camera_name')}, - }, - ), - migrations.CreateModel( - name='ObservatoryMatrix', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('active_flg', models.BooleanField(default='True')), - ('comment', models.TextField(blank=True, null=True, verbose_name='Comments (e.g. hyperlink to the observatory website, camera specifications, telescope info)')), - ('created', models.DateTimeField(auto_now_add=True, db_index=True)), - ('modified', models.DateTimeField(auto_now_add=True, null=True)), - ('number_of_uploaded_file', models.IntegerField(default=0)), - ('file_size', models.FloatField(default=0)), - ('last_file_process', models.DateTimeField(blank=True, null=True)), - ('camera', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='bhtom_observatory.camera')), - ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), - ], - options={ - 'verbose_name_plural': 'observatory matrix', - 'unique_together': {('user', 'camera')}, - }, - ), - ] \ No newline at end of file diff --git a/bhtom2/bhtom_observatory/migrations/0002_alter_observatory_comment_alter_observatory_lon.py b/bhtom2/bhtom_observatory/migrations/0002_alter_observatory_comment_alter_observatory_lon.py deleted file mode 100644 index 6f1ae9b..0000000 --- a/bhtom2/bhtom_observatory/migrations/0002_alter_observatory_comment_alter_observatory_lon.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 4.0.4 on 2024-05-22 08:17 - -import django.core.validators -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('bhtom_observatory', '0001_initial'), - ] - - operations = [ - migrations.AlterField( - model_name='observatory', - name='comment', - field=models.TextField(blank=True, null=True, verbose_name='Comments (e.g. camera specifications, telescope info)'), - ), - migrations.AlterField( - model_name='observatory', - name='lon', - field=models.FloatField(db_index=True, validators=[django.core.validators.MinValueValidator(-180.0, message='longitude must be greater than -180.'), django.core.validators.MaxValueValidator(180.0, message='longitude must be less than 180.')], verbose_name='Longitude (East is positive) [deg]'), - ), - ] \ No newline at end of file From 9134e834cb3bc89e3ee389239d436380ac6f2d78 Mon Sep 17 00:00:00 2001 From: Lukasz Wyrzykowski Date: Wed, 27 Nov 2024 13:14:00 +0100 Subject: [PATCH 05/11] observation id randomly generated added to REM --- bhtom2/bhtom_observations/facilities/rem.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bhtom2/bhtom_observations/facilities/rem.py b/bhtom2/bhtom_observations/facilities/rem.py index f6462d7..bb0be49 100644 --- a/bhtom2/bhtom_observations/facilities/rem.py +++ b/bhtom2/bhtom_observations/facilities/rem.py @@ -9,6 +9,7 @@ from django.conf import settings from datetime import datetime, timedelta +import random SUCCESSFUL_OBSERVING_STATES = ['COMPLETED'] FAILED_OBSERVING_STATES = ['WINDOW_EXPIRED', 'CANCELED', 'FAILURE_LIMIT_REACHED', 'NOT_ATTEMPTED'] @@ -364,7 +365,8 @@ def submit_observation(self, observation_payload): recipient_email = ["remobs@www.rem.inaf.it","wyrzykow@gmail.com"] # Send the email self.send_template_email(filled_template, recipient_email) - return [] + obs_id = random.randint(10000, 99999) + return [obs_id] From 950c8ecf17cb5cb05493ab963d689868ab4309d2 Mon Sep 17 00:00:00 2001 From: Yurii Purdenko Date: Wed, 27 Nov 2024 13:23:39 +0100 Subject: [PATCH 06/11] fix --- bhtom2/bhtom_observations/facilities/rem.py | 35 ++++++++++++--------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/bhtom2/bhtom_observations/facilities/rem.py b/bhtom2/bhtom_observations/facilities/rem.py index f6462d7..210dfce 100644 --- a/bhtom2/bhtom_observations/facilities/rem.py +++ b/bhtom2/bhtom_observations/facilities/rem.py @@ -10,6 +10,8 @@ from datetime import datetime, timedelta +logger: BHTOMLogger = BHTOMLogger(__name__, 'Bhtom: bhtom_rem.views') + SUCCESSFUL_OBSERVING_STATES = ['COMPLETED'] FAILED_OBSERVING_STATES = ['WINDOW_EXPIRED', 'CANCELED', 'FAILURE_LIMIT_REACHED', 'NOT_ATTEMPTED'] TERMINAL_OBSERVING_STATES = SUCCESSFUL_OBSERVING_STATES + FAILED_OBSERVING_STATES @@ -380,20 +382,25 @@ def date_to_julian_date(self,date_str): return julian_date #recipients can be a single string or a list of strings - def send_template_email(self,filled_template, recipients): + def send_template_email(self, filled_template, recipients): + # Ensure recipients is a list even if a single email is passed if isinstance(recipients, str): recipients = [recipients] # Convert single email to list - subject = "REM_OBS" #don't change! - message = filled_template # The filled template string - from_email = settings.EMAIL_HOST_USER # From email address - recipient_list = recipients - - # Send the email - send_mail( - subject, - message, - from_email, - recipient_list, - fail_silently=False, # Set to True in production to avoid raising errors - ) \ No newline at end of file + subject = "REM_OBS" # Don't change! + from_email = settings.EMAIL_HOST_USER # Ensure this is configured correctly + + try: + send_mail( + subject=subject, + message=filled_template, + from_email=from_email, + recipient_list=recipients, + fail_silently=False # Set to True in production to suppress errors + ) + print(f"Failed to send email: {e}") # Replace with proper logging + + except Exception as e: + # Optionally, log the error or take other actions + print(f"Failed to send email: {e}") # Replace with proper logging + return False # Indicate failure \ No newline at end of file From 723d96113e78ca9d592ed5742d9961ad35b65ebb Mon Sep 17 00:00:00 2001 From: Yurii Purdenko Date: Wed, 27 Nov 2024 13:46:04 +0100 Subject: [PATCH 07/11] fix: rem --- bhtom2/bhtom_observations/facilities/rem.py | 22 ++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/bhtom2/bhtom_observations/facilities/rem.py b/bhtom2/bhtom_observations/facilities/rem.py index 463b0e0..7646760 100644 --- a/bhtom2/bhtom_observations/facilities/rem.py +++ b/bhtom2/bhtom_observations/facilities/rem.py @@ -8,10 +8,11 @@ from django.core.mail import send_mail from django.conf import settings from datetime import datetime, timedelta +from bhtom2.utils.bhtom_logger import BHTOMLogger import random -logger: BHTOMLogger = BHTOMLogger(__name__, 'Bhtom: bhtom_rem.views') +logger: BHTOMLogger = BHTOMLogger(__name__, 'Bhtom: bhtom_observations.facilities.rem') SUCCESSFUL_OBSERVING_STATES = ['COMPLETED'] FAILED_OBSERVING_STATES = ['WINDOW_EXPIRED', 'CANCELED', 'FAILURE_LIMIT_REACHED', 'NOT_ATTEMPTED'] @@ -176,10 +177,13 @@ def all_data_products(self, observation_record): def submit_observation(self, observation_payload): #print(observation_payload) + logger.debug("submitted rem observations") # Retrieve target information using the target_id target_id = observation_payload['target_id'] - target = Target.objects.get(id=target_id) - + try: + target = Target.objects.get(id=target_id) + except Exception as e: + logger.error("No target id", str(e)) # Extract target details # removing spaces in target name (REM requirement) target_name = target.name.replace(" ", "_") # or use .replace(" ", "") @@ -394,15 +398,15 @@ def send_template_email(self, filled_template, recipients): try: send_mail( - subject=subject, - message=filled_template, - from_email=from_email, - recipient_list=recipients, + subject, + filled_template, + from_email, + recipients, fail_silently=False # Set to True in production to suppress errors ) - print(f"Failed to send email: {e}") # Replace with proper logging + logger.info(f"Email sent successfully to", recipients) except Exception as e: # Optionally, log the error or take other actions - print(f"Failed to send email: {e}") # Replace with proper logging + logger.info(f"Failed to send email: {e}") return False # Indicate failure \ No newline at end of file From 307bbd61b1caf04202ef0a3f11eefef1e014c7c9 Mon Sep 17 00:00:00 2001 From: Yurii Purdenko Date: Wed, 27 Nov 2024 16:15:35 +0100 Subject: [PATCH 08/11] fix: rem --- bhtom2/bhtom_observations/facilities/rem.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bhtom2/bhtom_observations/facilities/rem.py b/bhtom2/bhtom_observations/facilities/rem.py index 7646760..be9245b 100644 --- a/bhtom2/bhtom_observations/facilities/rem.py +++ b/bhtom2/bhtom_observations/facilities/rem.py @@ -29,7 +29,7 @@ class REMPhotometricSequenceForm(BaseManualObservationForm): -# name = forms.CharField() + name = forms.CharField() proposal_id = forms.ChoiceField(label="Proposal ID", choices=proposal_choices) @@ -97,7 +97,7 @@ def layout(self): filter_rows = "".join(f"{filter_option}{self.exposure_times.get(filter_option)}" for filter_option, _ in valid_filters) mag = self.mag_init return Div( -# Div('name'), + Div('name'), Div('proposal_id'), Div( Div('start', css_class='col'), @@ -177,7 +177,6 @@ def all_data_products(self, observation_record): def submit_observation(self, observation_payload): #print(observation_payload) - logger.debug("submitted rem observations") # Retrieve target information using the target_id target_id = observation_payload['target_id'] try: @@ -189,7 +188,6 @@ def submit_observation(self, observation_payload): target_name = target.name.replace(" ", "_") # or use .replace(" ", "") ra = target.ra dec = target.dec - template = """ [STARTREMOB] From 3e3fc7c2ad7d3c081614e921d0c3d2112b445aef Mon Sep 17 00:00:00 2001 From: Yurii Purdenko Date: Thu, 28 Nov 2024 17:12:26 +0100 Subject: [PATCH 09/11] feat: #159, dataproduct --- Documentation/DocumentationAPI.md | 4 +- bhtom2/bhtom_calibration/views.py | 4 ++ bhtom2/bhtom_common/serializers.py | 82 +++++++++++++++--------------- bhtom2/bhtom_common/views.py | 27 ++++++---- 4 files changed, 65 insertions(+), 52 deletions(-) diff --git a/Documentation/DocumentationAPI.md b/Documentation/DocumentationAPI.md index 54e60d6..549066f 100644 --- a/Documentation/DocumentationAPI.md +++ b/Documentation/DocumentationAPI.md @@ -1302,8 +1302,10 @@ API for DataProduct list, lists uploaded instrumental or fits files and tracks t ```json { "data_product_type": "photometry", + "id": 1, "status": "Dataproduct status", - "fits_data": "test", + "fits_data": "fits file name", + "photometry_data": "photometry file name", "camera" : "BIALKOW_ANDOR-DW432", "created_start": "2024-01-01", "created_end": "2024-01-02", diff --git a/bhtom2/bhtom_calibration/views.py b/bhtom2/bhtom_calibration/views.py index a431942..266a325 100644 --- a/bhtom2/bhtom_calibration/views.py +++ b/bhtom2/bhtom_calibration/views.py @@ -75,6 +75,8 @@ def post(self, request): serialized_data = serializers.serialize('json', [instance]) data = json.loads(serialized_data)[0] result = data["fields"] + file_url = request.build_absolute_uri(settings.DATA_MEDIA_PATH + str(instance.dataproduct.photometry_data)) + result["file_download_link"] = file_url if getPlot: if instance.calibration_plot: plot_path = settings.DATA_PLOTS_PATH + str(instance.calibration_plot) @@ -101,6 +103,8 @@ def post(self, request): serialized_data = serializers.serialize('json', [instance]) data = json.loads(serialized_data)[0] result = data["fields"] + file_url = request.build_absolute_uri(settings.DATA_MEDIA_PATH + str(instance.dataproduct.photometry_data)) + result["file_download_link"] = file_url if getPlot: if instance.calibration_plot: plot_path = settings.DATA_PLOTS_PATH + str(instance.calibration_plot) diff --git a/bhtom2/bhtom_common/serializers.py b/bhtom2/bhtom_common/serializers.py index 014866b..968fc32 100644 --- a/bhtom2/bhtom_common/serializers.py +++ b/bhtom2/bhtom_common/serializers.py @@ -1,5 +1,6 @@ from rest_framework import serializers from bhtom_base.bhtom_dataproducts.models import DataProduct, CCDPhotJob +from bhtom2.bhtom_calibration.models import Calibration_data from bhtom_base.bhtom_targets.models import Target from django_comments.models import Comment @@ -11,61 +12,62 @@ class DataProductSerializer(serializers.ModelSerializer): camera = serializers.SerializerMethodField() observatory_name = serializers.SerializerMethodField() observatory = serializers.SerializerMethodField() - fits_filter = serializers.SerializerMethodField() + calibration_data = serializers.SerializerMethodField() class Meta: model = DataProduct fields = '__all__' def get_user_name(self, obj): - user_name = obj.user.first_name + " " + obj.user.last_name - return user_name - + return f"{obj.user.first_name} {obj.user.last_name}" + def get_user(self, obj): - user = obj.user.username - return user - + return obj.user.username + def get_camera(self, obj): - camera = None try: - camera = obj.observatory.camera.prefix - except Exception as e: - camera = None - return camera - + return obj.observatory.camera.prefix + except AttributeError: + return None + def get_target_name(self, obj): - target_name = obj.target.name if obj.target else None - return target_name - + return obj.target.name if obj.target else None + def get_target(self, obj): - target = obj.target.id if obj.target else None - return target - - + return obj.target.id if obj.target else None + def get_observatory_name(self, obj): - observatory_name = None try: - observatory_name = obj.observatory.camera.observatory.name - except Exception as e: - observatory_name = None - return observatory_name - + return obj.observatory.camera.observatory.name + except AttributeError: + return None + def get_observatory(self, obj): - observatory = None try: - observatory = obj.observatory.camera.observatory.id - except Exception as e: - observatory = None - return observatory - - def get_fits_filter(self, obj): - try: - ccdphotjob = CCDPhotJob.objects.get(dataProduct=obj.id) - fits_filter = ccdphotjob.fits_filter - except Exception as e: - fits_filter = None - return fits_filter - + return obj.observatory.camera.observatory.id + except AttributeError: + return None + + + def get_calibration_data(self, obj): + + calibration_data = Calibration_data.objects.filter(dataproduct=obj) + return [ + { + 'id': cal.id, + 'time_photometry': cal.modified, + 'mjd': cal.mjd, + 'calib_survey_filter': f"{cal.use_catalog.survey}/{cal.use_catalog.filters}", + 'standardised_to': f"{cal.survey}/{cal.best_filter}" if cal.survey and cal.best_filter else None, + 'magnitude': cal.mag, + 'zp': cal.zeropoint, + 'scatter': cal.scatter, + 'number of datapoints used for calibration': cal.npoints, + 'outlier fraction': cal.outlier_fraction, + 'matching radius[arcsec]': cal.match_distans + } + for cal in calibration_data + ] diff --git a/bhtom2/bhtom_common/views.py b/bhtom2/bhtom_common/views.py index fbd3820..87898fe 100644 --- a/bhtom2/bhtom_common/views.py +++ b/bhtom2/bhtom_common/views.py @@ -36,6 +36,7 @@ from bhtom2.bhtom_common.serializers import DataProductSerializer,CommentSerializer from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger + logger: BHTOMLogger = BHTOMLogger(__name__, 'Bhtom: bhtom_common.views') @@ -721,9 +722,11 @@ class GetDataProductApi(views.APIView): request_body=openapi.Schema( type=openapi.TYPE_OBJECT, properties={ + 'id': openapi.Schema(type=openapi.TYPE_INTEGER), 'data_product_type': openapi.Schema(type=openapi.TYPE_STRING), 'status': openapi.Schema(type=openapi.TYPE_STRING), 'fits_data': openapi.Schema(type=openapi.TYPE_STRING), + 'photometry_data': openapi.Schema(type=openapi.TYPE_STRING), 'camera': openapi.Schema(type=openapi.TYPE_STRING), 'created_start': openapi.Schema(type=openapi.TYPE_STRING, format=openapi.FORMAT_DATETIME), 'created_end': openapi.Schema(type=openapi.TYPE_STRING, format=openapi.FORMAT_DATETIME), @@ -751,24 +754,29 @@ class GetDataProductApi(views.APIView): ) def post(self, request): query = Q() - + id = request.data.get('id', None) data_product_type = request.data.get('data_product_type', None) - status = request.data.get('status', None) + dp_status = request.data.get('status', None) fits_data = request.data.get('fits_data', None) + photometry_data = request.data.get('photometry_data', None) camera = request.data.get('camera', None) created_start = request.data.get('created_start', None) created_end = request.data.get('created_end', None) mjd = request.data.get('mjd', None) page = request.data.get('page', 1) + if id is not None: + query &= Q(id=id) + if photometry_data is not None: + query &= Q(data__contains=photometry_data) if data_product_type is not None: query &= Q(data_product_type=data_product_type) - if status is not None: - query &= Q(status=status) + if dp_status is not None: + query &= Q(status=dp_status) if camera is not None: query &= Q(observatory__camera__prefix=camera) if fits_data is not None: - query &= Q(fits_data=fits_data) + query &= Q(fits_data__contains=fits_data) if created_start is not None: query &= Q(created__gte=created_start) if created_end is not None: @@ -778,10 +786,8 @@ def post(self, request): queryset = DataProduct.objects.filter(query).distinct().order_by('-created') - # Determine page size based on user role page_size = 500 if not request.user.is_staff else 1000 - - paginator = Paginator(queryset, page_size) # Set page size based on user role + paginator = Paginator(queryset, page_size) try: data_products = paginator.page(page) @@ -790,6 +796,7 @@ def post(self, request): except EmptyPage: data_products = paginator.page(paginator.num_pages) + # Serialize the data serialized_queryset = self.serializer_class(data_products, many=True).data return Response({ @@ -797,9 +804,7 @@ def post(self, request): 'num_pages': paginator.num_pages, 'current_page': data_products.number, 'data': serialized_queryset - }, status=200) - - + }, status= status.HTTP_200_OK) class NewsletterView(LoginRequiredMixin, TemplateView): From 81cdd2301890553258816dff1e9ccfcd3c441084 Mon Sep 17 00:00:00 2001 From: Lukasz Wyrzykowski Date: Thu, 28 Nov 2024 23:18:23 +0100 Subject: [PATCH 10/11] fix(catalogs): temp extended error messaging in CatalogQueryView --- bhtom2/bhtom_catalogs/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bhtom2/bhtom_catalogs/views.py b/bhtom2/bhtom_catalogs/views.py index 46aae71..4f8fbb5 100644 --- a/bhtom2/bhtom_catalogs/views.py +++ b/bhtom2/bhtom_catalogs/views.py @@ -51,7 +51,9 @@ def form_valid(self, form): return self.form_invalid(form) except Exception as e: # Handle other exceptions, log the error, etc. - form.add_error('term', ValidationError('Object not found')) +# form.add_error('term', ValidationError(f'Object not found')) + sanitized_message = f"Object not found: {str(e)}" + form.add_error('term', ValidationError(sanitized_message)) logger.error("Oops something went wrong: " + str(e)) return self.form_invalid(form) From 96b5264781026d298010a6ffca04f72596941248 Mon Sep 17 00:00:00 2001 From: Lukasz Wyrzykowski Date: Fri, 29 Nov 2024 00:01:42 +0100 Subject: [PATCH 11/11] fix(catalogs): fix to error message for catalog search failures --- bhtom2/bhtom_catalogs/views.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/bhtom2/bhtom_catalogs/views.py b/bhtom2/bhtom_catalogs/views.py index 4f8fbb5..42429fb 100644 --- a/bhtom2/bhtom_catalogs/views.py +++ b/bhtom2/bhtom_catalogs/views.py @@ -50,10 +50,7 @@ def form_valid(self, form): form.add_error('term', ValidationError('Object not found')) return self.form_invalid(form) except Exception as e: - # Handle other exceptions, log the error, etc. -# form.add_error('term', ValidationError(f'Object not found')) - sanitized_message = f"Object not found: {str(e)}" - form.add_error('term', ValidationError(sanitized_message)) + form.add_error('term', ValidationError("Error while searching for target")) logger.error("Oops something went wrong: " + str(e)) return self.form_invalid(form)