From 7071e73b27f16d4a620f0f4d2dc75afd2906fc58 Mon Sep 17 00:00:00 2001 From: glaubervila Date: Tue, 23 Jan 2024 13:16:52 -0300 Subject: [PATCH] Closed #794 A table, endpoint and interfaces were created to manage the asteroid pipeline executions. --- backend/coreAdmin/urls.py | 4 +- backend/tno/admin.py | 14 ++ backend/tno/migrations/0049_asteroidjob.py | 29 +++ backend/tno/models/__init__.py | 1 + backend/tno/models/asteroid_job.py | 70 ++++++ backend/tno/serializers.py | 8 +- backend/tno/views/__init__.py | 1 + backend/tno/views/asteroid_job.py | 12 + frontend/src/components/Drawer/index.js | 7 +- .../PredictionEventsDataGrid/Columns.js | 2 +- .../pages/AsteroidJob/AsteroidJobDetail.js | 108 +++++++++ .../pages/AsteroidJob/AsteroidJobHistory.js | 211 ++++++++++++++++++ frontend/src/pages/AsteroidJob/index.js | 22 ++ frontend/src/routes/index.js | 27 ++- frontend/src/services/api/AsteroidJob.js | 37 +++ 15 files changed, 542 insertions(+), 11 deletions(-) create mode 100644 backend/tno/migrations/0049_asteroidjob.py create mode 100644 backend/tno/models/asteroid_job.py create mode 100644 backend/tno/views/asteroid_job.py create mode 100644 frontend/src/pages/AsteroidJob/AsteroidJobDetail.js create mode 100644 frontend/src/pages/AsteroidJob/AsteroidJobHistory.js create mode 100644 frontend/src/pages/AsteroidJob/index.js create mode 100644 frontend/src/services/api/AsteroidJob.js diff --git a/backend/coreAdmin/urls.py b/backend/coreAdmin/urls.py index 188182342..251b99952 100644 --- a/backend/coreAdmin/urls.py +++ b/backend/coreAdmin/urls.py @@ -36,7 +36,7 @@ from rest_framework.authtoken.views import obtain_auth_token from rest_framework.routers import DefaultRouter from skybot.views import PositionViewSet -from tno.views import AsteroidViewSet, UserViewSet, OccultationViewSet, LeapSecondViewSet, BspPlanetaryViewSet, CatalogViewSet, PredictionJobViewSet, PredictionJobResultViewSet +from tno.views import AsteroidJobViewSet, AsteroidViewSet, UserViewSet, OccultationViewSet, LeapSecondViewSet, BspPlanetaryViewSet, CatalogViewSet, PredictionJobViewSet, PredictionJobResultViewSet router = DefaultRouter() router.register(r"users", UserViewSet) @@ -59,6 +59,8 @@ router.register(r"skybot/position", PositionViewSet) + +router.register(r"asteroid_jobs", AsteroidJobViewSet) router.register(r"asteroids", AsteroidViewSet) router.register(r"occultations", OccultationViewSet, basename="occultations") diff --git a/backend/tno/admin.py b/backend/tno/admin.py index cca01beb5..77856dbe9 100644 --- a/backend/tno/admin.py +++ b/backend/tno/admin.py @@ -1,6 +1,7 @@ from django.contrib import admin from tno.models import JohnstonArchive +from tno.models import AsteroidJob from tno.models import Asteroid from tno.models import BspPlanetary from tno.models import LeapSecond @@ -20,6 +21,19 @@ class ProfileAdmin(admin.ModelAdmin): raw_id_fields = ("user",) +@admin.register(AsteroidJob) +class AsteroidJobAdmin(admin.ModelAdmin): + list_display = ( + "id", + "status", + "submit_time", + "start", + "end", + "exec_time", + "asteroids_before", + "asteroids_after", + "error" + ) @admin.register(Asteroid) diff --git a/backend/tno/migrations/0049_asteroidjob.py b/backend/tno/migrations/0049_asteroidjob.py new file mode 100644 index 000000000..bce002dea --- /dev/null +++ b/backend/tno/migrations/0049_asteroidjob.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2.18 on 2024-01-23 13:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tno', '0048_remove_predictionjobresult_asteroid'), + ] + + operations = [ + migrations.CreateModel( + name='AsteroidJob', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('status', models.IntegerField(choices=[(1, 'Idle'), (2, 'Running'), (3, 'Completed'), (4, 'Failed'), (5, 'Aborted'), (6, 'Warning'), (7, 'Aborting')], default=1, verbose_name='Status')), + ('submit_time', models.DateTimeField(auto_now_add=True, verbose_name='Submit Time')), + ('start', models.DateTimeField(blank=True, null=True, verbose_name='Start')), + ('end', models.DateTimeField(blank=True, null=True, verbose_name='Finish')), + ('exec_time', models.DurationField(blank=True, null=True, verbose_name='Execution Time')), + ('asteroids_before', models.IntegerField(default=0, help_text='Total asteroids antes da execução', verbose_name='Asteroids Before')), + ('asteroids_after', models.IntegerField(default=0, help_text='Total asteroids após a execução', verbose_name='Asteroids After')), + ('path', models.CharField(blank=True, help_text='Path to the directory where the job data is located.', max_length=2048, null=True, verbose_name='Path')), + ('error', models.TextField(blank=True, null=True, verbose_name='Error')), + ('traceback', models.TextField(blank=True, null=True, verbose_name='Traceback')), + ], + ), + ] diff --git a/backend/tno/models/__init__.py b/backend/tno/models/__init__.py index cd22a4c48..feb7e711f 100644 --- a/backend/tno/models/__init__.py +++ b/backend/tno/models/__init__.py @@ -1,4 +1,5 @@ from .profile import Profile +from .asteroid_job import AsteroidJob from .asteroid import Asteroid from .occultation import Occultation from .leadp_seconds import LeapSecond diff --git a/backend/tno/models/asteroid_job.py b/backend/tno/models/asteroid_job.py new file mode 100644 index 000000000..d928ae951 --- /dev/null +++ b/backend/tno/models/asteroid_job.py @@ -0,0 +1,70 @@ + +from django.conf import settings +from django.db import models + +class AsteroidJob (models.Model): + + # Status da execução. + status = models.IntegerField( + verbose_name="Status", + default=1, + choices=( + (1, "Idle"), + (2, "Running"), + (3, "Completed"), + (4, "Failed"), + (5, "Aborted"), + (6, "Warning"), + (7, "Aborting"), + ), + ) + + # Momento em que o Job foi submetido. + submit_time = models.DateTimeField( + verbose_name="Submit Time", + auto_now_add=True, + ) + + # Momento em que o Job foi criado. + start = models.DateTimeField( + verbose_name="Start", auto_now_add=False, null=True, blank=True + ) + + # Momento em que o Job foi finalizado. + end = models.DateTimeField( + verbose_name="Finish", auto_now_add=False, null=True, blank=True + ) + + # Tempo de duração do Job. + exec_time = models.DurationField( + verbose_name="Execution Time", null=True, blank=True + ) + + asteroids_before = models.IntegerField( + verbose_name="Asteroids Before", + help_text="Total asteroids antes da execução", + default=0, + ) + + asteroids_after = models.IntegerField( + verbose_name="Asteroids After", + help_text="Total asteroids após a execução", + default=0, + ) + + # Pasta onde estão os dados do Job. + path = models.CharField( + verbose_name="Path", + help_text="Path to the directory where the job data is located.", + max_length=2048, + null=True, + blank=True + ) + + error = models.TextField(verbose_name="Error", null=True, blank=True) + + traceback = models.TextField( + verbose_name="Traceback", null=True, blank=True) + + def __str__(self): + return str(self.id) diff --git a/backend/tno/serializers.py b/backend/tno/serializers.py index bf05fdf40..2c1689c0c 100644 --- a/backend/tno/serializers.py +++ b/backend/tno/serializers.py @@ -3,11 +3,12 @@ from rest_framework import serializers from skybot.models.position import Position -from tno.models import (Asteroid, BspPlanetary, Catalog, JohnstonArchive, +from tno.models import (AsteroidJob, Asteroid, BspPlanetary, Catalog, JohnstonArchive, LeapSecond, Occultation, PredictionJob, PredictionJobResult, PredictionJobStatus, Profile) + class UserSerializer(serializers.ModelSerializer): dashboard = serializers.SerializerMethodField() @@ -73,6 +74,11 @@ class Meta: ) +class AsteroidJobSerializer(serializers.ModelSerializer): + class Meta: + model = AsteroidJob + fields = '__all__' + class AsteroidSerializer(serializers.ModelSerializer): class Meta: model = Asteroid diff --git a/backend/tno/views/__init__.py b/backend/tno/views/__init__.py index b7da05bde..0ec6e957c 100644 --- a/backend/tno/views/__init__.py +++ b/backend/tno/views/__init__.py @@ -1,4 +1,5 @@ from tno.views.user import UserViewSet +from tno.views.asteroid_job import AsteroidJobViewSet from tno.views.asteroid import AsteroidViewSet from tno.views.occultation import OccultationViewSet from tno.views.johnston_archive import JohnstonArchiveViewSet diff --git a/backend/tno/views/asteroid_job.py b/backend/tno/views/asteroid_job.py new file mode 100644 index 000000000..21d68fe0e --- /dev/null +++ b/backend/tno/views/asteroid_job.py @@ -0,0 +1,12 @@ +from rest_framework import viewsets +from tno.models import AsteroidJob +from tno.serializers import AsteroidJobSerializer + + +class AsteroidJobViewSet(viewsets.ReadOnlyModelViewSet): + + queryset = AsteroidJob.objects.select_related().all() + serializer_class = AsteroidJobSerializer + # filter_fields = ("id", "name", "number", "dynclass", "base_dynclass") + # search_fields = ("name", "number") + diff --git a/frontend/src/components/Drawer/index.js b/frontend/src/components/Drawer/index.js index 870d27473..c2b3d2e0c 100644 --- a/frontend/src/components/Drawer/index.js +++ b/frontend/src/components/Drawer/index.js @@ -81,6 +81,8 @@ const routes = [ { path: '/dashboard/stats', title: 'Dashboard' }, { path: '/dashboard/data-preparation/des/orbittrace-detail/:id', title: 'Orbit Trace Details' }, { path: '/dashboard/data-preparation/des/orbittrace/asteroid/:id', title: 'Orbit Trace Asteroid' }, + { path: '/dashboard/asteroid_job', title: 'Asteroid Job' }, + { path: '/dashboard/asteroid_job/:id', title: 'Asteroid Job Detail' }, ] const useCurrentPath = () => { @@ -199,7 +201,10 @@ export default function PersistentDrawerLeft({ children }) { navigate('/')}> - + + navigate('/dashboard/asteroid_job')}> + +
diff --git a/frontend/src/components/PredictionEventsDataGrid/Columns.js b/frontend/src/components/PredictionEventsDataGrid/Columns.js index 181e0bc49..c082407f9 100644 --- a/frontend/src/components/PredictionEventsDataGrid/Columns.js +++ b/frontend/src/components/PredictionEventsDataGrid/Columns.js @@ -1,5 +1,5 @@ import Box from '@mui/material/Box'; -import moment from '../../../node_modules/moment/moment' +import moment from 'moment'; function ImageCell(props) { if (props.value == null) { return ( diff --git a/frontend/src/pages/AsteroidJob/AsteroidJobDetail.js b/frontend/src/pages/AsteroidJob/AsteroidJobDetail.js new file mode 100644 index 000000000..99b6eed22 --- /dev/null +++ b/frontend/src/pages/AsteroidJob/AsteroidJobDetail.js @@ -0,0 +1,108 @@ +import Grid from '@mui/material/Grid' +import Card from '@mui/material/Card'; +import Box from '@mui/material/Box'; +import CardContent from '@mui/material/CardContent'; +import { useParams } from 'react-router-dom' +import TextField from '@mui/material/TextField'; +import { getAsteroidJobById } from '../../services/api/AsteroidJob'; +import { useQuery } from 'react-query'; +import Alert from '@mui/material/Alert'; +import moment from 'moment'; +import ColumnStatus from '../../components/Table/ColumnStatus'; +function AsteroidJob() { + const { id } = useParams(); + + const { data, isLoading } = useQuery({ + queryKey: ['asteroidJob', { id }], + queryFn: getAsteroidJobById, + keepPreviousData: false, + refetchInterval: false, + refetchOnWindowFocus: false, + refetchOnmount: false, + refetchOnReconnect: false, + staleTime: 1 * 60 * 60 * 1000, + }) + + console.log(data) + return ( + + {data !== undefined && data.status === 4 && ( + + {data?.error} + + )} + + :not(style)': { m: 1, width: '25ch' }, + }} + noValidate + autoComplete="off" + > + + + + + + + + + + + + + ) +} + +export default AsteroidJob \ No newline at end of file diff --git a/frontend/src/pages/AsteroidJob/AsteroidJobHistory.js b/frontend/src/pages/AsteroidJob/AsteroidJobHistory.js new file mode 100644 index 000000000..a43f9d66e --- /dev/null +++ b/frontend/src/pages/AsteroidJob/AsteroidJobHistory.js @@ -0,0 +1,211 @@ +import React, { useState } from 'react'; +import { useQuery } from 'react-query'; +import { DataGrid } from '@mui/x-data-grid'; +import { listAllAsteroidJobs } from '../../services/api/AsteroidJob'; +import CustomToolbar from '../../components/CustomDataGrid/Toolbar' +import CustomPagination from '../../components/CustomDataGrid/Pagination'; +import ColumnStatus from '../../components/Table/ColumnStatus'; +import moment from 'moment'; +import { useNavigate } from 'react-router-dom' +import Button from '@mui/material/Button' +import InfoOutlinedIcon from '@mui/icons-material/InfoOutlined'; + +export function PredictionEventsDataGrid() { +const navigate = useNavigate() + const columns = [ + { + field: 'id', + headerName: 'ID', + description: 'Internal ID', + headerAlign: 'center', + align: 'center', + width: 100, + renderCell: (params) => ( + + ), + + }, + { + field: 'status', + headerName: 'Status', + description: '', + headerAlign: 'center', + align: 'center', + width: 100, + renderCell: (params) => ( + + ), + + }, + { + field: 'submit_time', + headerName: 'Submit', + description: '', + width: 180, + type: 'dateTime', + valueGetter: ({ value }) => value && new Date(value), + valueFormatter: (params) => { + if (params.value == null) { + return ''; + } + return `${moment(params.value).utc().format('YYYY-MM-DD HH:mm:ss')}`; + }, + }, + { + field: 'start', + headerName: 'Start', + description: '', + width: 180, + type: 'dateTime', + valueGetter: ({ value }) => value && new Date(value), + valueFormatter: (params) => { + if (params.value == null) { + return ''; + } + return `${moment(params.value).utc().format('YYYY-MM-DD HH:mm:ss')}`; + }, + }, + { + field: 'end', + headerName: 'End', + description: '', + width: 180, + type: 'dateTime', + valueGetter: ({ value }) => value && new Date(value), + valueFormatter: (params) => { + if (params.value == null) { + return ''; + } + return `${moment(params.value).utc().format('YYYY-MM-DD HH:mm:ss')}`; + }, + }, + { + field: 'exec_time', + headerName: 'Exec Time', + description: '', + width: 80, + type: 'string', + }, + { + field: 'asteroids_before', + headerName: 'Before', + description: '', + type: 'number', + headerAlign: 'center', + align: 'center', + }, + { + field: 'asteroids_after', + headerName: 'After', + description: '', + type: 'number', + headerAlign: 'center', + align: 'center', + }, + { + field: 'error', + headerName: 'Error', + description: '', + type: 'string', + }, + ] + const columnVisibilityModel = { + id: true, + status: true, + submit_time: false, + start: true, + end: true, + exec_time: true, + asteroids_before: true, + asteroids_after: true, + error: true + } + + const [queryOptions, setQueryOptions] = useState({ + paginationModel: { page: 0, pageSize: 25 }, + selectionModel: [], + sortModel: [{ field: 'date_time', sort: 'asc' }], + filters: {} +}) + + const { paginationModel, sortModel, filters } = queryOptions + + const { data, isLoading } = useQuery({ + queryKey: ['asteroidJobs', { paginationModel, sortModel, filters }], + queryFn: listAllAsteroidJobs, + keepPreviousData: false, + refetchInterval: false, + refetchOnWindowFocus: false, + refetchOnmount: false, + refetchOnReconnect: false, + staleTime: 1 * 60 * 60 * 1000, + }) + // Some API clients return undefined while loading + // Following lines are here to prevent `rowCountState` from being undefined during the loading + const [rowCountState, setRowCountState] = React.useState( + data?.count || 0, + ); + + React.useEffect(() => { + setRowCountState((prevRowCountState) => + data?.count !== undefined + ? data?.count + : prevRowCountState, + ); + }, [data?.count, setRowCountState]); + + return ( + { + setQueryOptions(prev => { + return { + ...prev, + paginationModel: { ...paginationModel } + } + }) + }} + sortingMode="server" + onSortModelChange={(sortModel) => { + setQueryOptions(prev => { + return { + ...prev, + sortModel: [...sortModel] + } + }) + }} + initialState={{ + pagination: { + paginationModel: { + pageSize: 25, + }, + }, + sorting: { + sortModel: queryOptions.sortModel, + }, + columns: { + columnVisibilityModel: { ...columnVisibilityModel } + } + }} + slots={{ + pagination: CustomPagination, + toolbar: CustomToolbar + }} + /> + ); + +} + +export default PredictionEventsDataGrid diff --git a/frontend/src/pages/AsteroidJob/index.js b/frontend/src/pages/AsteroidJob/index.js new file mode 100644 index 000000000..166d9c770 --- /dev/null +++ b/frontend/src/pages/AsteroidJob/index.js @@ -0,0 +1,22 @@ +import Grid from '@mui/material/Grid' +import Card from '@mui/material/Card'; +import CardContent from '@mui/material/CardContent'; +import AsteroidJobHistory from './AsteroidJobHistory' + +function AsteroidJob() { + + + return ( + + + + + + + + + + ) +} + +export default AsteroidJob \ No newline at end of file diff --git a/frontend/src/routes/index.js b/frontend/src/routes/index.js index f11e66b61..cfe4861dd 100644 --- a/frontend/src/routes/index.js +++ b/frontend/src/routes/index.js @@ -31,6 +31,9 @@ import FooterSupporte from '../components/PublicPortal/Footer/FooterSupporters' import PredictionEvents from '../pages/PredictionEvents/index' import PredictionEventDetail from '../pages/PredictionEvents/Detail' +import AsteroidJob from '../pages/AsteroidJob/index' +import AsteroidJobDetail from '../pages/AsteroidJob/AsteroidJobDetail' + export default function AppRoutes() { const { isAuthenticated, signIn } = useAuth() @@ -294,16 +297,26 @@ export default function AppRoutes() { } /> - {/* - - + + + } - /> */} + /> + + + + } + /> ) } diff --git a/frontend/src/services/api/AsteroidJob.js b/frontend/src/services/api/AsteroidJob.js new file mode 100644 index 000000000..d252785eb --- /dev/null +++ b/frontend/src/services/api/AsteroidJob.js @@ -0,0 +1,37 @@ +import { api } from './Api' + +export const listAllAsteroidJobs = ({ queryKey }) => { + const [_, params] = queryKey + + const { paginationModel, filters, sortModel } = params + const { pageSize } = paginationModel + + // Fix Current page + let page = paginationModel.page + 1 + + // Parse Sort options + let sortFields = [] + if (sortModel !== undefined && sortModel.length > 0) { + sortModel.forEach((e) => { + if (e.sort === 'asc') { + sortFields.push(e.field); + } else { + sortFields.push(`-${e.field}`); + } + }); + } + let ordering = sortFields.length !== 0 ? sortFields.join(',') : null + + const newFilters = {} + if (filters !== undefined) {} + + return api.get( + `/asteroid_jobs/`, { params: { page, pageSize, ordering, ...newFilters } }) + .then((res) => res.data); +}; + +export const getAsteroidJobById = ({ queryKey }) => { + const [_, params] = queryKey + const {id} = params + return api.get(`/asteroid_jobs/${id}`).then((res) => res.data) +} \ No newline at end of file