Skip to content

Commit

Permalink
Merge pull request #417 from linea-it/fix_filter_by_name
Browse files Browse the repository at this point in the history 
Fixed Filter objects query and interface
  • Loading branch information
@matheusallein
matheusallein authored Feb 20, 2020
2 parents 58a0f07 + 02ce8d6 commit 445a60c
Show file tree
Hide file tree
Showing 3 changed files with 128 additions and 104 deletions.
127 changes: 67 additions & 60 deletions core_admin/tno/skybotoutput.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,6 @@
import logging




class FilterObjects(DBBase):
"""
Expand All @@ -31,32 +29,8 @@ def get_base_stm(self):
cols = self.table.c

stm = select([
cols.id,
cols.dynclass.label("object_table"),
cols.name,
cols.num,
cols.ra,
cols.dec,
cols.raj2000,
cols.decj2000,
cols.d,
cols.dracosdec,
cols.ddec,
cols.dgeo,
cols.dhelio,
cols.phase,
cols.solelong,
cols.px,
cols.py,
cols.pz,
cols.vx,
cols.vy,
cols.vz,
cols.externallink,
cols.expnum,
cols.ccdnum,
cols.band,
cols.pointing_id,
func.count(cols.name).label("freq"),
null().label('filters'),
func.min(cols.mv).label('mag_min'),
Expand All @@ -73,7 +47,6 @@ def get_base_stm(self):
def get_objects_stm(self,
name=None, objectTable=None, magnitude=None, diffDateNights=None,
moreFilter=None, page=1, pageSize=100):

"""Applies the filters to the skybot output table and returns the list
of objects that meet the requirements.
Expand Down Expand Up @@ -116,13 +89,14 @@ def get_objects_stm(self,
if diffDateNights:
# Exemplo query com where por diffDateNights
# select name, min(mv), max(jdref) - min(jdref) as DiffDateNights from tno.skybot_output where dynclass like 'Centaur' group by name HAVING max(jdref) - min(jdref) > 1000 limit 10 ;
stm = stm.having((func.max(cols.jdref) - func.min(cols.jdref)) > float(diffDateNights))
stm = stm.having(
(func.max(cols.jdref) - func.min(cols.jdref)) > float(diffDateNights))

if len(terms):
stm = stm.where(and_(*terms))

# Agrupamento
stm = stm.group_by(cols.id, cols.name, cols.dynclass)
stm = stm.group_by(cols.name, cols.dynclass)

# Ordenacao
stm = stm.order_by(cols.name)
Expand All @@ -139,7 +113,6 @@ def get_objects_stm(self,
def get_objects(self,
name=None, objectTable=None, magnitude=None, diffDateNights=None,
moreFilter=None, page=1, pageSize=None):

"""Applies the filters to the skybot output table and returns the list
of objects that meet the requirements.
Expand All @@ -163,9 +136,8 @@ def get_objects(self,
return rows, totalSize

def get_objects_count(self,
name=None, objectTable=None, magnitude=None, diffDateNights=None,
moreFilter=None,):

name=None, objectTable=None, magnitude=None, diffDateNights=None,
moreFilter=None,):

stm = self.get_objects_stm(
name, objectTable, magnitude, diffDateNights, moreFilter)
Expand All @@ -174,6 +146,24 @@ def get_objects_count(self,

return totalSize

def get_observations_count(self,
name=None, objectTable=None, magnitude=None, diffDateNights=None,
moreFilter=None,):
# Para esta query em especifico dependendo do filtro que é utilizado, não é possivel contar
# a quantidade de observações. isso pq alguns filtros são aplicados em colunas geradas depois do agrupamento.
try:
stm = self.get_objects_stm(
name, objectTable, magnitude, diffDateNights, moreFilter)

stm = stm.group_by(None)
totalSize = self.stm_count(stm)

return totalSize
except:
# TODO: Para saber o total de observações quando o filtro diffDateNights está presente
# é preciso executar a query primeiro e fazer um Sum na quantidade de CCD Nums
return None

def create_object_list(self,
tablename, name, objectTable,
magnitude, diffDateNights, moreFilter):
Expand Down Expand Up @@ -252,9 +242,11 @@ def list_objects_by_table(self, tablename, schema=None, page=1, pageSize=100):
tbl = self.get_table(tablename, schema).alias('a')
tbl_ccd = self.get_table_ccdimage().alias('b')

stm_join = tbl.join(tbl_ccd, tbl.c.pointing_id == tbl_ccd.c.pointing_id, isouter=True)
stm_join = tbl.join(tbl_ccd, tbl.c.pointing_id ==
tbl_ccd.c.pointing_id, isouter=True)

stm = select([tbl, tbl_ccd.c.filename, tbl_ccd.c.file_size]).select_from(stm_join)
stm = select([tbl, tbl_ccd.c.filename, tbl_ccd.c.file_size]
).select_from(stm_join)

# Ordenacao
stm = stm.order_by(tbl.c.name)
Expand All @@ -281,7 +273,8 @@ def list_distinct_objects_by_table(self, tablename, schema=None, page=1, pageSiz
tbl = self.get_table(tablename, schema).alias('a')
tbl_ccd = self.get_table_ccdimage().alias('b')

stm_join = tbl.join(tbl_ccd, tbl.c.pointing_id == tbl_ccd.c.pointing_id, isouter=True)
stm_join = tbl.join(tbl_ccd, tbl.c.pointing_id ==
tbl_ccd.c.pointing_id, isouter=True)

stm = select([tbl.c.name, tbl.c.num]).select_from(stm_join)

Expand Down Expand Up @@ -310,9 +303,11 @@ def count_distinct_objects(self, tablename, schema=None):
tbl = self.get_table(tablename, schema).alias('a')
tbl_ccd = self.get_table_ccdimage().alias('b')

stm_join = tbl.join(tbl_ccd, tbl.c.pointing_id == tbl_ccd.c.pointing_id, isouter=True)
stm_join = tbl.join(tbl_ccd, tbl.c.pointing_id ==
tbl_ccd.c.pointing_id, isouter=True)

stm = select([func.count(func.distinct(tbl.c.name))]).select_from(stm_join)
stm = select([func.count(func.distinct(tbl.c.name))]
).select_from(stm_join)

distinct_objects = self.fetch_scalar(stm)

Expand All @@ -323,9 +318,11 @@ def count_distinct_pointing(self, tablename, schema=None):
tbl = self.get_table(tablename, schema).alias('a')
tbl_ccd = self.get_table_ccdimage().alias('b')

stm_join = tbl.join(tbl_ccd, tbl.c.pointing_id == tbl_ccd.c.pointing_id, isouter=True)
stm_join = tbl.join(tbl_ccd, tbl.c.pointing_id ==
tbl_ccd.c.pointing_id, isouter=True)

stm = select([func.count(func.distinct(tbl_ccd.c.filename))]).select_from(stm_join)
stm = select([func.count(func.distinct(tbl_ccd.c.filename))]
).select_from(stm_join)

distinct_pointings = self.fetch_scalar(stm)

Expand Down Expand Up @@ -354,11 +351,13 @@ def count_pointing_not_downloaded(self, tablename, schema=None):
tbl = self.get_table(tablename, schema).alias('a')
tbl_ccd = self.get_table_ccdimage().alias('b')

stm_join = tbl.join(tbl_ccd, tbl.c.pointing_id == tbl_ccd.c.pointing_id, isouter=True)
stm_join = tbl.join(tbl_ccd, tbl.c.pointing_id ==
tbl_ccd.c.pointing_id, isouter=True)

stm = select([func.count(tbl.c.name)]).select_from(stm_join)

stm = stm.where(and_(tbl.c.pointing_id.isnot(None), tbl_ccd.c.filename.is_(None)))
stm = stm.where(and_(tbl.c.pointing_id.isnot(
None), tbl_ccd.c.filename.is_(None)))

not_downloaded = self.fetch_scalar(stm)

Expand All @@ -369,7 +368,8 @@ def list_ccdimage_by_table(self, tablename, schema=None, page=1, pageSize=None):
tbl = self.get_table(tablename, schema).alias('a')
tbl_ccd = self.get_table_ccdimage().alias('b')

stm_join = tbl.join(tbl_ccd, tbl.c.pointing_id == tbl_ccd.c.pointing_id, isouter=True)
stm_join = tbl.join(tbl_ccd, tbl.c.pointing_id ==
tbl_ccd.c.pointing_id, isouter=True)

stm = select(tbl_ccd.c).select_from(stm_join)

Expand Down Expand Up @@ -400,7 +400,8 @@ def list_pointing_path_by_table(self, tablename, schema, page=1, pageSize=None):
tbl = self.get_table(tablename, schema).alias('a')
tbl_pointing = self.get_table_pointing().alias('b')

stm_join = tbl.join(tbl_pointing, tbl.c.pointing_id == tbl_pointing.c.id, isouter=True)
stm_join = tbl.join(tbl_pointing, tbl.c.pointing_id ==
tbl_pointing.c.id, isouter=True)

stm = select([
tbl_pointing.c.id,
Expand Down Expand Up @@ -429,14 +430,13 @@ def list_pointing_path_by_table(self, tablename, schema, page=1, pageSize=None):

return rows, totalSize



def count_ccdimage_size(self, tablename, schema=None):

tbl = self.get_table(tablename, schema).alias('a')
tbl_ccd = self.get_table_ccdimage().alias('b')

stm_join = tbl.join(tbl_ccd, tbl.c.pointing_id == tbl_ccd.c.pointing_id, isouter=True)
stm_join = tbl.join(tbl_ccd, tbl.c.pointing_id ==
tbl_ccd.c.pointing_id, isouter=True)

stm = select([func.sum(tbl_ccd.c.file_size)]).select_from(stm_join)

Expand All @@ -451,7 +451,8 @@ def ccd_images_by_object(self, name):
tbl = self.get_table_pointing()
tbl_skybot = self.get_table_skybot().alias('b')

stm_join = tbl.join(tbl_skybot, tbl.c.id == tbl_skybot.c.pointing_id, isouter=True)
stm_join = tbl.join(tbl_skybot, tbl.c.id ==
tbl_skybot.c.pointing_id, isouter=True)

stm = select(tbl.c).select_from(stm_join)

Expand All @@ -463,7 +464,6 @@ def ccd_images_by_object(self, name):

return rows, totalSize


def check_bsp_jpl_by_object(self, name, max_age):
"""
Retorna o registro do bsl jpl para o objeto,
Expand All @@ -477,7 +477,7 @@ def check_bsp_jpl_by_object(self, name, max_age):
and_(
tbl.c.name == name,
tbl.c.download_finish_time > (
func.now() - text('INTERVAL \'%s DAY\'' % int(max_age)))
func.now() - text('INTERVAL \'%s DAY\'' % int(max_age)))
)
)

Expand All @@ -495,12 +495,14 @@ def count_lines(self):
return self.get_count(self.tbl)

def count_unique_ccds(self):
stm = select([func.count(func.distinct(self.tbl.c.expnum, self.tbl.c.ccdnum)).label('unique_ccds')])
stm = select([func.count(func.distinct(self.tbl.c.expnum,
self.tbl.c.ccdnum)).label('unique_ccds')])

return self.fetch_scalar(stm)

def count_asteroids(self):
stm = select([func.count(func.distinct(self.tbl.c.name)).label('asteroids')])
stm = select(
[func.count(func.distinct(self.tbl.c.name)).label('asteroids')])

return self.fetch_scalar(stm)

Expand All @@ -510,7 +512,8 @@ def distinct_dynclass(self):
return self.fetch_all_dict(stm)

def count_asteroids_by_dynclass(self):
stm = select([self.tbl.c.dynclass, func.count(self.tbl.c.name).label('count')]).group_by(self.tbl.c.dynclass)
stm = select([self.tbl.c.dynclass, func.count(
self.tbl.c.name).label('count')]).group_by(self.tbl.c.dynclass)

return self.fetch_all_dict(stm)

Expand Down Expand Up @@ -538,7 +541,8 @@ def histogram(self, column, bin):
"""

stm = select([
(func.floor(self.tbl.c[column] / bin) * bin).label('bin'), func.count('*').label('count')
(func.floor(self.tbl.c[column] / bin) *
bin).label('bin'), func.count('*').label('count')
]).group_by('1')

return self.fetch_all_dict(stm)
Expand Down Expand Up @@ -572,7 +576,8 @@ def counts_by_bands(self):

results = list()
for band in bands:
results.append(dict({'name': band, 'band': self.count_by_band(band)}))
results.append(
dict({'name': band, 'band': self.count_by_band(band)}))

return results

Expand All @@ -589,12 +594,14 @@ def first(self):
# Count por faixa das exposições
def count_range_exposures(self, start, end):

stm = select([func.count()]).where(between(self.tbl.c.exptime, int(start.strip()), int(end.strip())))
stm = select([func.count()]).where(
between(self.tbl.c.exptime, int(start.strip()), int(end.strip())))

return self.fetch_scalar(stm)

def count_unique_exposures(self):
stm = select([func.count(func.distinct(self.tbl.c.expnum)).label('exposures')])
stm = select(
[func.count(func.distinct(self.tbl.c.expnum)).label('exposures')])

return self.fetch_scalar(stm)

Expand All @@ -611,7 +618,6 @@ def count_unique_exposures_by_period(self, date_initial=None, end_time=None):

return self.fetch_scalar(stm)


# Exposições
def unique_exposures(self):

Expand All @@ -631,7 +637,7 @@ def unique_exposures_by_period(self, date_initial=None, end_time=None):
stm = select([
func.count(func.distinct(self.tbl.c.expnum)).label('exposures'),
cast(self.tbl.c.date_obs, DATE).label('date_obs'),
]).where(and_(
]).where(and_(
cast(self.tbl.c.date_obs, DATE) >= date_initial.strftime("%Y-%m-%d"),
cast(self.tbl.c.date_obs, DATE) <= end_time.strftime("%Y-%m-%d")
)).group_by(cast(self.tbl.c.date_obs, DATE))
Expand All @@ -646,5 +652,6 @@ def counts_range_exposures(self):
for exptime in exptimes:
start, end = exptime.split(',')

results.append(dict({'name' : exptime, 'exposure' : self.count_range_exposures(start, end)}))
results.append(
dict({'name': exptime, 'exposure': self.count_range_exposures(start, end)}))
return results
Loading

0 comments on commit 445a60c

Please sign in to comment.