Allow upserts to do nothing

py/specprodDB/load.py

@@ -1223,13 +1223,15 @@ def convert(cls, data, survey=None, program=None, tileid=None, night=None,
         return [cls(**(dict([(col.name, dat) for col, dat in zip(cls.__table__.columns, row)]))) for row in data_rows]
 
 
-def upsert(rows):
+def upsert(rows, do_nothing=False):
     """Convert a list of ORM objects into an ``INSERT ... ON CONFLICT`` statement.
 
     Parameters
     ----------
     rows : :class:`list`
         A list of ORM objects. All items should be the same type.
+    do_nothing : :class:`bool`, optional
+        If ``True``, *do not* attempt to update existing rows.
 
     Returns
     -------
@@ -1244,9 +1246,12 @@ def upsert(rows):
         del rr['_sa_instance_state']
         inserts.append(rr)
     stmt = pg_insert(cls).values(inserts)
-    stmt = stmt.on_conflict_do_update(index_elements=[getattr(cls, pk.name)],
-                                      set_=dict([(c, getattr(stmt.excluded, c.name))
-                                                 for c in cls.__table__.columns if c.name != pk.name]))
+    if do_nothing:
+        stmt = stmt.on_conflict_do_nothing(index_elements=[getattr(cls, pk.name)])
+    else:
+        stmt = stmt.on_conflict_do_update(index_elements=[getattr(cls, pk.name)],
+                                          set_=dict([(c, getattr(stmt.excluded, c.name))
+                                                     for c in cls.__table__.columns if c.name != pk.name]))
     return stmt
 
 

py/specprodDB/tile.py

@@ -73,8 +73,10 @@ def potential_targets(tileid):
         A table containing potential target information.
     """
     potential_targets_table = Table.read(fiberassign_file(tileid), format='fits', hdu='TARGETS')
+    db.log.debug("Found %d potential targets.", len(potential_targets_table))
     no_sky_rows = no_sky(potential_targets_table)
     potential_targets_table = Table(potential_targets_table[no_sky_rows])
+    db.log.debug("%d potential targets remain after removing sky targets.", len(potential_targets_table))
     return potential_targets_table
 
 
@@ -96,6 +98,7 @@ def potential_photometry(tile, targets):
     :class:`~astropy.table.Table`
         A Table that will be the input to photometric search functions.
     """
+    db.log.debug("Checking for existing photometry.")
     potential_tractorphot_already_loaded = db.dbSession.query(db.Photometry.targetid).filter(db.Photometry.targetid.in_(targets['TARGETID'].tolist())).all()
     potential_tractorphot_not_already_loaded = np.ones((len(targets),), dtype=bool)
     if len(potential_tractorphot_already_loaded) > 0:
@@ -126,7 +129,9 @@ def targetphot(catalog):
     :class:`~astropy.table.Table`
         A Table containing the targeting data.
     """
+    db.log.debug("Starting gather_targetphot(); %d objects in input catalog.", len(catalog))
     potential_targetphot = gather_targetphot(catalog, racolumn='TARGET_RA', deccolumn='TARGET_DEC')
+    db.log.debug("Finished with gather_targetphot(); %d objects found.", len(potential_targetphot))
     potential_targetphot['SURVEY'] = catalog['SURVEY']
     potential_targetphot['PROGRAM'] = catalog['PROGRAM']
     potential_targetphot['TILEID'] = catalog['TILEID']
@@ -151,7 +156,9 @@ def tractorphot(catalog):
     :class:`~astropy.table.Table`
         A Table containing the photometry data.
     """
+    db.log.debug("Starting gather_tractorphot(); %d objects in input catalog.", len(catalog))
     potential_tractorphot = gather_tractorphot(catalog, racolumn='TARGET_RA', deccolumn='TARGET_DEC')
+    db.log.debug("Finished with gather_tractorphot(); %d objects found.", len(potential_tractorphot))
     assert (np.where(potential_tractorphot['RELEASE'] == 0)[0] == np.where(potential_tractorphot['BRICKNAME'] == '')[0]).all()
     return potential_tractorphot