Merge pull request #97 from dr-rodriguez/switch-to-felis

Switch to using Felis schema for tests

schema/schema.yaml

@@ -216,7 +216,7 @@ tables:
       "@id": "#Sources.source"
       datatype: string
       length: 100
-      description: Unique identfier for an object
+      description: Unique identifier for an object
       ivoa:ucd: meta.id;src;meta.main
       nullable: false
     - name: ra_deg
@@ -245,7 +245,7 @@ tables:
       "@id": "#Sources.shortname"
       datatype: string
       length: 30
-      description: Short identfier for an object
+      description: Short identifier for an object
       ivoa:ucd: meta.id
     - name: reference
       "@id": "#Sources.reference"
@@ -306,7 +306,7 @@ tables:
       "@id": "#Names.source"
       datatype: string
       length: 100
-      description: Main identfier for an object; links to Sources table
+      description: Main identifier for an object; links to Sources table
       ivoa:ucd: meta.id;meta.main
       nullable: false
     - name: other_name
@@ -348,7 +348,7 @@ tables:
       "@id": "#Photometry.source"
       datatype: string
       length: 100
-      description: Main identfier for an object; links to Sources table
+      description: Main identifier for an object; links to Sources table
       ivoa:ucd: meta.id;meta.main
       nullable: false
     - name: band
@@ -456,7 +456,7 @@ tables:
       "@id": "#Parallaxes.source"
       datatype: string
       length: 100
-      description: Main identfier for an object; links to Sources table
+      description: Main identifier for an object; links to Sources table
       ivoa:ucd: meta.id;meta.main
       nullable: false
     - name: parallax_mas
@@ -526,7 +526,7 @@ tables:
       "@id": "#RadialVelocities.source"
       datatype: string
       length: 100
-      description: Main identfier for an object; links to Sources table
+      description: Main identifier for an object; links to Sources table
       ivoa:ucd: meta.id;meta.main
       nullable: false
     - name: radial_velocity_kms

scripts/make_erd.py

@@ -2,17 +2,20 @@
 
 import sys
 
-from astrodbkit.astrodb import Database, create_database
+import yaml
 from eralchemy2 import render_er
+from felis.datamodel import Schema
+from felis.metadata import MetaDataBuilder
 
 sys.path.append("./")  # needed for github actions to find the template module
-from schema.schema_template import *
-from schema.schema_template import REFERENCE_TABLES
 
-# Connect to an in-memory sqlite database
-create_database(connection_string="sqlite://")
-db = Database("sqlite://", reference_tables=REFERENCE_TABLES)
+# Load up schema
+data = yaml.safe_load(open("schema/schema.yaml", "r"))
+schema = Schema.model_validate(data)
+
+# Create from Felis schema
+metadata = MetaDataBuilder(schema).build()
 
 # Create ER model from the database metadata
 filename = "schema/schema.png"
-render_er(db.metadata, filename)
+render_er(metadata, filename)

tests/conftest.py

@@ -1,42 +1,36 @@
-import pytest
 import os
-import logging
-from astrodbkit.astrodb import create_database, Database
 import sys
 
+import pytest
+from astrodbkit.astrodb import Database, create_database
+
 sys.path.append("./")  # needed for github actions to find the template module
 from schema.schema_template import REFERENCE_TABLES
-from schema.schema_template import *
+
+DB_PATH = "data"
+DB_NAME = "tests/astrodb_template_tests.sqlite"
+SCHEMA_PATH = "schema/schema.yaml"
+CONNECTION_STRING = "sqlite:///" + DB_NAME
 
 
 # Create a fresh template database for the data and integrity tests
-@pytest.fixture(scope="session", autouse=False)
+@pytest.fixture(scope="session", autouse=True)
 def db():
-    DB_NAME = "tests/astrodb_template_tests.sqlite"
-    DB_PATH = "data"
+
+    # Confirm the schema yaml file is present
+    assert os.path.exists(SCHEMA_PATH)
 
     # Remove any existing copy of the test database
     if os.path.exists(DB_NAME):
         os.remove(DB_NAME)
+        assert not os.path.exists(DB_NAME)
 
-    connection_string = "sqlite:///" + DB_NAME
-    create_database(connection_string)
+    # Create the database using the Felis schema
+    create_database(CONNECTION_STRING, felis_schema=SCHEMA_PATH)
     assert os.path.exists(DB_NAME)
 
-    # Connect to the new database
-    db = Database(connection_string, reference_tables=REFERENCE_TABLES)
-
-    # The input data is NOT correct; that needs to be fixed or this commented out
-    # Load data into an in-memory sqlite database first, for performance
-    db = Database(
-        "sqlite://", reference_tables=REFERENCE_TABLES
-    )  # creates and connects to a temporary in-memory database
-    db.load_database(
-        DB_PATH, verbose=False
-    )  # loads the data from the data files into the database
-    db.dump_sqlite(DB_NAME)  # dump in-memory database to file
-    db = Database(
-        "sqlite:///" + DB_NAME, reference_tables=REFERENCE_TABLES
-    )  # replace database object with new file version
+    # Connect and load to the database
+    db = Database(CONNECTION_STRING, reference_tables=REFERENCE_TABLES)
+    db.load_database(DB_PATH, verbose=False)
 
     return db

tests/test_database.py

@@ -2,19 +2,9 @@
 Functions to test the database and example files
 """
 
-from schema.schema_template import (
-    Instruments,
-    Names,
-    Photometry,
-    PhotometryFilters,
-    Publications,
-    Sources,
-    Telescopes,
-    Versions,
-    Regimes
-)
 from astrodbkit.astrodb import or_
-import numpy as np
+from sqlalchemy.ext.automap import automap_base
+
 
 def test_setup_db(db):
     # Some setup tasks to ensure some data exists in the database first
@@ -40,19 +30,47 @@ def test_setup_db(db):
         conn.commit()
 
 
+def test_table_presence(db):
+    # Confirm the tables that should be present
+
+    assert len(db.metadata.tables.keys()) == 11
+    assert "Sources" in db.metadata.tables.keys()
+    assert "Publications" in db.metadata.tables.keys()
+    assert "Names" in db.metadata.tables.keys()
+    assert "Telescopes" in db.metadata.tables.keys()
+    assert "Instruments" in db.metadata.tables.keys()
+    assert "PhotometryFilters" in db.metadata.tables.keys()
+    assert "Versions" in db.metadata.tables.keys()
+    assert "Parallaxes" in db.metadata.tables.keys()
+    assert "RadialVelocities" in db.metadata.tables.keys()
+    assert "Photometry" in db.metadata.tables.keys()
+    assert "Regimes" in db.metadata.tables.keys()
+
+
 def test_orm_use(db):
     # Tests validation using the SQLAlchemy ORM
 
+    Base = automap_base(metadata=db.metadata)
+    Base.prepare()
+
+    # Creating the actual Table objects
+    Sources = Base.classes.Sources
+    Names = Base.classes.Names
+
     # Adding and removing a basic source
     s = Sources(source="V4046 Sgr", ra_deg=273.54, dec_deg=-32.79, reference="Ref 1")
+    n = Names(source="V4046 Sgr", other_name="Hen 3-1636")
     with db.session as session:
         session.add(s)
+        session.add(n)
         session.commit()
 
     assert db.query(db.Sources).filter(db.Sources.c.source == "V4046 Sgr").count() == 1
+    assert db.query(db.Names).filter(db.Names.c.other_name == "Hen 3-1636").count() == 1
 
     # Remove added source so other tests don't include it
     with db.session as session:
+        session.delete(n)  # delete Names before Sources
         session.delete(s)
         session.commit()
 
@@ -64,6 +82,17 @@ def test_photometry(db):
     # Confirm the source isn't already present
     assert db.query(db.Sources).filter(db.Sources.c.source == "Fake V4046 Sgr").count() == 0
 
+    Base = automap_base(metadata=db.metadata)
+    Base.prepare()
+
+    # Creating the actual Table objects
+    Sources = Base.classes.Sources
+    Publications = Base.classes.Publications
+    Telescopes = Base.classes.Telescopes
+    Photometry = Base.classes.Photometry
+    PhotometryFilters = Base.classes.PhotometryFilters
+    Regimes = Base.classes.Regimes
+
     # Insert supporting data to (Sources, Publications, Telescopes, PhotometryFilters)
     s = Sources(source="V4046 Sgr", ra_deg=273.54, dec_deg=-32.79, reference="Ref 1")
     ref = Publications(reference="Cutri03")
@@ -134,10 +163,10 @@ def test_magnitudes(db):
 def test_parallax_error(db):
     # Verify that all sources have valid parallax errors
     t = (
-        db.query(db.Parallax.c.parallax_error)
+        db.query(db.Parallaxes.c.parallax_error)
         .filter(
             or_(
-                db.Parallax.c.parallax_error < 0,
+                db.Parallaxes.c.parallax_error < 0,
 
               )
         )
@@ -173,22 +202,3 @@ def test_coordinates(db):
         print(t)
 
     assert len(t) == 0, f"{len(t)} Sources failed coordinate checks"
-
-
-def test_sig_figs_parallax(db):
-    # verify that the precision on parallax isn't greater than the error's precision
-    t = (
-        db.query(db.Parallax.c.parallax_mas, db.Parallax.c.parallax_error)
-        .astropy()
-    )
-    # create empty table to add the results to
-
-
-    wrong_sig_figs = []
-    for i in t:
-        parallax_sig_figs = count_significant_digits(i['parallax_mas'])
-        error_sig_figs = count_significant_digits(i['parallax_error'])
-
-        if error_sig_figs >= parallax_sig_figs:
-            wrong_sig_figs.append(i)
-    assert len(wrong_sig_figs)==0, f"Parallax error has fewer significant figures than parallax for these sources: {wrong_sig_figs}"