Small changes

backend/ibutsu_server/db/models.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 from uuid import uuid4
 
 from ibutsu_server.db.base import Boolean
@@ -65,39 +66,39 @@ def to_dict(self):
 
 class Artifact(Model, FileMixin):
  __tablename__ = "artifacts"
- result_id = Column(PortableUUID(), ForeignKey("results.id"), nullable=False)
- filename = Column(Text)
+ result_id = Column(PortableUUID(), ForeignKey("results.id"), nullable=False, index=True)
+ filename = Column(Text, index=True)
  data = Column(mutable_json_type(dbtype=PortableJSON(), nested=True))
- upload_date = Column(DateTime)
+ upload_date = Column(DateTime, default=datetime.utcnow, index=True)
 
 
 class Group(Model, ModelMixin):
  __tablename__ = "groups"
- name = Column(Text)
+ name = Column(Text, index=True)
  projects = relationship("Project")
  data = Column(mutable_json_type(dbtype=PortableJSON(), nested=True))
 
 
 class Import(Model, ModelMixin):
  __tablename__ = "imports"
  file = relationship("ImportFile")
- filename = Column(Text)
- format = Column(Text)
+ filename = Column(Text, index=True)
+ format = Column(Text, index=True)
  data = Column(mutable_json_type(dbtype=PortableJSON(), nested=True))
- status = Column(Text)
+ status = Column(Text, index=True)
 
 
 class ImportFile(Model, FileMixin):
  __tablename__ = "import_files"
- import_id = Column(PortableUUID(), ForeignKey("imports.id"), nullable=False)
+ import_id = Column(PortableUUID(), ForeignKey("imports.id"), nullable=False, index=True)
 
 
 class Project(Model, ModelMixin):
  __tablename__ = "projects"
- name = Column(Text)
- title = Column(Text)
- owner_id = Column(Text)
- group_id = Column(PortableUUID(), ForeignKey("groups.id"))
+ name = Column(Text, index=True)
+ title = Column(Text, index=True)
+ owner_id = Column(Text, index=True)
+ group_id = Column(PortableUUID(), ForeignKey("groups.id"), index=True)
  reports = relationship("Report")
  results = relationship("Result")
  runs = relationship("Run")
@@ -106,64 +107,64 @@ class Project(Model, ModelMixin):
 
 class Report(Model, ModelMixin):
  __tablename__ = "reports"
- created = Column(DateTime)
- download_url = Column(Text)
- filename = Column(Text)
- mimetype = Column(Text)
- name = Column(Text)
+ created = Column(DateTime, default=datetime.utcnow, index=True)
+ download_url = Column(Text, index=True)
+ filename = Column(Text, index=True)
+ mimetype = Column(Text, index=True)
+ name = Column(Text, index=True)
  params = Column(mutable_json_type(dbtype=PortableJSON()))
- project_id = Column(PortableUUID(), ForeignKey("projects.id"))
+ project_id = Column(PortableUUID(), ForeignKey("projects.id"), index=True)
  file = relationship("ReportFile")
- status = Column(Text)
- url = Column(Text)
- view_url = Column(Text)
+ status = Column(Text, index=True)
+ url = Column(Text, index=True)
+ view_url = Column(Text, index=True)
 
 
 class ReportFile(Model, FileMixin):
  __tablename__ = "report_files"
- report_id = Column(PortableUUID(), ForeignKey("reports.id"), nullable=False)
- filename = Column(Text)
+ report_id = Column(PortableUUID(), ForeignKey("reports.id"), nullable=False, index=True)
+ filename = Column(Text, index=True)
  data = Column(mutable_json_type(dbtype=PortableJSON(), nested=True))
 
 
 class Result(Model, ModelMixin):
  __tablename__ = "results"
  artifacts = relationship("Artifact")
- component = Column(Text)
+ component = Column(Text, index=True)
  # this is metadata but it is a reserved attr
  data = Column(mutable_json_type(dbtype=PortableJSON(), nested=True))
- duration = Column(Float)
- env = Column(Text)
+ duration = Column(Float, index=True)
+ env = Column(Text, index=True)
  params = Column(mutable_json_type(dbtype=PortableJSON()))
- project_id = Column(PortableUUID(), ForeignKey("projects.id"))
- result = Column(Text)
- run_id = Column(PortableUUID(), ForeignKey("runs.id"))
- source = Column(Text)
- start_time = Column(DateTime)
- test_id = Column(Text)
+ project_id = Column(PortableUUID(), ForeignKey("projects.id"), index=True)
+ result = Column(Text, index=True)
+ run_id = Column(PortableUUID(), ForeignKey("runs.id"), index=True)
+ source = Column(Text, index=True)
+ start_time = Column(DateTime, default=datetime.utcnow, index=True)
+ test_id = Column(Text, index=True)
 
 
 class Run(Model, ModelMixin):
  __tablename__ = "runs"
- component = Column(Text)
- created = Column(DateTime)
+ component = Column(Text, index=True)
+ created = Column(DateTime, default=datetime.utcnow, index=True)
  # this is metadata but it is a reserved attr
  data = Column(mutable_json_type(dbtype=PortableJSON(), nested=True))
- duration = Column(Float)
- env = Column(Text)
- project_id = Column(PortableUUID(), ForeignKey("projects.id"))
+ duration = Column(Float, index=True)
+ env = Column(Text, index=True)
+ project_id = Column(PortableUUID(), ForeignKey("projects.id"), index=True)
  results = relationship("Result")
- source = Column(Text)
- start_time = Column(DateTime)
+ source = Column(Text, index=True)
+ start_time = Column(DateTime, default=datetime.utcnow, index=True)
  summary = Column(mutable_json_type(dbtype=PortableJSON()))
 
 
 class WidgetConfig(Model, ModelMixin):
  __tablename__ = "widget_configs"
- navigable = Column(Boolean)
+ navigable = Column(Boolean, index=True)
  params = Column(mutable_json_type(dbtype=PortableJSON()))
- project_id = Column(PortableUUID(), ForeignKey("projects.id"))
- title = Column(Text)
- type = Column(Text)
- weight = Column(Integer)
- widget = Column(Text)
+ project_id = Column(PortableUUID(), ForeignKey("projects.id"), index=True)
+ title = Column(Text, index=True)
+ type = Column(Text, index=True)
+ weight = Column(Integer, index=True)
+ widget = Column(Text, index=True)

backend/ibutsu_server/scripts/mongo2postgres.py

@@ -40,7 +40,7 @@
 TABLE_MAP = [
  ("groups", Group),
  ("projects", Project),
- ("widgetConfigs", WidgetConfig),
+ ("widgetConfig", WidgetConfig),
  ("runs", Run),
  ("results", Result),
 ]
@@ -63,27 +63,15 @@
  "project",
  "run",
 ]
+# fields that need to be typecast
+FIELDS_TO_TYPECAST = ["navigable", "weight"]
 
-# indexes for the tables
+# json indexes for the tables
 INDEXES = {
- "results": [
- "CREATE INDEX results_env ON results(env);",
- "CREATE INDEX results_component ON results(component);",
- "CREATE INDEX results_project ON results(project_id);",
- "CREATE INDEX results_start_time ON results(start_time);",
- "CREATE INDEX results_run ON results(run_id);",
- ],
- "runs": [
- "CREATE INDEX runs_env ON runs(env);",
- "CREATE INDEX runs_component ON runs(component);",
- "CREATE INDEX runs_project ON runs(project_id);",
- "CREATE INDEX runs_start_time ON runs(start_time);",
- ],
- "widget_configs": ["CREATE INDEX widget_configs_project ON widget_configs(project_id);"],
- "artifacts": [
- "CREATE INDEX artifact_result ON artifacts(result_id);",
- "CREATE INDEX artifact_upload_date ON artifacts(upload_date);",
- ],
+ # "results": [
+ # ],
+ # "runs": [
+ # ],
 }
 
 
@@ -144,10 +132,10 @@ def migrate_table(collection, Model, vprint):
  conn.execute(sql_index)
 
  # for runs and results, sort by descending start_time
- if Model.__tablename__ == "runs" or Model.__tablename__ == "results":
+ if Model.__tablename__ in ["runs", "results"]:
  sort = [("start_time", -1)]
  most_recent_record = collection.find_one(sort=sort)
- most_recent_start_time = most_recent_record["start_time"]
+ most_recent_start_time = float(most_recent_record["start_time"])
  # only include most recent runs and results
  filter_ = {"start_time": {"$gt": most_recent_start_time - MIGRATION_TIME_FRAME}}
  else:
@@ -168,7 +156,10 @@ def migrate_table(collection, Model, vprint):
  if row.get("start_time") and isinstance(row["start_time"], float):
  row["start_time"] = datetime.fromtimestamp(row["start_time"])
  if row.get("created"):
- row["created"] = datetime.fromisoformat(row["created"])
+ if isinstance(row["created"], str):
+ row["created"] = datetime.fromisoformat(row["created"])
+ else:
+ row.pop("created")
 
  # promote some metadata fields to the appropriate column
  for field in FIELDS_TO_PROMOTE:
@@ -189,14 +180,34 @@ def migrate_table(collection, Model, vprint):
  row["result_id"] = convert_objectid_to_uuid(row["metadata"][field])
  else:
  row["metadata"][field] = convert_objectid_to_uuid(row["metadata"][field])
+ if row.get(field):
+ if field == "project":
+ row["project_id"] = convert_objectid_to_uuid(row.pop(field))
+
+ # Table specific stuff
+ if Model.__tablename__ == "projects":
+ if row.get("group_id"):
+ # one of the projects has a group_id assigned (but no group exists in the DB)
+ row["group_id"] = None
+
+ if Model.__tablename__ == "widget_configs":
+ for field in FIELDS_TO_TYPECAST:
+ if row.get(field):
+ if field == "navigable":
+ row[field] = row[field].lower()[0] in ["t", "y"]
+ if field == "weight":
+ row[field] = int(row[field])
+ if row.get("params") and row["params"].get("sort_field"):
+ # we no longer use this field
+ row["params"].pop("sort_field")
 
  obj = Model.from_dict(**row)
  session.add(obj)
  if idx % ROWS_TO_COMMIT_AT_ONCE == 0:
  session.commit()
  session.commit()
  # at the end of the session do a little cleanup
- if Model.__tablename__ == "runs" or Model.__tablename__ == "results":
+ if Model.__tablename__ in ["runs", "results"]:
  conn = Base.metadata.bind.connect()
  # delete any results or runs without start_time
  sql_delete = f"DELETE FROM {Model.__tablename__} where start_time IS NULL;"