Create TestFlagBridge during test results processing (#740)

database/models/reports.py

@@ -381,3 +381,18 @@ class DailyTestRollup(CodecovBaseModel, MixinBaseClassNoExternalID):
             name="reports_dailytestrollups_repoid_date_branch_test",
         ),
     )
+
+
+class TestFlagBridge(CodecovBaseModel, MixinBaseClassNoExternalID):
+    __tablename__ = "reports_test_results_flag_bridge"
+
+    test_id = Column(types.Text, ForeignKey("reports_test.id"))
+    test = relationship(Test, backref=backref("test_flag_bridges"))
+
+    repoid = Column(types.Integer, ForeignKey("repos.repoid"))
+    repository = relationship("Repository", backref=backref("test_flag_bridges"))
+
+    flag_id = Column(
+        "flag_id", types.BigInteger, ForeignKey("reports_repositoryflag.id")
+    )
+    flag = relationship("RepositoryFlag", backref=backref("test_flag_bridges"))

requirements.in

@@ -1,5 +1,5 @@
 https://github.com/codecov/opentelem-python/archive/refs/tags/v0.0.4a1.tar.gz#egg=codecovopentelem
-https://github.com/codecov/shared/archive/403d5a21cbeb7a32a5c16d6fc83ac290824a2abc.tar.gz#egg=shared
+https://github.com/codecov/shared/archive/106b0ae2b9a2870899fa3903fc6da0a9ba67eef2.tar.gz#egg=shared
 https://github.com/codecov/test-results-parser/archive/1507de2241601d678e514c08b38426e48bb6d47d.tar.gz#egg=test-results-parser
 https://github.com/codecov/timestring/archive/d37ceacc5954dff3b5bd2f887936a98a668dda42.tar.gz#egg=timestring
 asgiref>=3.7.2

requirements.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
-#    pip-compile requirements.in
+#    pip-compile
 #
 amqp==5.2.0
     # via kombu
@@ -357,7 +357,7 @@ sentry-sdk[celery]==2.13.0
     # via
     #   -r requirements.in
     #   shared
-shared @ https://github.com/codecov/shared/archive/403d5a21cbeb7a32a5c16d6fc83ac290824a2abc.tar.gz
+shared @ https://github.com/codecov/shared/archive/106b0ae2b9a2870899fa3903fc6da0a9ba67eef2.tar.gz
     # via -r requirements.in
 six==1.16.0
     # via

tasks/test_results_processor.py

@@ -25,7 +25,9 @@
     DailyTestRollup,
     Flake,
     Repository,
+    RepositoryFlag,
     Test,
+    TestFlagBridge,
     TestInstance,
     Upload,
 )
@@ -49,6 +51,31 @@ def __init__(self, err_msg, file_content, parser="", parser_err_msg=""):
 class ParserNotSupportedError(Exception): ...
 
 
+def get_existing_tests(db_session: Session, repoid: int) -> dict[str, Test]:
+    existing_tests = db_session.query(Test).filter(Test.repoid == repoid).all()
+    return {test.id_: test for test in existing_tests}
+
+
+def get_repo_flags(
+    db_session: Session, repoid: int, flags: list[str]
+) -> dict[str, int]:
+    repo_flags: list[RepositoryFlag] = (
+        db_session.query(RepositoryFlag)
+        .filter(
+            RepositoryFlag.repository_id == repoid,
+            RepositoryFlag.flag_name.in_(flags),
+        )
+        .all()
+    )
+
+    # flag name => flag id
+    repo_flag_mapping: dict[str, int] = {
+        repo_flag.flag_name: repo_flag.id_ for repo_flag in repo_flags
+    }
+
+    return repo_flag_mapping
+
+
 class TestResultsProcessorTask(BaseCodecovTask, name=test_results_processor_task_name):
     __test__ = False
 
@@ -74,17 +101,15 @@ def run_impl(
         testrun_dict_list = []
         upload_list = []
 
-        f = db_session.query(Flake).all()
-
-        flakes = (
+        repo_flakes = (
             db_session.query(Flake)
             .filter(Flake.repoid == repoid, Flake.end_date.is_(None))
             .all()
         )
 
         flaky_test_set = set()
 
-        for flake in flakes:
+        for flake in repo_flakes:
             flaky_test_set.add(flake.testid)
 
         # process each report session's test information
@@ -125,12 +150,19 @@ def _bulk_write_tests_to_db(
         upload_id: int,
         branch: str,
         parsed_testruns: List[Testrun],
-        flags_hash: str,
         flaky_test_set: set[str],
+        flags: list[str],
     ):
         test_data = []
         test_instance_data = []
+        test_flag_bridge_data = []
         daily_totals = dict()
+        flags_hash = generate_flags_hash(flags)
+
+        repo_flags: dict[str, int] = get_repo_flags(db_session, repoid, flags)
+
+        existing_tests: dict[str, Test] = get_existing_tests(db_session, repoid)
+
         for testrun in parsed_testruns:
             # Build up the data for bulk insert
             name = testrun.name
@@ -150,6 +182,12 @@ def _bulk_write_tests_to_db(
                 )
             )
 
+            if test_id not in existing_tests:
+                test_flag_bridge_data += [
+                    {"test_id": test_id, "repoid": repoid, "flag_id": repo_flags[flag]}
+                    for flag in flags
+                ]
+
             test_instance_data.append(
                 dict(
                     test_id=test_id,
@@ -232,6 +270,15 @@ def create_daily_total():
         db_session.execute(insert_on_conflict_do_nothing)
         db_session.flush()
 
+        if len(test_flag_bridge_data):
+            insert_on_conflict_do_nothing_flags = (
+                insert(TestFlagBridge.__table__)
+                .values(test_flag_bridge_data)
+                .on_conflict_do_nothing()
+            )
+            db_session.execute(insert_on_conflict_do_nothing_flags)
+            db_session.flush()
+
         # Upsert Daily Test Totals
         rollup_table = DailyTestRollup.__table__
         stmt = insert(rollup_table).values(list(daily_totals.values()))
@@ -291,7 +338,6 @@ def process_individual_upload(
             return {
                 "successful": False,
             }
-        flags_hash = generate_flags_hash(upload_obj.flag_names)
         upload_id = upload_obj.id
         branch = upload_obj.report.commit.branch
         self._bulk_write_tests_to_db(
@@ -301,8 +347,8 @@ def process_individual_upload(
             upload_id,
             branch,
             parsed_testruns,
-            flags_hash,
             flaky_test_set,
+            upload_obj.flag_names,
         )
 
         return {

tasks/tests/unit/test_test_results_processor_task.py

@@ -6,8 +6,8 @@
 from test_results_parser import Outcome
 from time_machine import travel
 
-from database.models import CommitReport
-from database.models.reports import DailyTestRollup, Test, TestInstance
+from database.models import CommitReport, RepositoryFlag
+from database.models.reports import DailyTestRollup, Test, TestFlagBridge, TestInstance
 from database.tests.factories import CommitFactory, UploadFactory
 from database.tests.factories.reports import FlakeFactory
 from services.test_results import generate_test_id
@@ -562,6 +562,12 @@ def test_upload_processor_task_call_existing_test_diff_flags_hash(
         dbsession.add(upload)
         dbsession.flush()
         repoid = upload.report.commit.repoid
+        repo_flag = RepositoryFlag(
+            repository=upload.report.commit.repository, flag_name="hello_world"
+        )
+        upload.flags = [repo_flag]
+        dbsession.flush()
+
         redis_queue = [{"url": url, "upload_pk": upload.id_}]
         mocker.patch.object(TestResultsProcessorTask, "app", celery_app)
 
@@ -615,6 +621,18 @@ def test_upload_processor_task_call_existing_test_diff_flags_hash(
             .all()
         )
 
+        test_flag_bridges = dbsession.query(TestFlagBridge).all()
+
+        assert [bridge.test_id for bridge in test_flag_bridges] == [
+            tests[1].id,
+            tests[2].id,
+            tests[3].id,
+            tests[4].id,
+        ]
+        for bridge in test_flag_bridges:
+            assert bridge.flag == repo_flag
+            assert bridge.repoid == repoid
+
         assert len(tests) == 5
         assert len(test_instances) == 4
         assert len(failures) == 1