Optimize Report merging

services/processing/merging.py

@@ -23,7 +23,7 @@ def merge_reports(
     commit_yaml: UserYaml,
     master_report: Report,
     intermediate_reports: list[IntermediateReport],
-) -> MergeResult:
+) -> tuple[Report, MergeResult]:
     session_mapping: dict[int, int] = dict()
     deleted_sessions: set[int] = set()
 
@@ -34,9 +34,14 @@ def merge_reports(
 
         old_sessionid = next(iter(report.sessions))
         new_sessionid = master_report.next_session_number()
-        change_sessionid(report, old_sessionid, new_sessionid)
         session_mapping[intermediate_report.upload_id] = new_sessionid
 
+        if master_report.is_empty() and old_sessionid == new_sessionid:
+            # if the master report is empty, we can avoid a costly merge operation
+            master_report = report
+            continue
+
+        change_sessionid(report, old_sessionid, new_sessionid)
         session = report.sessions[new_sessionid]
 
         _session_id, session = master_report.add_session(
@@ -51,7 +56,7 @@ def merge_reports(
 
         master_report.merge(report)
 
-    return MergeResult(session_mapping, deleted_sessions)
+    return master_report, MergeResult(session_mapping, deleted_sessions)
 
 
 @sentry_sdk.trace
@@ -158,6 +163,7 @@ def make_upload_totals(
     )
 
 
+@sentry_sdk.trace
 def change_sessionid(report: EditableReport, old_id: int, new_id: int):
     """
     Modifies the `EditableReport`, changing the session with `old_id` to have `new_id` instead.

services/report/raw_upload_processor.py

@@ -108,7 +108,17 @@ def process_raw_upload(
             r.filename = current_filename
             raise
 
-        if report_from_file:
+        if not report_from_file:
+            continue
+        if report.is_empty():
+            # if the initial report is empty, we can avoid a costly merge operation
+            report = report_from_file
+        else:
+            # merging the smaller report into the larger one is faster,
+            # so swap the two reports in that case.
+            if len(report_from_file._files) > len(report._files):
+                report_from_file, report = report, report_from_file
+
             report.merge(report_from_file, joined=True)
 
     if not report:

tasks/compute_comparison.py

@@ -66,9 +66,8 @@ def run_impl(
 
         # At this point we can calculate the patch coverage
         # Because we have a HEAD report and a base commit to get the diff from
-        if comparison.patch_totals is None:
-            patch_totals = comparison_proxy.get_patch_totals()
-            comparison.patch_totals = minimal_totals(patch_totals)
+        patch_totals = comparison_proxy.get_patch_totals()
+        comparison.patch_totals = minimal_totals(patch_totals)
 
         if not comparison_proxy.has_project_coverage_base_report():
             comparison.error = CompareCommitError.missing_base_report.value

tasks/tests/integration/test_upload_e2e.py

@@ -141,9 +141,14 @@ def setup_mocks(
         "tasks.upload.fetch_commit_yaml_and_possibly_store",
         return_value=UserYaml(user_yaml or {}),
     )
+    mocker.patch(
+        "tasks.compute_comparison.get_current_yaml",
+        return_value=UserYaml(user_yaml or {}),
+    )
     # disable all the tasks being emitted from `UploadFinisher`.
     # ideally, we would really want to test their outcomes as well.
     mocker.patch("tasks.notify.NotifyTask.run_impl")
+    mocker.patch("tasks.sync_pull.PullSyncTask.run_impl")
     mocker.patch("tasks.save_commit_measurements.SaveCommitMeasurementsTask.run_impl")
 
     # force `report_json` to be written out to storage
@@ -238,6 +243,14 @@ def test_full_upload(
         f"UPDATE reports_upload SET id={upload_id} WHERE id={first_upload.id}"
     )
 
+    with run_tasks():
+        upload_task.apply_async(
+            kwargs={
+                "repoid": repoid,
+                "commitid": commitid,
+            }
+        )
+
     do_upload(
         b"""
 a.rs

tasks/upload_finisher.py

@@ -423,7 +423,9 @@ def perform_report_merging(
         archive_service, commit.commitid, upload_ids, intermediate_reports_in_redis
     )
 
-    merge_result = merge_reports(commit_yaml, master_report, intermediate_reports)
+    master_report, merge_result = merge_reports(
+        commit_yaml, master_report, intermediate_reports
+    )
 
     # Update the `Upload` in the database with the final session_id
     # (aka `order_number`) and other statuses