From d12c9105d49943f8b26113f712353466667bda52 Mon Sep 17 00:00:00 2001 From: joseph-sentry Date: Tue, 5 Nov 2024 13:29:21 -0500 Subject: [PATCH 1/3] fix: remove get_existing_tests from test process this query is very slow and we don't necessarily need it, a side effect is that this will mean the insert on conflict do nothing will contain rows that it doesn't need to but that's fine --- tasks/test_results_processor.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/tasks/test_results_processor.py b/tasks/test_results_processor.py index 26f17b98b..9002bf8cd 100644 --- a/tasks/test_results_processor.py +++ b/tasks/test_results_processor.py @@ -57,11 +57,6 @@ def __init__(self, err_msg, file_content, parser="", parser_err_msg=""): class ParserNotSupportedError(Exception): ... -def get_existing_tests(db_session: Session, repoid: int) -> dict[str, Test]: - existing_tests = db_session.query(Test).filter(Test.repoid == repoid).all() - return {test.id_: test for test in existing_tests} - - def get_repo_flags( db_session: Session, repoid: int, flags: list[str] ) -> dict[str, int]: @@ -218,8 +213,6 @@ def _bulk_write_tests_to_db( repo_flags: dict[str, int] = get_repo_flags(db_session, repoid, flags) - existing_tests: dict[str, Test] = get_existing_tests(db_session, repoid) - for p in parsing_results: framework = str(p.framework) if p.framework else None @@ -249,11 +242,9 @@ def _bulk_write_tests_to_db( computed_name=computed_name, ) - if test_id not in existing_tests: - test_flag_bridge_data += [ - {"test_id": test_id, "flag_id": repo_flags[flag]} - for flag in flags - ] + test_flag_bridge_data += [ + {"test_id": test_id, "flag_id": repo_flags[flag]} for flag in flags + ] test_instance_data.append( dict( @@ -353,7 +344,7 @@ def create_daily_total(): insert_on_conflict_do_nothing_flags = ( insert(TestFlagBridge.__table__) .values(test_flag_bridge_data) - .on_conflict_do_nothing() + .on_conflict_do_nothing(index_elements=["test_id", "flag_id"]) ) db_session.execute(insert_on_conflict_do_nothing_flags) db_session.flush() From 978cde4bf1cd6a8ed44e5a86f480eaea504816ae Mon Sep 17 00:00:00 2001 From: joseph-sentry Date: Wed, 6 Nov 2024 17:18:15 -0500 Subject: [PATCH 2/3] deps: update shared because we need the unique constraint on test flag bridge --- requirements.in | 4 ++++ requirements.txt | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/requirements.in b/requirements.in index 56bb05163..8120971f1 100644 --- a/requirements.in +++ b/requirements.in @@ -1,5 +1,9 @@ https://github.com/codecov/opentelem-python/archive/refs/tags/v0.0.4a1.tar.gz#egg=codecovopentelem +<<<<<<< Updated upstream https://github.com/codecov/shared/archive/83b3376fef128a8c8f4191212d89b97f3b507d15.tar.gz#egg=shared +======= +https://github.com/codecov/shared/archive/6958f32e314e6d728c1ee6a976c1cd7f49ad6f9f.tar.gz#egg=shared +>>>>>>> Stashed changes https://github.com/codecov/test-results-parser/archive/ef39a0888acd62d02a316a852a15d755c74e78c6.tar.gz#egg=test-results-parser https://github.com/codecov/timestring/archive/d37ceacc5954dff3b5bd2f887936a98a668dda42.tar.gz#egg=timestring asgiref>=3.7.2 diff --git a/requirements.txt b/requirements.txt index b866e5d9e..d2942620a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile requirements.in +# pip-compile # amqp==5.2.0 # via kombu @@ -362,7 +362,11 @@ sentry-sdk[celery]==2.13.0 # via # -r requirements.in # shared +<<<<<<< Updated upstream shared @ https://github.com/codecov/shared/archive/83b3376fef128a8c8f4191212d89b97f3b507d15.tar.gz +======= +shared @ https://github.com/codecov/shared/archive/6958f32e314e6d728c1ee6a976c1cd7f49ad6f9f.tar.gz +>>>>>>> Stashed changes # via -r requirements.in six==1.16.0 # via From 12baffc34055dd2e547473842fa17d59e15a9b9a Mon Sep 17 00:00:00 2001 From: joseph-sentry Date: Wed, 6 Nov 2024 17:22:04 -0500 Subject: [PATCH 3/3] fix --- requirements.in | 4 ---- requirements.txt | 4 ---- 2 files changed, 8 deletions(-) diff --git a/requirements.in b/requirements.in index 8120971f1..d1a526cfd 100644 --- a/requirements.in +++ b/requirements.in @@ -1,9 +1,5 @@ https://github.com/codecov/opentelem-python/archive/refs/tags/v0.0.4a1.tar.gz#egg=codecovopentelem -<<<<<<< Updated upstream -https://github.com/codecov/shared/archive/83b3376fef128a8c8f4191212d89b97f3b507d15.tar.gz#egg=shared -======= https://github.com/codecov/shared/archive/6958f32e314e6d728c1ee6a976c1cd7f49ad6f9f.tar.gz#egg=shared ->>>>>>> Stashed changes https://github.com/codecov/test-results-parser/archive/ef39a0888acd62d02a316a852a15d755c74e78c6.tar.gz#egg=test-results-parser https://github.com/codecov/timestring/archive/d37ceacc5954dff3b5bd2f887936a98a668dda42.tar.gz#egg=timestring asgiref>=3.7.2 diff --git a/requirements.txt b/requirements.txt index d2942620a..4cc91f026 100644 --- a/requirements.txt +++ b/requirements.txt @@ -362,11 +362,7 @@ sentry-sdk[celery]==2.13.0 # via # -r requirements.in # shared -<<<<<<< Updated upstream -shared @ https://github.com/codecov/shared/archive/83b3376fef128a8c8f4191212d89b97f3b507d15.tar.gz -======= shared @ https://github.com/codecov/shared/archive/6958f32e314e6d728c1ee6a976c1cd7f49ad6f9f.tar.gz ->>>>>>> Stashed changes # via -r requirements.in six==1.16.0 # via