diff --git a/docker/entrypoint_prod.sh b/docker/entrypoint_prod.sh index 7f105002c12..68402d3f873 100755 --- a/docker/entrypoint_prod.sh +++ b/docker/entrypoint_prod.sh @@ -52,6 +52,9 @@ elif [ "$1" == "worker_log_parser_fail_json_sheriffed" ]; then elif [ "$1" == "worker_log_parser_fail_json_unsheriffed" ]; then export REMAP_SIGTERM=SIGQUIT newrelic-admin run-program celery -A treeherder worker --without-gossip --without-mingle --without-heartbeat -Q log_parser_fail_json_unsheriffed --concurrency=7 +elif [ "$1" == "worker_perf_ingest" ]; then + export REMAP_SIGTERM=SIGQUIT + exec newrelic-admin run-program celery -A treeherder worker --without-gossip --without-mingle --without-heartbeat -Q perf_ingest --concurrency=7 # Tasks that don't need a dedicated worker. elif [ "$1" == "worker_misc" ]; then diff --git a/tests/etl/test_job_loader.py b/tests/etl/test_job_loader.py index 5f2814bf5b0..1cbed47ea1b 100644 --- a/tests/etl/test_job_loader.py +++ b/tests/etl/test_job_loader.py @@ -91,6 +91,27 @@ def test_job_transformation(pulse_jobs, transformed_pulse_jobs): assert transformed_pulse_jobs[idx] == json.loads(json.dumps(jl.transform(pulse_job))) +def test_job_transformation_extracts_perfherder_data(pulse_jobs): + jl = JobLoader() + pulse_job = pulse_jobs[0] + links = pulse_job.setdefault("jobInfo", {}).setdefault("links", []) + links.append( + { + "label": "artifact uploaded", + "linkText": "perfherder-data-decision.json", + "url": "https://example.mozilla.com/perfherder-data-decision.json", + } + ) + transformed_job = jl.transform(pulse_job) + assert transformed_job["job"]["perfherder_data_references"] == [ + { + "name": "perfherder-data-decision.json", + "url": "https://example.mozilla.com/perfherder-data-decision.json", + "parse_status": "pending", + } + ] + + @responses.activate def test_new_job_transformation(new_pulse_jobs, new_transformed_jobs, failure_classifications): jl = JobLoader() @@ -173,6 +194,27 @@ def test_ingest_pulse_jobs( ] == logs_expected +def test_ingest_pulse_job_with_perfherder_data( + pulse_jobs, push_stored, failure_classifications, mock_log_parser +): + jl = JobLoader() + pulse_job = copy.deepcopy(pulse_jobs[0]) + pulse_job["origin"]["revision"] = push_stored[0]["revision"] + pulse_job["jobInfo"]["links"] = [ + { + "label": "artifact uploaded", + "linkText": "perfherder-data-decision.json", + "url": "https://example.mozilla.com/perfherder-data-decision.json", + } + ] + + jl.process_job(pulse_job, "https://firefox-ci-tc.services.mozilla.com") + + names = list(JobLog.objects.filter(job_id=1).values_list("name", flat=True)) + assert "live_backing_log" in names + assert any(n.startswith("perfherder-data") for n in names) + + def test_ingest_pulse_job_with_long_job_type_name( pulse_jobs, test_repository, push_stored, failure_classifications, mock_log_parser ): diff --git a/tests/sample_data/pulse_consumer/taskcluster_transformed_jobs.json b/tests/sample_data/pulse_consumer/taskcluster_transformed_jobs.json index 49547d7c519..e6d76b86261 100644 --- a/tests/sample_data/pulse_consumer/taskcluster_transformed_jobs.json +++ b/tests/sample_data/pulse_consumer/taskcluster_transformed_jobs.json @@ -12,6 +12,7 @@ "job_guid": "037e6659-346e-426c-a3f3-cc8c9c81747c/0", "job_symbol": "a11y", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -48,6 +49,7 @@ "job_guid": "037749f1-b0c8-4112-b31e-264184cd1ce5/0", "job_symbol": "bc13", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -84,6 +86,7 @@ "job_guid": "038ede3c-f21a-4453-96c7-0f2854ebe608/0", "job_symbol": "mda2", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -120,6 +123,7 @@ "job_guid": "038015f4-45d7-4441-8257-e855293d1b95/0", "job_symbol": "X5", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -156,6 +160,7 @@ "job_guid": "0037ec90-e19a-4bb2-948f-402e51bdf6e0/0", "job_symbol": "bc2", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -192,6 +197,7 @@ "job_guid": "01c516b5-17fb-401f-b85d-04bc9d24e296/0", "job_symbol": "X3", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -228,6 +234,7 @@ "job_guid": "022f9b9a-e4f3-4bb7-9105-816148f9d9d7/0", "job_symbol": "dt7", "log_references": [], + "perfherder_data_references": [], "machine": "i-0429b6e2c0ac7474d", "machine_platform": { "architecture": "-", @@ -265,6 +272,7 @@ "job_guid": "02572f2a-6ac6-4bc7-bd15-7ac54d89e1a4/0", "job_symbol": "bc14", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -301,6 +309,7 @@ "job_guid": "029a2d36-0a9a-421e-8e18-35bc4cc49695/0", "job_symbol": "en-US", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -337,6 +346,7 @@ "job_guid": "02a32149-a0ea-4801-bb9d-f8dfc6328b64/0", "job_symbol": "dt2", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -373,6 +383,7 @@ "job_guid": "02c78770-a8cf-4618-be34-1b2fc07e7883/0", "job_symbol": "X2", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -416,6 +427,7 @@ "url": "https://taskcluster.net/api/queue/v1/task/B3EheQ-IQ5aPzw34KS_a6g/runs/0/artifacts/public/logs/live_backing.log" } ], + "perfherder_data_references": [], "machine": "t-mojave-r7-153", "machine_platform": { "architecture": "-", @@ -453,6 +465,7 @@ "job_guid": "045aa590-a31a-4d5a-afdd-13929544fa91/0", "job_symbol": "wpt7", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -489,6 +502,7 @@ "job_guid": "0477a30f-f46f-4d6d-8747-64c75dfc0301/0", "job_symbol": "11", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -525,6 +539,7 @@ "job_guid": "05758c69-457b-4c2b-a705-c25010b6d9ab/0", "job_symbol": "bc6", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -561,6 +576,7 @@ "job_guid": "059b986f-63d3-4d0f-a8a4-6cf1bd9bc0f4/0", "job_symbol": "14", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -597,6 +613,7 @@ "job_guid": "05d1df19-41cf-4cb9-b420-acf8ee834994/0", "job_symbol": "mda2", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -633,6 +650,7 @@ "job_guid": "05d1f2ff-8bd0-480d-9942-8c5773eea278/0", "job_symbol": "R5", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -669,6 +687,7 @@ "job_guid": "05f10ad3-417f-4d29-b6d1-30fbf7d19d33/0", "job_symbol": "Wr3", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -705,6 +724,7 @@ "job_guid": "060f21f8-2b7f-48f4-ab10-462ebcf01849/0", "job_symbol": "mda1", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -741,6 +761,7 @@ "job_guid": "0699d0a9-d6b6-4348-b35f-0e24d11513eb/0", "job_symbol": "Ru4", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -777,6 +798,7 @@ "job_guid": "0813b62b-c5c6-4046-8da5-7d4193026637/0", "job_symbol": "mda3", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -813,6 +835,7 @@ "job_guid": "08447b62-61bd-4260-9df9-1ebabdf5a455/0", "job_symbol": "rt", "log_references": [], + "perfherder_data_references": [], "machine": "i-0da13f5d4e5576fd5", "machine_platform": { "architecture": "-", @@ -850,6 +873,7 @@ "job_guid": "0cf0ae64-293e-4960-ba49-e7ed01bcadcf/0", "job_symbol": "GTest", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -886,6 +910,7 @@ "job_guid": "0ffe9649-c5a1-4cbc-acb9-22246ea7e3b2/0", "job_symbol": "R4", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -922,6 +947,7 @@ "job_guid": "0f186356-9fae-42d1-a144-b9b5c34c5d88/0", "job_symbol": "X3", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -965,6 +991,7 @@ "url": "https://taskcluster.net/api/queue/v1/task/ENXQRjSMRWyWK_zAEYEoXg/runs/0/artifacts/public/logs/live_backing.log" } ], + "perfherder_data_references": [], "machine": "t-yosemite-r7-098", "machine_platform": { "architecture": "-", @@ -1002,6 +1029,7 @@ "job_guid": "12f36ab3-2048-48d0-87f2-6c1f0d546749/0", "job_symbol": "rt", "log_references": [], + "perfherder_data_references": [], "machine": "i-062aa60828009efec", "machine_platform": { "architecture": "-", @@ -1039,6 +1067,7 @@ "job_guid": "17dddb99-eee2-48fd-a4d1-3b02c2a48157/0", "job_symbol": "bc9", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -1075,6 +1104,7 @@ "job_guid": "1608cd92-bbd7-4b45-830c-9254a555d9b0/0", "job_symbol": "R1", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -1111,6 +1141,7 @@ "job_guid": "163ddf0e-0aef-4578-ad09-55f64622546b/0", "job_symbol": "X4", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -1147,6 +1178,7 @@ "job_guid": "1641f3de-018f-4959-8be9-824eba96015f/0", "job_symbol": "C", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -1183,6 +1215,7 @@ "job_guid": "16c80b31-d2b0-4735-8903-0521632da0d1/0", "job_symbol": "c2", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -1219,6 +1252,7 @@ "job_guid": "19317164-95b3-48a4-8d73-f5c4960b561d/0", "job_symbol": "wpt16", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -1262,6 +1296,7 @@ "url": "https://taskcluster.net/api/queue/v1/task/GWxTc8p_RUa5_Xo4RtMiCA/runs/0/artifacts/public/logs/live_backing.log" } ], + "perfherder_data_references": [], "machine": "machine-39", "machine_platform": { "architecture": "-", @@ -1306,6 +1341,7 @@ "url": "https://taskcluster.net/api/queue/v1/task/HsvD9YlDQKaoP4AQsuGeJw/runs/0/artifacts/public/logs/live_backing.log" } ], + "perfherder_data_references": [], "machine": "t-yosemite-r7-123", "machine_platform": { "architecture": "-", @@ -1343,6 +1379,7 @@ "job_guid": "24a0ea3b-ecb2-4ad6-bbf7-e63dd83ab6c9/0", "job_symbol": "dt10", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -1379,6 +1416,7 @@ "job_guid": "24ce9e57-a4be-479f-83cb-4f208e8047ad/0", "job_symbol": "mda1", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -1415,6 +1453,7 @@ "job_guid": "2b6219ef-8996-41f8-849f-653a5e754b3d/0", "job_symbol": "1", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -1451,6 +1490,7 @@ "job_guid": "2b947cae-4800-4595-8cbc-7e09a6648fad/0", "job_symbol": "3", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -1494,6 +1534,7 @@ "url": "https://taskcluster.net/api/queue/v1/task/KYZKwkbvRc-qPE22g286yQ/runs/0/artifacts/public/logs/live_backing.log" } ], + "perfherder_data_references": [], "machine": "i-06c2c7e06fc36c6c5", "machine_platform": { "architecture": "-", @@ -1531,6 +1572,7 @@ "job_guid": "2a7063d0-a2dc-4e3b-b235-a4f1b792d5ab/0", "job_symbol": "rt", "log_references": [], + "perfherder_data_references": [], "machine": "i-019077061bddb4fe7", "machine_platform": { "architecture": "-", @@ -1575,6 +1617,7 @@ "url": "https://taskcluster.net/api/queue/v1/task/LBJoI2wUTNCteg-5CnaYZQ/runs/0/artifacts/public/logs/live_backing.log" } ], + "perfherder_data_references": [], "machine": "i-0b63c3556524f5295", "machine_platform": { "architecture": "-", @@ -1612,6 +1655,7 @@ "job_guid": "2d602bc3-8b79-41a0-8788-7b04268ea114/0", "job_symbol": "rt", "log_references": [], + "perfherder_data_references": [], "machine": "i-0be69dc84b0354bde", "machine_platform": { "architecture": "-", @@ -1656,6 +1700,7 @@ "url": "https://taskcluster.net/api/queue/v1/task/MrYMxyDDSwWXJey-uo4T3g/runs/0/artifacts/public/logs/live_backing.log" } ], + "perfherder_data_references": [], "machine": "i-08b6fd26055d2603e", "machine_platform": { "architecture": "-", @@ -1693,6 +1738,7 @@ "job_guid": "356cfeab-d328-48bf-9631-f3bb922c21b6/0", "job_symbol": "dt4", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -1736,6 +1782,7 @@ "url": "https://taskcluster.net/api/queue/v1/task/OiZYxvi5RfKol0A4SBbvOw/runs/0/artifacts/public/logs/live_backing.log" } ], + "perfherder_data_references": [], "machine": "i-0d0f91277e3f902cc", "machine_platform": { "architecture": "-", @@ -1780,6 +1827,7 @@ "url": "https://taskcluster.net/api/queue/v1/task/PCVDCxY9QM-lEgaGhrCPrw/runs/0/artifacts/public/logs/live_backing.log" } ], + "perfherder_data_references": [], "machine": "i-0bd696849d1151857", "machine_platform": { "architecture": "-", @@ -1817,6 +1865,7 @@ "job_guid": "3ebdf33c-d32a-4bc0-8063-a84707f970a3/0", "job_symbol": "Wr3", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -1853,6 +1902,7 @@ "job_guid": "4397606a-34e6-41a1-b76d-6f317fc99647/0", "job_symbol": "bc16", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -1896,6 +1946,7 @@ "url": "https://taskcluster.net/api/queue/v1/task/QDGdgQTQRC2xZNR3Zs_IhA/runs/0/artifacts/public/logs/live_backing.log" } ], + "perfherder_data_references": [], "machine": "i-0de7ae42ffc13af93", "machine_platform": { "architecture": "-", @@ -1933,6 +1984,7 @@ "job_guid": "41485368-dce6-47e4-a72e-e54cb1c9d260/0", "job_symbol": "rt", "log_references": [], + "perfherder_data_references": [], "machine": "i-0da13f5d4e5576fd5", "machine_platform": { "architecture": "-", @@ -1970,6 +2022,7 @@ "job_guid": "41b25c7a-e51b-405f-9200-7a7fb6e3786e/0", "job_symbol": "c1", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -2006,6 +2059,7 @@ "job_guid": "4cd050bd-f164-4742-a854-5cffed022423/0", "job_symbol": "16", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -2042,6 +2096,7 @@ "job_guid": "4da72c4f-5f85-48c8-9ad0-81a460ff9c4d/0", "job_symbol": "1", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -2078,6 +2133,7 @@ "job_guid": "4db3bf93-aeb7-4774-adea-104513dd92b5/0", "job_symbol": "Ru3", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -2114,6 +2170,7 @@ "job_guid": "4f03f140-f422-40b0-b48f-0a740ab7f8b3/0", "job_symbol": "wpt2", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -2150,6 +2207,7 @@ "job_guid": "50011ea6-cf30-48c9-a82f-8af431d8e55c/0", "job_symbol": "2", "log_references": [], + "perfherder_data_references": [], "machine": "i-07dbb95ac410df8f7", "machine_platform": { "architecture": "-", @@ -2187,6 +2245,7 @@ "job_guid": "5168cf32-9b2c-4e0a-b490-0ee18fd42d27/0", "job_symbol": "bc3", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -2230,6 +2289,7 @@ "url": "https://taskcluster.net/api/queue/v1/task/UZ37gpDqTRS1pBNnp0Spsg/runs/0/artifacts/public/logs/live_backing.log" } ], + "perfherder_data_references": [], "machine": "i-0d6489faaf7371ecb", "machine_platform": { "architecture": "-", @@ -2267,6 +2327,7 @@ "job_guid": "52818b84-7970-4f89-87de-1dee91c9bf22/0", "job_symbol": "Mn", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -2303,6 +2364,7 @@ "job_guid": "5314b88e-dea5-41dc-9462-73dc98dfb3fb/0", "job_symbol": "Wr1", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -2339,6 +2401,7 @@ "job_guid": "54653845-4285-49c6-8af1-e06290c8bd21/0", "job_symbol": "bc15", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -2375,6 +2438,7 @@ "job_guid": "56a6d16f-a6d9-4a5c-b058-e40deb6b64fa/0", "job_symbol": "14", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -2411,6 +2475,7 @@ "job_guid": "56ca5519-9623-480a-aab2-0e821a0bfeaf/0", "job_symbol": "R2", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -2447,6 +2512,7 @@ "job_guid": "5a4158c5-f970-4fbe-95cd-635cea1af289/0", "job_symbol": "Ru2", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -2483,6 +2549,7 @@ "job_guid": "5cec3615-811d-49bd-a1c1-2fe58f66c302/0", "job_symbol": "rt", "log_references": [], + "perfherder_data_references": [], "machine": "i-019077061bddb4fe7", "machine_platform": { "architecture": "-", @@ -2520,6 +2587,7 @@ "job_guid": "5d075f8e-3002-4490-a6a6-a2a32f12d15c/0", "job_symbol": "Wd1", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -2556,6 +2624,7 @@ "job_guid": "5e266e25-6ecc-43e1-ab0b-b895f3561677/0", "job_symbol": "7", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -2592,6 +2661,7 @@ "job_guid": "5f1ce78f-947d-4024-beee-2aa9b5e825b5/0", "job_symbol": "mda3", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -2628,6 +2698,7 @@ "job_guid": "637dadaf-dc34-4e5b-baed-dcfbe30882ca/0", "job_symbol": "c2", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -2664,6 +2735,7 @@ "job_guid": "63a65daf-4efc-46f7-a633-f9da6df884dc/0", "job_symbol": "10", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -2700,6 +2772,7 @@ "job_guid": "61453abf-cf6f-45de-baf6-46ae44b364e6/0", "job_symbol": "Ru5", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -2736,6 +2809,7 @@ "job_guid": "6180a789-8d48-4ee1-a5d1-f44d82192795/0", "job_symbol": "9", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -2772,6 +2846,7 @@ "job_guid": "6209c932-e913-4b47-9f24-7cd3239fc09c/0", "job_symbol": "Wr5", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -2808,6 +2883,7 @@ "job_guid": "64fae294-065c-4a29-9d98-b8fd7e5336e3/0", "job_symbol": "15", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -2844,6 +2920,7 @@ "job_guid": "66724e31-339c-4700-991b-4f55638227c5/0", "job_symbol": "c3", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -2887,6 +2964,7 @@ "url": "https://taskcluster.net/api/queue/v1/task/ZyNAZZR0SmOcgqC_k_wqsQ/runs/0/artifacts/public/logs/live_backing.log" } ], + "perfherder_data_references": [], "machine": "i-01e55895145a2d19c", "machine_platform": { "architecture": "-", @@ -2924,6 +3002,7 @@ "job_guid": "6b5b947a-142f-49c8-b290-ae835143506d/0", "job_symbol": "Ru1", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -2960,6 +3039,7 @@ "job_guid": "6b9505f1-f132-430f-9fc7-fdf6428f968d/0", "job_symbol": "tp6m-c-3", "log_references": [], + "perfherder_data_references": [], "machine": "pixel2-44", "machine_platform": { "architecture": "-", @@ -2997,6 +3077,7 @@ "job_guid": "68d7fc27-f72d-429e-888d-6253c4200f6d/0", "job_symbol": "TV", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -3033,6 +3114,7 @@ "job_guid": "68d944ea-d780-49b1-aab5-22b85d278c92/0", "job_symbol": "wpt4", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -3069,6 +3151,7 @@ "job_guid": "69ceefb0-c2c0-4a5e-974b-53bf57394b5d/0", "job_symbol": "Wr1", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -3105,6 +3188,7 @@ "job_guid": "69e5838d-1714-421b-9d53-86b1062090f0/0", "job_symbol": "a11y", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -3141,6 +3225,7 @@ "job_guid": "6f40e99d-2210-4329-a74d-3497c515a69a/0", "job_symbol": "R1", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -3177,6 +3262,7 @@ "job_guid": "6dadf806-7620-4ff9-afd6-0e41beb9579d/0", "job_symbol": "R7", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -3213,6 +3299,7 @@ "job_guid": "73d4925f-0c77-4e72-a6ec-c9cd122b7438/0", "job_symbol": "X5", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -3249,6 +3336,7 @@ "job_guid": "7547286c-0b8f-4c4a-87a2-4e2ba499750e/0", "job_symbol": "dt9", "log_references": [], + "perfherder_data_references": [], "machine": "unknown", "machine_platform": { "architecture": "-", @@ -3292,6 +3380,7 @@ "url": "https://taskcluster.net/api/queue/v1/task/eokGh4-OSbOqAzVhJypYPg/runs/0/artifacts/public/logs/live_backing.log" } ], + "perfherder_data_references": [], "machine": "i-09f5a2a05cb655f08", "machine_platform": { "architecture": "-", @@ -3329,6 +3418,7 @@ "job_guid": "7ae80204-f44d-443f-b3c8-ffc8f49727d7/0", "job_symbol": "rt", "log_references": [], + "perfherder_data_references": [], "machine": "i-0be69dc84b0354bde", "machine_platform": { "architecture": "-", @@ -3366,6 +3456,7 @@ "job_guid": "7eee1db9-28c1-46d7-be57-acbfa10fc1ad/0", "job_symbol": "rt", "log_references": [], + "perfherder_data_references": [], "machine": "i-062aa60828009efec", "machine_platform": { "architecture": "-", diff --git a/tests/sample_data/pulse_consumer/transformed_job_data.json b/tests/sample_data/pulse_consumer/transformed_job_data.json index 2440dd2c540..091ff1e998d 100644 --- a/tests/sample_data/pulse_consumer/transformed_job_data.json +++ b/tests/sample_data/pulse_consumer/transformed_job_data.json @@ -34,6 +34,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "bc5", "job_guid": "008dcdae-bde0-4834-a967-d13d681037ae/0", @@ -78,6 +79,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "dt10", "job_guid": "0401bb89-f4b5-41b1-820f-088e3fb36495/0", @@ -122,6 +124,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "J2", "job_guid": "6e58db2d-115d-4f82-8609-6276fc2e91b1/0", @@ -166,6 +169,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "bc11", "job_guid": "6ff402cc-c8d5-4262-8fc8-ee4897425e77/0", @@ -210,6 +214,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "bc11", "job_guid": "73677162-e702-48c5-8f95-391bef4afcf6/0", @@ -254,6 +259,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "bc9", "job_guid": "09a2ba36-57c1-49fe-85f8-d015aeb2890d/0", @@ -298,6 +304,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "J3", "job_guid": "0a29590a-79a2-4ca9-9a80-97bf87a1eae4/0", @@ -342,6 +349,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "bc6", "job_guid": "70f7bccb-4ef5-4a96-add3-d765033086ba/0", @@ -386,6 +394,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "J1", "job_guid": "719ee073-d258-41ad-943c-40bf108c48ba/0", @@ -430,6 +439,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "bc16", "job_guid": "741f11e4-05ce-4597-82a4-37d0958525a3/0", @@ -474,6 +484,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "J1", "job_guid": "7b560f96-5cfa-4cc6-b021-2a6e824471d6/0", @@ -518,6 +529,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "J3", "job_guid": "79dce0cc-2a61-4c04-be40-dfd30277fb78/0", @@ -562,6 +574,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "bc12", "job_guid": "789f4f1b-8d6d-49a5-b335-4d6e63bfae49/0", @@ -606,6 +619,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "bc2", "job_guid": "7a441a78-2ff2-4742-bc8c-f2b1dbc13b10/0", @@ -650,6 +664,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "J4", "job_guid": "15c6c852-855b-4b8b-adc4-5112f38c2b3f/0", @@ -694,6 +709,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "J2", "job_guid": "15c943ea-003e-4d31-a0be-afebf7e2d20e/0", @@ -738,6 +754,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "bc16", "job_guid": "14d4f704-b890-4473-b5e0-d360a278d042/0", @@ -782,6 +799,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "J4", "job_guid": "7ed5ec47-23b0-4607-a289-81f22135d438/0", @@ -826,6 +844,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "dt10", "job_guid": "19cbc73f-a1cb-49e3-bfac-a603376cbf4e/0", @@ -870,6 +889,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "bc", "job_guid": "18f2e4ef-c9ba-4b3e-934c-b24555c6b85f/0", @@ -914,6 +934,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "bc8", "job_guid": "23663e4c-135c-40f4-89ce-c2a507de6b7d/0", @@ -958,6 +979,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "bc13", "job_guid": "23e1e0ed-b338-4d43-aae0-9aa75f4a6dd2/0", @@ -1002,6 +1024,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "dt9", "job_guid": "3351028c-f241-4e59-b027-166ae695b2be/0", @@ -1040,6 +1063,7 @@ "state": "completed", "result": "retry", "log_references": [], + "perfherder_data_references": [], "tier": 1, "job_symbol": "J5", "job_guid": "30aabc98-c33e-4483-b1ce-d5cee6783efc/0", @@ -1084,6 +1108,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "bc9", "job_guid": "32badb89-fcc1-4097-9f51-b4b6ca6e106a/0", @@ -1128,6 +1153,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "bc8", "job_guid": "39161834-c844-4f4c-b118-cbde97826e8e/0", @@ -1172,6 +1198,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "bc3", "job_guid": "4ea5830c-6a16-49b0-87f7-74534f13c05a/0", @@ -1210,6 +1237,7 @@ "state": "completed", "result": "retry", "log_references": [], + "perfherder_data_references": [], "tier": 1, "job_symbol": "J5", "job_guid": "57caed20-3ae8-455f-86f5-bce3260952d0/0", @@ -1254,6 +1282,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "bc10", "job_guid": "5556b66a-6ccc-4bed-9c48-36e283d447b3/0", @@ -1298,6 +1327,7 @@ "parse_status": "pending" } ], + "perfherder_data_references": [], "tier": 1, "job_symbol": "bc3", "job_guid": "60838ae3-4d72-4761-afca-020571f3d506/0", diff --git a/treeherder/config/settings.py b/treeherder/config/settings.py index 8bd2994733c..abd74b21b21 100644 --- a/treeherder/config/settings.py +++ b/treeherder/config/settings.py @@ -332,6 +332,7 @@ ), Queue("store_pulse_pushes", Exchange("default"), routing_key="store_pulse_pushes"), Queue("statsd", Exchange("default"), routing_key="statsd"), + Queue("perf_ingest", Exchange("default"), routing_key="perf_ingest.normal"), ] # Force all queues to be explicitly listed in `CELERY_TASK_QUEUES` to help prevent typos diff --git a/treeherder/etl/job_loader.py b/treeherder/etl/job_loader.py index 3bdc52e4839..6e70e1b7e77 100644 --- a/treeherder/etl/job_loader.py +++ b/treeherder/etl/job_loader.py @@ -142,6 +142,7 @@ def transform(self, pulse_job): "machine": self._get_machine(pulse_job), "option_collection": self._get_option_collection(pulse_job), "log_references": self._get_log_references(pulse_job), + "perfherder_data_references": self._get_perfherder_data_references(pulse_job), }, "superseded": pulse_job.get("coalesced", []), "revision": pulse_job["origin"]["revision"], @@ -182,6 +183,25 @@ def _get_log_references(self, job): log_references.extend(self._get_errorsummary_log_references(job)) return log_references + def _get_perfherder_data_references(self, job): + performance_data_references = [] + for artifact in job.get("jobInfo", {}).get("links", []): + artifact_link = artifact.get("url") + if ( + artifact_link + and "perfherder-data" in artifact_link + and artifact_link.endswith(".json") + ): + performance_data_references.append( + { + "name": artifact.get("linkText"), + "url": artifact_link, + "parse_status": "pending", + } + ) + + return performance_data_references + def _get_errorsummary_log_references(self, job): log_references = [] try: diff --git a/treeherder/etl/jobs.py b/treeherder/etl/jobs.py index 2fdf9d7dccf..cfba7797e2d 100644 --- a/treeherder/etl/jobs.py +++ b/treeherder/etl/jobs.py @@ -277,6 +277,7 @@ def _load_job(repository, job_datum, push_id): push_id=push_id, ) + job_log_status_map = dict([(k, v) for (v, k) in JobLog.STATUSES]) log_refs = job_datum.get("log_references", []) job_logs = [] if log_refs: @@ -287,8 +288,7 @@ def _load_job(repository, job_datum, push_id): url = log.get("url") or "unknown" url = url[0:255] - parse_status_map = dict([(k, v) for (v, k) in JobLog.STATUSES]) - mapped_status = parse_status_map.get(log.get("parse_status")) + mapped_status = job_log_status_map.get(log.get("parse_status")) if mapped_status: parse_status = mapped_status else: @@ -302,6 +302,30 @@ def _load_job(repository, job_datum, push_id): _schedule_log_parsing(job, job_logs, result, repository) + perfherder_data_refs = job_datum.get("perfherder_data_references", []) + perfherder_datas = [] + if perfherder_data_refs: + for perfherder_data in perfherder_data_refs: + name = perfherder_data.get("name") or "unknown" + name = name[0:50] + + url = perfherder_data.get("url") or "unknown" + url = url[0:255] + + mapped_status = job_log_status_map.get(perfherder_data.get("parse_status")) + if mapped_status: + parse_status = mapped_status + else: + parse_status = JobLog.PENDING + + jl, _ = JobLog.objects.get_or_create( + job=job, name=name, url=url, defaults={"status": parse_status} + ) + + perfherder_datas.append(jl) + + _schedule_perfherder_ingest(job, perfherder_datas) + return job_guid @@ -363,6 +387,18 @@ def _schedule_log_parsing(job, job_logs, result, repository): parse_logs.apply_async(queue=queue, args=[job.id, [job_log.id], priority]) +def _schedule_perfherder_ingest(job, job_logs): + from treeherder.perf.tasks import ingest_perfherder_data + + for job_log in job_logs: + if job_log.status != JobLog.PENDING: + continue + + job_log_name = job_log.name.replace("-", "_") + if job_log_name.startswith("perfherder_data"): + ingest_perfherder_data.apply_async(queue="perf_ingest", args=[job.id, [job_log.id]]) + + def store_job_data(repository, original_data): """ Store job data instances into jobs db diff --git a/treeherder/etl/perf.py b/treeherder/etl/perf.py index b764c921bae..2d70c761913 100644 --- a/treeherder/etl/perf.py +++ b/treeherder/etl/perf.py @@ -333,12 +333,53 @@ def _load_perf_datum(job: Job, perf_datum: dict): generate_alerts.apply_async(args=[signature.id], queue="generate_perf_alerts") +def _is_suite_allowed(suites: list, framework_name: str) -> bool: + allowlist_frameworks_suites = { + "build_metrics": ["mach_artifact_toolchain"], + "browsertime": ["constant-regression"], + } + allowed = allowlist_frameworks_suites.get(framework_name) + if not allowed: + return False + if allowed == ["ALL"]: + return True + return any(suite["name"] in allowed for suite in suites) + + +def _should_ingest(framework_name: str, suites: list, is_perfherder_data_json: bool) -> bool: + """ + This function will be removed after migration to JSON artifact. + Ingestion policy: + - If (framework, suites) is allowlisted => JSON only + - If framework is NOT allowlisted => Log parsing only + - If framework is allowlisted but suites don't match => Log parsing only + """ + is_allowed = _is_suite_allowed(suites, framework_name) + if is_allowed: + # Allowlisted -> only JSON artifact + return is_perfherder_data_json + else: + # Not allowlisted (including unregistered frameworks (e.g, raptor)) -> only logs + return not is_perfherder_data_json + + def store_performance_artifact(job, artifact): blob = json.loads(artifact["blob"]) performance_data = blob["performance_data"] + log_url = blob.get("logurl", "") + is_perfherder_data_json = log_url.endswith(".json") and "perfherder-data" in log_url + if isinstance(performance_data, list): for perfdatum in performance_data: + framework_name = perfdatum["framework"]["name"] + suites = perfdatum.get("suites", []) + if not _should_ingest(framework_name, suites, is_perfherder_data_json): + continue _load_perf_datum(job, perfdatum) else: + framework_name = performance_data["framework"]["name"] + suites = performance_data.get("suites", []) + if not _should_ingest(framework_name, suites, is_perfherder_data_json): + return _load_perf_datum(job, performance_data) diff --git a/treeherder/log_parser/tasks.py b/treeherder/log_parser/tasks.py index c4e7f654c2a..3de8be95b99 100644 --- a/treeherder/log_parser/tasks.py +++ b/treeherder/log_parser/tasks.py @@ -118,7 +118,9 @@ def post_log_artifacts(job_log): serialized_artifacts = serialize_artifact_json_blobs(artifact_list) store_job_artifacts(serialized_artifacts) job_log.update_status(JobLog.PARSED) - logger.info("Stored artifact for %s %s", job_log.job.repository.name, job_log.job.id) + logger.info( + "Stored artifact for %s %s %s", job_log.job.repository.name, job_log.job.id, job_log.id + ) except Exception as e: logger.error("Failed to store parsed artifact for %s: %s", job_log.id, e) raise diff --git a/treeherder/perf/ingest_data.py b/treeherder/perf/ingest_data.py new file mode 100644 index 00000000000..14b68b7068e --- /dev/null +++ b/treeherder/perf/ingest_data.py @@ -0,0 +1,76 @@ +import json +import logging + +import newrelic.agent + +from treeherder.etl.artifact import serialize_artifact_json_blobs +from treeherder.etl.perf import store_performance_artifact +from treeherder.log_parser.utils import validate_perf_data +from treeherder.model.models import JobLog +from treeherder.utils.http import make_request + +logger = logging.getLogger(__name__) +MAX_JSON_SIZE = 5 * 1024 * 1024 + + +def post_perfherder_artifacts(job_log): + logger.info("Downloading/storing performance data for artifact %s", job_log.id) + + try: + with make_request(job_log.url, stream=False, timeout=60) as response: + download_size_in_bytes = int(response.headers.get("Content-Length", -1)) + newrelic.agent.add_custom_attribute("perf_json_size", download_size_in_bytes) + if download_size_in_bytes > 0 and download_size_in_bytes > MAX_JSON_SIZE: + job_log.update_status(JobLog.SKIPPED_SIZE) + logger.warning( + "Skipping perf json for %s: size %s bytes exceeds limit", + job_log.id, + download_size_in_bytes, + ) + return + raw = response.text + + data = json.loads(raw) + if not data: + logger.warning("Empty performance data for %s", job_log.id) + return + validate_perf_data(data) + perf_list = [data] + + artifact = {"logurl": job_log.url, "performance_data": perf_list} + artifact_list = [ + { + "job_guid": job_log.job.guid, + "name": "performance_data", + "type": "json", + "blob": json.dumps(artifact), + } + ] + except Exception as e: + job_log.update_status(JobLog.FAILED) + logger.error("Failed to download/parse performance data for %s: %s", job_log.id, e) + return + + try: + serialized_artifacts = serialize_artifact_json_blobs(artifact_list) + for artifact in serialized_artifacts: + job_guid = artifact.get("job_guid") + if not job_guid: + logger.error( + "Failed to store performance data for %s: Artifact with no job guid set, skipping", + job_log.id, + ) + continue + + store_performance_artifact(job_log.job, artifact) + + job_log.update_status(JobLog.PARSED) + logger.info( + "Stored performance data for %s %s %s", + job_log.job.repository.name, + job_log.job.id, + job_log.id, + ) + except Exception as e: + logger.error("Failed to store performance data for %s: %s", job_log.id, e) + raise diff --git a/treeherder/perf/tasks.py b/treeherder/perf/tasks.py index 53a0ec49d5f..e6543f9b897 100644 --- a/treeherder/perf/tasks.py +++ b/treeherder/perf/tasks.py @@ -1,12 +1,67 @@ +import logging + import newrelic.agent +from celery.exceptions import SoftTimeLimitExceeded +from treeherder.model.models import Job, JobLog from treeherder.perf.alerts import generate_new_alerts_in_series from treeherder.perf.models import PerformanceSignature from treeherder.workers.task import retryable_task +logger = logging.getLogger(__name__) + @retryable_task(name="generate-alerts", max_retries=10) def generate_alerts(signature_id): newrelic.agent.add_custom_attribute("signature_id", str(signature_id)) signature = PerformanceSignature.objects.get(id=signature_id) generate_new_alerts_in_series(signature) + + +@retryable_task(name="ingest-perfherder-data", max_retries=10) +def ingest_perfherder_data(job_id, job_log_ids): + from treeherder.perf.ingest_data import post_perfherder_artifacts + + newrelic.agent.add_custom_attribute("job_id", str(job_id)) + + job = Job.objects.get(id=job_id) + job_artifacts = JobLog.objects.filter(id__in=job_log_ids, job=job) + + if len(job_log_ids) != len(job_artifacts): + logger.warning( + "Failed to load all expected job ids: %s", ", ".join([str(j) for j in job_log_ids]) + ) + + first_exception = None + for job_artifact in job_artifacts: + job_artifact_name = job_artifact.name.replace("-", "_") + if not job_artifact_name.startswith("perfherder_data"): + continue + + newrelic.agent.add_custom_attribute(f"job_log_{job_artifact.name}_url", job_artifact.url) + logger.info("ingest_perfherder_data for %s", job_artifact.id) + + if job_artifact.status not in (JobLog.PENDING, JobLog.FAILED): + logger.info( + "Skipping ingest_perfherder_data for job %s since artifact already processed. Log Status: %s", + job_artifact.id, + job_artifact.status, + ) + continue + + try: + post_perfherder_artifacts(job_artifact) + except Exception as e: + if isinstance(e, SoftTimeLimitExceeded): + # stop parsing further logs but raise so NewRelic and + # Papertrail will still show output + raise + + if first_exception is None: + first_exception = e + + newrelic.agent.notice_error() + logger.exception("Failed ingesting perfherder JSON for log %s", job_artifact.id) + + if first_exception: + raise first_exception