Merge pull request #109 from nyaruka/update-fields

Add flow run path_nodes and path_times fields
nyaruka · Dec 5, 2024 · 81999dc · 81999dc
2 parents fb5ac37 + 629c2ef
commit 81999dc
Show file tree

Hide file tree

Showing 4 changed files with 61 additions and 43 deletions.
diff --git a/archives/archives_test.go b/archives/archives_test.go
@@ -225,9 +225,9 @@ func TestCreateRunArchive(t *testing.T) {
 	assert.NoError(t, err)
 
 	// should have two record
-	assert.Equal(t, 2, task.RecordCount)
-	assert.Equal(t, int64(458), task.Size)
-	assert.Equal(t, "7220a13c19f5b6065e7d4c419c114635", task.Hash)
+	assert.Equal(t, 3, task.RecordCount)
+	assert.Equal(t, int64(578), task.Size)
+	assert.Equal(t, "cd8ce82019986ac1f4ec1482aac7bca0", task.Hash)
 	assertArchiveFile(t, task, "runs1.jsonl")
 
 	DeleteArchiveFile(task)
@@ -459,7 +459,7 @@ func TestArchiveOrgRuns(t *testing.T) {
 		time.Date(2020, 2, 1, 0, 0, 0, 0, time.UTC),
 	)
 	assert.NoError(t, err)
-	assert.Equal(t, 3, count)
+	assert.Equal(t, 4, count)
 
 	// more recent run unaffected (even though it was parent)
 	count, err = getCountInRange(
@@ -511,7 +511,7 @@ func TestArchiveActiveOrgs(t *testing.T) {
 		"archiver.msgs_archives_failed":  {0},
 		"archiver.msgs_rollups_created":  {3},
 		"archiver.msgs_rollups_failed":   {0},
-		"archiver.runs_records_archived": {4},
+		"archiver.runs_records_archived": {5},
 		"archiver.runs_archives_created": {41},
 		"archiver.runs_archives_failed":  {1},
 		"archiver.runs_rollups_created":  {3},

diff --git a/archives/runs.go b/archives/runs.go
@@ -24,36 +24,47 @@ const (
 const sqlLookupRuns = `
 SELECT rec.uuid, rec.exited_on, row_to_json(rec)
 FROM (
-   SELECT
-	 fr.id as id,
-	 fr.uuid as uuid,
-     row_to_json(flow_struct) AS flow,
-     row_to_json(contact_struct) AS contact,
-     fr.responded,
-     (SELECT coalesce(jsonb_agg(path_data), '[]'::jsonb) from (
-		SELECT path_row ->> 'node_uuid' AS node, (path_row ->> 'arrived_on')::timestamptz as time
-		FROM jsonb_array_elements(fr.path::jsonb) AS path_row LIMIT 500) as path_data
-     ) as path,
-     (SELECT coalesce(jsonb_object_agg(values_data.key, values_data.value), '{}'::jsonb) from (
-		SELECT key, jsonb_build_object('name', value -> 'name', 'value', value -> 'value', 'input', value -> 'input', 'time', (value -> 'created_on')::text::timestamptz, 'category', value -> 'category', 'node', value -> 'node_uuid') as value
-		FROM jsonb_each(fr.results::jsonb)) AS values_data
-	 ) as values,
-     fr.created_on,
-     fr.modified_on,
-	 fr.exited_on,
-     CASE
-        WHEN status = 'C' THEN 'completed'
-        WHEN status = 'I' THEN 'interrupted'
-        WHEN status = 'X' THEN 'expired'
-        WHEN status = 'F' THEN 'failed'
-        ELSE NULL
-	 END as exit_type
-
-   FROM flows_flowrun fr
-   JOIN LATERAL (SELECT uuid, name FROM flows_flow WHERE flows_flow.id = fr.flow_id) AS flow_struct ON True
-   JOIN LATERAL (SELECT uuid, name FROM contacts_contact cc WHERE cc.id = fr.contact_id) AS contact_struct ON True
-   WHERE fr.org_id = $1 AND fr.modified_on >= $2 AND fr.modified_on < $3
-   ORDER BY fr.modified_on ASC, id ASC
+	SELECT
+		fr.id as id,
+		fr.uuid as uuid,
+		row_to_json(flow_struct) AS flow,
+		row_to_json(contact_struct) AS contact,
+		fr.responded,
+		(SELECT CASE
+			WHEN (fr.path_nodes IS NOT NULL AND fr.path_times IS NOT NULL)
+			THEN (
+				SELECT coalesce(jsonb_agg(path_data), '[]'::jsonb)
+				FROM (
+					SELECT node, time
+					FROM unnest(fr.path_nodes::text[] , fr.path_times::timestamptz[]) x(node, time) LIMIT 500)
+					as path_data)
+			ELSE (
+				SELECT coalesce(jsonb_agg(path_data), '[]'::jsonb)
+				FROM (
+					SELECT path_row ->> 'node_uuid' AS node, (path_row ->> 'arrived_on')::timestamptz as time
+					FROM jsonb_array_elements(fr.path::jsonb) AS path_row LIMIT 500)
+					as path_data)
+		END as path),
+		(SELECT coalesce(jsonb_object_agg(values_data.key, values_data.value), '{}'::jsonb) from (
+			SELECT key, jsonb_build_object('name', value -> 'name', 'value', value -> 'value', 'input', value -> 'input', 'time', (value -> 'created_on')::text::timestamptz, 'category', value -> 'category', 'node', value -> 'node_uuid') as value
+			FROM jsonb_each(fr.results::jsonb)) AS values_data
+		) as values,
+		fr.created_on,
+		fr.modified_on,
+		fr.exited_on,
+		CASE
+			WHEN status = 'C' THEN 'completed'
+			WHEN status = 'I' THEN 'interrupted'
+			WHEN status = 'X' THEN 'expired'
+			WHEN status = 'F' THEN 'failed'
+			ELSE NULL
+		END as exit_type
+
+	FROM flows_flowrun fr
+	JOIN LATERAL (SELECT uuid, name FROM flows_flow WHERE flows_flow.id = fr.flow_id) AS flow_struct ON True
+	JOIN LATERAL (SELECT uuid, name FROM contacts_contact cc WHERE cc.id = fr.contact_id) AS contact_struct ON True
+	WHERE fr.org_id = $1 AND fr.modified_on >= $2 AND fr.modified_on < $3
+	ORDER BY fr.modified_on ASC, id ASC
 ) as rec;`
 
 // writeRunRecords writes the runs in the archive's date range to the passed in writer

diff --git a/archives/testdata/runs1.jsonl b/archives/testdata/runs1.jsonl
@@ -1,2 +1,3 @@
 {"id":1,"uuid":"4ced1260-9cfe-4b7f-81dd-b637108f15b9","flow":{"uuid":"6639286a-9120-45d4-aa39-03ae3942a4a6","name":"Flow 1"},"contact":{"uuid":"3e814add-e614-41f7-8b5d-a07f670a698f","name":"Ajodinabiff Dane"},"responded":true,"path":[],"values":{},"created_on":"2017-08-12T19:11:59.890662+00:00","modified_on":"2017-08-12T19:11:59.890662+00:00","exited_on":"2017-08-12T19:11:59.890662+00:00","exit_type":"completed"}
 {"id":2,"uuid":"7d68469c-0494-498a-bdf3-bac68321fd6d","flow":{"uuid":"6639286a-9120-45d4-aa39-03ae3942a4a6","name":"Flow 1"},"contact":{"uuid":"3e814add-e614-41f7-8b5d-a07f670a698f","name":"Ajodinabiff Dane"},"responded":true,"path":[{"node": "10896d63-8df7-4022-88dd-a9d93edf355b", "time": "2017-08-12T13:07:24.049815+00:00"}],"values":{"agree": {"name": "Do you agree?", "node": "a0434c54-3e26-4eb0-bafc-46cdeaf435ac", "time": "2017-05-03T12:25:21.714339+00:00", "input": "A", "value": "A", "category": "Strongly agree"}},"created_on":"2017-08-12T19:11:59.890662+00:00","modified_on":"2017-08-12T19:11:59.890662+00:00","exited_on":"2017-08-12T19:11:59.890662+00:00","exit_type":"completed"}
+{"id":8,"uuid":"0c54f7b9-875b-4385-ae85-fb9e84f4b3d6","flow":{"uuid":"6639286a-9120-45d4-aa39-03ae3942a4a6","name":"Flow 1"},"contact":{"uuid":"3e814add-e614-41f7-8b5d-a07f670a698f","name":"Ajodinabiff Dane"},"responded":true,"path":[{"node": "d1a55403-83a3-42f1-b24c-6446bb18e6a6", "time": "2017-08-12T13:07:25.049815+00:00"}, {"node": "ce1bdc68-5c16-452e-a0ce-52440fc7bb9a", "time": "2017-08-12T13:07:26.049815+00:00"}, {"node": "1640b40d-63ed-43b0-a443-097ce8bb8710", "time": "2017-08-12T13:07:27.049815+00:00"}],"values":{"agree": {"name": "Do you agree?", "node": "a0434c54-3e26-4eb0-bafc-46cdeaf435ac", "time": "2017-05-03T12:25:21.714339+00:00", "input": "A", "value": "A", "category": "Strongly agree"}},"created_on":"2017-08-12T19:11:59.890662+00:00","modified_on":"2017-08-12T19:11:59.890662+00:00","exited_on":"2017-08-12T19:11:59.890662+00:00","exit_type":"completed"}
diff --git a/testdb.sql b/testdb.sql
@@ -209,6 +209,8 @@ CREATE TABLE flows_flowrun (
     start_id integer NULL REFERENCES flows_flowstart(id),
     results text NOT NULL,
     path text NOT NULL,
+    path_nodes uuid[] NULL,
+    path_times timestamp with time zone[] NULL,
     created_on timestamp with time zone NOT NULL,
     modified_on timestamp with time zone NOT NULL,
     exited_on timestamp with time zone NULL,
@@ -335,23 +337,27 @@ INSERT INTO flows_flowstart_groups(flowstart_id, contactgroup_id) VALUES
 INSERT INTO flows_flowstart_calls(flowstart_id, call_id) VALUES 
 (1, 1);
 
-INSERT INTO flows_flowrun(id, uuid, org_id, responded, contact_id, flow_id, results, path, created_on, modified_on, exited_on, status, start_id) VALUES
-(1, '4ced1260-9cfe-4b7f-81dd-b637108f15b9', 2, TRUE, 6, 1, '{}', '[]', '2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00', 'C', 1),
+INSERT INTO flows_flowrun(id, uuid, org_id, responded, contact_id, flow_id, results, path, path_nodes, path_times, created_on, modified_on, exited_on, status, start_id) VALUES
+(1, '4ced1260-9cfe-4b7f-81dd-b637108f15b9', 2, TRUE, 6, 1, '{}', '[]', NULL, NULL, '2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00', 'C', 1),
 (2, '7d68469c-0494-498a-bdf3-bac68321fd6d', 2, TRUE, 6, 1, 
 '{"agree": {"category": "Strongly agree", "node_uuid": "a0434c54-3e26-4eb0-bafc-46cdeaf435ac", "name": "Do you agree?", "value": "A", "created_on": "2017-05-03T12:25:21.714339+00:00", "input": "A"}}',
 '[{"uuid": "c3d0b417-db75-417c-8050-33776ec8f620", "node_uuid": "10896d63-8df7-4022-88dd-a9d93edf355b", "arrived_on": "2017-08-12T15:07:24.049815+02:00", "exit_uuid": "2f890507-2ad2-4bd1-92fc-0ca031155fca"}]', 
-'2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00', 'C', NULL),
+NULL, NULL, '2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00', 'C', NULL),
 (3, 'de782b35-a398-46ed-8550-34c66053841b', 3, TRUE, 7, 2, 
 '{"agree": {"category": "Strongly agree", "node_uuid": "084c8cf1-715d-4d0a-b38d-a616ed74e638", "name": "Agree", "value": "A", "created_on": "2017-05-03T12:25:21.714339+00:00", "input": "A"}, "confirm_agree": {"category": "Confirmed Strongly agree", "node_uuid": "a0434c54-3e26-4eb0-bafc-46cdeaf435ab", "name": "Do you agree?", "value": "A", "created_on": "2017-05-03T12:25:21.714339+00:00", "input": "A"}}',
 '[{"uuid": "600ac5b4-4895-4161-ad97-6e2f1bb48bcb", "node_uuid": "accbc6e2-b0df-46cd-9a76-bff0fdf4d753", "arrived_on": "2017-08-12T15:07:24.049815+02:00", "exit_uuid": "8249e2dc-c893-4200-b6d2-398d07a459bc"}]', 
-'2017-08-10 21:11:59.890662+02:00','2017-08-10 21:11:59.890662+02:00','2017-08-10 21:11:59.890662+02:00', 'C', NULL),
+NULL, NULL, '2017-08-10 21:11:59.890662+02:00','2017-08-10 21:11:59.890662+02:00','2017-08-10 21:11:59.890662+02:00', 'C', NULL),
 (4, '329a5d24-64fc-479c-8d24-9674c9b46530', 3, TRUE, 7, 2, 
 '{"agree": {"category": "Disagree", "node_uuid": "084c8cf1-715d-4d0a-b38d-a616ed74e638", "name": "Agree", "value": "B", "created_on": "2017-10-10T12:25:21.714339+00:00", "input": "B"}}',
 '[{"uuid": "babf4fc8-e12c-4bb9-a9dd-61178a118b5a", "node_uuid": "accbc6e2-b0df-46cd-9a76-bff0fdf4d753", "arrived_on": "2017-10-12T15:07:24.049815+02:00", "exit_uuid": "8249e2dc-c893-4200-b6d2-398d07a459bc"}]', 
-'2017-10-10 21:11:59.890662+02:00','2017-10-10 21:11:59.890662+02:00','2017-10-10 21:11:59.890662+02:00', 'C', NULL),
-(5, 'abed67d2-06b8-4749-8bb9-ecda037b673b', 3, TRUE, 7, 2, '{}', '[]', '2017-10-10 21:11:59.890663+02:00','2017-10-10 21:11:59.890662+02:00','2017-10-10 21:11:59.890662+02:00', 'C', NULL),
-(6, '6262eefe-a6e9-4201-9b76-a7f25e3b7f29', 3, TRUE, 7, 2, '{}', '[]', '2017-12-12 21:11:59.890662+02:00','2017-12-12 21:11:59.890662+02:00','2017-12-12 21:11:59.890662+02:00', 'C', NULL),
-(7, '6c0d7db9-076b-4edc-ab4b-38576ae394fc', 2, TRUE, 7, 2, '{}', '[]', '2017-08-13 13:11:59.890662+02:00','2017-08-14 16:11:59.890662+02:00', NULL, 'W', NULL);
+NULL, NULL, '2017-10-10 21:11:59.890662+02:00','2017-10-10 21:11:59.890662+02:00','2017-10-10 21:11:59.890662+02:00', 'C', NULL),
+(5, 'abed67d2-06b8-4749-8bb9-ecda037b673b', 3, TRUE, 7, 2, '{}', '[]', NULL, NULL, '2017-10-10 21:11:59.890663+02:00','2017-10-10 21:11:59.890662+02:00','2017-10-10 21:11:59.890662+02:00', 'C', NULL),
+(6, '6262eefe-a6e9-4201-9b76-a7f25e3b7f29', 3, TRUE, 7, 2, '{}', '[]', NULL, NULL, '2017-12-12 21:11:59.890662+02:00','2017-12-12 21:11:59.890662+02:00','2017-12-12 21:11:59.890662+02:00', 'C', NULL),
+(7, '6c0d7db9-076b-4edc-ab4b-38576ae394fc', 2, TRUE, 7, 2, '{}', '[]', NULL, NULL, '2017-08-13 13:11:59.890662+02:00','2017-08-14 16:11:59.890662+02:00', NULL, 'W', NULL),
+(8, '0c54f7b9-875b-4385-ae85-fb9e84f4b3d6', 2, TRUE, 6, 1,
+'{"agree": {"category": "Strongly agree", "node_uuid": "a0434c54-3e26-4eb0-bafc-46cdeaf435ac", "name": "Do you agree?", "value": "A", "created_on": "2017-05-03T12:25:21.714339+00:00", "input": "A"}}',
+'[]',
+'{"d1a55403-83a3-42f1-b24c-6446bb18e6a6","ce1bdc68-5c16-452e-a0ce-52440fc7bb9a","1640b40d-63ed-43b0-a443-097ce8bb8710"}', '{"2017-08-12T15:07:25.049815+02:00","2017-08-12T15:07:26.049815+02:00","2017-08-12T15:07:27.049815+02:00"}', '2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00', 'C', NULL);
 
 -- update run #5 to have a path longer than 500 steps
 UPDATE flows_flowrun SET path = s.path FROM (