Skip to content

update migration file for notekeeping purpose #1009

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
icecrasher321 merged 1 commit into from
Aug 18, 2025
Merged

update migration file for notekeeping purpose #1009

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 27 additions & 11 deletions apps/sim/db/migrations/0076_damp_vector.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
-- One-shot data migration to create/populate execution_data & cost, then drop legacy columns
-- Safe on reruns and across differing prior schemas

-- Note: Depending on runner timeouts, might have to be run manually
-- 1) Ensure execution_data exists (prefer rename if only metadata exists)
DO $$
BEGIN
Expand All @@ -24,19 +24,35 @@ DO $$
DECLARE
v_batch_size integer := 500; -- keep batches small to avoid timeouts/spills
v_rows_updated integer := 0;
v_rows_selected integer := 0;
v_last_id text := '';
v_last_created_at timestamp := '1970-01-01 00:00:00';
BEGIN
-- modest per-statement timeout; adjust based on observed per-batch runtime
PERFORM set_config('statement_timeout', '180s', true);
LOOP
WITH RECURSIVE candidate AS (
SELECT id
FROM workflow_execution_logs
WHERE cost IS NULL
ORDER BY id
LIMIT v_batch_size
),
CREATE TEMP TABLE IF NOT EXISTS _tmp_candidate_ids(id text, created_at timestamp) ON COMMIT DROP;
TRUNCATE _tmp_candidate_ids;
INSERT INTO _tmp_candidate_ids(id, created_at)
SELECT id, created_at
FROM workflow_execution_logs
WHERE (created_at, id) > (v_last_created_at, v_last_id) AND cost IS NULL
ORDER BY created_at, id
LIMIT v_batch_size;

SELECT COUNT(*) INTO v_rows_selected FROM _tmp_candidate_ids;
EXIT WHEN v_rows_selected = 0;
SELECT created_at, id
INTO v_last_created_at, v_last_id
FROM _tmp_candidate_ids
ORDER BY created_at DESC, id DESC
LIMIT 1;

WITH RECURSIVE
spans AS (
SELECT l.id, s.span
FROM workflow_execution_logs l
JOIN candidate c ON c.id = l.id
JOIN _tmp_candidate_ids c ON c.id = l.id
LEFT JOIN LATERAL jsonb_array_elements(
COALESCE(
CASE
Expand Down Expand Up @@ -105,7 +121,7 @@ BEGIN
NULLIF((l.execution_data->'tokenBreakdown'->>'prompt')::numeric, 0) AS prompt,
NULLIF((l.execution_data->'tokenBreakdown'->>'completion')::numeric, 0) AS completion
FROM workflow_execution_logs l
JOIN candidate c ON c.id = l.id
JOIN _tmp_candidate_ids c ON c.id = l.id
)
UPDATE workflow_execution_logs AS l
SET cost = jsonb_strip_nulls(
Expand Down Expand Up @@ -133,7 +149,7 @@ BEGIN
WHERE l.id = agg.id;

GET DIAGNOSTICS v_rows_updated = ROW_COUNT;
EXIT WHEN v_rows_updated = 0; -- no more rows to backfill
-- continue advancing by id until no more rows are selected
END LOOP;
END $$;--> statement-breakpoint

Expand Down