From b4aa81b4edcb4edae41680f8b7e6a86bc094d721 Mon Sep 17 00:00:00 2001 From: Aritra Basu <24430013+aritra24@users.noreply.github.com> Date: Fri, 14 Feb 2025 22:26:16 +0530 Subject: [PATCH 1/7] WIP: List Backfills --- .../ui/src/pages/Dag/Backfills/Backfills.tsx | 123 ++++++++++++++++++ airflow/ui/src/pages/Dag/Backfills/index.ts | 20 +++ airflow/ui/src/pages/Dag/Dag.tsx | 1 + airflow/ui/src/router.tsx | 2 + 4 files changed, 146 insertions(+) create mode 100644 airflow/ui/src/pages/Dag/Backfills/Backfills.tsx create mode 100644 airflow/ui/src/pages/Dag/Backfills/index.ts diff --git a/airflow/ui/src/pages/Dag/Backfills/Backfills.tsx b/airflow/ui/src/pages/Dag/Backfills/Backfills.tsx new file mode 100644 index 0000000000000..1dac1804a298b --- /dev/null +++ b/airflow/ui/src/pages/Dag/Backfills/Backfills.tsx @@ -0,0 +1,123 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Box, Heading, HStack, Text } from "@chakra-ui/react"; +import type { ColumnDef } from "@tanstack/react-table"; +import { MdOutlineContentCopy } from "react-icons/md"; +import { useParams } from "react-router-dom"; + +import { useBackfillServiceListBackfills } from "openapi/queries"; +import type { BackfillResponse } from "openapi/requests/types.gen"; +import { DataTable } from "src/components/DataTable"; +import { useTableURLState } from "src/components/DataTable/useTableUrlState"; +import { ErrorAlert } from "src/components/ErrorAlert"; +import { pluralize } from "src/utils"; + +const columns: Array> = [ + { + accessorKey: "id", + cell: ({ row }) => ( + + {row.original.id} + { + void navigator.clipboard.writeText(row.original.id.toString()); + }} + /> + + ), + enableSorting: true, + header: () => "ID", + }, + { + accessorKey: "reprocess_behavior", + enableSorting: false, + header: () => "Run Type", + }, + { + accessorKey: "max_active_runs", + enableSorting: false, + header: () => "Run", + }, + { + accessorKey: "date_range", + cell: ({ row }) => ( + + {new Date(row.original.from_date).toUTCString()} - {new Date(row.original.to_date).toUTCString()} + + ), + enableSorting: false, + header: () => "Backfill time range", + }, + { + accessorKey: "duration", + cell: ({ row }) => ( + + {row.original.completed_at === null + ? "" + : new Date( + Math.abs( + new Date(row.original.completed_at).getTime() - new Date(row.original.created_at).getTime(), + ), + ) + .toISOString() + .slice(11, 19)} + + ), + enableSorting: false, + header: () => "Duration", + }, +]; + +export const Backfills = () => { + const { setTableURLState, tableURLState } = useTableURLState(); + // const [searchParams, setSearchParams] = useSearchParams(); + + const { pagination, sorting } = tableURLState; + + const [sort] = sorting; + const orderBy = sort ? `${sort.desc ? "-" : ""}${sort.id}` : "-id"; + + const { dagId = "" } = useParams(); + + const { data, error, isFetching, isLoading } = useBackfillServiceListBackfills({ + dagId, + limit: pagination.pageSize, + offset: pagination.pageIndex * pagination.pageSize, + orderBy, + }); + + return ( + + + + {pluralize("Task", data ? data.total_entries : 0)} + + + + ); +}; diff --git a/airflow/ui/src/pages/Dag/Backfills/index.ts b/airflow/ui/src/pages/Dag/Backfills/index.ts new file mode 100644 index 0000000000000..bf74d95d55056 --- /dev/null +++ b/airflow/ui/src/pages/Dag/Backfills/index.ts @@ -0,0 +1,20 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export { Backfills } from "./Backfills"; diff --git a/airflow/ui/src/pages/Dag/Dag.tsx b/airflow/ui/src/pages/Dag/Dag.tsx index 4f5372689ab27..5bc8176bedad3 100644 --- a/airflow/ui/src/pages/Dag/Dag.tsx +++ b/airflow/ui/src/pages/Dag/Dag.tsx @@ -33,6 +33,7 @@ const tabs = [ { icon: , label: "Overview", value: "" }, { icon: , label: "Runs", value: "runs" }, { icon: , label: "Tasks", value: "tasks" }, + { label: "Backfills", value: "backfills" }, { icon: , label: "Events", value: "events" }, { icon: , label: "Code", value: "code" }, ]; diff --git a/airflow/ui/src/router.tsx b/airflow/ui/src/router.tsx index c33feb6bbefd4..d34f3a068f11d 100644 --- a/airflow/ui/src/router.tsx +++ b/airflow/ui/src/router.tsx @@ -47,6 +47,7 @@ import { TaskInstances } from "src/pages/TaskInstances"; import { Variables } from "src/pages/Variables"; import { XCom } from "src/pages/XCom"; +import { Backfills } from "./pages/Dag/Backfills"; import { queryClient } from "./queryClient"; export const routerConfig = [ @@ -109,6 +110,7 @@ export const routerConfig = [ { element: , index: true }, { element: , path: "runs" }, { element: , path: "tasks" }, + { element: , path: "backfills" }, { element: , path: "events" }, { element: , path: "code" }, ], From e5da76e6ab539546d8c8eb4bcb750084f0b4afe7 Mon Sep 17 00:00:00 2001 From: Aritra Basu <24430013+aritra24@users.noreply.github.com> Date: Fri, 21 Feb 2025 15:15:19 +0530 Subject: [PATCH 2/7] Resolve review comments --- .../ui/src/pages/Dag/Backfills/Backfills.tsx | 22 +++++++++++++------ airflow/ui/src/router.tsx | 2 +- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/airflow/ui/src/pages/Dag/Backfills/Backfills.tsx b/airflow/ui/src/pages/Dag/Backfills/Backfills.tsx index 1dac1804a298b..dd86dec98e42a 100644 --- a/airflow/ui/src/pages/Dag/Backfills/Backfills.tsx +++ b/airflow/ui/src/pages/Dag/Backfills/Backfills.tsx @@ -26,6 +26,8 @@ import type { BackfillResponse } from "openapi/requests/types.gen"; import { DataTable } from "src/components/DataTable"; import { useTableURLState } from "src/components/DataTable/useTableUrlState"; import { ErrorAlert } from "src/components/ErrorAlert"; +import Time from "src/components/Time"; +import { reprocessBehaviors } from "src/constants/reprocessBehaviourParams"; import { pluralize } from "src/utils"; const columns: Array> = [ @@ -46,19 +48,27 @@ const columns: Array> = [ }, { accessorKey: "reprocess_behavior", + cell: ({ row }) => ( + + { + reprocessBehaviors.find((rb: { value: string }) => rb.value === row.original.reprocess_behavior) + ?.label + } + + ), enableSorting: false, - header: () => "Run Type", + header: () => "Reprocess Behavior", }, { accessorKey: "max_active_runs", enableSorting: false, - header: () => "Run", + header: () => "Max Active Runs", }, { accessorKey: "date_range", cell: ({ row }) => ( - {new Date(row.original.from_date).toUTCString()} - {new Date(row.original.to_date).toUTCString()} + ), enableSorting: false, @@ -86,7 +96,6 @@ const columns: Array> = [ export const Backfills = () => { const { setTableURLState, tableURLState } = useTableURLState(); - // const [searchParams, setSearchParams] = useSearchParams(); const { pagination, sorting } = tableURLState; @@ -106,15 +115,14 @@ export const Backfills = () => { - {pluralize("Task", data ? data.total_entries : 0)} + {pluralize("Backfill", data ? data.total_entries : 0)} diff --git a/airflow/ui/src/router.tsx b/airflow/ui/src/router.tsx index d34f3a068f11d..f291393a985a4 100644 --- a/airflow/ui/src/router.tsx +++ b/airflow/ui/src/router.tsx @@ -25,6 +25,7 @@ import { BaseLayout } from "src/layouts/BaseLayout"; import { DagsLayout } from "src/layouts/DagsLayout"; import { AssetsList } from "src/pages/AssetsList"; import { Dag } from "src/pages/Dag"; +import { Backfills } from "src/pages/Dag/Backfills"; import { Code } from "src/pages/Dag/Code"; import { Overview } from "src/pages/Dag/Overview"; import { Tasks } from "src/pages/Dag/Tasks"; @@ -47,7 +48,6 @@ import { TaskInstances } from "src/pages/TaskInstances"; import { Variables } from "src/pages/Variables"; import { XCom } from "src/pages/XCom"; -import { Backfills } from "./pages/Dag/Backfills"; import { queryClient } from "./queryClient"; export const routerConfig = [ From d8ecd3623ec02156629bf4f3e15a19b2b07fa754 Mon Sep 17 00:00:00 2001 From: Aritra Basu <24430013+aritra24@users.noreply.github.com> Date: Sun, 23 Feb 2025 10:36:52 +0530 Subject: [PATCH 3/7] Removed id and split dates into from-to --- .../ui/src/pages/Dag/Backfills/Backfills.tsx | 52 +++++++------------ 1 file changed, 19 insertions(+), 33 deletions(-) diff --git a/airflow/ui/src/pages/Dag/Backfills/Backfills.tsx b/airflow/ui/src/pages/Dag/Backfills/Backfills.tsx index dd86dec98e42a..36876d1ff9d52 100644 --- a/airflow/ui/src/pages/Dag/Backfills/Backfills.tsx +++ b/airflow/ui/src/pages/Dag/Backfills/Backfills.tsx @@ -16,9 +16,8 @@ * specific language governing permissions and limitations * under the License. */ -import { Box, Heading, HStack, Text } from "@chakra-ui/react"; +import { Box, Heading, Text } from "@chakra-ui/react"; import type { ColumnDef } from "@tanstack/react-table"; -import { MdOutlineContentCopy } from "react-icons/md"; import { useParams } from "react-router-dom"; import { useBackfillServiceListBackfills } from "openapi/queries"; @@ -28,24 +27,9 @@ import { useTableURLState } from "src/components/DataTable/useTableUrlState"; import { ErrorAlert } from "src/components/ErrorAlert"; import Time from "src/components/Time"; import { reprocessBehaviors } from "src/constants/reprocessBehaviourParams"; -import { pluralize } from "src/utils"; +import { getDuration, pluralize } from "src/utils"; const columns: Array> = [ - { - accessorKey: "id", - cell: ({ row }) => ( - - {row.original.id} - { - void navigator.clipboard.writeText(row.original.id.toString()); - }} - /> - - ), - enableSorting: true, - header: () => "ID", - }, { accessorKey: "reprocess_behavior", cell: ({ row }) => ( @@ -57,40 +41,42 @@ const columns: Array> = [ ), enableSorting: false, - header: () => "Reprocess Behavior", + header: "Reprocess Behavior", }, { accessorKey: "max_active_runs", enableSorting: false, - header: () => "Max Active Runs", + header: "Max Active Runs", + }, + { + accessorKey: "date_from", + cell: ({ row }) => ( + + + ), + enableSorting: false, + header: "From", }, { - accessorKey: "date_range", + accessorKey: "date_to", cell: ({ row }) => ( - ), enableSorting: false, - header: () => "Backfill time range", + header: "To", }, { accessorKey: "duration", cell: ({ row }) => ( - {row.original.completed_at === null - ? "" - : new Date( - Math.abs( - new Date(row.original.completed_at).getTime() - new Date(row.original.created_at).getTime(), - ), - ) - .toISOString() - .slice(11, 19)} + {row.original.completed_at ?? getDuration(row.original.created_at, row.original.completed_at)} ), enableSorting: false, - header: () => "Duration", + header: "Duration", }, ]; From 0e2fa7da6fc0b8f24682279cd2d1935e29284b8d Mon Sep 17 00:00:00 2001 From: Aritra Basu <24430013+aritra24@users.noreply.github.com> Date: Sun, 23 Feb 2025 11:01:47 +0530 Subject: [PATCH 4/7] Updated duration --- airflow/ui/src/pages/Dag/Backfills/Backfills.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/airflow/ui/src/pages/Dag/Backfills/Backfills.tsx b/airflow/ui/src/pages/Dag/Backfills/Backfills.tsx index 36876d1ff9d52..874c304b5a322 100644 --- a/airflow/ui/src/pages/Dag/Backfills/Backfills.tsx +++ b/airflow/ui/src/pages/Dag/Backfills/Backfills.tsx @@ -72,7 +72,9 @@ const columns: Array> = [ accessorKey: "duration", cell: ({ row }) => ( - {row.original.completed_at ?? getDuration(row.original.created_at, row.original.completed_at)} + {row.original.completed_at === null + ? "" + : getDuration(row.original.created_at, row.original.completed_at)} ), enableSorting: false, From c6f66e63634f01b3e70c3bcfcdfd1518d857c556 Mon Sep 17 00:00:00 2001 From: Aritra Basu <24430013+aritra24@users.noreply.github.com> Date: Sun, 23 Feb 2025 11:19:31 +0530 Subject: [PATCH 5/7] Append s to duration field --- airflow/ui/src/pages/Dag/Backfills/Backfills.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/ui/src/pages/Dag/Backfills/Backfills.tsx b/airflow/ui/src/pages/Dag/Backfills/Backfills.tsx index 874c304b5a322..e99c7e3a8f488 100644 --- a/airflow/ui/src/pages/Dag/Backfills/Backfills.tsx +++ b/airflow/ui/src/pages/Dag/Backfills/Backfills.tsx @@ -74,7 +74,7 @@ const columns: Array> = [ {row.original.completed_at === null ? "" - : getDuration(row.original.created_at, row.original.completed_at)} + : `${getDuration(row.original.created_at, row.original.completed_at)}s`} ), enableSorting: false, From 30331499aeb777ae58ff5b602779d63c36a954a3 Mon Sep 17 00:00:00 2001 From: Aritra Basu <24430013+aritra24@users.noreply.github.com> Date: Wed, 26 Feb 2025 18:01:44 +0530 Subject: [PATCH 6/7] Remove sorting and adding new columns --- .../ui/src/pages/Dag/Backfills/Backfills.tsx | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/airflow/ui/src/pages/Dag/Backfills/Backfills.tsx b/airflow/ui/src/pages/Dag/Backfills/Backfills.tsx index e99c7e3a8f488..30dc9649b2350 100644 --- a/airflow/ui/src/pages/Dag/Backfills/Backfills.tsx +++ b/airflow/ui/src/pages/Dag/Backfills/Backfills.tsx @@ -48,6 +48,26 @@ const columns: Array> = [ enableSorting: false, header: "Max Active Runs", }, + { + accessorKey: "created_at", + cell: ({ row }) => ( + + + ), + enableSorting: false, + header: "Created at", + }, + { + accessorKey: "completed_at", + cell: ({ row }) => ( + + + ), + enableSorting: false, + header: "Completed at", + }, { accessorKey: "date_from", cell: ({ row }) => ( @@ -85,10 +105,7 @@ const columns: Array> = [ export const Backfills = () => { const { setTableURLState, tableURLState } = useTableURLState(); - const { pagination, sorting } = tableURLState; - - const [sort] = sorting; - const orderBy = sort ? `${sort.desc ? "-" : ""}${sort.id}` : "-id"; + const { pagination } = tableURLState; const { dagId = "" } = useParams(); @@ -96,7 +113,6 @@ export const Backfills = () => { dagId, limit: pagination.pageSize, offset: pagination.pageIndex * pagination.pageSize, - orderBy, }); return ( From 7b7e6c53d9d9c8e1abab3485c48c8a765c9cc724 Mon Sep 17 00:00:00 2001 From: Aritra Basu <24430013+aritra24@users.noreply.github.com> Date: Wed, 26 Feb 2025 18:25:32 +0530 Subject: [PATCH 7/7] Updated icon in dag --- airflow/ui/src/pages/Dag/Dag.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/ui/src/pages/Dag/Dag.tsx b/airflow/ui/src/pages/Dag/Dag.tsx index 5bc8176bedad3..da86ada2c93d3 100644 --- a/airflow/ui/src/pages/Dag/Dag.tsx +++ b/airflow/ui/src/pages/Dag/Dag.tsx @@ -17,7 +17,7 @@ * under the License. */ import { ReactFlowProvider } from "@xyflow/react"; -import { FiBarChart, FiCode } from "react-icons/fi"; +import { FiBarChart, FiCode, FiRotateCcw } from "react-icons/fi"; import { LuChartColumn } from "react-icons/lu"; import { MdOutlineEventNote } from "react-icons/md"; import { useParams } from "react-router-dom"; @@ -33,7 +33,7 @@ const tabs = [ { icon: , label: "Overview", value: "" }, { icon: , label: "Runs", value: "runs" }, { icon: , label: "Tasks", value: "tasks" }, - { label: "Backfills", value: "backfills" }, + { icon: , label: "Backfills", value: "backfills" }, { icon: , label: "Events", value: "events" }, { icon: , label: "Code", value: "code" }, ];