Add Rendered k8s pod spec tab to ti details view

airflow/www/package.json

@@ -51,6 +51,7 @@
     "@testing-library/jest-dom": "^5.16.0",
     "@testing-library/react": "^13.0.0",
     "@types/color": "^3.0.3",
+    "@types/json-to-pretty-yaml": "^1.2.1",
     "@types/react": "^18.0.12",
     "@types/react-dom": "^18.0.5",
     "@types/react-syntax-highlighter": "^15.5.6",
@@ -126,11 +127,12 @@
     "framer-motion": "^6.0.0",
     "jquery": ">=3.5.0",
     "jshint": "^2.13.4",
+    "json-to-pretty-yaml": "^1.2.2",
     "lodash": "^4.17.21",
     "moment-timezone": "^0.5.43",
     "react": "^18.0.0",
     "react-dom": "^18.0.0",
-    "react-icons": "^4.9.0",
+    "react-icons": "^5.1.0",
     "react-json-view": "^1.21.3",
     "react-markdown": "^8.0.4",
     "react-query": "^3.39.1",

airflow/www/static/js/api/index.ts

@@ -53,6 +53,7 @@ import { useTaskXcomEntry, useTaskXcomCollection } from "./useTaskXcom";
 import useEventLogs from "./useEventLogs";
 import useCalendarData from "./useCalendarData";
 import useCreateDatasetEvent from "./useCreateDatasetEvent";
+import useRenderedK8s from "./useRenderedK8s";
 
 axios.interceptors.request.use((config) => {
   config.paramsSerializer = {
@@ -102,4 +103,5 @@ export {
   useEventLogs,
   useCalendarData,
   useCreateDatasetEvent,
+  useRenderedK8s,
 };

airflow/www/static/js/api/useRenderedK8s.ts

@@ -0,0 +1,43 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import axios, { AxiosResponse } from "axios";
+import { useQuery } from "react-query";
+
+import { getMetaValue } from "src/utils";
+
+const url = getMetaValue("rendered_k8s_data_url");
+
+const useRenderedK8s = (
+  runId: string | null,
+  taskId: string | null,
+  mapIndex?: number
+) =>
+  useQuery(
+    ["rendered_k8s", runId, taskId, mapIndex],
+    async () =>
+      axios.get<AxiosResponse, any>(url, {
+        params: { run_id: runId, task_id: taskId, map_index: mapIndex },
+      }),
+    {
+      enabled: !!runId && !!taskId,
+    }
+  );
+
+export default useRenderedK8s;

airflow/www/static/js/dag/details/index.tsx

@@ -45,7 +45,7 @@ import {
   MdPlagiarism,
   MdEvent,
 } from "react-icons/md";
-import { BiBracket } from "react-icons/bi";
+import { BiBracket, BiLogoKubernetes } from "react-icons/bi";
 import URLSearchParamsWrapper from "src/utils/URLSearchParamWrapper";
 
 import Header from "./Header";
@@ -68,6 +68,7 @@ import TaskDetails from "./task";
 import AuditLog from "./AuditLog";
 import RunDuration from "./dag/RunDuration";
 import Calendar from "./dag/Calendar";
+import RenderedK8s from "./taskInstance/RenderedK8s";
 
 const dagId = getMetaValue("dag_id")!;
 
@@ -79,6 +80,8 @@ interface Props {
   ganttScrollRef: React.RefObject<HTMLDivElement>;
 }
 
+const isK8sExecutor = getMetaValue("k8s_or_k8scelery_executor") === "True";
+
 const tabToIndex = (tab?: string) => {
   switch (tab) {
     case "graph":
@@ -96,6 +99,8 @@ const tabToIndex = (tab?: string) => {
     case "xcom":
     case "calendar":
       return 6;
+    case "rendered_k8s":
+      return 7;
     case "details":
     default:
       return 0;
@@ -135,6 +140,9 @@ const indexToTab = (
       if (!runId && !taskId) return "calendar";
       if (isTaskInstance) return "xcom";
       return undefined;
+    case 7:
+      if (isTaskInstance && isK8sExecutor) return "rendered_k8s";
+      return undefined;
     default:
       return undefined;
   }
@@ -360,6 +368,14 @@ const Details = ({
               </Text>
             </Tab>
           )}
+          {isTaskInstance && isK8sExecutor && (
+            <Tab>
+              <BiLogoKubernetes size={16} />
+              <Text as="strong" ml={1}>
+                K8s Pod Spec
+              </Text>
+            </Tab>
+          )}
           {/* Match the styling of a tab but its actually a button */}
           {!!taskId && !!runId && (
             <Button
@@ -484,6 +500,11 @@ const Details = ({
               />
             </TabPanel>
           )}
+          {isTaskInstance && isK8sExecutor && (
+            <TabPanel height="100%">
+              <RenderedK8s />
+            </TabPanel>
+          )}
         </TabPanels>
       </Tabs>
     </Flex>

airflow/www/static/js/dag/details/taskInstance/Nav.tsx

@@ -26,9 +26,7 @@ import type { Task } from "src/types";
 import URLSearchParamsWrapper from "src/utils/URLSearchParamWrapper";
 
 const dagId = getMetaValue("dag_id");
-const isK8sExecutor = getMetaValue("k8s_or_k8scelery_executor") === "True";
 const taskInstancesUrl = getMetaValue("task_instances_list_url");
-const renderedK8sUrl = getMetaValue("rendered_k8s_url");
 const taskUrl = getMetaValue("task_url");
 const gridUrl = getMetaValue("grid_url");
 
@@ -49,7 +47,6 @@ const Nav = forwardRef<HTMLDivElement, Props>(
       map_index: mapIndex ?? -1,
     });
     const detailsLink = `${taskUrl}&${params}`;
-    const k8sLink = `${renderedK8sUrl}&${params}`;
     const listParams = new URLSearchParamsWrapper({
       _flt_3_dag_id: dagId,
       _flt_3_task_id: taskId,
@@ -77,9 +74,6 @@ const Nav = forwardRef<HTMLDivElement, Props>(
         {(!isMapped || mapIndex !== undefined) && (
           <>
             <LinkButton href={detailsLink}>More Details</LinkButton>
-            {isK8sExecutor && (
-              <LinkButton href={k8sLink}>K8s Pod Spec</LinkButton>
-            )}
             {isSubDag && (
               <LinkButton href={subDagLink}>Zoom into SubDag</LinkButton>
             )}

airflow/www/static/js/dag/details/taskInstance/RenderedK8s.tsx

@@ -0,0 +1,55 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import React, { useRef } from "react";
+import { Code } from "@chakra-ui/react";
+import YAML from "json-to-pretty-yaml";
+
+import { getMetaValue, useOffsetTop } from "src/utils";
+
+import useSelection from "src/dag/useSelection";
+import { useRenderedK8s } from "src/api";
+
+const isK8sExecutor = getMetaValue("k8s_or_k8scelery_executor") === "True";
+
+const RenderedK8s = () => {
+  const {
+    selected: { runId, taskId, mapIndex },
+  } = useSelection();
+
+  const { data: renderedK8s } = useRenderedK8s(runId, taskId, mapIndex);
+
+  const k8sRef = useRef<HTMLPreElement>(null);
+  const offsetTop = useOffsetTop(k8sRef);
+
+  if (!isK8sExecutor || !runId || !taskId) return null;
+
+  return (
+    <Code
+      mt={3}
+      ref={k8sRef}
+      maxHeight={`calc(100% - ${offsetTop}px)`}
+      overflowY="auto"
+    >
+      <pre>{YAML.stringify(renderedK8s)}</pre>
+    </Code>
+  );
+};
+
+export default RenderedK8s;

airflow/www/templates/airflow/dag.html

@@ -65,7 +65,7 @@
   <meta name="graph_url" content="{{ url_for('Airflow.graph', dag_id=dag.dag_id, root=root) }}">
   <meta name="task_url" content="{{ url_for('Airflow.task', dag_id=dag.dag_id) }}">
   <meta name="log_url" content="{{ url_for('Airflow.log', dag_id=dag.dag_id) }}">
-  <meta name="rendered_k8s_url" content="{{ url_for('Airflow.rendered_k8s', dag_id=dag.dag_id) }}">
+  <meta name="rendered_k8s_data_url" content="{{ url_for('Airflow.rendered_k8s_data', dag_id=dag.dag_id) }}">
   <meta name="task_instances_list_url" content="{{ url_for('TaskInstanceModelView.list') }}">
   <meta name="tag_index_url" content="{{ url_for('Airflow.index', tags='_TAG_NAME_') }}">
   <meta name="mapped_instances_api" content="{{ url_for('/api/v1.airflow_api_connexion_endpoints_task_instance_endpoint_get_mapped_task_instances', dag_id=dag.dag_id, dag_run_id='_DAG_RUN_ID_', task_id='_TASK_ID_') }}">

airflow/www/views.py

@@ -1493,7 +1493,7 @@ def rendered_k8s(self, *, session: Session = NEW_SESSION):
         form = DateTimeForm(data={"execution_date": dttm})
         root = request.args.get("root", "")
         map_index = request.args.get("map_index", -1, type=int)
-        logger.info("Retrieving rendered templates.")
+        logger.info("Retrieving rendered k8s.")
 
         dag: DAG = get_airflow_app().dag_bag.get_dag(dag_id)
         task = dag.get_task(task_id)
@@ -1537,6 +1537,48 @@ def rendered_k8s(self, *, session: Session = NEW_SESSION):
             title=title,
         )
 
+    @expose("/object/rendered-k8s")
+    @auth.has_access_dag("GET", DagAccessEntity.TASK_INSTANCE)
+    @provide_session
+    def rendered_k8s_data(self, *, session: Session = NEW_SESSION):
+        """Get rendered k8s yaml."""
+        if not settings.IS_K8S_OR_K8SCELERY_EXECUTOR:
+            return {"error": "Not a k8s or k8s_celery executor"}, 404
+        # This part is only used for k8s executor so providers.cncf.kubernetes must be installed
+        # with the get_rendered_k8s_spec method
+        from airflow.providers.cncf.kubernetes.template_rendering import get_rendered_k8s_spec
+
+        dag_id = request.args.get("dag_id")
+        task_id = request.args.get("task_id")
+        if task_id is None:
+            return {"error": "Task id not passed in the request"}, 404
+        run_id = request.args.get("run_id")
+        map_index = request.args.get("map_index", -1, type=int)
+        logger.info("Retrieving rendered k8s data.")
+
+        dag: DAG = get_airflow_app().dag_bag.get_dag(dag_id)
+        task = dag.get_task(task_id)
+        dag_run = dag.get_dagrun(run_id=run_id, session=session)
+        ti = dag_run.get_task_instance(task_id=task.task_id, map_index=map_index, session=session)
+
+        if not ti:
+            return {"error": f"can't find task instance {task.task_id}"}, 404
+        pod_spec = None
+        if not isinstance(ti, TaskInstance):
+            return {"error": f"{task.task_id} is not a task instance"}, 500
+        try:
+            pod_spec = get_rendered_k8s_spec(ti, session=session)
+        except AirflowException as e:
+            if not e.__cause__:
+                return {"error": f"Error rendering Kubernetes POD Spec: {e}"}, 500
+            else:
+                tmp = Markup("Error rendering Kubernetes POD Spec: {0}<br><br>Original error: {0.__cause__}")
+                return {"error": tmp.format(e)}, 500
+        except Exception as e:
+            return {"error": f"Error rendering Kubernetes Pod Spec: {e}"}, 500
+
+        return pod_spec
+
     @expose("/get_logs_with_metadata")
     @auth.has_access_dag("GET", DagAccessEntity.TASK_INSTANCE)
     @auth.has_access_dag("GET", DagAccessEntity.TASK_LOGS)

airflow/www/yarn.lock

@@ -3360,6 +3360,11 @@
   resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.7.tgz#98a993516c859eb0d5c4c8f098317a9ea68db9ad"
   integrity sha512-cxWFQVseBm6O9Gbw1IWb8r6OS4OhSt3hPZLkFApLjM8TEXROBuQGLAH2i2gZpcXdLBIrpXuTDhH7Vbm1iXmNGA==
 
+"@types/json-to-pretty-yaml@^1.2.1":
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/@types/json-to-pretty-yaml/-/json-to-pretty-yaml-1.2.1.tgz#bf193455477295d83c78f73c08d956f74321193e"
+  integrity sha512-+uOlBkCPkny6CE2a5IAR0Q21/ZE+90MsK7EfDblDdutcey+rbMDrp3i93M6MTwbMHFB75aIFR5fVXVcnLCkAiw==
+
 "@types/json5@^0.0.29":
   version "0.0.29"
   resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee"
@@ -8006,6 +8011,14 @@ json-stringify-safe@^5.0.1:
   resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb"
   integrity sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==
 
+json-to-pretty-yaml@^1.2.2:
+  version "1.2.2"
+  resolved "https://registry.yarnpkg.com/json-to-pretty-yaml/-/json-to-pretty-yaml-1.2.2.tgz#f4cd0bd0a5e8fe1df25aaf5ba118b099fd992d5b"
+  integrity sha512-rvm6hunfCcqegwYaG5T4yKJWxc9FXFgBVrcTZ4XfSVRwa5HA/Xs+vB/Eo9treYYHCeNM0nrSUr82V/M31Urc7A==
+  dependencies:
+    remedial "^1.0.7"
+    remove-trailing-spaces "^1.0.6"
+
 json5@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.2.tgz#63d98d60f21b313b77c4d6da18bfa69d80e1d593"
@@ -9875,10 +9888,10 @@ react-focus-lock@^2.9.1:
     use-callback-ref "^1.3.0"
     use-sidecar "^1.1.2"
 
-react-icons@^4.9.0:
-  version "4.9.0"
-  resolved "https://registry.yarnpkg.com/react-icons/-/react-icons-4.9.0.tgz#ba44f436a053393adb1bdcafbc5c158b7b70d2a3"
-  integrity sha512-ijUnFr//ycebOqujtqtV9PFS7JjhWg0QU6ykURVHuL4cbofvRCf3f6GMn9+fBktEFQOIVZnuAYLZdiyadRQRFg==
+react-icons@^5.1.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/react-icons/-/react-icons-5.1.0.tgz#9e7533cc256571a610c2a1ec8a7a143fb1222943"
+  integrity sha512-D3zug1270S4hbSlIRJ0CUS97QE1yNNKDjzQe3HqY0aefp2CBn9VgzgES27sRR2gOvFK+0CNx/BW0ggOESp6fqQ==
 
 react-is@^16.13.1, react-is@^16.7.0, react-is@^16.8.1:
   version "16.13.1"
@@ -10248,11 +10261,21 @@ remark-rehype@^10.0.0:
     mdast-util-to-hast "^12.1.0"
     unified "^10.0.0"
 
+remedial@^1.0.7:
+  version "1.0.8"
+  resolved "https://registry.yarnpkg.com/remedial/-/remedial-1.0.8.tgz#a5e4fd52a0e4956adbaf62da63a5a46a78c578a0"
+  integrity sha512-/62tYiOe6DzS5BqVsNpH/nkGlX45C/Sp6V+NtiN6JQNS1Viay7cWkazmRkrQrdFj2eshDe96SIQNIoMxqhzBOg==
+
 remove-accents@0.4.2:
   version "0.4.2"
   resolved "https://registry.yarnpkg.com/remove-accents/-/remove-accents-0.4.2.tgz#0a43d3aaae1e80db919e07ae254b285d9e1c7bb5"
   integrity sha1-CkPTqq4egNuRngeuJUsoXZ4ce7U=
 
+remove-trailing-spaces@^1.0.6:
+  version "1.0.8"
+  resolved "https://registry.yarnpkg.com/remove-trailing-spaces/-/remove-trailing-spaces-1.0.8.tgz#4354d22f3236374702f58ee373168f6d6887ada7"
+  integrity sha512-O3vsMYfWighyFbTd8hk8VaSj9UAGENxAtX+//ugIst2RMk5e03h6RoIS+0ylsFxY1gvmPuAY/PO4It+gPEeySA==
+
 require-directory@^2.1.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42"