more statuses for documents

backend/src/main/java/com/cloudera/cai/rag/files/RagFileIndexReconciler.java

@@ -115,30 +115,54 @@ public ReconcileResult reconcile(Set<RagDocument> documents) {
         log.info("Document already indexed: {}", document.filename());
         continue;
       }
+      // todo: is there a way to test intermediate state?
+      setToInProgress(document);
 
       IndexConfiguration indexConfiguration = fetchIndexConfiguration(document.dataSourceId());
-      document = doIndexing(document, indexConfiguration);
-      String updateSql =
-          """
+      RagDocument finalDocument = doIndexing(document, indexConfiguration);
+      updateFinalStatus(finalDocument);
+    }
+    return new ReconcileResult();
+  }
+
+  private void updateFinalStatus(RagDocument finalDocument) {
+    jdbi.useTransaction(
+        handle -> {
+          String updateSql =
+              """
+                UPDATE rag_data_source_document
+                SET vector_upload_timestamp = :uploadTimestamp, indexing_status = :indexingStatus, indexing_error = :indexingError, time_updated = :now
+                WHERE id = :id
+              """;
+          try (Update update = handle.createUpdate(updateSql)) {
+            update
+                .bind("id", finalDocument.id())
+                .bind("uploadTimestamp", finalDocument.vectorUploadTimestamp())
+                .bind("indexingStatus", finalDocument.indexingStatus())
+                .bind("indexingError", finalDocument.indexingError())
+                .bind("now", Instant.now())
+                .execute();
+          }
+        });
+  }
+
+  private void setToInProgress(RagDocument document) {
+    jdbi.useTransaction(
+        handle -> {
+          String updateSql =
+              """
         UPDATE rag_data_source_document
-        SET vector_upload_timestamp = :uploadTimestamp, indexing_status = :indexingStatus, indexing_error = :indexingError, time_updated = :now
+        SET indexing_status = :indexingStatus, time_updated = :now
         WHERE id = :id
       """;
-      RagDocument finalDocument = document;
-      jdbi.useHandle(
-          handle -> {
-            try (Update update = handle.createUpdate(updateSql)) {
-              update
-                  .bind("id", finalDocument.id())
-                  .bind("uploadTimestamp", finalDocument.vectorUploadTimestamp())
-                  .bind("indexingStatus", finalDocument.indexingStatus())
-                  .bind("indexingError", finalDocument.indexingError())
-                  .bind("now", Instant.now())
-                  .execute();
-            }
-          });
-    }
-    return new ReconcileResult();
+          try (Update update = handle.createUpdate(updateSql)) {
+            update
+                .bind("id", document.id())
+                .bind("indexingStatus", RagDocumentStatus.IN_PROGRESS)
+                .bind("now", Instant.now())
+                .execute();
+          }
+        });
   }
 
   private RagDocument doIndexing(RagDocument document, IndexConfiguration indexConfiguration) {

backend/src/main/java/com/cloudera/cai/rag/files/RagFileRepository.java

@@ -60,7 +60,7 @@ public RagFileRepository(Jdbi jdbi) {
     this.jdbi = jdbi;
   }
 
-  public Long saveDocumentMetadata(RagDocument ragDocument) {
+  public Long insertDocumentMetadata(RagDocument ragDocument) {
     return jdbi.inTransaction(
         handle -> {
           String insertSql =

backend/src/main/java/com/cloudera/cai/rag/files/RagFileService.java

@@ -88,7 +88,7 @@ public RagDocumentMetadata saveRagFile(MultipartFile file, Long dataSourceId, St
 
     ragFileUploader.uploadFile(file, s3Path);
     var ragDocument = createUnsavedDocument(file, documentId, s3Path, dataSourceId, actorCrn);
-    Long id = ragFileRepository.saveDocumentMetadata(ragDocument);
+    Long id = ragFileRepository.insertDocumentMetadata(ragDocument);
     log.info("Saved document with id: {}", id);
 
     ragFileIndexReconciler.submit(ragDocument.withId(id));

backend/src/main/java/com/cloudera/cai/rag/files/RagFileSummaryReconciler.java

@@ -114,6 +114,7 @@ public ReconcileResult reconcile(Set<RagDocument> documents) {
         log.info("Document already summarized: {}", document.filename());
         continue;
       }
+      setToInProgress(document);
       var updatedDocument = doSummarization(document);
       log.info("finished requesting summarization of file {}", document);
       String updateSql =
@@ -138,6 +139,25 @@ public ReconcileResult reconcile(Set<RagDocument> documents) {
     return new ReconcileResult();
   }
 
+  private void setToInProgress(RagDocument document) {
+    jdbi.useTransaction(
+        handle -> {
+          String updateSql =
+              """
+                UPDATE rag_data_source_document
+                SET summary_status = :summaryStatus, time_updated = :now
+                WHERE id = :id
+              """;
+          try (Update update = handle.createUpdate(updateSql)) {
+            update
+                .bind("id", document.id())
+                .bind("summaryStatus", Types.RagDocumentStatus.IN_PROGRESS)
+                .bind("now", Instant.now())
+                .execute();
+          }
+        });
+  }
+
   private RagDocument doSummarization(RagDocument document) {
     try {
       ragBackendClient.createSummary(document, bucketName);

backend/src/test/java/com/cloudera/cai/rag/TestData.java

@@ -90,6 +90,6 @@ public static Long createTestDocument(
             .createdById("doesn't matter")
             .updatedById("doesn't matter")
             .build();
-    return ragFileRepository.saveDocumentMetadata(ragDocument);
+    return ragFileRepository.insertDocumentMetadata(ragDocument);
   }
 }

backend/src/test/java/com/cloudera/cai/rag/datasources/DeleteDataSourceReconcilerTest.java

@@ -84,7 +84,7 @@ void reconcile() {
             .createdById("test-id")
             .build();
 
-    ragFileRepository.saveDocumentMetadata(document);
+    ragFileRepository.insertDocumentMetadata(document);
     reconciler.resync();
     await().until(reconciler::isEmpty);
     ragDataSourceRepository.deleteDataSource(dataSourceId);

backend/src/test/java/com/cloudera/cai/rag/files/RagFileIndexReconcilerTest.java

@@ -96,7 +96,7 @@ void reconcile() {
             .timeUpdated(Instant.now())
             .createdById("test-id")
             .build();
-    Long id = ragFileRepository.saveDocumentMetadata(document);
+    Long id = ragFileRepository.insertDocumentMetadata(document);
     assertThat(ragFileRepository.findDocumentByDocumentId(documentId).vectorUploadTimestamp())
         .isNull();
 
@@ -153,7 +153,7 @@ void reconcile_notFound() {
             .timeUpdated(Instant.now())
             .createdById("test-id")
             .build();
-    Long id = ragFileRepository.saveDocumentMetadata(document);
+    Long id = ragFileRepository.insertDocumentMetadata(document);
     assertThat(ragFileRepository.findDocumentByDocumentId(documentId).vectorUploadTimestamp())
         .isNull();
 
@@ -209,7 +209,7 @@ void reconcile_exception() {
             .timeUpdated(Instant.now())
             .createdById("test-id")
             .build();
-    Long id = ragFileRepository.saveDocumentMetadata(document);
+    Long id = ragFileRepository.insertDocumentMetadata(document);
     assertThat(ragFileRepository.findDocumentByDocumentId(documentId).vectorUploadTimestamp())
         .isNull();
 

backend/src/test/java/com/cloudera/cai/rag/files/RagFileSummaryReconcilerTest.java

@@ -99,7 +99,7 @@ void reconcile() {
             .timeUpdated(Instant.now())
             .createdById("test-id")
             .build();
-    Long id = ragFileRepository.saveDocumentMetadata(document);
+    Long id = ragFileRepository.insertDocumentMetadata(document);
     assertThat(ragFileRepository.findDocumentByDocumentId(documentId).summaryCreationTimestamp())
         .isNull();
 
@@ -158,7 +158,7 @@ void reconcile_notFound() {
             .timeUpdated(Instant.now())
             .createdById("test-id")
             .build();
-    Long id = ragFileRepository.saveDocumentMetadata(document);
+    Long id = ragFileRepository.insertDocumentMetadata(document);
     assertThat(ragFileRepository.findDocumentByDocumentId(documentId).summaryCreationTimestamp())
         .isNull();
 
@@ -215,7 +215,7 @@ void reconcile_exception() {
             .timeUpdated(Instant.now())
             .createdById("test-id")
             .build();
-    Long id = ragFileRepository.saveDocumentMetadata(document);
+    Long id = ragFileRepository.insertDocumentMetadata(document);
     assertThat(ragFileRepository.findDocumentByDocumentId(documentId).summaryCreationTimestamp())
         .isNull();
 
@@ -271,7 +271,7 @@ void reconcile_noSummarizationModel() {
             .timeUpdated(Instant.now())
             .createdById("test-id")
             .build();
-    ragFileRepository.saveDocumentMetadata(document);
+    ragFileRepository.insertDocumentMetadata(document);
     assertThat(ragFileRepository.findDocumentByDocumentId(documentId).summaryCreationTimestamp())
         .isNull();
 

ui/src/layout/Sidebar.tsx

@@ -37,15 +37,29 @@
  ******************************************************************************/
 
 import React, { useRef, useState } from "react";
-import { DatabaseFilled, DesktopOutlined } from "@ant-design/icons";
-import { Flex, Image, Layout, Menu, MenuProps, Tag, Typography } from "antd";
+import {
+  CloudOutlined,
+  DatabaseFilled,
+  DesktopOutlined,
+} from "@ant-design/icons";
+import {
+  Flex,
+  Image,
+  Layout,
+  Menu,
+  MenuProps,
+  Tag,
+  Tooltip,
+  Typography,
+} from "antd";
 import { useMatchRoute, useNavigate } from "@tanstack/react-router";
 import Images from "src/components/images/Images.ts";
 import LightbulbIcon from "src/cuix/icons/LightbulbIcon";
 import { cdlAmber200, cdlAmber900 } from "src/cuix/variables.ts";
 import ThumbUpIcon from "src/cuix/icons/ThumbUpIcon";
 import useModal from "src/utils/useModal.ts";
 import FeedbackModal from "src/components/Feedback/FeedbackModal.tsx";
+import "./style.css";
 
 const { Sider } = Layout;
 
@@ -100,7 +114,28 @@ const Sidebar: React.FC = () => {
 
   const baseItems: MenuItem[] = [
     {
-      label: (
+      label: collapsed ? (
+        <Tooltip title="Technical Preview">
+          <Tag
+            color={cdlAmber200}
+            style={{
+              borderRadius: 4,
+              height: 24,
+              width: 30,
+              marginLeft: 18,
+            }}
+          >
+            <Flex
+              gap={4}
+              justify="center"
+              align="center"
+              style={{ height: "100%" }}
+            >
+              <LightbulbIcon color="#000" />
+            </Flex>
+          </Tag>
+        </Tooltip>
+      ) : (
         <Tag
           color={cdlAmber200}
           style={{
@@ -145,7 +180,7 @@ const Sidebar: React.FC = () => {
     <div data-testid="data-management-nav">Models</div>,
     "models",
     navToModels,
-    <DatabaseFilled />,
+    <CloudOutlined />,
   );
 
   const feedbackItem = getItem(
@@ -176,6 +211,7 @@ const Sidebar: React.FC = () => {
       onCollapse={(value) => {
         setCollapsed(value);
       }}
+      style={{ transition: "none" }}
       width={250}
       ref={ref}
     >
@@ -184,13 +220,13 @@ const Sidebar: React.FC = () => {
           src={Images.ClouderaSmall}
           preview={false}
           height={36}
-          style={{ paddingRight: 5 }}
+          style={{ paddingLeft: 4 }}
         />
         {!collapsed ? (
           <Image
             src={Images.RagStudioProduct}
             preview={false}
-            style={{ transition: "ease-in" }}
+            style={{ transition: "ease-in", paddingLeft: 5 }}
           />
         ) : null}
       </div>

ui/src/layout/style.css

@@ -0,0 +1,41 @@
+/*
+ * CLOUDERA APPLIED MACHINE LEARNING PROTOTYPE (AMP)
+ * (C) Cloudera, Inc. 2024
+ * All rights reserved.
+ *
+ * Applicable Open Source License: Apache 2.0
+ *
+ * NOTE: Cloudera open source products are modular software products
+ * made up of hundreds of individual components, each of which was
+ * individually copyrighted.  Each Cloudera open source product is a
+ * collective work under U.S. Copyright Law. Your license to use the
+ * collective work is as provided in your written agreement with
+ * Cloudera.  Used apart from the collective work, this file is
+ * licensed for your use pursuant to the open source license
+ * identified above.
+ *
+ * This code is provided to you pursuant a written agreement with
+ * (i) Cloudera, Inc. or (ii) a third-party authorized to distribute
+ * this code. If you do not have a written agreement with Cloudera nor
+ * with an authorized and properly licensed third party, you do not
+ * have any rights to access nor to use this code.
+ *
+ * Absent a written agreement with Cloudera, Inc. ("Cloudera") to the
+ * contrary, A) CLOUDERA PROVIDES THIS CODE TO YOU WITHOUT WARRANTIES OF ANY
+ * KIND; (B) CLOUDERA DISCLAIMS ANY AND ALL EXPRESS AND IMPLIED
+ * WARRANTIES WITH RESPECT TO THIS CODE, INCLUDING BUT NOT LIMITED TO
+ * IMPLIED WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE; (C) CLOUDERA IS NOT LIABLE TO YOU,
+ * AND WILL NOT DEFEND, INDEMNIFY, NOR HOLD YOU HARMLESS FOR ANY CLAIMS
+ * ARISING FROM OR RELATED TO THE CODE; AND (D)WITH RESPECT TO YOUR EXERCISE
+ * OF ANY RIGHTS GRANTED TO YOU FOR THE CODE, CLOUDERA IS NOT LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, PUNITIVE OR
+ * CONSEQUENTIAL DAMAGES INCLUDING, BUT NOT LIMITED TO, DAMAGES
+ * RELATED TO LOST REVENUE, LOST PROFITS, LOSS OF INCOME, LOSS OF
+ * BUSINESS ADVANTAGE OR UNAVAILABILITY, OR LOSS OR CORRUPTION OF
+ * DATA.
+ */
+
+.ant-layout-sider-trigger {
+    transition: none !important;
+}