use dataspace analytics cache to get mapping model coverage analysis

finos · Jul 3, 2023 · c7f42a7 · c7f42a7
1 parent 3d079af
commit c7f42a7
Show file tree

Hide file tree

Showing 15 changed files with 166 additions and 33 deletions.
diff --git a/.changeset/curly-apes-yawn.md b/.changeset/curly-apes-yawn.md
@@ -0,0 +1,5 @@
+---
+'@finos/legend-application-query': patch
+'@finos/legend-query-builder': patch
+'@finos/legend-graph': patch
+---
diff --git a/.changeset/tame-papayas-study.md b/.changeset/tame-papayas-study.md
@@ -0,0 +1,5 @@
+---
+'@finos/legend-extension-dsl-data-space': patch
+---
+
+Use dataspace analytics cache to get mapping model coverage analysis
diff --git a/packages/legend-application-query/src/stores/LegendQueryApplicationPlugin.ts b/packages/legend-application-query/src/stores/LegendQueryApplicationPlugin.ts
@@ -54,7 +54,7 @@ export type QuerySetupActionConfiguration = {
 export type ExistingQueryEditorStateBuilder = (
   query: Query,
   editorStore: ExistingQueryEditorStore,
-) => QueryBuilderState | undefined;
+) => Promise<QueryBuilderState | undefined>;
 
 export type QueryEditorActionConfiguration = {
   key: string;

diff --git a/packages/legend-application-query/src/stores/QueryEditorStore.ts b/packages/legend-application-query/src/stores/QueryEditorStore.ts
@@ -998,7 +998,7 @@ export class ExistingQueryEditorStore extends QueryEditorStore {
         (plugin) => plugin.getExtraExistingQueryEditorStateBuilders?.() ?? [],
       );
     for (const builder of existingQueryEditorStateBuilders) {
-      queryBuilderState = builder(query, this);
+      queryBuilderState = await builder(query, this);
       if (queryBuilderState) {
         break;
       }

diff --git a/...ension-dsl-data-space/src/components/query/DSL_DataSpace_LegendQueryApplicationPlugin.tsx b/...ension-dsl-data-space/src/components/query/DSL_DataSpace_LegendQueryApplicationPlugin.tsx
@@ -49,6 +49,9 @@ import { getOwnDataSpace } from '../../graph-manager/DSL_DataSpace_GraphManagerH
 import { assertErrorThrown, LogEvent, uuid } from '@finos/legend-shared';
 import type { QueryBuilderState } from '@finos/legend-query-builder';
 import { DataSpaceQuerySetup } from './DataSpaceQuerySetup.js';
+import { DSL_DataSpace_getGraphManagerExtension } from '../../graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.js';
+import { StoreProjectData } from '@finos/legend-server-depot';
+import { retrieveAnalyticsResultCache } from '../../graph-manager/action/analytics/DataSpaceAnalysisHelper.js';
 
 export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryApplicationPlugin {
   constructor() {
@@ -94,10 +97,10 @@ export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryAppli
 
   override getExtraExistingQueryEditorStateBuilders(): ExistingQueryEditorStateBuilder[] {
     return [
-      (
+      async (
         query: Query,
         editorStore: ExistingQueryEditorStore,
-      ): QueryBuilderState | undefined => {
+      ): Promise<QueryBuilderState | undefined> => {
         const dataSpaceTaggedValue = query.taggedValues?.find(
           (taggedValue) =>
             taggedValue.profile === QUERY_PROFILE_PATH &&
@@ -120,7 +123,29 @@ export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryAppli
             // properly created from a data space, therefore, we cannot support this case
             return undefined;
           }
-          return new DataSpaceQueryBuilderState(
+          let dataSpaceAnalysisResult;
+          try {
+            const project = StoreProjectData.serialization.fromJson(
+              await editorStore.depotServerClient.getProject(
+                query.groupId,
+                query.artifactId,
+              ),
+            );
+            dataSpaceAnalysisResult =
+              await DSL_DataSpace_getGraphManagerExtension(
+                editorStore.graphManagerState.graphManager,
+              ).analyzeDataSpaceFromCache(() =>
+                retrieveAnalyticsResultCache(
+                  project,
+                  query.versionId,
+                  dataSpace.path,
+                  editorStore.depotServerClient,
+                ),
+              );
+          } catch {
+            // do nothing
+          }
+          const dataSpaceQueryBuilderState = new DataSpaceQueryBuilderState(
             editorStore.applicationStore,
             editorStore.graphManagerState,
             editorStore.depotServerClient,
@@ -230,7 +255,17 @@ export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryAppli
                 );
               }
             },
+            dataSpaceAnalysisResult,
           );
+          const mappingModelCoverageAnalysisResult =
+            dataSpaceAnalysisResult?.executionContextsIndex.get(
+              matchingExecutionContext.name,
+            )?.mappingModelCoverageAnalysisResult;
+          if (mappingModelCoverageAnalysisResult) {
+            dataSpaceQueryBuilderState.explorerState.mappingModelCoverageAnalysisResult =
+              mappingModelCoverageAnalysisResult;
+          }
+          return dataSpaceQueryBuilderState;
         }
         return undefined;
       },

diff --git a/...s/legend-extension-dsl-data-space/src/graph-manager/action/analytics/DataSpaceAnalysis.ts b/...s/legend-extension-dsl-data-space/src/graph-manager/action/analytics/DataSpaceAnalysis.ts
@@ -20,6 +20,7 @@ import {
   type PackageableRuntime,
   type PureModel,
   type DatasetSpecification,
+  MappingModelCoverageAnalysisResult,
 } from '@finos/legend-graph';
 import { prettyCONSTName, uuid } from '@finos/legend-shared';
 import type { DataSpaceSupportInfo } from '../../../graph/metamodel/pure/model/packageableElements/dataSpace/DSL_DataSpace_DataSpace.js';
@@ -32,7 +33,7 @@ export class DataSpaceExecutionContextAnalysisResult {
   mapping!: Mapping;
   defaultRuntime!: PackageableRuntime;
   compatibleRuntimes!: PackageableRuntime[];
-  // TODO: mapping coverage analysis
+  mappingModelCoverageAnalysisResult!: MappingModelCoverageAnalysisResult;
   datasets: DatasetSpecification[] = [];
 }
 

diff --git a/...dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.ts b/...dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.ts
@@ -33,6 +33,11 @@ export abstract class DSL_DataSpace_PureGraphManagerExtension extends AbstractPu
     cacheRetriever?: () => Promise<PlainObject<DataSpaceAnalysisResult>>,
     actionState?: ActionState,
   ): Promise<DataSpaceAnalysisResult>;
+
+  abstract analyzeDataSpaceFromCache(
+    cacheRetriever: () => Promise<PlainObject<DataSpaceAnalysisResult>>,
+    actionState?: ActionState,
+  ): Promise<DataSpaceAnalysisResult | undefined>;
 }
 
 export const DSL_DataSpace_getGraphManagerExtension = (

diff --git a/...ta-space/src/graph-manager/protocol/pure/v1/V1_DSL_DataSpace_PureGraphManagerExtension.ts b/...ta-space/src/graph-manager/protocol/pure/v1/V1_DSL_DataSpace_PureGraphManagerExtension.ts
@@ -30,6 +30,7 @@ import {
   GRAPH_MANAGER_EVENT,
   V1_buildDatasetSpecification,
   type PureProtocolProcessorPlugin,
+  V1_buildModelCoverageAnalysisResult,
 } from '@finos/legend-graph';
 import type { Entity } from '@finos/legend-storage';
 import {
@@ -101,25 +102,13 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu
     cacheRetriever?: () => Promise<PlainObject<DataSpaceAnalysisResult>>,
     actionState?: ActionState,
   ): Promise<DataSpaceAnalysisResult> {
-    let cachResult: PlainObject<V1_DataSpaceAnalysisResult> | undefined;
-    if (cacheRetriever) {
-      try {
-        actionState?.setMessage(
-          'Fetching data space analysis result from cache...',
-        );
-        cachResult = await cacheRetriever();
-      } catch (error) {
-        assertErrorThrown(error);
-        this.graphManager.logService.warn(
-          LogEvent.create(GRAPH_MANAGER_EVENT.CACHE_MANAGER_FAILURE),
-          `Can't fetch data space analysis result cache: ${error.message}`,
-        );
-      }
-    }
+    const cacheResult = cacheRetriever
+      ? await this.fetchDataSpaceAnalyticsFromCache(cacheRetriever, actionState)
+      : undefined;
     const engineClient = this.graphManager.engine.getEngineServerClient();
     let analysisResult: PlainObject<V1_DataSpaceAnalysisResult>;
-    if (cachResult) {
-      analysisResult = cachResult;
+    if (cacheResult) {
+      analysisResult = cacheResult;
     } else {
       actionState?.setMessage('Fetching project entities and dependencies...');
       const entities = await entitiesRetriever();
@@ -149,6 +138,42 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu
     );
   }
 
+  async analyzeDataSpaceFromCache(
+    cacheRetriever: () => Promise<PlainObject<DataSpaceAnalysisResult>>,
+    actionState?: ActionState,
+  ): Promise<DataSpaceAnalysisResult | undefined> {
+    const cacheResult = await this.fetchDataSpaceAnalyticsFromCache(
+      cacheRetriever,
+      actionState,
+    );
+    return cacheResult
+      ? this.buildDataSpaceAnalytics(
+          cacheResult,
+          this.graphManager.pluginManager.getPureProtocolProcessorPlugins(),
+        )
+      : undefined;
+  }
+
+  async fetchDataSpaceAnalyticsFromCache(
+    cacheRetriever: () => Promise<PlainObject<DataSpaceAnalysisResult>>,
+    actionState?: ActionState,
+  ): Promise<PlainObject<V1_DataSpaceAnalysisResult> | undefined> {
+    let cacheResult: PlainObject<V1_DataSpaceAnalysisResult> | undefined;
+    try {
+      actionState?.setMessage(
+        'Fetching data space analysis result from cache...',
+      );
+      cacheResult = await cacheRetriever();
+    } catch (error) {
+      assertErrorThrown(error);
+      this.graphManager.logService.warn(
+        LogEvent.create(GRAPH_MANAGER_EVENT.CACHE_MANAGER_FAILURE),
+        `Can't fetch data space analysis result cache: ${error.message}`,
+      );
+    }
+    return cacheResult;
+  }
+
   private async buildDataSpaceAnalytics(
     json: PlainObject<V1_DataSpaceAnalysisResult>,
     plugins: PureProtocolProcessorPlugin[],
@@ -313,6 +338,11 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu
       contextAnalysisResult.defaultRuntime = graph.getRuntime(
         context.defaultRuntime,
       );
+      contextAnalysisResult.mappingModelCoverageAnalysisResult =
+        V1_buildModelCoverageAnalysisResult(
+          context.mappingModelCoverageAnalysisResult,
+          contextAnalysisResult.mapping,
+        );
       contextAnalysisResult.compatibleRuntimes = context.compatibleRuntimes.map(
         (runtime) => graph.getRuntime(runtime),
       );

diff --git a/...sl-data-space/src/graph-manager/protocol/pure/v1/engine/analytics/V1_DataSpaceAnalysis.ts b/...sl-data-space/src/graph-manager/protocol/pure/v1/engine/analytics/V1_DataSpaceAnalysis.ts
@@ -22,6 +22,8 @@ import {
   type PureProtocolProcessorPlugin,
   V1_deserializeDatasetSpecification,
   V1_pureModelContextDataPropSchema,
+  type V1_MappingModelCoverageAnalysisResult,
+  V1_mappingModelCoverageAnalysisResultModelSchema,
 } from '@finos/legend-graph';
 import {
   SerializationFactory,
@@ -33,6 +35,7 @@ import {
   isString,
   isNonNullable,
   optionalCustomUsingModelSchema,
+  usingModelSchema,
 } from '@finos/legend-shared';
 import {
   createModelSchema,
@@ -81,6 +84,7 @@ class V1_DataSpaceExecutionContextAnalysisResult {
   mapping!: string;
   defaultRuntime!: string;
   compatibleRuntimes!: string[];
+  mappingModelCoverageAnalysisResult!: V1_MappingModelCoverageAnalysisResult;
   datasets: V1_DatasetSpecification[] = [];
 }
 
@@ -98,6 +102,9 @@ const V1_dataSpaceExecutionContextAnalysisResultModelSchema = (
     ),
     defaultRuntime: primitive(),
     description: optional(primitive()),
+    mappingModelCoverageAnalysisResult: usingModelSchema(
+      V1_mappingModelCoverageAnalysisResultModelSchema,
+    ),
     mapping: primitive(),
     name: primitive(),
     title: optional(primitive()),
@@ -404,6 +411,9 @@ const V1_dataSpaceAnalysisResultModelSchema = (
     executionContexts: customListWithSchema(
       V1_dataSpaceExecutionContextAnalysisResultModelSchema(plugins),
     ),
+    mappingModelCoverageAnalysisResult: usingModelSchema(
+      V1_mappingModelCoverageAnalysisResultModelSchema,
+    ),
     defaultExecutionContext: primitive(),
 
     elements: list(primitive()),

diff --git a/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceAdvancedSearchState.ts b/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceAdvancedSearchState.ts
@@ -180,13 +180,7 @@ export class DataSpaceAdvancedSearchState {
             dataSpace.versionId,
             this.depotServerClient,
           ),
-        () =>
-          retrieveAnalyticsResultCache(
-            project,
-            dataSpace.versionId,
-            dataSpace.path,
-            this.depotServerClient,
-          ),
+        undefined,
         this.loadDataSpaceState,
       )) as DataSpaceAnalysisResult;
       this.dataSpaceViewerState = new DataSpaceViewerState(

diff --git a/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryBuilderState.ts b/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryBuilderState.ts
@@ -48,6 +48,7 @@ import type {
 import { DATA_SPACE_ELEMENT_CLASSIFIER_PATH } from '../../graph-manager/protocol/pure/DSL_DataSpace_PureProtocolProcessorPlugin.js';
 import { type DataSpaceInfo, extractDataSpaceInfo } from './DataSpaceInfo.js';
 import { DataSpaceAdvancedSearchState } from './DataSpaceAdvancedSearchState.js';
+import type { DataSpaceAnalysisResult } from '../../graph-manager/action/analytics/DataSpaceAnalysis.js';
 
 export class DataSpaceQueryBuilderState extends QueryBuilderState {
   readonly depotServerClient: DepotServerClient;
@@ -74,6 +75,7 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState {
     | undefined;
   readonly onRuntimeChange?: ((val: Runtime) => void) | undefined;
   readonly onClassChange?: ((val: Class) => void) | undefined;
+  readonly dataSpaceAnalysisResult?: DataSpaceAnalysisResult | undefined;
 
   override TEMPORARY__setupPanelContentRenderer = (): React.ReactNode =>
     renderDataSpaceQueryBuilderSetupPanelContent(this);
@@ -104,6 +106,7 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState {
       entityPath: string | undefined,
     ) => Promise<void>,
     onDataSpaceChange: (val: DataSpaceInfo) => void,
+    dataSpaceAnalysisResult?: DataSpaceAnalysisResult | undefined,
     onExecutionContextChange?:
       | ((val: DataSpaceExecutionContext) => void)
       | undefined,
@@ -139,6 +142,7 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState {
     // NOTE: if we reuse this state in the future (e.g. in Studio), we might need
     // to turn this flag off
     this.isAdvancedDataSpaceSearchEnabled = true;
+    this.dataSpaceAnalysisResult = dataSpaceAnalysisResult;
   }
 
   override get sideBarClassName(): string | undefined {
@@ -219,6 +223,14 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState {
   ): void {
     const mapping = executionContext.mapping.value;
     this.changeMapping(mapping);
+    const mappingModelCoverageAnalysisResult =
+      this.dataSpaceAnalysisResult?.executionContextsIndex.get(
+        executionContext.name,
+      )?.mappingModelCoverageAnalysisResult;
+    if (mappingModelCoverageAnalysisResult) {
+      this.explorerState.mappingModelCoverageAnalysisResult =
+        mappingModelCoverageAnalysisResult;
+    }
     this.changeRuntime(new RuntimePointer(executionContext.defaultRuntime));
 
     const compatibleClasses = getMappingCompatibleClasses(

diff --git a/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryCreatorStore.ts b/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryCreatorStore.ts
@@ -31,7 +31,10 @@ import {
   createViewProjectHandler,
   createViewSDLCProjectHandler,
 } from '@finos/legend-application-query';
-import type { DepotServerClient } from '@finos/legend-server-depot';
+import {
+  type DepotServerClient,
+  StoreProjectData,
+} from '@finos/legend-server-depot';
 import {
   guaranteeNonNullable,
   guaranteeType,
@@ -49,6 +52,8 @@ import { generateDataSpaceQueryCreatorRoute } from '../../__lib__/query/DSL_Data
 import type { DataSpaceExecutionContext } from '../../graph/metamodel/pure/model/packageableElements/dataSpace/DSL_DataSpace_DataSpace.js';
 import type { QueryBuilderState } from '@finos/legend-query-builder';
 import type { ProjectGAVCoordinates } from '@finos/legend-storage';
+import { DSL_DataSpace_getGraphManagerExtension } from '../../graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.js';
+import { retrieveAnalyticsResultCache } from '../../graph-manager/action/analytics/DataSpaceAnalysisHelper.js';
 
 export const createQueryDataSpaceTaggedValue = (
   dataSpacePath: string,
@@ -120,6 +125,24 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore {
       ),
       `Can't find execution context '${this.executionContext}'`,
     );
+    let dataSpaceAnalysisResult;
+    try {
+      const project = StoreProjectData.serialization.fromJson(
+        await this.depotServerClient.getProject(this.groupId, this.artifactId),
+      );
+      dataSpaceAnalysisResult = await DSL_DataSpace_getGraphManagerExtension(
+        this.graphManagerState.graphManager,
+      ).analyzeDataSpaceFromCache(() =>
+        retrieveAnalyticsResultCache(
+          project,
+          this.versionId,
+          dataSpace.path,
+          this.depotServerClient,
+        ),
+      );
+    } catch {
+      // do nothing
+    }
     const queryBuilderState = new DataSpaceQueryBuilderState(
       this.applicationStore,
       this.graphManagerState,
@@ -153,6 +176,7 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore {
           );
         }
       },
+      dataSpaceAnalysisResult,
       (ec: DataSpaceExecutionContext) => {
         // runtime should already be set
         const runtimePointer = guaranteeType(

diff --git a/...ph/src/graph-manager/protocol/pure/v1/engine/analytics/V1_MappingModelCoverageAnalysis.ts b/...ph/src/graph-manager/protocol/pure/v1/engine/analytics/V1_MappingModelCoverageAnalysis.ts
@@ -144,6 +144,13 @@ export class V1_MappingModelCoverageAnalysisResult {
   );
 }
 
+export const V1_mappingModelCoverageAnalysisResultModelSchema =
+  createModelSchema(V1_MappingModelCoverageAnalysisResult, {
+    mappedEntities: list(
+      usingModelSchema(V1_MappedEntity.serialization.schema),
+    ),
+  });
+
 const buildMappedProperty = (protocol: V1_MappedProperty): MappedProperty =>
   protocol instanceof V1_EntityMappedProperty
     ? new EntityMappedProperty(