diff --git a/.changeset/modern-adults-play.md b/.changeset/modern-adults-play.md new file mode 100644 index 0000000000..385dff9211 --- /dev/null +++ b/.changeset/modern-adults-play.md @@ -0,0 +1,5 @@ +--- +'@finos/legend-extension-dsl-persistence': patch +--- + +Added persistence V2 spec diff --git a/packages/legend-extension-dsl-persistence/src/graph-manager/__tests__/DSL_Persistence_Roundtrip.test.ts b/packages/legend-extension-dsl-persistence/src/graph-manager/__tests__/DSL_Persistence_Roundtrip.test.ts index 6bf1dc7b7c..805aaa4dd8 100644 --- a/packages/legend-extension-dsl-persistence/src/graph-manager/__tests__/DSL_Persistence_Roundtrip.test.ts +++ b/packages/legend-extension-dsl-persistence/src/graph-manager/__tests__/DSL_Persistence_Roundtrip.test.ts @@ -28,6 +28,11 @@ import { } from '@finos/legend-graph/test'; import { unitTest } from '@finos/legend-shared/test'; import type { Entity } from '@finos/legend-storage'; +import { + TEST_DATA__roundtrip_append_only_allow_duplicates, + TEST_DATA__roundtrip_bitemporal_no_del_ind_user_specifies_from, + TEST_DATA__roundtrip_non_temporal_snapshot_date_time_audit, +} from './TEST_DATA_DSL_PersistenceV2_Roundtrip.js'; const pluginManager = new TEST__GraphManagerPluginManager(); pluginManager @@ -51,3 +56,18 @@ test(unitTest('DSL Persistence roundtrip'), async () => { pluginManager, ); }); + +test(unitTest('DSL Persistence V2 roundtrip'), async () => { + await TEST__checkBuildingElementsRoundtrip( + TEST_DATA__roundtrip_append_only_allow_duplicates as Entity[], + pluginManager, + ); + await TEST__checkBuildingElementsRoundtrip( + TEST_DATA__roundtrip_bitemporal_no_del_ind_user_specifies_from as Entity[], + pluginManager, + ); + await TEST__checkBuildingElementsRoundtrip( + TEST_DATA__roundtrip_non_temporal_snapshot_date_time_audit as Entity[], + pluginManager, + ); +}); diff --git a/packages/legend-extension-dsl-persistence/src/graph-manager/__tests__/TEST_DATA_DSL_PersistenceV2_Roundtrip.ts b/packages/legend-extension-dsl-persistence/src/graph-manager/__tests__/TEST_DATA_DSL_PersistenceV2_Roundtrip.ts new file mode 100644 index 0000000000..c4e5139bde --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph-manager/__tests__/TEST_DATA_DSL_PersistenceV2_Roundtrip.ts @@ -0,0 +1,853 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const TEST_DATA__roundtrip_append_only_allow_duplicates = [ + { + path: 'test::Person', + content: { + _type: 'class', + name: 'Person', + package: 'test', + properties: [ + { + multiplicity: { + lowerBound: 1, + upperBound: 1, + }, + name: 'name', + type: 'String', + }, + ], + }, + classifierPath: 'meta::pure::metamodel::type::Class', + }, + { + path: 'test::ServiceResult', + content: { + _type: 'class', + name: 'ServiceResult', + package: 'test', + properties: [ + { + multiplicity: { + lowerBound: 1, + upperBound: 1, + }, + name: 'ID', + type: 'String', + }, + { + multiplicity: { + lowerBound: 1, + upperBound: 1, + }, + name: 'NAME', + type: 'String', + }, + ], + }, + classifierPath: 'meta::pure::metamodel::type::Class', + }, + { + path: 'test::TestDatabase', + content: { + _type: 'relational', + filters: [], + includedStores: [], + joins: [], + name: 'TestDatabase', + package: 'test', + schemas: [ + { + name: 'default', + tables: [ + { + columns: [ + { + name: 'ID', + nullable: true, + type: { + _type: 'Integer', + }, + }, + { + name: 'NAME', + nullable: true, + type: { + _type: 'Varchar', + size: 100, + }, + }, + ], + name: 'personTable', + primaryKey: [], + }, + ], + views: [], + }, + ], + }, + classifierPath: 'meta::relational::metamodel::Database', + }, + { + path: 'test::Mapping', + content: { + _type: 'mapping', + classMappings: [], + enumerationMappings: [], + includedMappings: [], + name: 'Mapping', + package: 'test', + tests: [], + }, + classifierPath: 'meta::pure::mapping::Mapping', + }, + { + path: 'test::Service', + content: { + _type: 'service', + autoActivateUpdates: true, + documentation: 'test', + execution: { + _type: 'pureSingleExecution', + func: { + _type: 'lambda', + body: [ + { + _type: 'property', + parameters: [ + { + _type: 'var', + name: 'src', + }, + ], + property: 'name', + }, + ], + parameters: [ + { + _type: 'var', + class: 'test::Person', + multiplicity: { + lowerBound: 1, + upperBound: 1, + }, + name: 'src', + }, + ], + }, + mapping: 'test::Mapping', + runtime: { + _type: 'engineRuntime', + connections: [], + mappings: [ + { + path: 'test::Mapping', + type: 'MAPPING', + }, + ], + }, + }, + name: 'Service', + owners: [], + package: 'test', + pattern: 'test', + }, + classifierPath: 'meta::legend::service::metamodel::Service', + }, + { + path: 'test::TestPersistence', + content: { + _type: 'persistence', + documentation: 'This is test documentation.', + name: 'TestPersistence', + notifier: {}, + package: 'test', + serviceOutputTargets: [ + { + serviceOutput: { + _type: 'tdsServiceOutput', + datasetType: { + _type: 'delta', + actionIndicator: { + _type: 'noActionIndicator', + }, + }, + deduplication: { + _type: 'noDeduplication', + }, + keys: [], + }, + persistenceTarget: { + _type: 'relationalPersistenceTarget', + table: 'personTable', + database: 'test::TestDatabase', + temporality: { + _type: 'none', + auditing: { + _type: 'noAuditing', + }, + updatesHandling: { + _type: 'appendOnly', + appendStrategy: { + _type: 'allowDuplicates', + }, + }, + }, + }, + }, + ], + service: 'test::Service', + tests: [ + { + _type: 'test', + id: 'test1', + isTestDataFromServiceOutput: true, + testBatches: [ + { + assertions: [ + { + _type: 'equalToJson', + expected: { + _type: 'externalFormat', + contentType: 'application/json', + data: '[{"ID":1, "NAME":"ANDY"},{"ID":2, "NAME":"BRAD"}]', + }, + id: 'assert1', + }, + ], + id: 'testBatch1', + testData: { + connection: { + data: { + _type: 'externalFormat', + contentType: 'application/json', + data: '[{"ID":1, "NAME":"ANDY"},{"ID":2, "NAME":"BRAD"}]', + }, + }, + }, + }, + { + assertions: [ + { + _type: 'equalToJson', + expected: { + _type: 'externalFormat', + contentType: 'application/json', + data: '[{"ID":1, "NAME":"ANDY"},{"ID":2, "NAME":"BRAD"},{"ID":2, "NAME":"BRAD"},{"ID":3, "NAME":"CATHY"},{"ID":4, "NAME":"TOM"}]', + }, + id: 'assert1', + }, + ], + id: 'testBatch2', + testData: { + connection: { + data: { + _type: 'externalFormat', + contentType: 'application/json', + data: '[{"ID":2, "NAME":"BRAD"},{"ID":3, "NAME":"CATHY"},{"ID":4, "NAME":"TOM"}]', + }, + }, + }, + }, + ], + }, + ], + trigger: { + _type: 'manualTrigger', + }, + }, + classifierPath: 'meta::pure::persistence::metamodel::Persistence', + }, +]; + +export const TEST_DATA__roundtrip_bitemporal_no_del_ind_user_specifies_from = [ + { + path: 'test::Person', + content: { + _type: 'class', + name: 'Person', + package: 'test', + properties: [ + { + multiplicity: { + lowerBound: 1, + upperBound: 1, + }, + name: 'name', + type: 'String', + }, + ], + }, + classifierPath: 'meta::pure::metamodel::type::Class', + }, + { + path: 'test::ServiceResult', + content: { + _type: 'class', + name: 'ServiceResult', + package: 'test', + properties: [ + { + multiplicity: { + lowerBound: 1, + upperBound: 1, + }, + name: 'ID', + type: 'String', + }, + { + multiplicity: { + lowerBound: 1, + upperBound: 1, + }, + name: 'AMOUNT', + type: 'Integer', + }, + { + multiplicity: { + lowerBound: 1, + upperBound: 1, + }, + name: 'SOURCE_FROM', + type: 'String', + }, + ], + }, + classifierPath: 'meta::pure::metamodel::type::Class', + }, + { + path: 'test::TestDatabase', + content: { + _type: 'relational', + filters: [], + includedStores: [], + joins: [], + name: 'TestDatabase', + package: 'test', + schemas: [ + { + name: 'default', + tables: [ + { + columns: [ + { + name: 'ID', + nullable: false, + type: { + _type: 'Integer', + }, + }, + { + name: 'AMOUNT', + nullable: true, + type: { + _type: 'Integer', + }, + }, + { + name: 'FROM_Z', + nullable: false, + type: { + _type: 'Timestamp', + }, + }, + { + name: 'THRU_Z', + nullable: true, + type: { + _type: 'Timestamp', + }, + }, + { + name: 'IN_Z', + nullable: false, + type: { + _type: 'Timestamp', + }, + }, + { + name: 'OUT_Z', + nullable: true, + type: { + _type: 'Timestamp', + }, + }, + ], + name: 'personTable', + primaryKey: ['ID', 'FROM_Z', 'IN_Z'], + }, + ], + views: [], + }, + ], + }, + classifierPath: 'meta::relational::metamodel::Database', + }, + { + path: 'test::Mapping', + content: { + _type: 'mapping', + classMappings: [], + enumerationMappings: [], + includedMappings: [], + name: 'Mapping', + package: 'test', + tests: [], + }, + classifierPath: 'meta::pure::mapping::Mapping', + }, + { + path: 'test::Service', + content: { + _type: 'service', + autoActivateUpdates: true, + documentation: 'test', + execution: { + _type: 'pureSingleExecution', + func: { + _type: 'lambda', + body: [ + { + _type: 'property', + parameters: [ + { + _type: 'var', + name: 'src', + }, + ], + property: 'name', + }, + ], + parameters: [ + { + _type: 'var', + class: 'test::Person', + multiplicity: { + lowerBound: 1, + upperBound: 1, + }, + name: 'src', + }, + ], + }, + mapping: 'test::Mapping', + runtime: { + _type: 'engineRuntime', + connections: [], + mappings: [ + { + path: 'test::Mapping', + type: 'MAPPING', + }, + ], + }, + }, + name: 'Service', + owners: [], + package: 'test', + pattern: 'test', + test: { + _type: 'singleExecutionTest', + asserts: [], + data: 'test', + }, + }, + classifierPath: 'meta::legend::service::metamodel::Service', + }, + { + path: 'test::TestPersistence', + content: { + _type: 'persistence', + documentation: 'This is test documentation.', + name: 'TestPersistence', + notifier: {}, + package: 'test', + serviceOutputTargets: [ + { + serviceOutput: { + _type: 'tdsServiceOutput', + datasetType: { + _type: 'delta', + actionIndicator: { + _type: 'noActionIndicator', + }, + }, + deduplication: { + _type: 'noDeduplication', + }, + keys: ['ID'], + }, + persistenceTarget: { + _type: 'relationalPersistenceTarget', + table: 'personTable', + database: 'test::TestDatabase', + temporality: { + _type: 'bitemporalTemporality', + processingDimension: { + _type: 'processingTime', + timeIn: 'IN_Z', + timeOut: 'OUT_Z', + }, + sourceDerivedDimension: { + _type: 'sourceDerivedTime', + timeStart: 'FROM_Z', + timeEnd: 'THRU_Z', + sourceTimeFields: { + _type: 'sourceTimeStartAndEnd', + startField: 'SOURCE_FROM', + }, + }, + }, + }, + }, + ], + service: 'test::Service', + tests: [ + { + _type: 'test', + id: 'test1', + isTestDataFromServiceOutput: true, + testBatches: [ + { + assertions: [ + { + _type: 'equalToJson', + expected: { + _type: 'externalFormat', + contentType: 'application/json', + data: '[{"ID":1, "AMOUNT":100, "FROM_Z": "2022-01-01 00:00:00.0", "THRU_Z": "9999-12-31 23:59:59.0"},{"ID":2, "AMOUNT":200, "FROM_Z": "2022-01-01 00:00:00.0", "THRU_Z": "9999-12-31 23:59:59.0"},{"ID":3, "AMOUNT":400, "FROM_Z": "2022-01-01 00:00:00.0", "THRU_Z": "9999-12-31 23:59:59.0"}]', + }, + id: 'assert1', + }, + ], + id: 'testBatch1', + testData: { + connection: { + data: { + _type: 'externalFormat', + contentType: 'application/json', + data: '[{"ID":1, "AMOUNT":100, "SOURCE_FROM": "2022-01-01 00:00:00.0"},{"ID":2, "AMOUNT":200, "SOURCE_FROM": "2022-01-01 00:00:00.0"},{"ID":3, "AMOUNT":400, "SOURCE_FROM": "2022-01-01 00:00:00.0"}]', + }, + }, + }, + }, + { + assertions: [ + { + _type: 'equalToJson', + expected: { + _type: 'externalFormat', + contentType: 'application/json', + data: '[{"ID":1,"AMOUNT":100,"THRU_Z":"9999-12-31 23:59:59.0","FROM_Z":"2022-01-01 00:00:00.0"},{"ID":2,"AMOUNT":200,"THRU_Z":"9999-12-31 23:59:59.0","FROM_Z":"2022-01-01 00:00:00.0"},{"ID":3,"AMOUNT":400,"THRU_Z":"9999-12-31 23:59:59.0","FROM_Z":"2022-01-01 00:00:00.0"},{"ID":1,"AMOUNT":200,"THRU_Z":"9999-12-31 23:59:59.0","FROM_Z":"2022-02-01 00:00:00.0"},{"ID":2,"AMOUNT":400,"THRU_Z":"9999-12-31 23:59:59.0","FROM_Z":"2022-02-01 00:00:00.0"},{"ID":3,"AMOUNT":800,"THRU_Z":"9999-12-31 23:59:59.0","FROM_Z":"2022-01-01 00:00:00.0"},{"ID":1,"AMOUNT":100,"THRU_Z":"2022-02-01 00:00:00.0","FROM_Z":"2022-01-01 00:00:00.0"},{"ID":2,"AMOUNT":200,"THRU_Z":"2022-02-01 00:00:00.0","FROM_Z":"2022-01-01 00:00:00.0"}]', + }, + id: 'assert1', + }, + ], + id: 'testBatch2', + testData: { + connection: { + data: { + _type: 'externalFormat', + contentType: 'application/json', + data: '[{"ID":1, "AMOUNT":200, "SOURCE_FROM": "2022-02-01 00:00:00.0"},{"ID":2, "AMOUNT":400, "SOURCE_FROM": "2022-02-01 00:00:00.0"},{"ID":3, "AMOUNT":800, "SOURCE_FROM": "2022-01-01 00:00:00.0"}]', + }, + }, + }, + }, + ], + }, + ], + trigger: { + _type: 'manualTrigger', + }, + }, + classifierPath: 'meta::pure::persistence::metamodel::Persistence', + }, +]; + +export const TEST_DATA__roundtrip_non_temporal_snapshot_date_time_audit = [ + { + path: 'test::Person', + content: { + _type: 'class', + name: 'Person', + package: 'test', + properties: [ + { + multiplicity: { + lowerBound: 1, + upperBound: 1, + }, + name: 'name', + type: 'String', + }, + ], + }, + classifierPath: 'meta::pure::metamodel::type::Class', + }, + { + path: 'test::ServiceResult', + content: { + _type: 'class', + name: 'ServiceResult', + package: 'test', + properties: [ + { + multiplicity: { + lowerBound: 1, + upperBound: 1, + }, + name: 'ID', + type: 'String', + }, + { + multiplicity: { + lowerBound: 1, + upperBound: 1, + }, + name: 'NAME', + type: 'String', + }, + ], + }, + classifierPath: 'meta::pure::metamodel::type::Class', + }, + { + path: 'test::TestDatabase', + content: { + _type: 'relational', + filters: [], + includedStores: [], + joins: [], + name: 'TestDatabase', + package: 'test', + schemas: [ + { + name: 'default', + tables: [ + { + columns: [ + { + name: 'ID', + nullable: true, + type: { + _type: 'Integer', + }, + }, + { + name: 'NAME', + nullable: true, + type: { + _type: 'Varchar', + size: 100, + }, + }, + { + name: 'BATCH_TIME_IN', + nullable: true, + type: { + _type: 'Timestamp', + }, + }, + ], + name: 'personTable', + primaryKey: [], + }, + ], + views: [], + }, + ], + }, + classifierPath: 'meta::relational::metamodel::Database', + }, + { + path: 'test::Mapping', + content: { + _type: 'mapping', + classMappings: [], + enumerationMappings: [], + includedMappings: [], + name: 'Mapping', + package: 'test', + tests: [], + }, + classifierPath: 'meta::pure::mapping::Mapping', + }, + { + path: 'test::Service', + content: { + _type: 'service', + autoActivateUpdates: true, + documentation: 'test', + execution: { + _type: 'pureSingleExecution', + func: { + _type: 'lambda', + body: [ + { + _type: 'property', + parameters: [ + { + _type: 'var', + name: 'src', + }, + ], + property: 'name', + }, + ], + parameters: [ + { + _type: 'var', + class: 'test::Person', + multiplicity: { + lowerBound: 1, + upperBound: 1, + }, + name: 'src', + }, + ], + }, + mapping: 'test::Mapping', + runtime: { + _type: 'engineRuntime', + connections: [], + mappings: [ + { + path: 'test::Mapping', + type: 'MAPPING', + }, + ], + }, + }, + name: 'Service', + owners: [], + package: 'test', + pattern: 'test', + test: { + _type: 'singleExecutionTest', + asserts: [], + data: 'test', + }, + }, + classifierPath: 'meta::legend::service::metamodel::Service', + }, + { + path: 'test::TestPersistence', + content: { + _type: 'persistence', + documentation: 'This is test documentation.', + name: 'TestPersistence', + notifier: {}, + package: 'test', + serviceOutputTargets: [ + { + serviceOutput: { + _type: 'tdsServiceOutput', + datasetType: { + _type: 'snapshot', + partitioning: { + _type: 'noPartitioning', + emptyDatasetHandling: { + _type: 'noOp', + }, + }, + }, + deduplication: { + _type: 'noDeduplication', + }, + keys: [], + }, + persistenceTarget: { + _type: 'relationalPersistenceTarget', + table: 'personTable', + database: 'test::TestDatabase', + temporality: { + _type: 'none', + auditing: { + _type: 'auditingDateTime', + auditingDateTimeName: 'BATCH_TIME_IN', + }, + updatesHandling: { + _type: 'overwrite', + }, + }, + }, + }, + ], + service: 'test::Service', + tests: [ + { + _type: 'test', + id: 'test1', + isTestDataFromServiceOutput: true, + testBatches: [ + { + assertions: [ + { + _type: 'equalToJson', + expected: { + _type: 'externalFormat', + contentType: 'application/json', + data: '[{"ID":1, "NAME":"ANDY"},{"ID":2, "NAME":"BRAD"}]', + }, + id: 'assert1', + }, + ], + id: 'testBatch1', + testData: { + connection: { + data: { + _type: 'externalFormat', + contentType: 'application/json', + data: '[{"ID":1, "NAME":"ANDY"},{"ID":2, "NAME":"BRAD"}]', + }, + }, + }, + }, + { + assertions: [ + { + _type: 'equalToJson', + expected: { + _type: 'externalFormat', + contentType: 'application/json', + data: '[{"ID":2, "NAME":"BRADLEY"},{"ID":3, "NAME":"CATHY"}]', + }, + id: 'assert1', + }, + ], + id: 'testBatch2', + testData: { + connection: { + data: { + _type: 'externalFormat', + contentType: 'application/json', + data: '[{"ID":2, "NAME":"BRADLEY"},{"ID":3, "NAME":"CATHY"}]', + }, + }, + }, + }, + ], + }, + ], + trigger: { + _type: 'manualTrigger', + }, + }, + classifierPath: 'meta::pure::persistence::metamodel::Persistence', + }, +]; diff --git a/packages/legend-extension-dsl-persistence/src/graph-manager/action/changeDetection/DSL_Persistence_ObserverHelper.ts b/packages/legend-extension-dsl-persistence/src/graph-manager/action/changeDetection/DSL_Persistence_ObserverHelper.ts index 7288e772be..eab33411da 100644 --- a/packages/legend-extension-dsl-persistence/src/graph-manager/action/changeDetection/DSL_Persistence_ObserverHelper.ts +++ b/packages/legend-extension-dsl-persistence/src/graph-manager/action/changeDetection/DSL_Persistence_ObserverHelper.ts @@ -88,6 +88,7 @@ export const observe_PersistenceTestBatch = skipObservedWithContext( ): PersistenceTestBatch => { makeObservable(metamodel, { id: observable, + batchId: observable, assertions: observable, testData: observable, hashCode: computed, diff --git a/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_ActionIndicatorFields.ts b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_ActionIndicatorFields.ts new file mode 100644 index 0000000000..800fc92e6c --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_ActionIndicatorFields.ts @@ -0,0 +1,51 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type Hashable, hashArray } from '@finos/legend-shared'; +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../../../graph/DSL_Persistence_HashUtils.js'; + +export abstract class V1_ActionIndicatorFields implements Hashable { + abstract get hashCode(): string; +} + +export class V1_NoActionIndicator implements V1_ActionIndicatorFields { + get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.NO_ACTION_INDICATOR]); + } +} + +export abstract class V1_DeleteIndicator extends V1_ActionIndicatorFields { + deleteValues!: string[]; +} + +export class V1_DeleteIndicatorForGraphFetch extends V1_DeleteIndicator { + get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.DELETE_INDICATOR_FOR_GRAPH_FETCH, + ]); + } +} + +export class V1_DeleteIndicatorForTds extends V1_DeleteIndicator { + deleteField!: string; + + get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.DELETE_INDICATOR_FOR_TDS, + this.deleteField, + ]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_AppendStrategy.ts b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_AppendStrategy.ts new file mode 100644 index 0000000000..0cb5c0ab79 --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_AppendStrategy.ts @@ -0,0 +1,40 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type Hashable, hashArray } from '@finos/legend-shared'; +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../../../graph/DSL_Persistence_HashUtils.js'; + +export abstract class V1_AppendStrategy implements Hashable { + abstract get hashCode(): string; +} + +export class V1_AllowDuplicates extends V1_AppendStrategy implements Hashable { + override get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.ALLOW_DUPLICATES]); + } +} + +export class V1_FailOnDuplicates extends V1_AppendStrategy implements Hashable { + override get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.FAIL_ON_DUPLICATES]); + } +} + +export class V1_FilterDuplicates extends V1_AppendStrategy implements Hashable { + override get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.FAIL_ON_DUPLICATES]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_AuditingV2.ts b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_AuditingV2.ts new file mode 100644 index 0000000000..4a2ac72b15 --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_AuditingV2.ts @@ -0,0 +1,39 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../../../graph/DSL_Persistence_HashUtils.js'; +import { type Hashable, hashArray } from '@finos/legend-shared'; + +export abstract class V1_AuditingV2 implements Hashable { + abstract get hashCode(): string; +} + +export class V1_AuditingDateTimeV2 extends V1_AuditingV2 { + auditingDateTimeName!: string; + + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.AUDITING_DATE_TIME, + this.auditingDateTimeName, + ]); + } +} + +export class V1_NoAuditingV2 extends V1_AuditingV2 { + override get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.NO_AUDITING_V2]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_DatasetType.ts b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_DatasetType.ts new file mode 100644 index 0000000000..b283c0a9a0 --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_DatasetType.ts @@ -0,0 +1,40 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type Hashable, hashArray } from '@finos/legend-shared'; +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../../../graph/DSL_Persistence_HashUtils.js'; +import type { V1_ActionIndicatorFields } from './V1_DSL_Persistence_ActionIndicatorFields.js'; +import type { V1_Partitioning } from './V1_DSL_Persistence_Partitioning.js'; + +export abstract class V1_DatasetType implements Hashable { + abstract get hashCode(): string; +} + +export class V1_Snapshot extends V1_DatasetType { + partitioning!: V1_Partitioning; + + override get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.SNAPSHOT, this.partitioning]); + } +} + +export class V1_Delta extends V1_DatasetType { + actionIndicator!: V1_ActionIndicatorFields; + + override get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.DELTA, this.actionIndicator]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_Deduplication.ts b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_Deduplication.ts new file mode 100644 index 0000000000..0c2c63c224 --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_Deduplication.ts @@ -0,0 +1,53 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type Hashable, hashArray } from '@finos/legend-shared'; +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../../../graph/DSL_Persistence_HashUtils.js'; + +export abstract class V1_Deduplication implements Hashable { + abstract get hashCode(): string; +} + +export class V1_NoDeduplication extends V1_Deduplication { + override get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.NO_DEDUPLICATION]); + } +} + +export class V1_AnyVersion extends V1_Deduplication { + override get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.ANY_VERSION]); + } +} + +export abstract class V1_MaxVersion extends V1_Deduplication {} + +export class V1_MaxVersionForGraphFetch extends V1_MaxVersion { + override get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.MAX_VERSION_FOR_GRAPH_FETCH]); + } +} + +export class V1_MaxVersionForTds extends V1_MaxVersion { + versionField!: string; + + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.MAX_VERSION_FOR_TDS, + this.versionField, + ]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_EmptyDatasetHandling.ts b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_EmptyDatasetHandling.ts new file mode 100644 index 0000000000..85636b65eb --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_EmptyDatasetHandling.ts @@ -0,0 +1,34 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type Hashable, hashArray } from '@finos/legend-shared'; +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../../../graph/DSL_Persistence_HashUtils.js'; + +export abstract class V1_EmptyDatasetHandling implements Hashable { + abstract get hashCode(): string; +} + +export class V1_NoOp extends V1_EmptyDatasetHandling { + get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.NO_OP_DATASET_HANDLING]); + } +} + +export class V1_DeleteTargetDataset extends V1_EmptyDatasetHandling { + get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.DELETE_TARGET_DATASET]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_Partitioning.ts b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_Partitioning.ts new file mode 100644 index 0000000000..874cb12358 --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_Partitioning.ts @@ -0,0 +1,53 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type Hashable, hashArray } from '@finos/legend-shared'; +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../../../graph/DSL_Persistence_HashUtils.js'; +import type { V1_EmptyDatasetHandling } from './V1_DSL_Persistence_EmptyDatasetHandling.js'; + +export abstract class V1_Partitioning implements Hashable { + abstract get hashCode(): string; +} + +export class V1_NoPartitioning extends V1_Partitioning { + emptyDatasetHandling!: V1_EmptyDatasetHandling; + + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.NO_PARTITIONING, + this.emptyDatasetHandling, + ]); + } +} + +export abstract class V1_FieldBased extends V1_Partitioning {} + +export class V1_FieldBasedForGraphFetch extends V1_FieldBased { + override get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.FIELD_BASED_FOR_GRAPH_FETCH]); + } +} + +export class V1_FieldBasedForTds extends V1_FieldBased { + partitionFields: string[] = []; + + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.FIEDD_BASED_FOR_TDS, + hashArray(this.partitionFields), + ]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_Persistence.ts b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_Persistence.ts index e91e61a943..8c67ab5af6 100644 --- a/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_Persistence.ts +++ b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_Persistence.ts @@ -24,12 +24,14 @@ import { import { type Hashable, hashArray } from '@finos/legend-shared'; import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../../../graph/DSL_Persistence_HashUtils.js'; import type { V1_PersistenceTest } from './V1_DSL_Persistence_PersistenceTest.js'; +import type { V1_ServiceOutputTarget as V1_ServiceOutputTarget } from './V1_DSL_Persistence_ServiceOutputTarget.js'; export class V1_Persistence extends V1_PackageableElement implements Hashable { documentation!: string; trigger!: V1_Trigger; service!: string; - persister!: V1_Persister; + persister?: V1_Persister | undefined; + serviceOutputTargets?: V1_ServiceOutputTarget[] | undefined; notifier!: V1_Notifier; tests: V1_PersistenceTest[] = []; @@ -39,7 +41,8 @@ export class V1_Persistence extends V1_PackageableElement implements Hashable { this.documentation, this.trigger, this.service, - this.persister, + hashArray(this.serviceOutputTargets ?? []), + this.persister ?? '', this.notifier, hashArray(this.tests), ]); diff --git a/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_PersistenceTestBatch.ts b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_PersistenceTestBatch.ts index 5842ec59d9..fa941be20b 100644 --- a/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_PersistenceTestBatch.ts +++ b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_PersistenceTestBatch.ts @@ -29,6 +29,7 @@ export class V1_PersistenceTestBatch implements Hashable { return hashArray([ PERSISTENCE_HASH_STRUCTURE.PERSISTENCE_TEST_BATCH, this.id, + this.batchId, this.testData, hashArray(this.assertions), ]); diff --git a/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_PersistentTarget.ts b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_PersistentTarget.ts new file mode 100644 index 0000000000..874b2351ef --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_PersistentTarget.ts @@ -0,0 +1,41 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type Hashable, hashArray } from '@finos/legend-shared'; +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../../../graph/DSL_Persistence_HashUtils.js'; +import type { V1_Temporality } from './V1_DSL_Persistence_Temporality.js'; + +export abstract class V1_PersistenceTarget implements Hashable { + abstract get hashCode(): string; +} + +export class V1_RelationalPersistenceTarget + extends V1_PersistenceTarget + implements Hashable +{ + table!: string; + database!: string; + temporality!: V1_Temporality; + + get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.RELATIONAL_PERSISTENCE_TARGET, + this.table, + this.database, + this.temporality, + ]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_ProcessingDimension.ts b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_ProcessingDimension.ts new file mode 100644 index 0000000000..08fee3faea --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_ProcessingDimension.ts @@ -0,0 +1,65 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type Hashable, hashArray } from '@finos/legend-shared'; +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../../../graph/DSL_Persistence_HashUtils.js'; + +export abstract class V1_ProcessingDimension implements Hashable { + abstract get hashCode(): string; +} + +export class V1_BatchId extends V1_ProcessingDimension implements Hashable { + batchIdIn!: string; + batchIdOut!: string; + + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.BATCH_ID, + this.batchIdIn, + this.batchIdOut, + ]); + } +} + +export class V1_ProcessingDateTime extends V1_ProcessingDimension { + timeIn!: string; + timeOut!: string; + + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.PROCESSING_DATE_TIME, + this.timeIn, + this.timeOut, + ]); + } +} + +export class V1_BatchIdAndDateTime extends V1_ProcessingDimension { + batchIdIn!: string; + batchIdOut!: string; + timeIn!: string; + timeOut!: string; + + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.BATCH_ID_AND_DATE_TIME, + this.batchIdIn, + this.batchIdOut, + this.timeIn, + this.timeOut, + ]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_ServiceOutput.ts b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_ServiceOutput.ts new file mode 100644 index 0000000000..ddf49b6dce --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_ServiceOutput.ts @@ -0,0 +1,49 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type Hashable, hashArray } from '@finos/legend-shared'; +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../../../graph/DSL_Persistence_HashUtils.js'; +import type { V1_DatasetType } from './V1_DSL_Persistence_DatasetType.js'; +import type { V1_Deduplication } from './V1_DSL_Persistence_Deduplication.js'; + +export abstract class V1_ServiceOutput implements Hashable { + deduplication!: V1_Deduplication; + datasetType!: V1_DatasetType; + + abstract get hashCode(): string; +} + +export class V1_GraphFetchServiceOutput extends V1_ServiceOutput { + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.GRAPH_FETCH_SERVICE_OUTPUT, + this.deduplication, + this.datasetType, + ]); + } +} + +export class V1_TdsServiceOutput extends V1_ServiceOutput { + keys: string[] = []; + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.TDS_SERVICE_OUTPUT, + this.deduplication, + this.datasetType, + hashArray(this.keys), + ]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_ServiceOutputTarget.ts b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_ServiceOutputTarget.ts new file mode 100644 index 0000000000..ad86a16f3b --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_ServiceOutputTarget.ts @@ -0,0 +1,33 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../../../graph/DSL_Persistence_HashUtils.js'; +import { type Hashable, hashArray } from '@finos/legend-shared'; +import type { V1_PersistenceTarget } from './V1_DSL_Persistence_PersistentTarget.js'; +import type { V1_ServiceOutput } from './V1_DSL_Persistence_ServiceOutput.js'; + +export class V1_ServiceOutputTarget implements Hashable { + serviceOutput!: V1_ServiceOutput; + persistenceTarget!: V1_PersistenceTarget; + + get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.SERVICE_OUTPUT_TARGET, + this.serviceOutput, + this.persistenceTarget, + ]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_SourceDerivedDimension.ts b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_SourceDerivedDimension.ts new file mode 100644 index 0000000000..624cc2fd6f --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_SourceDerivedDimension.ts @@ -0,0 +1,37 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type Hashable, hashArray } from '@finos/legend-shared'; +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../../../graph/DSL_Persistence_HashUtils.js'; +import type { V1_SourceTimeFields } from './V1_DSL_Persistence_SourceTimeFields.js'; + +export abstract class V1_SourceDerivedDimension implements Hashable { + abstract get hashCode(): string; +} + +export class V1_SourceDerivedTime extends V1_SourceDerivedDimension { + timeStart!: string; + timeEnd!: string; + sourceTimeFields!: V1_SourceTimeFields; + + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.SOURCE_DERIVED_TIME, + this.timeStart, + this.timeEnd, + ]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_SourceTimeFields.ts b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_SourceTimeFields.ts new file mode 100644 index 0000000000..d0eb243fe7 --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_SourceTimeFields.ts @@ -0,0 +1,46 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type Hashable, hashArray } from '@finos/legend-shared'; +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../../../graph/DSL_Persistence_HashUtils.js'; + +export abstract class V1_SourceTimeFields implements Hashable { + abstract get hashCode(): string; +} + +export class V1_SourceTimeStart extends V1_SourceTimeFields { + startField!: string; + + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.SOURCE_TIME_START, + this.startField, + ]); + } +} + +export class V1_SourceTimeStartAndEnd extends V1_SourceTimeFields { + startField!: string; + endField!: string; + + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.SOURCE_TIME_START_AND_END, + this.startField, + this.endField, + ]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_Temporality.ts b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_Temporality.ts new file mode 100644 index 0000000000..daa0272e56 --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_Temporality.ts @@ -0,0 +1,63 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type Hashable, hashArray } from '@finos/legend-shared'; +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../../../graph/DSL_Persistence_HashUtils.js'; +import type { V1_UpdatesHandling } from './V1_DSL_Persistence_UpdatesHandling.js'; +import type { V1_ProcessingDimension } from './V1_DSL_Persistence_ProcessingDimension.js'; +import type { V1_SourceDerivedDimension } from './V1_DSL_Persistence_SourceDerivedDimension.js'; +import type { V1_AuditingV2 } from './V1_DSL_Persistence_AuditingV2.js'; + +export abstract class V1_Temporality implements Hashable { + abstract get hashCode(): string; +} + +export class V1_NonTemporal extends V1_Temporality { + auditing!: V1_AuditingV2; + updatesHandling!: V1_UpdatesHandling; + + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.NON_TEMPORAL, + this.auditing, + this.updatesHandling, + ]); + } +} + +export class V1_UniTemporal extends V1_Temporality { + processingDimension!: V1_ProcessingDimension; + + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.UNI_TEMPORAL, + this.processingDimension, + ]); + } +} + +export class V1_BiTemporal extends V1_Temporality { + processingDimension!: V1_ProcessingDimension; + sourceDerivedDimension!: V1_SourceDerivedDimension; + + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.UNI_TEMPORAL, + this.processingDimension, + this.sourceDerivedDimension, + ]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_UpdatesHandling.ts b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_UpdatesHandling.ts new file mode 100644 index 0000000000..206bae3a6f --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/model/packageableElements/persistence/V1_DSL_Persistence_UpdatesHandling.ts @@ -0,0 +1,46 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type Hashable, hashArray } from '@finos/legend-shared'; +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../../../graph/DSL_Persistence_HashUtils.js'; +import type { V1_AppendStrategy } from './V1_DSL_Persistence_AppendStrategy.js'; + +export abstract class V1_UpdatesHandling implements Hashable { + abstract get hashCode(): string; +} + +export class V1_AppendOnlyUpdatesHandling + extends V1_UpdatesHandling + implements Hashable +{ + appendStrategy!: V1_AppendStrategy; + + get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.APPEND_ONLY_UPDATES, + this.appendStrategy, + ]); + } +} + +export class V1_OverwriteUpdatesHandling + extends V1_UpdatesHandling + implements Hashable +{ + get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.OVERWRITE_UPDATES]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/transformation/pureGraph/from/V1_DSL_Persistence_TransformerHelper.ts b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/transformation/pureGraph/from/V1_DSL_Persistence_TransformerHelper.ts index bdff3017f1..56a33be4c5 100644 --- a/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/transformation/pureGraph/from/V1_DSL_Persistence_TransformerHelper.ts +++ b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/transformation/pureGraph/from/V1_DSL_Persistence_TransformerHelper.ts @@ -163,6 +163,164 @@ import { V1_PersistenceTestData } from '../../../model/packageableElements/persi import type { ConnectionTestData } from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_ConnectionTestData.js'; import { V1_ConnectionTestData } from '../../../model/packageableElements/persistence/V1_DSL_Persistence_ConnectionTestData.js'; import { V1_PersistenceTest } from '../../../model/packageableElements/persistence/V1_DSL_Persistence_PersistenceTest.js'; +import { V1_ServiceOutputTarget } from '../../../model/packageableElements/persistence/V1_DSL_Persistence_ServiceOutputTarget.js'; +import type { ServiceOutputTarget } from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_ServiceOutputTarget.js'; +import { + GraphFetchServiceOutput, + type ServiceOutput, + TdsServiceOutput, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_ServiceOutput.js'; +import { + V1_GraphFetchServiceOutput, + type V1_ServiceOutput, + V1_TdsServiceOutput, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_ServiceOutput.js'; +import { + type DatasetType, + Delta, + Snapshot, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_DatasetType.js'; +import { + type V1_DatasetType, + V1_Delta, + V1_Snapshot, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_DatasetType.js'; +import { + FieldBased, + FieldBasedForGraphFetch, + FieldBasedForTds, + NoPartitioning, + type Partitioning, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_Partitioning.js'; +import { + type V1_FieldBased, + V1_FieldBasedForGraphFetch, + V1_FieldBasedForTds, + V1_NoPartitioning, + type V1_Partitioning, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_Partitioning.js'; +import { + DeleteTargetDataset, + type EmptyDatasetHandling, + NoOp, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_EmptyDatasetHandling.js'; +import { + V1_DeleteTargetDataset, + type V1_EmptyDatasetHandling, + V1_NoOp, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_EmptyDatasetHandling.js'; +import { + type ActionIndicatorFields, + DeleteIndicator, + DeleteIndicatorForGraphFetch, + DeleteIndicatorForTds, + NoActionIndicator, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_ActionIndicatorFields.js'; +import { + type V1_ActionIndicatorFields, + type V1_DeleteIndicator, + V1_DeleteIndicatorForGraphFetch, + V1_DeleteIndicatorForTds, + V1_NoActionIndicator, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_ActionIndicatorFields.js'; +import { + AnyVersion, + type Deduplication, + MaxVersion, + MaxVersionForGraphFetch, + MaxVersionForTds, + NoDeduplication, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_Deduplication.js'; +import { + V1_AnyVersion, + type V1_Deduplication, + type V1_MaxVersion, + V1_MaxVersionForGraphFetch, + V1_MaxVersionForTds, + V1_NoDeduplication, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_Deduplication.js'; +import { + type PersistenceTarget, + RelationalPersistenceTarget, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_PersistentTarget.js'; +import { + BiTemporal, + NonTemporal, + type Temporality, + UniTemporal, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_Temporality.js'; +import { + V1_BiTemporal, + V1_NonTemporal, + type V1_Temporality, + V1_UniTemporal, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_Temporality.js'; +import { + AuditingDateTimeV2, + type AuditingV2, + NoAuditingV2, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_AuditingV2.js'; +import { + V1_AuditingDateTimeV2, + type V1_AuditingV2, + V1_NoAuditingV2, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_AuditingV2.js'; +import { + AppendOnlyUpdatesHandling, + OverwriteUpdatesHandling, + type UpdatesHandling, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_UpdatesHandling.js'; +import { + V1_AppendOnlyUpdatesHandling, + V1_OverwriteUpdatesHandling, + type V1_UpdatesHandling, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_UpdatesHandling.js'; +import { + AllowDuplicates, + type AppendStrategy, + FailOnDuplicates, + FilterDuplicates, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_AppendStrategy.js'; +import { + V1_AllowDuplicates, + type V1_AppendStrategy, + V1_FailOnDuplicates, + V1_FilterDuplicates, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_AppendStrategy.js'; +import { + BatchId, + BatchIdAndDateTime, + ProcessingDateTime, + type ProcessingDimension, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_ProcessingDimension.js'; +import { + V1_BatchId, + V1_BatchIdAndDateTime, + V1_ProcessingDateTime, + type V1_ProcessingDimension, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_ProcessingDimension.js'; +import { + type SourceDerivedDimension, + SourceDerivedTime, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_SourceDerivedDimension.js'; +import { + type V1_SourceDerivedDimension, + V1_SourceDerivedTime, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_SourceDerivedDimension.js'; +import { + type SourceTimeFields, + SourceTimeStart, + SourceTimeStartAndEnd, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_SourceTimeFields.js'; +import { + type V1_SourceTimeFields, + V1_SourceTimeStart, + V1_SourceTimeStartAndEnd, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_SourceTimeFields.js'; +import { + type V1_PersistenceTarget, + V1_RelationalPersistenceTarget, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_PersistentTarget.js'; /********** * trigger @@ -568,9 +726,12 @@ export const V1_transformTargetShape = ( **********/ export const V1_transformPersister = ( - element: Persister, + element: Persister | undefined, context: V1_GraphTransformerContext, -): V1_Persister => { +): V1_Persister | undefined => { + if (element === undefined) { + return element; + } if (element instanceof StreamingPersister) { const protocol = new V1_StreamingPersister(); protocol.sink = V1_transformSink(element.sink, context); @@ -588,6 +749,232 @@ export const V1_transformPersister = ( throw new UnsupportedOperationError(`Can't transform persister '${element}'`); }; +/*************** + * deduplication + ***************/ + +export const V1_transformMaxVersion = ( + element: MaxVersion, + context: V1_GraphTransformerContext, +): V1_MaxVersion => { + if (element instanceof MaxVersionForGraphFetch) { + return new V1_MaxVersionForGraphFetch(); + } else if (element instanceof MaxVersionForTds) { + const protocol = new V1_MaxVersionForTds(); + protocol.versionField = element.versionField; + return protocol; + } + throw new UnsupportedOperationError(`Can't transform MaxVersion`, element); +}; + +export const V1_transformDeduplication = ( + element: Deduplication, + context: V1_GraphTransformerContext, +): V1_Deduplication => { + if (element instanceof NoDeduplication) { + return new V1_NoDeduplication(); + } else if (element instanceof AnyVersion) { + return new V1_AnyVersion(); + } else if (element instanceof MaxVersion) { + return V1_transformMaxVersion(element, context); + } + throw new UnsupportedOperationError(`Can't transform deduplication`, element); +}; + +/********** + * auditing + **********/ + +const V1_transformAuditingV2 = ( + element: AuditingV2, + context: V1_GraphTransformerContext, +): V1_AuditingV2 => { + if (element instanceof NoAuditingV2) { + return new V1_NoAuditingV2(); + } else if (element instanceof AuditingDateTimeV2) { + const protocol = new V1_AuditingDateTimeV2(); + protocol.auditingDateTimeName = element.auditingDateTimeName; + return protocol; + } + throw new UnsupportedOperationError(`Can't transform AuditingV2`, element); +}; + +/****************** + * updates handling + ******************/ + +const V1_transformAppendOnlyUpdatesHandling = ( + element: AppendStrategy, + context: V1_GraphTransformerContext, +): V1_AppendStrategy => { + if (element instanceof AllowDuplicates) { + return new V1_AllowDuplicates(); + } else if (element instanceof FailOnDuplicates) { + return new V1_FailOnDuplicates(); + } else if (element instanceof FilterDuplicates) { + return new V1_FilterDuplicates(); + } + throw new UnsupportedOperationError( + `Can't transform AppendOnlyUpdatesHandling`, + element, + ); +}; + +const V1_transformUpdatesHandling = ( + element: UpdatesHandling, + context: V1_GraphTransformerContext, +): V1_UpdatesHandling => { + if (element instanceof AppendOnlyUpdatesHandling) { + const protocol = new V1_AppendOnlyUpdatesHandling(); + protocol.appendStrategy = V1_transformAppendOnlyUpdatesHandling( + element.appendStrategy, + context, + ); + return protocol; + } + if (element instanceof OverwriteUpdatesHandling) { + return new V1_OverwriteUpdatesHandling(); + } + throw new UnsupportedOperationError( + `Can't transform UpdatesHandling`, + element, + ); +}; + +/********************* + * processing dimesion + *********************/ + +const V1_transformProcessingDimension = ( + element: ProcessingDimension, + context: V1_GraphTransformerContext, +): V1_ProcessingDimension => { + if (element instanceof BatchId) { + const protocol = new V1_BatchId(); + protocol.batchIdIn = element.batchIdIn; + protocol.batchIdOut = element.batchIdOut; + return protocol; + } else if (element instanceof ProcessingDateTime) { + const protocol = new V1_ProcessingDateTime(); + protocol.timeIn = element.timeIn; + protocol.timeOut = element.timeOut; + return protocol; + } else if (element instanceof BatchIdAndDateTime) { + const protocol = new V1_BatchIdAndDateTime(); + protocol.batchIdIn = element.batchIdIn; + protocol.batchIdOut = element.batchIdOut; + protocol.timeIn = element.timeIn; + protocol.timeOut = element.timeOut; + return protocol; + } + throw new UnsupportedOperationError( + `Can't transform ProcessingDimension`, + element, + ); +}; + +const V1_transformSourceTimeFields = ( + element: SourceTimeFields, + context: V1_GraphTransformerContext, +): V1_SourceTimeFields => { + if (element instanceof SourceTimeStart) { + const protocol = new V1_SourceTimeStart(); + protocol.startField = element.startField; + return protocol; + } else if (element instanceof SourceTimeStartAndEnd) { + const protocol = new V1_SourceTimeStartAndEnd(); + protocol.endField = element.endField; + protocol.startField = element.startField; + return protocol; + } + throw new UnsupportedOperationError( + `Can't transform SourceTimeFields`, + element, + ); +}; + +const V1_transformSourceDerivedDimension = ( + element: SourceDerivedDimension, + context: V1_GraphTransformerContext, +): V1_SourceDerivedDimension => { + if (element instanceof SourceDerivedTime) { + const sourceDerivedTime = new V1_SourceDerivedTime(); + sourceDerivedTime.sourceTimeFields = V1_transformSourceTimeFields( + element.sourceTimeFields, + context, + ); + sourceDerivedTime.timeEnd = element.timeEnd; + sourceDerivedTime.timeStart = element.timeStart; + return sourceDerivedTime; + } + throw new UnsupportedOperationError( + `Can't transform SourceDerivedDimension`, + element, + ); +}; + +/************* + * temporality + *************/ + +export const V1_transformTemporality = ( + element: Temporality, + context: V1_GraphTransformerContext, +): V1_Temporality => { + if (element instanceof NonTemporal) { + const protocol = new V1_NonTemporal(); + protocol.auditing = V1_transformAuditingV2(element.auditing, context); + protocol.updatesHandling = V1_transformUpdatesHandling( + element.updatesHandling, + context, + ); + return protocol; + } else if (element instanceof UniTemporal) { + const protocol = new V1_UniTemporal(); + protocol.processingDimension = V1_transformProcessingDimension( + element.processingDimension, + context, + ); + return protocol; + } else if (element instanceof BiTemporal) { + const protocol = new V1_BiTemporal(); + protocol.processingDimension = V1_transformProcessingDimension( + element.processingDimension, + context, + ); + protocol.sourceDerivedDimension = V1_transformSourceDerivedDimension( + element.sourceDerivedDimension, + context, + ); + return protocol; + } + throw new UnsupportedOperationError(`Can't transform Temporality`, element); +}; + +/******************** + * persistence target + ********************/ + +export const V1_transformPersistenceTarget = ( + element: PersistenceTarget, + context: V1_GraphTransformerContext, +): V1_PersistenceTarget => { + if (element instanceof RelationalPersistenceTarget) { + const protocol = new V1_RelationalPersistenceTarget(); + protocol.database = element.database; + protocol.table = element.table; + protocol.temporality = V1_transformTemporality( + element.temporality, + context, + ); + return protocol; + } + throw new UnsupportedOperationError( + `Can't transform PersistenceTarget`, + element, + ); +}; + /********** * notifier **********/ @@ -649,6 +1036,7 @@ export const V1_transformPersistenceTestBatch = ( context: V1_GraphTransformerContext, ): V1_PersistenceTestBatch => { const persistenceTestBatch = new V1_PersistenceTestBatch(); + persistenceTestBatch.batchId = element.batchId; persistenceTestBatch.id = element.id; persistenceTestBatch.testData = V1_transformTestData( element.testData, @@ -660,6 +1048,190 @@ export const V1_transformPersistenceTestBatch = ( return persistenceTestBatch; }; +/****************** + * action indicator + ******************/ + +export const V1_transformDeleteIndicator = ( + element: DeleteIndicator, + context: V1_GraphTransformerContext, +): V1_DeleteIndicator => { + if (element instanceof DeleteIndicatorForGraphFetch) { + const protocol = new V1_DeleteIndicatorForGraphFetch(); + protocol.deleteValues = element.deleteValues; + return protocol; + } else if (element instanceof DeleteIndicatorForTds) { + const protocol = new V1_DeleteIndicatorForTds(); + protocol.deleteField = element.deleteField; + protocol.deleteValues = element.deleteValues; + return protocol; + } + throw new UnsupportedOperationError( + `Can't transform DeleteIndicator`, + element, + ); +}; + +export const V1_transformActionIndicator = ( + element: ActionIndicatorFields, + context: V1_GraphTransformerContext, +): V1_ActionIndicatorFields => { + if (element instanceof NoActionIndicator) { + return new V1_NoActionIndicator(); + } else if (element instanceof DeleteIndicator) { + return V1_transformDeleteIndicator(element, context); + } + throw new UnsupportedOperationError( + `Can't transform ActionIndicator`, + element, + ); +}; + +/************************ + * empty dataset handling + ***********************/ + +export const V1_transformEmptyDatasetHandling = ( + element: EmptyDatasetHandling, + context: V1_GraphTransformerContext, +): V1_EmptyDatasetHandling => { + if (element instanceof NoOp) { + return new V1_NoOp(); + } else if (element instanceof DeleteTargetDataset) { + return new V1_DeleteTargetDataset(); + } + throw new UnsupportedOperationError( + `Can't transform EmptyDatasetHandling`, + element, + ); +}; + +/************** + * partitioning + **************/ + +export const V1_transformFieldBasedPartitioning = ( + protocol: FieldBased, + context: V1_GraphTransformerContext, +): V1_FieldBased => { + if (protocol instanceof FieldBasedForGraphFetch) { + return new V1_FieldBasedForGraphFetch(); + } else if (protocol instanceof FieldBasedForTds) { + const fieldBasedForTds = new V1_FieldBasedForTds(); + fieldBasedForTds.partitionFields = protocol.partitionFields; + return fieldBasedForTds; + } + throw new UnsupportedOperationError( + `Can't transform FieldBasedPartitioning`, + protocol, + ); +}; + +export const V1_transformPartitioning = ( + element: Partitioning, + context: V1_GraphTransformerContext, +): V1_Partitioning => { + if (element instanceof NoPartitioning) { + const protocol = new V1_NoPartitioning(); + protocol.emptyDatasetHandling = V1_transformEmptyDatasetHandling( + element.emptyDatasetHandling, + context, + ); + return protocol; + } else if (element instanceof FieldBased) { + return V1_transformFieldBasedPartitioning(element, context); + } + throw new UnsupportedOperationError(`Can't transform Partitioning`, element); +}; + +/**************** + * dataset type + ****************/ + +export const V1_transformDatasetType = ( + element: DatasetType, + context: V1_GraphTransformerContext, +): V1_DatasetType => { + if (element instanceof Snapshot) { + const protocol = new V1_Snapshot(); + protocol.partitioning = V1_transformPartitioning( + element.partitioning, + context, + ); + return protocol; + } else if (element instanceof Delta) { + const protocol = new V1_Delta(); + protocol.actionIndicator = V1_transformActionIndicator( + element.actionIndicator, + context, + ); + return protocol; + } + throw new UnsupportedOperationError(`Can't transform DatasetType`, element); +}; + +/**************** + * service output + ****************/ + +export const V1_transformServiceOutput = ( + element: ServiceOutput, + context: V1_GraphTransformerContext, +): V1_ServiceOutput => { + if (element instanceof GraphFetchServiceOutput) { + const protocol = new V1_GraphFetchServiceOutput(); + protocol.datasetType = V1_transformDatasetType( + element.datasetType, + context, + ); + protocol.deduplication = V1_transformDeduplication( + element.deduplication, + context, + ); + return protocol; + } else if (element instanceof TdsServiceOutput) { + const protocol = new V1_TdsServiceOutput(); + protocol.datasetType = V1_transformDatasetType( + element.datasetType, + context, + ); + protocol.deduplication = V1_transformDeduplication( + element.deduplication, + context, + ); + protocol.keys = element.keys; + return protocol; + } + throw new UnsupportedOperationError(`Can't transform ServiceOutput`, element); +}; + +/************************ + * service output targets + ***********************/ + +export const V1_transformServiceOutputTargets = ( + element: ServiceOutputTarget[] | undefined, + context: V1_GraphTransformerContext, +): V1_ServiceOutputTarget[] | undefined => { + if (element === undefined) { + return element; + } + const protocol: V1_ServiceOutputTarget[] = []; + for (const v1ServiceOutputTarget of element) { + const serviceOutputTarget = new V1_ServiceOutputTarget(); + serviceOutputTarget.persistenceTarget = V1_transformPersistenceTarget( + v1ServiceOutputTarget.persistenceTarget, + context, + ); + serviceOutputTarget.serviceOutput = V1_transformServiceOutput( + v1ServiceOutputTarget.serviceOutput, + context, + ); + protocol.push(serviceOutputTarget); + } + return protocol; +}; + /********** * persistence **********/ @@ -671,12 +1243,16 @@ export const V1_transformPersistence = ( const protocol = new V1_Persistence(); V1_initPackageableElement(protocol, element); protocol.documentation = element.documentation; - protocol.trigger = V1_transformTrigger(element.trigger, context); - protocol.service = element.service.valueForSerialization ?? ''; - protocol.persister = V1_transformPersister(element.persister, context); protocol.notifier = V1_transformNotifier(element.notifier, context); + protocol.persister = V1_transformPersister(element.persister, context); + protocol.service = element.service.valueForSerialization ?? ''; + protocol.serviceOutputTargets = V1_transformServiceOutputTargets( + element.serviceOutputTargets, + context, + ); protocol.tests = element.tests.map((test) => guaranteeType(V1_transformAtomicTest(test, context), V1_PersistenceTest), ); + protocol.trigger = V1_transformTrigger(element.trigger, context); return protocol; }; diff --git a/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/transformation/pureGraph/to/V1_DSL_Persistence_BuilderHelper.ts b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/transformation/pureGraph/to/V1_DSL_Persistence_BuilderHelper.ts index 2d8d876fd4..e831a47354 100644 --- a/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/transformation/pureGraph/to/V1_DSL_Persistence_BuilderHelper.ts +++ b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/transformation/pureGraph/to/V1_DSL_Persistence_BuilderHelper.ts @@ -173,6 +173,164 @@ import type { V1_PersistenceTestData } from '../../../model/packageableElements/ import { PersistenceTestData } from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_PersistenceTestData.js'; import type { V1_ConnectionTestData } from '../../../model/packageableElements/persistence/V1_DSL_Persistence_ConnectionTestData.js'; import { ConnectionTestData } from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_ConnectionTestData.js'; +import type { V1_ServiceOutputTarget } from '../../../model/packageableElements/persistence/V1_DSL_Persistence_ServiceOutputTarget.js'; +import { + GraphFetchServiceOutput, + type ServiceOutput, + TdsServiceOutput, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_ServiceOutput.js'; +import { ServiceOutputTarget } from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_ServiceOutputTarget.js'; +import { + V1_GraphFetchServiceOutput, + type V1_ServiceOutput, + V1_TdsServiceOutput, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_ServiceOutput.js'; +import { + type V1_DatasetType, + V1_Delta, + V1_Snapshot, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_DatasetType.js'; +import { + type DatasetType, + Delta, + Snapshot, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_DatasetType.js'; +import { + type FieldBased, + FieldBasedForGraphFetch, + FieldBasedForTds, + NoPartitioning, + type Partitioning, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_Partitioning.js'; +import { + V1_FieldBased, + V1_FieldBasedForGraphFetch, + V1_FieldBasedForTds, + V1_NoPartitioning, + type V1_Partitioning, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_Partitioning.js'; +import { + V1_DeleteTargetDataset, + type V1_EmptyDatasetHandling, + V1_NoOp, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_EmptyDatasetHandling.js'; +import { + DeleteTargetDataset, + type EmptyDatasetHandling, + NoOp, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_EmptyDatasetHandling.js'; +import { + type V1_ActionIndicatorFields, + V1_DeleteIndicator, + V1_DeleteIndicatorForGraphFetch, + V1_DeleteIndicatorForTds, + V1_NoActionIndicator, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_ActionIndicatorFields.js'; +import { + type ActionIndicatorFields, + type DeleteIndicator, + DeleteIndicatorForGraphFetch, + DeleteIndicatorForTds, + NoActionIndicator, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_ActionIndicatorFields.js'; +import { + V1_AnyVersion, + type V1_Deduplication, + V1_MaxVersion, + V1_MaxVersionForGraphFetch, + V1_MaxVersionForTds, + V1_NoDeduplication, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_Deduplication.js'; +import { + AnyVersion, + type Deduplication, + type MaxVersion, + MaxVersionForGraphFetch, + MaxVersionForTds, + NoDeduplication, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_Deduplication.js'; +import { + type V1_PersistenceTarget, + V1_RelationalPersistenceTarget, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_PersistentTarget.js'; +import { + type PersistenceTarget, + RelationalPersistenceTarget, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_PersistentTarget.js'; +import { + V1_BiTemporal, + V1_NonTemporal, + type V1_Temporality, + V1_UniTemporal, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_Temporality.js'; +import { + BiTemporal, + NonTemporal, + type Temporality, + UniTemporal, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_Temporality.js'; +import { + V1_AuditingDateTimeV2, + type V1_AuditingV2, + V1_NoAuditingV2, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_AuditingV2.js'; +import { + AuditingDateTimeV2, + type AuditingV2, + NoAuditingV2, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_AuditingV2.js'; +import { + V1_AppendOnlyUpdatesHandling, + V1_OverwriteUpdatesHandling, + type V1_UpdatesHandling, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_UpdatesHandling.js'; +import { + AppendOnlyUpdatesHandling, + OverwriteUpdatesHandling, + type UpdatesHandling, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_UpdatesHandling.js'; +import { + V1_AllowDuplicates, + type V1_AppendStrategy, + V1_FailOnDuplicates, + V1_FilterDuplicates, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_AppendStrategy.js'; +import { + AllowDuplicates, + type AppendStrategy, + FailOnDuplicates, + FilterDuplicates, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_AppendStrategy.js'; +import { + V1_BatchId, + V1_BatchIdAndDateTime, + V1_ProcessingDateTime, + type V1_ProcessingDimension, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_ProcessingDimension.js'; +import { + BatchId, + BatchIdAndDateTime, + ProcessingDateTime, + type ProcessingDimension, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_ProcessingDimension.js'; +import { + type V1_SourceDerivedDimension, + V1_SourceDerivedTime, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_SourceDerivedDimension.js'; +import { + type SourceDerivedDimension, + SourceDerivedTime, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_SourceDerivedDimension.js'; +import { + type V1_SourceTimeFields, + V1_SourceTimeStart, + V1_SourceTimeStartAndEnd, +} from '../../../model/packageableElements/persistence/V1_DSL_Persistence_SourceTimeFields.js'; +import { + type SourceTimeFields, + SourceTimeStart, + SourceTimeStartAndEnd, +} from '../../../../../../../graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_SourceTimeFields.js'; /********** * trigger @@ -588,9 +746,12 @@ export const V1_buildTargetShape = ( **********/ export const V1_buildPersister = ( - protocol: V1_Persister, + protocol: V1_Persister | undefined, context: V1_GraphBuilderContext, -): Persister => { +): Persister | undefined => { + if (protocol === undefined) { + return protocol; + } if (protocol instanceof V1_StreamingPersister) { const persister = new StreamingPersister(); persister.sink = V1_buildSink(protocol.sink, context); @@ -636,6 +797,435 @@ export const V1_buildNotifier = ( return notifier; }; +/************************ + * empty dataset handling + ************************/ + +export const V1_buildEmptyDatasetHandling = ( + protocol: V1_EmptyDatasetHandling, + context: V1_GraphBuilderContext, +): EmptyDatasetHandling => { + if (protocol instanceof V1_NoOp) { + return new NoOp(); + } else if (protocol instanceof V1_DeleteTargetDataset) { + return new DeleteTargetDataset(); + } + throw new UnsupportedOperationError( + `Can't build EmptyDatasetHandling`, + protocol, + ); +}; + +/************** + * partitioning + **************/ + +export const V1_buildFieldBasedPartitioning = ( + protocol: V1_FieldBased, + context: V1_GraphBuilderContext, +): FieldBased => { + if (protocol instanceof V1_FieldBasedForGraphFetch) { + return new FieldBasedForGraphFetch(); + } else if (protocol instanceof V1_FieldBasedForTds) { + const fieldBasedForTds = new FieldBasedForTds(); + fieldBasedForTds.partitionFields = protocol.partitionFields; + return fieldBasedForTds; + } + throw new UnsupportedOperationError( + `Can't build FieldBasedPartitioning`, + protocol, + ); +}; + +export const V1_buildPartitioning = ( + protocol: V1_Partitioning, + context: V1_GraphBuilderContext, +): Partitioning => { + if (protocol instanceof V1_NoPartitioning) { + const noPartitioning = new NoPartitioning(); + noPartitioning.emptyDatasetHandling = V1_buildEmptyDatasetHandling( + protocol.emptyDatasetHandling, + context, + ); + return noPartitioning; + } else if (protocol instanceof V1_FieldBased) { + return V1_buildFieldBasedPartitioning(protocol, context); + } + throw new UnsupportedOperationError(`Can't build Partitioning`, protocol); +}; + +/****************** + * action indicator + ******************/ + +export const V1_buildDeleteIndicator = ( + protocol: V1_DeleteIndicator, + context: V1_GraphBuilderContext, +): DeleteIndicator => { + if (protocol instanceof V1_DeleteIndicatorForGraphFetch) { + const deleteIndicatorForGraphFetch = new DeleteIndicatorForGraphFetch(); + deleteIndicatorForGraphFetch.deleteValues = protocol.deleteValues; + return deleteIndicatorForGraphFetch; + } else if (protocol instanceof V1_DeleteIndicatorForTds) { + const deleteIndicatorForTds = new DeleteIndicatorForTds(); + deleteIndicatorForTds.deleteField = protocol.deleteField; + deleteIndicatorForTds.deleteValues = protocol.deleteValues; + return deleteIndicatorForTds; + } + throw new UnsupportedOperationError(`Can't build DeleteIndicator`, protocol); +}; + +export const V1_buildActionIndicator = ( + protocol: V1_ActionIndicatorFields, + context: V1_GraphBuilderContext, +): ActionIndicatorFields => { + if (protocol instanceof V1_NoActionIndicator) { + return new NoActionIndicator(); + } else if (protocol instanceof V1_DeleteIndicator) { + return V1_buildDeleteIndicator(protocol, context); + } + throw new UnsupportedOperationError(`Can't build ActionIndicator`, protocol); +}; + +/************** + * dataset type + **************/ + +export const V1_buildDatasetType = ( + protocol: V1_DatasetType, + context: V1_GraphBuilderContext, +): DatasetType => { + if (protocol instanceof V1_Snapshot) { + const snapshot = new Snapshot(); + snapshot.partitioning = V1_buildPartitioning( + protocol.partitioning, + context, + ); + return snapshot; + } else if (protocol instanceof V1_Delta) { + const delta = new Delta(); + delta.actionIndicator = V1_buildActionIndicator( + protocol.actionIndicator, + context, + ); + return delta; + } + throw new UnsupportedOperationError(`Can't build DatasetType`, protocol); +}; + +/****************** + * de-duplication + ******************/ + +export const V1_buildMaxVersion = ( + protocol: V1_MaxVersion, + context: V1_GraphBuilderContext, +): MaxVersion => { + if (protocol instanceof V1_MaxVersionForGraphFetch) { + return new MaxVersionForGraphFetch(); + } else if (protocol instanceof V1_MaxVersionForTds) { + const maxVeresionForTds = new MaxVersionForTds(); + maxVeresionForTds.versionField = protocol.versionField; + return maxVeresionForTds; + } + throw new UnsupportedOperationError( + `Can't build MaxVersion + `, + protocol, + ); +}; + +export const V1_buildDeduplication = ( + protocol: V1_Deduplication, + context: V1_GraphBuilderContext, +): Deduplication => { + if (protocol instanceof V1_NoDeduplication) { + return new NoDeduplication(); + } else if (protocol instanceof V1_AnyVersion) { + return new AnyVersion(); + } else if (protocol instanceof V1_MaxVersion) { + return V1_buildMaxVersion(protocol, context); + } + throw new UnsupportedOperationError(`Can't build Deduplication`, protocol); +}; + +/**************** + * service output + ****************/ + +export const V1_buildServiceOutput = ( + protocol: V1_ServiceOutput, + context: V1_GraphBuilderContext, +): ServiceOutput => { + if (protocol instanceof V1_GraphFetchServiceOutput) { + const serviceOutput = new GraphFetchServiceOutput(); + serviceOutput.datasetType = V1_buildDatasetType( + protocol.datasetType, + context, + ); + serviceOutput.deduplication = V1_buildDeduplication( + protocol.deduplication, + context, + ); + return serviceOutput; + } else if (protocol instanceof V1_TdsServiceOutput) { + const serviceOutput = new TdsServiceOutput(); + serviceOutput.datasetType = V1_buildDatasetType( + protocol.datasetType, + context, + ); + serviceOutput.deduplication = V1_buildDeduplication( + protocol.deduplication, + context, + ); + serviceOutput.keys = protocol.keys; + return serviceOutput; + } + throw new UnsupportedOperationError(`Can't build ServiceOutput`, protocol); +}; + +/************* + * auditing V2 + *************/ + +const V1_buildAuditingV2 = ( + protocol: V1_AuditingV2, + context: V1_GraphBuilderContext, +): AuditingV2 => { + if (protocol instanceof V1_NoAuditingV2) { + return new NoAuditingV2(); + } else if (protocol instanceof V1_AuditingDateTimeV2) { + const auditingDateTimeV2 = new AuditingDateTimeV2(); + auditingDateTimeV2.auditingDateTimeName = protocol.auditingDateTimeName; + return auditingDateTimeV2; + } + throw new UnsupportedOperationError(`Can't build AuditingV2`, protocol); +}; + +/****************** + * updates handling + ******************/ + +const V1_buildAppendOnlyUpdatesHandling = ( + protocol: V1_AppendStrategy, + context: V1_GraphBuilderContext, +): AppendStrategy => { + if (protocol instanceof V1_AllowDuplicates) { + return new AllowDuplicates(); + } else if (protocol instanceof V1_FailOnDuplicates) { + return new FailOnDuplicates(); + } else if (protocol instanceof V1_FilterDuplicates) { + return new FilterDuplicates(); + } + throw new UnsupportedOperationError( + `Can't build AppendOnlyUpdatesHandling`, + protocol, + ); +}; + +const V1_buildUpdatesHandling = ( + protocol: V1_UpdatesHandling, + context: V1_GraphBuilderContext, +): UpdatesHandling => { + if (protocol instanceof V1_AppendOnlyUpdatesHandling) { + const appendOnlyUpdatesHandling = new AppendOnlyUpdatesHandling(); + appendOnlyUpdatesHandling.appendStrategy = + V1_buildAppendOnlyUpdatesHandling(protocol.appendStrategy, context); + return appendOnlyUpdatesHandling; + } + if (protocol instanceof V1_OverwriteUpdatesHandling) { + return new OverwriteUpdatesHandling(); + } + throw new UnsupportedOperationError(`Can't build UpdatesHandling`, protocol); +}; + +/********************** + * processing dimension + **********************/ + +const V1_buildProcessingDimension = ( + protocol: V1_ProcessingDimension, + context: V1_GraphBuilderContext, +): ProcessingDimension => { + if (protocol instanceof V1_BatchId) { + const batchId = new BatchId(); + batchId.batchIdIn = protocol.batchIdIn; + batchId.batchIdOut = protocol.batchIdOut; + return batchId; + } else if (protocol instanceof V1_ProcessingDateTime) { + const processingDateTime = new ProcessingDateTime(); + processingDateTime.timeIn = protocol.timeIn; + processingDateTime.timeOut = protocol.timeOut; + return processingDateTime; + } else if (protocol instanceof V1_BatchIdAndDateTime) { + const batchIdAndDateTime = new BatchIdAndDateTime(); + batchIdAndDateTime.batchIdIn = protocol.batchIdIn; + batchIdAndDateTime.batchIdOut = protocol.batchIdOut; + batchIdAndDateTime.timeIn = protocol.timeIn; + batchIdAndDateTime.timeOut = protocol.timeOut; + return batchIdAndDateTime; + } + throw new UnsupportedOperationError( + `Can't build ProcessingDimension`, + protocol, + ); +}; + +const V1_buildSourceTimeFields = ( + protocol: V1_SourceTimeFields, + context: V1_GraphBuilderContext, +): SourceTimeFields => { + if (protocol instanceof V1_SourceTimeStart) { + const sourceTimeStart = new SourceTimeStart(); + sourceTimeStart.startField = protocol.startField; + return sourceTimeStart; + } else if (protocol instanceof V1_SourceTimeStartAndEnd) { + const sourceTimeStartAndEnd = new SourceTimeStartAndEnd(); + sourceTimeStartAndEnd.endField = protocol.endField; + sourceTimeStartAndEnd.startField = protocol.startField; + return sourceTimeStartAndEnd; + } + throw new UnsupportedOperationError(`Can't build SourceTimeFields`, protocol); +}; + +const V1_buildSourceDerivedDimension = ( + protocol: V1_SourceDerivedDimension, + context: V1_GraphBuilderContext, +): SourceDerivedDimension => { + if (protocol instanceof V1_SourceDerivedTime) { + const sourceDerivedTime = new SourceDerivedTime(); + sourceDerivedTime.sourceTimeFields = V1_buildSourceTimeFields( + protocol.sourceTimeFields, + context, + ); + sourceDerivedTime.timeEnd = protocol.timeEnd; + sourceDerivedTime.timeStart = protocol.timeStart; + + return sourceDerivedTime; + } + throw new UnsupportedOperationError( + `Can't build SourceDerivedDimension`, + protocol, + ); +}; + +/************* + * temporality + *************/ + +export const V1_buildTemporality = ( + protocol: V1_Temporality, + context: V1_GraphBuilderContext, +): Temporality => { + if (protocol instanceof V1_NonTemporal) { + const nonTemporal = new NonTemporal(); + nonTemporal.auditing = V1_buildAuditingV2(protocol.auditing, context); + nonTemporal.updatesHandling = V1_buildUpdatesHandling( + protocol.updatesHandling, + context, + ); + return nonTemporal; + } else if (protocol instanceof V1_UniTemporal) { + const uniTemporal = new UniTemporal(); + uniTemporal.processingDimension = V1_buildProcessingDimension( + protocol.processingDimension, + context, + ); + return uniTemporal; + } else if (protocol instanceof V1_BiTemporal) { + const biTemporal = new BiTemporal(); + biTemporal.processingDimension = V1_buildProcessingDimension( + protocol.processingDimension, + context, + ); + biTemporal.sourceDerivedDimension = V1_buildSourceDerivedDimension( + protocol.sourceDerivedDimension, + context, + ); + return biTemporal; + } + throw new UnsupportedOperationError(`Can't build Temporality`, protocol); +}; + +/******************** + * persistence target + ********************/ + +export const V1_buildPersistenceTarget = ( + protocol: V1_PersistenceTarget, + context: V1_GraphBuilderContext, +): PersistenceTarget => { + if (protocol instanceof V1_RelationalPersistenceTarget) { + const persistentTarget = new RelationalPersistenceTarget(); + persistentTarget.database = protocol.database; + persistentTarget.table = protocol.table; + persistentTarget.temporality = V1_buildTemporality( + protocol.temporality, + context, + ); + return persistentTarget; + } + throw new UnsupportedOperationError( + `Can't build PersistenceTarget`, + protocol, + ); +}; + +/************************ + * service output targets + ***********************/ + +export const V1_buildServiceOutputTargets = ( + protocol: V1_ServiceOutputTarget[] | undefined, + context: V1_GraphBuilderContext, +): ServiceOutputTarget[] | undefined => { + if (protocol === undefined) { + return protocol; + } + const serviceOutputTargets: ServiceOutputTarget[] = []; + for (const v1ServiceOutputTarget of protocol) { + const serviceOutputTarget = new ServiceOutputTarget(); + serviceOutputTarget.persistenceTarget = V1_buildPersistenceTarget( + v1ServiceOutputTarget.persistenceTarget, + context, + ); + serviceOutputTarget.serviceOutput = V1_buildServiceOutput( + v1ServiceOutputTarget.serviceOutput, + context, + ); + + serviceOutputTargets.push(serviceOutputTarget); + } + return serviceOutputTargets; +}; + +/************* + * persistence + *************/ + +export const V1_buildPersistence = ( + protocol: V1_Persistence, + context: V1_GraphBuilderContext, +): void => { + const path = V1_buildFullPath(protocol.package, protocol.name); + const persistence = getOwnPersistence(path, context.currentSubGraph); + persistence.documentation = guaranteeNonEmptyString( + protocol.documentation, + `Persistence 'documentation' field is missing or empty`, + ); + persistence.notifier = V1_buildNotifier(protocol.notifier, context); + persistence.persister = V1_buildPersister(protocol.persister, context); + persistence.service = context.resolveService(protocol.service); + persistence.serviceOutputTargets = V1_buildServiceOutputTargets( + protocol.serviceOutputTargets, + context, + ); + persistence.tests = protocol.tests + .map((test) => V1_buildAtomicTest(test, context)) + .map((e) => guaranteeType(e, PersistenceTest)); + persistence.trigger = V1_buildTrigger(protocol.trigger, context); +}; + /********** * test **********/ @@ -691,26 +1281,3 @@ export const V1_buildTestBatch = ( ); return testBatch; }); - -/********** - * persistence - **********/ - -export const V1_buildPersistence = ( - protocol: V1_Persistence, - context: V1_GraphBuilderContext, -): void => { - const path = V1_buildFullPath(protocol.package, protocol.name); - const persistence = getOwnPersistence(path, context.currentSubGraph); - persistence.documentation = guaranteeNonEmptyString( - protocol.documentation, - `Persistence 'documentation' field is missing or empty`, - ); - persistence.trigger = V1_buildTrigger(protocol.trigger, context); - persistence.service = context.resolveService(protocol.service); - persistence.persister = V1_buildPersister(protocol.persister, context); - persistence.notifier = V1_buildNotifier(protocol.notifier, context); - persistence.tests = protocol.tests - .map((test) => V1_buildAtomicTest(test, context)) - .map((e) => guaranteeType(e, PersistenceTest)); -}; diff --git a/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/transformation/pureProtocol/V1_DSL_Persistence_ProtocolHelper.ts b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/transformation/pureProtocol/V1_DSL_Persistence_ProtocolHelper.ts index 96e949075e..feb47cadd8 100644 --- a/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/transformation/pureProtocol/V1_DSL_Persistence_ProtocolHelper.ts +++ b/packages/legend-extension-dsl-persistence/src/graph-manager/protocol/pure/v1/transformation/pureProtocol/V1_DSL_Persistence_ProtocolHelper.ts @@ -98,6 +98,7 @@ import { UnsupportedOperationError, usingConstantValueSchema, usingModelSchema, + optionalCustom, optionalCustomList, customList, customListWithSchema, @@ -117,6 +118,85 @@ import { V1_PersistenceTest } from '../../model/packageableElements/persistence/ import { V1_PersistenceTestBatch } from '../../model/packageableElements/persistence/V1_DSL_Persistence_PersistenceTestBatch.js'; import { V1_ConnectionTestData } from '../../model/packageableElements/persistence/V1_DSL_Persistence_ConnectionTestData.js'; import { V1_PersistenceTestData } from '../../model/packageableElements/persistence/V1_DSL_Persistence_PersistenceTestData.js'; +import { V1_ServiceOutputTarget } from '../../model/packageableElements/persistence/V1_DSL_Persistence_ServiceOutputTarget.js'; +import { + V1_GraphFetchServiceOutput, + type V1_ServiceOutput, + V1_TdsServiceOutput, +} from '../../model/packageableElements/persistence/V1_DSL_Persistence_ServiceOutput.js'; +import { + type V1_PersistenceTarget, + V1_RelationalPersistenceTarget, +} from '../../model/packageableElements/persistence/V1_DSL_Persistence_PersistentTarget.js'; +import { + V1_AnyVersion, + type V1_Deduplication, + V1_MaxVersion, + V1_MaxVersionForGraphFetch, + V1_MaxVersionForTds, + V1_NoDeduplication, +} from '../../model/packageableElements/persistence/V1_DSL_Persistence_Deduplication.js'; +import { + type V1_DatasetType, + V1_Delta, + V1_Snapshot, +} from '../../model/packageableElements/persistence/V1_DSL_Persistence_DatasetType.js'; +import { + V1_FieldBased, + V1_FieldBasedForGraphFetch, + V1_FieldBasedForTds, + V1_NoPartitioning, + type V1_Partitioning, +} from '../../model/packageableElements/persistence/V1_DSL_Persistence_Partitioning.js'; +import { + V1_DeleteTargetDataset, + type V1_EmptyDatasetHandling, + V1_NoOp, +} from '../../model/packageableElements/persistence/V1_DSL_Persistence_EmptyDatasetHandling.js'; +import { + type V1_ActionIndicatorFields, + V1_DeleteIndicator, + V1_DeleteIndicatorForGraphFetch, + V1_DeleteIndicatorForTds, + V1_NoActionIndicator, +} from '../../model/packageableElements/persistence/V1_DSL_Persistence_ActionIndicatorFields.js'; +import { + V1_BiTemporal, + V1_NonTemporal, + type V1_Temporality, + V1_UniTemporal, +} from '../../model/packageableElements/persistence/V1_DSL_Persistence_Temporality.js'; +import { + V1_AuditingDateTimeV2, + type V1_AuditingV2, + V1_NoAuditingV2, +} from '../../model/packageableElements/persistence/V1_DSL_Persistence_AuditingV2.js'; +import { + V1_AppendOnlyUpdatesHandling, + V1_OverwriteUpdatesHandling, + type V1_UpdatesHandling, +} from '../../model/packageableElements/persistence/V1_DSL_Persistence_UpdatesHandling.js'; +import { + V1_AllowDuplicates, + type V1_AppendStrategy, + V1_FailOnDuplicates, + V1_FilterDuplicates, +} from '../../model/packageableElements/persistence/V1_DSL_Persistence_AppendStrategy.js'; +import { + V1_BatchId, + V1_BatchIdAndDateTime, + V1_ProcessingDateTime, + type V1_ProcessingDimension, +} from '../../model/packageableElements/persistence/V1_DSL_Persistence_ProcessingDimension.js'; +import { + type V1_SourceDerivedDimension, + V1_SourceDerivedTime, +} from '../../model/packageableElements/persistence/V1_DSL_Persistence_SourceDerivedDimension.js'; +import { + type V1_SourceTimeFields, + V1_SourceTimeStart, + V1_SourceTimeStartAndEnd, +} from '../../model/packageableElements/persistence/V1_DSL_Persistence_SourceTimeFields.js'; /********** * notifier @@ -1069,6 +1149,962 @@ export const V1_persistenceTestModelSchema = ( ), }); +/************* + * AuditingV2 + *************/ + +export enum V1_AuditingTypeV2 { + DATE_TIME_AUDITING_V2 = 'auditingDateTime', + NO_AUDITING_V2 = 'noAuditing', +} + +export const V1_auditingDateTimeV2ModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_AuditingDateTimeV2, { + _type: usingConstantValueSchema(V1_AuditingTypeV2.DATE_TIME_AUDITING_V2), + auditingDateTimeName: primitive(), + }); + +export const V1_noAuditingV2ModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_NoAuditingV2, { + _type: usingConstantValueSchema(V1_AuditingTypeV2.NO_AUDITING_V2), + }); + +export const V1_serializeAuditingV2 = ( + protocol: V1_AuditingV2, + plugins: PureProtocolProcessorPlugin[], +): PlainObject => { + if (protocol instanceof V1_AuditingDateTimeV2) { + return serialize(V1_auditingDateTimeV2ModelSchema(plugins), protocol); + } else if (protocol instanceof V1_NoAuditingV2) { + return serialize(V1_noAuditingV2ModelSchema(plugins), protocol); + } + throw new UnsupportedOperationError(`Can't serialize AuditingV2`, protocol); +}; + +export const V1_deserializeAuditingV2 = ( + json: PlainObject, + plugins: PureProtocolProcessorPlugin[], +): V1_AuditingV2 => { + if (json._type === V1_AuditingTypeV2.DATE_TIME_AUDITING_V2) { + return deserialize(V1_auditingDateTimeV2ModelSchema(plugins), json); + } else if (json._type === V1_AuditingTypeV2.NO_AUDITING_V2) { + return deserialize(V1_noAuditingV2ModelSchema(plugins), json); + } + throw new UnsupportedOperationError(`Can't deserialize AuditingV2`, json); +}; + +/****************** + * Append Strategy + ******************/ + +export enum V1_AppendStrategyType { + ALLOW_DUPLICATES = 'allowDuplicates', + FAIL_ON_DUPLICATES = 'failOnDuplicates', + FILTER_DUPLICATES = 'filterDuplicates', +} + +export const V1_failOnDuplicatesModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_FailOnDuplicates, { + _type: usingConstantValueSchema(V1_AppendStrategyType.FAIL_ON_DUPLICATES), + }); + +export const V1_allowDuplicatesModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_AllowDuplicates, { + _type: usingConstantValueSchema(V1_AppendStrategyType.ALLOW_DUPLICATES), + }); + +export const V1_filterDuplicatesModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_FilterDuplicates, { + _type: usingConstantValueSchema(V1_AppendStrategyType.FILTER_DUPLICATES), + }); + +export const V1_serializeAppendStrategy = ( + protocol: V1_AppendStrategy, + plugins: PureProtocolProcessorPlugin[], +): PlainObject => { + if (protocol instanceof V1_AllowDuplicates) { + return serialize(V1_allowDuplicatesModelSchema(plugins), protocol); + } else if (protocol instanceof V1_FailOnDuplicates) { + return serialize(V1_failOnDuplicatesModelSchema(plugins), protocol); + } else if (protocol instanceof V1_FilterDuplicates) { + return serialize(V1_filterDuplicatesModelSchema(plugins), protocol); + } + throw new UnsupportedOperationError( + `Can't serialize AppendStrategy`, + protocol, + ); +}; + +export const V1_deserializeAppendStrategy = ( + json: PlainObject, + plugins: PureProtocolProcessorPlugin[], +): V1_AppendStrategy => { + if (json._type === V1_AppendStrategyType.ALLOW_DUPLICATES) { + return deserialize(V1_allowDuplicatesModelSchema(plugins), json); + } else if (json._type === V1_AppendStrategyType.FAIL_ON_DUPLICATES) { + return deserialize(V1_failOnDuplicatesModelSchema(plugins), json); + } else if (json._type === V1_AppendStrategyType.FILTER_DUPLICATES) { + return deserialize(V1_filterDuplicatesModelSchema(plugins), json); + } + throw new UnsupportedOperationError(`Can't deserialize AppendStrategy`, json); +}; + +/****************** + * Updates handling + ******************/ + +export enum V1_UpdatesHandlingType { + APPEND_ONLY = 'appendOnly', + OVERWRITE = 'overwrite', +} + +export const V1_overwriteUpdatesHandlingModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_OverwriteUpdatesHandling, { + _type: usingConstantValueSchema(V1_UpdatesHandlingType.OVERWRITE), + }); + +export const V1_appendOnlyUpdatesHandlingModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_AppendOnlyUpdatesHandling, { + _type: usingConstantValueSchema(V1_UpdatesHandlingType.APPEND_ONLY), + appendStrategy: custom( + (val) => V1_serializeAppendStrategy(val, plugins), + (val) => V1_deserializeAppendStrategy(val, plugins), + ), + }); + +export const V1_serializeUpdatesHandling = ( + protocol: V1_UpdatesHandling, + plugins: PureProtocolProcessorPlugin[], +): PlainObject => { + if (protocol instanceof V1_AppendOnlyUpdatesHandling) { + return serialize( + V1_appendOnlyUpdatesHandlingModelSchema(plugins), + protocol, + ); + } else if (protocol instanceof V1_OverwriteUpdatesHandling) { + return serialize(V1_overwriteUpdatesHandlingModelSchema(plugins), protocol); + } + throw new UnsupportedOperationError( + `Can't serialize UpdatesHandling`, + protocol, + ); +}; + +export const V1_deserializeUpdatesHandling = ( + json: PlainObject, + plugins: PureProtocolProcessorPlugin[], +): V1_UpdatesHandling => { + if (json._type === V1_UpdatesHandlingType.APPEND_ONLY) { + return deserialize(V1_appendOnlyUpdatesHandlingModelSchema(plugins), json); + } else if (json._type === V1_UpdatesHandlingType.OVERWRITE) { + return deserialize(V1_overwriteUpdatesHandlingModelSchema(plugins), json); + } + throw new UnsupportedOperationError( + `Can't deserialize UpdatesHandling`, + json, + ); +}; + +/********************** + * Processing dimension + **********************/ + +export enum V1_ProcessingDimensionType { + BATCH_ID = 'batchId', + PROCESSING_DATE_TIME = 'processingTime', + BATCH_ID_AND_DATE_TIME = 'batchIdAndProcessingTime', +} + +export const V1_batchIdModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_BatchId, { + _type: usingConstantValueSchema(V1_ProcessingDimensionType.BATCH_ID), + batchIdIn: primitive(), + batchIdOut: primitive(), + }); + +export const V1_processingDateTimeModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_ProcessingDateTime, { + _type: usingConstantValueSchema( + V1_ProcessingDimensionType.PROCESSING_DATE_TIME, + ), + timeIn: primitive(), + timeOut: primitive(), + }); + +export const V1_batchIdAndDateTimeModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_BatchIdAndDateTime, { + _type: usingConstantValueSchema( + V1_ProcessingDimensionType.BATCH_ID_AND_DATE_TIME, + ), + batchIdIn: primitive(), + batchIdOut: primitive(), + timeIn: primitive(), + timeOut: primitive(), + }); + +export const V1_serializeProcessingDimension = ( + protocol: V1_ProcessingDimension, + plugins: PureProtocolProcessorPlugin[], +): PlainObject => { + if (protocol instanceof V1_BatchId) { + return serialize(V1_batchIdModelSchema(plugins), protocol); + } else if (protocol instanceof V1_ProcessingDateTime) { + return serialize(V1_processingDateTimeModelSchema(plugins), protocol); + } else if (protocol instanceof V1_BatchIdAndDateTime) { + return serialize(V1_batchIdAndDateTimeModelSchema(plugins), protocol); + } + throw new UnsupportedOperationError( + `Can't serialize ProcessingDimension`, + protocol, + ); +}; + +export const V1_deserializeProcessingDimension = ( + json: PlainObject, + plugins: PureProtocolProcessorPlugin[], +): V1_ProcessingDimension => { + if (json._type === V1_ProcessingDimensionType.BATCH_ID) { + return deserialize(V1_batchIdModelSchema(plugins), json); + } else if (json._type === V1_ProcessingDimensionType.BATCH_ID_AND_DATE_TIME) { + return deserialize(V1_batchIdAndDateTimeModelSchema(plugins), json); + } else if (json._type === V1_ProcessingDimensionType.PROCESSING_DATE_TIME) { + return deserialize(V1_processingDateTimeModelSchema(plugins), json); + } + throw new UnsupportedOperationError( + `Can't deserialize ProcessingDimension`, + json, + ); +}; + +/******************** + * Source time fields + ********************/ + +export enum V1_SourceTimeFieldsType { + SOURCE_TIME_START = 'sourceTimeStart', + SOURCE_TIME_START_AND_END = 'sourceTimeStartAndEnd', +} + +export const V1_sourceTimeStartModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_SourceTimeStart, { + _type: usingConstantValueSchema(V1_SourceTimeFieldsType.SOURCE_TIME_START), + startField: primitive(), + }); + +export const V1_sourceTimeStartAndEndModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_SourceTimeStartAndEnd, { + _type: usingConstantValueSchema( + V1_SourceTimeFieldsType.SOURCE_TIME_START_AND_END, + ), + endField: primitive(), + startField: primitive(), + }); + +export const V1_serializeSourceTimeFields = ( + protocol: V1_SourceTimeFields, + plugins: PureProtocolProcessorPlugin[], +): PlainObject => { + if (protocol instanceof V1_SourceTimeStart) { + return serialize(V1_sourceTimeStartModelSchema(plugins), protocol); + } else if (protocol instanceof V1_SourceTimeStartAndEnd) { + return serialize(V1_sourceTimeStartAndEndModelSchema(plugins), protocol); + } + throw new UnsupportedOperationError( + `Can't serialize SourceTimeFields`, + protocol, + ); +}; + +export const V1_deserializeSourceTimeFields = ( + json: PlainObject, + plugins: PureProtocolProcessorPlugin[], +): V1_SourceTimeFields => { + if (json._type === V1_SourceTimeFieldsType.SOURCE_TIME_START) { + return deserialize(V1_sourceTimeStartModelSchema(plugins), json); + } else if (json._type === V1_SourceTimeFieldsType.SOURCE_TIME_START_AND_END) { + return deserialize(V1_sourceTimeStartAndEndModelSchema(plugins), json); + } + throw new UnsupportedOperationError( + `Can't deserialize SourceTimeFields`, + json, + ); +}; + +/************************** + * Source derived dimension + **************************/ + +export enum V1_SourceDerivedDimensionType { + SOURCE_DERIVED_TIME = 'sourceDerivedTime', +} + +export const V1_sourceDerivedTimeModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_SourceDerivedTime, { + _type: usingConstantValueSchema( + V1_SourceDerivedDimensionType.SOURCE_DERIVED_TIME, + ), + sourceTimeFields: custom( + (val) => V1_serializeSourceTimeFields(val, plugins), + (val) => V1_deserializeSourceTimeFields(val, plugins), + ), + timeEnd: primitive(), + timeStart: primitive(), + }); + +export const V1_serializeSourceDerivedDimension = ( + protocol: V1_SourceDerivedDimension, + plugins: PureProtocolProcessorPlugin[], +): PlainObject => { + if (protocol instanceof V1_SourceDerivedTime) { + return serialize(V1_sourceDerivedTimeModelSchema(plugins), protocol); + } + throw new UnsupportedOperationError( + `Can't serialize SourceDerivedDimension`, + protocol, + ); +}; + +export const V1_deserializeSourceDerivedDimension = ( + json: PlainObject, + plugins: PureProtocolProcessorPlugin[], +): V1_SourceDerivedDimension => { + if (json._type === V1_SourceDerivedDimensionType.SOURCE_DERIVED_TIME) { + return deserialize(V1_sourceDerivedTimeModelSchema(plugins), json); + } + throw new UnsupportedOperationError( + `Can't deserialize SourceDerivedDimension`, + json, + ); +}; + +/************* + * Temporality + *************/ + +export enum V1_TemporalityType { + NON_TEMPORAL = 'none', + UNI_TEMPORAL = 'unitemporalTemporality', + BI_TEMPORAL = 'bitemporalTemporality', +} + +export const V1_nonTemporalModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_NonTemporal, { + _type: usingConstantValueSchema(V1_TemporalityType.NON_TEMPORAL), + auditing: custom( + (val) => V1_serializeAuditingV2(val, plugins), + (val) => V1_deserializeAuditingV2(val, plugins), + ), + updatesHandling: custom( + (val) => V1_serializeUpdatesHandling(val, plugins), + (val) => V1_deserializeUpdatesHandling(val, plugins), + ), + }); + +export const V1_biTemporalModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_BiTemporal, { + _type: usingConstantValueSchema(V1_TemporalityType.BI_TEMPORAL), + processingDimension: custom( + (val) => V1_serializeProcessingDimension(val, plugins), + (val) => V1_deserializeProcessingDimension(val, plugins), + ), + sourceDerivedDimension: custom( + (val) => V1_serializeSourceDerivedDimension(val, plugins), + (val) => V1_deserializeSourceDerivedDimension(val, plugins), + ), + }); + +export const V1_uniTemporalModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_UniTemporal, { + _type: usingConstantValueSchema(V1_TemporalityType.UNI_TEMPORAL), + processingDimension: custom( + (val) => V1_serializeProcessingDimension(val, plugins), + (val) => V1_deserializeProcessingDimension(val, plugins), + ), + }); + +export const V1_serializeTemporality = ( + protocol: V1_Temporality, + plugins: PureProtocolProcessorPlugin[], +): PlainObject => { + if (protocol instanceof V1_NonTemporal) { + return serialize(V1_nonTemporalModelSchema(plugins), protocol); + } else if (protocol instanceof V1_UniTemporal) { + return serialize(V1_uniTemporalModelSchema(plugins), protocol); + } else if (protocol instanceof V1_BiTemporal) { + return serialize(V1_biTemporalModelSchema(plugins), protocol); + } + throw new UnsupportedOperationError(`Can't serialize Temporality`, protocol); +}; + +export const V1_deserializeTemporality = ( + json: PlainObject, + plugins: PureProtocolProcessorPlugin[], +): V1_Temporality => { + if (json._type === V1_TemporalityType.NON_TEMPORAL) { + return deserialize(V1_nonTemporalModelSchema(plugins), json); + } else if (json._type === V1_TemporalityType.UNI_TEMPORAL) { + return deserialize(V1_uniTemporalModelSchema(plugins), json); + } else if (json._type === V1_TemporalityType.BI_TEMPORAL) { + return deserialize(V1_biTemporalModelSchema(plugins), json); + } + throw new UnsupportedOperationError(`Can't deserialize Temporality`, json); +}; + +/************************ + * persistent target type + ************************/ + +export enum V1_PersistentTargetType { + RELATIONAL_PERSISTENCE_TARGET = 'relationalPersistenceTarget', +} + +export const V1_relationalPersistenceTargetModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_RelationalPersistenceTarget, { + _type: usingConstantValueSchema( + V1_PersistentTargetType.RELATIONAL_PERSISTENCE_TARGET, + ), + database: primitive(), + table: primitive(), + temporality: custom( + (val) => V1_serializeTemporality(val, plugins), + (val) => V1_deserializeTemporality(val, plugins), + ), + }); + +export const V1_serializePersistentTarget = ( + protocol: V1_PersistenceTarget, + plugins: PureProtocolProcessorPlugin[], +): PlainObject => { + if (protocol instanceof V1_RelationalPersistenceTarget) { + return serialize( + V1_relationalPersistenceTargetModelSchema(plugins), + protocol, + ); + } + throw new UnsupportedOperationError( + `Can't serialize PersistentTarget`, + protocol, + ); +}; + +export const V1_deserializePersistentTarget = ( + json: PlainObject, + plugins: PureProtocolProcessorPlugin[], +): V1_PersistenceTarget => { + if (json._type === V1_PersistentTargetType.RELATIONAL_PERSISTENCE_TARGET) { + return deserialize( + V1_relationalPersistenceTargetModelSchema(plugins), + json, + ); + } + throw new UnsupportedOperationError( + `Can't deserialize PersistentTarget`, + json, + ); +}; + +/************************ + * empty dataset handling + ***********************/ + +export enum V1_EmptyDatasetHandlingType { + NO_OP = 'noOp', + DELETE_TARGET_DATASET = 'deleteTargetDataset', +} + +export const V1_noOpEmptyDatasetHandlingModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_NoOp, { + _type: usingConstantValueSchema(V1_EmptyDatasetHandlingType.NO_OP), + }); + +export const V1_deleteTargetDatasetModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_DeleteTargetDataset, { + _type: usingConstantValueSchema( + V1_EmptyDatasetHandlingType.DELETE_TARGET_DATASET, + ), + }); + +export const V1_serializeEmptyDatasetHandling = ( + protocol: V1_EmptyDatasetHandling, + plugins: PureProtocolProcessorPlugin[], +): PlainObject => { + if (protocol instanceof V1_NoOp) { + return serialize(V1_noOpEmptyDatasetHandlingModelSchema(plugins), protocol); + } else if (protocol instanceof V1_DeleteTargetDataset) { + return serialize(V1_deleteTargetDatasetModelSchema(plugins), protocol); + } + throw new UnsupportedOperationError( + `Can't serialize EmptyDataSetHandling`, + protocol, + ); +}; + +export const V1_deserializeEmptyDatasetHandling = ( + json: PlainObject, + plugins: PureProtocolProcessorPlugin[], +): V1_EmptyDatasetHandling => { + if (json._type === V1_EmptyDatasetHandlingType.NO_OP) { + return deserialize(V1_noOpEmptyDatasetHandlingModelSchema(plugins), json); + } else if (json._type === V1_EmptyDatasetHandlingType.DELETE_TARGET_DATASET) { + return deserialize(V1_deleteTargetDatasetModelSchema(plugins), json); + } + throw new UnsupportedOperationError( + `Can't deserialize EmptyDatasetHandling`, + json, + ); +}; + +/************** + * partitioning + **************/ + +export enum V1_PartitioningType { + NO_PARTITIONING = 'noPartitioning', + FIELD_BASED_FOR_GRAPH_FETCH = 'fieldBasedForGraphFetch', + FIELD_BASED_FOR_TDS = 'fieldBasedForTds', +} + +export const V1_fieldBasedForGraphFetchModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_FieldBasedForGraphFetch, { + _type: usingConstantValueSchema( + V1_PartitioningType.FIELD_BASED_FOR_GRAPH_FETCH, + ), + }); + +export const V1_fieldBasedForTdsModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_FieldBasedForGraphFetch, { + _type: usingConstantValueSchema( + V1_PartitioningType.FIELD_BASED_FOR_GRAPH_FETCH, + ), + }); + +export const V1_noPartitioningModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_NoPartitioning, { + _type: usingConstantValueSchema(V1_PartitioningType.NO_PARTITIONING), + emptyDatasetHandling: custom( + (val) => V1_serializeEmptyDatasetHandling(val, plugins), + (val) => V1_deserializeEmptyDatasetHandling(val, plugins), + ), + }); + +export const V1_serializeFieldBased = ( + protocol: V1_FieldBased, + plugins: PureProtocolProcessorPlugin[], +): PlainObject => { + if (protocol instanceof V1_FieldBasedForGraphFetch) { + return serialize(V1_fieldBasedForGraphFetchModelSchema(plugins), protocol); + } else if (protocol instanceof V1_FieldBasedForTds) { + return serialize(V1_fieldBasedForTdsModelSchema(plugins), protocol); + } + throw new UnsupportedOperationError(`Can't serialize FiedlBased`, protocol); +}; + +export const V1_serializePartitioning = ( + protocol: V1_Partitioning, + plugins: PureProtocolProcessorPlugin[], +): PlainObject => { + if (protocol instanceof V1_NoPartitioning) { + return serialize(V1_noPartitioningModelSchema(plugins), protocol); + } else if (protocol instanceof V1_FieldBased) { + return V1_serializeFieldBased(protocol, plugins); + } + throw new UnsupportedOperationError(`Can't serialize Partitioning`, protocol); +}; + +export const V1_deserializePartitioning = ( + json: PlainObject, + plugins: PureProtocolProcessorPlugin[], +): V1_Partitioning => { + if (json._type === V1_PartitioningType.NO_PARTITIONING) { + return deserialize(V1_noPartitioningModelSchema(plugins), json); + } else if (json._type === V1_PartitioningType.FIELD_BASED_FOR_GRAPH_FETCH) { + return deserialize(V1_fieldBasedForGraphFetchModelSchema(plugins), json); + } else if (json._type === V1_PartitioningType.FIELD_BASED_FOR_TDS) { + return deserialize(V1_fieldBasedForTdsModelSchema(plugins), json); + } + throw new UnsupportedOperationError(`Can't deserialize Partitioning`, json); +}; + +/******************* + * action indicator + ******************/ + +export enum V1_ActionIndicatorType { + NO_ACTION_INDICATOR = 'noActionIndicator', + DELETE_INDICATOR_FOR_GRAPH_FETCH = 'deleteIndicatorForGraphFetch', + DELETE_INDICATOR_FOR_TDS = 'deleteIndicatorForTds', +} + +export const V1_deleteIndicatorForGraphFetchModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_DeleteIndicatorForGraphFetch, { + _type: usingConstantValueSchema( + V1_ActionIndicatorType.DELETE_INDICATOR_FOR_GRAPH_FETCH, + ), + deleteValues: list(primitive()), + }); + +export const V1_deleteIndicatorForTdsModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_DeleteIndicatorForTds, { + _type: usingConstantValueSchema( + V1_ActionIndicatorType.DELETE_INDICATOR_FOR_TDS, + ), + deleteField: primitive(), + deleteValues: list(primitive()), + }); + +export const V1_noActionIndicatorModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_NoActionIndicator, { + _type: usingConstantValueSchema(V1_ActionIndicatorType.NO_ACTION_INDICATOR), + }); + +export const V1_serializeDeleteIndicator = ( + protocol: V1_DeleteIndicator, + plugins: PureProtocolProcessorPlugin[], +): PlainObject => { + if (protocol instanceof V1_DeleteIndicatorForGraphFetch) { + return serialize( + V1_deleteIndicatorForGraphFetchModelSchema(plugins), + protocol, + ); + } else if (protocol instanceof V1_DeleteIndicatorForTds) { + return serialize(V1_deleteIndicatorForTdsModelSchema(plugins), protocol); + } + throw new UnsupportedOperationError( + `Can't serialize DeleteIndicator`, + protocol, + ); +}; + +export const V1_deserializeActionIndicator = ( + json: PlainObject, + plugins: PureProtocolProcessorPlugin[], +): V1_ActionIndicatorFields => { + if (json._type === V1_ActionIndicatorType.NO_ACTION_INDICATOR) { + return deserialize(V1_noActionIndicatorModelSchema(plugins), json); + } else if ( + json._type === V1_ActionIndicatorType.DELETE_INDICATOR_FOR_GRAPH_FETCH + ) { + return deserialize( + V1_deleteIndicatorForGraphFetchModelSchema(plugins), + json, + ); + } else if (json._type === V1_ActionIndicatorType.DELETE_INDICATOR_FOR_TDS) { + return deserialize(V1_deleteIndicatorForTdsModelSchema(plugins), json); + } + throw new UnsupportedOperationError( + `Can't deserialize ActionIndicator`, + json, + ); +}; + +export const V1_serializeActionIndicator = ( + protocol: V1_ActionIndicatorFields, + plugins: PureProtocolProcessorPlugin[], +): PlainObject => { + if (protocol instanceof V1_NoActionIndicator) { + return serialize(V1_noActionIndicatorModelSchema(plugins), protocol); + } else if (protocol instanceof V1_DeleteIndicator) { + return V1_serializeDeleteIndicator(protocol, plugins); + } + throw new UnsupportedOperationError( + `Can't serialize ActionIndicator`, + protocol, + ); +}; + +/**************** + * dataset type + ***************/ + +export enum V1_DatasetTypeType { + SNAPSHOT = 'snapshot', + DELTA = 'delta', +} + +export const V1_snapshotModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_Snapshot, { + _type: usingConstantValueSchema(V1_DatasetTypeType.SNAPSHOT), + partitioning: custom( + (val) => V1_serializePartitioning(val, plugins), + (val) => V1_deserializePartitioning(val, plugins), + ), + }); + +export const V1_deltaModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_Delta, { + _type: usingConstantValueSchema(V1_DatasetTypeType.DELTA), + actionIndicator: custom( + (val) => V1_serializeActionIndicator(val, plugins), + (val) => V1_deserializeActionIndicator(val, plugins), + ), + }); + +export const V1_serializeDatasetType = ( + protocol: V1_DatasetType, + plugins: PureProtocolProcessorPlugin[], +): PlainObject => { + if (protocol instanceof V1_Snapshot) { + return serialize(V1_snapshotModelSchema(plugins), protocol); + } else if (protocol instanceof V1_Delta) { + return serialize(V1_deltaModelSchema(plugins), protocol); + } + throw new UnsupportedOperationError(`Can't serialize DatasetType`, protocol); +}; + +export const V1_deserializeDatasetType = ( + json: PlainObject, + plugins: PureProtocolProcessorPlugin[], +): V1_DatasetType => { + if (json._type === V1_DatasetTypeType.SNAPSHOT) { + return deserialize(V1_snapshotModelSchema(plugins), json); + } else if (json._type === V1_DatasetTypeType.DELTA) { + return deserialize(V1_deltaModelSchema(plugins), json); + } + throw new UnsupportedOperationError(`Can't deserialize DatasetType`, json); +}; + +/**************** + * De-duplication + ****************/ + +export enum V1_DeduplicationType { + NO_DEDUPLICATION = 'noDeduplication', + ANY_VERSION = 'anyVersion', + MAX_VERSION_FOR_GRAPH_FETCH = 'maxVersionForGraphFetch', + MAX_VERSION_FOR_TDS = 'maxVersionForTds', +} + +export const V1_NoDeduplicationModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_NoDeduplication, { + _type: usingConstantValueSchema(V1_DeduplicationType.NO_DEDUPLICATION), + }); + +export const V1_anyVersionModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_AnyVersion, { + _type: usingConstantValueSchema(V1_DeduplicationType.ANY_VERSION), + }); + +export const V1_maxVersionForGraphFetchModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_MaxVersionForGraphFetch, { + _type: usingConstantValueSchema( + V1_DeduplicationType.MAX_VERSION_FOR_GRAPH_FETCH, + ), + }); + +export const V1_maxVersionForTdsModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_MaxVersionForTds, { + _type: usingConstantValueSchema(V1_DeduplicationType.MAX_VERSION_FOR_TDS), + }); + +export const V1_serializeDeduplication = ( + protocol: V1_Deduplication, + plugins: PureProtocolProcessorPlugin[], +): PlainObject => { + if (protocol instanceof V1_NoDeduplication) { + return serialize(V1_NoDeduplicationModelSchema(plugins), protocol); + } else if (protocol instanceof V1_AnyVersion) { + return serialize(V1_anyVersionModelSchema(plugins), protocol); + } else if (protocol instanceof V1_MaxVersion) { + return serialize(V1_maxVersionForGraphFetchModelSchema(plugins), protocol); + } + throw new UnsupportedOperationError( + `Can't serialize Deduplication`, + protocol, + ); +}; + +export const V1_deserializeDeduplication = ( + json: PlainObject, + plugins: PureProtocolProcessorPlugin[], +): V1_Deduplication => { + if (json._type === V1_DeduplicationType.NO_DEDUPLICATION) { + return deserialize(V1_NoDeduplicationModelSchema(plugins), json); + } else if (json._type === V1_DeduplicationType.ANY_VERSION) { + return deserialize(V1_anyVersionModelSchema(plugins), json); + } else if (json._type === V1_DeduplicationType.MAX_VERSION_FOR_GRAPH_FETCH) { + return deserialize(V1_maxVersionForGraphFetchModelSchema(plugins), json); + } else if (json._type === V1_DeduplicationType.MAX_VERSION_FOR_TDS) { + return deserialize(V1_maxVersionForTdsModelSchema(plugins), json); + } + throw new UnsupportedOperationError(`Can't deserialize Deduplication`, json); +}; + +export const V1_serializeMaxVersionDeduplication = ( + protocol: V1_MaxVersion, + plugins: PureProtocolProcessorPlugin[], +): PlainObject => { + if (protocol instanceof V1_MaxVersionForGraphFetch) { + return serialize(V1_maxVersionForGraphFetchModelSchema(plugins), protocol); + } else if (protocol instanceof V1_MaxVersionForTds) { + return serialize(V1_maxVersionForTdsModelSchema(plugins), protocol); + } + throw new UnsupportedOperationError( + `Can't serialize MaxVersionDeduplication`, + protocol, + ); +}; + +/**************** + * service output + ***************/ + +export enum V1_ServiceOutputType { + GRAPH_FETCH_SERVICE_OUTPUT = 'graphFetchServiceOutput', + TDS_SERVICE_OUTPUT = 'tdsServiceOutput', +} + +export const V1_graphFetchServiceOutputModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_GraphFetchServiceOutput, { + _type: usingConstantValueSchema( + V1_ServiceOutputType.GRAPH_FETCH_SERVICE_OUTPUT, + ), + datasetType: custom( + (val) => V1_serializeDatasetType(val, plugins), + (val) => V1_deserializeDatasetType(val, plugins), + ), + deduplication: custom( + (val) => V1_serializeDeduplication(val, plugins), + (val) => V1_deserializeDeduplication(val, plugins), + ), + }); + +export const V1_tdsServiceOutputModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_TdsServiceOutput, { + _type: usingConstantValueSchema(V1_ServiceOutputType.TDS_SERVICE_OUTPUT), + datasetType: custom( + (val) => V1_serializeDatasetType(val, plugins), + (val) => V1_deserializeDatasetType(val, plugins), + ), + deduplication: custom( + (val) => V1_serializeDeduplication(val, plugins), + (val) => V1_deserializeDeduplication(val, plugins), + ), + keys: list(primitive()), + }); + +export const V1_serializeServiceOutput = ( + protocol: V1_ServiceOutput, + plugins: PureProtocolProcessorPlugin[], +): PlainObject => { + if (protocol instanceof V1_GraphFetchServiceOutput) { + return serialize(V1_graphFetchServiceOutputModelSchema(plugins), protocol); + } else if (protocol instanceof V1_TdsServiceOutput) { + return serialize(V1_tdsServiceOutputModelSchema(plugins), protocol); + } + throw new UnsupportedOperationError( + `Can't serialize ServiceOutput`, + protocol, + ); +}; + +export const V1_deserializeServiceOutput = ( + json: PlainObject, + plugins: PureProtocolProcessorPlugin[], +): V1_ServiceOutput => { + if (json._type === V1_ServiceOutputType.GRAPH_FETCH_SERVICE_OUTPUT) { + return deserialize(V1_graphFetchServiceOutputModelSchema(plugins), json); + } else if (json._type === V1_ServiceOutputType.TDS_SERVICE_OUTPUT) { + return deserialize(V1_tdsServiceOutputModelSchema(plugins), json); + } + throw new UnsupportedOperationError(`Can't deserialize ServiceOutput`, json); +}; + +/************************ + * service output target + ***********************/ + +const V1_serviceOutputTargetModelSchema = ( + plugins: PureProtocolProcessorPlugin[], +): ModelSchema => + createModelSchema(V1_ServiceOutputTarget, { + persistenceTarget: custom( + (val) => V1_serializePersistentTarget(val, plugins), + (val) => V1_deserializePersistentTarget(val, plugins), + ), + serviceOutput: custom( + (val) => V1_serializeServiceOutput(val, plugins), + (val) => V1_deserializeServiceOutput(val, plugins), + ), + }); + +export const V1_serializeServiceOutputTarget = ( + protocol: V1_ServiceOutputTarget, + plugins: PureProtocolProcessorPlugin[], +): PlainObject => + serialize(V1_serviceOutputTargetModelSchema(plugins), protocol); + +export const V1_deserializeServiceOutputTarget = ( + json: PlainObject, + plugins: PureProtocolProcessorPlugin[], +): V1_ServiceOutputTarget => + deserialize(V1_serviceOutputTargetModelSchema(plugins), json); + /********** * persistence **********/ @@ -1087,11 +2123,16 @@ export const V1_persistenceModelSchema = ( (val) => deserialize(V1_notifierModelSchema, val), ), package: primitive(), - persister: custom( + persister: optionalCustom( (val) => V1_serializePersister(val, plugins), (val) => V1_deserializePersister(val, plugins), ), service: primitive(), + serviceOutputTargets: optionalCustomList( + (val: V1_ServiceOutputTarget) => + V1_serializeServiceOutputTarget(val, plugins), + (val) => V1_deserializeServiceOutputTarget(val, plugins), + ), tests: optionalCustomList( (value: V1_AtomicTest) => V1_serializeAtomicTest(value, plugins), (value) => V1_deserializeAtomicTest(value, plugins), diff --git a/packages/legend-extension-dsl-persistence/src/graph/DSL_Persistence_HashUtils.ts b/packages/legend-extension-dsl-persistence/src/graph/DSL_Persistence_HashUtils.ts index a0da95cf8a..98d4745db4 100644 --- a/packages/legend-extension-dsl-persistence/src/graph/DSL_Persistence_HashUtils.ts +++ b/packages/legend-extension-dsl-persistence/src/graph/DSL_Persistence_HashUtils.ts @@ -17,6 +17,70 @@ export enum PERSISTENCE_HASH_STRUCTURE { PERSISTENCE = 'PERSISTENCE', + // service output target + SERVICE_OUTPUT_TARGET = 'SERVICE_OUTPUT_TARGET', + + // persistence target + RELATIONAL_PERSISTENCE_TARGET = 'RELATIONAL_PERSISTENCE_TARGET', + + // temporality + NON_TEMPORAL = 'NON_TEMPORAL', + UNI_TEMPORAL = 'UNI_TEMPORAL', + BI_TEMPORAL = 'BI_TEMPORAL', + + // updates handling + APPEND_ONLY_UPDATES = 'APPEND_ONLY_UPDATES', + OVERWRITE_UPDATES = 'OVERWRITE_UPDATES', + + // append strategy + ALLOW_DUPLICATES = 'ALLOW_DUPLICATES', + FAIL_ON_DUPLICATES = 'FAIL_ON_DUPLICATES', + FILTER_DUPLICATES = 'FILTER_DUPLICATES', + + // processing dimension + BATCH_ID = 'BATCH_ID', + PROCESSING_DATE_TIME = 'PROCESSING_DATE_TIME', + BATCH_ID_AND_DATE_TIME = 'BATCH_ID_AND_DATE_TIME', + + // source derived dimension + SOURCE_DERIVED_TIME = 'SOURCE_DERIVED_TIME', + + // source time fields + SOURCE_TIME_START = 'SOURCE_TIME_START', + SOURCE_TIME_START_AND_END = 'SOURCE_TIME_START_AND_END', + + // service output + GRAPH_FETCH_SERVICE_OUTPUT = 'GRAPH_FETCH_SERVICE_OUTPUT', + TDS_SERVICE_OUTPUT = 'TDS_SERVICE_OUTPUT', + + // de-duplication + NO_DEDUPLICATION = 'NO_DEDUPLICATION', + ANY_VERSION = 'ANY_VERSION', + MAX_VERSION_FOR_GRAPH_FETCH = 'MAX_VERSION_FOR_GRAPH_FETCH', + MAX_VERSION_FOR_TDS = 'MAX_VERSION_FOR_TDS', + + // dataset type + SNAPSHOT = 'SNAPSHOT', + DELTA = 'DELTA', + + // partitioning + NO_PARTITIONING = 'NO_PARTITIONING', + FIELD_BASED_FOR_GRAPH_FETCH = 'FIELD_BASED_FOR_GRAPH_FETCH', + FIEDD_BASED_FOR_TDS = 'FIEDD_BASED_FOR_TDS', + + // empty dataset handling + NO_OP_DATASET_HANDLING = 'NO_OP_DATASET_HANDLING', + DELETE_TARGET_DATASET = 'DELETE_TARGET_DATASET', + + // action indicator fields + NO_ACTION_INDICATOR = 'NO_ACTION_INDICATOR', + DELETE_INDICATOR_FOR_GRAPH_FETCH = 'DELETE_INDICATOR_FOR_GRAPH_FETCH', + DELETE_INDICATOR_FOR_TDS = 'DELETE_INDICATOR_FOR_TDS', + + // auditing v2 + AUDITING_DATE_TIME = 'AUDITING_DATE_TIME', + NO_AUDITING_V2 = 'NO_AUDITING', + // trigger MANUAL_TRIGGER = 'MANUAL_TRIGGER', CRON_TRIGGER = 'CRON_TRIGGER', diff --git a/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_ActionIndicatorFields.ts b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_ActionIndicatorFields.ts new file mode 100644 index 0000000000..82f4a97979 --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_ActionIndicatorFields.ts @@ -0,0 +1,51 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type Hashable, hashArray } from '@finos/legend-shared'; +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../DSL_Persistence_HashUtils.js'; + +export abstract class ActionIndicatorFields implements Hashable { + abstract get hashCode(): string; +} + +export class NoActionIndicator implements ActionIndicatorFields { + get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.NO_ACTION_INDICATOR]); + } +} + +export abstract class DeleteIndicator extends ActionIndicatorFields { + deleteValues!: string[]; +} + +export class DeleteIndicatorForGraphFetch extends DeleteIndicator { + get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.DELETE_INDICATOR_FOR_GRAPH_FETCH, + ]); + } +} + +export class DeleteIndicatorForTds extends DeleteIndicator { + deleteField!: string; + + get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.DELETE_INDICATOR_FOR_TDS, + this.deleteField, + ]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_AppendStrategy.ts b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_AppendStrategy.ts new file mode 100644 index 0000000000..d8a502dc67 --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_AppendStrategy.ts @@ -0,0 +1,40 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type Hashable, hashArray } from '@finos/legend-shared'; +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../DSL_Persistence_HashUtils.js'; + +export abstract class AppendStrategy implements Hashable { + abstract get hashCode(): string; +} + +export class AllowDuplicates extends AppendStrategy implements Hashable { + override get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.ALLOW_DUPLICATES]); + } +} + +export class FailOnDuplicates extends AppendStrategy implements Hashable { + override get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.FAIL_ON_DUPLICATES]); + } +} + +export class FilterDuplicates extends AppendStrategy implements Hashable { + override get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.FAIL_ON_DUPLICATES]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_AuditingV2.ts b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_AuditingV2.ts new file mode 100644 index 0000000000..af36219192 --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_AuditingV2.ts @@ -0,0 +1,39 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../DSL_Persistence_HashUtils.js'; +import { type Hashable, hashArray } from '@finos/legend-shared'; + +export abstract class AuditingV2 implements Hashable { + abstract get hashCode(): string; +} + +export class AuditingDateTimeV2 extends AuditingV2 { + auditingDateTimeName!: string; + + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.AUDITING_DATE_TIME, + this.auditingDateTimeName, + ]); + } +} + +export class NoAuditingV2 extends AuditingV2 { + override get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.NO_AUDITING_V2]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_DatasetType.ts b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_DatasetType.ts new file mode 100644 index 0000000000..a2f7060335 --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_DatasetType.ts @@ -0,0 +1,40 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type Hashable, hashArray } from '@finos/legend-shared'; +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../DSL_Persistence_HashUtils.js'; +import type { ActionIndicatorFields } from './DSL_Persistence_ActionIndicatorFields.js'; +import type { Partitioning } from './DSL_Persistence_Partitioning.js'; + +export abstract class DatasetType implements Hashable { + abstract get hashCode(): string; +} + +export class Snapshot extends DatasetType { + partitioning!: Partitioning; + + override get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.SNAPSHOT, this.partitioning]); + } +} + +export class Delta extends DatasetType { + actionIndicator!: ActionIndicatorFields; + + override get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.DELTA, this.actionIndicator]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_Deduplication.ts b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_Deduplication.ts new file mode 100644 index 0000000000..0a3d1d7361 --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_Deduplication.ts @@ -0,0 +1,53 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type Hashable, hashArray } from '@finos/legend-shared'; +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../DSL_Persistence_HashUtils.js'; + +export abstract class Deduplication implements Hashable { + abstract get hashCode(): string; +} + +export class NoDeduplication extends Deduplication { + override get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.NO_DEDUPLICATION]); + } +} + +export class AnyVersion extends Deduplication { + override get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.ANY_VERSION]); + } +} + +export abstract class MaxVersion extends Deduplication {} + +export class MaxVersionForGraphFetch extends MaxVersion { + override get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.MAX_VERSION_FOR_GRAPH_FETCH]); + } +} + +export class MaxVersionForTds extends MaxVersion { + versionField!: string; + + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.MAX_VERSION_FOR_TDS, + this.versionField, + ]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_EmptyDatasetHandling.ts b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_EmptyDatasetHandling.ts new file mode 100644 index 0000000000..5cee5394c9 --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_EmptyDatasetHandling.ts @@ -0,0 +1,34 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type Hashable, hashArray } from '@finos/legend-shared'; +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../DSL_Persistence_HashUtils.js'; + +export abstract class EmptyDatasetHandling implements Hashable { + abstract get hashCode(): string; +} + +export class NoOp extends EmptyDatasetHandling { + get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.NO_OP_DATASET_HANDLING]); + } +} + +export class DeleteTargetDataset extends EmptyDatasetHandling { + get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.DELETE_TARGET_DATASET]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_Partitioning.ts b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_Partitioning.ts new file mode 100644 index 0000000000..bb9fce82fb --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_Partitioning.ts @@ -0,0 +1,53 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type Hashable, hashArray } from '@finos/legend-shared'; +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../DSL_Persistence_HashUtils.js'; +import type { EmptyDatasetHandling } from './DSL_Persistence_EmptyDatasetHandling.js'; + +export abstract class Partitioning implements Hashable { + abstract get hashCode(): string; +} + +export class NoPartitioning extends Partitioning { + emptyDatasetHandling!: EmptyDatasetHandling; + + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.NO_PARTITIONING, + this.emptyDatasetHandling, + ]); + } +} + +export abstract class FieldBased extends Partitioning {} + +export class FieldBasedForGraphFetch extends FieldBased { + override get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.FIELD_BASED_FOR_GRAPH_FETCH]); + } +} + +export class FieldBasedForTds extends FieldBased { + partitionFields: string[] = []; + + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.FIEDD_BASED_FOR_TDS, + hashArray(this.partitionFields), + ]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_Persistence.ts b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_Persistence.ts index 984d539dea..3e0df5ef94 100644 --- a/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_Persistence.ts +++ b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_Persistence.ts @@ -27,6 +27,7 @@ import { } from '@finos/legend-graph'; import { type Hashable, hashArray } from '@finos/legend-shared'; import type { PersistenceTest } from './DSL_Persistence_PersistenceTest.js'; +import type { ServiceOutputTarget } from './DSL_Persistence_ServiceOutputTarget.js'; export class Persistence extends PackageableElement @@ -35,7 +36,8 @@ export class Persistence documentation!: string; trigger!: Trigger; service!: PackageableElementReference; - persister!: Persister; + persister?: Persister | undefined; + serviceOutputTargets?: ServiceOutputTarget[] | undefined; notifier!: Notifier; tests: PersistenceTest[] = []; @@ -45,7 +47,8 @@ export class Persistence this.documentation, this.trigger, this.service.valueForSerialization ?? '', - this.persister, + hashArray(this.serviceOutputTargets ?? []), + this.persister ?? '', this.notifier, hashArray(this.tests), ]); diff --git a/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_PersistenceTestBatch.ts b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_PersistenceTestBatch.ts index 96da286c80..01ddb92f69 100644 --- a/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_PersistenceTestBatch.ts +++ b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_PersistenceTestBatch.ts @@ -29,6 +29,7 @@ export class PersistenceTestBatch implements Hashable { return hashArray([ PERSISTENCE_HASH_STRUCTURE.PERSISTENCE_TEST_BATCH, this.id, + this.batchId, this.testData, hashArray(this.assertions), ]); diff --git a/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_PersistentTarget.ts b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_PersistentTarget.ts new file mode 100644 index 0000000000..d06fa3fc0a --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_PersistentTarget.ts @@ -0,0 +1,41 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type Hashable, hashArray } from '@finos/legend-shared'; +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../DSL_Persistence_HashUtils.js'; +import type { Temporality } from './DSL_Persistence_Temporality.js'; + +export abstract class PersistenceTarget implements Hashable { + abstract get hashCode(): string; +} + +export class RelationalPersistenceTarget + extends PersistenceTarget + implements Hashable +{ + table!: string; + database!: string; + temporality!: Temporality; + + get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.RELATIONAL_PERSISTENCE_TARGET, + this.table, + this.database, + this.temporality, + ]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_ProcessingDimension.ts b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_ProcessingDimension.ts new file mode 100644 index 0000000000..5a990d644c --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_ProcessingDimension.ts @@ -0,0 +1,65 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type Hashable, hashArray } from '@finos/legend-shared'; +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../DSL_Persistence_HashUtils.js'; + +export abstract class ProcessingDimension implements Hashable { + abstract get hashCode(): string; +} + +export class BatchId extends ProcessingDimension implements Hashable { + batchIdIn!: string; + batchIdOut!: string; + + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.BATCH_ID, + this.batchIdIn, + this.batchIdOut, + ]); + } +} + +export class ProcessingDateTime extends ProcessingDimension { + timeIn!: string; + timeOut!: string; + + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.PROCESSING_DATE_TIME, + this.timeIn, + this.timeOut, + ]); + } +} + +export class BatchIdAndDateTime extends ProcessingDimension { + batchIdIn!: string; + batchIdOut!: string; + timeIn!: string; + timeOut!: string; + + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.BATCH_ID_AND_DATE_TIME, + this.batchIdIn, + this.batchIdOut, + this.timeIn, + this.timeOut, + ]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_ServiceOutput.ts b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_ServiceOutput.ts new file mode 100644 index 0000000000..8d17a18a2b --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_ServiceOutput.ts @@ -0,0 +1,49 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type Hashable, hashArray } from '@finos/legend-shared'; +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../DSL_Persistence_HashUtils.js'; +import type { DatasetType } from './DSL_Persistence_DatasetType.js'; +import type { Deduplication } from './DSL_Persistence_Deduplication.js'; + +export abstract class ServiceOutput implements Hashable { + deduplication!: Deduplication; + datasetType!: DatasetType; + + abstract get hashCode(): string; +} + +export class GraphFetchServiceOutput extends ServiceOutput { + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.GRAPH_FETCH_SERVICE_OUTPUT, + this.deduplication, + this.datasetType, + ]); + } +} + +export class TdsServiceOutput extends ServiceOutput { + keys: string[] = []; + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.TDS_SERVICE_OUTPUT, + this.deduplication, + this.datasetType, + hashArray(this.keys), + ]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_ServiceOutputTarget.ts b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_ServiceOutputTarget.ts new file mode 100644 index 0000000000..3cbfd86374 --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_ServiceOutputTarget.ts @@ -0,0 +1,32 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../DSL_Persistence_HashUtils.js'; +import { type Hashable, hashArray } from '@finos/legend-shared'; +import type { PersistenceTarget } from './DSL_Persistence_PersistentTarget.js'; +import type { ServiceOutput } from './DSL_Persistence_ServiceOutput.js'; + +export class ServiceOutputTarget implements Hashable { + serviceOutput!: ServiceOutput; + persistenceTarget!: PersistenceTarget; + get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.SERVICE_OUTPUT_TARGET, + this.serviceOutput, + this.persistenceTarget, + ]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_SourceDerivedDimension.ts b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_SourceDerivedDimension.ts new file mode 100644 index 0000000000..c1e84450a6 --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_SourceDerivedDimension.ts @@ -0,0 +1,37 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type Hashable, hashArray } from '@finos/legend-shared'; +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../DSL_Persistence_HashUtils.js'; +import type { SourceTimeFields } from './DSL_Persistence_SourceTimeFields.js'; + +export abstract class SourceDerivedDimension implements Hashable { + abstract get hashCode(): string; +} + +export class SourceDerivedTime extends SourceDerivedDimension { + timeStart!: string; + timeEnd!: string; + sourceTimeFields!: SourceTimeFields; + + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.SOURCE_DERIVED_TIME, + this.timeStart, + this.timeEnd, + ]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_SourceTimeFields.ts b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_SourceTimeFields.ts new file mode 100644 index 0000000000..6aeca50d49 --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_SourceTimeFields.ts @@ -0,0 +1,46 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type Hashable, hashArray } from '@finos/legend-shared'; +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../DSL_Persistence_HashUtils.js'; + +export abstract class SourceTimeFields implements Hashable { + abstract get hashCode(): string; +} + +export class SourceTimeStart extends SourceTimeFields { + startField!: string; + + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.SOURCE_TIME_START, + this.startField, + ]); + } +} + +export class SourceTimeStartAndEnd extends SourceTimeFields { + startField!: string; + endField!: string; + + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.SOURCE_TIME_START_AND_END, + this.startField, + this.endField, + ]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_Temporality.ts b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_Temporality.ts new file mode 100644 index 0000000000..b5cf033551 --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_Temporality.ts @@ -0,0 +1,63 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type Hashable, hashArray } from '@finos/legend-shared'; +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../DSL_Persistence_HashUtils.js'; +import type { UpdatesHandling } from './DSL_Persistence_UpdatesHandling.js'; +import type { ProcessingDimension } from './DSL_Persistence_ProcessingDimension.js'; +import type { SourceDerivedDimension } from './DSL_Persistence_SourceDerivedDimension.js'; +import type { AuditingV2 } from './DSL_Persistence_AuditingV2.js'; + +export abstract class Temporality implements Hashable { + abstract get hashCode(): string; +} + +export class NonTemporal extends Temporality { + auditing!: AuditingV2; + updatesHandling!: UpdatesHandling; + + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.NON_TEMPORAL, + this.auditing, + this.updatesHandling, + ]); + } +} + +export class UniTemporal extends Temporality { + processingDimension!: ProcessingDimension; + + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.UNI_TEMPORAL, + this.processingDimension, + ]); + } +} + +export class BiTemporal extends Temporality { + processingDimension!: ProcessingDimension; + sourceDerivedDimension!: SourceDerivedDimension; + + override get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.UNI_TEMPORAL, + this.processingDimension, + this.sourceDerivedDimension, + ]); + } +} diff --git a/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_UpdatesHandling.ts b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_UpdatesHandling.ts new file mode 100644 index 0000000000..c55e71cecc --- /dev/null +++ b/packages/legend-extension-dsl-persistence/src/graph/metamodel/pure/model/packageableElements/persistence/DSL_Persistence_UpdatesHandling.ts @@ -0,0 +1,46 @@ +/** + * Copyright (c) 2020-present, Goldman Sachs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { type Hashable, hashArray } from '@finos/legend-shared'; +import { PERSISTENCE_HASH_STRUCTURE } from '../../../../../DSL_Persistence_HashUtils.js'; +import type { AppendStrategy } from './DSL_Persistence_AppendStrategy.js'; + +export abstract class UpdatesHandling implements Hashable { + abstract get hashCode(): string; +} + +export class AppendOnlyUpdatesHandling + extends UpdatesHandling + implements Hashable +{ + appendStrategy!: AppendStrategy; + + get hashCode(): string { + return hashArray([ + PERSISTENCE_HASH_STRUCTURE.APPEND_ONLY_UPDATES, + this.appendStrategy, + ]); + } +} + +export class OverwriteUpdatesHandling + extends UpdatesHandling + implements Hashable +{ + get hashCode(): string { + return hashArray([PERSISTENCE_HASH_STRUCTURE.OVERWRITE_UPDATES]); + } +} diff --git a/packages/legend-manual-tests/src/__tests__/roundtrip-grammar/cases/DSL_PersistenceV2-basic.pure b/packages/legend-manual-tests/src/__tests__/roundtrip-grammar/cases/DSL_PersistenceV2-basic.pure new file mode 100644 index 0000000000..317fd163a7 --- /dev/null +++ b/packages/legend-manual-tests/src/__tests__/roundtrip-grammar/cases/DSL_PersistenceV2-basic.pure @@ -0,0 +1,178 @@ +###Pure +Class test::Person +{ + name: String[1]; +} + +Class test::ServiceResult +{ + ID: String[1]; + NAME: String[1]; +} + + +###Mapping +Mapping test::Mapping +( +) + + +###Relational +Database test::TestDatabase +( + Table personTable + ( + ID INTEGER PRIMARY KEY, + NAME VARCHAR(100), + BATCH_IN INTEGER PRIMARY KEY, + BATCH_OUT INTEGER, + IN_Z TIMESTAMP PRIMARY KEY, + OUT_Z TIMESTAMP + ) +) + + +###Service +Service test::Service +{ + pattern: 'test'; + documentation: 'test'; + autoActivateUpdates: true; + execution: Single + { + query: src: test::Person[1]|$src.name; + mapping: test::Mapping; + runtime: + #{ + mappings: + [ + test::Mapping + ]; + }#; + } + testSuites: + [ + + ] +} + + +###Persistence +Persistence test::TestPersistence +{ + doc: 'This is test documentation.'; + trigger: Manual; + service: test::Service; + serviceOutputTargets: + [ + TDS + { + keys: + [ + ID + ] + datasetType: Snapshot + { + partitioning: None + { + emptyDatasetHandling: NoOp; + } + } + deduplication: None; + } + -> + Relational + #{ + table: personTable; + database: test::TestDatabase; + temporality: Unitemporal + { + processingDimension: BatchIdAndDateTime + { + batchIdIn: BATCH_IN; + batchIdOut: BATCH_OUT; + dateTimeIn: IN_Z; + dateTimeOut: OUT_Z; + } + } + }# + ]; + notifier: + { + notifyees: + [ + PagerDuty + { + url: 'http://xyz.com'; + }, + Email + { + address: 'http://xyz.com'; + } + ]; + } + tests: + [ + test1: + { + testBatches: + [ + testBatch1: + { + data: + { + connection: + { + ExternalFormat + #{ + contentType: 'application/json'; + data: '[{"ID":1, "NAME":"ANDY"},{"ID":2, "NAME":"BRAD"},{"ID":3, "NAME":"CATHY"}]'; + }# + } + } + asserts: + [ + assert1: + EqualToJson + #{ + expected: + ExternalFormat + #{ + contentType: 'application/json'; + data: '[{"ID":1, "NAME":"ANDY", "BATCH_IN":1, "BATCH_OUT":999999999},{"ID":2, "NAME":"BRAD", "BATCH_IN":1, "BATCH_OUT":999999999},{"ID":3, "NAME":"CATHY", "BATCH_IN":1, "BATCH_OUT":999999999}]'; + }#; + }# + ] + }, + testBatch2: + { + data: + { + connection: + { + ExternalFormat + #{ + contentType: 'application/json'; + data: '[{"ID":1, "NAME":"ANDY"},{"ID":3, "NAME":"CATHERINE"},{"ID":4, "NAME":"TOM"}]'; + }# + } + } + asserts: + [ + assert1: + EqualToJson + #{ + expected: + ExternalFormat + #{ + contentType: 'application/json'; + data: '[{"ID":1, "NAME":"ANDY", "BATCH_IN":1, "BATCH_OUT":999999999},{"ID":2, "NAME":"BRAD", "BATCH_IN":1, "BATCH_OUT":1}, {"ID":3, "NAME":"CATHY", "BATCH_IN":1, "BATCH_OUT":1}, {"ID":3, "NAME":"CATHERINE", "BATCH_IN":2, "BATCH_OUT":999999999}, {"ID":4, "NAME":"TOM", "BATCH_IN":2, "BATCH_OUT":999999999}]'; + }#; + }# + ] + } + ] + isTestDataFromServiceOutput: true; + } + ] +} diff --git a/packages/legend-manual-tests/src/__tests__/roundtrip-grammar/cases/DSL_PersistenceV2-bitemporal.pure b/packages/legend-manual-tests/src/__tests__/roundtrip-grammar/cases/DSL_PersistenceV2-bitemporal.pure new file mode 100644 index 0000000000..9de83f48ee --- /dev/null +++ b/packages/legend-manual-tests/src/__tests__/roundtrip-grammar/cases/DSL_PersistenceV2-bitemporal.pure @@ -0,0 +1,188 @@ +###Pure +Class test::Person +{ + name: String[1]; +} + +Class test::ServiceResult +{ + ID: String[1]; + AMOUNT: Integer[1]; + SOURCE_FROM: String[1]; + DELETED: String[1]; +} + + +###Mapping +Mapping test::Mapping +( +) + + +###Relational +Database test::TestDatabase +( + Table personTable + ( + ID INTEGER PRIMARY KEY, + AMOUNT INTEGER, + FROM_Z TIMESTAMP PRIMARY KEY, + THRU_Z TIMESTAMP, + BATCH_IN INTEGER PRIMARY KEY, + BATCH_OUT INTEGER + ) +) + + +###Service +Service test::Service +{ + pattern: 'test'; + documentation: 'test'; + autoActivateUpdates: true; + execution: Single + { + query: src: test::Person[1]|$src.name; + mapping: test::Mapping; + runtime: + #{ + mappings: + [ + test::Mapping + ]; + }#; + } + testSuites: + [ + + ] +} + + +###Persistence +Persistence test::TestPersistence +{ + doc: 'This is test documentation.'; + trigger: Manual; + service: test::Service; + serviceOutputTargets: + [ + TDS + { + keys: + [ + ID + ] + datasetType: Delta + { + actionIndicator: DeleteIndicator + { + deleteField: DELETED; + deleteValues: ['Y']; + } + } + deduplication: None; + } + -> + Relational + #{ + table: personTable; + database: test::TestDatabase; + temporality: Bitemporal + { + processingDimension: BatchId + { + batchIdIn: BATCH_IN; + batchIdOut: BATCH_OUT; + } + sourceDerivedDimension: DateTime + { + dateTimeStart: FROM_Z; + dateTimeEnd: THRU_Z; + sourceFields: Start + { + startField: SOURCE_FROM; + } + } + } + }# + ]; + notifier: + { + notifyees: + [ + PagerDuty + { + url: 'http://xyz.com'; + }, + Email + { + address: 'http://xyz.com'; + } + ]; + } + tests: + [ + test1: + { + testBatches: + [ + testBatch1: + { + data: + { + connection: + { + ExternalFormat + #{ + contentType: 'application/json'; + data: '[{"ID":1, "AMOUNT":100, "SOURCE_FROM": "2022-01-01 00:00:00.0","DELETED": "N"},{"ID":2, "AMOUNT":200, "SOURCE_FROM": "2022-01-01 00:00:00.0","DELETED": "N"},{"ID":3, "AMOUNT":400, "SOURCE_FROM": "2022-01-01 00:00:00.0","DELETED": "N"}]'; + }# + } + } + asserts: + [ + assert1: + EqualToJson + #{ + expected: + ExternalFormat + #{ + contentType: 'application/json'; + data: '[{"ID":1,"AMOUNT":100,"BATCH_IN":1,"THRU_Z":"9999-12-31 23:59:59.0","BATCH_OUT":999999999,"FROM_Z":"2022-01-01 00:00:00.0"},{"ID":2,"AMOUNT":200,"BATCH_IN":1,"THRU_Z":"9999-12-31 23:59:59.0","BATCH_OUT":999999999,"FROM_Z":"2022-01-01 00:00:00.0"},{"ID":3,"AMOUNT":400,"BATCH_IN":1,"THRU_Z":"9999-12-31 23:59:59.0","BATCH_OUT":999999999,"FROM_Z":"2022-01-01 00:00:00.0"}]'; + }#; + }# + ] + }, + testBatch2: + { + data: + { + connection: + { + ExternalFormat + #{ + contentType: 'application/json'; + data: '[{"ID":1, "AMOUNT":200, "SOURCE_FROM": "2022-02-01 00:00:00.0","DELETED": "N"},{"ID":2, "AMOUNT":200, "SOURCE_FROM": "2022-01-01 00:00:00.0","DELETED": "Y"},{"ID":3, "AMOUNT":800, "SOURCE_FROM": "2022-01-01 00:00:00.0", "DELETED": "N"}]'; + }# + } + } + asserts: + [ + assert1: + EqualToJson + #{ + expected: + ExternalFormat + #{ + contentType: 'application/json'; + data: '[{"ID":1,"AMOUNT":100,"BATCH_IN":1,"THRU_Z":"9999-12-31 23:59:59.0","BATCH_OUT":1,"FROM_Z":"2022-01-01 00:00:00.0"},{"ID":2,"AMOUNT":200,"BATCH_IN":1,"THRU_Z":"9999-12-31 23:59:59.0","BATCH_OUT":1,"FROM_Z":"2022-01-01 00:00:00.0"},{"ID":3,"AMOUNT":400,"BATCH_IN":1,"THRU_Z":"9999-12-31 23:59:59.0","BATCH_OUT":1,"FROM_Z":"2022-01-01 00:00:00.0"},{"ID":1,"AMOUNT":200,"BATCH_IN":2,"THRU_Z":"9999-12-31 23:59:59.0","BATCH_OUT":999999999,"FROM_Z":"2022-02-01 00:00:00.0"},{"ID":3,"AMOUNT":800,"BATCH_IN":2,"THRU_Z":"9999-12-31 23:59:59.0","BATCH_OUT":999999999,"FROM_Z":"2022-01-01 00:00:00.0"},{"ID":1,"AMOUNT":100,"BATCH_IN":2,"THRU_Z":"2022-02-01 00:00:00.0","BATCH_OUT":999999999,"FROM_Z":"2022-01-01 00:00:00.0"}]'; + }#; + }# + ] + } + ] + isTestDataFromServiceOutput: true; + } + ] +} diff --git a/packages/legend-manual-tests/src/__tests__/roundtrip-grammar/cases/DSL_PersistenceV2-unitemporal.pure b/packages/legend-manual-tests/src/__tests__/roundtrip-grammar/cases/DSL_PersistenceV2-unitemporal.pure new file mode 100644 index 0000000000..17ad88c549 --- /dev/null +++ b/packages/legend-manual-tests/src/__tests__/roundtrip-grammar/cases/DSL_PersistenceV2-unitemporal.pure @@ -0,0 +1,128 @@ +###Pure +Class test::Person +{ + name: String[1]; +} + +Class test::ServiceResult +{ + ID: String[1]; + AMOUNT: Integer[1]; + SOURCE_FROM: String[1]; + DELETED: String[1]; +} + + +###Mapping +Mapping test::Mapping +( +) + + +###Relational +Database test::TestDatabase +( + Table personTable + ( + ID INTEGER PRIMARY KEY, + AMOUNT INTEGER, + FROM_Z TIMESTAMP PRIMARY KEY, + THRU_Z TIMESTAMP, + BATCH_IN INTEGER PRIMARY KEY, + BATCH_OUT INTEGER + ) +) + + +###Service +Service test::Service +{ + pattern: 'test'; + documentation: 'test'; + autoActivateUpdates: true; + execution: Single + { + query: src: test::Person[1]|$src.name; + mapping: test::Mapping; + runtime: + #{ + mappings: + [ + test::Mapping + ]; + }#; + } + testSuites: + [ + + ] +} + + +###Persistence +Persistence test::TestPersistence +{ + doc: 'This is test documentation.'; + trigger: Manual; + service: test::Service; + serviceOutputTargets: + [ + TDS + { + keys: + [ + ID + ] + datasetType: Delta + { + actionIndicator: DeleteIndicator + { + deleteField: DELETED; + deleteValues: ['Y']; + } + } + deduplication: None; + } + -> + Relational + #{ + table: personTable; + database: test::TestDatabase; + temporality: Bitemporal + { + processingDimension: BatchId + { + batchIdIn: BATCH_IN; + batchIdOut: BATCH_OUT; + } + sourceDerivedDimension: DateTime + { + dateTimeStart: FROM_Z; + dateTimeEnd: THRU_Z; + sourceFields: Start + { + startField: SOURCE_FROM; + } + } + } + }# + ]; + notifier: + { + notifyees: + [ + PagerDuty + { + url: 'http://xyz.com'; + }, + Email + { + address: 'http://xyz.com'; + } + ]; + } + tests: + [ + + ] +}