fix: rebase on 3.5.0

backend/shared/model/command.model.ts

@@ -4,7 +4,7 @@ import { ScanModeCommandDTO } from './scan-mode.model';
 import { SouthConnectorCommandDTO } from './south-connector.model';
 import { SouthItemSettings, SouthSettings } from './south-settings.model';
 import { NorthConnectorCommandDTO } from './north-connector.model';
-import { NorthSettings } from './north-settings.model';
+import { NorthItemSettings, NorthSettings } from './north-settings.model';
 
 export const OIBUS_COMMAND_TYPES = [
   'update-version',
@@ -113,14 +113,14 @@ export interface OIBusDeleteSouthConnectorCommandDTO extends BaseOIBusCommandDTO
 export interface OIBusCreateNorthConnectorCommandDTO extends BaseOIBusCommandDTO {
   type: 'create-north';
   targetVersion: string;
-  commandContent: NorthConnectorCommandDTO<NorthSettings>;
+  commandContent: NorthConnectorCommandDTO<NorthSettings, NorthItemSettings>;
 }
 
 export interface OIBusUpdateNorthConnectorCommandDTO extends BaseOIBusCommandDTO {
   type: 'update-north';
   targetVersion: string;
   northConnectorId: string;
-  commandContent: NorthConnectorCommandDTO<NorthSettings>;
+  commandContent: NorthConnectorCommandDTO<NorthSettings, NorthItemSettings>;
 }
 
 export interface OIBusDeleteNorthConnectorCommandDTO extends BaseOIBusCommandDTO {

backend/shared/model/history-query.model.ts

@@ -1,8 +1,9 @@
 import { BaseEntity } from './types';
 import { SouthItemSettings, SouthSettings } from './south-settings.model';
-import { NorthSettings } from './north-settings.model';
+import { NorthItemSettings, NorthSettings } from './north-settings.model';
 import { OIBusSouthType } from './south-connector.model';
 import { OIBusNorthType } from './north-connector.model';
+import { Transformer } from '../../src/model/transformer.model';
 
 export const HISTORY_QUERY_STATUS = ['PENDING', 'RUNNING', 'PAUSED', 'FINISHED', 'ERRORED'] as const;
 export type HistoryQueryStatus = (typeof HISTORY_QUERY_STATUS)[number];
@@ -17,7 +18,8 @@ export interface HistoryQueryLightDTO extends BaseEntity {
   northType: OIBusNorthType;
 }
 
-export interface HistoryQueryDTO<S extends SouthSettings, N extends NorthSettings, I extends SouthItemSettings> extends BaseEntity {
+export interface HistoryQueryDTO<S extends SouthSettings, N extends NorthSettings, I extends SouthItemSettings, J extends NorthItemSettings>
+  extends BaseEntity {
   name: string;
   description: string;
   status: HistoryQueryStatus;
@@ -41,10 +43,18 @@ export interface HistoryQueryDTO<S extends SouthSettings, N extends NorthSetting
       archive: { enabled: boolean; retentionDuration: number };
     };
   };
-  items: Array<HistoryQueryItemDTO<I>>;
+  southItems: Array<HistoryQuerySouthItemDTO<I>>;
+  northItems: Array<HistoryQueryNorthItemDTO<J>>;
+  southTransformers: Array<{ order: number; transformer: Transformer }>;
+  northTransformers: Array<{ order: number; transformer: Transformer }>;
 }
 
-export interface HistoryQueryCommandDTO<S extends SouthSettings, N extends NorthSettings, I extends SouthItemSettings> {
+export interface HistoryQueryCommandDTO<
+  S extends SouthSettings,
+  N extends NorthSettings,
+  I extends SouthItemSettings,
+  J extends NorthItemSettings
+> {
   name: string;
   description: string;
   startTime: string;
@@ -71,7 +81,10 @@ export interface HistoryQueryCommandDTO<S extends SouthSettings, N extends North
       };
     };
   };
-  items: Array<HistoryQueryItemCommandDTO<I>>;
+  southItems: Array<HistoryQuerySouthItemCommandDTO<I>>;
+  northItems: Array<HistoryQueryNorthItemCommandDTO<J>>;
+  southTransformers: Array<{ order: number; id: string }>;
+  northTransformers: Array<{ order: number; id: string }>;
 }
 
 export interface HistoryQueryItemSearchParam {
@@ -80,13 +93,26 @@ export interface HistoryQueryItemSearchParam {
   page?: number;
 }
 
-export interface HistoryQueryItemDTO<T extends SouthItemSettings> extends BaseEntity {
+export interface HistoryQuerySouthItemDTO<T extends SouthItemSettings> extends BaseEntity {
   name: string;
   enabled: boolean;
   settings: T;
 }
 
-export interface HistoryQueryItemCommandDTO<T extends SouthItemSettings> {
+export interface HistoryQueryNorthItemDTO<T extends NorthItemSettings> extends BaseEntity {
+  name: string;
+  enabled: boolean;
+  settings: T;
+}
+
+export interface HistoryQuerySouthItemCommandDTO<T extends SouthItemSettings> {
+  id: string | null;
+  name: string;
+  enabled: boolean;
+  settings: T;
+}
+
+export interface HistoryQueryNorthItemCommandDTO<T extends NorthItemSettings> {
   id: string | null;
   name: string;
   enabled: boolean;

backend/shared/model/north-connector.model.ts

@@ -1,7 +1,8 @@
 import { OibFormControl } from './form.model';
 import { BaseEntity, Instant } from './types';
-import { NorthSettings } from './north-settings.model';
+import { NorthSettings, NorthItemSettings } from './north-settings.model';
 import { SouthConnectorLightDTO } from './south-connector.model';
+import { TransformerDTO } from './transformer.model';
 
 export const OIBUS_NORTH_CATEGORIES = ['debug', 'api', 'file'] as const;
 export type OIBusNorthCategory = (typeof OIBUS_NORTH_CATEGORIES)[number];
@@ -25,7 +26,7 @@ export interface NorthConnectorLightDTO extends BaseEntity {
   enabled: boolean;
 }
 
-export interface NorthConnectorDTO<T extends NorthSettings> extends BaseEntity {
+export interface NorthConnectorDTO<T extends NorthSettings, I extends NorthItemSettings> extends BaseEntity {
   name: string;
   type: OIBusNorthType;
   description: string;
@@ -49,9 +50,11 @@ export interface NorthConnectorDTO<T extends NorthSettings> extends BaseEntity {
     };
   };
   subscriptions: Array<SouthConnectorLightDTO>;
+  items: Array<NorthConnectorItemDTO<I>>;
+  transformers: Array<{ transformer: TransformerDTO; order: number }>;
 }
 
-export interface NorthConnectorCommandDTO<T extends NorthSettings> {
+export interface NorthConnectorCommandDTO<T extends NorthSettings, I extends NorthItemSettings> {
   name: string;
   type: OIBusNorthType;
   description: string;
@@ -76,6 +79,30 @@ export interface NorthConnectorCommandDTO<T extends NorthSettings> {
     };
   };
   subscriptions: Array<string>;
+  items: Array<NorthConnectorItemCommandDTO<I>>;
+  transformers: Array<{ id: string; order: number }>;
+}
+
+/**
+ * DTO used for an item to query within a north
+ */
+export interface NorthConnectorItemDTO<T extends NorthItemSettings> extends BaseEntity {
+  name: string;
+  enabled: boolean;
+  settings: T;
+}
+
+export interface NorthConnectorItemCommandDTO<T extends NorthItemSettings> {
+  id: string | null;
+  enabled: boolean;
+  name: string;
+  settings: T;
+}
+
+export interface NorthConnectorItemSearchParam {
+  name?: string;
+  enabled?: boolean;
+  page?: number;
 }
 
 export interface NorthConnectorManifest {
@@ -86,6 +113,7 @@ export interface NorthConnectorManifest {
     points: boolean;
   };
   settings: Array<OibFormControl>;
+  items: { settings: Array<OibFormControl> };
 }
 
 export interface NorthCacheFiles {

backend/shared/model/north-settings.model.ts

@@ -96,3 +96,18 @@ export type NorthSettings =
   | NorthFileWriterSettings
   | NorthOIAnalyticsSettings
   | NorthSFTPSettings;
+
+export interface NorthAmazonS3ItemSettings {}
+export interface NorthAzureBlobItemSettings {}
+export interface NorthConsoleItemSettings {}
+export interface NorthFileWriterItemSettings {}
+export interface NorthOIAnalyticsItemSettings {}
+export interface NorthSFTPItemSettings {}
+
+export type NorthItemSettings =
+  | NorthAmazonS3ItemSettings
+  | NorthAzureBlobItemSettings
+  | NorthConsoleItemSettings
+  | NorthFileWriterItemSettings
+  | NorthOIAnalyticsItemSettings
+  | NorthSFTPItemSettings;

backend/shared/model/south-connector.model.ts

@@ -1,6 +1,7 @@
 import { OibFormControl } from './form.model';
 import { BaseEntity, Instant } from './types';
 import { SouthItemSettings, SouthSettings } from './south-settings.model';
+import { TransformerDTO } from './transformer.model';
 
 export const OIBUS_SOUTH_CATEGORIES = ['file', 'iot', 'database', 'api'] as const;
 export type OIBusSouthCategory = (typeof OIBUS_SOUTH_CATEGORIES)[number];
@@ -51,6 +52,7 @@ export interface SouthConnectorDTO<T extends SouthSettings, I extends SouthItemS
   enabled: boolean;
   settings: T;
   items: Array<SouthConnectorItemDTO<I>>;
+  transformers: Array<{ transformer: TransformerDTO; order: number }>;
 }
 
 export interface SouthConnectorCommandDTO<T extends SouthSettings, I extends SouthItemSettings> {
@@ -60,6 +62,7 @@ export interface SouthConnectorCommandDTO<T extends SouthSettings, I extends Sou
   enabled: boolean;
   settings: T;
   items: Array<SouthConnectorItemCommandDTO<I>>;
+  transformers: Array<{ id: string; order: number }>;
 }
 
 /**

backend/shared/model/transformer.model.ts

@@ -1,13 +1,22 @@
 export interface TransformerDTO {
   id: string;
   name: string;
-  description: string | null;
+  description: string;
   inputType: string;
   outputType: string;
   code: string;
-  fileRegex: string | null;
 }
 
-export type TransformerFilterDTO = Partial<Pick<TransformerDTO, 'inputType' | 'outputType' | 'name'>>;
+export interface TransformerCommand {
+  name: string;
+  description: string;
+  inputType: string;
+  outputType: string;
+  code: string;
+}
 
-export type TransformerCommandDTO = Omit<TransformerDTO, 'id'>;
+export interface TransformerSearchParam {
+  name?: string;
+  inputType?: string;
+  outputType?: string;
+}

backend/shared/model/types.ts

@@ -142,4 +142,5 @@ export interface ConnectorManifest {
   name: string;
   description: string;
   settings: Array<OibFormControl>;
+  items: { settings: Array<OibFormControl> };
 }

backend/src/engine/data-stream-engine.spec.ts

@@ -5,7 +5,7 @@ import DataStreamEngine from './data-stream-engine';
 import testData from '../tests/utils/test-data';
 import { OIBusRawContent, OIBusTimeValueContent } from '../../shared/model/engine.model';
 import NorthConnector from '../north/north-connector';
-import { NorthSettings } from '../../shared/model/north-settings.model';
+import { NorthItemSettings, NorthSettings } from '../../shared/model/north-settings.model';
 import SouthConnector from '../south/south-connector';
 import { SouthItemSettings, SouthSettings } from '../../shared/model/south-settings.model';
 import NorthConnectorMock from '../tests/__mocks__/north-connector.mock';
@@ -53,8 +53,8 @@ const anotherLogger: pino.Logger = new PinoLogger();
 
 describe('DataStreamEngine', () => {
   let engine: DataStreamEngine;
-  const mockedNorth1 = new NorthConnectorMock(testData.north.list[0]) as unknown as NorthConnector<NorthSettings>;
-  const mockedNorth2 = new NorthConnectorMock(testData.north.list[1]) as unknown as NorthConnector<NorthSettings>;
+  const mockedNorth1 = new NorthConnectorMock(testData.north.list[0]) as unknown as NorthConnector<NorthSettings, NorthItemSettings>;
+  const mockedNorth2 = new NorthConnectorMock(testData.north.list[1]) as unknown as NorthConnector<NorthSettings, NorthItemSettings>;
   const mockedSouth1 = new SouthConnectorMock(testData.south.list[0]) as unknown as SouthConnector<SouthSettings, SouthItemSettings>;
   const mockedSouth2 = new SouthConnectorMock(testData.south.list[1]) as unknown as SouthConnector<SouthSettings, SouthItemSettings>;
 
@@ -355,9 +355,9 @@ describe('DataStreamEngine', () => {
   it('should manage item change', async () => {
     await engine.start([], [mockedSouth1]);
 
-    await engine.reloadItems('bad id');
+    await engine.reloadSouthItems('bad id');
     expect(mockedSouth1.onItemChange).not.toHaveBeenCalled();
-    await engine.reloadItems(testData.south.list[0].id);
+    await engine.reloadSouthItems(testData.south.list[0].id);
     expect(mockedSouth1.onItemChange).toHaveBeenCalled();
   });
 

backend/src/engine/data-stream-engine.ts

@@ -6,7 +6,7 @@ import { Instant } from '../../shared/model/types';
 import { BaseFolders } from '../model/types';
 import { NorthConnectorMetrics, OIBusContent, SouthConnectorMetrics } from '../../shared/model/engine.model';
 import { ScanMode } from '../model/scan-mode.model';
-import { NorthSettings } from '../../shared/model/north-settings.model';
+import { NorthItemSettings, NorthSettings } from '../../shared/model/north-settings.model';
 import { SouthItemSettings, SouthSettings } from '../../shared/model/south-settings.model';
 import { SouthConnectorEntity } from '../model/south-connector.model';
 import { NorthConnectorEntity } from '../model/north-connector.model';
@@ -23,7 +23,7 @@ const ARCHIVE_FOLDER = './archive';
 const ERROR_FOLDER = './error';
 
 export default class DataStreamEngine {
-  private northConnectors = new Map<string, NorthConnector<NorthSettings>>();
+  private northConnectors = new Map<string, NorthConnector<NorthSettings, NorthItemSettings>>();
   private northConnectorMetrics: Map<string, NorthConnectorMetricsService> = new Map<string, NorthConnectorMetricsService>();
   private southConnectors = new Map<string, SouthConnector<SouthSettings, SouthItemSettings>>();
   private southConnectorMetrics: Map<string, SouthConnectorMetricsService> = new Map<string, SouthConnectorMetricsService>();
@@ -124,7 +124,7 @@ export default class DataStreamEngine {
    * Creates CronJobs based on the ScanModes and starts them.
    */
   async start(
-    northConnectorList: Array<NorthConnector<NorthSettings>>,
+    northConnectorList: Array<NorthConnector<NorthSettings, NorthItemSettings>>,
     southConnectorList: Array<SouthConnector<SouthSettings, SouthItemSettings>>
   ): Promise<void> {
     for (const north of northConnectorList) {
@@ -194,7 +194,7 @@ export default class DataStreamEngine {
     });
   }
 
-  async createNorth<N extends NorthSettings>(north: NorthConnector<N>): Promise<void> {
+  async createNorth<N extends NorthSettings, I extends NorthItemSettings>(north: NorthConnector<N, I>): Promise<void> {
     this.northConnectors.set(north.settings.id, north);
     this.northConnectorMetrics.set(north.settings.id, new NorthConnectorMetricsService(north, this.northConnectorMetricsRepository));
   }
@@ -235,7 +235,7 @@ export default class DataStreamEngine {
   /**
    * Stops the north connector and deletes all cache inside the base folder
    */
-  async deleteNorth(north: NorthConnectorEntity<NorthSettings>): Promise<void> {
+  async deleteNorth(north: NorthConnectorEntity<NorthSettings, NorthItemSettings>): Promise<void> {
     await this.stopNorth(north.id);
     // this.homeMetricsService.removeNorth(northId);
     this.northConnectors.delete(north.id);
@@ -373,7 +373,7 @@ export default class DataStreamEngine {
     }
   }
 
-  async reloadItems(southId: string): Promise<void> {
+  async reloadSouthItems(southId: string): Promise<void> {
     await this.southConnectors.get(southId)?.onItemChange();
   }
 
@@ -391,7 +391,7 @@ export default class DataStreamEngine {
     }
   }
 
-  async reloadNorth(northConnector: NorthConnectorEntity<NorthSettings>) {
+  async reloadNorth(northConnector: NorthConnectorEntity<NorthSettings, NorthItemSettings>) {
     await this.stopNorth(northConnector.id);
     const north = this.northConnectors.get(northConnector.id);
     if (north) {

backend/src/engine/history-query-engine.ts

@@ -3,7 +3,7 @@ import HistoryQuery from './history-query';
 import path from 'node:path';
 import { HistoryQueryEntity } from '../model/histor-query.model';
 import { SouthItemSettings, SouthSettings } from '../../shared/model/south-settings.model';
-import { NorthSettings } from '../../shared/model/north-settings.model';
+import { NorthItemSettings, NorthSettings } from '../../shared/model/north-settings.model';
 import HistoryQueryMetricsService from '../service/metrics/history-query-metrics.service';
 import HistoryQueryMetricsRepository from '../repository/logs/history-query-metrics.repository';
 import { PassThrough } from 'node:stream';
@@ -93,7 +93,10 @@ export default class HistoryQueryEngine {
     this.historyQueryMetrics.get(historyId)?.resetMetrics();
   }
 
-  async reloadHistoryQuery(historyQuery: HistoryQueryEntity<SouthSettings, NorthSettings, SouthItemSettings>, resetCache: boolean) {
+  async reloadHistoryQuery(
+    historyQuery: HistoryQueryEntity<SouthSettings, NorthSettings, SouthItemSettings, NorthItemSettings>,
+    resetCache: boolean
+  ) {
     await this.stopHistoryQuery(historyQuery.id);
     this.historyQueries
       .get(historyQuery.id)
@@ -117,7 +120,9 @@ export default class HistoryQueryEngine {
   /**
    * Stops the History query and deletes all cache inside the base folder
    */
-  async deleteHistoryQuery(historyQuery: HistoryQueryEntity<SouthSettings, NorthSettings, SouthItemSettings>): Promise<void> {
+  async deleteHistoryQuery(
+    historyQuery: HistoryQueryEntity<SouthSettings, NorthSettings, SouthItemSettings, NorthItemSettings>
+  ): Promise<void> {
     await this.stopHistoryQuery(historyQuery.id);
     await this.resetCache(historyQuery.id);
   }