feat: easy data imports (#1916)

closes #1135

Co-authored-by: Simon <simon@aam-digital.com>

src/app/app.module.ts

@@ -68,7 +68,6 @@ import { AttendanceModule } from "./child-dev-project/attendance/attendance.modu
 import { NotesModule } from "./child-dev-project/notes/notes.module";
 import { SchoolsModule } from "./child-dev-project/schools/schools.module";
 import { ConflictResolutionModule } from "./conflict-resolution/conflict-resolution.module";
-import { DataImportModule } from "./features/data-import/data-import.module";
 import { HistoricalDataModule } from "./features/historical-data/historical-data.module";
 import { MatchingEntitiesModule } from "./features/matching-entities/matching-entities.module";
 import { ProgressDashboardWidgetModule } from "./features/progress-dashboard-widget/progress-dashboard-widget.module";
@@ -81,6 +80,7 @@ import { SessionService } from "./core/session/session-service/session.service";
 import { waitForChangeTo } from "./core/session/session-states/session-utils";
 import { LoginState } from "./core/session/session-states/login-state.enum";
 import { appInitializers } from "./app-initializers";
+import { ImportModule } from "./features/import/import.module";
 
 /**
  * Main entry point of the application.
@@ -115,7 +115,7 @@ import { appInitializers } from "./app-initializers";
     // conflict resolution
     ConflictResolutionModule,
     // feature module
-    DataImportModule,
+    ImportModule,
     FileModule,
     HistoricalDataModule,
     LocationModule,

src/app/app.routing.ts

@@ -62,6 +62,7 @@ export const allRoutes: Routes = [
   },
   { path: "login", component: LoginComponent },
   { path: "404", component: NotFoundComponent },
+
   {
     path: "**",
     pathMatch: "full",

src/app/core/config-setup/config-import/config-import.component.ts

@@ -3,7 +3,7 @@ import { RouteTarget } from "../../../app.routing";
 import {
   InputFileComponent,
   ParsedData,
-} from "../../../features/data-import/input-file/input-file.component";
+} from "../../input-file/input-file.component";
 import { ConfigImportParserService } from "../config-import-parser.service";
 import { MatFormFieldModule } from "@angular/material/form-field";
 import { FormsModule } from "@angular/forms";

src/app/core/config/config-fix.ts

@@ -286,7 +286,6 @@ export const defaultJsonConfig = {
   },
   "view:import": {
     "component": "Import",
-    "permittedUserRoles": ["admin_app"]
   },
   "view:user": {
     "component": "EntityList",

src/app/core/configurable-enum/basic-autocomplete/basic-autocomplete.component.ts

@@ -76,8 +76,9 @@ export class BasicAutocompleteComponent<O, V = O>
   @ViewChild(MatAutocompleteTrigger) autocomplete: MatAutocompleteTrigger;
 
   @Input() valueMapper = (option: O) => option as unknown as V;
-  @Input() optionToString = (option) => option?.toString();
+  @Input() optionToString = (option: O) => option?.toString();
   @Input() createOption: (input: string) => O;
+  @Input() hideOption: (option: O) => boolean = () => false;
   @Input() multi?: boolean;
 
   autocompleteForm = new FormControl("");
@@ -155,7 +156,7 @@ export class BasicAutocompleteComponent<O, V = O>
       // cannot setValue to "" here because the current selection would be lost
       this.autocompleteForm.setValue(this.displayText);
       this.autocompleteSuggestedOptions = concat(
-        of(this._options),
+        of(this._options.filter(({ initial }) => !this.hideOption(initial))),
         this.autocompleteSuggestedOptions.pipe(skip(1)),
       );
     }
@@ -171,10 +172,12 @@ export class BasicAutocompleteComponent<O, V = O>
   }
 
   private updateAutocomplete(inputText: string): SelectableOption<O, V>[] {
-    let filteredOptions = this._options;
+    let filteredOptions = this._options.filter(
+      (o) => !this.hideOption(o.initial)
+    );
     if (inputText) {
-      filteredOptions = this._options.filter((option) =>
-        option.asString.toLowerCase().includes(inputText.toLowerCase()),
+      filteredOptions = filteredOptions.filter((o) =>
+        o.asString.toLowerCase().includes(inputText.toLowerCase()),
       );
       this.showAddOption = !this._options.some(
         (o) => o.asString.toLowerCase() === inputText.toLowerCase(),

src/app/core/configurable-enum/configurable-enum-datatype/configurable-enum-datatype.ts

@@ -17,7 +17,7 @@ export class ConfigurableEnumDatatype
    * @param value Object to be saved as specified in config file; e.g. `{id: 'CALL', label:'Phone Call', color:'#FFFFFF'}`
    */
   public transformToDatabaseFormat(value: ConfigurableEnumValue): string {
-    return value.id;
+    return value?.id;
   }
 
   /**

src/app/core/entity-components/entity-list/entity-list.component.html

@@ -95,7 +95,7 @@ <h2>{{ listName }}</h2>
   <mat-form-field class="full-width filter-field">
     <mat-label
       i18n="Filter placeholder|Allows the user to filter through entities"
-    >Filter
+      >Filter
     </mat-label>
     <input
       class="full-width"
@@ -184,5 +184,22 @@ <h2>{{ listName }}</h2>
     ></fa-icon>
     <span i18n="Download list contents as CSV"> Download CSV </span>
   </button>
+
+  <button
+    mat-menu-item
+    angulartics2On="click"
+    [angularticsCategory]="entityConstructor?.ENTITY_TYPE"
+    angularticsAction="import_file"
+    [routerLink]="['/import']"
+    [queryParams]="{ entityType: entityConstructor?.ENTITY_TYPE }"
+  >
+    <fa-icon
+      class="color-accent standard-icon-with-text"
+      aria-label="import file"
+      icon="file-import"
+    ></fa-icon>
+    <span i18n> Import from file </span>
+  </button>
+
   <ng-content select="[mat-menu-item]"></ng-content>
 </mat-menu>

src/app/core/entity-components/entity-list/entity-list.component.ts

@@ -8,7 +8,7 @@ import {
   SimpleChanges,
   ViewChild,
 } from "@angular/core";
-import { ActivatedRoute, Router } from "@angular/router";
+import { ActivatedRoute, Router, RouterLink } from "@angular/router";
 import {
   ColumnGroupsConfig,
   EntityListConfig,
@@ -77,6 +77,7 @@ import { DisableEntityOperationDirective } from "../../permissions/permission-di
     ViewTitleComponent,
     ExportDataDirective,
     DisableEntityOperationDirective,
+    RouterLink,
   ],
   standalone: true,
 })

src/app/core/entity-components/entity-type-label/entity-type-label.pipe.spec.ts

@@ -0,0 +1,9 @@
+import { EntityTypeLabelPipe } from "./entity-type-label.pipe";
+import { entityRegistry } from "../../entity/database-entity.decorator";
+
+describe("EntityTypeLabelPipe", () => {
+  it("create an instance", () => {
+    const pipe = new EntityTypeLabelPipe(entityRegistry);
+    expect(pipe).toBeTruthy();
+  });
+});

src/app/core/entity-components/entity-utils/dynamic-form-components/edit-component.ts

@@ -57,7 +57,7 @@ export abstract class EditComponent<T> implements OnInit {
 
   ngOnInit() {
     if (!this.formFieldConfig?.forTable) {
-      this.label = this.formFieldConfig?.label || this.propertySchema?.label;
+      this.label = this.formFieldConfig?.label ?? this.propertySchema?.label;
     }
     if (this.formFieldConfig?.forTable) {
       this.tooltip = undefined;

src/app/core/entity/entity-config.service.ts

@@ -85,6 +85,8 @@ export class EntityConfigService {
     entityType.icon = (entityConfig.icon as IconName) ?? entityType.icon;
     entityType.color = entityConfig.color ?? entityType.color;
     entityType.route = entityConfig.route ?? entityType.route;
+
+    entityType._isCustomizedType = true;
   }
 
   /**

src/app/core/entity/entity-mapper.service.ts

@@ -156,7 +156,7 @@ export class EntityMapperService {
    * saved
    * @param entities The entities to save
    */
-  public async saveAll(entities: Entity[]): Promise<any> {
+  public async saveAll(entities: Entity[]): Promise<any[]> {
     entities.forEach((e) => this.setEntityMetadata(e));
     const rawData = entities.map((e) =>
       this.entitySchemaService.transformEntityToDatabaseFormat(e)
@@ -189,7 +189,7 @@ export class EntityMapperService {
     }
   }
 
-  private setEntityMetadata(entity: Entity) {
+  protected setEntityMetadata(entity: Entity) {
     const newMetadata = new UpdateMetadata(
       this.sessionService.getCurrentUser()?.name
     );

src/app/core/entity/mock-entity-mapper-service.ts

@@ -4,6 +4,7 @@ import { UpdatedEntity } from "./model/entity-update";
 import { Observable, Subject } from "rxjs";
 import { entityRegistry } from "./database-entity.decorator";
 import { HttpErrorResponse } from "@angular/common/http";
+import { TEST_USER } from "../../utils/mocked-testing.module";
 
 export function mockEntityMapper(
   withData: Entity[] = []
@@ -22,7 +23,12 @@ export class MockEntityMapperService extends EntityMapperService {
   private observables: Map<string, Subject<UpdatedEntity<any>>> = new Map();
 
   constructor() {
-    super(null, null, null, entityRegistry);
+    super(
+      null,
+      null,
+      { getCurrentUser: () => ({ name: TEST_USER }) } as any,
+      entityRegistry
+    );
   }
 
   private publishUpdates(type: string, update: UpdatedEntity<any>) {
@@ -41,6 +47,7 @@ export class MockEntityMapperService extends EntityMapperService {
     if (!this.data.get(type)) {
       this.data.set(type, new Map());
     }
+    super.setEntityMetadata(entity);
     const alreadyExists = this.contains(entity);
     this.data.get(type).set(entity.getId(), entity);
     this.publishUpdates(
@@ -74,7 +81,7 @@ export class MockEntityMapperService extends EntityMapperService {
    * @param id
    */
   public get(entityType: string, id: string): Entity {
-    const entityId = id.includes(":") ? id.split(":")[1] : id;
+    const entityId = Entity.extractEntityIdFromId(id);
     const result = this.data.get(entityType)?.get(entityId);
     if (!result) {
       throw new HttpErrorResponse({ status: 404 });

src/app/core/entity/model/entity.ts

@@ -62,6 +62,11 @@ export class Entity {
    */
   static schema: EntitySchema;
 
+  /**
+   * True if this type's schema has been customized dynamically from the config.
+   */
+  static _isCustomizedType?: boolean;
+
   /**
    * Defining which attribute values of an entity should be shown in the `.toString()` method.
    *

src/app/core/entity/schema/entity-schema.service.ts

@@ -110,9 +110,7 @@ export class EntitySchemaService {
         continue;
       }
 
-      const newValue = this.getDatatypeOrDefault(
-        schemaField.dataType
-      ).transformToObjectFormat(data[key], schemaField, this, data);
+      const newValue = this.valueToEntityFormat(data[key], schemaField, data);
       if (newValue !== undefined) {
         transformed[key] = newValue;
       }
@@ -133,7 +131,7 @@ export class EntitySchemaService {
   public loadDataIntoEntity(entity: Entity, data: any) {
     const transformed = this.transformDatabaseToEntityFormat(
       data,
-      (<typeof Entity>entity.constructor).schema
+      (<typeof Entity>entity.constructor).schema,
     );
     Object.assign(entity, transformed);
   }
@@ -145,7 +143,7 @@ export class EntitySchemaService {
    */
   public transformEntityToDatabaseFormat(
     entity: Entity,
-    schema?: EntitySchema
+    schema?: EntitySchema,
   ): any {
     if (!schema) {
       schema = entity.getSchema();
@@ -163,9 +161,7 @@ export class EntitySchemaService {
       }
 
       try {
-        data[key] = this.getDatatypeOrDefault(
-          schemaField.dataType
-        ).transformToDatabaseFormat(value, schemaField, this, entity);
+        data[key] = this.valueToDatabaseFormat(value, schemaField, entity);
       } catch (err) {
         throw new Error(`Transformation for ${key} failed: ${err}`);
       }
@@ -189,7 +185,7 @@ export class EntitySchemaService {
    */
   getComponent(
     propertySchema: EntitySchemaField,
-    mode: "view" | "edit" = "view"
+    mode: "view" | "edit" = "view",
   ): string {
     if (!propertySchema) {
       return undefined;
@@ -206,10 +202,42 @@ export class EntitySchemaService {
     }
 
     const innerDataType = this.getDatatypeOrDefault(
-      propertySchema.innerDataType
+      propertySchema.innerDataType,
     );
     if (innerDataType?.[componentAttribute]) {
       return innerDataType[componentAttribute];
     }
   }
+
+  /**
+   * Transform a single value into database format
+   * @param value
+   * @param schemaField
+   * @param entity
+   */
+  valueToDatabaseFormat(
+    value: any,
+    schemaField: EntitySchemaField,
+    entity?: Entity,
+  ) {
+    return this.getDatatypeOrDefault(
+      schemaField.dataType,
+    ).transformToDatabaseFormat(value, schemaField, this, entity);
+  }
+
+  /**
+   * Transform a single value into entity format
+   * @param value
+   * @param schemaField
+   * @param dataObject
+   */
+  valueToEntityFormat(
+    value: any,
+    schemaField: EntitySchemaField,
+    dataObject?: any,
+  ) {
+    return this.getDatatypeOrDefault(
+      schemaField.dataType,
+    ).transformToObjectFormat(value, schemaField, this, dataObject);
+  }
 }

src/app/features/data-import/input-file/input-file.component.html → src/app/core/input-file/input-file.component.html

@@ -5,9 +5,10 @@
   (change)="loadFile($event)"
 />
 <mat-form-field>
-  <mat-label i18n="Label for file select input">Selected File:</mat-label>
+  <mat-label i18n="Label for file select input">Select file</mat-label>
   <input
-    matInput readonly
+    matInput
+    readonly
     [formControl]="formControl"
     i18n-placeholder="placeholder for file-input"
     placeholder="No file selected"
@@ -16,6 +17,7 @@
   <button mat-icon-button matIconSuffix (click)="fileInput.click()">
     <fa-icon icon="upload"></fa-icon>
   </button>
-  <mat-error
-    *ngIf="formControl.invalid">{{ formControl.errors.fileInvalid || formControl.errors.parsingError }}</mat-error>
+  <mat-error *ngIf="formControl.invalid">{{
+    formControl.errors.fileInvalid || formControl.errors.parsingError
+  }}</mat-error>
 </mat-form-field>

src/app/features/data-import/input-file/input-file.component.spec.ts → src/app/core/input-file/input-file.component.spec.ts

@@ -2,7 +2,7 @@ import { ComponentFixture, TestBed } from "@angular/core/testing";
 
 import { InputFileComponent } from "./input-file.component";
 import { Papa } from "ngx-papaparse";
-import { MockedTestingModule } from "../../../utils/mocked-testing.module";
+import { MockedTestingModule } from "../../utils/mocked-testing.module";
 
 function mockFileEvent(mockFile: { name: string }): Event {
   return { target: { files: [mockFile] } } as unknown as Event;

src/app/features/data-import/input-file/input-file.component.ts → src/app/core/input-file/input-file.component.ts

@@ -1,5 +1,5 @@
 import { Component, EventEmitter, Input, Output } from "@angular/core";
-import { readFile } from "../../../utils/utils";
+import { readFile } from "../../utils/utils";
 import { Papa } from "ngx-papaparse";
 import { FormControl, ReactiveFormsModule } from "@angular/forms";
 import { MatFormFieldModule } from "@angular/material/form-field";
@@ -22,9 +22,9 @@ import { NgIf } from "@angular/common";
     ReactiveFormsModule,
     MatButtonModule,
     FontAwesomeModule,
-    NgIf
+    NgIf,
   ],
-  standalone: true
+  standalone: true,
 })
 export class InputFileComponent<T = any> {
   /** returns parsed data as an object on completing load after user selects a file */