feat: allow groupBy in exports (#1636)

closes #1150 

Co-authored-by: Sebastian Leidig <sebastian.leidig@gmail.com>

src/app/child-dev-project/children/children-list/children-list.component.spec.ts

@@ -16,7 +16,7 @@ import {
 } from "../../../core/entity-components/entity-list/EntityListConfig";
 import { School } from "../../schools/model/school";
 import { MockedTestingModule } from "../../../utils/mocked-testing.module";
-import { ExportService } from "../../../core/export/export-service/export.service";
+import { DownloadService } from "../../../core/export/download-service/download.service";
 
 describe("ChildrenListComponent", () => {
   let component: ChildrenListComponent;
@@ -88,7 +88,7 @@ describe("ChildrenListComponent", () => {
           useValue: mockChildrenService,
         },
         { provide: ActivatedRoute, useValue: routeMock },
-        { provide: ExportService, useValue: {} },
+        { provide: DownloadService, useValue: {} },
       ],
     }).compileComponents();
   }));

src/app/child-dev-project/notes/note-details/note-details.component.ts

@@ -6,7 +6,7 @@ import { INTERACTION_TYPE_CONFIG_ID } from "../model/interaction-type.interface"
 import { Child } from "../../children/model/child";
 import { User } from "../../../core/user/user";
 import { School } from "../../schools/model/school";
-import { ExportColumnConfig } from "../../../core/export/export-service/export-column-config";
+import { ExportColumnConfig } from "../../../core/export/data-transformation-service/export-column-config";
 import { ConfigService } from "../../../core/config/config.service";
 import { EntityListConfig } from "../../../core/entity-components/entity-list/EntityListConfig";
 import { compareEnums } from "../../../utils/utils";

src/app/core/admin/admin/admin.component.spec.ts

@@ -13,27 +13,22 @@ import { ConfirmationDialogService } from "../../confirmation-dialog/confirmatio
 import { SessionType } from "../../session/session-type";
 import { MockedTestingModule } from "../../../utils/mocked-testing.module";
 import { environment } from "../../../../environments/environment";
+import { DownloadService } from "../../export/download-service/download.service";
 
 describe("AdminComponent", () => {
   let component: AdminComponent;
   let fixture: ComponentFixture<AdminComponent>;
 
   const mockBackupService = jasmine.createSpyObj<BackupService>([
-    "getJsonExport",
-    "getCsvExport",
+    "getDatabaseExport",
     "clearDatabase",
-    "importJson",
+    "restoreData",
   ]);
+  let mockDownloadService: jasmine.SpyObj<DownloadService>;
 
   const confirmationDialogMock =
     jasmine.createSpyObj<ConfirmationDialogService>(["getConfirmation"]);
 
-  const tmplink: jasmine.SpyObj<HTMLAnchorElement> = jasmine.createSpyObj(
-    "mockLink",
-    ["click"],
-    ["href", "target", "download"]
-  );
-
   function createFileReaderMock(result: string = "") {
     const mockFileReader: any = {
       result: result,
@@ -48,6 +43,7 @@ describe("AdminComponent", () => {
 
   beforeEach(waitForAsync(() => {
     environment.session_type = SessionType.mock;
+    mockDownloadService = jasmine.createSpyObj(["triggerDownload"]);
 
     TestBed.configureTestingModule({
       imports: [AdminComponent, MockedTestingModule.withState()],
@@ -57,6 +53,7 @@ describe("AdminComponent", () => {
           provide: ConfirmationDialogService,
           useValue: confirmationDialogMock,
         },
+        { provide: DownloadService, useValue: mockDownloadService },
       ],
     }).compileComponents();
   }));
@@ -72,25 +69,22 @@ describe("AdminComponent", () => {
   });
 
   it("should call backup service for json export", fakeAsync(() => {
-    spyOn(document, "createElement").and.callFake(() => tmplink);
-    mockBackupService.getJsonExport.and.resolveTo("");
+    mockBackupService.getDatabaseExport.and.resolveTo([]);
     component.saveBackup();
-    expect(mockBackupService.getJsonExport).toHaveBeenCalled();
+    expect(mockBackupService.getDatabaseExport).toHaveBeenCalled();
     tick();
-    expect(tmplink.click).toHaveBeenCalled();
+    expect(mockDownloadService.triggerDownload).toHaveBeenCalled();
   }));
 
   it("should call backup service for csv export", fakeAsync(() => {
-    spyOn(document, "createElement").and.returnValue(tmplink);
-    mockBackupService.getCsvExport.and.resolveTo("");
+    mockBackupService.getDatabaseExport.and.resolveTo([]);
     component.saveCsvExport();
-    expect(mockBackupService.getCsvExport).toHaveBeenCalled();
+    expect(mockBackupService.getDatabaseExport).toHaveBeenCalled();
     tick();
-    expect(tmplink.click).toHaveBeenCalled();
+    expect(mockDownloadService.triggerDownload).toHaveBeenCalled();
   }));
 
   it("should call config service for configuration export", fakeAsync(() => {
-    spyOn(document, "createElement").and.returnValue(tmplink);
     const exportConfigSpy = spyOn(
       TestBed.inject(ConfigService),
       "exportConfig"
@@ -99,7 +93,7 @@ describe("AdminComponent", () => {
     component.downloadConfigClick();
     expect(exportConfigSpy).toHaveBeenCalled();
     tick();
-    expect(tmplink.click).toHaveBeenCalled();
+    expect(mockDownloadService.triggerDownload).toHaveBeenCalled();
   }));
 
   it("should save and apply new configuration", fakeAsync(() => {
@@ -114,25 +108,25 @@ describe("AdminComponent", () => {
 
   it("should open dialog and call backup service when loading backup", fakeAsync(() => {
     const mockFileReader = createFileReaderMock("[]");
-    mockBackupService.getJsonExport.and.resolveTo("[]");
+    mockBackupService.getDatabaseExport.and.resolveTo([]);
     confirmationDialogMock.getConfirmation.and.resolveTo(true);
 
     component.loadBackup({ target: { files: [] } } as any);
-    expect(mockBackupService.getJsonExport).toHaveBeenCalled();
+    expect(mockBackupService.getDatabaseExport).toHaveBeenCalled();
     tick();
     expect(mockFileReader.readAsText).toHaveBeenCalled();
     expect(confirmationDialogMock.getConfirmation).toHaveBeenCalled();
     flush();
     expect(mockBackupService.clearDatabase).toHaveBeenCalled();
-    expect(mockBackupService.importJson).toHaveBeenCalled();
+    expect(mockBackupService.restoreData).toHaveBeenCalled();
   }));
 
   it("should open dialog when clearing database", fakeAsync(() => {
-    mockBackupService.getJsonExport.and.resolveTo("");
+    mockBackupService.getDatabaseExport.and.resolveTo([]);
     confirmationDialogMock.getConfirmation.and.resolveTo(true);
 
     component.clearDatabase();
-    expect(mockBackupService.getJsonExport).toHaveBeenCalled();
+    expect(mockBackupService.getDatabaseExport).toHaveBeenCalled();
     tick();
     expect(confirmationDialogMock.getConfirmation).toHaveBeenCalled();
     flush();

src/app/core/admin/admin/admin.component.ts

@@ -13,6 +13,7 @@ import { ExtendedAlertConfig } from "../../alerts/alert-config";
 import { MatButtonModule } from "@angular/material/button";
 import { RouterLink } from "@angular/router";
 import { DatePipe, NgForOf } from "@angular/common";
+import { DownloadService } from "../../export/download-service/download.service";
 
 /**
  * Admin GUI giving administrative users different options/actions.
@@ -33,6 +34,7 @@ export class AdminComponent implements OnInit {
   constructor(
     private alertService: AlertService,
     private backupService: BackupService,
+    private downloadService: DownloadService,
     private db: Database,
     private confirmationDialog: ConfirmationDialogService,
     private snackBar: MatSnackBar,
@@ -62,58 +64,55 @@ export class AdminComponent implements OnInit {
    * Download a full backup of the database as (json) file.
    */
   async saveBackup() {
-    const backup = await this.backupService.getJsonExport();
-    this.startDownload(backup, "text/json", "backup.json");
+    const backup = await this.backupService.getDatabaseExport();
+    await this.downloadService.triggerDownload(backup, "json", "backup");
   }
 
   /**
    * Download a full export of the database as csv file.
    */
   async saveCsvExport() {
-    const csv = await this.backupService.getCsvExport();
-    this.startDownload(csv, "text/csv", "export.csv");
+    const backup = await this.backupService.getDatabaseExport();
+    await this.downloadService.triggerDownload(backup, "csv", "export");
   }
 
-  downloadConfigClick() {
+  async downloadConfigClick() {
     const configString = this.configService.exportConfig();
-    this.startDownload(configString, "text/json", "config.json");
+    await this.downloadService.triggerDownload(
+      configString,
+      "json",
+      "config.json"
+    );
   }
 
   async uploadConfigFile(inputEvent: Event) {
     const loadedFile = await readFile(this.getFileFromInputEvent(inputEvent));
     await this.configService.saveConfig(JSON.parse(loadedFile));
   }
 
-  private startDownload(data: string, type: string, name: string) {
-    const tempLink = document.createElement("a");
-    tempLink.href =
-      "data:" + type + ";charset=utf-8," + encodeURIComponent(data);
-    tempLink.target = "_blank";
-    tempLink.download = name;
-    tempLink.click();
-  }
-
   /**
    * Reset the database to the state from the loaded backup file.
    * @param inputEvent for the input where a file has been selected
    */
   async loadBackup(inputEvent: Event) {
-    const restorePoint = await this.backupService.getJsonExport();
-    const newData = await readFile(this.getFileFromInputEvent(inputEvent));
+    const restorePoint = await this.backupService.getDatabaseExport();
+    const dataToBeRestored = JSON.parse(
+      await readFile(this.getFileFromInputEvent(inputEvent))
+    );
 
     const confirmed = await this.confirmationDialog.getConfirmation(
       `Overwrite complete database?`,
       `Are you sure you want to restore this backup? This will
-      delete all ${JSON.parse(restorePoint).length} existing records,
-      restoring ${JSON.parse(newData).length} records from the loaded file.`
+      delete all ${restorePoint.length} existing records,
+      restoring ${dataToBeRestored.length} records from the loaded file.`
     );
 
     if (!confirmed) {
       return;
     }
 
     await this.backupService.clearDatabase();
-    await this.backupService.importJson(newData, true);
+    await this.backupService.restoreData(dataToBeRestored, true);
 
     const snackBarRef = this.snackBar.open(`Backup restored`, "Undo", {
       duration: 8000,
@@ -123,7 +122,7 @@ export class AdminComponent implements OnInit {
       .pipe(untilDestroyed(this))
       .subscribe(async () => {
         await this.backupService.clearDatabase();
-        await this.backupService.importJson(restorePoint, true);
+        await this.backupService.restoreData(restorePoint, true);
       });
   }
 
@@ -136,7 +135,7 @@ export class AdminComponent implements OnInit {
    * Reset the database removing all entities except user accounts.
    */
   async clearDatabase() {
-    const restorePoint = await this.backupService.getJsonExport();
+    const restorePoint = await this.backupService.getDatabaseExport();
 
     const confirmed = await this.confirmationDialog.getConfirmation(
       `Empty complete database?`,
@@ -156,7 +155,7 @@ export class AdminComponent implements OnInit {
       .onAction()
       .pipe(untilDestroyed(this))
       .subscribe(async () => {
-        await this.backupService.importJson(restorePoint, true);
+        await this.backupService.restoreData(restorePoint, true);
       });
   }
 }

src/app/core/admin/services/backup.service.spec.ts

@@ -3,8 +3,8 @@ import { TestBed } from "@angular/core/testing";
 import { BackupService } from "./backup.service";
 import { Database } from "../../database/database";
 import { PouchDatabase } from "../../database/pouch-database";
-import { ExportService } from "../../export/export-service/export.service";
-import { QueryService } from "../../../features/reporting/query.service";
+import { DownloadService } from "../../export/download-service/download.service";
+import { DataTransformationService } from "../../export/data-transformation-service/data-transformation.service";
 
 describe("BackupService", () => {
   let db: PouchDatabase;
@@ -15,8 +15,8 @@ describe("BackupService", () => {
     TestBed.configureTestingModule({
       providers: [
         BackupService,
-        ExportService,
-        { provide: QueryService, useValue: { queryData: () => [] } },
+        DownloadService,
+        { provide: DataTransformationService, useValue: {} },
         { provide: Database, useValue: db },
       ],
     });
@@ -44,30 +44,31 @@ describe("BackupService", () => {
     expect(resAfter).toBeEmpty();
   });
 
-  it("getJsonExport should return all records", async () => {
+  it("getDatabaseExport should return all records", async () => {
     await db.put({ _id: "Test:1", test: 1 });
     await db.put({ _id: "Test:2", test: 2 });
 
     const res = await db.getAll();
     expect(res).toHaveSize(2);
 
-    const jsonExport = await service.getJsonExport();
+    const result = await service.getDatabaseExport();
 
-    expect(jsonExport).toBe(
-      `[{"test":1,"_id":"Test:1","_rev":"${res[0]._rev}"},{"test":2,"_id":"Test:2","_rev":"${res[1]._rev}"}]`
-    );
+    expect(result).toEqual([
+      { test: 1, _id: "Test:1", _rev: res[0]._rev },
+      { test: 2, _id: "Test:2", _rev: res[1]._rev },
+    ]);
   });
 
-  it("getJsonExport | clearDatabase | importJson should restore all records", async () => {
+  it("getDatabaseExport | clearDatabase | restoreData should restore all records", async () => {
     await db.put({ _id: "Test:1", test: 1 });
     await db.put({ _id: "Test:2", test: 2 });
 
     const originalData = await db.getAll();
     expect(originalData).toHaveSize(2);
 
-    const backup = await service.getJsonExport();
+    const backup = await service.getDatabaseExport();
     await service.clearDatabase();
-    await service.importJson(backup, true);
+    await service.restoreData(backup, true);
 
     const res = await db.getAll();
     expect(res).toHaveSize(2);
@@ -78,35 +79,26 @@ describe("BackupService", () => {
       .toEqual(originalData.map(ignoreRevProperty));
   });
 
-  it("getCsvExport should contain a line for every record", async () => {
-    await db.put({ _id: "Test:1", test: 1 });
-    await db.put({ _id: "Test:2", test: 2 });
+  it("getDatabaseExport should contain an entry for every record", async () => {
+    const x1 = { _id: "Test:1", test: 1 };
+    const x2 = { _id: "Test:2", test: 2 };
+    await db.put(x1);
+    await db.put(x2);
 
     const res = await db.getAll();
     expect(res).toHaveSize(2);
 
-    const csvExport = await service.getCsvExport();
+    const result = await service.getDatabaseExport();
 
-    expect(csvExport.split(ExportService.SEPARATOR_ROW)).toHaveSize(2 + 1); // includes 1 header line
+    expect(result.map(ignoreRevProperty)).toEqual([x1, x2]);
   });
 
   it("importCsv should add records", async () => {
-    const csv =
-      "_id" +
-      ExportService.SEPARATOR_COL +
-      "test" +
-      ExportService.SEPARATOR_ROW +
-      '"Test:1"' +
-      ExportService.SEPARATOR_COL +
-      "1" +
-      ExportService.SEPARATOR_ROW +
-      '"Test:2"' +
-      ExportService.SEPARATOR_COL +
-      "2" +
-      ExportService.SEPARATOR_ROW;
-
-    await service.importCsv(csv, true);
-
+    const data = [
+      { _id: "Test:1", test: 1 },
+      { _id: "Test:2", test: 2 },
+    ];
+    await service.restoreData(data, true);
     const res = await db.getAll();
     expect(res).toHaveSize(2);
     expect(res.map(ignoreRevProperty)).toEqual([
@@ -115,20 +107,9 @@ describe("BackupService", () => {
     ]);
   });
 
-  it("importCsv should not add empty properties to records", async () => {
-    const csv =
-      "_id" +
-      ExportService.SEPARATOR_COL +
-      "other" +
-      ExportService.SEPARATOR_COL +
-      "test" +
-      ExportService.SEPARATOR_ROW +
-      '"Test:1"' +
-      ExportService.SEPARATOR_COL +
-      ExportService.SEPARATOR_COL +
-      "1";
-
-    await service.importCsv(csv);
+  it("restoreData should not add empty properties to records", async () => {
+    const data = [{ _id: "Test:1", test: 1 }];
+    await service.restoreData(data);
 
     const res = await db.getAll();
     expect(res).toHaveSize(1);