Skip to content

Commit

Permalink
fix: import UI refactoring (#1418)
Browse files Browse the repository at this point in the history 
Co-authored-by: Simon <simon@aam-digital.com>
  • Loading branch information
@sleidig @TheSlimvReal
sleidig and TheSlimvReal authored Oct 27, 2022
1 parent df76bca commit 087790c
Show file tree
Hide file tree
Showing 9 changed files with 383 additions and 341 deletions.
5 changes: 4 additions & 1 deletion src/app/features/data-import/data-import.module.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,11 @@ import { FontAwesomeModule } from "@fortawesome/angular-fontawesome";
import { MatAutocompleteModule } from "@angular/material/autocomplete";
import { ExportModule } from "../../core/export/export.module";
import { FlexModule } from "@angular/flex-layout";
import { InputFileComponent } from "./input-file/input-file.component";
import { MatExpansionModule } from "@angular/material/expansion";

@NgModule({
declarations: [DataImportComponent],
declarations: [DataImportComponent, InputFileComponent],
imports: [
CommonModule,
FormsModule,
Expand All @@ -30,6 +32,7 @@ import { FlexModule } from "@angular/flex-layout";
MatAutocompleteModule,
ExportModule,
FlexModule,
MatExpansionModule,
],
exports: [DataImportComponent],
providers: [DataImportService],
Expand Down
142 changes: 55 additions & 87 deletions src/app/features/data-import/data-import.service.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { ConfirmationDialogService } from "../../core/confirmation-dialog/confir
import { MatSnackBar, MatSnackBarRef } from "@angular/material/snack-bar";
import { NEVER, of } from "rxjs";
import { EntityMapperService } from "../../core/entity/entity-mapper.service";
import { Papa, ParseResult } from "ngx-papaparse";
import { ParseResult } from "ngx-papaparse";
import { ImportMetaData } from "./import-meta-data.type";
import { expectEntitiesToBeInDatabase } from "../../utils/expect-entity-data.spec";
import { Child } from "../../child-dev-project/children/model/child";
Expand All @@ -19,101 +19,82 @@ describe("DataImportService", () => {
let service: DataImportService;
let mockConfirmationService: jasmine.SpyObj<ConfirmationDialogService>;

beforeEach(
waitForAsync(() => {
mockConfirmationService = jasmine.createSpyObj(["getConfirmation"]);
mockConfirmationService.getConfirmation.and.resolveTo(true);
TestBed.configureTestingModule({
imports: [DataImportModule, DatabaseTestingModule, ChildrenModule],
providers: [
{
provide: ConfirmationDialogService,
useValue: mockConfirmationService,
},
],
});
service = TestBed.inject(DataImportService);
db = TestBed.inject(Database);
})
);
let mockParseResult: ParseResult;
const date1 = moment().subtract("10", "years");
const date2 = moment().subtract("12", "years");

beforeEach(waitForAsync(() => {
mockParseResult = {
meta: { fields: ["ID", "Name", "Birthday", "Age"] },
data: [
{
ID: 1,
Name: "First",
Birthday: date1.format("YYYY-MM-DD"),
notExistingProperty: "some value",
},
{
ID: 2,
Name: "Second",
Birthday: date2.format("YYYY-MM-DD"),
notExistingProperty: "another value",
},
],
} as ParseResult;

mockConfirmationService = jasmine.createSpyObj(["getConfirmation"]);
mockConfirmationService.getConfirmation.and.resolveTo(true);
TestBed.configureTestingModule({
imports: [DataImportModule, DatabaseTestingModule, ChildrenModule],
providers: [
{
provide: ConfirmationDialogService,
useValue: mockConfirmationService,
},
],
});
service = TestBed.inject(DataImportService);
db = TestBed.inject(Database);
}));

afterEach(() => db.destroy());

it("should be created", () => {
expect(service).toBeTruthy();
});

it("should only allow files that have a .csv extension", async () => {
mockFileReader();

const file = { name: "wrong_extension.xlsx" };
await expectAsync(service.validateCsvFile(file as File)).toBeRejected();

file.name = "good_extension.csv";
await expectAsync(service.validateCsvFile(file as File)).toBeResolved();
});

it("should throw error if file cannot be parsed", async () => {
mockFileReader();
const papa = TestBed.inject(Papa);
spyOn(papa, "parse").and.returnValue(undefined);
const file = { name: "file.csv" } as File;

await expectAsync(service.validateCsvFile(file)).toBeRejected();
});

it("should throw error if file is empty", async () => {
mockFileReader("");
const file = { name: "file.csv" } as File;

await expectAsync(service.validateCsvFile(file)).toBeRejected();
});

it("should restore database if snackbar is clicked", async () => {
const doc1 = { _id: "Doc:1" };
const doc2 = { _id: "Doc:2" };
await db.put(doc1);
await db.put(doc2);
mockFileReader();
mockSnackbar(true);
const parsedData = {
meta: { fields: ["ID", "Name", "Birthday", "Age"] },
data: [
{
_id: "Child:1",
name: "First",
},
],
} as ParseResult;
const importMeta: ImportMetaData = {
entityType: "Child",
columnMap: {
_id: "_id",
projectNumber: "projectNumber",
name: "name",
},
};
const file = { name: "some.csv" } as File;
const parseResult = await service.validateCsvFile(file);

await service.handleCsvImport(parseResult, importMeta);
await service.handleCsvImport(parsedData.data, importMeta);

await expectAsync(db.get(doc1._id)).toBeResolved();
await expectAsync(db.get(doc2._id)).toBeResolved();
await expectAsync(db.get("Child:1")).toBeRejected();
});

it("should use the passed component map to create the entity", async () => {
const birthday1 = moment().subtract("10", "years");
const birthday2 = moment().subtract("12", "years");
const csvData = {
meta: { fields: ["ID", "Name", "Birthday", "Age"] },
data: [
{
ID: 1,
Name: "First",
Birthday: birthday1.format("YYYY-MM-DD"),
notExistingProperty: "some value",
},
{
ID: 2,
Name: "Second",
Birthday: birthday2.format("YYYY-MM-DD"),
notExistingProperty: "another value",
},
],
} as ParseResult;
const csvData = mockParseResult;
const columnMap = {
ID: "_id",
Name: "name",
Expand All @@ -125,19 +106,19 @@ describe("DataImportService", () => {
columnMap: columnMap,
};

await service.handleCsvImport(csvData, importMeta);
await service.handleCsvImport(csvData.data, importMeta);

const entityMapper = TestBed.inject(EntityMapperService);
const firstChild = await entityMapper.load(Child, "1");
expect(firstChild._id).toBe("Child:1");
expect(firstChild.name).toBe("First");
expect(birthday1.isSame(firstChild.dateOfBirth, "day")).toBeTrue();
expect(date1.isSame(firstChild.dateOfBirth, "day")).toBeTrue();
expect(firstChild.age).toBe(10);
expect(firstChild).not.toHaveOwnProperty("notExistingProperty");
const secondChild = await entityMapper.load(Child, "2");
expect(secondChild._id).toBe("Child:2");
expect(secondChild.name).toBe("Second");
expect(birthday2.isSame(secondChild.dateOfBirth, "day")).toBeTrue();
expect(date2.isSame(secondChild.dateOfBirth, "day")).toBeTrue();
expect(secondChild.age).toBe(12);
expect(secondChild).not.toHaveOwnProperty("notExistingProperty");
});
Expand All @@ -156,7 +137,7 @@ describe("DataImportService", () => {
await db.put({ _id: `Child:${transactionID}-123` });
await db.put({ _id: `Child:${transactionID}-124` });

await service.handleCsvImport(csvData, importMeta);
await service.handleCsvImport(csvData.data, importMeta);

await expectEntitiesToBeInDatabase(
[Child.create("test1"), Child.create("test2")],
Expand Down Expand Up @@ -185,7 +166,7 @@ describe("DataImportService", () => {
dateFormat: "D/M/YYYY",
};

await service.handleCsvImport(csvData, importMeta);
await service.handleCsvImport(csvData.data, importMeta);

const entityMapper = TestBed.inject(EntityMapperService);
const test1 = await entityMapper.load(Child, "test1");
Expand All @@ -205,7 +186,7 @@ describe("DataImportService", () => {
columnMap: { name: "name", projectNumber: "projectNumber" },
};

await service.handleCsvImport(csvData, importMeta);
await service.handleCsvImport(csvData.data, importMeta);

expect(db.put).toHaveBeenCalledWith(
jasmine.objectContaining({
Expand All @@ -215,19 +196,6 @@ describe("DataImportService", () => {
);
});

function mockFileReader(
result = '_id,name,projectNumber\nChild:1,"John Doe",123'
) {
const fileReader: any = {
result: result,
addEventListener: (_str: string, fun: () => any) => fun(),
readAsText: () => {},
};
// mock FileReader constructor
spyOn(window, "FileReader").and.returnValue(fileReader);
return fileReader;
}

function mockSnackbar(clicked: boolean): jasmine.SpyObj<MatSnackBarRef<any>> {
const mockSnackBarRef = jasmine.createSpyObj<MatSnackBarRef<any>>(
"mockSnackBarRef",
Expand Down
49 changes: 9 additions & 40 deletions src/app/features/data-import/data-import.service.ts
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
import { Injectable } from "@angular/core";
import { Database } from "../../core/database/database";
import { Papa, ParseResult } from "ngx-papaparse";
import { BackupService } from "../../core/admin/services/backup.service";
import { ConfirmationDialogService } from "../../core/confirmation-dialog/confirmation-dialog.service";
import { MatSnackBar } from "@angular/material/snack-bar";
import { readFile } from "../../utils/utils";
import { ImportMetaData } from "./import-meta-data.type";
import { v4 as uuid } from "uuid";
import { Entity } from "../../core/entity/model/entity";
Expand All @@ -24,56 +22,27 @@ export class DataImportService {
dateOnlyEntitySchemaDatatype,
monthEntitySchemaDatatype,
].map((dataType) => dataType.name);

constructor(
private db: Database,
private papa: Papa,
private backupService: BackupService,
private confirmationDialog: ConfirmationDialogService,
private snackBar: MatSnackBar,
private entities: EntityRegistry
) {}

/**
* Validates and reads a CSV
* @param file a File Blob
*/
async validateCsvFile(file: File): Promise<ParseResult> {
if (!file.name.toLowerCase().endsWith(".csv")) {
throw new Error("Only .csv files are supported");
}
const csvData = await readFile(file);
const parsedCsvFile = this.parseCsvFile(csvData);

if (parsedCsvFile === undefined || parsedCsvFile.data === undefined) {
throw new Error("File could not be parsed");
}
if (parsedCsvFile.data.length === 0) {
throw new Error("File has no content");
}

return parsedCsvFile;
}

private parseCsvFile(csvString: string): ParseResult {
return this.papa.parse(csvString, {
header: true,
dynamicTyping: true,
skipEmptyLines: true,
});
}

/**
* Add the data from the loaded file to the database, inserting and updating records.
* If a transactionId is provided in the ImportMetaData, all records starting with this ID will be deleted from the database before importing
* @param csvFile The file object of the csv data to be loaded
* @param data The objects parsed from a file to be loaded
* @param importMeta Additional information required for importing the file
*/
async handleCsvImport(
csvFile: ParseResult,
data: any[],
importMeta: ImportMetaData
): Promise<void> {
const restorePoint = await this.backupService.getJsonExport();
const confirmed = await this.getUserConfirmation(csvFile, importMeta);
const confirmed = await this.getUserConfirmation(data, importMeta);
if (!confirmed) {
return;
}
Expand All @@ -82,7 +51,7 @@ export class DataImportService {
await this.deleteExistingRecords(importMeta);
}

await this.importCsvContentToDB(csvFile, importMeta);
await this.importCsvContentToDB(data, importMeta);

const snackBarRef = this.snackBar.open(
$localize`Import completed`,
Expand All @@ -98,12 +67,12 @@ export class DataImportService {
}

private getUserConfirmation(
csvFile: ParseResult,
data: any[],
importMeta: ImportMetaData
): Promise<boolean> {
const refTitle = $localize`Import new data?`;
let refText = $localize`Are you sure you want to import this file?
This will add or update ${csvFile.data.length} records from the loaded file.`;
This will add or update ${data.length} records from the loaded file.`;
if (importMeta.transactionId) {
refText = $localize`${refText} All existing records imported with the transaction id '${importMeta.transactionId}' will be deleted!`;
}
Expand All @@ -118,10 +87,10 @@ export class DataImportService {
}

private async importCsvContentToDB(
csv: ParseResult,
data: any[],
importMeta: ImportMetaData
): Promise<void> {
for (const row of csv.data) {
for (const row of data) {
const entity = this.createEntityWithRowData(row, importMeta);
this.createSearchIndices(importMeta, entity);
if (!entity["_id"]) {
Expand Down
Loading

0 comments on commit 087790c

Please sign in to comment.