Search improvement (#1739)

closes #1609

src/app/child-dev-project/attendance/model/recurring-activity.spec.ts

@@ -29,7 +29,5 @@ describe("RecurringActivity", () => {
     participants: ["1", "2"],
     linkedGroups: ["3"],
     excludedParticipants: ["5"],
-
-    searchIndices: ["test", "activity"],
   });
 });

src/app/child-dev-project/children/model/child.spec.ts

@@ -37,8 +37,6 @@ describe("Child", () => {
     dropoutDate: new Date("2022-03-31"),
     dropoutType: "unknown",
     dropoutRemarks: "no idea what happened",
-
-    searchIndices: ["Max", "projectNumber01"],
   });
 
   it("should determine isActive based on inferred state", () => {

src/app/child-dev-project/children/model/child.ts

@@ -57,6 +57,7 @@ export class Child extends Entity {
   @DatabaseField({
     label: $localize`:Label for the project number of a child:Project Number`,
     labelShort: $localize`:Short label for the project number:PN`,
+    searchable: true,
   })
   projectNumber: string;
 
@@ -135,19 +136,4 @@ export class Child extends Entity {
       super.isActive
     );
   }
-
-  /**
-   * @override see {@link Entity}
-   */
-  @DatabaseField() get searchIndices(): string[] {
-    let indices = [];
-
-    indices = indices.concat(this.toString().split(" "));
-    if (this.projectNumber !== undefined && this.projectNumber !== null) {
-      indices.push(this.projectNumber);
-    }
-    return indices;
-  }
-
-  set searchIndices(value) {}
 }

src/app/child-dev-project/schools/model/school.spec.ts

@@ -22,6 +22,5 @@ describe("School Entity", () => {
   testEntitySubclass("School", School, {
     _id: "School:some-id",
     name: "Max",
-    searchIndices: ["Max"],
   });
 });

src/app/core/entity/model/entity.spec.ts

@@ -51,29 +51,6 @@ describe("Entity", () => {
     expect(data.otherText).toBeUndefined();
   });
 
-  it("rawData() includes searchIndices containing toString parts", function () {
-    const id = "test1";
-    const entity = new Entity(id);
-    entity.toString = () => entity["name"];
-    entity["name"] = "John Doe";
-
-    const data = entitySchemaService.transformEntityToDatabaseFormat(entity);
-
-    expect(data.searchIndices).toBeDefined();
-    expect(data.searchIndices).toContain("John");
-    expect(data.searchIndices).toContain("Doe");
-  });
-
-  it("should not generate searchIndices for entities without a custom toString method", function () {
-    const id = "test1";
-    const entity = new Entity(id);
-    entity["name"] = "John Doe";
-
-    const data = entitySchemaService.transformEntityToDatabaseFormat(entity);
-
-    expect(data.searchIndices).toEqual([]);
-  });
-
   it("can perform a shallow copy of itself", () => {
     const id = "t1";
     const entity: Entity = new Entity(id);
@@ -176,9 +153,6 @@ export function testEntitySubclass(
       JSON.parse(JSON.stringify(expectedDatabaseFormat))
     );
     const rawData = schemaService.transformEntityToDatabaseFormat(entity);
-    if (rawData.searchIndices.length === 0) {
-      delete rawData.searchIndices;
-    }
     expect(rawData).toEqual(expectedDatabaseFormat);
   }));
 }

src/app/core/entity/model/entity.ts

@@ -194,31 +194,6 @@ export class Entity {
     this._id = Entity.createPrefixedId(this.getType(), newEntityId);
   }
 
-  /**
-   * Returns an array of strings by which the entity can be searched.
-   *
-   * By default the parts of the string representation (toString) split at spaces is used if it is present.
-   *
-   * <b>Overwrite this method in subtypes if you want an entity type to be searchable by other properties.</b>
-   */
-  @DatabaseField() get searchIndices(): string[] {
-    if (
-      this.getConstructor().toStringAttributes === Entity.toStringAttributes &&
-      this.toString() === this.entityId
-    ) {
-      // no indices for the default if an entity does not have a human-readable name
-      return [];
-    }
-
-    // default indices generated from toString
-    return this.toString().split(" ");
-  }
-
-  set searchIndices(value) {
-    // do nothing, always generated on the fly
-    // searchIndices is only saved to database so it can be used internally for database indexing
-  }
-
   /**
    * Check, if this entity is considered active.
    * This is either taken from the property "inactive" (configured) or "active" (not configured).

src/app/core/entity/schema/entity-schema-field.ts

@@ -47,6 +47,11 @@ export interface EntitySchemaField {
    */
   generateIndex?: boolean; // TODO: implement index support in EntitySchema
 
+  /**
+   * If set to `true`, the entity can be found in the global search by entering this value
+   */
+  searchable?: boolean;
+
   /**
    * Whether the field should be initialized with a default value if undefined
    * (which is then run through dataType transformation);

src/app/core/ui/search/search.component.spec.ts

@@ -7,14 +7,10 @@ import {
 } from "@angular/core/testing";
 
 import { SearchComponent } from "./search.component";
-import { Child } from "../../../child-dev-project/children/model/child";
-import { School } from "../../../child-dev-project/schools/model/school";
 import { DatabaseIndexingService } from "../../entity/database-indexing/database-indexing.service";
 import { Subscription } from "rxjs";
-import { Entity } from "../../entity/model/entity";
 import { MockedTestingModule } from "../../../utils/mocked-testing.module";
 import { SwUpdate } from "@angular/service-worker";
-import { UserRoleGuard } from "../../permissions/permission-guard/user-role.guard";
 
 describe("SearchComponent", () => {
   SearchComponent.INPUT_DEBOUNCE_TIME_MS = 4;
@@ -36,7 +32,6 @@ describe("SearchComponent", () => {
     TestBed.configureTestingModule({
       imports: [SearchComponent, MockedTestingModule.withState()],
       providers: [
-        UserRoleGuard,
         { provide: DatabaseIndexingService, useValue: mockIndexService },
         { provide: SwUpdate, useValue: {} },
       ],
@@ -69,11 +64,6 @@ describe("SearchComponent", () => {
 
     component.formControl.setValue("AB");
     tick(SearchComponent.INPUT_DEBOUNCE_TIME_MS * 2);
-    expect(component.state).toBe(component.TOO_FEW_CHARACTERS);
-    expect(mockIndexService.queryIndexRaw).not.toHaveBeenCalled();
-
-    component.formControl.setValue("ABC");
-    tick(SearchComponent.INPUT_DEBOUNCE_TIME_MS * 2);
     expect(component.state).toBe(component.NO_RESULTS);
     expect(mockIndexService.queryIndexRaw).toHaveBeenCalled();
 
@@ -97,58 +87,4 @@ describe("SearchComponent", () => {
     expectResultToBeEmpty(done);
     component.formControl.setValue(null);
   });
-
-  function expectResultToHave(queryResults: any, result: Entity, done: DoneFn) {
-    mockIndexService.queryIndexRaw.and.returnValue(
-      Promise.resolve(queryResults)
-    );
-
-    subscription = component.results.subscribe((next) => {
-      expect(next).toHaveSize(1);
-      expect(next[0]).toHaveId(result.getId());
-      expect(mockIndexService.queryIndexRaw).toHaveBeenCalled();
-      done();
-    });
-  }
-
-  function generateDemoData(): [Child, School, object] {
-    const child1 = new Child("1");
-    child1.name = "Adam X";
-    const school1 = new School("s1");
-    school1.name = "Anglo Primary";
-    const mockQueryResults = {
-      rows: [
-        { id: child1.getId(true), doc: { name: child1.name }, key: "adam" },
-        { id: child1.getId(true), doc: { name: child1.name }, key: "x" },
-        { id: school1.getId(true), doc: { name: school1.name }, key: "anglo" },
-        {
-          id: school1.getId(true),
-          doc: { name: school1.name },
-          key: "primary",
-        },
-      ],
-    };
-    return [child1, school1, mockQueryResults];
-  }
-
-  it("should set results correctly for search input", (done) => {
-    const [child1, , mockQueryResults] = generateDemoData();
-
-    expectResultToHave(mockQueryResults, child1, done);
-    component.formControl.setValue("Ada");
-  });
-
-  it("should not include duplicates in results", (done) => {
-    const [child1, , mockQueryResults] = generateDemoData();
-
-    expectResultToHave(mockQueryResults, child1, done);
-    component.formControl.setValue("Ada");
-  });
-
-  it("should only include results matching all search terms (words)", (done) => {
-    const [child1, , mockQueryResults] = generateDemoData();
-
-    expectResultToHave(mockQueryResults, child1, done);
-    component.formControl.setValue("A X");
-  });
 });

src/app/core/ui/search/search.component.ts

@@ -1,12 +1,9 @@
 import { Component, ViewEncapsulation } from "@angular/core";
 import { Entity } from "../../entity/model/entity";
-import { from, Observable } from "rxjs";
-import { concatMap, debounceTime, skipUntil, tap } from "rxjs/operators";
-import { DatabaseIndexingService } from "../../entity/database-indexing/database-indexing.service";
+import { Observable } from "rxjs";
+import { concatMap, debounceTime, tap } from "rxjs/operators";
 import { Router } from "@angular/router";
 import { FormControl, ReactiveFormsModule } from "@angular/forms";
-import { EntitySchemaService } from "../../entity/schema/entity-schema.service";
-import { EntityRegistry } from "../../entity/database-entity.decorator";
 import { UserRoleGuard } from "../../permissions/permission-guard/user-role.guard";
 import { MatFormFieldModule } from "@angular/material/form-field";
 import { FontAwesomeModule } from "@fortawesome/angular-fontawesome";
@@ -15,6 +12,7 @@ import { MatAutocompleteModule } from "@angular/material/autocomplete";
 import { AsyncPipe, NgForOf, NgSwitch, NgSwitchCase } from "@angular/common";
 import { DisplayEntityComponent } from "../../entity-components/entity-select/display-entity/display-entity.component";
 import { UntilDestroy, untilDestroyed } from "@ngneat/until-destroy";
+import { SearchService } from "./search.service";
 
 /**
  * General search box that provides results out of any kind of entities from the system
@@ -43,8 +41,8 @@ import { UntilDestroy, untilDestroyed } from "@ngneat/until-destroy";
   standalone: true,
 })
 export class SearchComponent {
-  static INPUT_DEBOUNCE_TIME_MS: number = 400;
-  MIN_CHARACTERS_FOR_SEARCH: number = 3;
+  static INPUT_DEBOUNCE_TIME_MS = 400;
+  MIN_CHARACTERS_FOR_SEARCH = 2;
 
   readonly NOTHING_ENTERED = 0;
   readonly TOO_FEW_CHARACTERS = 1;
@@ -60,15 +58,12 @@ export class SearchComponent {
   results: Observable<Entity[]>;
 
   constructor(
-    private indexingService: DatabaseIndexingService,
     private router: Router,
     private userRoleGuard: UserRoleGuard,
-    private entitySchemaService: EntitySchemaService,
-    private entities: EntityRegistry
+    private searchService: SearchService
   ) {
     this.results = this.formControl.valueChanges.pipe(
       debounceTime(SearchComponent.INPUT_DEBOUNCE_TIME_MS),
-      skipUntil(this.createSearchIndex()),
       tap((next) => (this.state = this.updateState(next))),
       concatMap((next: string) => this.searchResults(next)),
       untilDestroyed(this)
@@ -94,19 +89,10 @@ export class SearchComponent {
     if (this.state !== this.SEARCH_IN_PROGRESS) {
       return [];
     }
-    const searchTerms = next.toLowerCase().split(" ");
-    const entities = await this.indexingService.queryIndexRaw(
-      "search_index/by_name",
-      {
-        startkey: searchTerms[0],
-        endkey: searchTerms[0] + "\ufff0",
-        include_docs: true,
-      }
-    );
-    const filtered = this.prepareResults(entities.rows, searchTerms);
-    const uniques = this.uniquify(filtered);
-    this.state = uniques.length === 0 ? this.NO_RESULTS : this.SHOW_RESULTS;
-    return uniques;
+    const entities = await this.searchService.getSearchResults(next);
+    const filtered = this.prepareResults(entities);
+    this.state = filtered.length === 0 ? this.NO_RESULTS : this.SHOW_RESULTS;
+    return filtered;
   }
 
   async clickOption(optionElement) {
@@ -126,68 +112,9 @@ export class SearchComponent {
     return /^[a-zA-Z]+|\d+$/.test(searchText);
   }
 
-  private createSearchIndex(): Observable<void> {
-    // `emit(x)` to add x as a key to the index that can be searched
-    const searchMapFunction = `
-      (doc) => {
-        if (doc.hasOwnProperty("searchIndices")) {
-           doc.searchIndices.forEach(word => emit(word.toString().toLowerCase()));
-        }
-      }`;
-
-    const designDoc = {
-      _id: "_design/search_index",
-      views: {
-        by_name: {
-          map: searchMapFunction,
-        },
-      },
-    };
-
-    // TODO move this to a service so it is not executed whenever a user logs in
-    return from(this.indexingService.createIndex(designDoc));
-  }
-
-  private prepareResults(
-    rows: [{ key: string; id: string; doc: object }],
-    searchTerms: string[]
-  ): Entity[] {
-    return rows
-      .map((doc) => this.transformDocToEntity(doc))
-      .filter((entity) =>
-        this.userRoleGuard.checkRoutePermissions(entity.getConstructor().route)
-      )
-      .filter((entity) =>
-        this.containsSecondarySearchTerms(entity, searchTerms)
-      );
-  }
-
-  private containsSecondarySearchTerms(
-    entity: Entity,
-    searchTerms: string[]
-  ): boolean {
-    const searchIndices = entity.searchIndices.join(" ").toLowerCase();
-    return searchTerms.every((s) => searchIndices.includes(s));
-  }
-
-  private uniquify(entities: Entity[]): Entity[] {
-    const uniques = new Map<string, Entity>();
-    entities.forEach((e) => {
-      uniques.set(e.getId(), e);
-    });
-    return [...uniques.values()];
-  }
-
-  private transformDocToEntity(doc: {
-    key: string;
-    id: string;
-    doc: object;
-  }): Entity {
-    const ctor = this.entities.get(Entity.extractTypeFromId(doc.id));
-    const entity = doc.id ? new ctor(doc.id) : new ctor();
-    if (doc.doc) {
-      this.entitySchemaService.loadDataIntoEntity(entity, doc.doc);
-    }
-    return entity;
+  private prepareResults(entities: Entity[]): Entity[] {
+    return entities.filter((entity) =>
+      this.userRoleGuard.checkRoutePermissions(entity.getConstructor().route)
+    );
   }
 }