[Vis Augmenter] Add tests in OSD & AD plugin (#752) (#760)

* [Vis Augmenter / Feature Anywhere] Add test suite for vanilla OSD + helper fns for plugins (#725)

* feature anywhere initial tests

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* Add test suite

Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com>

* Remove unnecessary test case

Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com>

* Optimize getters

Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com>

---------

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>
Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com>
Co-authored-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* [Feature Anywhere / Vis Augmenter] Add test flows for integration with AD plugin (#727)

* feature anywhere initial tests

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* Add test suite

Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com>

* Add AD vis augmenter tests

Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com>

* More refactoring

Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com>

* More tests

Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com>

* Add test for AD cleanup scenario

Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com>

* Set up saved obj test suite

Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com>

* Add reminder TODO

Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com>

* Add tests regarding saved obj visibility

Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com>

* Add view events tests

Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com>

* cleanup

Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com>

* remove import

Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com>

---------

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>
Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com>
Co-authored-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>

* Fix NPE

Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com>

* Fix bug of non-empty dashboard; remove unnecessary test

Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com>

* Simplify dashboard creation

Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com>

* Update undefined check

Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com>

---------

Signed-off-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>
Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com>
Co-authored-by: Jovan Cvetkovic <jovanca.cvetkovic@gmail.com>
(cherry picked from commit 39444c1)

Co-authored-by: Tyler Ohlsen <ohltyler@amazon.com>

cypress/fixtures/dashboard/opensearch_dashboards/vis-augmenter/sample-data-simple/data.json

@@ -0,0 +1,27 @@
+[
+  {
+    "value1": 1,
+    "value2": 10,
+    "value3": 5
+  },
+  {
+    "value1": 5,
+    "value2": 1,
+    "value3": 3
+  },
+  {
+    "value1": 9,
+    "value2": 6,
+    "value3": 2
+  },
+  {
+    "value1": 2,
+    "value2": 1,
+    "value3": 1
+  },
+  {
+    "value1": 12,
+    "value2": 5,
+    "value3": 4
+  }
+]

cypress/fixtures/dashboard/opensearch_dashboards/vis-augmenter/sample-data-simple/index-pattern-fields.txt

@@ -0,0 +1 @@
+[{"count":0,"name":"@timestamp","type":"date","esTypes":["date"],"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":true},{"count":0,"name":"_id","type":"string","esTypes":["_id"],"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":false},{"count":0,"name":"_index","type":"string","esTypes":["_index"],"scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":false},{"count":0,"name":"_score","type":"number","scripted":false,"searchable":false,"aggregatable":false,"readFromDocValues":false},{"count":0,"name":"_source","type":"_source","esTypes":["_source"],"scripted":false,"searchable":false,"aggregatable":false,"readFromDocValues":false},{"count":0,"name":"_type","type":"string","scripted":false,"searchable":false,"aggregatable":false,"readFromDocValues":false},{"count":0,"name":"value1","type":"number","scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":false},{"count":0,"name":"value2","type":"number","scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":false},{"count":0,"name":"value3","type":"number","scripted":false,"searchable":true,"aggregatable":true,"readFromDocValues":false}]

cypress/fixtures/dashboard/opensearch_dashboards/vis-augmenter/sample-data-simple/index-settings.txt

@@ -0,0 +1 @@
+{"mappings":{"properties":{"value1":{"type":"integer"},"value2":{"type":"integer"},"value3":{"type":"integer"},"@timestamp":{"type":"date", "format":"epoch_millis"}}},"settings":{"index":{"number_of_shards":"1","number_of_replicas":"1"}}}

cypress/integration/core-opensearch-dashboards/opensearch-dashboards/apps/vis-augmenter/dashboard_spec.js

@@ -0,0 +1,129 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import {
+  INDEX_PATTERN_FILEPATH_SIMPLE,
+  INDEX_SETTINGS_FILEPATH_SIMPLE,
+  SAMPLE_DATA_FILEPATH_SIMPLE,
+} from '../../../../../utils/constants';
+import {
+  deleteVisAugmenterData,
+  bootstrapDashboard,
+} from '../../../../../utils/dashboards/vis-augmenter/helpers';
+
+describe('Vis augmenter - existing dashboards work as expected', () => {
+  describe('dashboard with ineligible, eligible, and vega visualizations', () => {
+    const indexName = 'vis-augmenter-sample-index';
+    const indexPatternName = 'vis-augmenter-sample-*';
+    const dashboardName = 'Vis Augmenter Dashboard';
+    const visualizationSpecs = [
+      {
+        name: 'count-agg-vis',
+        type: 'line',
+        indexPattern: indexPatternName,
+        metrics: [],
+      },
+      {
+        name: 'single-metric-vis',
+        type: 'line',
+        indexPattern: indexPatternName,
+        metrics: [
+          {
+            aggregation: 'Average',
+            field: 'value1',
+          },
+        ],
+      },
+      {
+        name: 'multi-metric-vis',
+        type: 'line',
+        indexPattern: indexPatternName,
+        metrics: [
+          {
+            aggregation: 'Average',
+            field: 'value1',
+          },
+          {
+            aggregation: 'Average',
+            field: 'value2',
+          },
+          {
+            aggregation: 'Max',
+            field: 'value3',
+          },
+        ],
+      },
+      {
+        name: 'area-vis',
+        type: 'area',
+        indexPattern: indexPatternName,
+        metrics: [
+          {
+            aggregation: 'Max',
+            field: 'value2',
+          },
+        ],
+      },
+      {
+        name: 'vega-vis',
+        type: 'vega',
+        indexPattern: indexPatternName,
+        metrics: [],
+      },
+    ];
+
+    const visualizationNames = visualizationSpecs.map(
+      (visualizationSpec) => visualizationSpec.name
+    );
+
+    before(() => {
+      // Create a dashboard and add some visualizations
+      bootstrapDashboard(
+        INDEX_SETTINGS_FILEPATH_SIMPLE,
+        INDEX_PATTERN_FILEPATH_SIMPLE,
+        SAMPLE_DATA_FILEPATH_SIMPLE,
+        indexName,
+        indexPatternName,
+        dashboardName,
+        visualizationSpecs
+      );
+    });
+
+    beforeEach(() => {
+      cy.visitDashboard(dashboardName);
+    });
+
+    after(() => {
+      deleteVisAugmenterData(
+        indexName,
+        indexPatternName,
+        visualizationNames,
+        dashboardName
+      );
+    });
+
+    it('View events option does not exist for any visualization', () => {
+      visualizationNames.forEach((visualizationName) => {
+        cy.getVisPanelByTitle(visualizationName)
+          .openVisContextMenu()
+          .getMenuItems()
+          .contains('View Events')
+          .should('not.exist');
+      });
+    });
+
+    it('Validate non-vega visualizations are not rendered with vega under the hood', () => {
+      visualizationSpecs.forEach((visualizationSpec) => {
+        cy.getVisPanelByTitle(visualizationSpec.name).within(() => {
+          if (visualizationSpec.type === 'vega') {
+            cy.get('.vgaVis__view').should('exist');
+          } else {
+            cy.get('.vgaVis__view').should('not.exist');
+          }
+        });
+      });
+    });
+  });
+});

cypress/integration/plugins/anomaly-detection-dashboards-plugin/sample_detector_spec.js

@@ -3,27 +3,9 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { AD_URL } from '../../../utils/constants';
+import { createSampleDetector } from '../../../utils/helpers';
 
 context('Sample detectors', () => {
-  // Helper fn that takes in a button test ID to determine
-  // the sample detector to create
-  const createSampleDetector = (createButtonDataTestSubj) => {
-    cy.visit(AD_URL.OVERVIEW);
-
-    cy.getElementByTestId('overviewTitle').should('exist');
-    cy.getElementByTestId('viewSampleDetectorLink').should('not.exist');
-    cy.getElementByTestId(createButtonDataTestSubj).click();
-    cy.visit(AD_URL.OVERVIEW);
-
-    // Check that the details page defaults to real-time, and shows detector is initializing
-    cy.getElementByTestId('viewSampleDetectorLink').click();
-    cy.getElementByTestId('detectorNameHeader').should('exist');
-    cy.getElementByTestId('sampleIndexDetailsCallout').should('exist');
-    cy.getElementByTestId('realTimeResultsHeader').should('exist');
-    cy.getElementByTestId('detectorStateInitializing').should('exist');
-  };
-
   beforeEach(() => {
     cy.deleteAllIndices();
     cy.deleteADSystemIndices();

cypress/integration/plugins/anomaly-detection-dashboards-plugin/vis_augmenter/associate_detector_spec.js

@@ -0,0 +1,145 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import {
+  deleteVisAugmenterData,
+  bootstrapDashboard,
+  openAddAnomalyDetectorFlyout,
+  openAssociatedDetectorsFlyout,
+  createDetectorFromVis,
+  associateDetectorFromVis,
+  unlinkDetectorFromVis,
+  ensureDetectorIsLinked,
+  ensureDetectorDetails,
+  openDetectorDetailsPageFromFlyout,
+} from '../../../../utils/helpers';
+import {
+  INDEX_PATTERN_FILEPATH_SIMPLE,
+  INDEX_SETTINGS_FILEPATH_SIMPLE,
+  SAMPLE_DATA_FILEPATH_SIMPLE,
+} from '../../../../utils/constants';
+
+describe('Anomaly detection integration with vis augmenter', () => {
+  const indexName = 'ad-vis-augmenter-sample-index';
+  const indexPatternName = 'ad-vis-augmenter-sample-*';
+  const dashboardName = 'AD Vis Augmenter Dashboard';
+  const detectorName = 'ad-vis-augmenter-detector';
+  const visualizationName = 'single-metric-vis';
+  const visualizationSpec = {
+    name: visualizationName,
+    type: 'line',
+    indexPattern: indexPatternName,
+    metrics: [
+      {
+        aggregation: 'Average',
+        field: 'value1',
+      },
+    ],
+  };
+
+  before(() => {
+    // Create a dashboard and add some visualizations
+    cy.wait(5000);
+    bootstrapDashboard(
+      INDEX_SETTINGS_FILEPATH_SIMPLE,
+      INDEX_PATTERN_FILEPATH_SIMPLE,
+      SAMPLE_DATA_FILEPATH_SIMPLE,
+      indexName,
+      indexPatternName,
+      dashboardName,
+      [visualizationSpec]
+    );
+  });
+
+  after(() => {
+    deleteVisAugmenterData(
+      indexName,
+      indexPatternName,
+      [visualizationName],
+      dashboardName
+    );
+    cy.deleteADSystemIndices();
+  });
+
+  beforeEach(() => {});
+
+  afterEach(() => {});
+
+  it('Shows empty state when no associated detectors', () => {
+    openAssociatedDetectorsFlyout(dashboardName, visualizationName);
+    cy.getElementByTestId('emptyAssociatedDetectorFlyoutMessage');
+  });
+
+  it('Create new detector from visualization', () => {
+    openAddAnomalyDetectorFlyout(dashboardName, visualizationName);
+    createDetectorFromVis(detectorName);
+
+    ensureDetectorIsLinked(dashboardName, visualizationName, detectorName);
+
+    // Since this detector is created based off of vis metrics, we assume here
+    // the number of features will equal the number of metrics we have specified.
+    ensureDetectorDetails(detectorName, visualizationSpec.metrics.length);
+
+    unlinkDetectorFromVis(dashboardName, visualizationName, detectorName);
+  });
+
+  it('Associate existing detector - creation flow', () => {
+    openAddAnomalyDetectorFlyout(dashboardName, visualizationName);
+
+    cy.get('.euiFlyout').find('.euiTitle').contains('Add anomaly detector');
+    // ensuring the flyout is defaulting to detector creation vs. association
+    cy.getElementByTestId('adAnywhereCreateDetectorButton');
+    cy.get('[id="add-anomaly-detector__existing"]').click();
+
+    associateDetectorFromVis(detectorName);
+
+    ensureDetectorIsLinked(dashboardName, visualizationName, detectorName);
+    unlinkDetectorFromVis(dashboardName, visualizationName, detectorName);
+  });
+
+  it('Associate existing detector - associated detectors flow', () => {
+    openAssociatedDetectorsFlyout(dashboardName, visualizationName);
+    cy.getElementByTestId('associateDetectorButton').click();
+    associateDetectorFromVis(detectorName);
+
+    ensureDetectorIsLinked(dashboardName, visualizationName, detectorName);
+    unlinkDetectorFromVis(dashboardName, visualizationName, detectorName);
+  });
+
+  it('Deleting linked detector shows error once and removes from associated detectors list', () => {
+    openAssociatedDetectorsFlyout(dashboardName, visualizationName);
+    cy.getElementByTestId('associateDetectorButton').click();
+    associateDetectorFromVis(detectorName);
+    ensureDetectorIsLinked(dashboardName, visualizationName, detectorName);
+    openDetectorDetailsPageFromFlyout();
+    cy.getElementByTestId('configurationsTab').click();
+    cy.getElementByTestId('detectorNameHeader').within(() => {
+      cy.contains(detectorName);
+    });
+
+    cy.getElementByTestId('actionsButton').click();
+    cy.getElementByTestId('deleteDetectorItem').click();
+    cy.getElementByTestId('typeDeleteField').type('delete', { force: true });
+    cy.getElementByTestId('confirmButton').click();
+    cy.wait(5000);
+
+    cy.visitDashboard(dashboardName);
+
+    // Expect an error message to show up
+    cy.getElementByTestId('errorToastMessage').parent().find('button').click();
+    cy.get('.euiModal');
+    cy.get('.euiModalFooter').find('button').click();
+    cy.wait(2000);
+
+    // Expect associated detector list to be empty (the association should be removed)
+    openAssociatedDetectorsFlyout(dashboardName, visualizationName);
+    cy.getElementByTestId('emptyAssociatedDetectorFlyoutMessage');
+    cy.wait(2000);
+
+    // Reload the dashboard - error toast shouldn't show anymore
+    cy.visitDashboard(dashboardName);
+    cy.getElementByTestId('errorToastMessage').should('not.exist');
+  });
+});