Plugin E2E: Make it possible to test query editor in panel edit page

.github/workflows/playwright.yml

@@ -11,6 +11,8 @@ jobs:
       fail-fast: false
       matrix:
         GRAFANA_VERSION: ['latest', '10.0.5', '9.5.5', '9.2.5']
+        GOOGLE_SHEETS_VERSION: ['1.2.4', '1.2.0']
+    name: E2E Tests - Grafana@${{ matrix.GRAFANA_VERSION }} GoogleSheets@${{ matrix.GOOGLE_SHEETS_VERSION }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -31,7 +33,7 @@ jobs:
         run: npx playwright install --with-deps chromium
 
       - name: Start Grafana
-        run: docker run --rm -d -p 3000:3000 --name=grafana --env GF_INSTALL_PLUGINS=grafana-googlesheets-datasource grafana/grafana:${{ matrix.GRAFANA_VERSION }}; sleep 30
+        run: docker run --rm -d -p 3000:3000 --name=grafana --volume ./packages/plugin-e2e/provisioning:/etc/grafana/provisioning --env GOOGLE_JWT_FILE=${{secrets.GOOGLE_JWT_FILE}} --env "GF_INSTALL_PLUGINS=grafana-googlesheets-datasource ${{matrix.GOOGLE_SHEETS_VERSION}}" grafana/grafana:${{ matrix.GRAFANA_VERSION }}; sleep 30
 
       - name: Run Playwright tests
         run: npm run playwright:test --w @grafana/plugin-e2e
@@ -41,6 +43,6 @@ jobs:
       - uses: actions/upload-artifact@v3
         if: always()
         with:
-          name: playwright-report-${{ matrix.GRAFANA_VERSION }}
+          name: playwright-report-Grafana${{ matrix.GRAFANA_VERSION }}-GoogleSheets${{ matrix.GOOGLE_SHEETS_VERSION }}
           path: packages/plugin-e2e/playwright-report/
           retention-days: 30

packages/plugin-e2e/docker-compose.yaml

@@ -4,11 +4,12 @@ services:
   grafana:
     image: grafana/${GRAFANA_IMAGE:-grafana-enterprise}:${GRAFANA_VERSION:-main}
     environment:
-      - GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-googlesheets-datasource
+      - GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-googlesheets-datasource 1.2.4
       - GF_AUTH_ANONYMOUS_ENABLED=true
       - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
       - GF_AUTH_ANONYMOUS_ORG_NAME=Main Org.
       - GF_AUTH_ANONYMOUS_ORG_ID=1
+      - GOOGLE_JWT_FILE=${GOOGLE_JWT_FILE}
     ports:
       - 3000:3000/tcp
     volumes:

packages/plugin-e2e/playwright.config.ts

@@ -38,7 +38,7 @@ export default defineConfig<PluginOptions>({
       name: 'authenticate',
       testMatch: [/.*auth\.setup\.ts/],
     },
-    // 2. Run all tests in parallel with Chrome.
+    // 2. Run all tests in parallel using Chrome.
     {
       name: 'chromium',
       use: {

packages/plugin-e2e/provisioning/datasources/google-sheets-datasource-jwt.yaml

@@ -0,0 +1,17 @@
+# config file version
+apiVersion: 1
+
+deleteDatasources:
+  - name: Google Sheets Service Account
+    orgId: 1
+
+datasources:
+  - editable: true
+    enabled: true
+    jsonData:
+      authType: jwt
+    name: Google Sheets Service Account
+    secureJsonData:
+      jwt: ${GOOGLE_JWT_FILE}
+    type: grafana-googlesheets-datasource
+    version: 1

packages/plugin-e2e/src/api.ts

@@ -1,9 +1,10 @@
-import { test as base, expect as baseExpect } from '@playwright/test';
+import { test as base, expect as baseExpect, selectors } from '@playwright/test';
 import { E2ESelectors } from './e2e-selectors/types';
 import fixtures from './fixtures';
-import { DataSourceConfigPage } from './models';
 import matchers from './matchers';
-import { CreateDataSourcePageArgs } from './types';
+import { CreateDataSourceArgs, CreateDataSourcePageArgs, DataSource, ReadProvisionArgs } from './types';
+import { PanelEditPage, GrafanaPage, DataSourceConfigPage, DashboardPage } from './models';
+import { grafanaE2ESelectorEngine } from './selectorEngine';
 
 export type PluginOptions = {
   selectorRegistration: void;
@@ -23,6 +24,30 @@ export type PluginFixture = {
    */
   selectors: E2ESelectors;
 
+  /**
+   * Isolated {@link DashboardPage} instance for each test.
+   *
+   * Navigates to a new dashboard page and adds a new panel.
+   *
+   * Use {@link PanelEditPage.setVisualization} to change the visualization
+   * Use {@link PanelEditPage.datasource.set} to change the datasource
+   * Use {@link PanelEditPage.getQueryEditorEditorRow} to retrieve the query
+   * editor row locator for a given query refId
+   */
+  newDashboardPage: DashboardPage;
+
+  /**
+   * Isolated {@link PanelEditPage} instance for each test.
+   *
+   * Navigates to a new dashboard page, adds a new panel and moves to the panel edit page. 
+   *
+   * Use {@link PanelEditPage.setVisualization} to change the visualization
+   * Use {@link PanelEditPage.datasource.set} to change the datasource
+   * Use {@link ExplorePage.getQueryEditorEditorRow} to retrieve the query
+   * editor row locator for a given query refId
+   */
+  panelEditPage: PanelEditPage;
+
   /**
    * Fixture command that will create an isolated DataSourceConfigPage instance for a given data source type.
    *
@@ -31,6 +56,15 @@ export type PluginFixture = {
    */
   createDataSourceConfigPage: (args: CreateDataSourcePageArgs) => Promise<DataSourceConfigPage>;
 
+  /**
+   * Fixture command that creates a data source via the Grafana API.
+   *
+   * If you have tests that depend on the the existance of a data source,
+   * you may use this command in a setup project. Read more about setup projects
+   * here: https://playwright.dev/docs/auth#basic-shared-account-in-all-tests
+   */
+  createDataSource: (args: CreateDataSourceArgs) => Promise<DataSource>;
+
   /**
    * Fixture command that login to Grafana using the Grafana API. 
    * If the same credentials should be used in every test, 
@@ -56,11 +90,28 @@ export type PluginFixture = {
    * test.use({ storageState: { cookies: [], origins: [] } });
    */
   login: () => Promise<void>;
+
+  /**
+   * Fixture command that reads a the yaml file for a provisioned dashboard
+   * or data source and returns it as json.
+   */
+  readProvision<T = any>(args: ReadProvisionArgs): Promise<T>;
 };
 
 // extend Playwright with Grafana plugin specific fixtures
 export const test = base.extend<PluginFixture & PluginOptions>(fixtures);
 
 export const expect = baseExpect.extend(matchers);
 
+/** Register a custom selector engine that resolves locators for Grafana E2E selectors
+ *
+ * The same functionality is available in the {@link GrafanaPage.getByTestIdOrAriaLabel} method. However,
+ * by registering the selector engine, one can resolve locators by Grafana E2E selectors also within a locator.
+ *
+ * Example:
+ * const queryEditorRow = await panelEditPage.getQueryEditorRow('A'); // returns a locator
+ * queryEditorRow.locator(`selector=${selectors.components.TimePicker.openButton}`).click();
+ * */
+selectors.register('selector', grafanaE2ESelectorEngine);
+
 export { selectors } from '@playwright/test';

packages/plugin-e2e/src/e2e-selectors/types.ts

@@ -6,7 +6,7 @@ export type E2ESelectors = {
 
 export type APIs = {
   DataSource: {
-    getResource: string;
+    resource: string;
     healthCheck: string;
   };
 };

packages/plugin-e2e/src/e2e-selectors/versioned/components.ts

@@ -9,7 +9,7 @@ export const versionedComponents = {
   },
   TimePicker: {
     openButton: {
-      '8.1.0': 'data-testid TimePicker open button',
+      '8.1.0': 'data-testid TimePicker Open Button',
       [MIN_GRAFANA_VERSION]: 'TimePicker open button',
     },
     fromField: 'Time Range from field',

packages/plugin-e2e/src/fixtures/commands/createDataSourceConfigPage.ts

@@ -27,7 +27,7 @@ const createDataSourceConfigPage: CreateDataSourceConfigPageFixture = async (
     await datasourceConfigPage.goto();
     return datasourceConfigPage;
   });
-  deleteDataSource && datasourceConfigPage?.deleteDataSource();
+  deleteDataSource && (await datasourceConfigPage?.deleteDataSource());
 };
 
 export default createDataSourceConfigPage;

packages/plugin-e2e/src/fixtures/commands/readProvision.ts

@@ -0,0 +1,22 @@
+import { TestFixture } from '@playwright/test';
+import { promises } from 'fs';
+import { resolve as resolvePath } from 'path';
+import { parse as parseYml } from 'yaml';
+import { PluginFixture, PluginOptions } from '../../api';
+import { ReadProvisionArgs } from '../../types';
+import { PlaywrightCombinedArgs } from '../types';
+
+type ReadProvisionFixture = TestFixture<
+  <T = any>(args: ReadProvisionArgs) => Promise<T>,
+  PluginFixture & PluginOptions & PlaywrightCombinedArgs
+>;
+
+const readProvision: ReadProvisionFixture = async ({}, use) => {
+  await use(async ({ filePath }) => {
+    const resolvedPath = resolvePath(process.cwd(), 'provisioning', filePath);
+    const contents = await promises.readFile(resolvedPath, 'utf8');
+    return parseYml(contents);
+  });
+};
+
+export default readProvision;

packages/plugin-e2e/src/fixtures/index.ts

@@ -2,10 +2,18 @@ import grafanaVersion from './grafanaVersion';
 import selectors from './selectors';
 import login from './commands/login';
 import createDataSourceConfigPage from './commands/createDataSourceConfigPage';
+import panelEditPage from './panelEditPage';
+import createDataSource from './commands/createDataSource';
+import readProvision from './commands/readProvision';
+import newDashboardPage from './newDashboardPage';
 
 export default {
   selectors,
   grafanaVersion,
   login,
   createDataSourceConfigPage,
+  newDashboardPage,
+  panelEditPage,
+  createDataSource,
+  readProvision,
 };

packages/plugin-e2e/src/fixtures/newDashboardPage.ts

@@ -0,0 +1,14 @@
+import { TestFixture, expect } from '@playwright/test';
+import { PluginFixture, PluginOptions } from '../api';
+import { DashboardPage } from '../models';
+import { PlaywrightCombinedArgs } from './types';
+
+type NewDashboardPageFixture = TestFixture<DashboardPage, PluginFixture & PluginOptions & PlaywrightCombinedArgs>;
+
+const newDashboardPage: NewDashboardPageFixture = async ({ page, request, selectors, grafanaVersion }, use) => {
+  const newDashboardPage = new DashboardPage({ page, selectors, grafanaVersion, request }, expect);
+  await newDashboardPage.goto();
+  await use(newDashboardPage);
+};
+
+export default newDashboardPage;

packages/plugin-e2e/src/fixtures/panelEditPage.ts

@@ -0,0 +1,13 @@
+import { TestFixture } from '@playwright/test';
+import { PluginFixture, PluginOptions } from '../api';
+import { PanelEditPage } from '../models';
+import { PlaywrightCombinedArgs } from './types';
+
+type PanelEditPageFixture = TestFixture<PanelEditPage, PluginFixture & PluginOptions & PlaywrightCombinedArgs>;
+
+const panelEditPage: PanelEditPageFixture = async ({ newDashboardPage }, use) => {
+  const panelEditPage = await newDashboardPage.addPanel();
+  await use(panelEditPage);
+};
+
+export default panelEditPage;

packages/plugin-e2e/src/matchers/index.ts

@@ -1,5 +1,7 @@
 import toBeOK from './toBeOK';
+import toHavePanelError from './toHavePanelError';
 
 export default {
   toBeOK,
+  toHavePanelError,
 };

packages/plugin-e2e/src/matchers/toHavePanelError.ts

@@ -0,0 +1,26 @@
+import { expect } from '@playwright/test';
+import { PanelError } from '../types';
+import { getMessage } from './utils';
+
+const toHavePanelError = async (panelError: PanelError, options?: { timeout?: number }) => {
+  let pass = true;
+  let actual;
+  let message: any = 'A panel error to be displayed';
+
+  try {
+    const numberOfErrors = await panelError.getPanelError().count();
+    await expect(numberOfErrors).toBe(1);
+  } catch (_) {
+    message = getMessage(message, 'No panel error was found on the page');
+    actual = await panelError.getPanelError().count();
+    pass = false;
+  }
+
+  return {
+    message: () => message,
+    pass,
+    actual,
+  };
+};
+
+export default toHavePanelError;

packages/plugin-e2e/src/models/DashboardPage.ts

@@ -0,0 +1,65 @@
+const gte = require('semver/functions/gte');
+
+import { GotoDashboardArgs, PluginTestCtx } from '../types';
+
+import { Expect } from '@playwright/test';
+import { DataSourcePicker } from './DataSourcePicker';
+import { GrafanaPage } from './GrafanaPage';
+import { PanelEditPage } from './PanelEditPage';
+import { TimeRange } from './TimeRange';
+
+export class DashboardPage extends GrafanaPage {
+  dataSourcePicker: any;
+  timeRange: TimeRange;
+
+  constructor(ctx: PluginTestCtx, expect: Expect<any>, protected readonly dashboardUid?: string) {
+    super(ctx, expect);
+    this.dataSourcePicker = new DataSourcePicker(ctx, expect);
+    this.timeRange = new TimeRange(ctx, this.expect);
+  }
+
+  async goto(opts?: GotoDashboardArgs) {
+    const uid = opts?.uid || this.dashboardUid;
+    let url = uid ? this.ctx.selectors.pages.Dashboard.url(uid) : this.ctx.selectors.pages.AddDashboard.url;
+    if (opts?.queryParams) {
+      url += `?${opts.queryParams.toString()}`;
+    }
+    await this.ctx.page.goto(url, {
+      waitUntil: 'networkidle',
+    });
+    if (opts?.timeRange) {
+      await this.timeRange.set(opts.timeRange);
+    }
+  }
+
+  async gotoPanelEditPage(panelId: string) {
+    const url = this.ctx.selectors.pages.Dashboard.url(this.dashboardUid ?? '');
+    await this.ctx.page.goto(`${url}?editPanel=${panelId}`, {
+      waitUntil: 'networkidle',
+    });
+    return new PanelEditPage(this.ctx, this.expect);
+  }
+
+  async addPanel(): Promise<PanelEditPage> {
+    if (gte(this.ctx.grafanaVersion, '10.0.0')) {
+      //TODO: add new selector and use it in grafana/ui
+      const title = gte(this.ctx.grafanaVersion, '10.1.0') ? 'Add button' : 'Add panel button';
+      await this.getByTestIdOrAriaLabel(this.ctx.selectors.components.PageToolbar.itemButton(title)).click();
+      await this.getByTestIdOrAriaLabel(
+        this.ctx.selectors.pages.AddDashboard.itemButton('Add new visualization menu item')
+      ).click();
+    } else {
+      await this.getByTestIdOrAriaLabel(this.ctx.selectors.pages.AddDashboard.addNewPanel).click();
+    }
+
+    return new PanelEditPage(this.ctx, this.expect);
+  }
+
+  async deleteDashboard() {
+    await this.ctx.request.delete(`/api/datasources/uid/${this.dashboardUid}`);
+  }
+
+  async refreshDashboard() {
+    await this.ctx.page.getByTestId(this.ctx.selectors.components.RefreshPicker.runButtonV2).click();
+  }
+}

packages/plugin-e2e/src/models/DataSourcePicker.ts

@@ -0,0 +1,21 @@
+import { Expect } from '@playwright/test';
+import { PluginTestCtx } from '../types';
+import { GrafanaPage } from './GrafanaPage';
+
+export class DataSourcePicker extends GrafanaPage {
+  constructor(ctx: PluginTestCtx, expect: Expect<any>) {
+    super(ctx, expect);
+  }
+
+  async set(name: string) {
+    await this.getByTestIdOrAriaLabel(this.ctx.selectors.components.DataSourcePicker.container)
+      .locator('input')
+      .fill(name);
+
+    // this is a hack to get the selection to work in 10.ish versions of Grafana.
+    // TODO: investigate if the select component can somehow be refactored so that its easier to test with playwright
+    await this.ctx.page.keyboard.press('ArrowDown');
+    await this.ctx.page.keyboard.press('ArrowUp');
+    await this.ctx.page.keyboard.press('Enter');
+  }
+}