Plugin E2E: Make it possible to test query editor in panel edit page (#…

…551)

Co-authored-by: Levente Balogh <balogh.levente.hu@gmail.com>

.github/workflows/playwright.yml

@@ -11,6 +11,8 @@ jobs:
  fail-fast: false
  matrix:
  GRAFANA_VERSION: ['latest', '10.0.5', '9.5.5', '9.2.5']
+ GOOGLE_SHEETS_VERSION: ['1.2.4', '1.2.0']
+ name: E2E Tests - Grafana@${{ matrix.GRAFANA_VERSION }} GoogleSheets@${{ matrix.GOOGLE_SHEETS_VERSION }}
  runs-on: ubuntu-latest
  steps:
  - uses: actions/checkout@v3
@@ -31,7 +33,7 @@ jobs:
  run: npx playwright install --with-deps chromium
 
  - name: Start Grafana
- run: docker run --rm -d -p 3000:3000 --name=grafana --env GF_INSTALL_PLUGINS=grafana-googlesheets-datasource grafana/grafana:${{ matrix.GRAFANA_VERSION }}; sleep 30
+ run: docker run --rm -d -p 3000:3000 --name=grafana --volume ./packages/plugin-e2e/provisioning:/etc/grafana/provisioning --env GOOGLE_JWT_FILE=${{secrets.GOOGLE_JWT_FILE}} --env "GF_INSTALL_PLUGINS=grafana-googlesheets-datasource ${{matrix.GOOGLE_SHEETS_VERSION}}" grafana/grafana:${{ matrix.GRAFANA_VERSION }}; sleep 30
 
  - name: Run Playwright tests
  run: npm run playwright:test --w @grafana/plugin-e2e
@@ -41,6 +43,6 @@ jobs:
  - uses: actions/upload-artifact@v3
  if: always()
  with:
- name: playwright-report-${{ matrix.GRAFANA_VERSION }}
+ name: playwright-report-Grafana${{ matrix.GRAFANA_VERSION }}-GoogleSheets${{ matrix.GOOGLE_SHEETS_VERSION }}
  path: packages/plugin-e2e/playwright-report/
  retention-days: 30

packages/plugin-e2e/docker-compose.yaml

@@ -4,11 +4,12 @@ services:
  grafana:
  image: grafana/${GRAFANA_IMAGE:-grafana-enterprise}:${GRAFANA_VERSION:-main}
  environment:
- - GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-googlesheets-datasource
+ - GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-googlesheets-datasource 1.2.4
  - GF_AUTH_ANONYMOUS_ENABLED=true
  - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
  - GF_AUTH_ANONYMOUS_ORG_NAME=Main Org.
  - GF_AUTH_ANONYMOUS_ORG_ID=1
+ - GOOGLE_JWT_FILE=${GOOGLE_JWT_FILE}
  ports:
  - 3000:3000/tcp
  volumes:

packages/plugin-e2e/playwright.config.ts

@@ -38,7 +38,7 @@ export default defineConfig<PluginOptions>({
  name: 'authenticate',
  testMatch: [/.*auth\.setup\.ts/],
  },
- // 2. Run all tests in parallel with Chrome.
+ // 2. Run all tests in parallel using Chrome.
  {
  name: 'chromium',
  use: {

packages/plugin-e2e/provisioning/datasources/google-sheets-datasource-jwt.yaml

@@ -0,0 +1,17 @@
+# config file version
+apiVersion: 1
+
+deleteDatasources:
+ - name: Google Sheets Service Account
+ orgId: 1
+
+datasources:
+ - editable: true
+ enabled: true
+ jsonData:
+ authType: jwt
+ name: Google Sheets Service Account
+ secureJsonData:
+ jwt: ${GOOGLE_JWT_FILE}
+ type: grafana-googlesheets-datasource
+ version: 1

packages/plugin-e2e/src/api.ts

@@ -1,9 +1,10 @@
-import { test as base, expect as baseExpect } from '@playwright/test';
+import { test as base, expect as baseExpect, selectors } from '@playwright/test';
 import { E2ESelectors } from './e2e-selectors/types';
 import fixtures from './fixtures';
-import { DataSourceConfigPage } from './models';
 import matchers from './matchers';
-import { CreateDataSourcePageArgs } from './types';
+import { CreateDataSourceArgs, CreateDataSourcePageArgs, DataSource, ReadProvisionArgs } from './types';
+import { PanelEditPage, GrafanaPage, DataSourceConfigPage, DashboardPage } from './models';
+import { grafanaE2ESelectorEngine } from './selectorEngine';
 
 export type PluginOptions = {
  selectorRegistration: void;
@@ -23,6 +24,30 @@ export type PluginFixture = {
  */
  selectors: E2ESelectors;
 
+ /**
+ * Isolated {@link DashboardPage} instance for each test.
+ *
+ * Navigates to a new dashboard page and adds a new panel.
+ *
+ * Use {@link PanelEditPage.setVisualization} to change the visualization
+ * Use {@link PanelEditPage.datasource.set} to change the datasource
+ * Use {@link PanelEditPage.getQueryEditorEditorRow} to retrieve the query
+ * editor row locator for a given query refId
+ */
+ newDashboardPage: DashboardPage;
+
+ /**
+ * Isolated {@link PanelEditPage} instance for each test.
+ *
+ * Navigates to a new dashboard page, adds a new panel and moves to the panel edit page. 
+ *
+ * Use {@link PanelEditPage.setVisualization} to change the visualization
+ * Use {@link PanelEditPage.datasource.set} to change the datasource
+ * Use {@link ExplorePage.getQueryEditorEditorRow} to retrieve the query
+ * editor row locator for a given query refId
+ */
+ panelEditPage: PanelEditPage;
+
  /**
  * Fixture command that will create an isolated DataSourceConfigPage instance for a given data source type.
  *
@@ -31,6 +56,15 @@ export type PluginFixture = {
  */
  createDataSourceConfigPage: (args: CreateDataSourcePageArgs) => Promise<DataSourceConfigPage>;
 
+ /**
+ * Fixture command that creates a data source via the Grafana API.
+ *
+ * If you have tests that depend on the the existance of a data source,
+ * you may use this command in a setup project. Read more about setup projects
+ * here: https://playwright.dev/docs/auth#basic-shared-account-in-all-tests
+ */
+ createDataSource: (args: CreateDataSourceArgs) => Promise<DataSource>;
+
  /**
  * Fixture command that login to Grafana using the Grafana API. 
  * If the same credentials should be used in every test, 
@@ -56,11 +90,28 @@ export type PluginFixture = {
  * test.use({ storageState: { cookies: [], origins: [] } });
  */
  login: () => Promise<void>;
+
+ /**
+ * Fixture command that reads a the yaml file for a provisioned dashboard
+ * or data source and returns it as json.
+ */
+ readProvision<T = any>(args: ReadProvisionArgs): Promise<T>;
 };
 
 // extend Playwright with Grafana plugin specific fixtures
 export const test = base.extend<PluginFixture & PluginOptions>(fixtures);
 
 export const expect = baseExpect.extend(matchers);
 
+/** Register a custom selector engine that resolves locators for Grafana E2E selectors
+ *
+ * The same functionality is available in the {@link GrafanaPage.getByTestIdOrAriaLabel} method. However,
+ * by registering the selector engine, one can resolve locators by Grafana E2E selectors also within a locator.
+ *
+ * Example:
+ * const queryEditorRow = await panelEditPage.getQueryEditorRow('A'); // returns a locator
+ * queryEditorRow.locator(`selector=${selectors.components.TimePicker.openButton}`).click();
+ * */
+selectors.register('selector', grafanaE2ESelectorEngine);
+
 export { selectors } from '@playwright/test';

packages/plugin-e2e/src/e2e-selectors/types.ts

@@ -6,7 +6,7 @@ export type E2ESelectors = {
 
 export type APIs = {
  DataSource: {
- getResource: string;
+ resource: string;
  healthCheck: string;
  };
 };

packages/plugin-e2e/src/e2e-selectors/versioned/components.ts

@@ -9,7 +9,7 @@ export const versionedComponents = {
  },
  TimePicker: {
  openButton: {
- '8.1.0': 'data-testid TimePicker open button',
+ '8.1.0': 'data-testid TimePicker Open Button',
  [MIN_GRAFANA_VERSION]: 'TimePicker open button',
  },
  fromField: 'Time Range from field',

packages/plugin-e2e/src/fixtures/commands/createDataSourceConfigPage.ts

@@ -27,7 +27,7 @@ const createDataSourceConfigPage: CreateDataSourceConfigPageFixture = async (
  await datasourceConfigPage.goto();
  return datasourceConfigPage;
  });
- deleteDataSource && datasourceConfigPage?.deleteDataSource();
+ deleteDataSource && (await datasourceConfigPage?.deleteDataSource());
 };
 
 export default createDataSourceConfigPage;

packages/plugin-e2e/src/fixtures/commands/readProvision.ts

@@ -0,0 +1,22 @@
+import { TestFixture } from '@playwright/test';
+import { promises } from 'fs';
+import { resolve as resolvePath } from 'path';
+import { parse as parseYml } from 'yaml';
+import { PluginFixture, PluginOptions } from '../../api';
+import { ReadProvisionArgs } from '../../types';
+import { PlaywrightCombinedArgs } from '../types';
+
+type ReadProvisionFixture = TestFixture<
+ <T = any>(args: ReadProvisionArgs) => Promise<T>,
+ PluginFixture & PluginOptions & PlaywrightCombinedArgs
+>;
+
+const readProvision: ReadProvisionFixture = async ({}, use) => {
+ await use(async ({ filePath }) => {
+ const resolvedPath = resolvePath(process.cwd(), 'provisioning', filePath);
+ const contents = await promises.readFile(resolvedPath, 'utf8');
+ return parseYml(contents);
+ });
+};
+
+export default readProvision;

packages/plugin-e2e/src/fixtures/index.ts

@@ -2,10 +2,18 @@ import grafanaVersion from './grafanaVersion';
 import selectors from './selectors';
 import login from './commands/login';
 import createDataSourceConfigPage from './commands/createDataSourceConfigPage';
+import panelEditPage from './panelEditPage';
+import createDataSource from './commands/createDataSource';
+import readProvision from './commands/readProvision';
+import newDashboardPage from './newDashboardPage';
 
 export default {
  selectors,
  grafanaVersion,
  login,
  createDataSourceConfigPage,
+ newDashboardPage,
+ panelEditPage,
+ createDataSource,
+ readProvision,
 };

packages/plugin-e2e/src/fixtures/newDashboardPage.ts

@@ -0,0 +1,14 @@
+import { TestFixture, expect } from '@playwright/test';
+import { PluginFixture, PluginOptions } from '../api';
+import { DashboardPage } from '../models';
+import { PlaywrightCombinedArgs } from './types';
+
+type NewDashboardPageFixture = TestFixture<DashboardPage, PluginFixture & PluginOptions & PlaywrightCombinedArgs>;
+
+const newDashboardPage: NewDashboardPageFixture = async ({ page, request, selectors, grafanaVersion }, use) => {
+ const newDashboardPage = new DashboardPage({ page, selectors, grafanaVersion, request }, expect);
+ await newDashboardPage.goto();
+ await use(newDashboardPage);
+};
+
+export default newDashboardPage;

packages/plugin-e2e/src/fixtures/panelEditPage.ts

@@ -0,0 +1,13 @@
+import { TestFixture } from '@playwright/test';
+import { PluginFixture, PluginOptions } from '../api';
+import { PanelEditPage } from '../models';
+import { PlaywrightCombinedArgs } from './types';
+
+type PanelEditPageFixture = TestFixture<PanelEditPage, PluginFixture & PluginOptions & PlaywrightCombinedArgs>;
+
+const panelEditPage: PanelEditPageFixture = async ({ newDashboardPage }, use) => {
+ const panelEditPage = await newDashboardPage.addPanel();
+ await use(panelEditPage);
+};
+
+export default panelEditPage;

packages/plugin-e2e/src/matchers/index.ts

@@ -1,5 +1,7 @@
 import toBeOK from './toBeOK';
+import toHavePanelError from './toHavePanelError';
 
 export default {
  toBeOK,
+ toHavePanelError,
 };

packages/plugin-e2e/src/matchers/toHavePanelError.ts

@@ -0,0 +1,26 @@
+import { expect } from '@playwright/test';
+import { PanelError } from '../types';
+import { getMessage } from './utils';
+
+const toHavePanelError = async (panelError: PanelError, options?: { timeout?: number }) => {
+ let pass = true;
+ let actual;
+ let message: any = 'A panel error to be displayed';
+
+ try {
+ const numberOfErrors = await panelError.getPanelError().count();
+ await expect(numberOfErrors).toBe(1);
+ } catch (_) {
+ message = getMessage(message, 'No panel error was found on the page');
+ actual = await panelError.getPanelError().count();
+ pass = false;
+ }
+
+ return {
+ message: () => message,
+ pass,
+ actual,
+ };
+};
+
+export default toHavePanelError;

packages/plugin-e2e/src/models/DashboardPage.ts

@@ -0,0 +1,65 @@
+const gte = require('semver/functions/gte');
+
+import { GotoDashboardArgs, PluginTestCtx } from '../types';
+
+import { Expect } from '@playwright/test';
+import { DataSourcePicker } from './DataSourcePicker';
+import { GrafanaPage } from './GrafanaPage';
+import { PanelEditPage } from './PanelEditPage';
+import { TimeRange } from './TimeRange';
+
+export class DashboardPage extends GrafanaPage {
+ dataSourcePicker: any;
+ timeRange: TimeRange;
+
+ constructor(ctx: PluginTestCtx, expect: Expect<any>, protected readonly dashboardUid?: string) {
+ super(ctx, expect);
+ this.dataSourcePicker = new DataSourcePicker(ctx, expect);
+ this.timeRange = new TimeRange(ctx, this.expect);
+ }
+
+ async goto(opts?: GotoDashboardArgs) {
+ const uid = opts?.uid || this.dashboardUid;
+ let url = uid ? this.ctx.selectors.pages.Dashboard.url(uid) : this.ctx.selectors.pages.AddDashboard.url;
+ if (opts?.queryParams) {
+ url += `?${opts.queryParams.toString()}`;
+ }
+ await this.ctx.page.goto(url, {
+ waitUntil: 'networkidle',
+ });
+ if (opts?.timeRange) {
+ await this.timeRange.set(opts.timeRange);
+ }
+ }
+
+ async gotoPanelEditPage(panelId: string) {
+ const url = this.ctx.selectors.pages.Dashboard.url(this.dashboardUid ?? '');
+ await this.ctx.page.goto(`${url}?editPanel=${panelId}`, {
+ waitUntil: 'networkidle',
+ });
+ return new PanelEditPage(this.ctx, this.expect);
+ }
+
+ async addPanel(): Promise<PanelEditPage> {
+ if (gte(this.ctx.grafanaVersion, '10.0.0')) {
+ //TODO: add new selector and use it in grafana/ui
+ const title = gte(this.ctx.grafanaVersion, '10.1.0') ? 'Add button' : 'Add panel button';
+ await this.getByTestIdOrAriaLabel(this.ctx.selectors.components.PageToolbar.itemButton(title)).click();
+ await this.getByTestIdOrAriaLabel(
+ this.ctx.selectors.pages.AddDashboard.itemButton('Add new visualization menu item')
+ ).click();
+ } else {
+ await this.getByTestIdOrAriaLabel(this.ctx.selectors.pages.AddDashboard.addNewPanel).click();
+ }
+
+ return new PanelEditPage(this.ctx, this.expect);
+ }
+
+ async deleteDashboard() {
+ await this.ctx.request.delete(`/api/datasources/uid/${this.dashboardUid}`);
+ }
+
+ async refreshDashboard() {
+ await this.ctx.page.getByTestId(this.ctx.selectors.components.RefreshPicker.runButtonV2).click();
+ }
+}

packages/plugin-e2e/src/models/DataSourcePicker.ts

@@ -0,0 +1,21 @@
+import { Expect } from '@playwright/test';
+import { PluginTestCtx } from '../types';
+import { GrafanaPage } from './GrafanaPage';
+
+export class DataSourcePicker extends GrafanaPage {
+ constructor(ctx: PluginTestCtx, expect: Expect<any>) {
+ super(ctx, expect);
+ }
+
+ async set(name: string) {
+ await this.getByTestIdOrAriaLabel(this.ctx.selectors.components.DataSourcePicker.container)
+ .locator('input')
+ .fill(name);
+
+ // this is a hack to get the selection to work in 10.ish versions of Grafana.
+ // TODO: investigate if the select component can somehow be refactored so that its easier to test with playwright
+ await this.ctx.page.keyboard.press('ArrowDown');
+ await this.ctx.page.keyboard.press('ArrowUp');
+ await this.ctx.page.keyboard.press('Enter');
+ }
+}