fix(heatmap): remove values when brushing only over axes

packages/charts/src/chart_types/heatmap/renderer/dom/highlighter.tsx

@@ -39,11 +39,10 @@ export const HighlighterCellsComponent: FC<HighlighterCellsProps> = ({
 }) => {
   if (!initialized || dragShape === null) return null;
 
-  const maskId = `echHighlighterMask__${chartId}`;
   return (
     <svg className="echHighlighter" width="100%" height="100%">
       <defs>
-        <mask id={maskId}>
+        <mask id={`echHighlighterMask__${chartId}`}>
           {/* the entire chart */}
           {brushMask.visible && (
             <rect
@@ -84,7 +83,7 @@ export const HighlighterCellsComponent: FC<HighlighterCellsProps> = ({
             y={0}
             width={canvasDimension.width + canvasDimension.left}
             height={canvasDimension.height}
-            mask={`url(#${maskId})`}
+            mask={`url(#echHighlighterMask__${chartId})`}
             fill={brushMask.fill}
           />
         )}

packages/charts/src/chart_types/heatmap/state/selectors/get_brush_area.ts

@@ -35,6 +35,7 @@ export const getBrushAreaSelector = createCustomCachedSelector(
     if (!isDragging || !mouseDownPosition || !dragShape) {
       return null;
     }
+
     const start = {
       x: mouseDownPosition.position.x - chartDimensions.left,
       y: mouseDownPosition.position.y,

packages/charts/src/chart_types/heatmap/state/selectors/get_picked_cells.test.ts

@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Store } from 'redux';
+
+import { MockGlobalSpec, MockSeriesSpec } from '../../../../mocks/specs/specs';
+import { MockStore } from '../../../../mocks/store/store';
+import { ScaleType } from '../../../../scales/constants';
+import { onPointerMove, onMouseDown, onMouseUp } from '../../../../state/actions/mouse';
+import { GlobalChartState } from '../../../../state/chart_state';
+import { createOnBrushEndCaller } from './on_brush_end_caller';
+
+describe('Heatmap picked cells', () => {
+  let store: Store<GlobalChartState>;
+  let onBrushEndMock = jest.fn();
+
+  beforeEach(() => {
+    store = MockStore.default({ width: 300, height: 300, top: 0, left: 0 }, 'chartId');
+    onBrushEndMock = jest.fn();
+    MockStore.addSpecs(
+      [
+        MockGlobalSpec.settingsNoMargins(),
+        MockSeriesSpec.heatmap({
+          xScaleType: ScaleType.Ordinal,
+          data: [
+            { x: 'a', y: 'ya', value: 1 },
+            { x: 'b', y: 'ya', value: 2 },
+            { x: 'c', y: 'ya', value: 3 },
+            { x: 'a', y: 'yb', value: 4 },
+            { x: 'b', y: 'yb', value: 5 },
+            { x: 'c', y: 'yb', value: 6 },
+            { x: 'a', y: 'yc', value: 7 },
+            { x: 'b', y: 'yc', value: 8 },
+            { x: 'c', y: 'yc', value: 9 },
+          ],
+          config: {
+            brushTool: { visible: true },
+            grid: {
+              cellHeight: {
+                max: 'fill',
+              },
+              cellWidth: {
+                max: 'fill',
+              },
+            },
+            xAxisLabel: {
+              visible: true,
+            },
+            yAxisLabel: {
+              visible: true,
+            },
+            margin: { top: 0, bottom: 0, left: 0, right: 0 },
+            onBrushEnd: onBrushEndMock,
+          },
+        }),
+      ],
+      store,
+    );
+  });
+
+  it('should pick cells', () => {
+    const caller = createOnBrushEndCaller();
+    store.dispatch(onPointerMove({ x: 50, y: 50 }, 0));
+    store.dispatch(onMouseDown({ x: 50, y: 50 }, 100));
+    store.dispatch(onPointerMove({ x: 150, y: 250 }, 200));
+    store.dispatch(onMouseUp({ x: 150, y: 250 }, 300));
+    caller(store.getState());
+    const brushEvent = onBrushEndMock.mock.calls[0][0];
+    expect(brushEvent.x.length).toBe(2);
+  });
+  it('should not include x values if only dragging along y-axis', () => {
+    const caller = createOnBrushEndCaller();
+    store.dispatch(onPointerMove({ x: 0, y: 50 }, 0));
+    store.dispatch(onMouseDown({ x: 0, y: 50 }, 100));
+    store.dispatch(onPointerMove({ x: 0, y: 20 }, 200));
+    store.dispatch(onMouseUp({ x: 0, y: 20 }, 300));
+    caller(store.getState());
+    const brushEvent = onBrushEndMock.mock.calls[0][0];
+    expect(brushEvent.x.length).toBe(0);
+  });
+});

packages/charts/src/chart_types/heatmap/state/selectors/get_picked_cells.ts

@@ -9,16 +9,34 @@
 import { createCustomCachedSelector } from '../../../../state/create_selector';
 import { getLastDragSelector } from '../../../../state/selectors/get_last_drag';
 import { PickDragFunction } from '../../layout/types/viewmodel_types';
+import { computeChartDimensionsSelector } from './compute_chart_dimensions';
 import { geometries } from './geometries';
+import { getGridHeightParamsSelector } from './get_grid_full_height';
 
 /** @internal */
 export const getPickedCells = createCustomCachedSelector(
-  [geometries, getLastDragSelector],
-  (geoms, dragState): ReturnType<PickDragFunction> | null => {
+  [geometries, getLastDragSelector, computeChartDimensionsSelector, getGridHeightParamsSelector],
+  (geoms, dragState, canvasDimensions, gridParams): ReturnType<PickDragFunction> | null => {
     if (!dragState) {
       return null;
     }
 
+    // the pointer is not on the cells but over the y- axis and does not cross the y-axis
+    if (dragState.start.position.x < canvasDimensions.left && dragState.end.position.x < canvasDimensions.left) {
+      const fittedDragStateStart = { x: canvasDimensions.left, y: dragState.start.position.y };
+      const { y, cells } = geoms.pickDragArea([fittedDragStateStart, dragState.end.position]);
+      return { x: [], y, cells };
+    }
+
+    // the pointer is not on the cells by over the x-axis and does not cross the x-axis
+    if (
+      dragState.start.position.y > gridParams.gridCellHeight * gridParams.pageSize &&
+      dragState.end.position.y > gridParams.gridCellHeight * gridParams.pageSize
+    ) {
+      const fittedDragStateStart = { x: dragState.start.position.x, y: canvasDimensions.height };
+      const { x, cells } = geoms.pickDragArea([fittedDragStateStart, dragState.end.position]);
+      return { x, y: [], cells };
+    }
     return geoms.pickDragArea([dragState.start.position, dragState.end.position]);
   },
 );