fix(Group, Controls): Fix interactive group actions when negative scaling is involved

.codesandbox/templates/vanilla/src/index.ts

@@ -1,6 +1,6 @@
 import * as fabric from 'fabric';
 import './styles.css';
-import { testCase } from './testcases/nestedSelections';
+import { testCase } from './testcases/flippedInteractive';
 
 const el = document.getElementById('canvas');
 const canvas = (window.canvas = new fabric.Canvas(el));

.codesandbox/templates/vanilla/src/testcases/flippedInteractive.ts

@@ -0,0 +1,69 @@
+import * as fabric from 'fabric';
+
+export function testCase(canvas: fabric.Canvas, objectCaching = true) {
+  function renderNamedControl(
+    this: fabric.Control,
+    ctx: CanvasRenderingContext2D,
+    left: number,
+    top: number,
+    styleOverride: any,
+    fabricObject: fabric.FabricObject
+  ) {
+    ctx.save();
+    ctx.font = '12px Arial';
+    ctx.fillStyle = 'black';
+    ctx.fillText(this.name || '??', left - 12, top - 3);
+    ctx.restore();
+  }
+
+  canvas.preserveObjectStacking = true;
+  const rect = new fabric.Rect({
+    left: 100,
+    top: 50,
+    width: 100,
+    height: 100,
+    flipX: false,
+    fill: 'blue',
+    padding: 20,
+  });
+
+  const controls = rect.controls;
+
+  Object.entries(controls).forEach(([key, control]) => {
+    control.name = key;
+    control.render = renderNamedControl;
+  });
+
+  const rect2 = new fabric.Rect({
+    top: 200,
+    width: 50,
+    height: 50,
+    fill: 'red',
+    flipX: true,
+    opacity: 1,
+    controls,
+    padding: 20,
+  });
+
+  const g = new fabric.Group([rect], {
+    subTargetCheck: true,
+    interactive: true,
+    objectCaching,
+    skewX: 0,
+    angle: 90,
+    flipY: true,
+    controls,
+  });
+  const g2 = new fabric.Group([rect2], {
+    top: 100,
+    left: 100,
+    subTargetCheck: true,
+    interactive: true,
+    objectCaching,
+    angle: 0,
+    flipY: false,
+    controls,
+  });
+  canvas.add(g);
+  canvas.add(g2);
+}

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## [next]
 
+- fix(Group, Controls): Fix interactive group actions when negative scaling is involved [#9811](https://github.com/fabricjs/fabric.js/pull/9811)
 - fix(): Replace 'hasOwn' with 'in' operator in typeAssertions check [#9812](https://github.com/fabricjs/fabric.js/pull/9812)
 
 ## [6.0.0-rc1]

src/shapes/Object/InteractiveObject.ts

@@ -247,8 +247,13 @@ export class InteractiveFabricObject<
       ]),
       transformOptions = this.group
         ? qrDecompose(this.calcTransformMatrix())
-        : undefined,
-      dim = this._calculateCurrentDimensions(transformOptions),
+        : undefined;
+    // decomposing could bring negative scaling and `_calculateCurrentDimensions` can't take it
+    if (transformOptions) {
+      transformOptions.scaleX = Math.abs(transformOptions.scaleX);
+      transformOptions.scaleY = Math.abs(transformOptions.scaleY);
+    }
+    const dim = this._calculateCurrentDimensions(transformOptions),
       coords: Record<string, TOCoord> = {};
 
     this.forEachControl((control, key) => {

src/shapes/Object/ObjectOrigin.ts

@@ -49,6 +49,9 @@ export class ObjectOrigin<EventSpec>
    */
   _getTransformedDimensions(options: any = {}): Point {
     const dimOptions = {
+      // if scaleX or scaleY are negative numbers,
+      // this will return dimensions that are negative.
+      // and this will break assumptions around the codebase
       scaleX: this.scaleX,
       scaleY: this.scaleY,
       skewX: this.skewX,

test/visual/control_rendering.js

@@ -300,6 +300,63 @@
     fabricClass: 'Canvas',
   });
 
+  function controlboxFlippedXInGroup(canvas, callback) {
+    var rect = new fabric.Rect({
+      width: 90, height: 90, padding: 9, angle: 0, flipX: true,
+      cornerSize: 12, cornerColor: 'green', cornerStrokeColor: 'blue',
+      transparentCorners: false, borderScaleFactor: 3,
+      fill: 'red', top: 50, left: 35,
+    });
+    var group = new fabric.Group([rect], {
+      interactive: true,
+      subTargetCheck: true,
+    });
+    canvas.add(group);
+    canvas.setActiveObject(rect);
+    canvas.renderAll();
+    callback(canvas.lowerCanvasEl);
+  }
+
+  tests.push({
+    test: 'controlbox with flipped X in group',
+    code: controlboxFlippedXInGroup,
+    golden: 'controls11group.png',
+    percentage: 0.004,
+    width: 150,
+    height: 180,
+    fabricClass: 'Canvas',
+  });
+
+  function controlboxFlippedYInRotatedGroup(canvas, callback) {
+    var rect = new fabric.Rect({
+      width: 90, height: 90, padding: 9, angle: 0, flipX: false,
+      cornerSize: 12, cornerColor: 'green', cornerStrokeColor: 'blue',
+      transparentCorners: false, borderScaleFactor: 3,
+      fill: 'red', top: 50, left: 35,
+    });
+    var group = new fabric.Group([rect], {
+      angle: 90,
+      left: 110,
+      flipY: true,
+      interactive: true,
+      subTargetCheck: true,
+    });
+    canvas.add(group);
+    canvas.setActiveObject(rect);
+    canvas.renderAll();
+    callback(canvas.lowerCanvasEl);
+  }
+
+  tests.push({
+    test: 'controlbox with flipped Y in rotated group',
+    code: controlboxFlippedYInRotatedGroup,
+    golden: 'controls11group90r.png',
+    percentage: 0.004,
+    width: 180,
+    height: 180,
+    fabricClass: 'Canvas',
+  });
+
   function controlboxOpacitySingle(canvas, callback) {
     var rect = new fabric.Rect({
       width: 90, height: 90, padding: 3, opacity: 0.4,