Merge pull request #4563 from wix/feat/copilot

Detox Copilot: initial version (core).

.github/workflows/rapid-test.yml

@@ -47,6 +47,9 @@ jobs:
       - name: Generation tests
         run: npm test
         working-directory: generation
+      - name: Copilot unit tests
+        run: npm test
+        working-directory: detox-copilot
       - name: Pack Allure results
         run: zip -r allure-results.zip detox/allure-results detox/test/allure-results generation/allure-results
       - name: Upload Allure results

detox-copilot/.gitignore

@@ -0,0 +1 @@
+dist/

detox-copilot/README.md

@@ -0,0 +1,16 @@
+# Detox Copilot
+
+## Work in Progress
+
+Detox Copilot is a Detox plugin that leverages large language models (LLM) to seamlessly invoke Detox actions.
+
+It provides APIs to perform actions and assertions within your Detox tests while interfacing with an LLM service to enhance the testing process.
+
+## API Overview
+
+We will provide a high-level overview of the API that Detox Copilot will expose, this is a work in progress and the final API may differ. We will also provide a more extensive documentation once the API is finalized.
+
+- `copilot.init(config)`: Initializes the Copilot with the provided configuration, must be called before using Copilot, e.g. `copilot.init(...)`
+- `copilot.reset()`: Resets the Copilot by clearing the previous steps, e.g. `copilot.reset()`
+- `act(prompt)`: Semantic action invocation, e.g. `copilot.act('tap the sign-in button')`
+- `assert(prompt)`: Semantic assertion invocation, e.g. `copilot.assert('the sign-in button is visible')`

detox-copilot/jest.config.js

@@ -0,0 +1,10 @@
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  roots: ['<rootDir>/src'],
+  testMatch: ['**/__tests__/**/*.ts', '**/?(*.)+(spec|test).ts'],
+  moduleFileExtensions: ['ts', 'js', 'json', 'node'],
+  moduleNameMapper: {
+    '^@/(.*)$': '<rootDir>/src/$1'
+  },
+};

detox-copilot/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "detox-copilot",
+  "version": "0.0.0",
+  "description": "A Detox-based plugin that leverages AI to seamlessly invoke UI testing framework operations",
+  "keywords": [
+    "detox",
+    "copilot",
+    "ai"
+  ],
+  "author": "Asaf Korem <asaf.korem@gmail.com>",
+  "homepage": "https://github.com/wix/Detox/detox-copilot",
+  "license": "MIT",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "publishConfig": {
+    "registry": "https://registry.npmjs.org"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/wix/Detox.git"
+  },
+  "scripts": {
+    "build": "tsc",
+    "test": "jest",
+    "prepublishOnly": "npm run build"
+  },
+  "bugs": {
+    "url": "https://github.com/wix/Detox/issues"
+  },
+  "devDependencies": {
+    "@types/jest": "^29.5.12",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.2.4",
+    "typescript": "^5.5.4"
+  }
+}

detox-copilot/src/Copilot.test.ts

@@ -0,0 +1,122 @@
+import { Copilot } from '@/Copilot';
+import { StepPerformer } from '@/actions/StepPerformer';
+import { CopilotError } from '@/errors/CopilotError';
+
+jest.mock('@/actions/StepPerformer');
+
+describe('Copilot', () => {
+    let mockConfig: Config;
+
+    beforeEach(() => {
+        mockConfig = {
+            frameworkDriver: {
+                captureSnapshotImage: jest.fn(),
+                captureViewHierarchyString: jest.fn(),
+                availableAPI: {
+                    matchers: [],
+                    actions: [],
+                    assertions: []
+                },
+            },
+            promptHandler: {
+                runPrompt: jest.fn(),
+                isSnapshotImageSupported: jest.fn().mockReturnValue(true)
+            }
+        };
+        jest.spyOn(console, 'error').mockImplementation(() => {});
+    });
+
+    afterEach(() => {
+        jest.resetAllMocks();
+        (console.error as jest.Mock).mockRestore();
+        Copilot['instance'] = undefined;
+    });
+
+    describe('getInstance', () => {
+        it('should return the same instance after initialization', () => {
+            Copilot.init(mockConfig);
+
+            const instance1 = Copilot.getInstance();
+            const instance2 = Copilot.getInstance();
+
+            expect(instance1).toBe(instance2);
+        });
+
+        it('should throw CopilotError if getInstance is called before init', () => {
+            expect(() => Copilot.getInstance()).toThrow(CopilotError);
+            expect(() => Copilot.getInstance()).toThrow('Copilot has not been initialized. Please call the `init()` method before using it.');
+        });
+    });
+
+    describe('init', () => {
+        it('should create a new instance of Copilot', () => {
+            Copilot.init(mockConfig);
+            expect(Copilot.getInstance()).toBeInstanceOf(Copilot);
+        });
+
+        it('should overwrite existing instance when called multiple times', () => {
+            Copilot.init(mockConfig);
+            const instance1 = Copilot.getInstance();
+
+            Copilot.init(mockConfig);
+            const instance2 = Copilot.getInstance();
+
+            expect(instance1).not.toBe(instance2);
+        });
+
+        it('should throw an error if config is invalid', () => {
+            const invalidConfig = {} as Config;
+
+            expect(() => Copilot.init(invalidConfig)).toThrow();
+        });
+    });
+
+    describe('execute', () => {
+        it('should call StepPerformer.perform with the given step', async () => {
+            Copilot.init(mockConfig);
+            const instance = Copilot.getInstance();
+            const step: ExecutionStep = { type: 'action', value: 'tap button' };
+
+            await instance.perform(step);
+
+            expect(StepPerformer.prototype.perform).toHaveBeenCalledWith(step, []);
+        });
+
+        it('should return the result from StepPerformer.perform', async () => {
+            (StepPerformer.prototype.perform as jest.Mock).mockResolvedValue(true);
+            Copilot.init(mockConfig);
+            const instance = Copilot.getInstance();
+
+            const result = await instance.perform({ type: 'action', value: 'tap button' });
+
+            expect(result).toBe(true);
+        });
+
+        it('should accumulate previous steps', async () => {
+            Copilot.init(mockConfig);
+            const instance = Copilot.getInstance();
+            const step1: ExecutionStep = { type: 'action', value: 'tap button 1' };
+            const step2 : ExecutionStep = { type: 'action', value: 'tap button 2' };
+
+            await instance.perform(step1);
+            await instance.perform(step2);
+
+            expect(StepPerformer.prototype.perform).toHaveBeenLastCalledWith(step2, [step1]);
+        });
+    });
+
+    describe('reset', () => {
+        it('should clear previous steps', async () => {
+            Copilot.init(mockConfig);
+            const instance = Copilot.getInstance();
+            const step1: ExecutionStep = { type: 'action', value: 'tap button 1' };
+            const step2: ExecutionStep = { type: 'action', value: 'tap button 2' };
+
+            await instance.perform(step1);
+            instance.reset();
+            await instance.perform(step2);
+
+            expect(StepPerformer.prototype.perform).toHaveBeenLastCalledWith(step2, []);
+        });
+    });
+});

detox-copilot/src/Copilot.ts

@@ -0,0 +1,70 @@
+import {CopilotError} from "@/errors/CopilotError";
+import {PromptCreator} from "@/utils/PromptCreator";
+import {CodeEvaluator} from "@/utils/CodeEvaluator";
+import {SnapshotManager} from "@/utils/SnapshotManager";
+import {StepPerformer} from "@/actions/StepPerformer";
+
+/**
+ * The main Copilot class that provides AI-assisted testing capabilities for a given underlying testing framework.
+ * @note Originally, this class is designed to work with Detox, but it can be extended to work with other frameworks.
+ */
+export class Copilot {
+    // Singleton instance of Copilot
+    static instance?: Copilot;
+
+    private readonly promptCreator: PromptCreator;
+    private readonly codeEvaluator: CodeEvaluator;
+    private readonly snapshotManager: SnapshotManager;
+    private previousSteps: ExecutionStep[] = [];
+    private stepPerformer: StepPerformer;
+
+    private constructor(config: Config) {
+        this.promptCreator = new PromptCreator(config.frameworkDriver.availableAPI);
+        this.codeEvaluator = new CodeEvaluator();
+        this.snapshotManager = new SnapshotManager(config.frameworkDriver);
+        this.stepPerformer = new StepPerformer(this.promptCreator, this.codeEvaluator, this.snapshotManager, config.promptHandler);
+    }
+
+    /**
+     * Gets the singleton instance of Copilot.
+     * @returns The Copilot instance.
+     */
+    static getInstance(): Copilot {
+        if (!Copilot.instance) {
+            throw new CopilotError('Copilot has not been initialized. Please call the `init()` method before using it.');
+        }
+
+        return Copilot.instance;
+    }
+
+    /**
+     * Initializes the Copilot with the provided configuration, must be called before using Copilot.
+     * @param config The configuration options for Copilot.
+     */
+    static init(config: Config): void {
+        Copilot.instance = new Copilot(config);
+    }
+
+    /**
+     * Performs a test step based on the given prompt.
+     * @param step The step describing the operation to perform.
+     */
+    async perform(step: ExecutionStep): Promise<any> {
+        const result = await this.stepPerformer.perform(step, this.previousSteps);
+        this.didPerformStep(step);
+
+        return result;
+    }
+
+    /**
+     * Resets the Copilot by clearing the previous steps.
+     * @note This must be called before starting a new test flow, in order to clean context from previous tests.
+     */
+    reset(): void {
+        this.previousSteps = [];
+    }
+
+    private didPerformStep(step: ExecutionStep): void {
+        this.previousSteps = [...this.previousSteps, step];
+    }
+}

detox-copilot/src/actions/StepPerformer.test.ts

@@ -0,0 +1,111 @@
+import { StepPerformer } from '@/actions/StepPerformer';
+import { PromptCreator } from '@/utils/PromptCreator';
+import { CodeEvaluator } from '@/utils/CodeEvaluator';
+import { SnapshotManager } from '@/utils/SnapshotManager';
+
+jest.mock('@/utils/PromptCreator');
+jest.mock('@/utils/CodeEvaluator');
+jest.mock('@/utils/SnapshotManager');
+
+describe('StepPerformer', () => {
+    let stepPerformer: StepPerformer;
+    let mockPromptCreator: jest.Mocked<PromptCreator>;
+    let mockCodeEvaluator: jest.Mocked<CodeEvaluator>;
+    let mockSnapshotManager: jest.Mocked<SnapshotManager>;
+    let mockPromptHandler: jest.Mocked<PromptHandler>;
+
+    beforeEach(() => {
+        const availableAPI = { matchers: [], actions: [], assertions: [] };
+
+        mockPromptCreator = new PromptCreator(availableAPI) as jest.Mocked<PromptCreator>;
+        mockCodeEvaluator = new CodeEvaluator() as jest.Mocked<CodeEvaluator>;
+        mockSnapshotManager = new SnapshotManager({} as any) as jest.Mocked<SnapshotManager>;
+        mockPromptHandler = {
+            runPrompt: jest.fn(),
+            isSnapshotImageSupported: jest.fn().mockReturnValue(true)
+        } as jest.Mocked<PromptHandler>;
+
+        stepPerformer = new StepPerformer(
+            mockPromptCreator,
+            mockCodeEvaluator,
+            mockSnapshotManager,
+            mockPromptHandler
+        );
+    });
+
+    const createStep = (value: string): ExecutionStep => ({ type: 'action', value });
+
+    interface SetupMockOptions {
+        isSnapshotSupported?: boolean;
+        snapshotData?: string | null;
+        viewHierarchy?: string;
+        promptResult?: string;
+        codeEvaluationResult?: string;
+    }
+
+    const setupMocks = ({
+                            isSnapshotSupported = true,
+                            snapshotData = 'snapshot_data',
+                            viewHierarchy = '<view></view>',
+                            promptResult = 'generated code',
+                            codeEvaluationResult = 'success'
+                        }: SetupMockOptions = {}) => {
+        mockPromptHandler.isSnapshotImageSupported.mockReturnValue(isSnapshotSupported);
+        mockSnapshotManager.captureSnapshotImage.mockResolvedValue(snapshotData as string);
+        mockSnapshotManager.captureViewHierarchyString.mockResolvedValue(viewHierarchy);
+        mockPromptCreator.createPrompt.mockReturnValue('generated prompt');
+        mockPromptHandler.runPrompt.mockResolvedValue(promptResult);
+        mockCodeEvaluator.evaluate.mockResolvedValue(codeEvaluationResult);
+    };
+
+    it('should perform a step successfully with snapshot image support', async () => {
+        const step = createStep('tap button');
+        setupMocks();
+
+        const result = await stepPerformer.perform(step);
+
+        expect(result).toBe('success');
+        expect(mockPromptCreator.createPrompt).toHaveBeenCalledWith(step, '<view></view>', true, []);
+        expect(mockPromptHandler.runPrompt).toHaveBeenCalledWith('generated prompt', 'snapshot_data');
+        expect(mockCodeEvaluator.evaluate).toHaveBeenCalledWith('generated code');
+    });
+
+    it('should perform a step successfully without snapshot image support', async () => {
+        const step = createStep('tap button');
+        setupMocks({ isSnapshotSupported: false });
+
+        const result = await stepPerformer.perform(step);
+
+        expect(result).toBe('success');
+        expect(mockPromptCreator.createPrompt).toHaveBeenCalledWith(step, '<view></view>', false, []);
+    });
+
+    it('should perform a step with null snapshot', async () => {
+        const step = createStep('tap button');
+        setupMocks({ snapshotData: null });
+
+        const result = await stepPerformer.perform(step);
+
+        expect(result).toBe('success');
+        expect(mockPromptHandler.runPrompt).toHaveBeenCalledWith('generated prompt', null);
+    });
+
+    it('should perform a step successfully with previous steps', async () => {
+        const step = createStep('current step');
+        const previousSteps = [createStep('previous step')];
+        setupMocks();
+
+        const result = await stepPerformer.perform(step, previousSteps);
+
+        expect(result).toBe('success');
+        expect(mockPromptCreator.createPrompt).toHaveBeenCalledWith(step, '<view></view>', true, previousSteps);
+    });
+
+    it('should throw an error if code evaluation fails', async () => {
+        const step = createStep('tap button');
+        setupMocks();
+        mockCodeEvaluator.evaluate.mockRejectedValue(new Error('Evaluation failed'));
+
+        await expect(stepPerformer.perform(step)).rejects.toThrow('Evaluation failed');
+    });
+});