Extending AI Actions

code_samples/ai_actions/assets/js/addAudioModule.js

@@ -0,0 +1,4 @@
+import { addModule } from '../../vendor/ibexa/connector-ai/src/bundle/Resources/public/js/core/create.ai.module';
+import TranscribeAudio from './transcribe.audio';
+
+addModule(TranscribeAudio);

code_samples/ai_actions/assets/js/transcribe.audio.js

@@ -0,0 +1,67 @@
+import BaseAIComponent from '../../vendor/ibexa/connector-ai/src/bundle/Resources/public/js/core/base.ai.component';
+
+export default class TranscribeAudio extends BaseAIComponent {
+    constructor(mainElement, config) {
+        super(mainElement, config);
+
+        this.requestHeaders = {
+            Accept: 'application/vnd.ibexa.api.ai.AudioText+json',
+            'Content-Type': 'application/vnd.ibexa.api.ai.TranscribeAudio+json',
+        };
+    }
+
+    getAudioInBase64() {
+        const request = new XMLHttpRequest();
+        request.open('GET', this.inputElement.href, false);
+        request.overrideMimeType('text/plain; charset=x-user-defined');
+        request.send();
+
+        if (request.status === 200) {
+            return this.convertToBase64(request.responseText);
+        }
+        else {
+            this.processError('Error occured when decoding the file.');
+        }
+    }
+
+    getRequestBody() {
+        const body = {
+            TranscribeAudio: {
+                Audio: {
+                    base64: this.getAudioInBase64(),
+                },
+                RuntimeContext: {},
+            },
+        };
+
+        if (this.languageCode) {
+            body.TranscribeAudio.RuntimeContext.languageCode = this.languageCode;
+        }
+
+        return JSON.stringify(body);
+    }
+
+    afterFetchData(response) {
+        super.afterFetchData();
+
+        if (response) {
+            this.outputElement.value = response.AudioText.Text.text[0];
+        }
+    }
+
+    toggle(forceEnabled) {
+        super.toggle(forceEnabled);
+
+        this.outputElement.disabled = !forceEnabled || !this.outputElement.disabled;
+    }
+
+    convertToBase64(data) {
+        let binary = '';
+
+        for (let i = 0; i < data.length; i++) {
+            binary += String.fromCharCode(data.charCodeAt(i) & 0xff);
+        }
+
+        return btoa(binary);
+    }
+}

code_samples/ai_actions/config/packages/ibexa_admin_ui.yaml

@@ -0,0 +1,7 @@
+ibexa:
+    system:
+        admin_group:
+            admin_ui_forms:
+                content_edit:
+                    form_templates:
+                        - { template: '@ibexadesign/admin/ui/fieldtype/edit/form_fields_binary_ai.html.twig', priority: -10 } }

code_samples/ai_actions/config/services.yaml

@@ -0,0 +1,81 @@
+# This file is the entry point to configure your own services.
+# Files in the packages/ subdirectory configure your dependencies.
+
+# Put parameters here that don't need to change on each machine where the app is deployed
+# https://symfony.com/doc/current/best_practices.html#use-parameters-for-application-configuration
+parameters:
+
+services:
+    # default configuration for services in *this* file
+    _defaults:
+        autowire: true      # Automatically injects dependencies in your services.
+        autoconfigure: true # Automatically registers your services as commands, event subscribers, etc.
+
+    # makes classes in src/ available to be used as services
+    # this creates a service per class whose id is the fully-qualified class name
+    App\:
+        resource: '../src/'
+        exclude:
+            - '../src/DependencyInjection/'
+            - '../src/Entity/'
+            - '../src/Kernel.php'
+
+    # add more service definitions when explicit configuration is needed
+    # please note that last definitions always *replace* previous ones
+
+    App\Command\AddMissingAltTextCommand:
+        arguments:
+            $projectDir: '%kernel.project_dir%'
+
+    App\AI\Handler\LLaVATextToTextActionHandler:
+        tags:
+            - { name: ibexa.ai.action.handler, priority: 0 }
+            - { name: ibexa.ai.action.handler.text_to_text, priority: 0 }
+
+    app.connector_ai.action_configuration.handler.llava_text_to_text.form_mapper.options:
+        class: Ibexa\Bundle\ConnectorAi\Form\FormMapper\ActionConfiguration\ActionHandlerOptionsFormMapper
+        arguments:
+            $formType: 'App\Form\Type\TextToTextOptionsType'
+        tags:
+            - name: ibexa.connector_ai.action_configuration.form_mapper.options
+              type: !php/const \App\AI\Handler\LLaVaTextToTextActionHandler::IDENTIFIER
+
+    App\AI\ActionType\TranscribeAudioActionType:
+        arguments:
+            $actionHandlers: !tagged_iterator
+                tag: app.connector_ai.action.handler.audio_to_text
+                default_index_method: getIdentifier
+                index_by: key
+        tags:
+            - { name: ibexa.ai.action.type, identifier: !php/const \App\AI\ActionType\TranscribeAudioActionType::IDENTIFIER }
+
+    app.connector_ai.action_configuration.handler.transcribe_audio.form_mapper.options:
+        class: Ibexa\Bundle\ConnectorAi\Form\FormMapper\ActionConfiguration\ActionTypeOptionsFormMapper
+        arguments:
+            $formType: 'App\Form\Type\TranscribeAudioOptionsType'
+        tags:
+            - name: ibexa.connector_ai.action_configuration.form_mapper.action_type_options
+              type: !php/const \App\AI\ActionType\TranscribeAudioActionType::IDENTIFIER
+
+    App\AI\Handler\WhisperAudioToTextActionHandler:
+        tags:
+            - { name: ibexa.ai.action.handler, priority: 0 }
+            - { name: app.connector_ai.action.handler.audio_to_text, priority: 0 }
+
+    Ibexa\Contracts\ConnectorAi\ActionConfiguration\OptionsFormatterInterface:
+        alias: Ibexa\ConnectorAi\ActionConfiguration\JsonOptionsFormatter
+
+#REST services
+    App\AI\REST\Input\Parser\TranscribeAudio:
+        parent: Ibexa\Rest\Server\Common\Parser
+        tags:
+            - { name: ibexa.rest.input.parser, mediaType: application/vnd.ibexa.api.ai.TranscribeAudio }
+
+    App\AI\REST\Output\Resolver\AudioTextResolver:
+        tags:
+            - { name: ibexa.ai.action.mime_type, key: application/vnd.ibexa.api.ai.AudioText }
+
+    App\AI\REST\Output\ValueObjectVisitor\AudioText:
+        parent: Ibexa\Contracts\Rest\Output\ValueObjectVisitor
+        tags:
+            - { name: ibexa.rest.output.value_object.visitor, type: App\AI\REST\Value\AudioText }

code_samples/ai_actions/src/AI/Action/TranscribeAudioAction.php

@@ -0,0 +1,33 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\AI\Action;
+
+use App\AI\DataType\Audio;
+use Ibexa\Contracts\ConnectorAi\Action\Action;
+
+final class TranscribeAudioAction extends Action
+{
+    private Audio $audio;
+
+    public function __construct(Audio $audio)
+    {
+        $this->audio = $audio;
+    }
+
+    public function getParameters(): array
+    {
+        return [];
+    }
+
+    public function getInput(): Audio
+    {
+        return $this->audio;
+    }
+
+    public function getActionTypeIdentifier(): string
+    {
+        return 'transcribe_audio';
+    }
+}

code_samples/ai_actions/src/AI/ActionType/TranscribeAudioActionType.php

@@ -0,0 +1,69 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\AI\ActionType;
+
+use App\AI\Action\TranscribeAudioAction;
+use App\AI\DataType\Audio;
+use Ibexa\Contracts\ConnectorAi\Action\DataType\Text;
+use Ibexa\Contracts\ConnectorAi\ActionInterface;
+use Ibexa\Contracts\ConnectorAi\ActionType\ActionTypeInterface;
+use Ibexa\Contracts\ConnectorAi\DataType;
+use Ibexa\Contracts\Core\Exception\InvalidArgumentException;
+
+final class TranscribeAudioActionType implements ActionTypeInterface
+{
+    public const IDENTIFIER = 'transcribe_audio';
+
+    /** @var iterable<\Ibexa\Contracts\ConnectorAi\Action\ActionHandlerInterface> */
+    private iterable $actionHandlers;
+
+    /** @param iterable<\Ibexa\Contracts\ConnectorAi\Action\ActionHandlerInterface> $actionHandlers*/
+    public function __construct(iterable $actionHandlers)
+    {
+        $this->actionHandlers = $actionHandlers;
+    }
+
+    public function getIdentifier(): string
+    {
+        return self::IDENTIFIER;
+    }
+
+    public function getName(): string
+    {
+        return 'Transcribe audio';
+    }
+
+    public function getInputIdentifier(): string
+    {
+        return Audio::getIdentifier();
+    }
+
+    public function getOutputIdentifier(): string
+    {
+        return Text::getIdentifier();
+    }
+
+    public function getOptions(): array
+    {
+        return [];
+    }
+
+    public function createAction(DataType $input, array $parameters = []): ActionInterface
+    {
+        if (!$input instanceof Audio) {
+            throw new InvalidArgumentException(
+                'audio',
+                'expected \App\AI\DataType\Audio type, ' . get_debug_type($input) . ' given.'
+            );
+        }
+
+        return new TranscribeAudioAction($input);
+    }
+
+    public function getActionHandlers(): iterable
+    {
+        return $this->actionHandlers;
+    }
+}

code_samples/ai_actions/src/AI/DataType/Audio.php

@@ -0,0 +1,39 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\AI\DataType;
+
+use Ibexa\Contracts\ConnectorAi\DataType;
+
+/**
+ * @implements DataType<string>
+ */
+final class Audio implements DataType
+{
+    /** @var non-empty-array<string> */
+    private array $base64;
+
+    /**
+     * @param non-empty-array<string> $base64
+     */
+    public function __construct(array $base64)
+    {
+        $this->base64 = $base64;
+    }
+
+    public function getBase64(): string
+    {
+        return reset($this->base64);
+    }
+
+    public function getList(): array
+    {
+        return $this->base64;
+    }
+
+    public static function getIdentifier(): string
+    {
+        return 'audio';
+    }
+}

code_samples/ai_actions/src/AI/Handler/LLaVaTextToTextActionHandler.php

@@ -0,0 +1,80 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\AI\Handler;
+
+use Ibexa\Contracts\ConnectorAi\Action\ActionHandlerInterface;
+use Ibexa\Contracts\ConnectorAi\Action\DataType\Text;
+use Ibexa\Contracts\ConnectorAi\Action\Response\TextResponse;
+use Ibexa\Contracts\ConnectorAi\Action\TextToText\Action as TextToTextAction;
+use Ibexa\Contracts\ConnectorAi\ActionInterface;
+use Ibexa\Contracts\ConnectorAi\ActionResponseInterface;
+use Symfony\Contracts\HttpClient\HttpClientInterface;
+
+final class LLaVaTextToTextActionHandler implements ActionHandlerInterface
+{
+    private HttpClientInterface $client;
+
+    private string $host;
+
+    public const IDENTIFIER = 'LLaVATextToText';
+
+    public function __construct(HttpClientInterface $client, string $host = 'http://localhost:8080')
+    {
+        $this->client = $client;
+        $this->host = $host;
+    }
+
+    public function supports(ActionInterface $action): bool
+    {
+        return $action instanceof TextToTextAction;
+    }
+
+    public function handle(ActionInterface $action, array $context = []): ActionResponseInterface
+    {
+        /** @var \Ibexa\Contracts\ConnectorAi\Action\DataType\Text */
+        $input = $action->getInput();
+        $text = $this->sanitizeInput($input->getText());
+
+        $systemMessage = $action->hasActionContext() ? $action->getActionContext()->getActionHandlerOptions()->get('system_prompt', '') : '';
+
+        $response = $this->client->request(
+            'POST',
+            sprintf('%s/v1/chat/completions', $this->host),
+            [
+                'headers' => [
+                    'Authorization: Bearer no-key',
+                ],
+                'json' => [
+                    'model' => 'LLaMA_CPP',
+                    'messages' => [
+                        (object)[
+                            'role' => 'system',
+                            'content' => $systemMessage,
+                        ],
+                        (object)[
+                            'role' => 'user',
+                            'content' => $text,
+                        ],
+                    ],
+                    'temperature' => 0.7,
+                ],
+            ]
+        );
+
+        $output = strip_tags(json_decode($response->getContent(), true)['choices'][0]['message']['content']);
+
+        return new TextResponse(new Text([$output]));
+    }
+
+    public static function getIdentifier(): string
+    {
+        return self::IDENTIFIER;
+    }
+
+    private function sanitizeInput(string $text): string
+    {
+        return str_replace(["\n", "\r"], ' ', $text);
+    }
+}