[WIP] POST API changes for integration

config/core.entity_form_display.reliefweb_post_api_provider.reliefweb_post_api_provider.default.yml

@@ -10,6 +10,7 @@ dependencies:
     - field.field.reliefweb_post_api_provider.reliefweb_post_api_provider.field_quota
     - field.field.reliefweb_post_api_provider.reliefweb_post_api_provider.field_rate_limit
     - field.field.reliefweb_post_api_provider.reliefweb_post_api_provider.field_resource_status
+    - field.field.reliefweb_post_api_provider.reliefweb_post_api_provider.field_skip_queue
     - field.field.reliefweb_post_api_provider.reliefweb_post_api_provider.field_source
     - field.field.reliefweb_post_api_provider.reliefweb_post_api_provider.field_user
     - field.field.reliefweb_post_api_provider.reliefweb_post_api_provider.field_webhook_url
@@ -74,6 +75,13 @@ content:
     region: content
     settings: {  }
     third_party_settings: {  }
+  field_skip_queue:
+    type: boolean_checkbox
+    weight: 14
+    region: content
+    settings:
+      display_label: true
+    third_party_settings: {  }
   field_source:
     type: reliefweb_entity_reference_select
     weight: 3

config/core.entity_view_display.reliefweb_post_api_provider.reliefweb_post_api_provider.default.yml

@@ -10,6 +10,7 @@ dependencies:
     - field.field.reliefweb_post_api_provider.reliefweb_post_api_provider.field_quota
     - field.field.reliefweb_post_api_provider.reliefweb_post_api_provider.field_rate_limit
     - field.field.reliefweb_post_api_provider.reliefweb_post_api_provider.field_resource_status
+    - field.field.reliefweb_post_api_provider.reliefweb_post_api_provider.field_skip_queue
     - field.field.reliefweb_post_api_provider.reliefweb_post_api_provider.field_source
     - field.field.reliefweb_post_api_provider.reliefweb_post_api_provider.field_user
     - field.field.reliefweb_post_api_provider.reliefweb_post_api_provider.field_webhook_url
@@ -90,6 +91,16 @@ content:
     third_party_settings: {  }
     weight: 2
     region: content
+  field_skip_queue:
+    type: boolean
+    label: above
+    settings:
+      format: default
+      format_custom_false: ''
+      format_custom_true: ''
+    third_party_settings: {  }
+    weight: 13
+    region: content
   field_source:
     type: entity_reference_label
     label: above
@@ -133,6 +144,15 @@ content:
     third_party_settings: {  }
     weight: 1
     region: content
+  status:
+    type: boolean
+    label: above
+    settings:
+      format: default
+      format_custom_false: ''
+      format_custom_true: ''
+    third_party_settings: {  }
+    weight: 12
+    region: content
 hidden:
   key: true
-  status: true

config/field.field.reliefweb_post_api_provider.reliefweb_post_api_provider.field_skip_queue.yml

@@ -0,0 +1,22 @@
+uuid: e61ae584-476f-4b69-a560-421db571f92b
+langcode: en
+status: true
+dependencies:
+  config:
+    - field.storage.reliefweb_post_api_provider.field_skip_queue
+  module:
+    - reliefweb_post_api
+id: reliefweb_post_api_provider.reliefweb_post_api_provider.field_skip_queue
+field_name: field_skip_queue
+entity_type: reliefweb_post_api_provider
+bundle: reliefweb_post_api_provider
+label: 'Skip queue'
+description: 'Skip the queue and process the submissions directly.'
+required: false
+translatable: false
+default_value: {  }
+default_value_callback: ''
+settings:
+  on_label: 'On'
+  off_label: 'Off'
+field_type: boolean

config/field.storage.reliefweb_post_api_provider.field_skip_queue.yml

@@ -0,0 +1,18 @@
+uuid: f0c717e9-6ca8-488f-935b-b61c4647394a
+langcode: en
+status: true
+dependencies:
+  module:
+    - reliefweb_post_api
+id: reliefweb_post_api_provider.field_skip_queue
+field_name: field_skip_queue
+entity_type: reliefweb_post_api_provider
+type: boolean
+settings: {  }
+module: core
+locked: false
+cardinality: 1
+translatable: true
+indexes: {  }
+persist_with_no_fields: false
+custom_storage: false

html/modules/custom/reliefweb_post_api/drush.services.yml

@@ -1,6 +1,6 @@
 services:
   reliefweb_post_api.commands:
     class: \Drupal\reliefweb_post_api\Commands\ReliefWebPostApiCommands
-    arguments: ['@queue', '@plugin.manager.reliefweb_post_api.content_processor']
+    arguments: ['@reliefweb_post_api.queue.database', '@plugin.manager.reliefweb_post_api.content_processor']
     tags:
       - { name: drush.command }

html/modules/custom/reliefweb_post_api/reliefweb_post_api.module

@@ -7,26 +7,17 @@
 
 declare(strict_types=1);
 
-use Drupal\Core\Entity\EntityInterface;
 use Drupal\reliefweb_post_api\Entity\Provider;
 
 /**
- * Implements hook_entity_create().
+ * Implements hook_reliefweb_api_post_indexing().
  */
-function reliefweb_post_api_entity_create(EntityInterface $entity): void {
-  Provider::notifyProvider($entity);
-}
-
-/**
- * Implements hook_entity_update().
- */
-function reliefweb_post_api_entity_update(EntityInterface $entity): void {
-  Provider::notifyProvider($entity);
-}
-
-/**
- * Implements hook_entity_delete().
- */
-function reliefweb_post_api_entity_delete(EntityInterface $entity): void {
-  Provider::notifyProvider($entity);
+function reliefweb_post_api_reliefweb_api_post_indexing(string $entity_type_id, string $bundle, string|int $id) {
+  // We notify the POST API provider, if any, after the document is indexed in
+  // the API otherwise the provider's webhook implementation will not be able to
+  // to retrieve the data from the API to confirm the status.
+  $entity = \Drupal::entityTypeManager()->getStorage($entity_type_id)->load($id);
+  if (!empty($entity)) {
+    Provider::notifyProvider($entity);
+  }
 }

html/modules/custom/reliefweb_post_api/reliefweb_post_api.services.yml

@@ -6,3 +6,7 @@ services:
     class: Drupal\reliefweb_post_api\Theme\ThemeNegotiator
     tags:
       - { name: theme_negotiator, priority: 0 }
+  reliefweb_post_api.queue.database:
+    class: Drupal\reliefweb_post_api\Queue\ReliefWebPostApiDatabaseQueueFactory
+    parent: queue.database
+    arguments: ['@datetime.time']

html/modules/custom/reliefweb_post_api/schemas/v2/report.json

@@ -119,6 +119,34 @@
                         "format": "iri",
                         "pattern": "\\.pdf$"
                     },
+                    "uuid": {
+                        "description": "The universally unique identifier (UUID) version 5 generated from the URL property above and the UUID of the document as namespace.",
+                        "type": "string",
+                        "format": "uuid"
+                    },
+                    "filename": {
+                        "description": "File name",
+                        "type": "string",
+                        "maxLength": 255,
+                        "allOf": [
+                            {
+                                "description": "Must contain letters (any language).",
+                                "pattern": "\\p{L}+"
+                            },
+                            {
+                                "description": "No control characters or separators except for spaces.",
+                                "pattern": "^([^\\p{Z}\\p{C}]|[ \u3000])+$"
+                            },
+                            {
+                                "description": "No system character: '<>:\"/\\|?*'.",
+                                "pattern": "^[^<>:\"/\\|?*]+$"
+                            }
+                        ],
+                        "not": {
+                            "description": "No leading, trailing or consecutive spaces.",
+                            "pattern": "(?:^[ \u3000]|[ \u3000]$|[ \u3000]{2,})"
+                        }
+                    },
                     "checksum": {
                         "description": "SHA-256 checksum of the file",
                         "type": "string",
@@ -149,19 +177,24 @@
                         "enum": ["ar", "en", "es", "fr", "ru"]
                     }
                 },
-                "required": ["url", "checksum"]
+                "required": ["url", "uuid", "filename", "checksum"]
             },
             "maxItems": 10
         },
         "image": {
-            "description": "Image to illustrate the document with a minimum width of 700px.",
+            "description": "Image to illustrate the document with a minimum width of 700px and a maximum size of 5MB.",
             "type": "object",
             "properties": {
                 "url": {
                     "type": "string",
                     "format": "iri",
                     "pattern": "\\.(jpg|png|webp)$"
                 },
+                "uuid": {
+                    "description": "The universally unique identifier (UUID) version 5 generated from the URL property above and the UUID of the document as namespace.",
+                    "type": "string",
+                    "format": "uuid"
+                },
                 "checksum": {
                     "description": "SHA-256 checksum of the image",
                     "type": "string",
@@ -206,7 +239,7 @@
                     }
                 }
             },
-            "required": ["url", "checksum", "description", "copyright"]
+            "required": ["url", "uuid", "checksum", "description", "copyright"]
         },
         "disaster": {
             "description": "Document disaster(s) as a list of IDs from https://api.reliefweb.int/v1/disasters.",
@@ -253,5 +286,5 @@
         "published",
         "body"
     ],
-    "additionalProperties": false
+    "unevaluatedProperties": false
 }

html/modules/custom/reliefweb_post_api/src/Commands/ReliefWebPostApiCommands.php

@@ -4,8 +4,8 @@
 
 namespace Drupal\reliefweb_post_api\Commands;
 
-use Drupal\Core\Queue\QueueFactory;
 use Drupal\reliefweb_post_api\Plugin\ContentProcessorPluginManagerInterface;
+use Drupal\reliefweb_post_api\Queue\ReliefWebPostApiDatabaseQueueFactory;
 use Drush\Commands\DrushCommands;
 
 /**
@@ -14,10 +14,15 @@
 class ReliefWebPostApiCommands extends DrushCommands {
 
   /**
-   * {@inheritdoc}
+   * Constructor.
+   *
+   * @param \Drupal\reliefweb_post_api\Queue\ReliefWebPostApiDatabaseQueueFactory $queueFactory
+   *   The ReliefWeb POST API queue factory.
+   * @param \Drupal\reliefweb_post_api\Plugin\ContentProcessorPluginManagerInterface $contentProcessorPluginManager
+   *   The ReliefWeb POST API content processor plugin manager.
    */
   public function __construct(
-    protected QueueFactory $queueFactory,
+    protected ReliefWebPostApiDatabaseQueueFactory $queueFactory,
     protected ContentProcessorPluginManagerInterface $contentProcessorPluginManager,
   ) {}
 

html/modules/custom/reliefweb_post_api/src/Controller/ReliefWebPostApi.php

@@ -8,9 +8,10 @@
 use Drupal\Core\Controller\ControllerBase;
 use Drupal\Core\Database\Connection;
 use Drupal\Core\Extension\ExtensionPathResolver;
-use Drupal\Core\Queue\QueueFactory;
 use Drupal\reliefweb_post_api\Entity\ProviderInterface;
+use Drupal\reliefweb_post_api\Plugin\ContentProcessorException;
 use Drupal\reliefweb_post_api\Plugin\ContentProcessorPluginManagerInterface;
+use Drupal\reliefweb_post_api\Queue\ReliefWebPostApiDatabaseQueueFactory;
 use Symfony\Component\DependencyInjection\ContainerInterface;
 use Symfony\Component\HttpFoundation\JsonResponse;
 use Symfony\Component\HttpFoundation\RequestStack;
@@ -33,8 +34,8 @@ class ReliefWebPostApi extends ControllerBase {
    *
    * @param \Symfony\Component\HttpFoundation\RequestStack $requestStack
    *   The request stack.
-   * @param \Drupal\Core\Queue\QueueFactory $queueFactory
-   *   The queue factory.
+   * @param \Drupal\reliefweb_post_api\Queue\ReliefWebPostApiDatabaseQueueFactory $queueFactory
+   *   The ReliefWeb POST API queue factory.
    * @param \Drupal\Core\Extension\ExtensionPathResolver $pathResolver
    *   The path resolver service.
    * @param \Drupal\Core\Database\Connection $database
@@ -46,7 +47,7 @@ class ReliefWebPostApi extends ControllerBase {
    */
   public function __construct(
     protected RequestStack $requestStack,
-    protected QueueFactory $queueFactory,
+    protected ReliefWebPostApiDatabaseQueueFactory $queueFactory,
     protected ExtensionPathResolver $pathResolver,
     protected Connection $database,
     protected TimeInterface $time,
@@ -59,7 +60,9 @@ public function __construct(
   public static function create(ContainerInterface $container) {
     return new static(
       $container->get('request_stack'),
-      $container->get('queue'),
+      // We use our own queue that stores unique submissions based on the
+      // resource UUID.
+      $container->get('reliefweb_post_api.queue.database'),
       $container->get('extension.path.resolver'),
       $container->get('database'),
       $container->get('datetime.time'),
@@ -173,16 +176,32 @@ public function postContent(string $resource, string $uuid): JsonResponse {
         throw new BadRequestHttpException("Invalid data:\n\n" . $exception->getMessage());
       }
 
-      // Queue the data so it can be processed later (ex: drush command).
-      try {
-        $queue = $this->queueFactory->get('reliefweb_post_api');
-        $queue->createItem($data);
-      }
-      catch (\Exception $exception) {
-        throw new HttpException(500, 'Internal server error.');
+      // Process the document directly.
+      if ($provider->skipQueue()) {
+        try {
+          $plugin->process($data);
+        }
+        catch (ContentProcessorException $exception) {
+          throw new BadRequestHttpException("Invalid data:\n\n" . $exception->getMessage());
+        }
+        catch (\Exception $exception) {
+          throw new HttpException(500, 'Internal server error.');
+        }
+
+        $response = new JsonResponse('Document processed.', 200);
       }
+      // Queue the data so it can be processed later (ex: drush command).
+      else {
+        try {
+          $queue = $this->queueFactory->get('reliefweb_post_api');
+          $queue->createItem($data);
+        }
+        catch (\Exception $exception) {
+          throw new HttpException(500, 'Internal server error.');
+        }
 
-      $response = new JsonResponse('Document queued for processing.', 200);
+        $response = new JsonResponse('Document queued for processing.', 202);
+      }
     }
     catch (HttpException $exception) {
       $response = new JsonResponse($exception->getMessage(), $exception->getStatusCode());