async/split on demand, and only if required

Signed-off-by: Maxence Lange <maxence@artificial-owl.com>

appinfo/routes.php

@@ -101,7 +101,8 @@
 	],
 
 	'routes' => [
-		['name' => 'EventWrapper#asyncBroadcast', 'url' => '/async/{token}/', 'verb' => 'POST'],
+		['name' => 'EventWrapper#asyncBroadcast', 'url' => '/async/broadcast/{token}', 'verb' => 'POST'],
+		['name' => 'EventWrapper#asyncInternal', 'url' => '/async/internal/{token}', 'verb' => 'POST'],
 
 		['name' => 'Remote#appService', 'url' => '/', 'verb' => 'GET'],
 		['name' => 'Remote#test', 'url' => '/test', 'verb' => 'GET'],
@@ -114,7 +115,7 @@
 		['name' => 'Remote#inherited', 'url' => '/inherited/{circleId}/', 'verb' => 'GET'],
 		['name' => 'Remote#memberships', 'url' => '/memberships/{circleId}/', 'verb' => 'GET'],
 
-		['name' => 'Sync#getSyncedItem', 'url' => '/sync/item', 'verb' => 'GET'],
+		['name' => 'Sync#syncItem', 'url' => '/sync/item', 'verb' => 'GET'],
 		['name' => 'Sync#updateSyncedItem', 'url' => '/sync/item', 'verb' => 'PUT'],
 //		['name' => 'Remote#syncItem', 'url' => '/sync/item/{singleId}', 'verb' => 'GET'],
 		['name' => 'Sync#syncShare', 'url' => '/sync/share', 'verb' => 'POST'],

lib/CircleSharesManager.php

@@ -34,6 +34,7 @@
 use Exception;
 use OCA\Circles\Exceptions\CircleSharesManagerException;
 use OCA\Circles\Model\Probes\CircleProbe;
+use OCA\Circles\Model\SyncedItemLock;
 use OCA\Circles\Service\CircleService;
 use OCA\Circles\Service\ConfigService;
 use OCA\Circles\Service\DebugService;
@@ -208,13 +209,30 @@ public function deleteShare(string $itemId, string $circleId): void {
 
 	/**
 	 * @param string $itemId
+	 * @param string $updateType
+	 * @param string $updateTypeId
 	 * @param array $extraData
+	 * @param bool $sumCheck
 	 *
 	 * @throws CircleSharesManagerException
+	 * @throws Exceptions\FederatedEventException
+	 * @throws Exceptions\FederatedItemException
+	 * @throws Exceptions\FederatedSyncConflictException
+	 * @throws Exceptions\FederatedSyncManagerNotFoundException
+	 * @throws Exceptions\InitiatorNotConfirmedException
+	 * @throws Exceptions\OwnerNotFoundException
+	 * @throws Exceptions\RemoteInstanceException
+	 * @throws Exceptions\RemoteNotFoundException
+	 * @throws Exceptions\RemoteResourceNotFoundException
+	 * @throws Exceptions\RequestBuilderException
+	 * @throws Exceptions\UnknownRemoteException
 	 */
 	public function updateItem(
 		string $itemId,
-		array $extraData = []
+		string $updateType = '',
+		string $updateTypeId = '',
+		array $extraData = [],
+		bool $sumCheck = true
 	): void {
 		$this->mustHaveOrigin();
 
@@ -226,20 +244,13 @@ public function updateItem(
 				'appId' => $this->originAppId,
 				'itemType' => $this->originItemType,
 				'itemId' => $itemId,
+				'updateType' => $updateType,
+				'updateTypeId' => $updateTypeId,
 				'extraData' => $extraData
 			]
 		);
 
 		try {
-//			$this->mustHaveOrigin();
-
-//			// TODO: verify rules that apply when sharing to a circle
-//			$probe = new CircleProbe();
-//			$probe->includeSystemCircles()
-//				  ->mustBeMember();
-//
-//			$circle = $this->circleService->getCircle($circleId, $probe);
-//
 			// get valid SyncedItem based on appId, itemType, itemId
 			$syncedItem = $this->federatedSyncItemService->initSyncedItem(
 				$this->originAppId,
@@ -260,6 +271,7 @@ public function updateItem(
 			$this->federatedSyncItemService->requestSyncedItemUpdate(
 				$this->federatedUserService->getCurrentEntity(),
 				$syncedItem,
+				new SyncedItemLock($updateType, $updateTypeId, $sumCheck),
 				$extraData
 			);
 		} catch (Exception $e) {

lib/Controller/EventWrapperController.php

@@ -32,6 +32,8 @@
 namespace OCA\Circles\Controller;
 
 use OCA\Circles\AppInfo\Application;
+use OCA\Circles\Exceptions\EventWrapperNotFoundException;
+use OCA\Circles\Service\AsyncService;
 use OCA\Circles\Service\ConfigService;
 use OCA\Circles\Service\EventWrapperService;
 use OCA\Circles\Service\FederatedEventService;
@@ -66,6 +68,8 @@ class EventWrapperController extends Controller {
 	/** @var RemoteDownstreamService */
 	private $remoteDownstreamService;
 
+	private AsyncService $asyncService;
+
 	/** @var ConfigService */
 	private $configService;
 
@@ -79,6 +83,7 @@ class EventWrapperController extends Controller {
 	 * @param FederatedEventService $federatedEventService
 	 * @param RemoteUpstreamService $remoteUpstreamService
 	 * @param RemoteDownstreamService $remoteDownstreamService
+	 * @param AsyncService $asyncService
 	 * @param ConfigService $configService
 	 */
 	public function __construct(
@@ -88,13 +93,15 @@ public function __construct(
 		FederatedEventService $federatedEventService,
 		RemoteUpstreamService $remoteUpstreamService,
 		RemoteDownstreamService $remoteDownstreamService,
+		AsyncService $asyncService,
 		ConfigService $configService
 	) {
 		parent::__construct($appName, $request);
 		$this->eventWrapperService = $eventWrapperService;
 		$this->federatedEventService = $federatedEventService;
 		$this->remoteUpstreamService = $remoteUpstreamService;
 		$this->remoteDownstreamService = $remoteDownstreamService;
+		$this->asyncService = $asyncService;
 		$this->configService = $configService;
 
 		$this->setup('app', Application::APP_ID);
@@ -105,7 +112,7 @@ public function __construct(
 	/**
 	 * Called locally.
 	 *
-	 * Async process and broadcast the event to every instances of GS
+	 * Async process and broadcast the event to every instance of GS
 	 * This should be initiated by the instance that owns the Circles.
 	 *
 	 * @PublicPage
@@ -116,19 +123,15 @@ public function __construct(
 	 * @return DataResponse
 	 */
 	public function asyncBroadcast(string $token): DataResponse {
-		$wrappers = $this->remoteUpstreamService->getEventsByToken($token);
+		$wrappers = $this->eventWrapperService->getBroadcastByToken($token);
 		if (empty($wrappers) && $token !== 'test-dummy-token') {
 			return new DataResponse([], Http::STATUS_OK);
 		}
 
 		// closing socket, keep current process running.
-		$this->async();
-
-		foreach ($wrappers as $wrapper) {
-			$this->eventWrapperService->manageWrapper($wrapper);
-		}
-
-		$this->eventWrapperService->confirmStatus($token);
+		$this->asyncService->setSplittable(true);
+		$this->asyncService->split();
+		$this->eventWrapperService->performBroadcast($token, $wrappers);
 
 		// so circles:check can check async is fine
 		if ($token === 'test-dummy-token') {
@@ -140,6 +143,31 @@ public function asyncBroadcast(string $token): DataResponse {
 	}
 
 
+	/**
+	 * Called locally.
+	 *
+	 * Async process and continue using IInternalAsync
+	 *
+	 * @PublicPage
+	 * @NoCSRFRequired
+	 *
+	 * @param string $token
+	 *
+	 * @return DataResponse
+	 * @throws EventWrapperNotFoundException
+	 */
+	public function asyncInternal(string $token): DataResponse {
+		$this->asyncService->setSplittable(true);
+		$this->asyncService->split();
+
+		$this->eventWrapperService->performInternal($token);
+
+		// exit() or useless log will be generated
+		exit();
+	}
+
+
+
 //	/**
 //	 * Status Event. This is an event to check status of items between instances.
 //	 *

lib/Controller/SyncController.php

@@ -35,6 +35,7 @@
 use OCA\Circles\Exceptions\FederatedItemBadRequestException;
 use OCA\Circles\Model\SyncedItem;
 use OCA\Circles\Model\SyncedWrapper;
+use OCA\Circles\Service\AsyncService;
 use OCA\Circles\Service\DebugService;
 use OCA\Circles\Service\FederatedSyncItemService;
 use OCA\Circles\Service\FederatedSyncShareService;
@@ -54,20 +55,23 @@ class SyncController extends Controller {
 	private FederatedSyncItemService $federatedSyncItemService;
 	private FederatedSyncShareService $federatedSyncShareService;
 	private DebugService $debugService;
+	private AsyncService $asyncService;
 
 	public function __construct(
 		string $appName,
 		IRequest $request,
 		SignedControllerService $signedControllerService,
 		FederatedSyncItemService $federatedSyncItemService,
 		FederatedSyncShareService $federatedSyncShareService,
+		AsyncService $asyncService,
 		DebugService $debugService
 	) {
 		parent::__construct($appName, $request);
 
 		$this->signedControllerService = $signedControllerService;
 		$this->federatedSyncItemService = $federatedSyncItemService;
 		$this->federatedSyncShareService = $federatedSyncShareService;
+		$this->asyncService = $asyncService;
 		$this->debugService = $debugService;
 	}
 
@@ -78,7 +82,7 @@ public function __construct(
 	 *
 	 * @return DataResponse
 	 */
-	public function getSyncedItem(): DataResponse {
+	public function syncItem(): DataResponse {
 		try {
 			/** @var SyncedItem $item */
 			$item = $this->signedControllerService->extractObjectFromRequest(
@@ -145,8 +149,8 @@ public function updateSyncedItem(): DataResponse {
 				throw new FederatedItemBadRequestException();
 			}
 
-			$item = $wrapper->getItem();
-			$local = $this->federatedSyncItemService->getLocalSyncedItem($item->getSingleId());
+			$syncedItem = $wrapper->getItem();
+			$local = $this->federatedSyncItemService->getLocalSyncedItem($syncedItem->getSingleId());
 
 			// confirm that remote is in a circle with a share on the item
 			$this->federatedSyncShareService->confirmRemoteInstanceAccess(
@@ -162,18 +166,25 @@ public function updateSyncedItem(): DataResponse {
 				]
 			);
 
-			$updated = $this->federatedSyncItemService->requestSyncedItemUpdate(
+			$this->asyncService->setSplittable(true);
+			$this->federatedSyncItemService->requestSyncedItemUpdate(
 				$wrapper->getFederatedUser(),
 				$local,
-				$wrapper->getExtraData()
+				$wrapper->getLock(),
+				$wrapper->getExtraData(),
+				$syncedItem->getChecksum()
 			);
 
-			return new DataResponse($updated);
+			if (!$this->asyncService->isAsynced()) {
+				return new DataResponse(['success' => true]);
+			}
 		} catch (Exception $e) {
 			$this->e($e);
 
 			return $this->signedControllerService->exceptionResponse($e, Http::STATUS_UNAUTHORIZED);
 		}
+
+		exit();
 	}
 
 }

lib/Db/CoreQueryBuilder.php

@@ -56,33 +56,33 @@ class CoreQueryBuilder extends ExtendedQueryBuilder {
 	use TArrayTools;
 
 
-	public const SINGLE = 'cs';
-	public const CIRCLE = 'cc';
-	public const MEMBER = 'mm';
-	public const OWNER = 'wn';
-	public const FEDERATED_EVENT = 'ev';
-	public const REMOTE = 'rm';
-	public const BASED_ON = 'on';
-	public const INITIATOR = 'in';
-	public const DIRECT_INITIATOR = 'di';
-	public const MEMBERSHIPS = 'ms';
-	public const CONFIG = 'cf';
-	public const UPSTREAM_MEMBERSHIPS = 'up';
-	public const INHERITANCE_FROM = 'ih';
-	public const INHERITED_BY = 'by';
-	public const INVITED_BY = 'nv';
-	public const MOUNT = 'mo';
-	public const MOUNTPOINT = 'mp';
-	public const SHARE = 'sh';
-	public const FILE_CACHE = 'fc';
-	public const STORAGES = 'st';
-	public const TOKEN = 'tk';
-	public const OPTIONS = 'pt';
-	public const HELPER = 'hp';
-	public const SYNC_ITEM = 'si';
-	public const SYNC_SHARE = 'ss';
-	public const SYNC_LOCK = 'sl';
-	public const DEBUG = 'bg';
+	public const SINGLE = 'ca';
+	public const CIRCLE = 'cb';
+	public const MEMBER = 'cc';
+	public const OWNER = 'cd';
+	public const FEDERATED_EVENT = 'ce';
+	public const REMOTE = 'cf';
+	public const BASED_ON = 'cg';
+	public const INITIATOR = 'ch';
+	public const DIRECT_INITIATOR = 'ci';
+	public const MEMBERSHIPS = 'cj';
+	public const CONFIG = 'ck';
+	public const UPSTREAM_MEMBERSHIPS = 'cl';
+	public const INHERITANCE_FROM = 'cm';
+	public const INHERITED_BY = 'cn';
+	public const INVITED_BY = 'co';
+	public const MOUNT = 'cp';
+	public const MOUNTPOINT = 'cq';
+	public const SHARE = 'cr';
+	public const FILE_CACHE = 'cs';
+	public const STORAGES = 'ct';
+	public const TOKEN = 'cu';
+	public const OPTIONS = 'cv';
+	public const HELPER = 'cw';
+	public const SYNC_ITEM = 'cx';
+	public const SYNC_SHARE = 'cy';
+	public const SYNC_LOCK = 'cz';
+	public const DEBUG = 'c0';
 
 
 	public static $SQL_PATH = [

lib/Db/CoreRequestBuilder.php

@@ -124,6 +124,8 @@ class CoreRequestBuilder {
 		self::TABLE_EVENT => [
 			'token',
 			'event',
+			'store',
+			'event_type',
 			'result',
 			'instance',
 			'interface',
@@ -160,7 +162,6 @@ class CoreRequestBuilder {
 		],
 		self::TABLE_SYNC_LOCK => [
 			'id',
-			'single_id',
 			'update_type',
 			'update_type_id',
 			'time'