Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feature/CONNECTOR-30/caal-service #5

Merged
merged 41 commits into from
Oct 3, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
83dfcfe
added more formfields to editJob and updated migration
remko48 Oct 1, 2024
967b358
Updated synchronisation
remko48 Oct 1, 2024
9cc6ca5
Merge remote-tracking branch 'origin/feature/CONNECTOR-17/job-dialog'…
rubenvdlinde Oct 1, 2024
89fba5c
fixing the ts objects
rubenvdlinde Oct 1, 2024
aa5c807
Small fixes
rubenvdlinde Oct 1, 2024
a181bab
updated arguments
remko48 Oct 2, 2024
4f73ed7
updated arguments
remko48 Oct 2, 2024
614fe4e
Merge remote-tracking branch 'origin/feature/jobs-arguments' into fea…
rubenvdlinde Oct 2, 2024
04f82b4
Hotfix
rubenvdlinde Oct 2, 2024
015991f
Added refresh
remko48 Oct 2, 2024
98c26f0
Merge remote-tracking branch 'origin/feature/jobs-arguments' into fea…
rubenvdlinde Oct 2, 2024
27c7124
Auto stash before merge of "feature/CONNECTOR-30/caal-service" and "o…
rubenvdlinde Oct 2, 2024
af3d305
Add more functionality to log entities
rubenvdlinde Oct 2, 2024
f49bfe4
Stupid database fix
rubenvdlinde Oct 2, 2024
40b7ad3
CSS updates
remko48 Oct 2, 2024
6148f71
Added bugfix on DeleteJobArgument
remko48 Oct 2, 2024
6a9390d
updated sourcelog
remko48 Oct 2, 2024
c8cfc08
Hotfix: Remove twig for now
rubenvdlinde Oct 2, 2024
1824722
OKe die was duf
rubenvdlinde Oct 2, 2024
c2de35f
Added configurations to source
remko48 Oct 2, 2024
152f4b1
Updated editSource
remko48 Oct 2, 2024
616abc6
Added view log
remko48 Oct 2, 2024
a658411
Updated deleteSource
remko48 Oct 2, 2024
886de6c
Merge branch 'feature/configuration-source' into feature/CONNECTOR-30…
remko48 Oct 2, 2024
31ff3cf
Fixed console error
remko48 Oct 2, 2024
9c79eeb
Fixes for synchronysation
rubenvdlinde Oct 2, 2024
addb12c
small oversight
rubenvdlinde Oct 2, 2024
1251cc6
Merge remote-tracking branch 'origin/feature/CONNECTOR-40/source-logs…
rubenvdlinde Oct 2, 2024
5615332
Bit more loging and cleanup
rubenvdlinde Oct 2, 2024
db6d65f
Wierd constant fix
rubenvdlinde Oct 2, 2024
ee45376
Bit of improvements
rubenvdlinde Oct 2, 2024
3237de2
Fixed a typo
rubenvdlinde Oct 2, 2024
49f96e9
Kleine grafische fixes
rubenvdlinde Oct 2, 2024
f1d5f63
Merge branch 'feature/CONNECTOR-30/caal-service' into feature/CONNECT…
remko48 Oct 3, 2024
0b990a9
Updated mapping to fit new key value flow
remko48 Oct 3, 2024
c71b5d3
Merge branch 'feature/CONNECTOR-21/mappings' into feature/CONNECTOR-3…
remko48 Oct 3, 2024
74d788e
cleanup
remko48 Oct 3, 2024
5a5d553
Merge branch 'feature/CONNECTOR-30/caal-service' into feature/CONNECT…
remko48 Oct 3, 2024
a402dd2
css update
remko48 Oct 3, 2024
a5963ca
Merge branch 'feature/CONNECTOR-21/mappings' into feature/CONNECTOR-3…
remko48 Oct 3, 2024
fa7f88e
Merge remote-tracking branch 'origin/development' into feature/CONNEC…
remko48 Oct 3, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions lib/Action/SynchronizationAction.php
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,10 @@ public function __construct(
//@todo: make this a bit more generic :')
public function run($argument)
{
//@todo: for testing purposes
if (!isset($argument['synchronizationId'])) {
$argument['synchronizationId'] = 1;
}

$response = [];

Expand Down Expand Up @@ -59,9 +63,9 @@ public function run($argument)

// Doing the synchronization
$response['stackTrace'][] = 'Doing the synchronization';
$this->synchronizationService->synchronize($synchronization);
$objects = $this->synchronizationService->synchronize($synchronization);

// @todo: implement this
$response['stackTrace'][] = 'Synchronized '.count($objects).' successfully';

// Lets report back about what we have just done
return $response;
Expand Down
4 changes: 2 additions & 2 deletions lib/Db/JobLog.php
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,8 @@ public function jsonSerialize(): array
'arguments' => $this->arguments,
'executionTime' => $this->executionTime,
'userId' => $this->userId,
'stackTrace' => $this->trace,
'expires' => $this->lastRun,
'stackTrace' => $this->stackTrace,
'expires' => $this->expires,
'lastRun' => $this->lastRun,
'nextRun' => $this->nextRun,
'created' => $this->created,
Expand Down
12 changes: 3 additions & 9 deletions lib/Db/SynchronizationContract.php
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,7 @@
*/
class SynchronizationContract extends Entity implements JsonSerializable
{
protected ?string $name = null; // The name of the contract
protected ?string $description = null; // The description of the contract
protected ?string $synchronization = null; // The synchronization that this contract belongs to
protected ?string $synchronizationId = null; // The synchronization that this contract belongs to
// Source
protected ?string $sourceId = null; // The id of the object in the source
protected ?string $sourceHash = null; // The hash of the object in the source
Expand All @@ -34,9 +32,7 @@ class SynchronizationContract extends Entity implements JsonSerializable


public function __construct() {
$this->addType('name', 'string');
$this->addType('description', 'string');
$this->addType('synchronization', 'string');
$this->addType('synchronizationId', 'string');
$this->addType('sourceId', 'string');
$this->addType('sourceHash', 'string');
$this->addType('sourceLastChanged', 'datetime');
Expand Down Expand Up @@ -85,9 +81,7 @@ public function jsonSerialize(): array
{
return [
'id' => $this->id,
'name' => $this->name,
'description' => $this->description,
'synchronization' => $this->synchronization,
'synchronizationId' => $this->synchronization,
'sourceId' => $this->sourceId,
'sourceHash' => $this->sourceHash,
'sourceLastChanged' => $this->sourceLastChanged,
Expand Down
17 changes: 13 additions & 4 deletions lib/Db/SynchronizationContractMapper.php
Original file line number Diff line number Diff line change
Expand Up @@ -28,20 +28,24 @@ public function find(int $id): SynchronizationContract
return $this->findEntity(query: $qb);
}

public function findOnSource(string $target, string $sourceId): SynchronizationContract|bool
public function findOnSynchronizationIdSourceId(string $synchronizationId, string $sourceId): ?SynchronizationContract
{
$qb = $this->db->getQueryBuilder();

$qb->select('*')
->from('openconnector_synchronization_contracts')
->where(
$qb->expr()->eq('synchronization_id', $qb->createNamedParameter($synchronization))
$qb->expr()->eq('synchronization_id', $qb->createNamedParameter($synchronizationId))
)
->andWhere(
$qb->expr()->eq('source_id', $qb->createNamedParameter($sourceId))
);

return $this->findEntity(query: $qb);
try {
return $this->findEntity($qb);
} catch (\OCP\AppFramework\Db\DoesNotExistException $e) {
return null;
}
}


Expand All @@ -58,7 +62,12 @@ public function findOnTarget(string $synchronization, string $targetId): Synchro
$qb->expr()->eq('target_id', $qb->createNamedParameter($targetId))
);

return $this->findEntity(query: $qb);

try {
return $this->findEntity($qb);
} catch (\OCP\AppFramework\Db\DoesNotExistException $e) {
return null;
}
}


Expand Down
2 changes: 0 additions & 2 deletions lib/Migration/Version0Date20240826193657.php
Original file line number Diff line number Diff line change
Expand Up @@ -222,8 +222,6 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt
if (!$schema->hasTable('openconnector_synchronization_contracts')) {
$table = $schema->createTable('openconnector_synchronization_contracts');
$table->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'length' => 20]);
$table->addColumn('name', Types::STRING, ['notnull' => true, 'length' => 255]);
$table->addColumn('description', Types::TEXT, ['notnull' => false]);
$table->addColumn('synchronization_id', Types::STRING, ['notnull' => true, 'length' => 255]);
// Source
$table->addColumn('source_id', Types::STRING, ['notnull' => false, 'length' => 255]);
Expand Down
95 changes: 50 additions & 45 deletions lib/Service/SynchronizationService.php
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

namespace OCA\OpenConnector\Service;

use OCA\OpenConnector\Db\CallLog;
use OCA\OpenConnector\Db\Source;
use OCA\OpenConnector\Db\SourceMapper;
use OCA\OpenConnector\Db\Synchronization;
Expand Down Expand Up @@ -53,78 +54,76 @@ public function __construct(
*/
public function synchronize(Synchronization $synchronization)
{
$this->synchronization = $synchronization;
$objectList = $this->getAllObjectsFromSource($synchronization);

foreach($objectList as $object) {
foreach($objectList as $key => $object) {
// Get the synchronization contract for this object
$synchronizationContract = $this->synchronizationContractMapper->findOnSource($synchronization->id, $object['id']);
if(!$synchronizationContract) {
$synchronizationContract = $this->synchronizationContractMapper->findOnSynchronizationIdSourceId($synchronization->id, $object['id']);

if (!($synchronizationContract instanceof SynchronizationContract)) {
$synchronizationContract = new SynchronizationContract();
$synchronizationContract->setSynchronizationId($synchronization->id);
$synchronizationContract->setSourceId($object['id']);
$synchronizationContract->setSourceHash(md5(serialize($object)));
// @todo: should we do this here
$this->synchronizationContractMapper->insert($synchronizationContract);
}

$synchronizationContract = $this->synchronizeContract($synchronizationContract, $synchronization, $object);
$objectList[$key] = $this->synchronizationContractMapper->insert($synchronizationContract);

$this->synchronizeContract($synchronizationContract);
}
else{
// @todo this is wierd
$synchronizationContract = $this->synchronizeContract($synchronizationContract, $synchronization, $object);
$objectList[$key] = $this->synchronizationContractMapper->update($synchronizationContract);
}
}

return $objectList;
}

/**
* @param SynchronizationContract $synchronizationContract
* @return void
* Synchronize a contract
* @param SynchronizationContract $synchronizationContract
* @param Synchronization $synchronization
* @param array $object
*
* @return SynchronizationContract
*/
public function synchronizeContract(SynchronizationContract $synchronizationContract, $object = null)
public function synchronizeContract(SynchronizationContract $synchronizationContract, Synchronization $synchronization = null, array $object = [])
{
// The function can be called solo set let's make sure we have the full synchronization object
if(!$this->synchronization){
$this->synchronization = $this->synchronizationMapper->findById($synchronizationContract->getSynchronizationId());
}

// We should have an object but lets make sure we have the full object
if(!$object){
$object = $this->getAllObjectsFromSource($synchronizationContract);
}

// Let create a source hash for the object
$sourceHash = md5(serialize($object));
$synchronizationContract->sourceLastChecked(new DateTime());
$synchronizationContract->setSourceLastChecked(new DateTime());

// Lets prevent pointless updates @todo acount for omnidirectional sync
if($sourceHash === $synchronizationContract->getSourceHash()){
// The object has not changed
return $this->synchronizationContractMapper->update($synchronizationContract);
return $synchronizationContract; // Fix: Add $ before synchronizationContract
}

// The object has changed, oke let do mappig and bla die bla
$synchronizationContract->setSourceHash($sourceHash);
$synchronizationContract->sourceLastChanged(new DateTime());
$synchronizationContract->setSourceLastChanged(new DateTime());

// let do the mapping if provided
if($synchronizationContract->getSourceTargetMapping()){
$targetObject = $this->mappingService->mapping($synchronizationContract->getSourceTargetMapping(), $object);
if($synchronization->getSourceTargetMapping()){
$targetObject = $this->mappingService->mapping($synchronization->getSourceTargetMapping(), $object);
}
else{
$targetObject = $object;
}


// set the target hash
$targetHash = md5(serialize($targetObject));
$synchronizationContract->setTargetHash($targetHash);
$synchronizationContract->targetLastChanged(new DateTime());
$synchronizationContract->targetLastSynced(new DateTime());
$synchronizationContract->sourceLastSynced(new DateTime());
$synchronizationContract->setTargetLastChanged(new DateTime());
$synchronizationContract->setTargetLastSynced(new DateTime());
$synchronizationContract->setSourceLastSynced(new DateTime());

// Do the magic!!

$this->updateTarget($synchronizationContract, $targetObject);

// Save results
$this->synchronizationContractMapper->update($synchronizationContract);

return $synchronizationContract;

}
Expand All @@ -137,9 +136,9 @@ public function synchronizeContract(SynchronizationContract $synchronizationCont
*/
public function updateTarget(SynchronizationContract $synchronizationContract, array $targetObject)
{
// The function can be called solo set let's make sure we have the full synchronization object
if(!$this->synchronization){
$this->synchronization = $this->synchronizationMapper->findById($synchronizationContract->getSynchronizationId());
// The function can be called solo set let's make sure we have the full synchronization object
if(!$synchronization){
$synchronization = $this->synchronizationMapper->find($synchronizationContract->getSynchronizationId());
}

// Lets check if we need to create or update
Expand All @@ -148,22 +147,21 @@ public function updateTarget(SynchronizationContract $synchronizationContract, a
$update = true;
}

$type = $synchronizationContract->getTargetType();
$type = $synchronization->getTargetType();

switch($type){
case 'register/schema':
// Setup the object service
$this->objectService = $this->containerInterface->get('OCA\OpenRegister\Service\ObjectService');
$objectService = $this->containerInterface->get('OCA\OpenRegister\Service\ObjectService');
// if we alreadey have an id, we need to get the object and update it
if($synchronizationContract->getTargetId()){
$targetObject['id'] = $synchronizationContract->getTargetId();
}
// Extract register and schema from the targetId
$targetId = $this->synchronization->getTargetId();
$targetId = $synchronization->getTargetId();
list($register, $schema) = explode('/', $targetId);

// Save the object to the target
$target = $this->objectService->saveObject($register, $schema, $targetObject);
$target = $objectService->saveObject($register, $schema, $targetObject);
// Get the id form the target object
$synchronizationContract->setTargetId($target->getUuid());
break;
Expand All @@ -174,6 +172,8 @@ public function updateTarget(SynchronizationContract $synchronizationContract, a
case 'database':
//@todo: implement
break;
default:
throw new \Exception("Unsupported target type: $type");
}
}

Expand All @@ -186,6 +186,9 @@ public function updateTarget(SynchronizationContract $synchronizationContract, a
public function getAllObjectsFromSource(Synchronization $synchronization)
{
$objects = [];

$type = $synchronization->getSourceType();

switch($type){
case 'register/schema':
// Setup the object service
Expand All @@ -194,9 +197,9 @@ public function getAllObjectsFromSource(Synchronization $synchronization)
break;
case 'api':
//@todo: implement
$source = $this->sourceMapper->get($synchronization->getSourceUrl());
$sourceObject = $this->callService->get($source->getUrl());
$objects[] = $this->getAllObjectsFromJson($sourceObject, synchronization, $source);
$source = $this->sourceMapper->find($synchronization->getSourceId());
$sourceObject = $this->callService->call($source);
$objects = $this->getAllObjectsFromArray($sourceObject, $source);
break;
case 'database':
//@todo: implement
Expand All @@ -205,11 +208,13 @@ public function getAllObjectsFromSource(Synchronization $synchronization)
return $objects;
}

public function getAllObjectsFromArray(array $sourceObject, Synchronization $synchronization, Source $source)
public function getAllObjectsFromArray(CallLog $callLog, Source $source)
{

// lekeer hacky (only works on github for now)
//$sourceObject = $this->callService->get($source->getUrl());
return $sourceObject['items'];
$response = $callLog->getResponse();
$body = json_decode($response['body'], true);
return $body['items'];
}
}
2 changes: 1 addition & 1 deletion src/modals/mappingMapping/EditMappingMapping.vue
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import { mappingStore, navigationStore } from '../../store/store.js'
<div class="form-group">
<NcTextField
id="key"
label="Key"
label="Key*"
required
:error="checkIfKeyIsUnique(mappingItem.key)"
:helper-text="checkIfKeyIsUnique(mappingItem.key) ? 'This key is already in use. Please choose a different key name.' : ''"
Expand Down
3 changes: 3 additions & 0 deletions src/views/Job/JobDetails.vue
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,9 @@ export default {
jobStore.setJobArgumentKey(false)
} else { jobStore.setJobArgumentKey(jobArgumentKey) }
},
refreshJobLogs() {
jobStore.refreshJobLogs()
},
},
}
</script>
Expand Down
2 changes: 1 addition & 1 deletion src/views/Mapping/MappingDetails.vue
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import { mappingStore, navigationStore } from '../../store/store.js'
</template>
Edit
</NcActionButton>
<NcActionButton @click="navigationStore.setModal('editMappingMapping')">
<NcActionButton @click="addMappingMapping()">
<template #icon>
<MapPlus :size="20" />
</template>
Expand Down
2 changes: 1 addition & 1 deletion src/views/Source/SourceDetails.vue
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ import { sourceStore, navigationStore, logStore } from '../../store/store.js'
<NcListItem v-for="(log, i) in sourceStore.sourceLogs"
:key="log.id + i"
:class="checkIfStatusIsOk(log.statusCode) ? 'okStatus' : 'errorStatus'"
:name="log.response.body"
:name="`${log.statusMessage} (response time: ${(log.response.responseTime / 1000).toFixed(3)} seconds)`"
:bold="false"
:counter-number="log.statusCode"
:force-display-actions="true"
Expand Down