feat: Add scripts to enbale neo4j persistance after tao install #3907

yaraslau-kavaliou · 2023-10-12T12:35:31Z

Ticket - https://oat-sa.atlassian.net/browse/REL-1294

In order to setup neo4j ontology persistence right after tao installation I propose to use already existing migration script with some additional scripts for setup. As long we are going to support sql database along with neo4j this approach has to be valid and allow us to move forward.

How to test:

Use composer.json and seed.json (only postInstall scripts can be borrowed) files provided
Call taoDockerize install to install tao
Authoring should work without issues and use neo4j database

seed.json file which I used to install tao:

{
  "seed": {
    "identifier": "studio-neo4j"
  },
  "extensions": [
    "generis",
    "tao",
    "funcAcl",
    "taoBackOffice",
    "taoDacSimple",
    "taoDeliverConnect",
    "taoDelivery",
    "taoDeliveryRdf",
    "taoGroups",
    "taoItems",
    "taoLti",
    "taoLtiConsumer",
    "taoMediaManager",
    "taoOauth",
    "taoQtiItem",
    "taoQtiTest",
    "taoQtiTestPreviewer",
    "taoResultServer",
    "taoScheduler",
    "taoStyles",
    "taoSystemStatus",
    "taoTaskQueue",
    "taoTestPreviewUILoader",
    "taoTests",
    "taoTestTaker",
    "xmlEdit",
    "xmlEditRp",
    "qtiItemPci",
    "taoAdvancedSearch",
    "taoStudio"
  ],
  "super-user": {
    "login": "admin",
    "firstname": "John",
    "lastname": "Doe",
    "email": "",
    "password": "admin"
  },
  "configuration": {
    "global": {
      "lang": "en-US",
      "mode": "debug",
      "instance_name": "studio-neo4j",
      "namespace": "https://studio-neo4j.docker.localhost/ontologies/tao.rdf",
      "url": "https://studio-neo4j.docker.localhost/",
      "session_name": "tao_studio-neo4j",
      "timezone": "Europe/Luxembourg",
      "import_data": true
    },
    "generis": {
      "persistences": {
        "default": {
          "driver": "pdo_mysql",
          "host": "studio-neo4j-mariadb",
          "port": 3306,
          "dbname": "studio-neo4j",
          "user": "root",
          "password": "r00t"
        },
        "neo4j": {
          "driver": "phpneo4j",
          "host": "studio-neo4j-neo4j",
          "port": 7687,
          "dbname": "neo4j",
          "user": "neo4j",
          "password": "neo4studio-neo4j"
        },
        "redis": {
          "driver": "phpredis",
          "host": "studio-neo4j-redis",
          "port": 6379
        },
        "authKeyValue": {
          "driver": "phpredis",
          "host": "studio-neo4j-redis",
          "port": 6379
        },
        "keyValueResult": {
          "driver": "phpredis",
          "host": "studio-neo4j-redis",
          "port": 6379
        }
      },
      "log": {
        "type": "configurableService",
        "class": "oat\\oatbox\\log\\LoggerService",
        "options": {
          "logger": {
            "class": "oat\\oatbox\\log\\logger\\TaoMonolog",
            "options": {
              "name": "tao",
              "handlers": [
                {
                  "class": "Monolog\\Handler\\StreamHandler",
                  "options": [
                    "/var/www/html/data/tao/log/tao-community.log",
                    100
                  ]
                }
              ]
            }
          }
        }
      }
    }
  },
  "postInstall": [
    {
      "class": "oat\\tao\\scripts\\install\\CleanGraphDatabase",
      "params": [
        "neo4j"
      ]
    },
    {
      "class": "oat\\tao\\scripts\\install\\EnableGraphDatabase",
      "params": [
        "neo4j"
      ]
    },
    {
      "class": "oat\\tao\\scripts\\tools\\MigrateSqlToNeo4j",
      "params": [
        "-u",
        "-i",
        "-s",
        "10000",
        "-n",
        "10000",
        "-vvvv"
      ]
    }
  ]
}

composer.json I used:

{
  "name": "oat-sa/tao-studio",
  "license": "GPL-2.0-only",
  "config": {
    "platform": {
      "php": "7.4"
    },
    "preferred-install": "source",
    "allow-plugins": {
      "oat-sa/oatbox-extension-installer": true,
      "oat-sa/composer-npm-bridge": true,
      "php-http/discovery": true
    }
  },
  "support": {
    "issues": "http://forge.taotesting.com",
    "forum": "http://forum.taotesting.com"
  },
  "authors": [
    {
      "homepage": "http://www.taotesting.com",
      "name": "Open Assessment Technologies S.A."
    }
  ],
  "keywords": [
    "tao",
    "oat",
    "computer-based-assessment"
  ],
  "repositories": [
    {
      "type": "vcs",
      "url": "git@github.com:oat-sa/extension-tao-advanced-search.git"
    },
    {
      "type": "vcs",
      "url": "git@github.com:oat-sa/extension-tao-deliver-connect.git"
    },
    {
      "type": "vcs",
      "url": "https://github.com/oat-sa/extension-tao-premium.git"
    },
    {
      "type": "vcs",
      "url": "https://github.com/oat-sa/extension-tao-styles.git"
    },
    {
      "type": "vcs",
      "url": "git@github.com:oat-sa/extension-tao-system-status.git"
    },
    {
      "type": "vcs",
      "url": "git@github.com:oat-sa/extension-tao-test-preview-ui-loader.git"
    },
    {
      "type": "vcs",
      "url": "git@github.com:oat-sa/extension-tao-studio.git"
    },
    {
      "type": "vcs",
      "url": "git@github.com:oat-sa/lib-generis-gcp.git"
    }
  ],
  "homepage": "http://www.taotesting.com",
  "prefer-stable": true,
  "require": {
    "oat-sa/generis": "dev-feat/REL-1197/rdf-import-export-neo4j as 15.28.0",
    "oat-sa/tao-core": "dev-feat/REL-1197/rdf-import-export-neo4j as 53.7.0",
    "oat-sa/extension-tao-funcacl": "7.4.1",
    "oat-sa/extension-tao-dac-simple": "7.10.6",
    "oat-sa/extension-tao-itemqti": "30.2.8",
    "oat-sa/extension-tao-testqti": "47.3.0",
    "oat-sa/extension-tao-item": "12.1.1",
    "oat-sa/extension-tao-itemqti-pci": "8.10.2",
    "oat-sa/extension-tao-test": "16.0.4",
    "oat-sa/extension-tao-lti": "dev-feat/REL-1275/script-create-lti-platform as 15.13.4",
    "oat-sa/extension-tao-mediamanager": "12.37.1",
    "oat-sa/extension-tao-backoffice": "6.12.7",
    "oat-sa/extension-tao-task-queue": "6.8.2",
    "oat-sa/extension-tao-testqti-previewer": "dev-feat/REL-1197/rdf-import-export-neo4j as 3.10.0",
    "oat-sa/extension-tao-advanced-search": "1.14.0",
    "oat-sa/extension-tao-styles": "4.0.5",
    "oat-sa/extension-tao-system-status": "1.6.3",
    "oat-sa/extension-tao-test-preview-ui-loader": "1.26.5",
    "oat-sa/extension-tao-outcome": "dev-feat/REL-1197/rdf-import-export-neo4j as 13.7.2",
    "oat-sa/extension-tao-xmledit": "4.2.0",
    "oat-sa/extension-tao-xmledit-responseprocessing": "2.3.0",
    "oat-sa/extension-tao-lti-consumer": "2.4.1",
    "oat-sa/extension-tao-deliver-connect": "5.17.0",
    "oat-sa/extension-tao-studio": "0.3.0",
    "oat-sa/lib-generis-gcp": "0.2.0"
  },
  "require-dev": {
    "mikey179/vfsstream": "~1",
    "nimut/phpunit-merger": "~1",
    "phpdocumentor/reflection-docblock": "5.3.0 as 2.0.5",
    "phpspec/prophecy": "^1.15",
    "phpunit/phpunit": "^8.5"
  },
  "minimum-stability": "dev",
  "description": "TAO is an Open Source e-Testing platform that empowers you to build, deliver, and share innovative and engaging assessments online – in any language or subject matter."
}

codecov-commenter · 2023-10-12T12:37:52Z

Codecov Report

All modified lines are covered by tests ✅

Comparison is base (86fb138) 27.55% compared to head (a5cdf12) 27.55%.
Report is 1 commits behind head on develop.

Additional details and impacted files

@@            Coverage Diff             @@
##             develop    #3907   +/-   ##
==========================================
  Coverage      27.55%   27.55%           
  Complexity     10976    10976           
==========================================
  Files            922      922           
  Lines          33910    33910           
==========================================
  Hits            9345     9345           
  Misses         24565    24565

☔ View full report in Codecov by Sentry.
📢 Have feedback on the report? Share it here.

vbyndych

Review Checklist

vbyndych · 2023-10-13T08:04:20Z

scripts/install/CleanGraphDatabase.php

+            ->getPersistenceById($persistenceId);
+
+        $persistence->run('MATCH (n) DETACH DELETE n');
+        $persistence->run('CALL apoc.schema.assert({}, {}, true) YIELD label, key RETURN *');


question: It is not clear to me what is the purpose of this call, as we do not check results after it, could you clarify?

It removes all constraints

Thank you for clarifying that. Just a note (it is not a big deal), I think YIELD and RETURN sections are not required here as we do not use function results after execution.

scripts/install/EnableGraphDatabase.php

scripts/install/CleanGraphDatabase.php

scripts/install/EnableGraphDatabase.php

augustas

Works good, would also remove SmoothModel dependency 👍🏻

vbyndych

Looks good to me now.

github-actions · 2023-10-17T11:09:21Z

Version

Target Version	53.9.0
Last version	53.8.0

There are 0 BREAKING CHANGE, 2 features, 1 fix

yaraslau-kavaliou force-pushed the feat/REL-1294/neo4j-install branch from 5d03162 to 0ca4070 Compare October 12, 2023 12:36

yaraslau-kavaliou force-pushed the feat/REL-1294/neo4j-install branch from 0ca4070 to fd462e0 Compare October 12, 2023 12:38

feat: Add scripts to enbale neo4j persistance after tao install

403457a

yaraslau-kavaliou force-pushed the feat/REL-1294/neo4j-install branch from fd462e0 to 403457a Compare October 12, 2023 12:44

fix: Code style fix

c7ca9d4

yaraslau-kavaliou requested review from vbyndych and augustas October 12, 2023 12:48

vbyndych approved these changes Oct 16, 2023

View reviewed changes

vbyndych self-assigned this Oct 16, 2023

yaraslau-kavaliou requested a review from vbyndych October 16, 2023 22:26

augustas reviewed Oct 17, 2023

View reviewed changes

scripts/install/EnableGraphDatabase.php Outdated Show resolved Hide resolved

augustas approved these changes Oct 17, 2023

View reviewed changes

vbyndych approved these changes Oct 17, 2023

View reviewed changes

yaraslau-kavaliou force-pushed the feat/REL-1294/neo4j-install branch from 38bcfc5 to 31bf921 Compare October 17, 2023 11:07

feat: Add logging for graph db installation

a5cdf12

yaraslau-kavaliou force-pushed the feat/REL-1294/neo4j-install branch from 31bf921 to a5cdf12 Compare October 17, 2023 11:08

yaraslau-kavaliou merged commit 78b6f69 into develop Oct 17, 2023
4 checks passed

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

feat: Add scripts to enbale neo4j persistance after tao install #3907

feat: Add scripts to enbale neo4j persistance after tao install #3907

yaraslau-kavaliou commented Oct 12, 2023

codecov-commenter commented Oct 12, 2023 •

edited

Loading

vbyndych left a comment

vbyndych Oct 13, 2023

yaraslau-kavaliou Oct 16, 2023

vbyndych Oct 17, 2023

augustas left a comment

vbyndych left a comment

github-actions bot commented Oct 17, 2023

feat: Add scripts to enbale neo4j persistance after tao install #3907

feat: Add scripts to enbale neo4j persistance after tao install #3907

Conversation

yaraslau-kavaliou commented Oct 12, 2023

codecov-commenter commented Oct 12, 2023 • edited Loading

Codecov Report

vbyndych left a comment

Choose a reason for hiding this comment

vbyndych Oct 13, 2023

Choose a reason for hiding this comment

yaraslau-kavaliou Oct 16, 2023

Choose a reason for hiding this comment

vbyndych Oct 17, 2023

Choose a reason for hiding this comment

augustas left a comment

Choose a reason for hiding this comment

vbyndych left a comment

Choose a reason for hiding this comment

github-actions bot commented Oct 17, 2023

Version

codecov-commenter commented Oct 12, 2023 •

edited

Loading