project: upgrade invenio and elasticsearch

This commit upgrade invenio to version 3.3 and elasticsearch to version 7.9.1.

* Builds elasticsearch container from image with version 7.9.1
* Upgrades dependencies in `pyproject.toml`
* Removes `celery.contrib.pytest` in `pytest.ini` as this plugin is now already registered by `invenio-pytest`.
* Moves templates and mappings from `v6` to `v7`.
* Removes first level of `mappings` property in all mappings files.
* Adds `**kwargs` to all hooks methods of marshmallow to avoid an error after upgrading to version 3.8.
* Removes `permissions` property when loading data in marshmallow schema, to avoid an error.
* Removes `permalink` property when loading data in marshmallow documents schema, to avoid an error.
* Closes #345.

Co-Authored-by: Sébastien Délèze <sebastien.deleze@rero.ch>

docker-services.yml

@@ -74,11 +74,13 @@ services:
       - "5672:5672"
   es:
     build: ./docker/elasticsearch/
-    image: elasticsearch-icu
+    image: elasticsearch-icu-7.9.1
     restart: "always"
     environment:
       - bootstrap.memory_lock=true
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - "discovery.type=single-node"
+      - "indices.query.bool.max_clause_count=3000"
     ulimits:
       memlock:
         soft: -1
@@ -91,7 +93,7 @@ services:
       - "9200:9200"
       - "9300:9300"
   kibana:
-    image: docker.elastic.co/kibana/kibana-oss:6.6.0
+    image: docker.elastic.co/kibana/kibana-oss:7.9.1
     environment:
       - "ELASTICSEARCH_HOSTS=http://es:9200"
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"

docker/elasticsearch/Dockerfile

@@ -1,2 +1,2 @@
-FROM docker.elastic.co/elasticsearch/elasticsearch-oss:6.6.2
+FROM docker.elastic.co/elasticsearch/elasticsearch-oss:7.9.1
 RUN bin/elasticsearch-plugin install analysis-icu

pyproject.toml

@@ -9,17 +9,17 @@ license = "GNU Affero General Public License v3.0"
 python = "^3.6"
 Babel = ">=2.4.0"
 Flask-BabelEx = ">=0.9.3"
-invenio-logging = { version = ">=1.2.0,<1.3.0", extras = ["sentry-sdk", "sentry"] }
+invenio-logging = { version = ">=1.3.0,<1.4.0", extras = ["sentry-sdk", "sentry"] }
 invenio-oaiharvester = {tag = "v1.0.0a4", git = "https://github.com/inveniosoftware/invenio-oaiharvester.git"}
-invenio = {version = "==3.2.1", extras = ["base", "metadata", "files", "postgresql", "auth", "elasticsearch6" ]}
+invenio = {version = ">=3.3.0,<3.4.0", extras = ["base", "metadata", "files", "postgresql", "auth", "elasticsearch7" ]}
 uwsgi = ">=2.0"
 uwsgitop = ">=0.11"
 uwsgi-tools = ">=1.1.1"
 orcid = "*"
 python-slugify = "*"
 python3-saml = "*"
 xmltodict = "*"
-marshmallow = "<=3.0.0b6"
+marshmallow = ">=3.0.0,<4.0.0"
 pycountry = "*"
 flask-wiki = {git = "https://github.com/rero/flask-wiki.git"}
 markdown-captions = "*"

pytest.ini

@@ -14,8 +14,7 @@
 
 [pytest]
 pep8ignore = docs/conf.py ALL
-; Add pluging for celery fixtures manually. This can be removed when invenio-celery will be on version 1.2.1
-addopts = --pep8 --doctest-glob="*.rst" --doctest-modules --cov=sonar --cov-report=term-missing --ignore=setup.py -p celery.contrib.pytest
+addopts = --pep8 --doctest-glob="*.rst" --doctest-modules --cov=sonar --cov-report=term-missing --ignore=setup.py
 testpaths = docs tests sonar
 ; Not displaying all the PendingDeprecationWarnings from invenio
 filterwarnings =