Merge branch 'master' into get-base-url-with-root-path

.github/workflows/deps_lint.yml

@@ -141,7 +141,7 @@ jobs:
 
  services:
  mongo:
- image: mongo:4.2
+ image: mongo:4
  ports:
  - 27017:27017
  postgres:
@@ -156,6 +156,14 @@ jobs:
  --health-retries 5
  ports:
  - 5432:5432
+ elasticsearch:
+ image: elasticsearch:6.8.13
+ ports:
+ - 9200:9200
+ - 9300:9300
+ env:
+ discovery.type: single-node
+
 
  steps:
  - uses: actions/checkout@v2
@@ -176,20 +184,29 @@ jobs:
  pip install -r requirements.txt
  pip install -r requirements-dev.txt
 
+ - name: Run all tests (using `mongomock`)
+ run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml tests/
+ env:
+ OPTIMADE_DATABASE_BACKEND: 'mongomock'
+
+
  - name: Run server tests (using a real MongoDB)
- run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml tests/server
+ run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server
  env:
- OPTIMADE_CI_FORCE_MONGO: 1
+ OPTIMADE_DATABASE_BACKEND: 'mongodb'
 
- - name: Run all tests (using `mongomock`)
- run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/
+ - name: Run server tests (using Elasticsearch)
+ run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server
  env:
- OPTIMADE_CI_FORCE_MONGO: 0
+ OPTIMADE_DATABASE_BACKEND: 'elastic'
 
  - name: Install adapter conversion dependencies
  run: |
  pip install -r requirements-client.txt
  # AiiDA-specific
+ # sqlalchemy install is temporary and has been fixed in aiidateam/aiida-core#4809,
+ # but is still not released in a new AiiDA version.
+ pip install sqlalchemy==1.3.23
  reentry scan
 
  - name: Setup up environment for AiiDA
@@ -200,9 +217,25 @@ jobs:
  - name: Run previously skipped tests for adapter conversion
  run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/adapters/
 
- - name: Run tests for validator only to assess coverage
+ - name: Run tests for validator only to assess coverage (mongomock)
  if: matrix.python-version == 3.8
- run: pytest -rs --cov=./optimade/ --cov-report=xml:validator_cov.xml tests/server/test_server_validation.py
+ run: pytest -rs --cov=./optimade/ --cov-report=xml:validator_cov.xml --cov-append tests/server/test_server_validation.py
+ env:
+ OPTIMADE_DATABASE_BACKEND: 'mongomock'
+
+ - name: Run tests for validator only to assess coverage (Elasticsearch)
+ if: matrix.python-version == 3.8
+ run: pytest -rs --cov=./optimade/ --cov-report=xml:validator_cov.xml --cov-append tests/server/test_server_validation.py
+ env:
+ OPTIMADE_DATABASE_BACKEND: 'elastic'
+ OPTIMADE_INSERT_TEST_DATA: false # Must be specified as previous steps will have already inserted the test data
+
+ - name: Run tests for validator only to assess coverage (MongoDB)
+ if: matrix.python-version == 3.8
+ run: pytest -rs --cov=./optimade/ --cov-report=xml:validator_cov.xml --cov-append tests/server/test_server_validation.py
+ env:
+ OPTIMADE_DATABASE_BACKEND: 'mongodb'
+ OPTIMADE_INSERT_TEST_DATA: false # Must be specified as previous steps will have already inserted the test data
 
  - name: Upload coverage to Codecov
  if: matrix.python-version == 3.8 && github.repository == 'Materials-Consortia/optimade-python-tools'
@@ -238,10 +271,13 @@ jobs:
  pip install -U setuptools
  pip install -e .
  pip install -r requirements.txt
+ pip install -r requirements-dev.txt
  pip install -r requirements-docs.txt
 
  - name: Build
- run: mkdocs build --strict --verbose
+ run: |
+ invoke create-api-reference-docs
+ mkdocs build --strict --verbose
 
  test_build:
  runs-on: ubuntu-latest

INSTALL.md

@@ -44,7 +44,7 @@ py.test
 # Install pre-commit environment (e.g., auto-formats code on `git commit`)
 pre-commit install
 
-# Optional: Install MongoDB (and set `use_real_mongo = true`)
+# Optional: Install MongoDB (and set `database_backend = mongodb`)
 # Below method installs in conda environment and
 # - starts server in background
 # - ensures and uses ~/dbdata directory to store data

README.md

@@ -11,10 +11,8 @@ The aim of OPTIMADE is to develop a common API, compliant with the [JSON API 1.0
 This is to enable interoperability among databases that contain calculated properties of existing and hypothetical materials.
 
 This repository contains a library of tools for implementing and consuming [OPTIMADE](https://www.optimade.org) APIs using Python.
-It also contains a server validator tool, which may be called from the shell or used as a GitHub Action from [optimade-validator-action](https://github.com/Materials-Consortia/optimade-validator-action).
-
-_Disclaimer_: While the package supports `elasticsearch-dsl` v6 & v7 and `django` v2 & v3, all tests are performed with the latest supported version.
-If you experience any issues with the older versions, you are most welcome to contribute to the repository (see below under [Contributing](#contributing)).
+Server implementations can make use of the supported MongoDB (v4) and Elasticsearch (v6) database backends, or plug in a custom backend implementation.
+The package also contains a server validator tool, which may be called from the shell (`optimade-validator`) or used as a GitHub Action from [optimade-validator-action](https://github.com/Materials-Consortia/optimade-validator-action).
 
 ## Status
 

docs/api_reference/server/entry_collections/elasticsearch.md

@@ -0,0 +1,3 @@
+# elasticsearch
+
+::: optimade.server.entry_collections.elasticsearch

docs/static/default_config.json

@@ -1,7 +1,7 @@
 {
  "config_file": "~/.optimade.json",
  "debug": false,
- "use_real_mongo": false,
+ "insert_test_data": true,
  "mongo_database": "optimade",
  "mongo_uri": "localhost:27017",
  "links_collection": "links",

optimade/__init__.py

@@ -1,2 +1,21 @@
 __version__ = "0.13.3"
 __api_version__ = "1.0.1"
+
+
+import sys
+
+if sys.version_info.minor == 6:
+ # Python 3.6
+ import warnings
+
+ warnings.filterwarnings(
+ action="once",
+ message=r"v0\.14 of the `optimade` package.*",
+ category=DeprecationWarning,
+ append=False,
+ )
+ warnings.warn(
+ "v0.14 of the `optimade` package will be the last to support Python 3.6. "
+ "Please upgrade to Python 3.7+ to use v0.15 and later versions of `optimade`.",
+ DeprecationWarning,
+ )

optimade/filtertransformers/django.py

@@ -1,7 +1,7 @@
 import warnings
 
 warnings.warn(
- "Django functionality is deprecated and will be removed in later versions (unless support is requested).",
+ "Django functionality is deprecated and will be removed in a later version (unless support is requested).",
  DeprecationWarning,
  stacklevel=2,
 )

optimade/filtertransformers/elasticsearch.py

@@ -2,7 +2,6 @@
 
 from lark import v_args
 from elasticsearch_dsl import Q, Text, Keyword, Integer, Field
-from optimade.models import CHEMICAL_SYMBOLS, ATOMIC_NUMBERS
 from optimade.filtertransformers import BaseTransformer
 from optimade.server.exceptions import BadRequest
 
@@ -14,7 +13,7 @@
 _has_operators = {"ALL": "must", "ANY": "should"}
 _length_quantities = {
  "elements": "nelements",
- "elements_rations": "nelements",
+ "elements_ratios": "nelements",
  "dimension_types": "dimension_types",
 }
 
@@ -130,42 +129,50 @@ def _has_query_op(self, quantities, op, predicate_zip_list):
  # in elastic search. Only supported for elements, where we can construct
  # an anonymous "formula" based on elements sorted by order number and
  # can do a = comparision to check if all elements are contained
- if len(quantities) > 1:
- raise NotImplementedError("HAS ONLY is not supported with zip")
- quantity = quantities[0]
 
- if quantity.has_only_quantity is None:
- raise NotImplementedError(
- "HAS ONLY is not supported by %s" % quantity.name
- )
-
- def values():
- for predicates in predicate_zip_list:
- if len(predicates) != 1:
- raise NotImplementedError("Tuples not supported in HAS ONLY")
- op, value = predicates[0]
- if op != "=":
- raise NotImplementedError(
- "Predicated not supported in HAS ONLY"
- )
- if not isinstance(value, str):
- raise NotImplementedError("Only strings supported in HAS ONLY")
- yield value
-
- try:
- order_numbers = list([ATOMIC_NUMBERS[element] for element in values()])
- order_numbers.sort()
- value = "".join(
- [CHEMICAL_SYMBOLS[number - 1] for number in order_numbers]
- )
- except KeyError:
- raise NotImplementedError(
- "HAS ONLY is only supported for chemical symbols"
- )
+ # @ml-evs: Disabling this HAS ONLY workaround as tests are not passing
+ raise NotImplementedError(
+ "HAS ONLY queries are not currently supported by the Elasticsearch backend."
+ )
 
- return Q("term", **{quantity.has_only_quantity.name: value})
+ # from optimade.models import CHEMICAL_SYMBOLS, ATOMIC_NUMBERS
+
+ # if len(quantities) > 1:
+ # raise NotImplementedError("HAS ONLY is not supported with zip")
+ # quantity = quantities[0]
+
+ # if quantity.has_only_quantity is None:
+ # raise NotImplementedError(
+ # "HAS ONLY is not supported by %s" % quantity.name
+ # )
+
+ # def values():
+ # for predicates in predicate_zip_list:
+ # if len(predicates) != 1:
+ # raise NotImplementedError("Tuples not supported in HAS ONLY")
+ # op, value = predicates[0]
+ # if op != "=":
+ # raise NotImplementedError(
+ # "Predicated not supported in HAS ONLY"
+ # )
+ # if not isinstance(value, str):
+ # raise NotImplementedError("Only strings supported in HAS ONLY")
+ # yield value
+
+ # try:
+ # order_numbers = list([ATOMIC_NUMBERS[element] for element in values()])
+ # order_numbers.sort()
+ # value = "".join(
+ # [CHEMICAL_SYMBOLS[number - 1] for number in order_numbers]
+ # )
+ # except KeyError:
+ # raise NotImplementedError(
+ # "HAS ONLY is only supported for chemical symbols"
+ # )
+
+ # return Q("term", **{quantity.has_only_quantity.name: value})
  else:
- raise NotImplementedError
+ raise NotImplementedError(f"Unrecognised operation {op}.")
 
  queries = [
  self._has_query(quantities, predicates) for predicates in predicate_zip_list
@@ -320,12 +327,15 @@ def set_zip_op_rhs(self, args):
  return lambda quantity: self._has_query_op([quantity] + add_on, op, values)
 
  def property_zip_addon(self, args):
+ raise NotImplementedError("Correlated list queries are not supported.")
  return args
 
  def value_zip(self, args):
+ raise NotImplementedError("Correlated list queries are not supported.")
  return self.value_list(args)
 
  def value_zip_list(self, args):
+ raise NotImplementedError("Correlated list queries are not supported.")
  return args
 
  def value_list(self, args):

optimade/filtertransformers/mongo.py

@@ -61,11 +61,11 @@ def value_list(self, arg):
 
  def value_zip(self, arg):
  # value_zip: [ OPERATOR ] value ":" [ OPERATOR ] value (":" [ OPERATOR ] value)*
- raise NotImplementedError
+ raise NotImplementedError("Correlated list queries are not supported.")
 
  def value_zip_list(self, arg):
  # value_zip_list: value_zip ( "," value_zip )*
- raise NotImplementedError
+ raise NotImplementedError("Correlated list queries are not supported.")
 
  def expression(self, arg):
  # expression: expression_clause ( OR expression_clause )
@@ -158,11 +158,11 @@ def length_op_rhs(self, arg):
  def set_zip_op_rhs(self, arg):
  # set_zip_op_rhs: property_zip_addon HAS ( value_zip | ONLY value_zip_list | ALL value_zip_list |
  # ANY value_zip_list )
- raise NotImplementedError
+ raise NotImplementedError("Correlated list queries are not supported.")
 
  def property_zip_addon(self, arg):
  # property_zip_addon: ":" property (":" property)*
- raise NotImplementedError
+ raise NotImplementedError("Correlated list queries are not supported.")
 
  def _recursive_expression_phrase(self, arg):
  """Helper function for parsing `expression_phrase`. Recursively sorts out
@@ -226,15 +226,15 @@ def _apply_aliases(self, filter_: dict) -> dict:
  return filter_
 
  def check_for_alias(prop, expr):
- return self.mapper.alias_for(prop) != prop
+ return self.mapper.get_backend_field(prop) != prop
 
  def apply_alias(subdict, prop, expr):
  if isinstance(subdict, dict):
- subdict[self.mapper.alias_for(prop)] = self._apply_aliases(
+ subdict[self.mapper.get_backend_field(prop)] = self._apply_aliases(
  subdict.pop(prop)
  )
  elif isinstance(subdict, str):
- subdict = self.mapper.alias_for(subdict)
+ subdict = self.mapper.get_backend_field(subdict)
 
  return subdict
 
@@ -396,7 +396,7 @@ def replace_str_id_with_objectid(subdict, prop, expr):
  val = subdict[prop][operator]
  if operator not in ("$eq", "$ne"):
  if self.mapper is not None:
- prop = self.mapper.alias_of(prop)
+ prop = self.mapper.get_optimade_field(prop)
  raise NotImplementedError(
  f"Operator {operator} not supported for query on field {prop!r}, can only test for equality"
  )
@@ -417,7 +417,7 @@ def _apply_mongo_date_filter(self, filter_: dict) -> dict:
  def check_for_timestamp_field(prop, _):
  """ Find cases where the query dict is operating on a timestamp field. """
  if self.mapper is not None:
- prop = self.mapper.alias_of(prop)
+ prop = self.mapper.get_optimade_field(prop)
  return prop == "last_modified"
 
  def replace_str_date_with_datetime(subdict, prop, expr):