Merge pull request #1791 from sphuber/verdi

Merge develop into verdi
aiidateam · Jul 24, 2018 · fb9e2eb · fb9e2eb
2 parents aa8537a + 2823588
commit fb9e2eb
Show file tree

Hide file tree

Showing 131 changed files with 1,918 additions and 1,625 deletions.
diff --git a/.ci/before_install.sh b/.ci/before_install.sh
@@ -5,17 +5,17 @@ set -ev
 
 if [[ "$TEST_TYPE" == "tests" ]]
 then
-    THEKEY=`cat ${HOME}/.ssh/id_rsa.pub`
+    THEKEY=`ssh-keygen -y -f "${HOME}/.ssh/id_rsa"`
     echo 'AUTHORIZED_KEY='"$THEKEY" > ${TRAVIS_BUILD_DIR}/torquessh.env
-    docker build -t torquessh ${TRAVIS_BUILD_DIR}/.ci/torquessh-doubler
+    docker build -t torquessh "${TRAVIS_BUILD_DIR}/.ci/torquessh-doubler"
     # Run it in the background, mapping port 22 of the container
     # to port 10022 outside, and passing the environment variable
-    docker run -d --privileged -p=10022:22 --name="torquesshmachine" --env-file ${TRAVIS_BUILD_DIR}/torquessh.env torquessh
+    docker run -d --privileged -p=10022:22 --name="torquesshmachine" --env-file "${TRAVIS_BUILD_DIR}/torquessh.env" torquessh
     # Docker ps to see what is going on
     echo "Running docker ps to see if the 'torquessh' docker image is up..."
     docker ps    
     # Wait for SSH to be up
-    ${TRAVIS_BUILD_DIR}/.ci/wait-for-it.sh localhost:10022 -t 0
+    "${TRAVIS_BUILD_DIR}"/.ci/wait-for-it.sh localhost:10022 -t 0
 
     # I will add the key to the known_hosts later, to give the time to ssh
     # to be really up - see the before_script script

diff --git a/.ci/polish/__init__.py b/.ci/polish/__init__.py
@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+###########################################################################
+# Copyright (c), The AiiDA team. All rights reserved.                     #
+# This file is part of the AiiDA code.                                    #
+#                                                                         #
+# The code is hosted on GitHub at https://github.com/aiidateam/aiida_core #
+# For further information on the license, see the LICENSE.txt file        #
+# For further information please visit http://www.aiida.net               #
+###########################################################################
diff --git a/.ci/polish/cli.py b/.ci/polish/cli.py
@@ -1,5 +1,13 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
+###########################################################################
+# Copyright (c), The AiiDA team. All rights reserved.                     #
+# This file is part of the AiiDA code.                                    #
+#                                                                         #
+# The code is hosted on GitHub at https://github.com/aiidateam/aiida_core #
+# For further information on the license, see the LICENSE.txt file        #
+# For further information please visit http://www.aiida.net               #
+###########################################################################
 import click
 from aiida.utils.cli import command
 from aiida.utils.cli import options

diff --git a/.ci/polish/expression.py b/.ci/polish/expression.py
@@ -1,4 +1,12 @@
 # -*- coding: utf-8 -*-
+###########################################################################
+# Copyright (c), The AiiDA team. All rights reserved.                     #
+# This file is part of the AiiDA code.                                    #
+#                                                                         #
+# The code is hosted on GitHub at https://github.com/aiidateam/aiida_core #
+# For further information on the license, see the LICENSE.txt file        #
+# For further information please visit http://www.aiida.net               #
+###########################################################################
 import operator as operators
 import random
 from collections import deque

diff --git a/.ci/polish/polish_workchains/__init__.py b/.ci/polish/polish_workchains/__init__.py
@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+###########################################################################
+# Copyright (c), The AiiDA team. All rights reserved.                     #
+# This file is part of the AiiDA code.                                    #
+#                                                                         #
+# The code is hosted on GitHub at https://github.com/aiidateam/aiida_core #
+# For further information on the license, see the LICENSE.txt file        #
+# For further information please visit http://www.aiida.net               #
+###########################################################################
diff --git a/.ci/polish/workchain.py b/.ci/polish/workchain.py
@@ -1,4 +1,12 @@
 # -*- coding: utf-8 -*-
+###########################################################################
+# Copyright (c), The AiiDA team. All rights reserved.                     #
+# This file is part of the AiiDA code.                                    #
+#                                                                         #
+# The code is hosted on GitHub at https://github.com/aiidateam/aiida_core #
+# For further information on the license, see the LICENSE.txt file        #
+# For further information please visit http://www.aiida.net               #
+###########################################################################
 import os
 import errno
 from collections import deque

diff --git a/.ci/prep_ssh.sh b/.ci/prep_ssh.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Be verbose, and stop with error as soon there's one
+set -ev
+
+# This is needed for the SSH tests (being able to ssh to localhost)
+# And will also be used for the docker test.
+
+# Make sure we don't overwrite an existing SSH key (seems to be the case for private Travis instances)
+[ -f "${HOME}/.ssh/id_rsa" ] || ssh-keygen -q -t rsa -N "" -f "${HOME}/.ssh/id_rsa"
+
+# Extract the public key directly from the private key without requiring .pub file (again: private Travis instances)
+ssh-keygen -y -f "${HOME}/.ssh/id_rsa" >> "${HOME}/.ssh/authorized_keys"
+
+# Register the current hosts fingerprint
+ssh-keyscan -H localhost >> "${HOME}/.ssh/known_hosts"
diff --git a/.ci/test_daemon.py b/.ci/test_daemon.py
@@ -19,6 +19,7 @@
 from aiida.orm.data.int import Int
 from aiida.orm.data.str import Str
 from aiida.orm.data.list import List
+from aiida.orm.calculation import JobCalculation
 from aiida.work.launch import run_get_node, submit
 from aiida.work.persistence import ObjectLoader
 from workchains import (
@@ -76,8 +77,8 @@ def validate_calculations(expected_results):
     for pk, expected_dict in expected_results.iteritems():
         calc = load_node(pk)
         if not calc.is_finished_ok:
-            print 'Calculation<{}> not finished ok: process_state<{}> finish_status<{}>'.format(
-                pk, calc.process_state, calc.finish_status)
+            print 'Calculation<{}> not finished ok: process_state<{}> exit_status<{}>'.format(
+                pk, calc.process_state, calc.exit_status)
             print_logshow(pk)
             valid = False
 
@@ -118,8 +119,8 @@ def validate_workchains(expected_results):
 
         # I check only if this_valid, otherwise calc could not exist
         if this_valid and not calc.is_finished_ok:
-            print 'Calculation<{}> not finished ok: process_state<{}> finish_status<{}>'.format(
-                pk, calc.process_state, calc.finish_status)
+            print 'Calculation<{}> not finished ok: process_state<{}> exit_status<{}>'.format(
+                pk, calc.process_state, calc.exit_status)
             print_logshow(pk)
             valid = False
             this_valid = False
@@ -142,16 +143,26 @@ def validate_cached(cached_calcs):
     for calc in cached_calcs:
 
         if not calc.is_finished_ok:
-            print 'Cached calculation<{}> not finished ok: process_state<{}> finish_status<{}>'.format(
-                pk, calc.process_state, calc.finish_status)
-            print_logshow(pk)
+            print 'Cached calculation<{}> not finished ok: process_state<{}> exit_status<{}>'.format(
+                calc.pk, calc.process_state, calc.exit_status)
+            print_logshow(calc.pk)
             valid = False
 
         if '_aiida_cached_from' not in calc.extras() or calc.get_hash() != calc.get_extra('_aiida_hash'):
-            print 'Cached calculation<{}> has invalid hash'.format(pk)
-            print_logshow(pk)
+            print 'Cached calculation<{}> has invalid hash'.format(calc.pk)
+            print_logshow(calc.pk)
             valid = False
 
+        if isinstance(calc, JobCalculation):
+            if 'raw_input' not in calc.folder.get_content_list():
+                print "Cached calculation <{}> does not have a 'raw_input' folder".format(calc.pk)
+                print_logshow(calc.pk)
+                valid = False
+            original_calc = load_node(calc.get_extra('_aiida_cached_from'))
+            if 'raw_input' not in original_calc.folder.get_content_list():
+                print "Original calculation <{}> does not have a 'raw_input' folder after being cached from.".format(original_calc.pk)
+                valid = False
+
     return valid
 
 

diff --git a/.ci/test_script.sh b/.ci/test_script.sh
@@ -17,6 +17,10 @@ case "$TEST_TYPE" in
     tests)
         CI_DIR="${TRAVIS_BUILD_DIR}/.ci"
 
+        # make sure we have the correct pg_ctl in our path for pgtest, to prevent issue #1722
+        # this must match the version request in travis.yml
+        export PATH="/usr/lib/postgresql/9.5/bin:${PATH}"
+
         # Add the .ci folder to the python path so workchains within it can be found by the daemon
         export PYTHONPATH="${PYTHONPATH}:${CI_DIR}"
 

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -65,6 +65,23 @@
         aiida/restapi/resources.py|
         aiida/restapi/translator/data/cif.py|
         aiida/utils/fixtures.py|
+        aiida/work/awaitable.py|
+        aiida/work/context.py|
+        aiida/work/exceptions.py|
+        aiida/work/exit_code.py|
+        aiida/work/futures.py|
+        aiida/work/launch.py|
+        aiida/work/persistence.py|
+        aiida/work/ports.py|
+        aiida/work/process_builder.py|
+        aiida/work/process_spec.py|
+        aiida/work/rmq.py|
+        aiida/work/runners.py|
+        aiida/work/test_utils.py|
+        aiida/work/transports.py|
+        aiida/work/utils.py|
+        aiida/work/workchain.py|
+        aiida/work/workfunctions.py|
         docs/update_req_for_rtd.py # a|b -> match a OR b
       )$  # $ -> end of file path, to add a directory, give full/path/.*
 

diff --git a/.pylintrc b/.pylintrc
@@ -136,7 +136,7 @@ function-name-hint=(([a-z][a-z0-9_]{2,40})|(_[a-z0-9_]*))$
 function-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
 
 # Good variable names which should always be accepted, separated by a comma
-good-names=i,j,k,ex,Run,_, _INPUT_FILE_NAME, _OUTPUT_FILE_NAME
+good-names=i,j,k,ex,Run,_, _INPUT_FILE_NAME, _OUTPUT_FILE_NAME, pk
 
 # Include a hint for the correct naming format with invalid-name
 include-naming-hint=no

diff --git a/.travis.yml b/.travis.yml
@@ -14,10 +14,12 @@ services:
     - docker
 
 addons:
+    # make sure the path in .ci/test_script.sh matches the version requested here
     postgresql: "9.5"
 
     apt:
         packages:
+            - postgresql-server-dev-9.5
             - texlive-base
             - texlive-generic-recommended
             - texlive-fonts-recommended
@@ -37,6 +39,7 @@ before_install:
 
     # Needed to have 'locate' work properly
     - sudo updatedb
+    - .ci/prep_ssh.sh
 
     # Build the docker image if needed
     - .ci/before_install.sh

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -17,6 +17,9 @@
 - Convention of leading underscores for non-storable inputs has been replaced with a proper `non_db` attribute of the `Port` class [[#1105]](https://github.com/aiidateam/aiida_core/pull/1105)
 - Implemented a Sphinx extension for the `WorkChain` class to automatically generate documentation from the workchain definition [[#1155]](https://github.com/aiidateam/aiida_core/pull/1155)
 - Added new feature for a `WorkChain` to expose the inputs and outputs of another `WorkChain`, which is perfect for writing modular workflows [[#1170]](https://github.com/aiidateam/aiida_core/pull/1170)
+- Add built-in support and API for exit codes in WorkChains [[#1640]](https://github.com/aiidateam/aiida_core/pull/1640), [[#1704]](https://github.com/aiidateam/aiida_core/pull/1704), [[#1681]](https://github.com/aiidateam/aiida_core/pull/1681)
+- Overload PortNamespace mutable properties upon exposing [[#1635]](https://github.com/aiidateam/aiida_core/pull/1635)
+- Implement the Exit exception to allow setting finish status for workfunctions [[#1631]](https://github.com/aiidateam/aiida_core/pull/1631)
 
 ### Verdi
 - Added the command `verdi group rename` [[#1224]](https://github.com/aiidateam/aiida_core/pull/1224)
@@ -33,21 +36,34 @@
 - Implemented the `DbImporter` for the Materials Platform of Data Science API, which exposed the content of the Pauling file [[#1238]](https://github.com/aiidateam/aiida_core/pull/1238)
 - Implement the `has_atomic_sites` and `has_unknown_species` properties for the `CifData` class [[#1257]](https://github.com/aiidateam/aiida_core/pull/1257)
 - Default library used in `_get_aiida_structure` to convert the `CifData` to `StructureData` has been changed from `ase` to `pymatgen` [[#1257]](https://github.com/aiidateam/aiida_core/pull/1257)
+- Do not allow the copy or deepcopy of Node, except for Data  [[#1705]](https://github.com/aiidateam/aiida_core/pull/1705)
+- Added element X to the elements list in order to support unknown species [[#1613]]
+- Enable use of tuple in QueryBuilder.append for all ORM classes [[#1608]](https://github.com/aiidateam/aiida_core/pull/1608), [[#1607]](https://github.com/aiidateam/aiida_core/pull/1607)
 
-### Documentation:
+### Database
+- Allow PostgreSQL connections via unix sockets [[#1721]](https://github.com/aiidateam/aiida_core/pull/1721)
+- Creating unique constraint & indexes at the db_dbgroup_dbnodes table in SQLA [[#1680]](https://github.com/aiidateam/aiida_core/pull/1680)
+- Performance improvement for adding nodes to group [[#1677]](https://github.com/aiidateam/aiida_core/pull/1677)
+
+
+### Documentation
 - Big reorganization of the documentation structure [[#1299]](https://github.com/aiidateam/aiida_core/pull/1299)
 - Added section on the basics of workchains and workfunctions [[#1384]](https://github.com/aiidateam/aiida_core/pull/1384)
 - Added section on how to launch workchains and workfunctions [[#1385]](https://github.com/aiidateam/aiida_core/pull/1385)
 - Added section on how to monitor workchains and workfunctions[[#1387]](https://github.com/aiidateam/aiida_core/pull/1387)
 - Added section on the concept of the `Process` [[#1395]](https://github.com/aiidateam/aiida_core/pull/1395)
 - Added section on advance concepts of the `WorkChain` class, as well as best-practices on designing/writing workchains [[#1459]](https://github.com/aiidateam/aiida_core/pull/1459)
+- Remove outdated or duplicated docs for legacy and new workflow system [[#1718]]
+
 
 ### Bug fixes
 - Fixed a problem with the temporary folder containing the files of the `retrieve_temporary_list` that could be cleaned before parsing finished [[#1168]](https://github.com/aiidateam/aiida_core/pull/1168)
 - Fixed a bug in the `store` method of `CifData` which would raise and exception when called more than once [[#1136]](https://github.com/aiidateam/aiida_core/pull/1136)
 - Restored a proper implementation of mutability for `Node` attributes [[#1181]](https://github.com/aiidateam/aiida_core/pull/1181)
 - Restore the correct plugin type string based on the module path of the base `Data` types [[#1192]](https://github.com/aiidateam/aiida_core/pull/1192)
 - Fix bug in `verdi export create` when only exporting computers [[#1448]](https://github.com/aiidateam/aiida_core/pull/1448)
+- Fix copying of the calculation raw input folder in caching [[#1745]](https://github.com/aiidateam/aiida_core/pull/1745)
+- Fix sphinxext command by allowing whitespace in argument [[#1644]](https://github.com/aiidateam/aiida_core/pull/1644)
 
 
 ## v0.12.1

diff --git a/aiida/__init__.py b/aiida/__init__.py
@@ -8,13 +8,13 @@
 # For further information please visit http://www.aiida.net               #
 ###########################################################################
 import warnings
+
 from aiida.common.log import configure_logging
 from aiida.common.setup import get_property
 
 __copyright__ = u"Copyright (c), This file is part of the AiiDA platform. For further information please visit http://www.aiida.net/. All rights reserved."
 __license__ = "MIT license, see LICENSE.txt file."
-__version__ = "1.0.0a1"
-__version__ = "0.12.1"
+__version__ = "1.0.0a2"
 __authors__ = "The AiiDA team."
 __paper__ = """G. Pizzi, A. Cepellotti, R. Sabatini, N. Marzari, and B. Kozinsky, "AiiDA: automated interactive infrastructure and database for computational science", Comp. Mat. Sci 111, 218-230 (2016); http://dx.doi.org/10.1016/j.commatsci.2015.09.013 - http://www.aiida.net."""
 __paper_short__ = """G. Pizzi et al., Comp. Mat. Sci 111, 218 (2016)."""
@@ -27,6 +27,7 @@
     # in Python 2.7 it is suppressed by default
     warnings.simplefilter('default', DeprecationWarning)
 
+
 def try_load_dbenv(*argc, **argv):
     """
     Run `load_dbenv` unless the dbenv has already been loaded.
@@ -36,6 +37,7 @@ def try_load_dbenv(*argc, **argv):
         return True
     return False
 
+
 def load_dbenv(*argc, **argv):
     """
     Alias for `load_dbenv` from `aiida.backends.utils`
@@ -52,9 +54,21 @@ def is_dbenv_loaded(*argc, **argv):
     return is_dbenv_loaded(*argc, **argv)
 
 
+def get_strict_version():
+    """
+    Return a distutils StrictVersion instance with the current distribution version
+
+    :returns: StrictVersion instance with the current version
+    """
+    from distutils.version import StrictVersion
+    return StrictVersion(__version__)
+
+
 def get_version():
     """
-    Very simple function to get a string with the version number.
+    Return the current distribution version
+
+    :returns: a string with the current version
     """
     return __version__
 
@@ -71,6 +85,7 @@ def _get_raw_file_header():
 {}
 """.format(__version__, __paper__)
 
+
 def get_file_header(comment_char="# "):
     """
     Get a string to be put as header of files created with AiiDA;
@@ -80,4 +95,4 @@ def get_file_header(comment_char="# "):
     :return: a (multiline) string
     """
     lines = _get_raw_file_header().splitlines()
-    return "\n".join("{}{}".format(comment_char, line) for line in lines)
+    return '\n'.join('{}{}'.format(comment_char, line) for line in lines)
diff --git a/aiida/backends/sqlalchemy/migrations/versions/0aebbeab274d_base_data_plugin_type_string.py b/aiida/backends/sqlalchemy/migrations/versions/0aebbeab274d_base_data_plugin_type_string.py
@@ -1,3 +1,12 @@
+# -*- coding: utf-8 -*-
+###########################################################################
+# Copyright (c), The AiiDA team. All rights reserved.                     #
+# This file is part of the AiiDA code.                                    #
+#                                                                         #
+# The code is hosted on GitHub at https://github.com/aiidateam/aiida_core #
+# For further information on the license, see the LICENSE.txt file        #
+# For further information please visit http://www.aiida.net               #
+###########################################################################
 """Correct the type string for the base data types
 
 Revision ID: 0aebbeab274d

diff --git a/...nds/sqlalchemy/migrations/versions/59edaf8a8b79_adding_indexes_and_constraints_to_the_.py b/...nds/sqlalchemy/migrations/versions/59edaf8a8b79_adding_indexes_and_constraints_to_the_.py
@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+###########################################################################
+# Copyright (c), The AiiDA team. All rights reserved.                     #
+# This file is part of the AiiDA code.                                    #
+#                                                                         #
+# The code is hosted on GitHub at https://github.com/aiidateam/aiida_core #
+# For further information on the license, see the LICENSE.txt file        #
+# For further information please visit http://www.aiida.net               #
+###########################################################################
+"""Adding indexes and constraints to the dbnode-dbgroup relationship table
+
+Revision ID: 59edaf8a8b79
+Revises: a514d673c163
+Create Date: 2018-06-22 14:50:18.447211
+
+"""
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = '59edaf8a8b79'
+down_revision = 'a514d673c163'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_index('db_dbgroup_dbnodes_dbnode_id_idx', 'db_dbgroup_dbnodes',
+                    ['dbnode_id'])
+    op.create_index('db_dbgroup_dbnodes_dbgroup_id_idx', 'db_dbgroup_dbnodes',
+                    ['dbgroup_id'])
+    op.create_unique_constraint(
+        'db_dbgroup_dbnodes_dbgroup_id_dbnode_id_key', 'db_dbgroup_dbnodes',
+        ['dbgroup_id', 'dbnode_id'])
+
+
+def downgrade():
+    op.drop_index('db_dbgroup_dbnodes_dbnode_id_idx', 'db_dbgroup_dbnodes')
+    op.drop_index('db_dbgroup_dbnodes_dbgroup_id_idx', 'db_dbgroup_dbnodes')
+    op.drop_constraint('db_dbgroup_dbnodes_dbgroup_id_dbnode_id_key',
+                       'db_dbgroup_dbnodes')