project: use poetry

This PR moves from `pipenv` to `poetry` and refactors the way that docker images are built.

* Replaces installation of `pipenv` by `poetry` in Travis script.
* Renames `run-tests.sh` script to `scripts/test`.
* Removes `check-manifest` call in test script as it causes errors without pipenv.
* Updates the page for contributing.
* Updates the page for installing the project.
* Refactors Dockerfile.base to use a slim python 3.6 image.
* Replaces installation of `pipenv` by `poetry` in Dockerfile.base.
* Runs bootstrap script in Dockerfile.
* Removes useless files.
* Updates `build-images.sh` script in order to use hash from `poetry.lock`.
* Adds `deploy` option to bootstrap script when it is called from docker.
* Adds `ui` option to specify a tarball file to install for SONAR-UI.
* Adds a check to all scripts, to avoid to run script outside poetry environment.
* Closes #341.
* Closes #363.

Co-Authored-by: Sébastien Délèze <sebastien.deleze@rero.ch>

.travis.yml

@@ -54,13 +54,14 @@ before_install:
   # Chrome webdriver for Selenium
   - "PATH=$PATH:$HOME/webdrivers"
   - "nvm install 8; nvm use 8"
-  - "travis_retry pip install --upgrade pip setuptools py pipenv"
+  - "travis_retry pip install --upgrade pip setuptools py \"poetry<1.1.0\""
   - "travis_retry pip install twine wheel coveralls"
   # patch default version of travis which is marked as unsafe by pipenv check
   - "travis_retry pip install --upgrade numpy"
 
 install:
   - "./scripts/bootstrap"
+  - "poetry show"
 
 before_script:
   # https://docs.travis-ci.com/user/gui-and-headless-browsers/
@@ -69,7 +70,7 @@ before_script:
   - "./docker/wait-for-services.sh"
 
 script:
-  - ./run-tests.sh
+  - "./scripts/test"
 
 after_success:
   - coveralls

CONTRIBUTING.rst

@@ -78,14 +78,7 @@ Ready to contribute? Here's how to set up `sonar` for local development.
 
       $ git clone git@github.com:your_name_here/sonar.git
 
-3. Install your local copy. Assuming you have
-   `pipenv <https://docs.pipenv.org/install/#installing-pipenv>`_, installed, this is how you
-   set up your fork for local development:
-
-   .. code-block:: console
-
-      $ cd sonar/
-      $ pipenv run install -e .[all]
+3. Install your local using the `installation procedure`.
 
 4. Create a branch for local development:
 
@@ -99,7 +92,7 @@ Ready to contribute? Here's how to set up `sonar` for local development.
 
    .. code-block:: console
 
-      $ ./run-tests.sh
+      $ poetry run ./scripts/test
 
    The tests will provide you with test coverage and also check PEP8
    (code style), PEP257 (documentation), flake8 as well as build the Sphinx

Dockerfile

@@ -27,23 +27,17 @@
 ARG VERSION=latest
 FROM sonar-base:${VERSION}
 
-ARG UI_TGZ=""
-
-COPY ./ .
+# Copy files
+COPY ./ ${WORKING_DIR}/src
+WORKDIR ${WORKING_DIR}/src
 COPY ./docker/uwsgi/ ${INVENIO_INSTANCE_PATH}
 
-RUN pip install . && \
-    invenio collect -v  && \
-    invenio webpack create && \
-    # --unsafe needed because we are running as root
-    invenio webpack install --unsafe && \
-    if [ "${UI_TGZ}" != "" ] ; then invenio webpack install $PWD/data/$UI_TGZ --unsafe ; fi && \
-    invenio webpack build &&  \
-    invenio utils compile-json ./sonar/modules/documents/jsonschemas/documents/document-v1.0.0_src.json -o ./sonar/modules/documents/jsonschemas/documents/document-v1.0.0.json && \
-    invenio utils compile-json ./sonar/modules/deposits/jsonschemas/deposits/deposit-v1.0.0_src.json -o ./sonar/modules/deposits/jsonschemas/deposits/deposit-v1.0.0.json && \
-    python ./setup.py compile_catalog && \
-    pip install . && \
-    mkdir ${INVENIO_INSTANCE_PATH}/static/sonar-ui && \
-    cp -R ${INVENIO_INSTANCE_PATH}/assets/node_modules/@rero/sonar-ui/dist/sonar/* ${INVENIO_INSTANCE_PATH}/static/sonar-ui
+# Change owner
+RUN chown -R invenio:invenio ${WORKING_DIR}
+
+# Run bootstrap
+ENV TERM=xterm-256color
+ARG UI_TGZ=""
+RUN poetry run ./scripts/bootstrap --deploy --ui ${UI_TGZ}
 
 ENTRYPOINT [ "bash", "-c"]

Dockerfile.base

@@ -22,9 +22,39 @@
 # and includes Pip, Pipenv, Node.js, NPM and some few standard libraries
 # Invenio usually needs.
 
-FROM inveniosoftware/centos7-python:3.6
+FROM python:3.6-slim-stretch
 
-RUN yum -y install libxml2-devel xmlsec1-devel xmlsec1-openssl-devel libtool-ltdl-devel xpdf ghostscript ImageMagick
+# require debian packages
+RUN apt-get upgrade -y && apt-get update -y
+RUN apt-get install --no-install-recommends -y git vim-tiny curl gcc gnupg libc6-dev libxml2-dev libxmlsec1-dev libxmlsec1-openssl xpdf xpdf-utils ghostscript imagemagick && rm -rf /var/lib/apt/lists/*
+RUN pip install --upgrade setuptools wheel pip "poetry<1.1.0"
 
-COPY Pipfile Pipfile.lock ./
-RUN pipenv install --deploy --system
+# # Install Node
+RUN curl -sL https://deb.nodesource.com/setup_12.x | bash -
+RUN apt-get install --no-install-recommends -y nodejs && rm -rf /var/lib/apt/lists/*
+RUN npm install --silent  node-sass@4.14.1 clean-css-cli@4.3.0 uglify-js@3.9.4 requirejs@2.3.6
+
+# Env variables
+ENV WORKING_DIR=/invenio
+ENV INVENIO_INSTANCE_PATH=${WORKING_DIR}/var/instance
+
+# Create directories
+RUN mkdir -p ${INVENIO_INSTANCE_PATH}
+RUN mkdir -p ${WORKING_DIR}/src
+RUN mkdir -p ${WORKING_DIR}/instance
+
+# copy everything inside /src
+COPY ./ ${WORKING_DIR}/src
+WORKDIR ${WORKING_DIR}/src
+
+# copy uwsgi config files
+COPY ./docker/uwsgi/ ${INVENIO_INSTANCE_PATH}
+
+# Set folder permissions
+ENV INVENIO_USER_ID=1000
+RUN useradd invenio --uid ${INVENIO_USER_ID}  --home ${WORKING_DIR} && \
+    chown -R invenio:invenio ${WORKING_DIR} && \
+    chmod -R go+w ${WORKING_DIR}
+
+# Install dependencies
+RUN poetry install --no-dev

INSTALL.rst

@@ -14,98 +14,96 @@
     You should have received a copy of the GNU Affero General Public License
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-Installation
-============
 
-First you need to install
-`pipenv <https://docs.pipenv.org/install/#installing-pipenv>`_, it will handle
-the virtual environment creation for the project in order to sandbox our Python
-environment, as well as manage the dependency installation, among other things.
+Installation
+------------
 
-Start all dependent services using docker-compose (this will start PostgreSQL,
-Elasticsearch 6, RabbitMQ and Redis):
+First, create your working directory and ``cd`` into it. Clone the project into this directory:
 
 .. code-block:: console
 
-    $ docker-compose up -d
+    $ git clone https://github.com/rero/sonar.git
 
-.. note::
+You need to install `poetry`, it will handle the virtual environment creation for the project
+in order to sandbox our Python environment, as well as manage the dependency installation,
+among other things.
 
-    Make sure you have `enough virtual memory
-    <https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html#docker-cli-run-prod-mode>`_
-    for Elasticsearch in Docker:
+.. code-block:: console
 
-    .. code-block:: shell
+    $ pyenv install 3.6.10
+    $ cd sonar
+    $ pyenv local 3.6.10
+    $ pip install poetry
 
-        # Linux
-        $ sysctl -w vm.max_map_count=262144
+Next, ``cd`` into the project directory and bootstrap the instance (this will install
+all Python dependencies and build all static assets):
 
-        # macOS
-        $ screen ~/Library/Containers/com.docker.docker/Data/com.docker.driver.amd64-linux/tty
-        <enter>
-        linut00001:~# sysctl -w vm.max_map_count=262144
+.. code-block:: console
 
+    $ cd sonar
+    $ poetry run ./scripts/bootstrap
 
-Next, bootstrap the instance (this will install all Python dependencies and
-build all static assets):
+Start all dependent services using docker-compose (this will start PostgreSQL,
+Elasticsearch 6, RabbitMQ and Redis):
 
 .. code-block:: console
 
-    $ ./scripts/bootstrap
+    $ docker-compose up -d
+
+
+Make sure you have `enough virtual memory
+<https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html#docker-cli-run-prod-mode>`_
+for Elasticsearch in Docker:
+
+.. code-block:: shell
+
+    # Linux
+    $ sysctl -w vm.max_map_count=262144
+
+    # macOS
+    $ screen ~/Library/Containers/com.docker.docker/Data/com.docker.driver.amd64-linux/tty
+    <enter>
+    linut00001:~# sysctl -w vm.max_map_count=262144
+
 
 Next, create database tables, search indexes and message queues:
 
 .. code-block:: console
 
-    $ ./scripts/setup
+    $ poetry run ./scripts/setup
 
 Running
 -------
 Start the webserver and the celery worker:
 
 .. code-block:: console
 
-    $ ./scripts/server
+    $ poetry run ./scripts/server
 
 Start a Python shell:
 
 .. code-block:: console
 
-    $ ./scripts/console
-
-Upgrading
----------
-In order to upgrade an existing instance simply run:
-
-.. code-block:: console
-
-    $ ./scripts/update
+    $ poetry run ./scripts/console
 
 Testing
 -------
 Run the test suite via the provided script:
 
 .. code-block:: console
 
-    $ ./run-tests.sh
+    $ poetry run ./scripts/test
 
 By default, end-to-end tests are skipped. You can include the E2E tests like
 this:
 
 .. code-block:: console
 
-    $ env E2E=yes ./run-tests.sh
+    $ env E2E=yes poetry run ./scripts/test
 
 For more information about end-to-end testing see `pytest-invenio
-<https://pytest-invenio.readthedocs.io/en/latest/usage.html#running-e2e-tests>`_
+<https://pytest-invenio.readthedocs.io/en/latest/usage.html#running-e2e-tests>`_.
 
-Documentation
--------------
-You can build the documentation with:
-
-.. code-block:: console
-
-    $ pipenv run build_sphinx
 
 Production environment
 ----------------------
@@ -123,14 +121,23 @@ already done:
 
 .. code-block:: console
 
-    $ docker-compose -f docker-compose.full.yml run --rm web-ui ./scripts/setup
+    $ docker-compose -f docker-compose.full.yml run --rm web-ui "poetry run ./scripts/setup"
 
 In addition to the normal ``docker-compose.yml``, this one will start:
 
-- HAProxy (load balancer) -- https://127.0.0.1 and http://127.0.0.1:8080
+- HAProxy (load balancer)
 - Nginx (web frontend)
 - UWSGI (application container)
 - Celery (background task worker)
-- Flower (Celery monitoring) -- http://127.0.0.1:5555
-- Kibana (Elasticsearch inspection) -- http://127.0.0.1:5601
-- RabbitMQ (message queue) -- http://guest:guest@127.0.0.1:15672
+- Celery (background task beat)
+- Flower (Celery monitoring)
+
+
+
+.. References:
+.. _GitHub: https://github.com/rero/sonar
+.. _Travis: https://travis-ci.org/rero/sonar
+.. _Docker: https://hub.docker.com/r/rero/sonar/
+.. _npm: https://www.npmjs.com/org/rero
+.. _Gitter: https://gitter.im/rero/interne
+.. _Weblate: https://hosted.weblate.org/projects/rero_plus/sonar/

MANIFEST.in

@@ -12,15 +12,6 @@
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 # GNU Affero General Public License for more details.
 
-# To update your MANIFEST.in, use the following commands:
-#
-#  git init  # if not already done
-#  git add -A
-#  pipenv run check-manifest -u  # to install run "pipenv install --dev"
-#  # please sort the lines in this file after running the commands
-
-# Check manifest will not automatically add these two files:
-
 exclude sonar/modules/documents/jsonschemas/documents/document-v1.0.0.json
 exclude sonar/modules/deposits/jsonschemas/deposits/deposit-v1.0.0.json