feat(databases): create CACHE structure to ftp Directories & add CNES…

… database (#152)

* chore(Databases): explicity require load() content on Databases

* Finish CNES

* remove lru_cache; returning [] when it should return content

* Include download and async download methods on Databases

* test

* return the database when loading

* Update SINAN Notebook

* Update SIM Notebook

* Improving Directory class

* Rebase

* fix poetry lock

* add tomli

* Recursive Directory instantiation

* Refining Dirs CACHE and navigation

* Minor changes

* Remove requirements.txt

* Move databases to their own modules, reducing the initial load time

* Poetry lock

* fix(docs): remove unecessary steps to build notebooks container

* fix(databases): fix CNES.get_files and make Database.__content__ a dictionary

* fix(Directory): fix wrong __content__ ref on directory loading

* fix(File): fix File __info__ & info properties

* fix(ftp): closing all possible opened ftp connections

* docs(dbs): move CNES & SIM notebooks to databases docs

* fix(sphinx): sphinx needs nbsphinx to be installed by conda

* fix(ftp): loading directories when loading dbs

* docs(dbs): move SINAN notebook to databases docs

* docs(dbs): move SINASC notebook to databases docs

* docs(dbs): Include Databases on sphinx index

* fix(online_data): fix references on online_data for databases usage

* fix(Directory): remove unnecessary root loading

* docs(dbs): create SIA notebook on databases docs & minor changes

* docs(dbs): create SIH notebook on databases docs & rerun database notebooks

* docs(all): comment broken notebooks to fix in next PR, move documentation to a better structure

* fix(dockerfile): fix notebooks paths on container

.github/workflows/python-package.yml

@@ -26,7 +26,7 @@ jobs:
         mamba-version: "*"
         environment-file: conda/dev.yaml
         channels: conda-forge,nodefaults
-        activate-environment: env-pysus
+        activate-environment: pysus
         use-mamba: true
         miniforge-variant: Mambaforge
 

.github/workflows/release.yaml

@@ -24,7 +24,7 @@ jobs:
         mamba-version: "*"
         environment-file: conda/dev.yaml
         channels: conda-forge,nodefaults
-        activate-environment: env-pysus
+        activate-environment: pysus
         use-mamba: true
         miniforge-variant: Mambaforge
 

Makefile

@@ -36,21 +36,6 @@ SEMANTIC_RELEASE = npx --yes \
           semantic-release
 
 
-#* Poetry
-.PHONY: poetry-download
-poetry-download:
-	curl -sSL https://install.python-poetry.org | $(PYTHON) -
-
-.PHONY: poetry-remove
-poetry-remove:
-	curl -sSL https://install.python-poetry.org | $(PYTHON) - --uninstall
-
-#* Installation
-.PHONY: install
-install:
-	poetry lock -n && poetry export --without-hashes > requirements.txt
-	poetry build && poetry install
-
 .PHONY: pre-commit-install
 pre-commit-install:
 	poetry run pre-commit install
@@ -79,8 +64,9 @@ test-jupyter-pysus: ## run pytest for notebooks inside jupyter container
 
 .PHONY: test
 test: ## run tests quickly with the default Python
+	cp docs/source/**/*.ipynb pysus/Notebooks
 	poetry run pytest -vv pysus/tests/
-	poetry run pytest --nbmake --nbmake-timeout=800 docs/source/*.ipynb pysus/Notebooks/*.ipynb
+	poetry run pytest --nbmake --nbmake-timeout=800 pysus/Notebooks/*.ipynb
 
 coverage: ## check code coverage quickly with the default Python
 	coverage run --source pysus/tests/ -m pytest

conda/dev.yaml

@@ -1,18 +1,12 @@
-name: env-pysus
+name: pysus
 channels:
   - conda-forge
   - nodefaults
 dependencies:
+  - python >= 3.11
   - cffi
-  - docker-compose
   - gcc
   - make
+  - nbsphinx
   - pip
-  - psycopg2
-  - python 3.9.*
   - poetry >= 1.3.2
-  - pip:
-    - urllib3 ==1.26.15
-    - requests
-    - pandas
-    - sqlalchemy

docker/Dockerfile

@@ -14,8 +14,11 @@ ENV PATH "/opt/poetry/bin:$PATH"
 
 RUN apt-get -qq update --yes \
   && apt-get -qq install --yes --no-install-recommends \
-  build-essential firefox\
-  ca-certificates sudo curl \
+  build-essential \
+  firefox \
+  ca-certificates \
+  sudo \
+  curl \
   && rm -rf /var/lib/apt/lists/*
 
 RUN useradd -ms /bin/bash pysus \
@@ -29,27 +32,18 @@ RUN useradd -ms /bin/bash pysus \
 
 COPY --chown=pysus:pysus conda/dev.yaml /tmp/dev.yaml
 COPY --chown=pysus:pysus docker/scripts/entrypoint.sh /entrypoint.sh
-COPY --chown=pysus:pysus docker/scripts/poetry_install.sh /tmp/poetry_install.sh
+COPY --chown=pysus:pysus docker/scripts/poetry-install.sh /tmp/poetry-install.sh
 COPY --chown=pysus:pysus pyproject.toml poetry.lock LICENSE README.md /usr/src/
 COPY --chown=pysus:pysus pysus /usr/src/pysus
-COPY --chown=pysus:pysus docs/source/*.ipynb /home/pysus/Notebooks/
-
-RUN curl -sSL https://install.python-poetry.org/ | POETRY_HOME=/opt/poetry python && \
-    cd /usr/local/bin && \
-    ln -s /opt/poetry/bin/poetry && \
-    poetry config virtualenvs.create false
-
-RUN chmod -R a+rwx /home/pysus/.config/pypoetry/ \
-    && chown -R pysus:pysus /opt/conda/bin/
+COPY --chown=pysus:pysus docs/source/**/*.ipynb /home/pysus/Notebooks/
 
 USER pysus
 
 RUN mamba env create -n $ENV_NAME --file /tmp/dev.yaml \
-  && cd /usr/src \
-  && poetry build \
-  && poetry install \
   && mamba clean -afy
 
+RUN cd /usr/src/ && bash /tmp/poetry-install.sh
+
 WORKDIR /home/pysus/Notebooks
 
 ENTRYPOINT ["bash", "/activate.sh"]

docker/scripts/poetry-install.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -ex
+
+poetry config virtualenvs.create false
+poetry install --all-extras --with docs

docs/requirements.txt

@@ -1,2 +1,2 @@
 nbsphinx
-sphinx
+sphinx