From 927a268da8c7ae1071d3c2cae9edbd721f566917 Mon Sep 17 00:00:00 2001 From: Arnaldo Gualberto Date: Wed, 31 Aug 2022 21:21:36 -0300 Subject: [PATCH 1/4] =?UTF-8?q?=E2=9E=95=20chore:=20add=20kedro-docker=20d?= =?UTF-8?q?ependency?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/requirements.lock | 11 +++++++++-- src/requirements.txt | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/requirements.lock b/src/requirements.lock index d3079c52..7719eacb 100644 --- a/src/requirements.lock +++ b/src/requirements.lock @@ -7,7 +7,9 @@ aiofiles==0.6.0 # via kedro-viz anyconfig==0.10.1 - # via kedro + # via + # kedro + # kedro-docker anyio==3.6.1 # via # jupyter-server @@ -205,8 +207,11 @@ jupyterlab-widgets==1.1.1 kedro==0.18.2 # via # -r C:\Users\arnal\Documents\GitHub\caRtola\src\requirements.txt + # kedro-docker # kedro-telemetry # kedro-viz +kedro-docker==0.3.0 + # via -r C:\Users\arnal\Documents\GitHub\caRtola\src\requirements.txt kedro-telemetry==0.2.1 # via -r C:\Users\arnal\Documents\GitHub\caRtola\src\requirements.txt kedro-viz==4.7.1 @@ -388,7 +393,9 @@ rich==12.5.1 rope==0.21.1 # via kedro semver==2.13.0 - # via kedro-viz + # via + # kedro-docker + # kedro-viz send2trash==1.8.0 # via # jupyter-server diff --git a/src/requirements.txt b/src/requirements.txt index 0221dbf0..482e0ce3 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -5,6 +5,7 @@ isort~=5.0 jupyter~=1.0 jupyterlab~=3.0 kedro~=0.18.2 +kedro-docker~=0.3.0 kedro-viz==4.7.1 kedro-telemetry~=0.2.0 nbstripout~=0.4 From e9c03bd433d1c87aa88a9994e02a190602fda01e Mon Sep 17 00:00:00 2001 From: Arnaldo Gualberto Date: Thu, 1 Sep 2022 21:13:46 -0300 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=93=8C=20chore:=20add=20missing=20dep?= =?UTF-8?q?endencies?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/requirements.lock | 135 ++++++++++++++++++++---------------------- src/requirements.txt | 3 +- 2 files changed, 66 insertions(+), 72 deletions(-) diff --git a/src/requirements.lock b/src/requirements.lock index 7719eacb..ecdc1098 100644 --- a/src/requirements.lock +++ b/src/requirements.lock @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with python 3.10 # To update, run: # -# pip-compile --output-file='C:\Users\arnal\Documents\GitHub\caRtola\src\requirements.lock' 'C:\Users\arnal\Documents\GitHub\caRtola\src\requirements.txt' +# pip-compile --output-file=/Users/arnaldo/Documents/github/caRtola/src/requirements.lock /Users/arnaldo/Documents/github/caRtola/src/requirements.txt # aiofiles==0.6.0 # via kedro-viz @@ -14,6 +14,10 @@ anyio==3.6.1 # via # jupyter-server # watchgod +appnope==0.1.3 + # via + # ipykernel + # ipython argon2-cffi==21.3.0 # via # jupyter-server @@ -25,8 +29,6 @@ arrow==1.2.2 # via jinja2-time asgiref==3.5.2 # via uvicorn -atomicwrites==1.4.1 - # via pytest attrs==21.4.0 # via # jsonschema @@ -36,14 +38,12 @@ babel==2.10.3 # via jupyterlab-server backcall==0.2.0 # via ipython -backports-cached-property==1.0.2 - # via strawberry-graphql beautifulsoup4==4.11.1 # via nbconvert binaryornot==0.4.4 # via cookiecutter -black==22.6.0 - # via -r C:\Users\arnal\Documents\GitHub\caRtola\src\requirements.txt +black==22.8.0 + # via -r /Users/arnaldo/Documents/github/caRtola/src/requirements.txt bleach==5.0.1 # via nbconvert build==0.8.0 @@ -66,20 +66,13 @@ click==8.1.3 # pip-tools # strawberry-graphql # uvicorn -colorama==0.4.5 - # via - # build - # click - # ipython - # pytest - # uvicorn commonmark==0.9.1 # via rich cookiecutter==2.1.1 # via kedro -coverage[toml]==6.4.2 +coverage[toml]==6.4.4 # via pytest-cov -debugpy==1.6.2 +debugpy==1.6.3 # via ipykernel decorator==5.1.1 # via ipython @@ -88,15 +81,13 @@ defusedxml==0.7.1 dynaconf==3.1.9 # via kedro entrypoints==0.4 - # via - # jupyter-client - # nbconvert + # via jupyter-client fastapi==0.66.1 # via kedro-viz fastjsonschema==2.16.1 # via nbformat flake8==4.0.1 - # via -r C:\Users\arnal\Documents\GitHub\caRtola\src\requirements.txt + # via -r /Users/arnaldo/Documents/github/caRtola/src/requirements.txt fsspec==2022.1.0 # via kedro gitdb==4.0.9 @@ -105,8 +96,6 @@ gitpython==3.1.27 # via kedro graphql-core==3.2.1 # via strawberry-graphql -greenlet==1.1.2 - # via sqlalchemy h11==0.13.0 # via uvicorn httptools==0.4.0 @@ -119,7 +108,7 @@ importlib-metadata==4.12.0 # via kedro iniconfig==1.1.1 # via pytest -ipykernel==6.15.1 +ipykernel==6.15.2 # via # ipywidgets # jupyter @@ -129,7 +118,7 @@ ipykernel==6.15.1 # qtconsole ipython==7.34.0 # via - # -r C:\Users\arnal\Documents\GitHub\caRtola\src\requirements.txt + # -r /Users/arnaldo/Documents/github/caRtola/src/requirements.txt # ipykernel # ipywidgets # jupyter-console @@ -137,14 +126,13 @@ ipython==7.34.0 # kedro-viz ipython-genutils==0.2.0 # via - # ipywidgets # nbclassic # notebook # qtconsole -ipywidgets==7.7.1 +ipywidgets==8.0.1 # via jupyter isort==5.10.1 - # via -r C:\Users\arnal\Documents\GitHub\caRtola\src\requirements.txt + # via -r /Users/arnaldo/Documents/github/caRtola/src/requirements.txt jedi==0.18.1 # via ipython jinja2==3.1.2 @@ -161,15 +149,15 @@ jinja2-time==0.2.0 # via cookiecutter jmespath==0.10.0 # via kedro -json5==0.9.8 +json5==0.9.10 # via jupyterlab-server -jsonschema==4.9.0 +jsonschema==4.15.0 # via # jupyterlab-server # nbformat jupyter==1.0.0 - # via -r C:\Users\arnal\Documents\GitHub\caRtola\src\requirements.txt -jupyter-client==7.3.4 + # via -r /Users/arnaldo/Documents/github/caRtola/src/requirements.txt +jupyter-client==7.3.5 # via # ipykernel # jupyter-console @@ -196,45 +184,47 @@ jupyter-server==1.18.1 # jupyterlab-server # nbclassic # notebook-shim -jupyterlab==3.4.4 - # via -r C:\Users\arnal\Documents\GitHub\caRtola\src\requirements.txt +jupyterlab==3.4.5 + # via -r /Users/arnaldo/Documents/github/caRtola/src/requirements.txt jupyterlab-pygments==0.2.2 # via nbconvert -jupyterlab-server==2.15.0 +jupyterlab-server==2.15.1 # via jupyterlab -jupyterlab-widgets==1.1.1 +jupyterlab-widgets==3.0.2 # via ipywidgets kedro==0.18.2 # via - # -r C:\Users\arnal\Documents\GitHub\caRtola\src\requirements.txt + # -r /Users/arnaldo/Documents/github/caRtola/src/requirements.txt # kedro-docker # kedro-telemetry # kedro-viz kedro-docker==0.3.0 - # via -r C:\Users\arnal\Documents\GitHub\caRtola\src\requirements.txt + # via -r /Users/arnaldo/Documents/github/caRtola/src/requirements.txt kedro-telemetry==0.2.1 - # via -r C:\Users\arnal\Documents\GitHub\caRtola\src\requirements.txt + # via -r /Users/arnaldo/Documents/github/caRtola/src/requirements.txt kedro-viz==4.7.1 - # via -r C:\Users\arnal\Documents\GitHub\caRtola\src\requirements.txt + # via -r /Users/arnaldo/Documents/github/caRtola/src/requirements.txt +lxml==4.9.1 + # via nbconvert markupsafe==2.1.1 # via # jinja2 # nbconvert -matplotlib-inline==0.1.3 +matplotlib-inline==0.1.6 # via # ipykernel # ipython mccabe==0.6.1 # via flake8 -mistune==0.8.4 +mistune==2.0.4 # via nbconvert mypy-extensions==0.4.3 # via black nbclassic==0.4.3 # via jupyterlab -nbclient==0.6.6 +nbclient==0.6.7 # via nbconvert -nbconvert==6.5.0 +nbconvert==7.0.0 # via # jupyter # jupyter-server @@ -249,7 +239,7 @@ nbformat==5.4.0 # nbstripout # notebook nbstripout==0.6.0 - # via -r C:\Users\arnal\Documents\GitHub\caRtola\src\requirements.txt + # via -r /Users/arnaldo/Documents/github/caRtola/src/requirements.txt nest-asyncio==1.5.5 # via # ipykernel @@ -257,16 +247,15 @@ nest-asyncio==1.5.5 # nbclassic # nbclient # notebook -networkx==2.8.5 +networkx==2.8.6 # via kedro-viz notebook==6.4.12 # via # jupyter # jupyterlab - # widgetsnbextension notebook-shim==0.1.0 # via nbclassic -numpy==1.23.1 +numpy==1.23.2 # via pandas packaging==21.3 # via @@ -278,23 +267,25 @@ packaging==21.3 # nbconvert # pytest # qtpy -pandas==1.4.3 +pandas==1.4.4 # via kedro-viz pandocfilters==1.5.0 # via nbconvert parso==0.8.3 # via jedi -pathspec==0.9.0 +pathspec==0.10.0 # via black pep517==0.13.0 # via build +pexpect==4.8.0 + # via ipython pickleshare==0.7.5 # via ipython pip-tools==6.8.0 # via kedro platformdirs==2.5.2 # via black -plotly==5.9.0 +plotly==5.10.0 # via kedro-viz pluggy==1.0.0 # via @@ -311,17 +302,21 @@ prompt-toolkit==3.0.30 # jupyter-console psutil==5.9.1 # via ipykernel +ptyprocess==0.7.0 + # via + # pexpect + # terminado py==1.11.0 # via pytest pycodestyle==2.8.0 # via flake8 pycparser==2.21 # via cffi -pydantic==1.9.1 +pydantic==1.10.1 # via fastapi pyflakes==2.4.0 # via flake8 -pygments==2.12.0 +pygments==2.13.0 # via # ipython # jupyter-console @@ -335,13 +330,13 @@ pyrsistent==0.18.1 # via jsonschema pytest==6.2.5 # via - # -r C:\Users\arnal\Documents\GitHub\caRtola\src\requirements.txt + # -r /Users/arnaldo/Documents/github/caRtola/src/requirements.txt # pytest-cov # pytest-mock pytest-cov==3.0.0 - # via -r C:\Users\arnal\Documents\GitHub\caRtola\src\requirements.txt + # via -r /Users/arnaldo/Documents/github/caRtola/src/requirements.txt pytest-mock==1.13.0 - # via -r C:\Users\arnal\Documents\GitHub\caRtola\src\requirements.txt + # via -r /Users/arnaldo/Documents/github/caRtola/src/requirements.txt python-dateutil==2.8.2 # via # arrow @@ -356,22 +351,16 @@ python-multipart==0.0.5 # via strawberry-graphql python-slugify==6.1.2 # via cookiecutter -pytz==2022.1 +pytz==2022.2.1 # via # babel # pandas -pywin32==304 - # via jupyter-core -pywinpty==2.0.6 - # via - # jupyter-server - # terminado pyyaml==6.0 # via # cookiecutter # kedro # uvicorn -pyzmq==23.2.0 +pyzmq==23.2.1 # via # ipykernel # jupyter-client @@ -379,9 +368,9 @@ pyzmq==23.2.0 # nbclassic # notebook # qtconsole -qtconsole==5.3.1 +qtconsole==5.3.2 # via jupyter -qtpy==2.1.0 +qtpy==2.2.0 # via qtconsole requests==2.25.1 # via @@ -408,15 +397,15 @@ six==1.16.0 # python-multipart smmap==5.0.0 # via gitdb -sniffio==1.2.0 +sniffio==1.3.0 # via anyio soupsieve==2.3.2.post1 # via beautifulsoup4 -sqlalchemy==1.4.39 +sqlalchemy==1.4.40 # via kedro-viz starlette==0.14.2 # via fastapi -strawberry-graphql==0.122.1 +strawberry-graphql==0.127.4 # via kedro-viz tenacity==8.0.1 # via plotly @@ -469,10 +458,14 @@ typing-extensions==4.3.0 # via # pydantic # strawberry-graphql -urllib3==1.26.11 +unidecode==1.3.4 + # via -r /Users/arnaldo/Documents/github/caRtola/src/requirements.txt +urllib3==1.26.12 # via requests uvicorn[standard]==0.17.6 # via kedro-viz +uvloop==0.16.0 + # via uvicorn watchgod==0.8.2 # via # kedro-viz @@ -483,13 +476,13 @@ webencodings==0.5.1 # via # bleach # tinycss2 -websocket-client==1.3.3 +websocket-client==1.4.0 # via jupyter-server websockets==10.3 # via uvicorn wheel==0.37.1 # via pip-tools -widgetsnbextension==3.6.1 +widgetsnbextension==4.0.2 # via ipywidgets zipp==3.8.1 # via importlib-metadata diff --git a/src/requirements.txt b/src/requirements.txt index 482e0ce3..9420d2ee 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -4,7 +4,7 @@ ipython>=7.31.1, <8.0 isort~=5.0 jupyter~=1.0 jupyterlab~=3.0 -kedro~=0.18.2 +kedro>=0.18.0 kedro-docker~=0.3.0 kedro-viz==4.7.1 kedro-telemetry~=0.2.0 @@ -12,3 +12,4 @@ nbstripout~=0.4 pytest-cov~=3.0 pytest-mock>=1.7.1, <2.0 pytest~=6.2 +unidecode~=1.3.4 From 883cc670a6e89f926f8a99f96d1760a0167b9f29 Mon Sep 17 00:00:00 2001 From: Arnaldo Gualberto Date: Thu, 1 Sep 2022 21:14:03 -0300 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=90=9B=20fix:=20=5F=5Fdefault=5F=5F?= =?UTF-8?q?=20pipeline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/cartola/pipeline_registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cartola/pipeline_registry.py b/src/cartola/pipeline_registry.py index 6562fc42..a554b1f7 100644 --- a/src/cartola/pipeline_registry.py +++ b/src/cartola/pipeline_registry.py @@ -79,7 +79,7 @@ def register_pipelines() -> Dict[str, Pipeline]: return { "__default__": ( - pipe_2014 + pipe_2015 + pipe_2016 + pipe_2017 + pipe_2018 + pipe_2019 + pipe_2020 + pipe_2021 + pipe_2022, + pipe_2014 + pipe_2015 + pipe_2016 + pipe_2017 + pipe_2018 + pipe_2019 + pipe_2020 + pipe_2021 + pipe_2022 ), "2014": pipe_2014, "2015": pipe_2015, From 4435ad7839c32d568dae4f8a43ecacbf9ac00a08 Mon Sep 17 00:00:00 2001 From: Arnaldo Gualberto Date: Thu, 1 Sep 2022 21:15:13 -0300 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=90=B3=20feat:=20support=20to=20docke?= =?UTF-8?q?r?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dive-ci | 13 +++++++++++++ .dockerignore | 26 ++++++++++++++++++++++++++ Dockerfile | 23 +++++++++++++++++++++++ 3 files changed, 62 insertions(+) create mode 100644 .dive-ci create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dive-ci b/.dive-ci new file mode 100644 index 00000000..447a947f --- /dev/null +++ b/.dive-ci @@ -0,0 +1,13 @@ +rules: + # If the efficiency is measured below X%, mark as failed. + # Expressed as a ratio between 0-1. + lowestEfficiency: 0.95 + + # If the amount of wasted space is at least X or larger than X, mark as failed. + # Expressed in B, KB, MB, and GB. + highestWastedBytes: 50MB + + # If the amount of wasted space makes up for X% or more of the image, mark as failed. + # Note: the base image layer is NOT included in the total image size. + # Expressed as a ratio between 0-1; fails if the threshold is met or crossed. + highestUserWastedPercent: 0.20 diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..7227c16c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,26 @@ +########################## +# Kedro PROJECT + +# ignore Dockerfile and .dockerignore +Dockerfile +.dockerignore + +# ignore potentially sensitive credentials files +conf/**/*credentials* + +# ignore all local configuration +conf/local +!conf/local/.gitkeep + +# ignore everything in the following folders +data +logs +notebooks +references +results + +# except the following +!logs/.gitkeep +!notebooks/.gitkeep +!references/.gitkeep +!results/.gitkeep diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..cf12cf06 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +ARG BASE_IMAGE=python:3.6-buster +FROM $BASE_IMAGE + +# install project requirements +COPY src/requirements.txt /tmp/requirements.txt +RUN pip install -r /tmp/requirements.txt && rm -f /tmp/requirements.txt + +# add kedro user +ARG KEDRO_UID=999 +ARG KEDRO_GID=0 +RUN groupadd -f -g ${KEDRO_GID} kedro_group && \ +useradd -d /home/kedro -s /bin/bash -g ${KEDRO_GID} -u ${KEDRO_UID} kedro + +# copy the whole project except what is in .dockerignore +WORKDIR /home/kedro +COPY . . +RUN chown -R kedro:${KEDRO_GID} /home/kedro +USER kedro +RUN chmod -R a+w /home/kedro + +EXPOSE 8888 + +CMD ["kedro", "run"]