From e02940cfdb221bf446dedfc42d0724d437c238b5 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Fri, 21 Sep 2018 11:47:03 -0700 Subject: [PATCH 1/5] [AIRFLOW-3100][AIRFLOW-3101] Add installing wheel and tox to the Dockerfile, add ~/.local/bin to the path. Comment on how to run individual tests & document workaround for now with installing tox when launching image. --- CONTRIBUTING.md | 9 +++++++++ scripts/ci/docker-compose.yml | 1 + scripts/ci/kubernetes/docker/Dockerfile | 2 ++ 3 files changed, 12 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 152d5d9aabf16..f114c66585342 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -146,9 +146,18 @@ There are three ways to setup an Apache Airflow development environment. # From the container pip install -e .[devel] # Run all the tests with python and mysql through tox + pip install tox tox -e py35-backend_mysql ``` + If you wish to run individual tests inside of docker enviroment you can do as follows: + + ```bash + # From the container (with your desired enviroment) with druid hook + tox -e py35-backend_mysql -- tests/hooks/test_druid_hook.py + ``` + + ### Running unit tests To run tests locally, once your unit test environment is setup (directly on your diff --git a/scripts/ci/docker-compose.yml b/scripts/ci/docker-compose.yml index 4accf119f68f2..9a5b410ea8210 100644 --- a/scripts/ci/docker-compose.yml +++ b/scripts/ci/docker-compose.yml @@ -66,6 +66,7 @@ services: init: true environment: - USER=airflow + - PATH=$PATH:~/.local/bin - SLUGIFY_USES_TEXT_UNIDECODE=yes - TOX_ENV - PYTHON_VERSION diff --git a/scripts/ci/kubernetes/docker/Dockerfile b/scripts/ci/kubernetes/docker/Dockerfile index 93b20dbcd22fa..19d5b915672c9 100644 --- a/scripts/ci/kubernetes/docker/Dockerfile +++ b/scripts/ci/kubernetes/docker/Dockerfile @@ -36,6 +36,8 @@ RUN apt-get update -y && apt-get install -y \ && apt-get clean RUN pip install --upgrade pip +RUN pip install wheel +RUN pip install tox # Since we install vanilla Airflow, we also want to have support for Postgres and Kubernetes RUN pip install -U setuptools && \ From 4552b10f527e7d005eb71b56317a01c65d21408c Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Fri, 21 Sep 2018 15:02:01 -0700 Subject: [PATCH 2/5] Install wheel and tox at the same time in the dockerfile --- scripts/ci/kubernetes/docker/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/ci/kubernetes/docker/Dockerfile b/scripts/ci/kubernetes/docker/Dockerfile index 19d5b915672c9..d215612d94864 100644 --- a/scripts/ci/kubernetes/docker/Dockerfile +++ b/scripts/ci/kubernetes/docker/Dockerfile @@ -36,8 +36,7 @@ RUN apt-get update -y && apt-get install -y \ && apt-get clean RUN pip install --upgrade pip -RUN pip install wheel -RUN pip install tox +RUN pip install wheel tox # Since we install vanilla Airflow, we also want to have support for Postgres and Kubernetes RUN pip install -U setuptools && \ From 1ea8f5b71adc66ad76affc4255ff33f066c35330 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Mon, 24 Sep 2018 19:31:13 -0700 Subject: [PATCH 3/5] remove wheel/tox since we only need it in the CI side --- scripts/ci/kubernetes/docker/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/ci/kubernetes/docker/Dockerfile b/scripts/ci/kubernetes/docker/Dockerfile index d215612d94864..93b20dbcd22fa 100644 --- a/scripts/ci/kubernetes/docker/Dockerfile +++ b/scripts/ci/kubernetes/docker/Dockerfile @@ -36,7 +36,6 @@ RUN apt-get update -y && apt-get install -y \ && apt-get clean RUN pip install --upgrade pip -RUN pip install wheel tox # Since we install vanilla Airflow, we also want to have support for Postgres and Kubernetes RUN pip install -U setuptools && \ From 3cf12c1673b7cf851edcdf75a5cacce30480ca36 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Mon, 24 Sep 2018 20:33:55 -0700 Subject: [PATCH 4/5] Use sudo -H as per the warning in travis about file permissions. --- scripts/ci/run-ci.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci/run-ci.sh b/scripts/ci/run-ci.sh index 1a65bf1ef6443..f16ab5284dd44 100755 --- a/scripts/ci/run-ci.sh +++ b/scripts/ci/run-ci.sh @@ -32,8 +32,8 @@ else PIP=pip fi -sudo $PIP install --upgrade pip -sudo $PIP install tox +sudo -H $PIP install --upgrade pip +sudo -H $PIP install tox cd $AIRFLOW_ROOT && $PIP --version && tox --version From db7279c675a956e96a49ec8417d262def3b6037e Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Mon, 24 Sep 2018 20:50:14 -0700 Subject: [PATCH 5/5] Set ADDITIONAL_PATH instead of PATH since using $PATH refs the host env rather than docker env :( --- scripts/ci/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/docker-compose.yml b/scripts/ci/docker-compose.yml index 9a5b410ea8210..101ad9529731e 100644 --- a/scripts/ci/docker-compose.yml +++ b/scripts/ci/docker-compose.yml @@ -66,7 +66,7 @@ services: init: true environment: - USER=airflow - - PATH=$PATH:~/.local/bin + - ADDITIONAL_PATH=~/.local/bin - SLUGIFY_USES_TEXT_UNIDECODE=yes - TOX_ENV - PYTHON_VERSION