|
| 1 | +#!/usr/bin/env bash |
| 2 | +# Licensed to the Apache Software Foundation (ASF) under one |
| 3 | +# or more contributor license agreements. See the NOTICE file |
| 4 | +# distributed with this work for additional information |
| 5 | +# regarding copyright ownership. The ASF licenses this file |
| 6 | +# to you under the Apache License, Version 2.0 (the |
| 7 | +# "License"); you may not use this file except in compliance |
| 8 | +# with the License. You may obtain a copy of the License at |
| 9 | +# |
| 10 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | +# |
| 12 | +# Unless required by applicable law or agreed to in writing, |
| 13 | +# software distributed under the License is distributed on an |
| 14 | +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 15 | +# KIND, either express or implied. See the License for the |
| 16 | +# specific language governing permissions and limitations |
| 17 | +# under the License. |
| 18 | + |
| 19 | +# This autodetection of PYTHON_VERSION works in this sequence: |
| 20 | +# |
| 21 | +# 1) When running builds on DockerHub we cannot pass different environment |
| 22 | +# variables there, so we detect python version based on the image |
| 23 | +# name being build (airflow:master-python3.7 -> PYTHON_VERSION=3.7). |
| 24 | +# DockerHub adds index.docker.io in front of the image name. |
| 25 | +# |
| 26 | +# 2) When running builds on Travis CI. We use python version determined |
| 27 | +# from default python3 version available on the path. This way we |
| 28 | +# do not have to specify PYTHON_VERSION separately in each job. |
| 29 | +# We just specify which host python version is used for that job. |
| 30 | +# This makes a nice UI experience where you see python version in |
| 31 | +# Travis UI. |
| 32 | +# |
| 33 | +# 3) Running builds locally via scripts where we can pass PYTHON_VERSION |
| 34 | +# as environment variable. |
| 35 | +# |
| 36 | +# 4) Running builds locally for the first time with Breeze. By default |
| 37 | +# we determine the version based on default python3 version we have |
| 38 | +# in the host system (3.5, 3.6 or 3.7) and we use this one. |
| 39 | +# |
| 40 | +# 5) Selecting python version with Breeze's --python switch. This will |
| 41 | +# override python version but it will also store the last used version |
| 42 | +# of python in .build directory so that it is automatically used next |
| 43 | +# time. |
| 44 | + |
| 45 | +# shellcheck source=common/_default_branch.sh |
| 46 | +. "${AIRFLOW_SOURCES}/common/_default_branch.sh" |
| 47 | + |
| 48 | +# You can override DOCKERHUB_USER to use your own DockerHub account and play with your |
| 49 | +# own docker images. In this case you can build images locally and push them |
| 50 | +export DOCKERHUB_USER=${DOCKERHUB_USER:="apache"} |
| 51 | + |
| 52 | +# You can override DOCKERHUB_REPO to use your own DockerHub repository and play with your |
| 53 | +# own docker images. In this case you can build images locally and push them |
| 54 | +export DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow"} |
| 55 | + |
| 56 | +# if AIRFLOW_CONTAINER_BRANCH_NAME is not set it will be set to either SOURCE_BRANCH |
| 57 | +# (if overridden by configuration) or default branch configured in /common/_default_branch.sh |
| 58 | +SOURCE_BRANCH=${SOURCE_BRANCH:=${DEFAULT_BRANCH}} |
| 59 | + |
| 60 | +AIRFLOW_CONTAINER_BRANCH_NAME=${AIRFLOW_CONTAINER_BRANCH_NAME:=${SOURCE_BRANCH}} |
| 61 | + |
| 62 | +echo |
| 63 | +echo "Branch: ${AIRFLOW_CONTAINER_BRANCH_NAME}" |
| 64 | +echo |
| 65 | + |
| 66 | +# You can override AUTODETECT_PYTHON_BINARY if you want to use different version but for now we assume |
| 67 | +# that the binary used will be the default python 3 version available on the path |
| 68 | +# unless it is overridden with PYTHON_VERSION variable in the next step |
| 69 | +if ! AUTODETECT_PYTHON_BINARY=${AUTODETECT_PYTHON_BINARY:=$(command -v python3)}; then |
| 70 | + echo >&2 |
| 71 | + echo >&2 "Error: You must have python3 in your PATH" |
| 72 | + echo >&2 |
| 73 | + exit 1 |
| 74 | +fi |
| 75 | + |
| 76 | +# Determine python version. This can be either specified by AUTODETECT_PYTHON_VERSION variable or it is |
| 77 | +# auto-detected from the version of the python3 binary (or AUTODETECT_PYTHON_BINARY you specify as |
| 78 | +# environment variable). |
| 79 | +# Note that even if we auto-detect it here, it can be further overridden in case IMAGE_NAME is speecified |
| 80 | +# as environment variable. The reason is that IMAGE_NAME is the only differentiating factor we can have |
| 81 | +# in the DockerHub build. We cannot specify different environment variables for different image names |
| 82 | +# so we use IMAGE_NAME to determine which PYTHON_VERSION is actually used for this particular build. |
| 83 | +# It's cumbersome - we can improve it in the future by swtching away from DockerHub builds - only use |
| 84 | +# DockerHub to store the images but not to run it to build the images. If we switch to another CI that |
| 85 | +# will let us build images outside of DockerHub and push them there, we will be able to get rid of this. |
| 86 | +# This will probably be necessary as we scale up becasue DockerHub build are very slow and they are |
| 87 | +# already queueing a lot. |
| 88 | +export AUTODETECT_PYTHON_VERSION=${PYTHON_VERSION:=$(${AUTODETECT_PYTHON_BINARY} -c \ |
| 89 | + 'import sys;print("%s.%s" % (sys.version_info.major, sys.version_info.minor))')} |
| 90 | + |
| 91 | +# IMAGE_NAME might come from DockerHub and we decide which PYTHON_VERSION to use based on it (see below) |
| 92 | +# In case IMAGE_NAME is not set we determine PYTHON_VERSION based on the default python on the path |
| 93 | +# So in virtualenvs it will use the virtualenv's PYTHON_VERSION and in DockerHub it will determine |
| 94 | +# PYTHHON_VERSION from the IMAGE_NAME set in the DockerHub build. |
| 95 | +# See comment above in PYTHON_VERSION - we will be able to get rid of this cumbersomness when we switch |
| 96 | +# to a different CI system and start pushing images to DockerHub rather than build it there. |
| 97 | +export BASE_IMAGE_NAME=${IMAGE_NAME:=${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CONTAINER_BRANCH_NAME}-python${AUTODETECT_PYTHON_VERSION}} |
| 98 | + |
| 99 | +echo |
| 100 | +echo "BASE_IMAGE_NAME=${BASE_IMAGE_NAME:=} (final image will have -ci, -ci-slim suffixes)" |
| 101 | +echo |
| 102 | + |
| 103 | +# Remove index.docker.io/ prefix as it is added by default by DockerHub |
| 104 | +export LOCAL_BASE_IMAGE_NAME=${BASE_IMAGE_NAME#index.docker.io/} |
| 105 | +echo |
| 106 | +echo "LOCAL_BASE_IMAGE_NAME=${LOCAL_BASE_IMAGE_NAME}" |
| 107 | +echo |
| 108 | + |
| 109 | +if [[ ! ${LOCAL_BASE_IMAGE_NAME} == ${DOCKERHUB_USER}/${DOCKERHUB_REPO}* ]]; then |
| 110 | + echo >&2 |
| 111 | + echo >&2 "ERROR: The ${LOCAL_BASE_IMAGE_NAME} does not start with ${DOCKERHUB_USER}/${DOCKERHUB_REPO}" |
| 112 | + echo >&2 |
| 113 | + exit 1 |
| 114 | +fi |
| 115 | + |
| 116 | +# Extract IMAGE_TAG from LOCAL_BASE_IMAGE_NAME (apache/airflow:master-python3.6 -> master-python3.6) |
| 117 | +IMAGE_TAG=${LOCAL_BASE_IMAGE_NAME##${DOCKERHUB_USER}/${DOCKERHUB_REPO}:} |
| 118 | +echo |
| 119 | +echo "Image tag: ${IMAGE_TAG}" |
| 120 | +echo |
| 121 | + |
| 122 | +# Extract TAG_PREFIX from IMAGE_TAG (master-python3.6 -> master) |
| 123 | +TAG_PREFIX=${IMAGE_TAG%-*} |
| 124 | +echo |
| 125 | +echo "Image tag prefix: ${TAG_PREFIX}" |
| 126 | +echo |
| 127 | + |
| 128 | +# Extract python version from image name we want to build in case it was passed |
| 129 | +# Via IMAGE_NAME. The bash construct below extracts last field after '-' delimiter |
| 130 | +# So for example 'airflow:master-python3.6' will produce AUTODETECT_LONG_PYTHON_VERSION=python3.6 |
| 131 | +export AUTODETECT_LONG_PYTHON_VERSION="${LOCAL_BASE_IMAGE_NAME##*-}" |
| 132 | + |
| 133 | +echo |
| 134 | +echo "AUTODETECT_LONG_PYTHON_VERSION=${AUTODETECT_LONG_PYTHON_VERSION}" |
| 135 | +echo |
| 136 | + |
| 137 | +# Verify that Auto-detected python version follows the expected pattern of python3.Y |
| 138 | +if [[ ! ${AUTODETECT_LONG_PYTHON_VERSION} =~ python[3]\.[0-9]+ ]]; then |
| 139 | + echo >&2 |
| 140 | + echo >&2 "ERROR: Python version extracted from IMAGE_NAME does not match the python3.Y format" |
| 141 | + echo >&2 |
| 142 | + echo >&2 "The IMAGE_NAME format should be '<BRANCH>-python3.Y'" |
| 143 | + echo >&2 |
| 144 | + exit 1 |
| 145 | +fi |
| 146 | + |
| 147 | +# Now set the auto-detected python version based on the auto-detected long version |
| 148 | +export PYTHON_VERSION=${AUTODETECT_LONG_PYTHON_VERSION#python} |
| 149 | + |
| 150 | +echo |
| 151 | +echo "PYTHON_VERSION=${PYTHON_VERSION}" |
| 152 | +echo |
0 commit comments