Dockerfiles Python 3 cleanup (#3601)

.travis.yml

@@ -16,10 +16,9 @@ addons:
     packages:
       - libtool-bin
       - libcppunit-dev
-      - python3
       - pkg-config
       - python3-dev
-      - python3-wheel
+      - python3-venv
       - wget
       - zip
       - zlib1g-dev

WORKSPACE

@@ -181,10 +181,6 @@ REQUESTS_SRC = "https://pypi.python.org/packages/d9/03/155b3e67fe35fe5b6f4227a8d
 
 SETUPTOOLS_WHEEL = "https://pypi.python.org/packages/a0/df/635cdb901ee4a8a42ec68e480c49f85f4c59e8816effbf57d9e6ee8b3588/setuptools-46.1.3-py3-none-any.whl"
 
-VIRTUALENV_SRC = "https://pypi.python.org/packages/d4/0c/9840c08189e030873387a73b90ada981885010dd9aea134d6de30cd24cb8/virtualenv-15.1.0.tar.gz"
-
-VIRTUALENV_PREFIX = "virtualenv-15.1.0"
-
 WHEEL_SRC = "https://pypi.python.org/packages/c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/wheel-0.29.0.tar.gz"
 
 http_file(
@@ -229,21 +225,6 @@ http_file(
     urls = [SETUPTOOLS_WHEEL],
 )
 
-http_archive(
-    name = "virtualenv",
-    build_file_content = "\n".join([
-        "load(\"@rules_python//python:defs.bzl\", \"py_binary\")",
-        "py_binary(",
-        "    name = 'virtualenv',",
-        "    srcs = ['virtualenv.py'],",
-        "    data = glob(['**/*']),",
-        "    visibility = ['//visibility:public'],",
-        ")",
-    ]),
-    sha256 = "02f8102c2436bb03b3ee6dede1919d1dac8a427541652e5ec95171ec8adbc93a",
-    strip_prefix = VIRTUALENV_PREFIX,
-    urls = [VIRTUALENV_SRC],
-)
 # end pex repos
 
 # protobuf dependencies for C++ and Java

bazel_configure.py

@@ -37,6 +37,7 @@
 import datetime
 import platform
 import subprocess
+import tempfile
 
 sys.path.append('third_party/python/semver')
 import semver
@@ -293,6 +294,11 @@ def discover_jdk():
   print('Using %s:\t%s' % ('JDK'.ljust(20), jdk_path))
   return jdk_path
 
+def test_venv():
+  with tempfile.TemporaryDirectory() as tmpdirname:
+    if subprocess.run(["python3", "-m", "venv", tmpdirname]).returncode != 0:
+      fail("Python3 venv module is not installed.")
+
 ######################################################################
 # Discover the linker directory
 ######################################################################
@@ -413,7 +419,8 @@ def main():
   env_map['AUTOMAKE'] = discover_tool('automake', 'Automake', 'AUTOMAKE', '1.9.6')
   env_map['AUTOCONF'] = discover_tool('autoconf', 'Autoconf', 'AUTOCONF', '2.6.3')
   env_map['MAKE'] = discover_tool('make', 'Make', 'MAKE', '3.81')
-  env_map['PYTHON3'] = discover_tool('python3', 'Python3', 'PYTHON3', '3.4')
+  env_map['PYTHON3'] = discover_tool('python3', 'Python3', 'PYTHON3', '3.6')
+  test_venv()
 
   if platform == 'Darwin':
     env_map['LIBTOOL'] = discover_tool('glibtool', 'Libtool', 'LIBTOOL', '2.4.2')

docker/compile/Dockerfile.centos7

@@ -36,7 +36,6 @@ RUN yum -y install \
       libtool \
       make \
       patch \
-      python \
       python3-devel \
       zip \
       unzip \

docker/compile/Dockerfile.ubuntu18.04

@@ -29,18 +29,19 @@ RUN apt-get update && apt-get -y install \
       libunwind8 \
       libcppunit-dev \
       patch \
-      python \
       python3-dev \
+      python3-venv \
       pkg-config \
       wget \
       zip \
-      virtualenv \
       unzip \
       git \
       curl \
       tree \
       openjdk-11-jdk-headless
 
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+
 ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
 
 ENV bazelVersion 3.4.1

docker/compile/Dockerfile.ubuntu20.04

@@ -31,12 +31,12 @@ RUN apt-get update && apt-get -y install \
       libunwind8 \
       libcppunit-dev \
       patch \
+      python-is-python3 \
       python3-dev \
-      python \
+      python3-venv \
       pkg-config \
       wget \
       zip \
-      virtualenv \
       unzip \
       git \
       curl \

docker/test/Dockerfile.ubuntu18.04

@@ -29,15 +29,17 @@ RUN apt-get update && apt-get -y install \
       libcppunit-dev \
       patch \
       python3-dev \
+      python3-venv \
       wget \
       zip \
-      virtualenv \
       unzip \
       git \
       curl \
       tree \
       openjdk-11-jdk-headless
 
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+
 ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
 
 ENV bazelVersion 3.4.1

scripts/packages/BUILD

@@ -604,8 +604,9 @@ genrule(
         "export RELEASE_FILE_DIR=$$(pwd)",
         "export TMP_DIR=$$(mktemp -d -t heronpy.XXXXX)",
         "echo $$TMP_DIR",
-        "$(location @virtualenv//:virtualenv) --no-download --quiet --clear $$TMP_DIR/venv",
-        "PS1= source $$TMP_DIR/venv/bin/activate",
+        # Create the virtual environment
+        'python3 -m venv $$TMP_DIR/venv --clear',
+        "VIRTUAL_ENV_DISABLE_PROMPT=1 source $$TMP_DIR/venv/bin/activate",
         "export HERONPY_DIR=$$TMP_DIR/heronpy",
         "export HERONPYAPI_UNZIP=$$TMP_DIR/heronapiunzip",
         "export HERONPYSTREAMLET_UNZIP=$$TMP_DIR/heronstreamletunzip",
@@ -661,7 +662,6 @@ genrule(
         "//heronpy/connectors:heron-pythonconnectors-package",
         "//heronpy/proto:proto-py-package",
         "//heronpy/streamlet:heron-python-streamlet-api-package",
-        "@virtualenv",
     ],
 )
 

tools/rules/pex/BUILD

@@ -25,10 +25,9 @@ LINUX_EXECUTE = [
 POST_EXECUTE = [
     'ln -sf "$$OUTDIR" "$$TMPF"',
     'VENV="$${TMPF}/venv"',
-    '$(location @virtualenv//:virtualenv) --no-download --quiet --clear "$$VENV"',
-    '# this is just to keep the activate script happy',
-    'PS1=',
-    'source "$$VENV/bin/activate"',
+    # Create the virtual environment
+    'python3 -m venv $$VENV --clear',
+    'VIRTUAL_ENV_DISABLE_PROMPT=1 source "$$VENV/bin/activate"',
 
     'pip install pex \
             --quiet --no-cache-dir --no-index --build $(@D)/pexbuild \
@@ -70,6 +69,5 @@ genrule(
     executable = True,
     message = "Bootstrapping pex",
     output_to_bindir = True,
-    tools = ["@virtualenv"],
     visibility = ["//visibility:public"],
 )

website2/website/scripts/python-doc-gen.sh

@@ -36,7 +36,7 @@ VENV=./venv/
 echo $VENV
 PIP_LOCATION=${HERON_ROOT_DIR}/bazel-bin/scripts/packages
 
-virtualenv "$VENV"
+python3 -m venv "$VENV"
 source "$VENV/bin/activate"
 pip3 install pdoc==0.3.2
 pip3 install --ignore-installed six