Set up Bazel build system to outsource parameterization

.gitignore

@@ -4,3 +4,7 @@
 
 # Ignore Python cache files after running tests
 */__pycache__/
+
+# Ignore all bazel-* symlinks. There is no full list since this can change
+# based on the name of the directory bazel is cloned into.
+/bazel-*

BUILD

@@ -0,0 +1,91 @@
+package(default_visibility = ["//visibility:public"])
+
+test_suite(
+ name = "DataprocInitActionsTestSuite",
+ tests = [
+ ":test_cloud_sql_proxy",
+ ":test_dr_elephant",
+ ":test_hive_hcatalog",
+ ":test_starburst_presto",
+ "//bigtable:test_bigtable",
+ "//conda:test_conda",
+ "//connectors:test_connectors",
+ "//datalab:test_datalab",
+ "//drill:test_drill",
+ "//flink:test_flink",
+ "//ganglia:test_ganglia",
+ "//gpu:test_gpu",
+ "//hbase:test_hbase",
+ "//hue:test_hue",
+ "//jupyter:test_jupyter",
+ "//kafka:test_kafka",
+ "//livy:test_livy",
+ "//oozie:test_oozie",
+ "//presto:test_presto",
+ "//ranger:test_ranger",
+ "//rapids:test_rapids",
+ "//rstudio:test_rstudio",
+ "//solr:test_solr",
+ "//tez:test_tez",
+ "//tony:test_tony",
+ ],
+)
+
+py_test(
+ name = "test_cloud_sql_proxy",
+ size = "enormous",
+ srcs = ["cloud-sql-proxy/test_cloud_sql_proxy.py"],
+ data = ["cloud-sql-proxy/cloud-sql-proxy.sh"],
+ local = True,
+ shard_count = 3,
+ deps = [
+ ":pyspark_metastore_test",
+ "//integration_tests:dataproc_test_case",
+ "@io_abseil_py//absl/testing:parameterized",
+ ],
+)
+
+py_test(
+ name = "test_dr_elephant",
+ size = "enormous",
+ srcs = ["dr-elephant/test_dr_elephant.py"],
+ data = ["dr-elephant/dr-elephant.sh"],
+ local = True,
+ shard_count = 2,
+ deps = [
+ "//integration_tests:dataproc_test_case",
+ "@io_abseil_py//absl/testing:parameterized",
+ ],
+)
+
+py_test(
+ name = "test_hive_hcatalog",
+ size = "enormous",
+ srcs = ["hive-hcatalog/test_hive_hcatalog.py"],
+ data = ["hive-hcatalog/hive-hcatalog.sh"],
+ local = True,
+ shard_count = 6,
+ deps = [
+ "//integration_tests:dataproc_test_case",
+ "@io_abseil_py//absl/testing:parameterized",
+ ],
+)
+
+py_test(
+ name = "test_starburst_presto",
+ size = "enormous",
+ srcs = ["starburst-presto/test_starburst_presto.py"],
+ data = ["starburst-presto/presto.sh"],
+ local = True,
+ shard_count = 4,
+ deps = [
+ "//integration_tests:dataproc_test_case",
+ "@io_abseil_py//absl/testing:parameterized",
+ ],
+)
+
+py_library(
+ name = "pyspark_metastore_test",
+ testonly = True,
+ srcs = ["cloud-sql-proxy/pyspark_metastore_test.py"],
+)

WORKSPACE

@@ -0,0 +1,25 @@
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+ name = "rules_python",
+ sha256 = "aa96a691d3a8177f3215b14b0edc9641787abaaa30363a080165d06ab65e1161",
+ url = "https://github.com/bazelbuild/rules_python/releases/download/0.0.1/rules_python-0.0.1.tar.gz",
+)
+
+http_archive(
+ name = "io_abseil_py",
+ sha256 = "9b324bc96587819a1420c592cf5d54424456645719f1de6bb22f7045bb6dbc6b",
+ strip_prefix = "abseil-py-master",
+ url = "https://github.com/abseil/abseil-py/archive/master.zip",
+)
+
+http_archive(
+ name = "six_archive",
+ build_file = "@io_abseil_py//third_party:six.BUILD",
+ sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a",
+ strip_prefix = "six-1.10.0",
+ urls = [
+ "http://mirror.bazel.build/pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz",
+ "https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz",
+ ],
+)

bigtable/BUILD

@@ -0,0 +1,20 @@
+package(default_visibility = ["//visibility:public"])
+
+py_test(
+ name = "test_bigtable",
+ size = "enormous",
+ srcs = ["test_bigtable.py"],
+ data = ["bigtable.sh"],
+ local = True,
+ shard_count = 3,
+ deps = [
+ ":run_hbase_commands",
+ "//integration_tests:dataproc_test_case",
+ "@io_abseil_py//absl/testing:parameterized",
+ ],
+)
+
+py_library(
+ name = "run_hbase_commands",
+ srcs = ["run_hbase_commands.py"],
+)

bigtable/test_bigtable.py

@@ -10,9 +10,9 @@
  See: https://cloud.google.com/bigtable/docs/cbt-overview
 """
 import os
-import unittest
 
-from parameterized import parameterized
+from absl.testing import absltest
+from absl.testing import parameterized
 
 from integration_tests.dataproc_test_case import DataprocTestCase
 
@@ -48,13 +48,14 @@ def tearDown(self):
  def _validate_bigtable(self):
  _, stdout, _ = self.assert_command(
  'cbt -instance {} count test-bigtable '.format(self.db_name))
- self.assertEqual(int(float(stdout)), 4,
-   "Invalid BigTable instance count")
+ self.assertEqual(
+ int(float(stdout)), 4, "Invalid BigTable instance count")
 
  def verify_instance(self, name):
  self.upload_test_file(
- os.path.join(os.path.dirname(os.path.abspath(__file__)),
- self.TEST_SCRIPT_FILE_NAME), name)
+ os.path.join(
+ os.path.dirname(os.path.abspath(__file__)),
+ self.TEST_SCRIPT_FILE_NAME), name)
  self.assert_instance_command(
  name, "python {}".format(self.TEST_SCRIPT_FILE_NAME))
  self._validate_bigtable()
@@ -64,26 +65,19 @@ def verify_instance(self, name):
  admin commands are provided from text file.
  """
 
- @parameterized.expand(
- [
- ("SINGLE", "1.2", ["m"]),
- ("STANDARD", "1.2", ["m"]),
- ("HA", "1.2", ["m-0"]),
- ("SINGLE", "1.3", ["m"]),
- ("STANDARD", "1.3", ["m"]),
- ("HA", "1.3", ["m-0"]),
- ],
- testcase_func_name=DataprocTestCase.generate_verbose_test_name)
- def test_bigtable(self, configuration, dataproc_version, machine_suffixes):
- self.createCluster(configuration,
- self.INIT_ACTIONS,
- dataproc_version,
- metadata=self.metadata)
+ @parameterized.parameters(
+ ("SINGLE", ["m"]),
+ ("STANDARD", ["m"]),
+ ("HA", ["m-0"]),
+ )
+ def test_bigtable(self, configuration, machine_suffixes):
+ self.createCluster(
+ configuration, self.INIT_ACTIONS, metadata=self.metadata)
 
  for machine_suffix in machine_suffixes:
  self.verify_instance("{}-{}".format(self.getClusterName(),
  machine_suffix))
 
 
 if __name__ == '__main__':
- unittest.main()
+ absltest.main()

cloud-sql-proxy/test_cloud_sql.py → cloud-sql-proxy/test_cloud_sql_proxy.py

@@ -1,8 +1,8 @@
 import json
 import logging
-import unittest
 
-from parameterized import parameterized
+from absl.testing import absltest
+from absl.testing import parameterized
 
 from integration_tests.dataproc_test_case import DataprocTestCase
 
@@ -50,31 +50,23 @@ def __submit_pyspark_job(self, cluster_name):
  cluster_name, 'pyspark',
  '{}/{}'.format(self.INIT_ACTIONS_REPO, self.TEST_SCRIPT_FILE_NAME))
 
- @parameterized.expand(
- [
- ("SINGLE", "1.2"),
- ("STANDARD", "1.2"),
- ("HA", "1.2"),
- ("SINGLE", "1.3"),
- ("STANDARD", "1.3"),
- ("HA", "1.3"),
- ("SINGLE", "1.4"),
- ("STANDARD", "1.4"),
- ("HA", "1.4"),
- ],
- testcase_func_name=DataprocTestCase.generate_verbose_test_name)
- def test_cloud_sql_proxy(self, configuration, dataproc_version):
+ @parameterized.parameters(
+ "SINGLE",
+ "STANDARD",
+ "HA",
+ )
+ def test_cloud_sql_proxy(self, configuration):
  metadata = 'hive-metastore-instance={}:{}'.format(
  self.PROJECT_METADATA, self.DB_NAME)
- self.createCluster(configuration,
-  self.INIT_ACTIONS,
-  dataproc_version,
-  machine_type="n1-standard-2",
-  metadata=metadata,
-  scopes='sql-admin')
+ self.createCluster(
+ configuration,
+ self.INIT_ACTIONS,
+ machine_type="n1-standard-2",
+ metadata=metadata,
+ scopes='sql-admin')
 
  self.verify_cluster(self.getClusterName())
 
 
 if __name__ == '__main__':
- unittest.main()
+ absltest.main()

cloudbuild/Dockerfile

@@ -7,13 +7,10 @@ RUN useradd -m -d /home/ia-tests -s /bin/bash ia-tests
 
 COPY --chown=ia-tests:ia-tests . /init-actions
 
-# Install Pip
-RUN apt-get -y update
-RUN apt-get -y install python3-pip
-
-# Install fastunit
-WORKDIR /tmp/fastunit
-RUN git clone https://github.com/ityoung/python3-fastunit.git .
-RUN python3 setup.py install
+# Install Bazel:
+# https://docs.bazel.build/versions/master/install-ubuntu.html
+RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list
+RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add -
+RUN apt-get update && apt-get install -y openjdk-8-jdk python3-setuptools bazel
 
 USER ia-tests

cloudbuild/cloudbuild.yaml

@@ -1,15 +1,42 @@
 steps:
  # Create Docker image from regular Dockerfile
  - name: 'gcr.io/cloud-builders/docker'
+ id: 'docker-build'
  args: ['build', '--tag=gcr.io/$PROJECT_ID/init-actions-image:$BUILD_ID', '-f', 'cloudbuild/Dockerfile', '.']
+
  # Push Docker image to GCR
  - name: 'gcr.io/cloud-builders/docker'
+ id: 'gcr-push'
  args: ['push', 'gcr.io/$PROJECT_ID/init-actions-image:$BUILD_ID']
 
- # Run presubmit script
+ # Run presubmit tests in parallel for 1.2 image
+ - name: 'gcr.io/cloud-builders/kubectl'
+ id: 'dataproc-1.2-tests'
+ waitFor: ['gcr-push']
+ entrypoint: 'bash'
+ args: ['cloudbuild/run-presubmit-on-k8s.sh', 'gcr.io/$PROJECT_ID/init-actions-image:$BUILD_ID', '$BUILD_ID', '1.2']
+ env:
+ - 'COMMIT_SHA=$COMMIT_SHA'
+ - 'CLOUDSDK_COMPUTE_ZONE=us-central1-f'
+ - 'CLOUDSDK_CONTAINER_CLUSTER=init-actions-presubmit'
+
+ # Run presubmit tests in parallel for 1.3 image
+ - name: 'gcr.io/cloud-builders/kubectl'
+ id: 'dataproc-1.3-tests'
+ waitFor: ['gcr-push']
+ entrypoint: 'bash'
+ args: ['cloudbuild/run-presubmit-on-k8s.sh', 'gcr.io/$PROJECT_ID/init-actions-image:$BUILD_ID', '$BUILD_ID', '1.3']
+ env:
+ - 'COMMIT_SHA=$COMMIT_SHA'
+ - 'CLOUDSDK_COMPUTE_ZONE=us-central1-f'
+ - 'CLOUDSDK_CONTAINER_CLUSTER=init-actions-presubmit'
+
+ # Run presubmit tests in parallel for 1.4 image
  - name: 'gcr.io/cloud-builders/kubectl'
+ id: 'dataproc-1.4-tests'
+ waitFor: ['gcr-push']
  entrypoint: 'bash'
- args: ['cloudbuild/run-presubmit-on-k8s.sh', 'gcr.io/$PROJECT_ID/init-actions-image:$BUILD_ID', '$BUILD_ID']
+ args: ['cloudbuild/run-presubmit-on-k8s.sh', 'gcr.io/$PROJECT_ID/init-actions-image:$BUILD_ID', '$BUILD_ID', '1.4']
  env:
  - 'COMMIT_SHA=$COMMIT_SHA'
  - 'CLOUDSDK_COMPUTE_ZONE=us-central1-f'