feat: add gapic options as inputs to generate_library.sh (#2121)

* feat: add gapic options as inputs

* fix unit tests

* fix showcase generation

* make sure gapic option exists in `proto_path`

* add docs

* change log format

* add two libraries in integration test

* add a warning if gapic option does not exist

* change grpc_service_config to service_config

* search for gapic options in `proto_path` if they are empty

* format gapic options

* fix unit tests

library_generation/README.md

@@ -122,6 +122,24 @@ The default value is `true`.
 
 Use `--rest_numeric_enums` to specify the value.
 
+### gapic_yaml (optional)
+One of GAPIC options passed to the generator.
+The default value is an empty string.
+
+Use `--gapic_yaml` to specify the value.
+
+### service_config (optional)
+One of GAPIC options passed to the generator.
+The default value is an empty string.
+
+Use `--service_config` to specify the value.
+
+### service_yaml (optional)
+One of GAPIC options passed to the generator.
+The default value is an empty string.
+
+Use `--service_yaml` to specify the value.
+
 ### include_samples (optional)
 Whether generates code samples. The value is either `true` or `false`. 
 The default value is `true`.

library_generation/generate_library.sh

@@ -45,6 +45,18 @@ case $key in
     rest_numeric_enums="$2"
     shift
     ;;
+  --gapic_yaml)
+    gapic_yaml="$2"
+    shift
+    ;;
+  --service_config)
+    service_config="$2"
+    shift
+    ;;
+  --service_yaml)
+    service_yaml="$2"
+    shift
+    ;;
   --include_samples)
     include_samples="$2"
     shift
@@ -89,6 +101,18 @@ if [ -z "${rest_numeric_enums}" ]; then
   rest_numeric_enums="true"
 fi
 
+if [ -z "${gapic_yaml}" ]; then
+  gapic_yaml=""
+fi
+
+if [ -z "${service_config}" ]; then
+  service_config=""
+fi
+
+if [ -z "${service_yaml}" ]; then
+  service_yaml=""
+fi
+
 if [ -z "${include_samples}" ]; then
   include_samples="true"
 fi
@@ -166,7 +190,7 @@ if [[ "${proto_only}" == "false" ]]; then
   "$protoc_path"/protoc --experimental_allow_proto3_optional \
   "--plugin=protoc-gen-java_gapic=${script_dir}/gapic-generator-java-wrapper" \
   "--java_gapic_out=metadata:${destination_path}/java_gapic_srcjar_raw.srcjar.zip" \
-  "--java_gapic_opt=$(get_gapic_opts)" \
+  "--java_gapic_opt=$(get_gapic_opts "${transport}" "${rest_numeric_enums}" "${gapic_yaml}" "${service_config}" "${service_yaml}")" \
   ${proto_files} ${gapic_additional_protos}
 
   unzip -o -q "${destination_path}/java_gapic_srcjar_raw.srcjar.zip" -d "${destination_path}"

library_generation/test/generate_library_integration_test.sh

@@ -68,9 +68,18 @@ grep -v '^ *#' < "${proto_path_list}" | while IFS= read -r line; do
   gapic_additional_protos=$(get_gapic_additional_protos_from_BUILD "${proto_build_file_path}")
   transport=$(get_transport_from_BUILD "${proto_build_file_path}")
   rest_numeric_enums=$(get_rest_numeric_enums_from_BUILD "${proto_build_file_path}")
+  gapic_yaml=$(get_gapic_yaml_from_BUILD "${proto_build_file_path}")
+  service_config=$(get_service_config_from_BUILD "${proto_build_file_path}")
+  service_yaml=$(get_service_yaml_from_BUILD "${proto_build_file_path}")
   include_samples=$(get_include_samples_from_BUILD "${proto_build_file_path}")
   popd # output_folder
-  echo "GAPIC options are transport=${transport}, rest_numeric_enums=${rest_numeric_enums}, include_samples=${include_samples}."
+  echo "GAPIC options are
+    transport=${transport},
+    rest_numeric_enums=${rest_numeric_enums},
+    gapic_yaml=${gapic_yaml},
+    service_config=${service_config},
+    service_yaml=${service_yaml},
+    include_samples=${include_samples}."
   # generate GAPIC client library
   echo "Generating library from ${proto_path}, to ${destination_path}..."
   "${library_generation_dir}"/generate_library.sh \
@@ -82,6 +91,9 @@ grep -v '^ *#' < "${proto_path_list}" | while IFS= read -r line; do
   --gapic_additional_protos "${gapic_additional_protos}" \
   --transport "${transport}" \
   --rest_numeric_enums "${rest_numeric_enums}" \
+  --gapic_yaml "${gapic_yaml}" \
+  --service_config "${service_config}" \
+  --service_yaml "${service_yaml}" \
   --include_samples "${include_samples}"
   echo "Generate library finished."
   echo "Checking out googleapis-gen repository..."

library_generation/test/generate_library_unit_tests.sh

@@ -46,7 +46,7 @@ get_gapic_opts_with_rest_test() {
   local transport="grpc"
   local rest_numeric_enums="true"
   local gapic_opts
-  gapic_opts="$(get_gapic_opts)"
+  gapic_opts="$(get_gapic_opts "${transport}" "${rest_numeric_enums}" "" "" "")"
   assertEquals \
   "transport=grpc,rest-numeric-enums,grpc-service-config=${proto_path}/example_grpc_service_config.json,gapic-config=${proto_path}/example_gapic.yaml,api-service-config=${proto_path}/example.yaml" \
   "${gapic_opts}"
@@ -57,9 +57,20 @@ get_gapic_opts_without_rest_test() {
   local transport="grpc"
   local rest_numeric_enums="false"
   local gapic_opts
-  gapic_opts="$(get_gapic_opts)"
+  gapic_opts="$(get_gapic_opts "${transport}" "${rest_numeric_enums}" "" "" "")"
   assertEquals \
-  "transport=grpc,grpc-service-config=${proto_path}/example_grpc_service_config.json,gapic-config=${proto_path}/example_gapic.yaml,api-service-config=${proto_path}/example.yaml" \
+  "transport=grpc,,grpc-service-config=${proto_path}/example_grpc_service_config.json,gapic-config=${proto_path}/example_gapic.yaml,api-service-config=${proto_path}/example.yaml" \
+  "$gapic_opts"
+}
+
+get_gapic_opts_with_non_default_test() {
+  local proto_path="${script_dir}/resources/gapic_options"
+  local transport="grpc"
+  local rest_numeric_enums="false"
+  local gapic_opts
+  gapic_opts="$(get_gapic_opts "${transport}" "${rest_numeric_enums}" "${proto_path}/example_gapic.yaml" "${proto_path}/example_grpc_service_config.json" "${proto_path}/example.yaml")"
+  assertEquals \
+  "transport=grpc,,grpc-service-config=${proto_path}/example_grpc_service_config.json,gapic-config=${proto_path}/example_gapic.yaml,api-service-config=${proto_path}/example.yaml" \
   "$gapic_opts"
 }
 
@@ -304,6 +315,7 @@ test_list=(
   get_protobuf_version_failed_with_invalid_generator_version_test
   get_gapic_opts_with_rest_test
   get_gapic_opts_without_rest_test
+  get_gapic_opts_with_non_default_test
   remove_grpc_version_test
   download_generator_success_with_valid_version_test
   download_generator_failed_with_invalid_version_test

library_generation/test/resources/proto_path_list.txt

@@ -6,7 +6,9 @@ google/cloud/apigeeconnect/v1 google-cloud-apigeeconnect-v1-java
 google/cloud/asset/v1 google-cloud-asset-v1-java
 google/cloud/compute/v1 google-cloud-compute-v1-java
 google/cloud/kms/v1 google-cloud-kms-v1-java
+google/cloud/optimization/v1 google-cloud-optimization-v1-java
 google/cloud/redis/v1 google-cloud-redis-v1-java
+google/cloud/videointelligence/v1p3beta1 google-cloud-videointelligence-v1p3beta1-java
 google/example/library/v1 google-cloud-example-library-v1-java
 google/devtools/containeranalysis/v1 google-cloud-devtools-containeranalysis-v1-java
 google/firestore/bundle google-cloud-firestore-bundle-v1-java

library_generation/test/test_utilities.sh

@@ -44,6 +44,46 @@ __get_config_from_BUILD() {
   echo "${result}"
 }
 
+__get_gapic_option_from_BUILD() {
+  local build_file=$1
+  local pattern=$2
+  local gapic_option
+  local file_path
+  gapic_option=$(grep "${pattern}" "${build_file}" |\
+    head -1 |\
+    sed 's/.*\"\([^]]*\)\".*/\1/g' |\
+    sed 's/^[[:space:]]*//;s/[[:space:]]*$//'
+  )
+  if [ -z "${gapic_option}" ] || [[ "${gapic_option}" == *"None"* ]]; then
+    echo ""
+    return
+  fi
+
+  if [[ "${gapic_option}" == ":"* ]] || [[ "${gapic_option}" == "*"* ]]; then
+    # if gapic_option starts with : or *, remove the first character.
+    gapic_option="${gapic_option:1}"
+  elif [[ "${gapic_option}" == "//"* ]]; then
+    # gapic option is a bazel target, use the file path and name directly.
+    # remove the leading "//".
+    gapic_option="${gapic_option:2}"
+    # replace ":" with "/"
+    gapic_option="${gapic_option//://}"
+    echo "${gapic_option}"
+    return
+  fi
+
+  file_path="${build_file%/*}"
+  # Make sure gapic option (*.yaml or *.json) exists in proto_path; otherwise
+  # reset gapic option to empty string.
+  if [ -f "${file_path}/${gapic_option}" ]; then
+    gapic_option="${file_path}/${gapic_option}"
+  else
+    echo "WARNING: file ${file_path}/${gapic_option} does not exist, reset gapic option to empty string." >&2
+    gapic_option=""
+  fi
+  echo "${gapic_option}"
+}
+
 __get_iam_policy_from_BUILD() {
   local build_file=$1
   local contains_iam_policy
@@ -190,6 +230,27 @@ get_rest_numeric_enums_from_BUILD() {
   echo "${rest_numeric_enums}"
 }
 
+get_gapic_yaml_from_BUILD() {
+  local build_file=$1
+  local gapic_yaml
+  gapic_yaml=$(__get_gapic_option_from_BUILD "${build_file}" "gapic_yaml = ")
+  echo "${gapic_yaml}"
+}
+
+get_service_config_from_BUILD() {
+  local build_file=$1
+  local service_config
+  service_config=$(__get_gapic_option_from_BUILD "${build_file}" "grpc_service_config = ")
+  echo "${service_config}"
+}
+
+get_service_yaml_from_BUILD() {
+  local build_file=$1
+  local service_yaml
+  service_yaml=$(__get_gapic_option_from_BUILD "${build_file}" "service_yaml")
+  echo "${service_yaml}"
+}
+
 get_include_samples_from_BUILD() {
   local build_file=$1
   local include_samples

library_generation/utilities.sh

@@ -56,23 +56,30 @@ unzip_src_files() {
 
 # get gapic options from .yaml and .json files from proto_path.
 get_gapic_opts() {
-  local gapic_config
-  local grpc_service_config
-  local api_service_config
-  gapic_config=$(find "${proto_path}" -type f -name "*gapic.yaml")
-  if [ -z "${gapic_config}" ]; then
-    gapic_config=""
-  else
-    gapic_config="gapic-config=${gapic_config},"
-  fi
-  grpc_service_config=$(find "${proto_path}" -type f -name "*service_config.json")
-  api_service_config=$(find "${proto_path}" -maxdepth 1 -type f \( -name "*.yaml" ! -name "*gapic*.yaml" \))
+  local transport=$1
+  local rest_numeric_enums=$2
+  local gapic_yaml=$3
+  local service_config=$4
+  local service_yaml=$5
   if [ "${rest_numeric_enums}" == "true" ]; then
-    rest_numeric_enums="rest-numeric-enums,"
+    rest_numeric_enums="rest-numeric-enums"
   else
     rest_numeric_enums=""
   fi
-  echo "transport=${transport},${rest_numeric_enums}grpc-service-config=${grpc_service_config},${gapic_config}api-service-config=${api_service_config}"
+  # If any of the gapic options is empty (default value), try to search for
+  # it in proto_path.
+  if [[ "${gapic_yaml}" == "" ]]; then
+    gapic_yaml=$(find "${proto_path}" -type f -name "*gapic.yaml")
+  fi
+
+  if [[ "${service_config}" == "" ]]; then
+    service_config=$(find "${proto_path}" -type f -name "*service_config.json")
+  fi
+
+  if [[ "${service_yaml}" == "" ]]; then
+    service_yaml=$(find "${proto_path}" -maxdepth 1 -type f \( -name "*.yaml" ! -name "*gapic*.yaml" \))
+  fi
+  echo "transport=${transport},${rest_numeric_enums},grpc-service-config=${service_config},gapic-config=${gapic_yaml},api-service-config=${service_yaml}"
 }
 
 remove_grpc_version() {

showcase/scripts/generate_showcase.sh

@@ -49,6 +49,9 @@ ggj_version=$(get_version_from_versions_txt ../../versions.txt "gapic-generator-
 gapic_additional_protos="google/iam/v1/iam_policy.proto google/cloud/location/locations.proto"
 rest_numeric_enums="false"
 transport="grpc+rest"
+gapic_yaml=""
+service_config="schema/google/showcase/v1beta1/showcase_grpc_service_config.json"
+service_yaml="schema/google/showcase/v1beta1/showcase_v1beta1.yaml"
 include_samples="false"
 rm -rdf output/showcase-output
 mkdir output/showcase-output
@@ -59,6 +62,9 @@ bash "${SCRIPT_DIR}/../../library_generation/generate_library.sh" \
   --gapic_generator_version "${ggj_version}" \
   --gapic_additional_protos "${gapic_additional_protos}" \
   --rest_numeric_enums "${rest_numeric_enums}" \
+  --gapic_yaml "${gapic_yaml}" \
+  --service_config "${service_config}" \
+  --service_yaml "${service_yaml}" \
   --include_samples "${include_samples}" \
   --transport "${transport}"