From 04f2a46f6904533f598d1e0505c15bc51399455d Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Thu, 9 Nov 2023 19:38:18 +0000 Subject: [PATCH 1/2] fix: confirm owlbot-copy succeeeded --- library_generation/test/generate_library_integration_test.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/library_generation/test/generate_library_integration_test.sh b/library_generation/test/generate_library_integration_test.sh index 64b526e47c..c29aaa6659 100755 --- a/library_generation/test/generate_library_integration_test.sh +++ b/library_generation/test/generate_library_integration_test.sh @@ -173,6 +173,11 @@ grep -v '^ *#' < "${proto_path_list}" | while IFS= read -r line; do echo "Generate library finished." echo "Compare generation result..." if [ $enable_postprocessing == "true" ]; then + if [ $(find "${output_folder}/workspace" -name '*.java' | wc -l) -eq 0 ]; + then + echo 'no java files found in workspace. This probably means that owlbot copy failed' + exit 1 + fi echo "Checking out repository..." pushd "${target_folder}" SOURCE_DIFF_RESULT=0 From e1d029ba484f26969ff4413016d1bbfbbe20c566 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Thu, 9 Nov 2023 20:20:15 +0000 Subject: [PATCH 2/2] fix: use correct path as input to owlbot-copy --- library_generation/generate_library.sh | 18 +++++++++++++----- library_generation/postprocess_library.sh | 14 +++++++++----- .../test/generate_library_integration_test.sh | 5 ++++- .../test/resources/proto_path_list.txt | 2 +- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/library_generation/generate_library.sh b/library_generation/generate_library.sh index 7df2b226e9..2debbe929c 100755 --- a/library_generation/generate_library.sh +++ b/library_generation/generate_library.sh @@ -72,6 +72,10 @@ case $key in versions_file="$2" shift ;; + --googleapis_gen_folder_name) + googleapis_gen_folder_name="$2" + shift + ;; *) echo "Invalid option: [$1]" exit 1 @@ -300,10 +304,13 @@ then echo "post processing is disabled" exit 0 fi -if [ -z "${versions_file}" ];then - echo "no versions.txt argument provided. Please provide one in order to enable post-processing" - exit 1 -fi +for necessary_argument in versions_file googleapis_gen_folder_name; do + if [ -z "${necessary_argument}" ];then + echo "no ${necessary_argument} argument provided. Please provide one in order to enable post-processing" + exit 1 + fi +done + workspace="${output_folder}/workspace" if [ -d "${workspace}" ]; then rm -rdf "${workspace}" @@ -316,7 +323,8 @@ bash -x "${script_dir}/postprocess_library.sh" "${workspace}" \ "${destination_path}" \ "${proto_path}" \ "${versions_file}" \ - "${output_folder}" + "${output_folder}" \ + "${googleapis_gen_folder_name}" # for post-procesed libraries, remove pre-processed folders pushd "${output_folder}/${destination_path}" diff --git a/library_generation/postprocess_library.sh b/library_generation/postprocess_library.sh index 880139db73..e002d9839e 100755 --- a/library_generation/postprocess_library.sh +++ b/library_generation/postprocess_library.sh @@ -15,6 +15,8 @@ # folder structure to run `owlbot-cli copy-code` # 5 - versions_file: path to file containing versions to be applied to the poms # 6 - output_folder: main workspace of the generation process +# 7 - googleapis_gen_folder_name: folder name in googleapis-gen/proto_path. This +# is necessary to work with owlbot CLI's copy-code workspace=$1 scripts_root=$2 @@ -22,6 +24,7 @@ destination_path=$3 proto_path=$4 versions_file=$5 output_folder=$6 +googleapis_gen_folder_name=$7 source "${scripts_root}"/utilities.sh @@ -58,16 +61,17 @@ pre_processed_libs_folder="${output_folder}/pre-processed" # By default (thanks to generation templates), .OwlBot.yaml `deep-copy` section # references a wildcard pattern matching a folder # ending with `-java` at the leaf of proto_path. -mkdir -p "${pre_processed_libs_folder}/${proto_path}/generated-java" +mkdir -p "${pre_processed_libs_folder}/${proto_path}/${googleapis_gen_folder_name}" folder_name=$(extract_folder_name "${destination_path}") +ggen_name="${googleapis_gen_folder_name}" copy_directory_if_exists "${output_folder}/${destination_path}/proto-${folder_name}" \ - "${pre_processed_libs_folder}/${proto_path}/generated-java/proto-google-cloud-${folder_name}" + "${pre_processed_libs_folder}/${proto_path}/${ggen_name}/proto-${ggen_name}" copy_directory_if_exists "${output_folder}/${destination_path}/grpc-${folder_name}" \ - "${pre_processed_libs_folder}/${proto_path}/generated-java/grpc-google-cloud-${folder_name}" + "${pre_processed_libs_folder}/${proto_path}/${ggen_name}/grpc-${ggen_name}" copy_directory_if_exists "${output_folder}/${destination_path}/gapic-${folder_name}" \ - "${pre_processed_libs_folder}/${proto_path}/generated-java/gapic-google-cloud-${folder_name}" + "${pre_processed_libs_folder}/${proto_path}/${ggen_name}/gapic-${ggen_name}" copy_directory_if_exists "${output_folder}/${destination_path}/samples" \ - "${pre_processed_libs_folder}/${proto_path}/generated-java/samples" + "${pre_processed_libs_folder}/${proto_path}/${ggen_name}/samples" pushd "${pre_processed_libs_folder}" # create an empty repository so owl-bot-copy can process this as a repo # (cannot process non-git-repositories) diff --git a/library_generation/test/generate_library_integration_test.sh b/library_generation/test/generate_library_integration_test.sh index c29aaa6659..4c1ff49b57 100755 --- a/library_generation/test/generate_library_integration_test.sh +++ b/library_generation/test/generate_library_integration_test.sh @@ -90,6 +90,8 @@ grep -v '^ *#' < "${proto_path_list}" | while IFS= read -r line; do pushd "${output_folder}" echo "Checking out googleapis-gen repository..." sparse_clone "${googleapis_gen_url}" "${proto_path}" + googleapis_gen_folder_name=$(basename \ + $(find "${output_folder}/googleapis-gen/${proto_path}" -maxdepth 1 -name '*-java')) destination_path=$(compute_destination_path "${proto_path}" "${output_folder}") # parse GAPIC options from proto_path/BUILD.bazel proto_build_file_path="${proto_path}/BUILD.bazel" @@ -146,7 +148,8 @@ grep -v '^ *#' < "${proto_path_list}" | while IFS= read -r line; do --service_yaml "${service_yaml}" \ --include_samples "${include_samples}" \ --enable_postprocessing "true" \ - --versions_file "${output_folder}/google-cloud-java/versions.txt" + --versions_file "${output_folder}/google-cloud-java/versions.txt" \ + --googleapis_gen_folder_name "${googleapis_gen_folder_name}" else "${library_generation_dir}"/generate_library.sh \ -p "${proto_path}" \ diff --git a/library_generation/test/resources/proto_path_list.txt b/library_generation/test/resources/proto_path_list.txt index 8943c03716..f56e78643c 100755 --- a/library_generation/test/resources/proto_path_list.txt +++ b/library_generation/test/resources/proto_path_list.txt @@ -17,7 +17,7 @@ google/cloud/redis/v1 java-redis false google/cloud/redis/v1beta1 java-redis false # google/example/library/v1 google-cloud-example-library-v1-java null false google/devtools/containeranalysis/v1 java-containeranalysis false -google/iam/v1 java-iam false +google/iam/v2 java-iam false google/iam/credentials/v1 java-iamcredentials false google/logging/v2 java-logging true google/pubsub/v1 java-pubsub true