feat: enable generation with postprocessing of multiple service versions

.github/workflows/verify_library_generation.yaml

@@ -25,7 +25,7 @@ jobs:
  cache: maven
  - uses: actions/setup-python@v4
  with:
- python-version: '3.11'
+ python-version: 3.11
  - name: install pyenv
  shell: bash
  run: |
@@ -36,10 +36,23 @@ jobs:
  export PATH="$PYENV_ROOT/bin:$PATH"
  echo "PYENV_ROOT=${PYENV_ROOT}" >> $GITHUB_ENV
  echo "PATH=${PATH}" >> $GITHUB_ENV
- # init pyenv
- eval "$(pyenv init --path)"
- eval "$(pyenv init -)"
+
  set +ex
+ - name: install python dependencies
+ shell: bash
+ run: |
+ set -ex
+ pushd library_generation
+ pip install -r requirements.in
+ popd
+
+ - name: install utils (macos)
+ if: matrix.os == 'macos-12'
+ shell: bash
+ run: |
+ brew update --preinstall
+ # we need the `realpath` command to be available
+ brew install coreutils
  - name: install docker (ubuntu)
  if: matrix.os == 'ubuntu-22.04'
  shell: bash
@@ -69,10 +82,30 @@ jobs:
  runs-on: ${{ matrix.os }}
  steps:
  - uses: actions/checkout@v3
- - name: Run unit tests
+ - name: install utils (macos)
+ if: matrix.os == 'macos-12'
+ shell: bash
+ run: |
+ brew update --preinstall
+ brew install coreutils
+ - uses: actions/setup-python@v4
+ with:
+ python-version: 3.11
+ - name: install python dependencies
+ shell: bash
+ run: |
+ set -ex
+ pushd library_generation
+ pip install -r requirements.in
+ popd
+ - name: Run shell unit tests
  run: |
  set -x
  library_generation/test/generate_library_unit_tests.sh
+ - name: Run python unit tests
+ run: |
+ set -x
+ python -m unittest library_generation/test/unit_tests.py
  lint:
  runs-on: ubuntu-22.04
  steps:

library_generation/.gitignore

@@ -0,0 +1 @@
+__pycache__/

library_generation/generate_composed_library.py

@@ -0,0 +1,141 @@
+"""
+This script allows generation of libraries that are composed of more than one
+service version. It is achieved by calling `generate_library.sh` without
+postprocessing for all service versions and then calling
+postprocess_library.sh at the end, once all libraries are ready.
+
+Prerequisites
+- Needs a folder named `output` in current working directory. This folder
+is automatically detected by `generate_library.sh` and this script ensures it
+contains the necessary folders and files, specifically:
+ - A "google" folder found in the googleapis/googleapis repository
+ - A "grafeas" folder found in the googleapis/googleapis repository
+Note: googleapis repo is found in https://github.com/googleapis/googleapis.
+"""
+
+import click
+import utilities as util
+import os
+import sys
+import subprocess
+import json
+from model.GenerationConfig import GenerationConfig
+from model.LibraryConfig import LibraryConfig
+from model.ClientInputs import parse as parse_build_file
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+
+"""
+Main function in charge of generating libraries composed of more than one
+service or service version.
+Arguments
+ - config: a GenerationConfig object representing a parsed configuration
+ yaml
+ - library: a LibraryConfig object contained inside config, passed here for
+ convenience and to prevent all libraries to be processed
+ - enable_postprocessing: true if postprocessing should be done on the generated
+ libraries
+ - repository_path: path to the repository where the generated files will be
+ sent. If not specified, it will default to the one defined in the configuration yaml
+ and will be downloaded. The versions file will be inferred from this folder
+"""
+def generate_composed_library(
+ config: GenerationConfig,
+ library: LibraryConfig,
+ repository_path: str,
+ enable_postprocessing: bool = True,
+) -> None:
+ output_folder = util.sh_util('get_output_folder')
+
+ print(f'output_folder: {output_folder}')
+ print('library: ', library)
+ os.makedirs(output_folder, exist_ok=True)
+
+ googleapis_commitish = config.googleapis_commitish
+ if library.googleapis_commitish is not None:
+ googleapis_commitish = library.googleapis_commitish
+ print('using library-specific googleapis commitish: ' + googleapis_commitish)
+ else:
+ print('using common googleapis_commitish')
+
+ print('removing old googleapis folders and files')
+ util.delete_if_exists(f'{output_folder}/google')
+ util.delete_if_exists(f'{output_folder}/grafeas')
+
+ print('downloading googleapis')
+ util.sh_util(f'download_googleapis_files_and_folders "{output_folder}" "{googleapis_commitish}"')
+
+ is_monorepo = len(config.libraries) > 1
+
+ base_arguments = []
+ base_arguments += util.create_argument('gapic_generator_version', config)
+ base_arguments += util.create_argument('grpc_version', config)
+ base_arguments += util.create_argument('protobuf_version', config)
+
+ library_name = f'java-{library.api_shortname}'
+ library_path = None
+
+ versions_file = ''
+ if is_monorepo:
+ print('this is a monorepo library')
+ destination_path = config.destination_path + '/' + library_name
+ library_folder = destination_path.split('/')[-1]
+ if repository_path is None:
+ print(f'sparse_cloning monorepo with {library_name}')
+ repository_path = f'{output_folder}/{config.destination_path}'
+ clone_out = util.sh_util(f'sparse_clone "https://github.com/googleapis/{MONOREPO_NAME}.git" "{library_folder} google-cloud-pom-parent google-cloud-jar-parent versions.txt .github"', cwd=output_folder)
+ print(clone_out)
+ library_path = f'{repository_path}/{library_name}'
+ versions_file = f'{repository_path}/versions.txt'
+ else:
+ print('this is a HW library')
+ destination_path = library_name
+ if repository_path is None:
+ repository_path = f'{output_folder}/{destination_path}'
+ util.delete_if_exists(f'{output_folder}/{destination_path}')
+ clone_out = util.sh_util(f'git clone "https://github.com/googleapis/{destination_path}.git"', cwd=output_folder)
+ print(clone_out)
+ library_path = f'{repository_path}'
+ versions_file = f'{repository_path}/versions.txt'
+
+ owlbot_cli_source_folder = util.sh_util('mktemp -d')
+ for gapic in library.gapic_configs:
+
+ effective_arguments = list(base_arguments)
+ effective_arguments += util.create_argument('proto_path', gapic)
+
+ build_file_folder = f'{output_folder}/{gapic.proto_path}'
+ print(f'build_file_folder: {build_file_folder}')
+ client_inputs = parse_build_file(build_file_folder, gapic.proto_path)
+ effective_arguments += [
+ '--proto_only', client_inputs.proto_only,
+ '--gapic_additional_protos', client_inputs.additional_protos,
+ '--transport', client_inputs.transport,
+ '--rest_numeric_enums', client_inputs.rest_numeric_enum,
+ '--gapic_yaml', client_inputs.gapic_yaml,
+ '--service_config', client_inputs.service_config,
+ '--service_yaml', client_inputs.service_yaml,
+ '--include_samples', client_inputs.include_samples,
+ ]
+ service_version = gapic.proto_path.split('/')[-1]
+ temp_destination_path = f'java-{library.api_shortname}-{service_version}'
+ effective_arguments += [ '--destination_path', temp_destination_path ]
+ print('arguments: ')
+ print(effective_arguments)
+ print(f'Generating library from {gapic.proto_path} to {destination_path}...')
+ util.run_process_and_print_output(['bash', '-x', f'{script_dir}/generate_library.sh',
+ *effective_arguments], 'Library generation')
+
+
+ if enable_postprocessing:
+ util.sh_util(f'build_owlbot_cli_source_folder "{library_path}"'
+ + f' "{owlbot_cli_source_folder}" "{output_folder}/{temp_destination_path}"'
+ + f' "{gapic.proto_path}"',
+ cwd=output_folder)
+
+ if enable_postprocessing:
+ # call postprocess library
+ util.run_process_and_print_output([f'{script_dir}/postprocess_library.sh',
+ f'{library_path}', '', versions_file, owlbot_cli_source_folder,
+ config.owlbot_cli_image, config.synthtool_commitish, str(is_monorepo).lower()], 'Library postprocessing')
+

library_generation/generate_library.sh

@@ -60,18 +60,10 @@ case $key in
  include_samples="$2"
  shift
  ;;
- --enable_postprocessing)
- enable_postprocessing="$2"
- shift
- ;;
  --os_architecture)
  os_architecture="$2"
  shift
  ;;
- --versions_file)
- versions_file="$2"
- shift
- ;;
  *)
  echo "Invalid option: [$1]"
  exit 1
@@ -85,6 +77,11 @@ script_dir=$(dirname "$(readlink -f "$0")")
 source "${script_dir}"/utilities.sh
 output_folder="$(get_output_folder)"
 
+if [ -z "${gapic_generator_version}" ]; then
+ echo 'missing required argument --gapic_generator_version'
+ exit 1
+fi
+
 if [ -z "${protobuf_version}" ]; then
  protobuf_version=$(get_protobuf_version "${gapic_generator_version}")
 fi
@@ -125,10 +122,6 @@ if [ -z "${include_samples}" ]; then
  include_samples="true"
 fi
 
-if [ -z "$enable_postprocessing" ]; then
- enable_postprocessing="true"
-fi
-
 if [ -z "${os_architecture}" ]; then
  os_architecture=$(detect_os_architecture)
 fi
@@ -305,34 +298,7 @@ popd # output_folder
 pushd "${temp_destination_path}"
 rm -rf java_gapic_srcjar java_gapic_srcjar_raw.srcjar.zip java_grpc.jar java_proto.jar temp-codegen.srcjar
 popd # destination path
-##################### Section 5 #####################
-# post-processing
-#####################################################
-if [ "${enable_postprocessing}" != "true" ];
-then
- echo "post processing is disabled"
- cp -r ${temp_destination_path}/* "${output_folder}/${destination_path}"
- rm -rdf "${temp_destination_path}"
- exit 0
-fi
-if [ -z "${versions_file}" ];then
- echo "no versions.txt argument provided. Please provide one in order to enable post-processing"
- exit 1
-fi
-workspace="${output_folder}/workspace"
-if [ -d "${workspace}" ]; then
- rm -rdf "${workspace}"
-fi
-
-mkdir -p "${workspace}"
-
-# if destination_path is not empty, it will be used as a starting workspace for
-# postprocessing
-if [[ $(find "${output_folder}/${destination_path}" -mindepth 1 -maxdepth 1 -type d,f | wc -l) -gt 0 ]];then
- workspace="${output_folder}/${destination_path}"
-fi
-
-bash -x "${script_dir}/postprocess_library.sh" "${workspace}" \
- "${temp_destination_path}" \
- "${versions_file}"
 
+cp -r ${temp_destination_path}/* "${output_folder}/${destination_path}"
+rm -rdf "${temp_destination_path}"
+exit 0

library_generation/main.py

@@ -0,0 +1,77 @@
+"""
+Parses a config yaml and generates libraries via generate_composed_library.py
+"""
+
+import click
+from generate_composed_library import generate_composed_library
+from typing import Dict
+from model.GenerationConfig import GenerationConfig
+from collections.abc import Sequence
+from absl import app
+
+@click.group(invoke_without_command=False)
+@click.pass_context
+@click.version_option(message="%(version)s")
+def main(ctx):
+ pass
+
+@main.command()
+@click.option(
+ "--generation-config-yaml",
+ required=True,
+ type=str,
+ help="""
+ Path to generation_config.yaml that contains the metadata about library generation
+ """
+)
+@click.option(
+ "--enable-postprocessing",
+ required=False,
+ default=True,
+ type=bool,
+ help="""
+ Path to repository where generated files will be merged into, via owlbot copy-code.
+ Specifying this option enables postprocessing
+ """
+)
+@click.option(
+ "--target-library-api-shortname",
+ required=False,
+ type=str,
+ help="""
+ If specified, only the `library` with api_shortname = target-library-api-shortname will
+ be generated. If not specified, all libraries in the configuration yaml will be generated
+ """
+)
+@click.option(
+ "--repository-path",
+ required=False,
+ type=str,
+ help="""
+ If specified, the generated files will be sent to this location. If not specified, the
+ repository will be pulled into output_folder and move the generated files there
+ """
+)
+def generate_from_yaml(
+ generation_config_yaml: str,
+ enable_postprocessing: bool,
+ target_library_api_shortname: str,
+ repository_path: str
+) -> None:
+ config = GenerationConfig.from_yaml(generation_config_yaml)
+ target_libraries = config.libraries
+ if target_library_api_shortname is not None:
+ target_libraries = [library for library in config.libraries
+ if library.api_shortname == target_library_api_shortname]
+ for library in target_libraries:
+ print(f'generating library {library.api_shortname}')
+ generate_composed_library(
+ config, library, repository_path, enable_postprocessing
+ )
+
+
+
+
+
+if __name__ == "__main__":
+ main()