Skip to content

Commit 86c5250

Browse files
meltsufinmeltsufin
authored
feat(internal/container/java): enable multi-version API generation (#2699)
Updates `generate` package to support the generation of client libraries with multiple API versions. - The `Generate` function now iterates through all API versions specified in the `generate-request.json` and processes each version independently. - The `restructureOutput` function has been updated to handle versioned output directories. - The `invokeProtoc` function now processes a single API at a time, allowing for version-specific output configurations. - Added `extractVersion` helper function to parse the API version from the path. - Updated `protoc.go` to include `common_resources.proto` in the compilation. Tests: - Refactored `TestRestructureOutput` to be table-driven and added a new test case to verify multi-version generation. - Updated `TestGenerate` to correctly mock `protoc` output for multi-version scenarios. - Modified `run-generate-library.sh` to use a multi-version `secret-manager-generate-request.json` for testing.
1 parent c58cf35 commit 86c5250

File tree

6 files changed

+282
-115
lines changed

6 files changed

+282
-115
lines changed

‎internal/container/java/generate/generator.go‎

Lines changed: 90 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -43,20 +43,41 @@ var (
4343
// the entire generation process.
4444
func Generate(ctx context.Context, cfg *generate.Config) error {
4545
slog.Debug("librariangen: generate command started")
46+
libraryID := cfg.Request.ID
47+
for _, api := range cfg.Request.APIs {
48+
if err := processAPI(ctx, cfg, libraryID, api); err != nil {
49+
return err
50+
}
51+
}
52+
53+
// Generate pom.xml files
54+
if err := pom.Generate(cfg.Context.OutputDir, libraryID); err != nil {
55+
return fmt.Errorf("librariangen: failed to generate poms for API %s: %w", libraryID, err)
56+
}
57+
58+
slog.Debug("librariangen: generate command finished")
59+
return nil
60+
}
61+
62+
func processAPI(ctx context.Context, cfg *generate.Config, libraryID string, api message.API) error {
63+
version := extractVersion(api.Path)
64+
if version == "" {
65+
slog.Warn("skipping api with no version", "api", api.Path)
66+
return nil
67+
}
68+
slog.Info("processing api", "path", api.Path, "version", version)
4669
outputConfig := &protoc.OutputConfig{
47-
GAPICDir: filepath.Join(cfg.Context.OutputDir, "gapic"),
48-
GRPCDir: filepath.Join(cfg.Context.OutputDir, "grpc"),
49-
ProtoDir: filepath.Join(cfg.Context.OutputDir, "proto"),
70+
GAPICDir: filepath.Join(cfg.Context.OutputDir, version, "gapic"),
71+
GRPCDir: filepath.Join(cfg.Context.OutputDir, version, "grpc"),
72+
ProtoDir: filepath.Join(cfg.Context.OutputDir, version, "proto"),
5073
}
5174
defer func() {
5275
if err := cleanupIntermediateFiles(outputConfig); err != nil {
53-
slog.Error("librariangen: failed to clean up intermediate files", "error", err)
76+
slog.Error("failed to cleanup", "err", err)
5477
}
5578
}()
5679

57-
generateReq := cfg.Request
58-
59-
if err := invokeProtoc(ctx, cfg.Context, generateReq, outputConfig); err != nil {
80+
if err := invokeProtoc(ctx, cfg.Context, &api, outputConfig); err != nil {
6081
return fmt.Errorf("librariangen: gapic generation failed: %w", err)
6182
}
6283
// Unzip the temp-codegen.srcjar.
@@ -66,45 +87,47 @@ func Generate(ctx context.Context, cfg *generate.Config) error {
6687
return fmt.Errorf("librariangen: failed to unzip %s: %w", srcjarPath, err)
6788
}
6889

69-
if err := restructureOutput(cfg.Context.OutputDir, generateReq.ID); err != nil {
90+
if err := restructureOutput(cfg.Context.OutputDir, libraryID, version); err != nil {
7091
return fmt.Errorf("librariangen: failed to restructure output: %w", err)
7192
}
7293

73-
// Generate pom.xml files
74-
if err := pom.Generate(cfg.Context.OutputDir, generateReq.ID); err != nil {
75-
return fmt.Errorf("librariangen: failed to generate poms for API %s: %w", generateReq.ID, err)
76-
}
77-
78-
slog.Debug("librariangen: generate command finished")
7994
return nil
8095
}
8196

97+
func extractVersion(path string) string {
98+
parts := strings.Split(path, "/")
99+
for i := len(parts) - 1; i >= 0; i-- {
100+
if strings.HasPrefix(parts[i], "v") {
101+
return parts[i]
102+
}
103+
}
104+
return ""
105+
}
106+
82107
// invokeProtoc handles the protoc GAPIC generation logic for the 'generate' CLI command.
83108
// It reads a request file, and for each API specified, it invokes protoc
84109
// to generate the client library. It returns the module path and the path to the service YAML.
85-
func invokeProtoc(ctx context.Context, genCtx *generate.Context, generateReq *message.Library, outputConfig *protoc.OutputConfig) error {
86-
for _, api := range generateReq.APIs {
87-
apiServiceDir := filepath.Join(genCtx.SourceDir, api.Path)
88-
slog.Info("processing api", "service_dir", apiServiceDir)
89-
bazelConfig, err := bazelParse(apiServiceDir)
90-
if err != nil {
91-
return fmt.Errorf("librariangen: failed to parse BUILD.bazel for %s: %w", apiServiceDir, err)
92-
}
93-
args, err := protocBuild(apiServiceDir, bazelConfig, genCtx.SourceDir, outputConfig)
94-
if err != nil {
95-
return fmt.Errorf("librariangen: failed to build protoc command for api %q in library %q: %w", api.Path, generateReq.ID, err)
96-
}
110+
func invokeProtoc(ctx context.Context, genCtx *generate.Context, api *message.API, outputConfig *protoc.OutputConfig) error {
111+
apiServiceDir := filepath.Join(genCtx.SourceDir, api.Path)
112+
slog.Info("processing api", "service_dir", apiServiceDir)
113+
bazelConfig, err := bazelParse(apiServiceDir)
114+
if err != nil {
115+
return fmt.Errorf("librariangen: failed to parse BUILD.bazel for %s: %w", apiServiceDir, err)
116+
}
117+
args, err := protocBuild(apiServiceDir, bazelConfig, genCtx.SourceDir, outputConfig)
118+
if err != nil {
119+
return fmt.Errorf("librariangen: failed to build protoc command for api %q: %w", api.Path, err)
120+
}
97121

98-
// Create protoc output directories.
99-
for _, dir := range []string{outputConfig.ProtoDir, outputConfig.GRPCDir, outputConfig.GAPICDir} {
100-
if err := os.MkdirAll(dir, 0755); err != nil {
101-
return err
102-
}
122+
// Create protoc output directories.
123+
for _, dir := range []string{outputConfig.ProtoDir, outputConfig.GRPCDir, outputConfig.GAPICDir} {
124+
if err := os.MkdirAll(dir, 0755); err != nil {
125+
return err
103126
}
127+
}
104128

105-
if err := execvRun(ctx, args, genCtx.OutputDir); err != nil {
106-
return fmt.Errorf("librariangen: protoc failed for api %q in library %q: %w, execvRun error: %v", api.Path, generateReq.ID, err, err)
107-
}
129+
if err := execvRun(ctx, args, genCtx.OutputDir); err != nil {
130+
return fmt.Errorf("librariangen: protoc failed for api %q: %w, execvRun error: %v", api.Path, err, err)
108131
}
109132
return nil
110133
}
@@ -126,23 +149,21 @@ func moveFiles(sourceDir, targetDir string) error {
126149
return nil
127150
}
128151

129-
func restructureOutput(outputDir, libraryID string) error {
152+
func restructureOutput(outputDir, libraryID, version string) error {
130153
slog.Debug("librariangen: restructuring output directory", "dir", outputDir)
131154

132155
// Define source and destination directories.
133-
gapicSrcDir := filepath.Join(outputDir, "gapic", "src", "main", "java")
134-
gapicTestDir := filepath.Join(outputDir, "gapic", "src", "test", "java")
135-
protoSrcDir := filepath.Join(outputDir, "proto")
136-
resourceNameSrcDir := filepath.Join(outputDir, "gapic", "proto", "src", "main", "java")
137-
grpcSrcDir := filepath.Join(outputDir, "grpc")
138-
samplesDir := filepath.Join(outputDir, "gapic", "samples", "snippets")
139-
140-
// TODO(meltsufin): currently we assume we have a single API variant v1
156+
gapicSrcDir := filepath.Join(outputDir, version, "gapic", "src", "main", "java")
157+
gapicTestDir := filepath.Join(outputDir, version, "gapic", "src", "test", "java")
158+
protoSrcDir := filepath.Join(outputDir, version, "proto")
159+
resourceNameSrcDir := filepath.Join(outputDir, version, "gapic", "proto", "src", "main", "java")
160+
samplesDir := filepath.Join(outputDir, version, "gapic", "samples", "snippets")
161+
141162
gapicDestDir := filepath.Join(outputDir, fmt.Sprintf("google-cloud-%s", libraryID), "src", "main", "java")
142163
gapicTestDestDir := filepath.Join(outputDir, fmt.Sprintf("google-cloud-%s", libraryID), "src", "test", "java")
143-
protoDestDir := filepath.Join(outputDir, fmt.Sprintf("proto-google-cloud-%s-v1", libraryID), "src", "main", "java")
144-
resourceNameDestDir := filepath.Join(outputDir, fmt.Sprintf("proto-google-cloud-%s-v1", libraryID), "src", "main", "java")
145-
grpcDestDir := filepath.Join(outputDir, fmt.Sprintf("grpc-google-cloud-%s-v1", libraryID), "src", "main", "java")
164+
protoDestDir := filepath.Join(outputDir, fmt.Sprintf("proto-google-cloud-%s-%s", libraryID, version), "src", "main", "java")
165+
resourceNameDestDir := filepath.Join(outputDir, fmt.Sprintf("proto-google-cloud-%s-%s", libraryID, version), "src", "main", "java")
166+
grpcDestDir := filepath.Join(outputDir, fmt.Sprintf("grpc-google-cloud-%s-%s", libraryID, version), "src", "main", "java")
146167
samplesDestDir := filepath.Join(outputDir, "samples", "snippets")
147168

148169
// Create destination directories.
@@ -161,20 +182,37 @@ func restructureOutput(outputDir, libraryID string) error {
161182
}
162183
}
163184

185+
// Remove the location classes from the proto output to avoid conflicts with
186+
// proto-google-common-protos.
187+
if err := os.RemoveAll(filepath.Join(protoSrcDir, "com", "google", "cloud", "location")); err != nil {
188+
return err
189+
}
190+
if err := os.Remove(filepath.Join(protoSrcDir, "google", "cloud", "CommonResources.java")); err != nil {
191+
return err
192+
}
193+
164194
// Move files that won't have conflicts.
165195
moves := map[string]string{
166-
gapicSrcDir: gapicDestDir,
167-
gapicTestDir: gapicTestDestDir,
168-
protoSrcDir: protoDestDir,
169-
grpcSrcDir: grpcDestDir,
170-
samplesDir: samplesDestDir,
196+
filepath.Join(outputDir, version, "proto"): protoDestDir,
197+
filepath.Join(outputDir, version, "grpc"): grpcDestDir,
171198
}
172199
for src, dest := range moves {
173200
if err := moveFiles(src, dest); err != nil {
174201
return err
175202
}
176203
}
177204

205+
// Merge the gapic source and test files.
206+
if err := copyAndMerge(gapicSrcDir, gapicDestDir); err != nil {
207+
return err
208+
}
209+
if err := copyAndMerge(gapicTestDir, gapicTestDestDir); err != nil {
210+
return err
211+
}
212+
if err := copyAndMerge(samplesDir, samplesDestDir); err != nil {
213+
return err
214+
}
215+
178216
// Merge the resource name files into the proto destination.
179217
if err := copyAndMerge(resourceNameSrcDir, resourceNameDestDir); err != nil {
180218
return err
@@ -214,12 +252,7 @@ func copyAndMerge(src, dest string) error {
214252

215253
func cleanupIntermediateFiles(outputConfig *protoc.OutputConfig) error {
216254
slog.Debug("librariangen: cleaning up intermediate files")
217-
for _, path := range []string{outputConfig.GAPICDir, outputConfig.GRPCDir, outputConfig.ProtoDir} {
218-
if err := os.RemoveAll(path); err != nil {
219-
return fmt.Errorf("failed to clean up intermediate file at %s: %w", path, err)
220-
}
221-
}
222-
return nil
255+
return os.RemoveAll(filepath.Dir(outputConfig.GAPICDir))
223256
}
224257

225258
func unzip(src, dest string) error {

0 commit comments

Comments
 (0)