Support subservices (#559)

## Changes
Support subservices
Test PR with custom OpenAPI spec
#558

Docs for linked PR

https://databricks-sdk-py--558.org.readthedocs.build/en/558/workspace/settings/index.html#

## Tests

- [X] `make test` run locally
- [X] `make fmt` applied
- [X] relevant integration tests applied

.codegen/__init__.py.tmpl

@@ -59,7 +59,7 @@ class WorkspaceClient:
         self._dbutils = _make_dbutils(self._config)
         self._api_client = client.ApiClient(self._config)
 
-        {{- range .Services}}{{if not .IsAccounts}}
+        {{- range .Services}}{{if and (not .IsAccounts) (not .HasParent)}}
         self._{{.SnakeName}} = {{template "api" .}}(self._api_client){{end -}}{{end}}
 
     @property
@@ -74,7 +74,7 @@ class WorkspaceClient:
     def dbutils(self) -> dbutils.RemoteDbUtils:
         return self._dbutils
 
-    {{- range .Services}}{{if not .IsAccounts}}
+    {{- range .Services}}{{if and (not .IsAccounts) (not .HasParent)}}
     @property
     def {{.SnakeName}}(self) -> {{template "api" .}}:
         {{if .Description}}"""{{.Summary}}"""{{end}}
@@ -113,7 +113,7 @@ class AccountClient:
         self._config = config.copy()
         self._api_client = client.ApiClient(self._config)
 
-        {{- range .Services}}{{if .IsAccounts}}
+        {{- range .Services}}{{if and .IsAccounts (not .HasParent)}}
         self._{{(.TrimPrefix "account").SnakeName}} = {{template "api" .}}(self._api_client){{end -}}{{end}}
 
     @property
@@ -124,7 +124,7 @@ class AccountClient:
     def api_client(self) -> client.ApiClient:
         return self._api_client
 
-    {{- range .Services}}{{if .IsAccounts}}
+    {{- range .Services}}{{if and .IsAccounts (not .HasParent)}}
     @property
     def {{(.TrimPrefix "account").SnakeName}}(self) -> {{template "api" .}}:{{if .Description}}
         """{{.Summary}}"""{{end}}

.codegen/service.py.tmpl

@@ -106,6 +106,16 @@ class {{.Name}}API:{{if .Description}}
     {{end}}
     def __init__(self, api_client):
         self._api = api_client
+        {{range .Subservices}}
+        self._{{.SnakeName}} = {{.PascalName}}API(self._api){{end}}
+
+    {{range .Subservices}}
+    @property
+    def {{.SnakeName}}(self) -> {{.PascalName}}API:
+        {{if .Description}}"""{{.Summary}}"""{{end}}
+        return self._{{.SnakeName}}
+    {{end}}
+
     {{range .Waits}}
     def {{template "safe-snake-name" .}}(self{{range .Binding}}, {{template "safe-snake-name" .PollField}}: {{template "type" .PollField.Entity}}{{end}},
       timeout=timedelta(minutes={{.Timeout}}), callback: Optional[Callable[[{{.Poll.Response.PascalName}}], None]] = None) -> {{.Poll.Response.PascalName}}:

docs/conf.py

@@ -5,6 +5,8 @@
 
 # -- Project information -----------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+import sys
+sys.path.append(".")
 
 project = 'Databricks SDK for Python'
 copyright = '2023, Databricks'

docs/dbdataclasses/sql.rst

@@ -64,6 +64,8 @@ These dataclasses are used in the SDK to represent API requests and responses fo
 
 .. py:class:: ChannelName
 
+   Name of the channel
+
    .. py:attribute:: CHANNEL_NAME_CURRENT
       :value: "CHANNEL_NAME_CURRENT"
 

docs/gen-client-docs.py

@@ -54,6 +54,23 @@ def __str__(self):
         return ret
 
 
+@dataclass
+class PropertyDoc:
+    name: str
+    doc: Optional[str]
+    tpe: Optional[str]
+
+    def as_rst(self) -> str:
+        out = ['', f'    .. py:property:: {self.name}']
+        if self.tpe is not None:
+            out.append(f'        :type: {self.tpe}')
+        if self.doc is not None:
+            # This is a class doc, which comes with 4 indentation spaces. 
+            # Here we are using the doc as property doc, which needs 8 indentation spaces.
+            formatted_doc = re.sub(r'\n', '\n    ', self.doc)
+            out.append(f'\n        {formatted_doc}')
+        return "\n".join(out)
+
 @dataclass
 class MethodDoc:
     method_name: str
@@ -85,6 +102,7 @@ class ServiceDoc:
     service_name: str
     class_name: str
     methods: list[MethodDoc]
+    property: list[PropertyDoc]
     doc: str
     tag: Tag
 
@@ -104,6 +122,9 @@ def as_rst(self) -> str:
                 continue
             out.append(rst)
 
+        for p in self.property:
+            out.append(p.as_rst())
+
         return "\n".join(out)
 
     def usage_example(self, m):
@@ -275,6 +296,23 @@ def _to_typed_args(argspec: inspect.FullArgSpec, required: bool) -> list[TypedAr
             out.append(TypedArgument(name=arg, tpe=tpe, default=defaults.get(arg)))
         return out
 
+    def class_properties(self, inst) -> list[PropertyDoc]:
+        property_docs = []
+        for name in dir(inst):
+            if name[0] == '_':
+                # private members
+                continue
+            instance_attr = getattr(inst, name)
+            if name.startswith('_'):
+                continue
+            if inspect.ismethod(instance_attr):
+                continue
+            property_docs.append(
+                PropertyDoc(name=name,
+                            doc=instance_attr.__doc__,
+                            tpe=instance_attr.__class__.__name__))
+        return property_docs
+
     def class_methods(self, inst) -> list[MethodDoc]:
         method_docs = []
         for name in dir(inst):
@@ -293,24 +331,27 @@ def class_methods(self, inst) -> list[MethodDoc]:
                           return_type=Generator._get_type_from_annotations(args.annotations, 'return')))
         return method_docs
 
-    def service_docs(self, client_inst) -> list[ServiceDoc]:
-        client_prefix = 'w' if isinstance(client_inst, WorkspaceClient) else 'a'
+    def service_docs(self, client_inst, client_prefix: str) -> list[ServiceDoc]:
         ignore_client_fields = ('config', 'dbutils', 'api_client', 'get_workspace_client', 'get_workspace_id')
         all = []
-        for service_name, service_inst in inspect.getmembers(client_inst):
+        for service_name, service_inst in inspect.getmembers(client_inst, lambda o: not inspect.ismethod(o)):
             if service_name.startswith('_'):
                 continue
             if service_name in ignore_client_fields:
                 continue
             class_doc = service_inst.__doc__
             class_name = service_inst.__class__.__name__
+            print(f'Processing service {client_prefix}.{service_name}')
+            all += self.service_docs(service_inst, client_prefix + "." + service_name)
+
             all.append(
                 ServiceDoc(client_prefix=client_prefix,
                            service_name=service_name,
                            class_name=class_name,
                            doc=class_doc,
                            tag=self._get_tag_name(service_inst.__class__.__name__, service_name),
-                           methods=self.class_methods(service_inst)))
+                           methods=self.class_methods(service_inst),
+                           property=self.class_properties(service_inst)))
         return all
 
     @staticmethod
@@ -354,11 +395,12 @@ def _get_tag_name(self, class_name, service_name) -> Tag:
     def load_client(self, client, folder, label, description):
         client_services = []
         package_to_services = collections.defaultdict(list)
-        service_docs = self.service_docs(client)
+        client_prefix = 'w' if isinstance(client, WorkspaceClient) else 'a'
+        service_docs = self.service_docs(client, client_prefix)
         for svc in service_docs:
             client_services.append(svc.service_name)
             package = svc.tag.package.name
-            package_to_services[package].append(svc.service_name)
+            package_to_services[package].append((svc.service_name, svc.client_prefix + "." + svc.service_name))
             self._make_folder_if_not_exists(f'{__dir__}/{folder}/{package}')
             with open(f'{__dir__}/{folder}/{package}/{svc.service_name}.rst', 'w') as f:
                 f.write(svc.as_rst())
@@ -367,7 +409,10 @@ def load_client(self, client, folder, label, description):
             if pkg.name not in package_to_services:
                 continue
             ordered_packages.append(pkg.name)
-            self._write_client_package_doc(folder, pkg, package_to_services[pkg.name])
+            # Order services inside the package by full path to have subservices grouped together
+            package_to_services[pkg.name].sort(key=lambda x: x[1])
+            ordered_services_names = [x[0] for x in package_to_services[pkg.name]]
+            self._write_client_package_doc(folder, pkg, ordered_services_names)
         self._write_client_packages(folder, label, description, ordered_packages)
 
     def _write_client_packages(self, folder: str, label: str, description: str, packages: list[str]):
@@ -390,7 +435,7 @@ def _write_client_package_doc(self, folder: str, pkg: Package, services: list[st
         """Writes out the index for a single package supported by a client."""
         self._make_folder_if_not_exists(f'{__dir__}/{folder}/{pkg.name}')
         with open(f'{__dir__}/{folder}/{pkg.name}/index.rst', 'w') as f:
-            all = "\n   ".join(sorted(services))
+            all = "\n   ".join(services)
             f.write(f'''
 {pkg.label}
 {'=' * len(pkg.label)}