Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Structure style: namespace-clusters #573

Merged
merged 2 commits into from
Aug 1, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions docs/codegen.rst
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,33 @@ type substitutions, redefines.
$ xsdata schema.xsd --package models --structure-style namespaces


Since v21.8, the generator converts namespaces to packages similar to jaxb in order
to facilitate runs against multiple schemas from the same vendor.

.. list-table::
:widths: 20 20
:header-rows: 1

* - Examples (before naming conventions)
-
* - http://www.w3.org/XML/1998/namespace
- org.w3.xml.1998.namespace
* - myNS.tempuri.org
- org.tempuri.myNS
* - urn:xmlns:25hoursaday-com:address
- com.25hoursaday.address


**namespace-clusters**

This style combines the clusters and the namespace styles. It will fail if there
are strongly connected classes in the same graph from different namespaces.

.. code-block:: console

$ xsdata schema.xsd --package models --structure-style namespace-clusters


**single-package**

This style will group all classes together into a single package eliminating imports
Expand Down
83 changes: 75 additions & 8 deletions tests/codegen/handlers/test_class_designate.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
from xsdata.codegen.container import ClassContainer
from xsdata.codegen.handlers import ClassDesignateHandler
from xsdata.exceptions import CodeGenerationError
from xsdata.models.config import GeneratorAlias
from xsdata.models.config import GeneratorConfig
from xsdata.models.config import StructureStyle
from xsdata.models.enums import Namespace
Expand Down Expand Up @@ -56,8 +58,8 @@ def test_group_by_filenames(self):

def test_group_by_namespace(self):
classes = [
ClassFactory.create(qname="{a}a", location="foo"),
ClassFactory.create(qname="{a}b", location="foo"),
ClassFactory.create(qname="{myNS.tempuri.org}a", location="foo"),
ClassFactory.create(qname="{myNS.tempuri.org}b", location="foo"),
ClassFactory.create(qname="b", location="bar"),
]

Expand All @@ -66,13 +68,13 @@ def test_group_by_namespace(self):
self.config.output.package = "bar"

self.handler.run()
self.assertEqual("bar", classes[0].package)
self.assertEqual("bar", classes[1].package)
self.assertEqual("bar", classes[2].package)
self.assertEqual("bar.org.tempuri", classes[0].package)
self.assertEqual("bar.org.tempuri", classes[1].package)
self.assertEqual("", classes[2].package)

self.assertEqual("a", classes[0].module)
self.assertEqual("a", classes[1].module)
self.assertEqual("", classes[2].module)
self.assertEqual("myNS", classes[0].module)
self.assertEqual("myNS", classes[1].module)
self.assertEqual("bar", classes[2].module)

def test_group_all_together(self):
classes = [
Expand Down Expand Up @@ -128,3 +130,68 @@ def test_group_by_strong_components(self):
self.assertEqual("class_E", classes[1].module)
self.assertEqual("class_E", classes[2].module)
self.assertEqual("class_E", classes[3].module)

def test_group_by_namespace_clusters(self):
classes = [
ClassFactory.create("{urn:foo-bar:com}a"),
ClassFactory.create("{urn:foo-bar:add}b"),
ClassFactory.create("{urn:foo-bar:add}c"),
ClassFactory.create("{urn:foo-bar:add}d"),
]

classes[0].attrs.append(AttrFactory.reference(classes[1].qname))
classes[1].attrs.append(AttrFactory.reference(classes[2].qname))
classes[2].attrs.append(AttrFactory.reference(classes[3].qname))
classes[3].attrs.append(AttrFactory.reference(classes[1].qname, circular=True))

self.config.output.structure = StructureStyle.NAMESPACE_CLUSTERS
self.config.output.package = "models"
self.container.extend(classes)

self.handler.run()
self.assertEqual("a", classes[0].module)
self.assertEqual("d", classes[1].module)
self.assertEqual("d", classes[2].module)
self.assertEqual("d", classes[3].module)

self.assertEqual("models.bar.foo.com", classes[0].package)
self.assertEqual("models.bar.foo.add", classes[1].package)
self.assertEqual("models.bar.foo.add", classes[2].package)
self.assertEqual("models.bar.foo.add", classes[3].package)

def test_group_by_namespace_clusters_raises_exception(self):
classes = [
ClassFactory.create("{urn:foo-bar:com}a"),
ClassFactory.create("{urn:foo-bar:add}b"),
ClassFactory.create("{urn:foo-bar:exc}c"),
ClassFactory.create("{urn:foo-bar:add}d"),
]

classes[0].attrs.append(AttrFactory.reference(classes[1].qname))
classes[1].attrs.append(AttrFactory.reference(classes[2].qname))
classes[2].attrs.append(AttrFactory.reference(classes[3].qname))
classes[3].attrs.append(AttrFactory.reference(classes[1].qname, circular=True))

self.config.output.structure = StructureStyle.NAMESPACE_CLUSTERS
self.config.output.package = "models"
self.container.extend(classes)

with self.assertRaises(CodeGenerationError) as cm:
self.handler.run()

self.assertEqual(
"Found strongly connected classes from different namespaces, "
"grouping them is impossible!",
str(cm.exception),
)

def test_combine_ns_package(self):
namespace = "urn:foo-bar:add"
result = self.handler.combine_ns_package(namespace)
self.assertEqual(["generated", "bar", "foo", "add"], result)

alias = GeneratorAlias(source=namespace, target="add.again")
self.config.aliases.package_name.append(alias)

result = self.handler.combine_ns_package(namespace)
self.assertEqual(["generated", "add", "again"], result)
8 changes: 8 additions & 0 deletions tests/codegen/test_container.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,14 @@ def process_class(x: Class, step: int):
self.assertEqual(second, self.container.find_inner(obj, "{a}b"))
mock_process_class.assert_called_once_with(first, Steps.FLATTEN)

def test_first(self):
obj = ClassFactory.create()
self.container.add(obj)
self.assertEqual(obj, self.container.first(obj.qname))

with self.assertRaises(KeyError) as cm:
self.container.first("aa")

def test_process_class(self):
target = ClassFactory.create(
inner=[ClassFactory.elements(2), ClassFactory.elements(1)]
Expand Down
23 changes: 23 additions & 0 deletions tests/utils/test_namespaces.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from xsdata.utils.namespaces import load_prefix
from xsdata.utils.namespaces import prefix_exists
from xsdata.utils.namespaces import split_qname
from xsdata.utils.namespaces import to_package_name


class NamespacesTests(TestCase):
Expand Down Expand Up @@ -112,3 +113,25 @@ def test_is_default(self):
self.assertTrue(is_default("foo", {"": "foo"}))
self.assertTrue(is_default("foo", {None: "foo"}))
self.assertTrue(is_default("foo", {"a": "foo", None: "foo"}))

def test_to_package_name(self):
cases = {
"http://www.w3.org/XML/1998/namespace": "org.w3.XML.1998.namespace",
"http://www.w3.org/XML/2008/06/xlink.xsd": "org.w3.XML.2008.06.xlink",
"http://xsdtesting": "xsdtesting",
"http://xsdtesting:8080": "xsdtesting",
"http://xsdtesting:8080#target": "xsdtesting",
"myNS.tempuri.org": "org.tempuri.myNS",
"ElemDecl/disallowedSubst": "ElemDecl.disallowedSubst",
"http://xstest-tns/schema11": "xstest-tns.schema11",
"http://uri.etsi.org/#": "org.etsi.uri",
"urn:xmlns:25hoursaday-com:address": "com.25hoursaday.address",
"urn:my.test:SchemaB": "my.test.SchemaB",
"http://": "",
"": "",
" ": "",
None: "",
}

for uri, package in cases.items():
self.assertEqual(package, to_package_name(uri))
2 changes: 2 additions & 0 deletions xsdata/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,8 @@ def download(source: str, output: str):
"\n\n"
"clusters: group by strong connected dependencies"
"\n\n"
"namespace-clusters: group by strong connected dependencies and namespaces"
"\n\n"
"single-package: group all classes together"
),
default="filenames",
Expand Down
7 changes: 7 additions & 0 deletions xsdata/codegen/container.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,13 @@ def find_inner(self, source: Class, qname: str) -> Class:

return inner

def first(self, qname: str) -> Class:
classes = self.data.get(qname)
if not classes:
raise KeyError(f"Class {qname} not found")

return classes[0]

def process(self):
"""
Run all the process handlers.
Expand Down
64 changes: 48 additions & 16 deletions xsdata/codegen/handlers/class_designate.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@
from collections import defaultdict
from pathlib import Path
from typing import Iterable
from typing import Iterator
from typing import List
from typing import Optional
from typing import Set
from urllib.parse import urlparse

from toposort import toposort_flatten
Expand All @@ -11,10 +14,12 @@
from xsdata.codegen.models import Class
from xsdata.codegen.models import get_location
from xsdata.codegen.models import get_target_namespace
from xsdata.exceptions import CodeGenerationError
from xsdata.models.config import StructureStyle
from xsdata.models.enums import COMMON_SCHEMA_DIR
from xsdata.utils import collections
from xsdata.utils.graphs import strongly_connected_components
from xsdata.utils.namespaces import to_package_name
from xsdata.utils.package import module_name


Expand All @@ -32,6 +37,8 @@ def run(self):
self.group_all_together()
elif structure_style == StructureStyle.CLUSTERS:
self.group_by_strong_components()
elif structure_style == StructureStyle.NAMESPACE_CLUSTERS:
self.group_by_namespace_clusters()
else:
self.group_by_filenames()

Expand All @@ -57,10 +64,12 @@ def group_by_filenames(self):

def group_by_namespace(self):
"""Group classes by their target namespace."""
package = self.container.config.output.package
groups = collections.group_by(self.container, key=get_target_namespace)
for namespace, classes in groups.items():
self.assign(classes, package, namespace or "")
parts = self.combine_ns_package(namespace)
module = parts.pop()
package = ".".join(parts)
self.assign(classes, package, module)

def group_all_together(self):
"""Group all classes together in the same module."""
Expand All @@ -72,24 +81,35 @@ def group_all_together(self):

def group_by_strong_components(self):
"""Find circular imports and cluster their classes together."""
edges = {}
class_map = {}
for obj in self.container:
edges[obj.qname] = set(obj.dependencies(True))
class_map[obj.qname] = obj

groups = strongly_connected_components(edges)
package = self.container.config.output.package
for group in self.strongly_connected_classes():
classes = self.sorted_classes(group)
module = classes[0].name
self.assign(classes, package, module)

def group_by_namespace_clusters(self):
for group in self.strongly_connected_classes():
classes = self.sorted_classes(group)
if len(set(map(get_target_namespace, classes))) > 1:
raise CodeGenerationError(
"Found strongly connected classes from different "
"namespaces, grouping them is impossible!"
)

for group in groups:
group_edges = {
qname: set(class_map[qname].dependencies()).intersection(group)
for qname in group
}
classes = [class_map[qname] for qname in toposort_flatten(group_edges)]
parts = self.combine_ns_package(classes[0].target_namespace)
module = classes[0].name
self.assign(classes, ".".join(parts), module)

self.assign(classes, package, module)
def sorted_classes(self, qnames: Set[str]) -> List[Class]:
edges = {
qname: set(self.container.first(qname).dependencies()).intersection(qnames)
for qname in qnames
}
return [self.container.first(qname) for qname in toposort_flatten(edges)]

def strongly_connected_classes(self) -> Iterator[Set[str]]:
edges = {obj.qname: list(set(obj.dependencies(True))) for obj in self.container}
return strongly_connected_components(edges)

@classmethod
def assign(cls, classes: Iterable[Class], package: str, module: str):
Expand Down Expand Up @@ -118,3 +138,15 @@ def group_common_paths(cls, paths: Iterable[str]) -> List[List[str]]:
groups[index].append(path)

return list(groups.values())

def combine_ns_package(self, namespace: Optional[str]) -> List[str]:
result = self.container.config.output.package.split(".")
aliases = self.container.config.aliases.package_name
alias = collections.first(x.target for x in aliases if x.source == namespace)

if alias:
result.extend(alias.split("."))
else:
result.extend(to_package_name(namespace).split("."))

return list(filter(None, result))
5 changes: 5 additions & 0 deletions xsdata/codegen/mixins.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,11 @@ def find(self, qname: str, condition: Callable = return_true) -> Optional[Class]
def find_inner(self, source: Class, qname: str) -> Class:
"""Search by qualified name for a specific inner class or fail."""

@abc.abstractmethod
def first(self, qname: str) -> Class:
"""Search by qualified name for a specific class and return the first
available."""

@abc.abstractmethod
def add(self, item: Class):
"""Add class item to the container."""
Expand Down
2 changes: 2 additions & 0 deletions xsdata/models/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,14 @@ class StructureStyle(Enum):
:cvar NAMESPACES: namespaces
:cvar CLUSTERS: clusters
:cvar SINGLE_PACKAGE: single-package
:cvar NAMESPACE_CLUSTERS: namespace-clusters
"""

FILENAMES = "filenames"
NAMESPACES = "namespaces"
CLUSTERS = "clusters"
SINGLE_PACKAGE = "single-package"
NAMESPACE_CLUSTERS = "namespace-clusters"


class NameCase(Enum):
Expand Down
Loading