Python: move model generation to Ruby templates

.github/workflows/test-python.yml

@@ -27,6 +27,8 @@ jobs:
             python-version: "3.11"
           - os: ubuntu-latest
             python-version: "3.12"
+          - os: ubuntu-latest
+            python-version: "3.13"
           - os: ubuntu-latest
             python-version: "pypy3.9"
           - os: ubuntu-latest

.pre-commit-config.yaml

@@ -2,6 +2,7 @@
 # See https://pre-commit.com/hooks.html for more hooks
 ---
 files: ^python/
+exclude: ^python/src/cucumber_messages/_messages\.py
 repos:
   - repo: https://github.com/psf/black
     rev: 24.10.0

codegen/codegen.rb

@@ -11,6 +11,7 @@
 require_relative 'generators/markdown'
 require_relative 'generators/perl'
 require_relative 'generators/php'
+require_relative 'generators/python'
 require_relative 'generators/ruby'
 require_relative 'generators/typescript'
 

codegen/generators/python.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module Generator
+  class Python < Base
+    def array_type_for(type_name)
+      inner_type = if language_translations_for_data_types.values.include?(type_name)
+        type_name  # Keep primitive types as is
+      else
+        class_name(type_name)  # CamelCase for complex types
+      end
+      "list[#{inner_type}]"
+    end
+
+    def format_description(raw_description, indent_string: '    ')
+      return '""" """' if raw_description.nil?
+
+      formatted = raw_description
+                   .split("\n")
+                   .map { |line| "#{line}" }
+                   .join("\n#{indent_string}")
+      %("""\n#{indent_string}#{formatted}\n#{indent_string}""")
+    end
+
+    def language_translations_for_data_types
+      {
+        'integer' => 'int',
+        'string' => 'str',
+        'boolean' => 'bool',
+        'array' => 'list'
+      }
+    end
+
+    private
+
+    def default_value(parent_type_name, property_name, property)
+      if property['type'] == 'string'
+        default_value_for_string(parent_type_name, property_name, property)
+      elsif property['type'] == 'integer'
+        '0'
+      elsif property['type'] == 'boolean'
+        'False'
+      elsif property['type'] == 'array'
+        '[]'
+      elsif property['$ref']
+        "#{class_name(type_for(parent_type_name, nil, property))}()"
+      else
+        'None'
+      end
+    end
+
+    def default_value_for_string(parent_type_name, property_name, property)
+      if property['enum']
+        enum_type_name = type_for(parent_type_name, property_name, property)
+        "#{class_name(enum_type_name)}.#{enum_constant(property['enum'][0])}"
+      else
+        '""'
+      end
+    end
+
+    def type_for(parent_type_name, property_name, property)
+      if property['$ref']
+        property_type_from_ref(property['$ref'])
+      elsif property['type']
+        property_type_from_type(parent_type_name, property_name, property, type: property['type'])
+      else
+        raise "Property #{property_name} did not define 'type' or '$ref'"
+      end
+    end
+
+    def property_type_from_type(parent_type_name, property_name, property, type:)
+      if type == 'array'
+        array_type_for(type_for(parent_type_name, nil, property['items']))
+      elsif property['enum']
+        enum_name(parent_type_name, property_name, property['enum'])
+      else
+        language_translations_for_data_types.fetch(type)
+      end
+    end
+
+    def enum_constant(value)
+      value.gsub(/[.\/+]/, '_').downcase
+    end
+
+    def enum_name(parent_type_name, property_name, enum)
+      "#{class_name(parent_type_name)}#{capitalize(property_name)}".tap do |name|
+        @enum_set.add({ name: name, values: enum })
+      end
+    end
+
+    def property_type_from_ref(ref)
+      class_name(ref)
+    end
+
+    def class_name(ref)
+      return ref if language_translations_for_data_types.values.include?(ref)
+
+      # Remove .json extension if present
+      name = ref.sub(/\.json$/, '')
+      # Get the basename without path
+      name = File.basename(name)
+      # Convert each word to proper case, handling camelCase and snake_case
+      parts = name.gsub(/[._-]/, '_').split('_').map do |part|
+        # Split by any existing camelCase
+        subparts = part.scan(/[A-Z][a-z]*|[a-z]+/)
+        subparts.map(&:capitalize).join
+      end
+      # Join all parts to create final CamelCase name
+      parts.join
+    end
+  end
+end

codegen/templates/python.py.erb

@@ -0,0 +1,65 @@
+# This code was generated using the code generator from cucumber-messages.
+# Manual changes will be lost if the code is regenerated.
+# Generator: cucumber-messages-python
+
+from __future__ import annotations
+from dataclasses import dataclass
+from enum import Enum
+from typing import Optional
+
+<%- @enums.each do |enum| -%>
+class <%= enum[:name] %>(Enum):
+    <%- enum[:values].each do |value| -%>
+    <%= value.downcase.gsub(/[.\/+\s-]/, '_') %> = "<%= value %>"
+    <%- end -%>
+
+<%- end -%>
+<%- @schemas.each do |key, definition| -%>
+@dataclass
+class <%= class_name(key) %>:
+    <%- if definition['description'] -%>
+    <%= format_description(definition['description']) %>
+    <%- end -%>
+    <%- if definition['properties'].any? -%>
+    <%-
+        required_fields = definition['required'] || []
+        properties = definition['properties'].sort_by do |name, *|
+            [required_fields.include?(name) ? 0 : 1, name]
+        end
+    -%>
+    <%- properties.each do |property_name, property| -%>
+    <%-
+        snake_name = property_name.gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+                                 .gsub(/([a-z\d])([A-Z])/, '\1_\2')
+                                 .downcase
+
+        property_type = type_for(key, property_name, property)
+        is_required = required_fields.include?(property_name)
+        is_list = property_type.start_with?('list[')
+
+        if is_list
+            list_type = property_type.match(/list\[(.*?)\]/)
+            inner_type = list_type[1]
+            if inner_type =~ /^[A-Z]/
+                property_type = "list['#{class_name(inner_type)}']"
+            else
+                property_type = "list[#{inner_type}]"
+            end
+        elsif property_type =~ /^[A-Z]/
+            property_type = "'#{class_name(property_type)}'"
+        end
+    -%>
+    <%- if property['description'] -%>
+    <%= format_description(property['description']) %>
+    <%- end -%>
+    <%- if is_required -%>
+    <%= snake_name %>: <%= property_type %>
+    <%- else -%>
+    <%= snake_name %>: Optional[<%= property_type %>] = None
+    <%- end -%>
+    <%- end -%>
+    <%- else -%>
+    pass
+    <%- end -%>
+
+<%- end -%>

python/Makefile

@@ -2,36 +2,16 @@ schemas = $(shell find ../jsonschema -name "*.json")
 
 .DEFAULT_GOAL = help
 
-MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
-HERE := $(dir $(MKFILE_PATH))
-
 help: ## Show this help
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make <target>\n\nWhere <target> is one of:\n"} /^[$$()% a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
-generate: require install-deps
-	datamodel-codegen \
-		--output-model-type "dataclasses.dataclass" \
-		--input $(HERE)../jsonschema/Envelope.json \
-		--output $(HERE)src/cucumber_messages/_messages.py \
-		--input-file-type=jsonschema \
-		--class-name Envelope  \
-		--target-python-version=3.9 \
-		--allow-extra-fields \
-		--allow-population-by-field-name \
-		--snake-case-field \
-		--use-standard-collections \
-		--use-double-quotes \
-		--use-exact-imports \
-		--use-field-description \
-		--use-union-operator \
-		--disable-timestamp
+generate: require src/cucumber_messages/_messages.py
 
-require: ## Check requirements for the code generation (python is required)
-	@python --version >/dev/null 2>&1 || (echo "ERROR: python is required."; exit 1)
+require: ## Check requirements for the code generation (ruby is required)
+	@ruby --version >/dev/null 2>&1 || (echo "ERROR: ruby is required."; exit 1)
 
-clean: ## Stub for the ancestor Makefile
-	rm -rf $(HERE)src/cucumber_messages/_messages.py.
+clean: ## Remove automatically generated files and related artifacts
+	rm -f src/cucumber_messages/_messages.py
 
-install-deps: ## Install generation dependencies
-	python -m ensurepip --upgrade
-	pip install $(HERE)[generation]
+src/cucumber_messages/_messages.py: $(schemas) ../codegen/codegen.rb ../codegen/templates/python.py.erb
+	ruby ../codegen/codegen.rb Generator::Python python.py.erb > $@

python/pyproject.toml

@@ -50,6 +50,8 @@ test = [
 ]
 test-coverage = [
   "coverage",
+  "GitPython",
+  "packaging",
   "pytest"
 ]
 

python/src/cucumber_messages/__init__.py

@@ -1,14 +1,4 @@
-from . import _messages
-from . import json_converter
+from . import _messages, json_converter
 from ._messages import *
 
-# Renaming types because of confusing collision naming
-HookType = Type
-PickleStepType = Type1
-ExpressionType = Type2
-
-serializer: json_converter.DataclassSerializer = json_converter.DataclassSerializer(module_scope=_messages)
-
-del Type
-del Type1
-del Type2
+message_converter: json_converter.JsonDataclassConverter = json_converter.JsonDataclassConverter(module_scope=_messages)