From beabc3d71007fb75f9e89835e65e87f57a16450d Mon Sep 17 00:00:00 2001 From: Sean O'Brien <60306702+stobrien89@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:43:16 -0400 Subject: [PATCH 1/6] add lazy model generation --- .../types/.smithy-python-manifest.json | 4 +- .../types/src/example_types/models.py | 75 ++ .../types/src/example_types/models.pyi | 135 +++ .../python/codegen/PythonSymbolProvider.java | 13 + .../python/codegen/reserved-member-names.txt | 14 + .../codegen/PythonSymbolProviderTest.java | 62 ++ ...y-core-enhancement-model-target-names.json | 4 + .../src/smithy_core/_dynamic_models.py | 165 +++- .../src/smithy_core/_model_naming.py | 17 +- .../tests/unit/test_dynamic_models.py | 307 ++++++- .../tests/unit/test_model_naming.py | 85 +- ...mithy-python-feature-model-generation.json | 4 + .../smithy-python/src/smithy_python/files.py | 6 +- .../src/smithy_python/generation.py | 52 +- .../smithy-python/src/smithy_python/models.py | 457 ++++++++++ .../src/smithy_python/symbols.py | 43 +- .../smithy-python/src/smithy_python/writer.py | 77 +- .../smithy-python/tests/unit/test_models.py | 841 ++++++++++++++++++ .../smithy-python/tests/unit/test_symbols.py | 83 +- .../smithy-python/tests/unit/test_writer.py | 31 +- pyproject.toml | 1 + scripts/api-snapshot/api.json | 3 + scripts/codegen-output-py/generate.py | 12 +- uv.lock | 12 +- 24 files changed, 2422 insertions(+), 81 deletions(-) create mode 100644 codegen-output-py/types/src/example_types/models.py create mode 100644 codegen-output-py/types/src/example_types/models.pyi create mode 100644 packages/smithy-core/.changes/next-release/smithy-core-enhancement-model-target-names.json create mode 100644 packages/smithy-python/.changes/next-release/smithy-python-feature-model-generation.json create mode 100644 packages/smithy-python/src/smithy_python/models.py create mode 100644 packages/smithy-python/tests/unit/test_models.py diff --git a/codegen-output-py/types/.smithy-python-manifest.json b/codegen-output-py/types/.smithy-python-manifest.json index 9c7015360..f5426b5de 100644 --- a/codegen-output-py/types/.smithy-python-manifest.json +++ b/codegen-output-py/types/.smithy-python-manifest.json @@ -1,6 +1,8 @@ { "files": { - "src/example_types/_private/schemas.py": "d0f35e8979c4aa9d49a1d9c0c8efeb860420bc0ceab3ba3edcc9488c053e9110" + "src/example_types/_private/schemas.py": "d0f35e8979c4aa9d49a1d9c0c8efeb860420bc0ceab3ba3edcc9488c053e9110", + "src/example_types/models.py": "6042e2e6454b40fef929d96cb7608ae8c591ac7205577f14337435718bb7c077", + "src/example_types/models.pyi": "bb3d59e91999a62438fce52fe3be551c63f998565e4f9640e4fb1e275b972b14" }, "version": 1 } diff --git a/codegen-output-py/types/src/example_types/models.py b/codegen-output-py/types/src/example_types/models.py new file mode 100644 index 000000000..aacb742b3 --- /dev/null +++ b/codegen-output-py/types/src/example_types/models.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +from smithy_core._dynamic_models import ModelRegistry +from smithy_core.exceptions import ModeledError + +from ._private import schemas + + +class ServiceError(ModeledError): + """ + Base error for all errors in the service. + + Some exceptions do not extend from this class, including synthetic, + implicit, and shared exception types. + """ + + +_MODEL_INDEX = { + "GetExampleInput": "GET_EXAMPLE_INPUT", + "GetExampleOutput": "GET_EXAMPLE_OUTPUT", + "ExampleError": "EXAMPLE_ERROR", + "RenamedExample": "RENAMED_EXAMPLE", + "ExampleValue": "EXAMPLE_VALUE", + "ExampleValueText": "EXAMPLE_VALUE", + "ExampleValueNumber": "EXAMPLE_VALUE", + "ExampleValueEmpty": "EXAMPLE_VALUE", + "ExampleValueUnknown": "EXAMPLE_VALUE", + "Status": "STATUS", + "Code": "CODE", +} + +_RENAMES = { + "RenamedExample": "RENAMED_EXAMPLE", + "ExampleValue": "EXAMPLE_VALUE", +} + +_TARGET_NAMES = { + "example#Example": "RenamedExample", + "example#Value": "ExampleValue", +} + +_MODEL_NAMES = (*tuple(_MODEL_INDEX), "Unit") +__all__ = ("ServiceError", *_MODEL_NAMES) + + +_registry = ModelRegistry( + schemas, + renames=_RENAMES, + index=_MODEL_INDEX, + target_names=_TARGET_NAMES, + shape_names=frozenset( + { + "Code", + "ExampleError", + "ExampleValue", + "GetExampleInput", + "GetExampleOutput", + "RenamedExample", + "Status", + "Unit", + } + ), + service_error="ServiceError", +) +del schemas, ModelRegistry, ModeledError + + +def __getattr__(name: str): + value = _registry.build(name) + globals()[name] = value + return value + + +def __dir__() -> list[str]: + return sorted({*globals(), *__all__}) diff --git a/codegen-output-py/types/src/example_types/models.pyi b/codegen-output-py/types/src/example_types/models.pyi new file mode 100644 index 000000000..a4ba94f5e --- /dev/null +++ b/codegen-output-py/types/src/example_types/models.pyi @@ -0,0 +1,135 @@ +from __future__ import annotations + +from dataclasses import dataclass +from enum import IntEnum, StrEnum +from typing import Self + +from smithy_core.deserializers import ShapeDeserializer +from smithy_core.documents import Document +from smithy_core.exceptions import ModeledError +from smithy_core.response import ResponseMetadata +from smithy_core.serializers import ShapeSerializer +from smithy_core.types import UnknownEnumMixin + +__all__ = [ + "Code", + "ExampleError", + "ExampleValue", + "ExampleValueEmpty", + "ExampleValueNumber", + "ExampleValueText", + "ExampleValueUnknown", + "GetExampleInput", + "GetExampleOutput", + "RenamedExample", + "ServiceError", + "Status", + "Unit", +] + +class _SerdeShape: + def serialize(self, serializer: ShapeSerializer) -> None: ... + def serialize_members(self, serializer: ShapeSerializer) -> None: ... + @classmethod + def deserialize(cls, deserializer: ShapeDeserializer) -> Self: ... + +class ServiceError(ModeledError): + """ + Base error for all errors in the service. + + Some exceptions do not extend from this class, including synthetic, + implicit, and shared exception types. + """ + +@dataclass(kw_only=True) +class GetExampleInput(_SerdeShape): + "Input for retrieving an example." + + id: str | None = ... + +@dataclass(kw_only=True) +class GetExampleOutput(_SerdeShape): + "Dataclass for GetExampleOutput structure." + + example: RenamedExample | None = ... + value: ExampleValue | None = ... + response_metadata: ResponseMetadata = ... + """ + Metadata about the response that produced this output. + + Use this to recover the request identifiers a service's support team + needs in order to investigate a call. Members of the metadata are + individually optional. + """ + +@dataclass(kw_only=True) +class ExampleError(ServiceError, _SerdeShape): + "An example modeled error." + + retry_after_seconds: int | None = ... + response_metadata: ResponseMetadata = ... + """ + Metadata about the response that produced this output. + + Use this to recover the request identifiers a service's support team + needs in order to investigate a call. Members of the metadata are + individually optional. + """ + +@dataclass(kw_only=True) +class RenamedExample(_SerdeShape): + "A representative generated structure." + + name: str + "The example name." + count: int | None = ... + tags: list[str] | None = ... + attributes: dict[str, Document] | None = ... + choice: ExampleValue | None = ... + status: str | None = ... + code: int | None = ... + +@dataclass +class ExampleValueText(_SerdeShape): + value: str + +@dataclass +class ExampleValueNumber(_SerdeShape): + value: int + +@dataclass +class ExampleValueEmpty(_SerdeShape): + value: Unit + +@dataclass +class ExampleValueUnknown(_SerdeShape): + """ + Represents an unknown variant. + + If you receive this value, you will need to update your library to + receive the parsed value. + + This value may not be deliberately sent. + """ + + tag: str + +ExampleValue = ( + ExampleValueText | ExampleValueNumber | ExampleValueEmpty | ExampleValueUnknown +) +"A representative generated union." + +class _ExampleValueDeserializer: + def deserialize(self, deserializer: ShapeDeserializer) -> ExampleValue: ... + +class Status(UnknownEnumMixin, StrEnum): + ACTIVE = "active" + INACTIVE = "inactive" + +class Code(UnknownEnumMixin, IntEnum): + OK = 200 + NOT_FOUND = 404 + +@dataclass(kw_only=True) +class Unit(_SerdeShape): + pass diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSymbolProvider.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSymbolProvider.java index 62b5b12e4..13e5883fc 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSymbolProvider.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSymbolProvider.java @@ -147,6 +147,9 @@ public String toMemberName(MemberShape shape) { } var memberName = escaper.escapeMemberName(CaseUtils.toSnakeCase(shape.getMemberName())); + if (memberName.startsWith("__") && !memberName.endsWith("__")) { + memberName += "___"; + } // Escape words that are only reserved for error members. if (shape.hasTrait(ErrorTrait.class)) { @@ -170,6 +173,16 @@ && carriesResponseMetadata(container)) { } if (container.isEnumShape() || container.isIntEnumShape()) { + if (memberName.startsWith("__") + && memberName.endsWith("__") + && !memberName.endsWith("___")) { + memberName += "_"; + } else if (memberName.startsWith("_") + && !memberName.startsWith("__") + && memberName.endsWith("_") + && !memberName.endsWith("__")) { + memberName += "_"; + } memberName = memberName.toUpperCase(Locale.ENGLISH); } return memberName; diff --git a/codegen/core/src/main/resources/software/amazon/smithy/python/codegen/reserved-member-names.txt b/codegen/core/src/main/resources/software/amazon/smithy/python/codegen/reserved-member-names.txt index 1831fb341..b18e8e1a8 100644 --- a/codegen/core/src/main/resources/software/amazon/smithy/python/codegen/reserved-member-names.txt +++ b/codegen/core/src/main/resources/software/amazon/smithy/python/codegen/reserved-member-names.txt @@ -81,3 +81,17 @@ str # that are likely imported in the generated code (e.g. datetime) # We only escape the types we use. datetime + +# Generated model runtime methods. +serialize +serialize_members +deserialize + +# Dataclass-generated methods and hooks. +__init__ +__post_init__ +__repr__ +__annotations__ + +# Generated model runtime methods. +_smithy_default diff --git a/codegen/core/src/test/java/software/amazon/smithy/python/codegen/PythonSymbolProviderTest.java b/codegen/core/src/test/java/software/amazon/smithy/python/codegen/PythonSymbolProviderTest.java index 0102a560c..3941b92e2 100644 --- a/codegen/core/src/test/java/software/amazon/smithy/python/codegen/PythonSymbolProviderTest.java +++ b/codegen/core/src/test/java/software/amazon/smithy/python/codegen/PythonSymbolProviderTest.java @@ -137,6 +137,68 @@ public void testUnrelatedMembersOnOutputsAndErrorsAreUnaffected() { assertEquals("retry_after", memberName(provider, model, "ThingError$retryAfter")); } + @Test + public void testDataclassHooksAreEscaped() { + Model model = loadModel(""" + $version: "2" + namespace smithy.example + + service TestService { + version: "2024-01-01" + operations: [GetThing] + } + + operation GetThing { + input: GetThingInput + } + + structure GetThingInput { + __init__: String + __post_init__: String + _smithy_default: String + __repr__: String + __annotations__: String + __value: String + } + """); + PythonSymbolProvider provider = createProvider(model); + + assertEquals("__init___", memberName(provider, model, "GetThingInput$__init__")); + assertEquals("__post_init___", memberName(provider, model, "GetThingInput$__post_init__")); + assertEquals("_smithy_default_", memberName(provider, model, "GetThingInput$_smithy_default")); + assertEquals("__repr___", memberName(provider, model, "GetThingInput$__repr__")); + assertEquals("__annotations___", memberName(provider, model, "GetThingInput$__annotations__")); + assertEquals("__value___", memberName(provider, model, "GetThingInput$__value")); + } + + @Test + public void testEnumSunderNamesAreEscaped() { + Model model = loadModel(""" + $version: "2" + namespace smithy.example + + service TestService { + version: "2024-01-01" + } + + enum StringValues { + _VALUE_ = "value" + __DUNDER__ = "dunder" + } + + intEnum IntValues { + _VALUE_ = 1 + __DUNDER__ = 2 + } + """); + PythonSymbolProvider provider = createProvider(model); + + assertEquals("_VALUE__", memberName(provider, model, "StringValues$_VALUE_")); + assertEquals("_VALUE__", memberName(provider, model, "IntValues$_VALUE_")); + assertEquals("__DUNDER___", memberName(provider, model, "StringValues$__DUNDER__")); + assertEquals("__DUNDER___", memberName(provider, model, "IntValues$__DUNDER__")); + } + private static final String RESPONSE_METADATA_MODEL = """ $version: "2" namespace smithy.example diff --git a/packages/smithy-core/.changes/next-release/smithy-core-enhancement-model-target-names.json b/packages/smithy-core/.changes/next-release/smithy-core-enhancement-model-target-names.json new file mode 100644 index 000000000..723b0ddcb --- /dev/null +++ b/packages/smithy-core/.changes/next-release/smithy-core-enhancement-model-target-names.json @@ -0,0 +1,4 @@ +{ + "type": "enhancement", + "description": "Added optional full shape ID target mappings to `ModelRegistry` so generators can resolve renamed and same-named cross-namespace model targets without ambiguous local-name lookups." +} diff --git a/packages/smithy-core/src/smithy_core/_dynamic_models.py b/packages/smithy-core/src/smithy_core/_dynamic_models.py index 028323346..1ee10c042 100644 --- a/packages/smithy-core/src/smithy_core/_dynamic_models.py +++ b/packages/smithy-core/src/smithy_core/_dynamic_models.py @@ -18,7 +18,7 @@ import datetime from base64 import b64decode -from collections.abc import Collection +from collections.abc import Collection, Mapping from dataclasses import field as _field from dataclasses import make_dataclass from decimal import Decimal @@ -30,7 +30,7 @@ from ._model_naming import member_field_name, union_unknown_name, variant_class_name from ._schema_compact import hydrate from .deserializers import ShapeDeserializer -from .exceptions import ModeledError +from .exceptions import ModeledError, SerializationError from .schemas import Schema, SchemaExtension from .serializers import ShapeSerializer from .shapes import ShapeID, ShapeType @@ -151,14 +151,15 @@ class ModelRegistry: def __init__( self, schemas_module: ModuleType, - renames: dict[str, str] | None = None, + renames: Mapping[str, str] | None = None, *, - index: dict[str, str] | None = None, + index: Mapping[str, str] | None = None, + target_names: Mapping[str, str] | None = None, shape_names: Collection[str] | None = None, service_error: str | None = None, ) -> None: self._schemas = schemas_module - self._renames = renames or {} + self._renames = dict(renames or {}) self._classes: dict[str, type] = {} self._union_schemas: dict[str, Schema] = {} # Reentrant because build() recurses: unions build their variants, and a @@ -169,13 +170,22 @@ def __init__( # A codegen-supplied index (name -> schema-const, variant/unknown names already # disambiguated) lets the registry resolve a build() without hydrating every # schema in the module just to map names; the scanning fallback hydrates all. - self._index: dict[str, str] | None = index + self._index = dict(index) if index is not None else None + # Full shape ID -> generated public model name. Full IDs keep member-target + # resolution unambiguous when service renames distinguish same-named shapes + # from different namespaces. + self._target_names = dict(target_names or {}) + self._renamed_schemas = { + schema_name: model_name for model_name, schema_name in self._renames.items() + } self._shape_names: set[str] = set(shape_names) if shape_names else set() # Generated clients name their per-service error base (a ModeledError subclass # emitted into the models module). Modeled errors extend it so `except # ServiceError` catches them; a hand-built registry omits it and falls back to # ModeledError. self._service_error_name = service_error + if service_error is not None: + self._shape_names.add(service_error) self._codec = SchemaExtension(self._resolve_member_codec) self._field_names_ext = SchemaExtension(self._compute_field_names) @@ -230,7 +240,7 @@ def build(self, name: str) -> type: # Event-stream union deserializer helper: _Deserializer().deserialize(d). if name.startswith("_") and name.endswith("Deserializer"): union_name = name[1 : -len("Deserializer")] - if union_name in index: + if union_name in index or union_name in self._renames: built = self._build_union_deserializer(union_name) self._classes[name] = built return built @@ -253,14 +263,26 @@ def build(self, name: str) -> type: return built raise AttributeError(name) schema = hydrate(getattr(self._schemas, const)) + canonical = self._renamed_schemas.get(const) if schema.shape_type == ShapeType.UNION: - self._build_union(schema.id.name, schema) - return self._classes[name] + self._build_union(canonical or schema.id.name, schema) + built = self._classes.get(name) + if built is None and name == schema.id.name: + built = self._classes[canonical or schema.id.name] + if built is None: + raise AttributeError(name) + self._classes[name] = built + return built + canonical = canonical or name + if cached := self._classes.get(canonical): + self._classes[name] = cached + return cached if schema.shape_type in (ShapeType.ENUM, ShapeType.INT_ENUM): - built = self._build_enum(name, schema) + built = self._build_enum(canonical, schema) else: - built = self._build_struct(name, schema) + built = self._build_struct(canonical, schema) built.__module__ = module + self._classes[canonical] = built self._classes[name] = built return built @@ -323,15 +345,16 @@ def _error_correct( def _correction_default(self, member: Schema) -> Any: st = member.shape_type if st == ShapeType.STRUCTURE: - target = member.expect_member_target().id.name + target = self._target_name(member.expect_member_target()) return self.build(target)._smithy_default() # type: ignore[attr-defined] if st == ShapeType.UNION: - target = member.expect_member_target().id.name - return self.build(union_unknown_name(target, self._shape_names))(tag="") + target = self._target_name(member.expect_member_target()) + return self.build(self._union_unknown_name(target))(tag="") return _CORRECTION_SCALARS[st]() def _build_struct(self, name: str, schema: Schema) -> type: - is_error = schema.get_trait(ErrorTrait) is not None + error_trait = schema.get_trait(ErrorTrait) + is_error = error_trait is not None # Per Smithy @input structure constraints, every member of an input # structure is effectively optional regardless of @required; only # non-input structures make a @required, non-@default member mandatory. @@ -339,11 +362,25 @@ def _build_struct(self, name: str, schema: Schema) -> type: field_names = self._field_names(schema) fields: list[Any] = [] for i, member in enumerate(schema.members_by_index): + if is_error and field_names[i] == "message": + if member.get_trait(SensitiveTrait) is not None: + fields.append( + ( + "message", + str, + _field(default="", kw_only=False, repr=False), + ) + ) + continue spec = self._field_spec(member, input_nullable) if spec is _MANDATORY: fields.append((field_names[i], _py_type(member))) else: fields.append((field_names[i], _py_type(member), spec)) + if error_trait is not None: + fields.append( + ("fault", str | None, _field(default=error_trait.fault.value)) + ) bases: tuple[type, ...] = (self._error_base(),) if is_error else () # Operation outputs carry a non-modeled response_metadata attribute, excluded # from eq and repr. Errors inherit it from ModeledError instead. @@ -456,16 +493,32 @@ def _default_factory(self, member: Schema, trait: DefaultTrait) -> Any: return None def _build_union(self, name: str, schema: Schema) -> None: + if name in self._union_schemas: + return module = self._models_module_name for member in schema.members_by_index: - vname = variant_class_name( - name, member.expect_member_name(), self._shape_names - ) + vname = self._union_variant_name(name, member.expect_member_name()) variant = self._build_variant(vname, schema, member.expect_member_index()) variant.__module__ = module self._classes[vname] = variant - unknown = union_unknown_name(name, self._shape_names) - unknown_cls = make_dataclass(unknown, [("tag", str)]) + unknown = self._union_unknown_name(name) + + def reject_serialization(self: Any, serializer: ShapeSerializer) -> None: + raise SerializationError("Unknown union variants may not be serialized.") + + @classmethod # type: ignore[misc] + def deserialize(cls: type, deserializer: ShapeDeserializer) -> Any: + raise NotImplementedError + + unknown_cls = make_dataclass( + unknown, + [("tag", str)], + namespace={ + "serialize": reject_serialization, + "serialize_members": reject_serialization, + "deserialize": deserialize, + }, + ) unknown_cls.__module__ = module self._classes[unknown] = unknown_cls self._union_schemas[name] = schema @@ -474,6 +527,55 @@ def _build_union(self, name: str, schema: Schema) -> None: marker = make_dataclass(name, []) marker.__module__ = module self._classes[name] = marker + original = schema.id.name + index = self._ensure_index() + const = self._renames.get(name, index.get(name)) + if ( + original != name + and original not in self._renames + and const is not None + and index.get(original) == const + ): + self._classes.setdefault(original, marker) + self._union_schemas.setdefault(original, schema) + for member in schema.members_by_index: + member_name = member.expect_member_name() + canonical_variant = self._union_variant_name(name, member_name) + original_variant = self._union_variant_name(original, member_name) + if ( + original_variant not in self._renames + and index.get(original_variant, const) == const + ): + self._classes.setdefault( + original_variant, self._classes[canonical_variant] + ) + original_unknown = self._union_unknown_name(original) + if ( + original_unknown not in self._renames + and index.get(original_unknown, const) == const + ): + self._classes.setdefault(original_unknown, unknown_cls) + + def _union_variant_name(self, container: str, member: str) -> str: + capitalized = member[:1].upper() + member[1:] + return self._union_synthetic_name(container, capitalized) + + def _union_unknown_name(self, container: str) -> str: + return self._union_synthetic_name(container, "Unknown") + + def _union_synthetic_name(self, container: str, suffix: str) -> str: + base = container + suffix + disambiguated = container + "_" + suffix + index = self._ensure_index() + const = self._renames.get(container, index.get(container)) + if const is not None: + for candidate in (base, disambiguated): + if self._renames.get(candidate, index.get(candidate)) == const: + return candidate + owner = self._renames.get(base, index.get(base)) + if owner is not None and owner != const: + return disambiguated + return disambiguated if base in self._shape_names else base def _build_variant(self, vname: str, union_schema: Schema, index: int) -> type: registry = self @@ -539,15 +641,13 @@ def consume(member_schema: Schema, de: ShapeDeserializer) -> None: member = by_index[member_schema.expect_member_index()] _, reader = member.get_extension(self._codec) variant = self.build( - variant_class_name(name, member.expect_member_name(), self._shape_names) + self._union_variant_name(name, member.expect_member_name()) ) result[0] = variant(value=reader(de, member)) deserializer.read_struct(schema, consumer=consume) if result[0] is None: - result[0] = self.build(union_unknown_name(name, self._shape_names))( - tag="unknown" - ) + result[0] = self.build(self._union_unknown_name(name))(tag="unknown") return result[0] def _is_streaming(self, member: Schema) -> bool: @@ -566,7 +666,7 @@ def _resolve_member_codec(self, member: Schema) -> tuple[_Writer, _Reader]: # coerce the raw primitive back through the enum class (matching the eager # EnumGenerator's `EnumClass(read_string(...))`). is_int = st == ShapeType.INT_ENUM - enum_cls = self.build(member.expect_member_target().id.name) + enum_cls = self.build(self._target_name(member.expect_member_target())) write_name = "write_integer" if is_int else "write_string" read_name = "read_integer" if is_int else "read_string" @@ -581,7 +681,7 @@ def re(d: ShapeDeserializer, m: Schema) -> Any: if scalar is not None: return scalar if st == ShapeType.STRUCTURE: - target = member.expect_member_target().id.name + target = self._target_name(member.expect_member_target()) def wr(s: ShapeSerializer, m: Schema, v: Any) -> None: s.write_struct(m, v) @@ -597,7 +697,7 @@ def rr(d: ShapeDeserializer, m: Schema) -> Any: return wr, rr if st == ShapeType.UNION: - target = member.expect_member_target().id.name + target = self._target_name(member.expect_member_target()) self.build(target) # ensure union schema is registered def wu(s: ShapeSerializer, m: Schema, v: Any) -> None: @@ -619,6 +719,17 @@ def ru(d: ShapeDeserializer, m: Schema) -> Any: ) raise NotImplementedError(f"no codec for shape type {st}") + def _target_name(self, schema: Schema) -> str: + if generated := self._target_names.get(str(schema.id)): + return generated + index = self._ensure_index() + const = index.get(schema.id.name) + return ( + self._renamed_schemas.get(const, schema.id.name) + if const is not None + else schema.id.name + ) + def _list_codec(self, elem: Schema, sparse: bool) -> tuple[_Writer, _Reader]: def w(s: ShapeSerializer, m: Schema, v: list[Any]) -> None: ew, _ = elem.get_extension(self._codec) diff --git a/packages/smithy-core/src/smithy_core/_model_naming.py b/packages/smithy-core/src/smithy_core/_model_naming.py index 05245e2e8..1fb098e3f 100644 --- a/packages/smithy-core/src/smithy_core/_model_naming.py +++ b/packages/smithy-core/src/smithy_core/_model_naming.py @@ -18,7 +18,8 @@ """and as assert async await break class continue def del elif else except finally for from global if import in is lambda nonlocal not or pass raise return try while with yield bool bytes bytearray dict float int list str - datetime""".split() + datetime serialize serialize_members deserialize __init__ __post_init__ + _smithy_default __repr__ __annotations__""".split() ) # reserved-error-member-names.txt _RESERVED_ERROR_MEMBERS = frozenset( @@ -54,14 +55,24 @@ def member_field_name( return "message" name = to_snake_case(wire_name) - if name in _RESERVED_MEMBERS: + if name.startswith("__") and not name.endswith("__"): + name += "___" + elif name in _RESERVED_MEMBERS: name += "_" if is_error and name in _RESERVED_ERROR_MEMBERS: name += "_" if name == _RESPONSE_METADATA_MEMBER and carries_response_metadata: name += "_" if is_enum: - name = name.upper() + name = _escape_enum_underscores(name).upper() + return name + + +def _escape_enum_underscores(name: str) -> str: + if name.startswith("__"): + return name + "_" if name.endswith("__") and not name.endswith("___") else name + if name.startswith("_") and name.endswith("_") and not name.endswith("__"): + return name + "_" return name diff --git a/packages/smithy-core/tests/unit/test_dynamic_models.py b/packages/smithy-core/tests/unit/test_dynamic_models.py index 60d4363dc..6f342ca03 100644 --- a/packages/smithy-core/tests/unit/test_dynamic_models.py +++ b/packages/smithy-core/tests/unit/test_dynamic_models.py @@ -3,19 +3,40 @@ # pyright: reportPrivateUsage=false from enum import IntEnum, StrEnum from types import ModuleType -from typing import Any, cast +from typing import Any, Protocol, cast +import pytest from smithy_core import _schema_compact from smithy_core._dynamic_models import ModelRegistry from smithy_core._schema_compact import StaticSchema from smithy_core.deserializers import ShapeDeserializer -from smithy_core.exceptions import ModeledError +from smithy_core.exceptions import ModeledError, SerializationError +from smithy_core.schemas import Schema +from smithy_core.serializers import ShapeSerializer from smithy_core.shapes import ShapeType from smithy_core.types import UnknownEnumMixin NS = "ns.example" +class _UnknownVariant(Protocol): + def serialize(self, serializer: ShapeSerializer) -> None: ... + + def serialize_members(self, serializer: ShapeSerializer) -> None: ... + + +class _UnknownVariantClass(Protocol): + def __call__(self, *, tag: str) -> _UnknownVariant: ... + + def deserialize(self, deserializer: ShapeDeserializer) -> _UnknownVariant: ... + + +class _ModeledErrorClass(Protocol): + def __call__(self) -> ModeledError: ... + + def deserialize(self, deserializer: ShapeDeserializer) -> ModeledError: ... + + def _struct(name: str, members: tuple[tuple[str, object], ...]) -> StaticSchema: return ( ShapeType.STRUCTURE.value, @@ -30,12 +51,17 @@ def _struct(name: str, members: tuple[tuple[str, object], ...]) -> StaticSchema: _ERROR_TRAIT = "smithy.api#error" -def _error_struct(name: str, members: tuple[tuple[str, object], ...]) -> StaticSchema: +def _error_struct( + name: str, + members: tuple[tuple[str, object], ...], + *, + fault: str = "client", +) -> StaticSchema: return ( ShapeType.STRUCTURE.value, NS, name, - {_ERROR_TRAIT: "client"}, + {_ERROR_TRAIT: fault}, tuple(m[0] for m in members), tuple((m[1], 0) for m in members), ) @@ -128,6 +154,249 @@ def test_supplied_index_matches_scanning_fallback() -> None: assert _fields(scanned) == _fields(supplied) +def test_supplied_index_preserves_distinct_same_named_structures() -> None: + first = ( + ShapeType.STRUCTURE.value, + "a.example", + "Thing", + 0, + ("first",), + ((ShapeType.STRING.value, 0),), + ) + second = ( + ShapeType.STRUCTURE.value, + "b.example", + "Thing", + 0, + ("second",), + ((ShapeType.STRING.value, 0),), + ) + registry = ModelRegistry( + _module(FIRST=first, SECOND=second), + index={"FirstThing": "FIRST", "SecondThing": "SECOND"}, + ) + + first_cls = registry.build("FirstThing") + second_cls = registry.build("SecondThing") + assert first_cls is not second_cls + assert first_cls.__name__ == "FirstThing" + assert second_cls.__name__ == "SecondThing" + assert _fields(first_cls) == ["first"] + assert _fields(second_cls) == ["second"] + + +def test_registry_copies_supplied_name_maps() -> None: + renames = {"Alias": "FOO"} + index = {"Alias": "FOO"} + registry = ModelRegistry( + _module(FOO=_struct("Foo", ())), + renames=renames, + index=index, + ) + renames.clear() + index.clear() + + assert registry.build("Alias").__name__ == "Alias" + + +def test_alias_and_canonical_struct_lookups_share_class_identity() -> None: + registry = ModelRegistry( + _module(FOO=_struct("Foo", ())), + renames={"Alias": "FOO"}, + ) + + alias = registry.build("Alias") + assert registry.build("Foo") is alias + assert registry.build("Alias") is alias + + +def test_alias_and_canonical_union_lookups_share_class_identity() -> None: + registry = ModelRegistry( + _module(FOO=_union("Foo", (("value", ShapeType.STRING.value),))), + renames={"Alias": "FOO"}, + ) + + alias = registry.build("Alias") + variant = registry.build("AliasValue") + unknown = registry.build("AliasUnknown") + assert registry.build("Foo") is alias + assert registry.build("Alias") is alias + assert registry.build("AliasValue") is variant + assert registry.build("FooValue") is variant + assert registry.build("FooUnknown") is unknown + + class _De: + def read_struct(self, schema: Schema, consumer: Any) -> None: + consumer(schema.members_by_index[0], self) + + def read_string(self, schema: Schema) -> str: + return "value" + + alias_deserializer = registry.build("_AliasDeserializer")() + original_deserializer = registry.build("_FooDeserializer")() + alias_value = alias_deserializer.deserialize(cast(ShapeDeserializer, _De())) + original_value = original_deserializer.deserialize(cast(ShapeDeserializer, _De())) + assert type(alias_value) is variant + assert type(original_value) is variant + assert alias_value.value == original_value.value == "value" + + +def test_renamed_union_does_not_replace_same_named_shape() -> None: + renamed = ( + ShapeType.UNION.value, + "a.example", + "Thing", + 0, + ("value",), + ((ShapeType.STRING.value, 0),), + ) + other = ( + ShapeType.STRUCTURE.value, + "b.example", + "Thing", + 0, + ("other",), + ((ShapeType.STRING.value, 0),), + ) + registry = ModelRegistry( + _module(A_OTHER=other, Z_RENAMED=renamed), + renames={"FirstThing": "Z_RENAMED"}, + ) + + first = registry.build("FirstThing") + thing = registry.build("Thing") + assert first is not thing + assert _fields(thing) == ["other"] + + +def test_supplied_union_names_override_original_shape_names() -> None: + union = ( + ShapeType.UNION.value, + "a.example", + "Foo", + 0, + ("item",), + ((ShapeType.STRING.value, 0),), + ) + item = ( + ShapeType.STRUCTURE.value, + "b.example", + "OldItem", + 0, + ("item",), + ((ShapeType.STRING.value, 0),), + ) + unknown = ( + ShapeType.STRUCTURE.value, + "b.example", + "OldUnknown", + 0, + ("unknown",), + ((ShapeType.STRING.value, 0),), + ) + registry = ModelRegistry( + _module(FOO=union, ITEM=item, UNKNOWN=unknown), + renames={ + "Choice": "FOO", + "ChoiceItem": "ITEM", + "ChoiceUnknown": "UNKNOWN", + }, + index={ + "Foo": "FOO", + "Choice_Item": "FOO", + "Choice_Unknown": "FOO", + "OldItem": "ITEM", + "OldUnknown": "UNKNOWN", + }, + shape_names={"Foo", "OldItem", "OldUnknown"}, + ) + + item_cls = registry.build("ChoiceItem") + unknown_cls = registry.build("ChoiceUnknown") + registry.build("Choice") + assert registry.build("Choice_Item") is not item_cls + assert registry.build("Choice_Unknown") is not unknown_cls + assert registry.build("ChoiceItem") is item_cls + assert registry.build("ChoiceUnknown") is unknown_cls + + +def test_renamed_union_aliases_do_not_replace_renamed_models() -> None: + registry = ModelRegistry( + _module( + FOO=_union("Foo", (("value", ShapeType.STRING.value),)), + VALUE=_struct("ValueModel", (("value", ShapeType.STRING.value),)), + UNKNOWN=_struct("UnknownModel", (("value", ShapeType.STRING.value),)), + ), + renames={ + "Alias": "FOO", + "FooValue": "VALUE", + "FooUnknown": "UNKNOWN", + }, + ) + + registry.build("Alias") + assert _fields(registry.build("FooValue")) == ["value"] + assert _fields(registry.build("FooUnknown")) == ["value"] + assert registry.build("FooValue") is not registry.build("AliasValue") + assert registry.build("FooUnknown") is not registry.build("AliasUnknown") + + +def test_renamed_union_original_alias_does_not_replace_renamed_model() -> None: + registry = ModelRegistry( + _module( + FOO=_union("Foo", (("value", ShapeType.STRING.value),)), + MODEL=_struct("Model", (("value", ShapeType.STRING.value),)), + ), + renames={"Alias": "FOO", "Foo": "MODEL"}, + ) + + registry.build("Alias") + assert _fields(registry.build("Foo")) == ["value"] + assert registry.build("Foo") is not registry.build("Alias") + + +def test_target_names_resolve_renames_by_full_shape_id() -> None: + _schema_compact._cache.clear() + first = ( + ShapeType.STRUCTURE.value, + "a.example", + "Thing", + 0, + ) + second = ( + ShapeType.STRUCTURE.value, + "b.example", + "Thing", + 0, + ) + container = _struct("Container", (("value", lambda: second),)) + registry = ModelRegistry( + _module(FIRST=first, SECOND=second, CONTAINER=container), + renames={"FirstThing": "FIRST", "SecondThing": "SECOND"}, + index={ + "FirstThing": "FIRST", + "SecondThing": "SECOND", + "Container": "CONTAINER", + }, + target_names={ + "a.example#Thing": "FirstThing", + "b.example#Thing": "SecondThing", + f"{NS}#Container": "Container", + }, + shape_names={"FirstThing", "SecondThing", "Container"}, + ) + schema = _schema_compact.hydrate(container) + member = schema.members_by_index[0] + _, reader = member.get_extension(registry._codec) + + class _De: + def read_struct(self, schema: object, consumer: object) -> None: + pass + + result = reader(cast(ShapeDeserializer, _De()), member) + assert type(result).__name__ == "SecondThing" + + def test_supplied_index_disambiguated_union_variant() -> None: # Union FooWarning.message collides with struct FooWarningMessage; codegen emits # the "_"-disambiguated variant name in the index. The real struct must still @@ -220,6 +489,20 @@ def test_modeled_error_falls_back_to_modeled_error_without_service_error() -> No assert cls.__mro__[1] is ModeledError +def test_modeled_error_uses_modeled_fault() -> None: + _schema_compact._cache.clear() + mod = _module(BOOM=_error_struct("Boom", (), fault="server")) + registry = ModelRegistry(mod, index={"Boom": "BOOM"}, shape_names={"Boom"}) + cls = cast(_ModeledErrorClass, registry.build("Boom")) + + class _De: + def read_struct(self, schema: Schema, consumer: Any) -> None: + pass + + assert cls().fault == "server" + assert cls.deserialize(cast(ShapeDeserializer, _De())).fault == "server" + + def test_enum_member_deserialize_coerces_to_enum_class() -> None: # A member whose target is an enum must deserialize to the enum class, not the # raw wire string (the eager generator wrapped it as EnumClass(read_string(...))). @@ -265,6 +548,22 @@ def test_union_classes_report_models_module() -> None: assert registry.build(name).__module__ == "ns.example.models" +def test_unknown_union_variant_rejects_serde() -> None: + _schema_compact._cache.clear() + mod = _module(FOO=_union("Foo", (("a", ShapeType.STRING.value),))) + registry = ModelRegistry(mod, index={"Foo": "FOO"}, shape_names={"Foo"}) + registry.build("Foo") + unknown = cast(_UnknownVariantClass, registry.build("FooUnknown")) + value = unknown(tag="future") + + with pytest.raises(SerializationError): + value.serialize(cast(ShapeSerializer, object())) + with pytest.raises(SerializationError): + value.serialize_members(cast(ShapeSerializer, object())) + with pytest.raises(NotImplementedError): + unknown.deserialize(cast(ShapeDeserializer, object())) + + def test_concurrent_build_returns_one_class() -> None: import threading diff --git a/packages/smithy-core/tests/unit/test_model_naming.py b/packages/smithy-core/tests/unit/test_model_naming.py index d47d7e495..d0198c905 100644 --- a/packages/smithy-core/tests/unit/test_model_naming.py +++ b/packages/smithy-core/tests/unit/test_model_naming.py @@ -1,6 +1,89 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from smithy_core._model_naming import union_unknown_name, variant_class_name +from smithy_core._model_naming import ( + member_field_name, + to_snake_case, + union_unknown_name, + variant_class_name, +) + + +def test_snake_case_preserves_prefix_for_leading_digit() -> None: + assert to_snake_case("_1") == "_1" + assert to_snake_case("_2HTTPServer") == "_2_http_server" + + +def test_snake_case_preserves_existing_underscores() -> None: + assert to_snake_case("_value") == "_value" + assert to_snake_case("value_") == "value_" + assert to_snake_case("some__value") == "some__value" + + +def test_enum_names_escape_before_uppercasing() -> None: + assert ( + member_field_name( + "CLASS", + is_error=False, + carries_response_metadata=False, + is_enum=True, + ) + == "CLASS_" + ) + + +def test_enum_sunder_names_are_escaped() -> None: + assert ( + member_field_name( + "_VALUE_", + is_error=False, + carries_response_metadata=False, + is_enum=True, + ) + == "_VALUE__" + ) + + +def test_enum_dunder_names_are_escaped() -> None: + assert ( + member_field_name( + "__VALUE__", + is_error=False, + carries_response_metadata=False, + is_enum=True, + ) + == "__VALUE___" + ) + + +def test_runtime_method_names_are_reserved() -> None: + for name in ( + "serialize", + "serializeMembers", + "deserialize", + "__init__", + "__post_init__", + "_smithy_default", + "__repr__", + "__annotations__", + ): + assert member_field_name( + name, + is_error=False, + carries_response_metadata=False, + is_enum=False, + ).endswith("_") + + +def test_double_leading_underscore_names_avoid_name_mangling() -> None: + assert ( + member_field_name( + "__value", + is_error=False, + carries_response_metadata=False, + is_enum=False, + ) + == "__value___" + ) def test_variant_class_name_default() -> None: diff --git a/packages/smithy-python/.changes/next-release/smithy-python-feature-model-generation.json b/packages/smithy-python/.changes/next-release/smithy-python-feature-model-generation.json new file mode 100644 index 000000000..d5c28cdf5 --- /dev/null +++ b/packages/smithy-python/.changes/next-release/smithy-python-feature-model-generation.json @@ -0,0 +1,4 @@ +{ + "type": "feature", + "description": "Added native generation of lazy runtime model modules and complete static model stubs for standalone types packages." +} diff --git a/packages/smithy-python/src/smithy_python/files.py b/packages/smithy-python/src/smithy_python/files.py index 736f71821..79fa17069 100644 --- a/packages/smithy-python/src/smithy_python/files.py +++ b/packages/smithy-python/src/smithy_python/files.py @@ -53,7 +53,11 @@ def python_module(self, path: str, module: str) -> _writer.PythonWriter: raise _exceptions.CodegenError( f"Generated path {path!r} is already a text file" ) - writer = _writer.PythonWriter(module, dependencies=self._dependencies) + writer = _writer.PythonWriter( + module, + dependencies=self._dependencies, + stub=path.endswith(".pyi"), + ) self._files[path] = _PythonFile(module, writer) return writer diff --git a/packages/smithy-python/src/smithy_python/generation.py b/packages/smithy-python/src/smithy_python/generation.py index 3bec23e70..9d2b4ece7 100644 --- a/packages/smithy-python/src/smithy_python/generation.py +++ b/packages/smithy-python/src/smithy_python/generation.py @@ -25,6 +25,9 @@ "TypesSettings", ] +_SERVICE_ERROR = "ServiceError" +_UNIT = _model.ShapeId("smithy.api", "Unit") + @dataclass(frozen=True, slots=True) class ArtifactSettings: @@ -47,10 +50,15 @@ class TypesSettings(ArtifactSettings): class ModelPlan: """Derived semantic data used to emit lazy model declarations.""" + shapes: tuple[_model.Shape, ...] declaration_names: Mapping[_model.ShapeId, str] renames: Mapping[str, str] union_variant_names: Mapping[_model.ShapeId, str] union_unknown_names: Mapping[_model.ShapeId, str] + registry_index: Mapping[str, str] + target_names: Mapping[str, str] + shape_names: frozenset[str] + includes_unit: bool @classmethod def from_shapes( @@ -60,31 +68,55 @@ def from_shapes( schema_names: Mapping[_model.ShapeId, str], ) -> ModelPlan: """Plan declaration and union names in stable shape order.""" + model_shapes: list[_model.Shape] = [] declarations: dict[_model.ShapeId, str] = {} variants: dict[_model.ShapeId, str] = {} unknowns: dict[_model.ShapeId, str] = {} + index: dict[str, str] = {} + renames: dict[str, str] = {} + target_names: dict[str, str] = {} + includes_unit = any( + shape.type not in (_model.ShapeType.ENUM, _model.ShapeType.INT_ENUM) + and any(member.target == _UNIT for member in shape.members.values()) + for shape in shapes + ) for shape in shapes: if shape.type not in _model.DECLARATION_TYPES or shape.has_trait( "streaming" ): continue - declarations[shape.id] = symbols.declaration_name(shape.id) + model_shapes.append(shape) + declaration = symbols.declaration_name(shape.id) + schema = schema_names[shape.id] + declarations[shape.id] = declaration + index[declaration] = schema + if declaration != shape.id.name: + renames[declaration] = schema + target_names[str(shape.id)] = declaration if shape.type is _model.ShapeType.UNION: for member in shape.members.values(): - variants[member.id] = symbols.union_variant_name(member.id) - unknowns[shape.id] = symbols.union_unknown_name(shape.id) + variant = symbols.union_variant_name(member.id) + variants[member.id] = variant + index[variant] = schema + unknown = symbols.union_unknown_name(shape.id) + unknowns[shape.id] = unknown + index[unknown] = schema return cls( + shapes=tuple(model_shapes), declaration_names=MappingProxyType(declarations), - renames=MappingProxyType( - { - declaration: schema_names[shape_id] - for shape_id, declaration in declarations.items() - if declaration != shape_id.name - } - ), + renames=MappingProxyType(renames), union_variant_names=MappingProxyType(variants), union_unknown_names=MappingProxyType(unknowns), + registry_index=MappingProxyType(index), + target_names=MappingProxyType(target_names), + shape_names=frozenset( + ( + *declarations.values(), + *(("Unit",) if includes_unit else ()), + ) + ), + includes_unit=includes_unit, ) diff --git a/packages/smithy-python/src/smithy_python/models.py b/packages/smithy-python/src/smithy_python/models.py new file mode 100644 index 000000000..14ac548dc --- /dev/null +++ b/packages/smithy-python/src/smithy_python/models.py @@ -0,0 +1,457 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Emission of lazy runtime models and their static typing surface.""" + +from __future__ import annotations + +from collections.abc import Callable, Collection, Mapping +from dataclasses import replace +from json import dumps +from textwrap import wrap +from typing import TYPE_CHECKING, Final + +from . import dependencies as _dependencies +from . import exceptions as _exceptions +from . import model as _model +from . import symbols as _symbols + +if TYPE_CHECKING: + from .generation import GenerationContext, ModelPlan, TypesSettings + from .writer import PythonWriter + +__all__ = ["ModelGenerator"] + + +class ModelGenerator: + """Emit lazy runtime models and the complete static model stub.""" + + def generate(self, context: GenerationContext[TypesSettings]) -> None: + """Register ``models.py`` and ``models.pyi`` with the file manifest.""" + _validate_member_names(context) + package = context.settings.package + directory = package.replace(".", "/") + _write_runtime( + context.files.python_module( + f"src/{directory}/models.py", f"{package}.models" + ), + context.plan.models, + ) + _StubEmitter( + context.files.python_module( + f"src/{directory}/models.pyi", f"{package}.models" + ), + context, + ).write() + + +def _core_reference(name: str, module: str) -> _symbols.TypeReference: + return _symbols.TypeReference( + name, f"smithy_core.{module}", dependencies=(_dependencies.SMITHY_CORE,) + ) + + +_MODEL_REGISTRY: Final = _core_reference("ModelRegistry", "_dynamic_models") +_MODELED_ERROR: Final = _core_reference("ModeledError", "exceptions") +_SHAPE_SERIALIZER: Final = _core_reference("ShapeSerializer", "serializers") +_SHAPE_DESERIALIZER: Final = _core_reference("ShapeDeserializer", "deserializers") +_UNKNOWN_ENUM_MIXIN: Final = _core_reference("UnknownEnumMixin", "types") +_RESPONSE_METADATA: Final = _core_reference("ResponseMetadata", "response") +_DATACLASS: Final = _symbols.TypeReference("dataclass", "dataclasses") +_SELF: Final = _symbols.TypeReference("Self", "typing") +_STR_ENUM: Final = _symbols.TypeReference("StrEnum", "enum") +_INT_ENUM: Final = _symbols.TypeReference("IntEnum", "enum") +_SCHEMAS: Final = _symbols.TypeReference("schemas", "._private") + +_ERROR_MESSAGE_NAMES: Final = frozenset({"errormessage", "error_message", "message"}) +_DOCSTRING_WIDTH: Final = 72 +_MODEL_MEMBER_NAMES: Final = frozenset( + { + "bool", + "bytes", + "bytearray", + "dict", + "float", + "int", + "list", + "str", + "datetime", + "serialize", + "serialize_members", + "deserialize", + "__init__", + "__post_init__", + "_smithy_default", + "__repr__", + "__annotations__", + } +) +_ERROR_MEMBER_NAMES: Final = frozenset( + {"is_retry_safe", "retry_after", "is_throttling_error", "fault"} +) +_RESPONSE_METADATA_DOC: Final = ( + "Metadata about the response that produced this output.\n\n" + "Use this to recover the request identifiers a service's support team needs " + "in order to investigate a call. Members of the metadata are individually " + "optional." +) +_SERVICE_ERROR_DOC: Final = ( + "Base error for all errors in the service.\n\n" + "Some exceptions do not extend from this class, including synthetic, implicit, " + "and shared exception types." +) +_UNKNOWN_UNION_DOC: Final = ( + "Represents an unknown variant.\n\n" + "If you receive this value, you will need to update your library to receive " + "the parsed value.\n\n" + "This value may not be deliberately sent." +) + + +def _write_runtime(writer: PythonWriter, plan: ModelPlan) -> None: + exports = _model_exports(plan) + writer.line("class ServiceError(", _MODELED_ERROR, "):") + with writer.indent(): + _write_docstring(writer, _SERVICE_ERROR_DOC) + writer.line() + writer.line() + _write_mapping(writer, "_MODEL_INDEX", plan.registry_index) + writer.line() + _write_mapping(writer, "_RENAMES", plan.renames) + writer.line() + _write_mapping(writer, "_TARGET_NAMES", plan.target_names) + writer.line() + additional = exports[len(plan.registry_index) + 1 :] + if additional: + writer.line( + "_MODEL_NAMES = (*tuple(_MODEL_INDEX), ", + ", ".join(map(_quote, additional)), + ")", + ) + else: + writer.line("_MODEL_NAMES = tuple(_MODEL_INDEX)") + writer.line("__all__ = (", _quote(exports[0]), ", *_MODEL_NAMES)") + writer.line() + writer.line() + writer.line("_registry = ", _MODEL_REGISTRY, "(") + with writer.indent(): + writer.line(_SCHEMAS, ",") + writer.line("renames=_RENAMES,") + writer.line("index=_MODEL_INDEX,") + writer.line("target_names=_TARGET_NAMES,") + _write_string_set(writer, "shape_names", plan.shape_names, suffix=",") + writer.line('service_error="ServiceError",') + writer.line(")") + writer.line("del ", _SCHEMAS, ", ", _MODEL_REGISTRY, ", ", _MODELED_ERROR) + writer.line() + writer.line() + writer.line("def __getattr__(name: str):") + with writer.indent(): + writer.line("value = _registry.build(name)") + writer.line("globals()[name] = value") + writer.line("return value") + writer.line() + writer.line() + writer.line("def __dir__() -> list[str]:") + with writer.indent(): + writer.line("return sorted({*globals(), *__all__})") + + +class _StubEmitter: + def __init__( + self, + writer: PythonWriter, + context: GenerationContext[TypesSettings], + ) -> None: + self.writer = writer + self.model = context.model + self.plan = context.plan.models + self.symbols = context.symbols + self.module = f"{context.settings.package}.models" + operations = tuple(context.model.iter_shapes(_model.OperationShape)) + self.inputs = {operation.input for operation in operations} | { + shape.id for shape in self.plan.shapes if shape.has_trait("input") + } + self.outputs = {operation.output for operation in operations} | { + shape.id for shape in self.plan.shapes if shape.has_trait("output") + } + + def write(self) -> None: + writer = self.writer + writer.line("__all__ = [") + with writer.indent(): + for name in sorted(_model_exports(self.plan)): + writer.line(_quote(name), ",") + writer.line("]") + writer.line() + writer.line("class _SerdeShape:") + with writer.indent(): + writer.line( + "def serialize(self, serializer: ", + _SHAPE_SERIALIZER, + ") -> None: ...", + ) + writer.line( + "def serialize_members(self, serializer: ", + _SHAPE_SERIALIZER, + ") -> None: ...", + ) + writer.line("@classmethod") + writer.line( + "def deserialize(cls, deserializer: ", + _SHAPE_DESERIALIZER, + ") -> ", + _SELF, + ": ...", + ) + writer.line() + writer.line("class ServiceError(", _MODELED_ERROR, "):") + with writer.indent(): + _write_docstring(writer, _SERVICE_ERROR_DOC) + + for shape in self.plan.shapes: + writer.line() + match shape.type: + case _model.ShapeType.STRUCTURE: + self._write_structure(shape) + case _model.ShapeType.UNION: + self._write_union(shape) + case _model.ShapeType.ENUM | _model.ShapeType.INT_ENUM: + self._write_enum(shape) + case _: + pass + if self.plan.includes_unit: + writer.line() + writer.line("@", _DATACLASS, "(kw_only=True)") + writer.line("class Unit(_SerdeShape):") + with writer.indent(): + writer.line("pass") + + def _write_structure(self, shape: _model.Shape) -> None: + writer = self.writer + name = self.plan.declaration_names[shape.id] + error = shape.has_trait("error") + writer.line("@", _DATACLASS, "(kw_only=True)") + writer.line( + "class ", + name, + "(ServiceError, _SerdeShape):" if error else "(_SerdeShape):", + ) + with writer.indent(): + members = tuple( + member + for member in shape.members.values() + if not error or member.name.lower() not in _ERROR_MESSAGE_NAMES + ) + carries_metadata = error or shape.id in self.outputs + _write_docstring( + writer, + _documentation(shape) or f"Dataclass for {shape.id.name} structure.", + ) + if members or carries_metadata: + writer.line() + for member in members: + reference = self.symbols.type_reference(member.target) + has_default = member.has_trait("default") or self.model.get_target( + member + ).has_trait("default") + mandatory = ( + shape.id not in self.inputs + and member.has_trait("required") + and not has_default + ) + if not mandatory: + reference = replace(reference, nullable=True) + writer.line( + self.symbols.member_name(member.id), + ": ", + reference, + "" if mandatory else " = ...", + ) + if docs := _documentation(member): + _write_docstring(writer, docs) + if carries_metadata: + writer.line("response_metadata: ", _RESPONSE_METADATA, " = ...") + _write_docstring(writer, _RESPONSE_METADATA_DOC) + + def _write_union(self, shape: _model.Shape) -> None: + writer = self.writer + variants: list[_symbols.TypeReference] = [] + for member in shape.members.values(): + name = self.plan.union_variant_names[member.id] + variants.append(_symbols.TypeReference(name, self.module)) + writer.line("@", _DATACLASS) + writer.line("class ", name, "(_SerdeShape):") + with writer.indent(): + if docs := _documentation(member): + _write_docstring(writer, docs) + writer.line() + writer.line("value: ", self.symbols.type_reference(member.target)) + writer.line() + + unknown = self.plan.union_unknown_names[shape.id] + variants.append(_symbols.TypeReference(unknown, self.module)) + writer.line("@", _DATACLASS) + writer.line("class ", unknown, "(_SerdeShape):") + with writer.indent(): + _write_docstring(writer, _UNKNOWN_UNION_DOC) + writer.line() + writer.line("tag: str") + writer.line() + + name = self.plan.declaration_names[shape.id] + writer.line(name, " = ", *_join_types(variants)) + if docs := _documentation(shape): + _write_docstring(writer, docs) + writer.line() + writer.line("class _", name, "Deserializer:") + with writer.indent(): + writer.line( + "def deserialize(self, deserializer: ", + _SHAPE_DESERIALIZER, + ") -> ", + _symbols.TypeReference(name, self.module), + ": ...", + ) + + def _write_enum(self, shape: _model.Shape) -> None: + writer = self.writer + name = self.plan.declaration_names[shape.id] + base = _INT_ENUM if shape.type is _model.ShapeType.INT_ENUM else _STR_ENUM + writer.line("class ", name, "(", _UNKNOWN_ENUM_MIXIN, ", ", base, "):") + with writer.indent(): + docs = _documentation(shape) + if docs: + _write_docstring(writer, docs) + if shape.members: + writer.line() + for member in shape.members.values(): + value = member.get_trait("enumValue") + writer.line( + self.symbols.member_name(member.id), + " = ", + _literal(member.name if value is None else value), + ) + if docs := _documentation(member): + _write_docstring(writer, docs) + if not docs and not shape.members: + writer.line("pass") + + +def _model_exports(plan: ModelPlan) -> tuple[str, ...]: + exports = ("ServiceError", *plan.registry_index) + return (*exports, "Unit") if plan.includes_unit else exports + + +def _write_mapping(writer: PythonWriter, name: str, values: Mapping[str, str]) -> None: + if not values: + writer.line(name, " = {}") + return + writer.line(name, " = {") + with writer.indent(): + for key, value in values.items(): + writer.line(_quote(key), ": ", _quote(value), ",") + writer.line("}") + + +def _write_string_set( + writer: PythonWriter, + name: str, + values: Collection[str], + *, + suffix: str = "", +) -> None: + if not values: + writer.line(name, "=frozenset()", suffix) + return + writer.line(name, "=frozenset(") + with writer.indent(): + writer.line("{") + with writer.indent(): + for value in sorted(values): + writer.line(_quote(value), ",") + writer.line("}") + writer.line(")", suffix) + + +def _write_docstring(writer: PythonWriter, value: str) -> None: + lines = [ + wrapped + for line in value.split("\n") + for wrapped in ( + wrap( + line, + width=_DOCSTRING_WIDTH, + break_long_words=False, + break_on_hyphens=False, + ) + or [""] + ) + ] + if len(lines) > 1: + writer.line('"""') + for line in lines: + writer.line(line.replace("\\", "\\\\").replace('"""', '\\"""')) + writer.line('"""') + return + writer.line(_quote(lines[0])) + + +def _literal(value: object) -> str: + return _quote(value) if isinstance(value, str) else repr(value) + + +def _quote(value: str) -> str: + return dumps(value, ensure_ascii=False) + + +def _join_types( + types: Collection[_symbols.TypeReference], +) -> tuple[str | _symbols.TypeReference, ...]: + parts: list[str | _symbols.TypeReference] = [] + for reference in types: + if parts: + parts.append(" | ") + parts.append(reference) + return tuple(parts) + + +def _documentation(shape: _model.Shape) -> str | None: + value = shape.get_trait("documentation") + return value if isinstance(value, str) else None + + +def _validate_member_names(context: GenerationContext[TypesSettings]) -> None: + reserved = _model_reserved_member_names(context.model) + for shape in context.plan.models.shapes: + names: set[str] = set() + shape_reserved = reserved(shape) + for member in shape.members.values(): + name = context.symbols.member_name(member.id) + if name in shape_reserved: + raise _exceptions.CodegenError( + f"Symbol provider did not reserve model member name {name!r} " + f"for {member.id}" + ) + if name in names: + raise _exceptions.CodegenError( + f"Generated model member name collision {name!r} in {shape.id}" + ) + names.add(name) + + +def _model_reserved_member_names( + model: _model.Model, +) -> Callable[[_model.Shape], Collection[str]]: + outputs = { + operation.output for operation in model.iter_shapes(_model.OperationShape) + } | {shape.id for shape in model.iter_shapes() if shape.has_trait("output")} + + def reserved(shape: _model.Shape) -> Collection[str]: + names = set(_MODEL_MEMBER_NAMES) + if shape.has_trait("error"): + names.update(_ERROR_MEMBER_NAMES) + if shape.has_trait("error") or shape.id in outputs: + names.add("response_metadata") + return names + + return reserved diff --git a/packages/smithy-python/src/smithy_python/symbols.py b/packages/smithy-python/src/smithy_python/symbols.py index 1fe1d417e..3dfa2be3b 100644 --- a/packages/smithy-python/src/smithy_python/symbols.py +++ b/packages/smithy-python/src/smithy_python/symbols.py @@ -69,6 +69,13 @@ def __init__( self._union_variant_names: dict[ShapeId, str] = {} self._union_unknown_names: dict[ShapeId, str] = {} self._selected = frozenset(shape.id for shape in selection.shapes) + self._reserved_declarations = _RESERVED_DECLARATIONS + if any( + shape.type not in _ENUMS + and any(member.target == _UNIT for member in shape.members.values()) + for shape in selection.shapes + ): + self._reserved_declarations |= {"Unit"} self._shape_names: frozenset[str] = frozenset() declarations: dict[str, ShapeId] = {} for shape in selection.shapes: @@ -84,7 +91,7 @@ def __init__( members: dict[str, ShapeId] = {} for member in shape.members.values(): self._check_collision(members, self.member_name(member.id), member.id) - self._shape_names = frozenset(declarations) + self._shape_names = frozenset((*declarations, *self._reserved_declarations)) for shape in selection.shapes: if shape.type is not ShapeType.UNION or shape.has_trait("streaming"): continue @@ -106,7 +113,7 @@ def declaration_name(self, shape_id: ShapeId | str) -> str: raise ModelError(f"No generated declaration for {shape.id}") service = self._selection.service name = service.rename.get(shape.id, shape.id.name) if service else shape.id.name - result = _name(name, pascal=True) + result = _name(name, pascal=True, reserved=self._reserved_declarations) if not re.fullmatch(r"[A-Za-z0-9_]+", name) or not result.isidentifier(): raise ModelError( f"Invalid Python declaration name {result!r} for {shape.id}" @@ -211,7 +218,7 @@ def type_reference(self, shape_id: ShapeId | str) -> TypeReference: pending.append((current, True)) pending.extend((target, False) for target in reversed(targets)) elif current == ShapeId("smithy.api", "Unit"): - resolved[current] = TypeReference("None") + resolved[current] = TypeReference("Unit", f"{self._package}.models") elif shape.type in (ShapeType.STRUCTURE, ShapeType.UNION): resolved[current] = TypeReference( self.declaration_name(current), f"{self._package}.models" @@ -237,6 +244,7 @@ def member_name(self, shape_id: ShapeId | str) -> str: return _name( member.name, constant=container.type in _ENUMS, + preserve_underscores=True, reserved=self._reserved_member_names.get(container.id, ()), ) @@ -279,6 +287,8 @@ def _shape(self, shape_id: ShapeId | str) -> Shape: "not or pass raise return try while with yield".split() ) _ENUMS = frozenset({ShapeType.ENUM, ShapeType.INT_ENUM}) +_RESERVED_DECLARATIONS = frozenset({"ServiceError"}) +_UNIT = ShapeId("smithy.api", "Unit") _ACRONYM_BOUNDARY = re.compile(r"([A-Z]+)([A-Z][a-z])") _WORD_BOUNDARY = re.compile(r"([a-z0-9])([A-Z])") @@ -321,6 +331,7 @@ def _name( *, pascal: bool = False, constant: bool = False, + preserve_underscores: bool = False, reserved: Collection[str] = (), ) -> str: if pascal: @@ -328,16 +339,34 @@ def _name( else: value = _ACRONYM_BOUNDARY.sub(r"\1_\2", value) value = _WORD_BOUNDARY.sub(r"\1_\2", value) - result = "_".join(word.lower() for word in value.split("_") if word) - if constant: - result = result.upper() + result = ( + value.lower() + if preserve_underscores + else "_".join(word.lower() for word in value.split("_") if word) + ) if result[:1].isdigit(): result = "_" + result - if result in _KEYWORDS or result in reserved: + if result.startswith("__") and not result.endswith("__"): + result += "___" + elif ( + result in _KEYWORDS + or result in reserved + or (constant and result.upper() in reserved) + ): result += "_" + if constant: + result = _escape_enum_underscores(result).upper() return result +def _escape_enum_underscores(name: str) -> str: + if name.startswith("__"): + return name + "_" if name.endswith("__") and not name.endswith("___") else name + if name.startswith("_") and name.endswith("_") and not name.endswith("__"): + return name + "_" + return name + + def _plan_schema_names(selection: Selection) -> dict[ShapeId, str]: names: dict[ShapeId, str] = {} used: set[str] = set() diff --git a/packages/smithy-python/src/smithy_python/writer.py b/packages/smithy-python/src/smithy_python/writer.py index d91ffdf2e..efcf45e4b 100644 --- a/packages/smithy-python/src/smithy_python/writer.py +++ b/packages/smithy-python/src/smithy_python/writer.py @@ -58,11 +58,16 @@ class PythonWriter: """Write trusted Python lines, retaining type references until rendering.""" def __init__( - self, module: str, *, dependencies: _dependencies.DependencySet | None = None + self, + module: str, + *, + dependencies: _dependencies.DependencySet | None = None, + stub: bool = False, ) -> None: _validate_module(module, relative=False) self._module = _binding(module) self._dependencies = dependencies + self._stub = stub self._depth = 0 self._lines: list[tuple[int, tuple[str | TypeReference, ...]]] = [] @@ -91,9 +96,10 @@ def render(self) -> str: names, imports = self._plan_imports(*self._names()) header = "from __future__ import annotations\n" if imports: - header += "\n" + "\n".join(item.statement() for item in imports) + "\n" + header += "\n" + _render_imports(imports) + "\n" body = self._body(lambda reference: self._annotation(reference, names)) - return header + ("\n\n" + "\n".join(body) + "\n" if body else "") + separator = "\n" if self._stub else "\n\n" + return header + (separator + "\n".join(body) + "\n" if body else "") def _body(self, reference: Callable[[TypeReference], str]) -> list[str]: body: list[str] = [] @@ -139,6 +145,7 @@ def _plan_imports( self, module_names: frozenset[str], shadows: list[frozenset[str]] ) -> tuple[dict[tuple[str | None, str], str], list[_Import]]: identities: set[tuple[str | None, str]] = set() + third_party_roots: set[str] = set() # The local names visible where each identity is referenced. References # in one scope share a set, so keep the distinct sets, not their union. local_names: dict[tuple[str | None, str], set[frozenset[str]]] = {} @@ -153,6 +160,8 @@ def _plan_imports( if ref.module is not None: _validate_identifier(ref.name, "reference name") _validate_module(ref.module, relative=True) + if ref.dependencies and not ref.module.startswith("."): + third_party_roots.add(_module_root(ref.module)) identities.add((ref.module, ref.name)) key = _identity(ref.module, ref.name) local_names.setdefault(key, set()).add(shadow) @@ -195,7 +204,17 @@ def _plan_imports( else name ) names[(module, name)] = alias - imports.append((_Import(module, name, alias), scoped)) + imports.append( + ( + _Import( + module, + name, + alias, + third_party=_module_root(module) in third_party_roots, + ), + scoped, + ) + ) qualified: set[frozenset[str]] = set() for key, (module, name) in representatives.items(): module_binding, name_binding = key @@ -211,7 +230,9 @@ def _plan_imports( elif module_binding == self._module: names[(module, name)] = name if qualified: - imports.append((_Import("builtins", None, "_builtins"), qualified)) + imports.append( + (_Import("builtins", None, "_builtins", third_party=False), qualified) + ) # Check the complete plan, including unaliased imports. No binding gets # priority merely because its reference was encountered first. @@ -273,12 +294,7 @@ class _Import: module: str name: str | None alias: str - - def statement(self) -> str: - if self.name is None: - return f"import {self.module} as {self.alias}" - suffix = f" as {self.alias}" if self.alias != self.name else "" - return f"from {self.module} import {self.name}{suffix}" + third_party: bool @dataclass(eq=False, slots=True) @@ -319,11 +335,50 @@ def _binding(name: str) -> str: return normalize("NFKC", name) +def _render_imports(imports: list[_Import]) -> str: + sections: dict[int, dict[str, list[_Import]]] = {} + for item in imports: + section = 2 if item.module.startswith(".") else 1 if item.third_party else 0 + sections.setdefault(section, {}).setdefault(item.module, []).append(item) + + rendered: list[str] = [] + for section in sorted(sections): + lines: list[str] = [] + for module, items in sections[section].items(): + for item in items: + if item.name is None: + lines.append(f"import {module} as {item.alias}") + named = [item for item in items if item.name is not None] + if named: + lines.extend(_render_from_import(module, named)) + rendered.append("\n".join(lines)) + return "\n\n".join(rendered) + + +def _render_from_import(module: str, imports: list[_Import]) -> list[str]: + names: list[str] = [] + for item in imports: + if item.name is None: + continue + names.append( + item.name if item.alias == item.name else f"{item.name} as {item.alias}" + ) + prefix = f"from {module} import " + joined = ", ".join(names) + if len(prefix) + len(joined) <= 88: + return [prefix + joined] + return [prefix + "(", *(f" {name}," for name in names), ")"] + + def _identity(module: str | None, name: str) -> tuple[str | None, str]: """Return a reference's module and name as Python binds them.""" return (None if module is None else _binding(module), _binding(name)) +def _module_root(module: str) -> str: + return _binding(module.lstrip(".").split(".", 1)[0]) + + def _validate_identifier(value: str, kind: str) -> None: """Require a Python identifier that is not a hard keyword.""" if not value.isidentifier() or keyword.iskeyword(_binding(value)): diff --git a/packages/smithy-python/tests/unit/test_models.py b/packages/smithy-python/tests/unit/test_models.py new file mode 100644 index 000000000..cd9f752ce --- /dev/null +++ b/packages/smithy-python/tests/unit/test_models.py @@ -0,0 +1,841 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# pyright: reportPrivateUsage=false + +from __future__ import annotations + +import importlib +import json +import pickle +import subprocess +import sys +from collections.abc import Mapping +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Any + +import pytest +from smithy_core.types import UnknownEnumMixin +from smithy_json import JSONCodec +from smithy_python.dependencies import SMITHY_CORE, DependencySet +from smithy_python.files import FileManifest +from smithy_python.generation import GenerationContext, GenerationPlan, TypesSettings +from smithy_python.model import load_model +from smithy_python.models import ModelGenerator, _model_reserved_member_names +from smithy_python.schemas import SchemaGenerator +from smithy_python.selection import select_type_shapes +from smithy_python.symbols import SymbolProvider + +_PACKAGE = "generated_types" + + +def _run_tool(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "-m", *args], + capture_output=True, + text=True, + check=False, + ) + + +def _context( + model_data: Mapping[str, object], + *, + service_id: str | None = "example#Service", + package: str = _PACKAGE, +) -> GenerationContext[TypesSettings]: + model = load_model(model_data) + selection = select_type_shapes( + model, + service_id=service_id, + generate_inputs_and_outputs=True, + ) + settings = TypesSettings(package, "1.0.0", None) + symbols = SymbolProvider( + model, + selection, + package=settings.package, + reserved_member_names=_model_reserved_member_names(model), + ) + dependencies = DependencySet() + return GenerationContext( + model=model, + selection=selection, + settings=settings, + symbols=symbols, + plan=GenerationPlan.from_selection(model, selection, symbols), + dependencies=dependencies, + files=FileManifest(dependencies), + ) + + +def _model() -> dict[str, object]: + return { + "smithy": "2.0", + "shapes": { + "example#Service": { + "type": "service", + "version": "1", + "operations": [{"target": "example#Operation"}], + "rename": { + "example#Original": "Renamed", + "example#Choice": "Selection", + "example#Color": "Tone", + }, + }, + "example#Operation": { + "type": "operation", + "input": {"target": "example#Input"}, + "output": {"target": "example#Output"}, + "errors": [{"target": "example#Problem"}], + }, + "example#Input": { + "type": "structure", + "traits": {"smithy.api#documentation": "Input docs."}, + "members": { + "required": { + "target": "smithy.api#String", + "traits": {"smithy.api#required": {}}, + }, + "list": {"target": "smithy.api#String"}, + "renamed": {"target": "example#Original"}, + "choice": {"target": "example#Choice"}, + "color": {"target": "example#Color"}, + }, + }, + "example#Output": { + "type": "structure", + "members": { + "responseMetadata": {"target": "smithy.api#String"}, + "code": {"target": "example#Code"}, + }, + }, + "example#Problem": { + "type": "structure", + "traits": { + "smithy.api#error": "client", + "smithy.api#documentation": "Problem docs.", + }, + "members": { + "message": { + "target": "smithy.api#String", + "traits": {"smithy.api#sensitive": {}}, + }, + "fault": {"target": "smithy.api#String"}, + "detail": {"target": "example#Original"}, + }, + }, + "example#Original": { + "type": "structure", + "traits": {"smithy.api#documentation": "Renamed structure."}, + "members": { + "value": { + "target": "smithy.api#String", + "traits": { + "smithy.api#required": {}, + "smithy.api#documentation": "Value docs.", + }, + }, + "count": { + "target": "smithy.api#PrimitiveInteger", + "traits": {"smithy.api#required": {}}, + }, + }, + }, + "example#Choice": { + "type": "union", + "traits": {"smithy.api#documentation": "Choice docs."}, + "members": { + "text": { + "target": "smithy.api#String", + "traits": {"smithy.api#documentation": "Text docs."}, + }, + "item": {"target": "example#Original"}, + }, + }, + "example#Color": { + "type": "enum", + "traits": {"smithy.api#documentation": "Color docs."}, + "members": { + "RED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "red", + "smithy.api#documentation": "Red docs.", + }, + } + }, + }, + "example#Code": { + "type": "intEnum", + "members": { + "OK": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": 200}, + } + }, + }, + }, + } + + +def _generate(tmp_path: Path) -> tuple[GenerationContext[TypesSettings], Path]: + context = _context(_model()) + SchemaGenerator().generate(context) + ModelGenerator().generate(context) + context.files.text_file(f"src/{_PACKAGE}/__init__.py", "") + context.files.text_file(f"src/{_PACKAGE}/_private/__init__.py", "") + context.files.commit(tmp_path) + return context, tmp_path / "src" + + +def _generate_custom( + tmp_path: Path, + model: Mapping[str, object], + package: str, +) -> Path: + context = _context(model, service_id=None, package=package) + SchemaGenerator().generate(context) + ModelGenerator().generate(context) + context.files.text_file(f"src/{package}/__init__.py", "") + context.files.text_file(f"src/{package}/_private/__init__.py", "") + context.files.commit(tmp_path) + return tmp_path / "src" + + +def test_model_plan_precomputes_registry_metadata() -> None: + plan = _context(_model()).plan.models + + assert plan.renames == { + "Renamed": "RENAMED", + "Selection": "SELECTION", + "Tone": "TONE", + } + assert "Original" not in plan.registry_index + assert plan.registry_index["SelectionText"] == "SELECTION" + assert plan.registry_index["SelectionUnknown"] == "SELECTION" + assert plan.target_names["example#Original"] == "Renamed" + assert plan.target_names["example#Choice"] == "Selection" + assert plan.target_names["example#Color"] == "Tone" + assert tuple(plan.registry_index) == ( + "Input", + "Output", + "Problem", + "Renamed", + "Selection", + "SelectionText", + "SelectionItem", + "SelectionUnknown", + "Tone", + "Code", + ) + assert plan.shape_names == { + "Input", + "Output", + "Problem", + "Renamed", + "Selection", + "Tone", + "Code", + } + + +def test_model_generator_emits_runtime_and_stub(tmp_path: Path) -> None: + context, source_root = _generate(tmp_path) + + runtime = (source_root / _PACKAGE / "models.py").read_text() + stub = (source_root / _PACKAGE / "models.pyi").read_text() + assert "from smithy_core._dynamic_models import ModelRegistry" in runtime + service_error_doc = ( + "class ServiceError(ModeledError):\n" + ' """\n' + " Base error for all errors in the service.\n" + "\n" + " Some exceptions do not extend from this class, including synthetic,\n" + " implicit, and shared exception types.\n" + ' """' + ) + assert service_error_doc in runtime + assert '"Renamed": "RENAMED"' in runtime + assert '"Original": "RENAMED"' not in runtime + assert "_MODEL_NAMES = tuple(_MODEL_INDEX)" in runtime + assert '__all__ = ("ServiceError", *_MODEL_NAMES)' in runtime + assert "globals()[name] = value" in runtime + assert "def __dir__() -> list[str]:" in runtime + + assert "class Input(_SerdeShape):" in stub + assert service_error_doc in stub + assert '"Input docs."' in stub + assert "required: str | None = ..." in stub + assert "list_: str | None = ..." in stub + assert "class Problem(ServiceError, _SerdeShape):" in stub + assert "message:" not in stub + assert "fault_: str | None = ..." in stub + assert "class Renamed(_SerdeShape):" in stub + assert '"Value docs."' in stub + assert "count: int | None = ..." in stub + assert "class SelectionText(_SerdeShape):" in stub + assert "Selection = SelectionText | SelectionItem | SelectionUnknown" in stub + assert "class Tone(UnknownEnumMixin, StrEnum):" in stub + assert 'RED = "red"' in stub + assert "class Code(UnknownEnumMixin, IntEnum):" in stub + assert "OK = 200" in stub + assert "response_metadata_: str | None = ..." in stub + assert "response_metadata: ResponseMetadata = ..." in stub + assert SMITHY_CORE in context.dependencies + + +def test_generated_models_are_lazy_and_runtime_compatible( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _, source_root = _generate(tmp_path) + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{_PACKAGE}.models") + + assert "Renamed" not in vars(models) + assert "Renamed" in dir(models) + renamed = models.Renamed + assert vars(models)["Renamed"] is renamed + assert models.Renamed is renamed + assert renamed.__module__ == f"{_PACKAGE}.models" + value = renamed(value="hello") + assert value.value == "hello" + assert value.count == 0 + assert pickle.loads(pickle.dumps(value)) == value + + def get_input(_: int) -> type[Any]: + return models.Input + + with ThreadPoolExecutor(max_workers=8) as executor: + inputs = tuple(executor.map(get_input, range(32))) + assert len({id(model) for model in inputs}) == 1 + input_ = inputs[0]( + required="yes", + list_="value", + renamed=value, + choice=models.SelectionText(value="selected"), + color=models.Tone.RED, + ) + assert input_.list_ == "value" + assert input_.renamed is value + decoded = JSONCodec().deserialize( + ( + b'{"required":"yes","renamed":{"value":"hello"},' + b'"choice":{"text":"selected"},"color":"red"}' + ), + inputs[0], + ) + assert type(decoded.renamed) is renamed + assert decoded.renamed.value == "hello" + assert type(decoded.choice) is models.SelectionText + assert decoded.choice.value == "selected" + assert decoded.color is models.Tone.RED + assert json.loads(JSONCodec().serialize(input_)) == { + "required": "yes", + "list": "value", + "renamed": {"value": "hello", "count": 0}, + "choice": {"text": "selected"}, + "color": "red", + } + + assert issubclass(models.Problem, models.ServiceError) + problem = models.Problem("bad", fault_="client", detail=value) + assert problem.message == "bad" + assert problem.fault_ == "client" + assert "bad" not in repr(problem) + assert models.Problem().message == "" + + assert models.Tone.RED == "red" + assert issubclass(models.Tone, UnknownEnumMixin) + assert models.Code.OK == 200 + assert models.SelectionText(value="text").value == "text" + assert models.SelectionItem(value=value).value is value + assert models.SelectionUnknown(tag="future").tag == "future" + + namespace: dict[str, object] = {} + exec(f"from {_PACKAGE}.models import *", namespace) + assert tuple(name for name in models.__all__ if name not in namespace) == () + assert "Original" not in models.__all__ + with pytest.raises(AttributeError): + models.Original + + +def test_generated_model_names_do_not_shadow_runtime_helpers( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + package = "generated_collisions" + source_root = _generate_custom( + tmp_path, + { + "smithy": "2.0", + "shapes": { + f"example#{name}": {"type": "structure"} + for name in ("ServiceError", "ModelRegistry", "ModeledError") + }, + }, + package, + ) + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{package}.models") + + assert models.ServiceError_.__name__ == "ServiceError_" + assert models.ModelRegistry.__name__ == "ModelRegistry" + assert models.ModeledError.__name__ == "ModeledError" + assert models.ServiceError is not models.ServiceError_ + assert {"ServiceError_", "ModelRegistry", "ModeledError"} <= set(models.__all__) + + +def test_explicit_input_and_output_traits_match_runtime_and_stub( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + package = "generated_io_traits" + source_root = _generate_custom( + tmp_path, + { + "smithy": "2.0", + "shapes": { + "example#Input": { + "type": "structure", + "traits": {"smithy.api#input": {}}, + "members": { + "required": { + "target": "smithy.api#String", + "traits": {"smithy.api#required": {}}, + } + }, + }, + "example#Output": { + "type": "structure", + "traits": {"smithy.api#output": {}}, + "members": {"responseMetadata": {"target": "smithy.api#String"}}, + }, + }, + }, + package, + ) + stub = (source_root / package / "models.pyi").read_text() + assert "required: str | None = ..." in stub + assert "response_metadata_: str | None = ..." in stub + assert "response_metadata: ResponseMetadata = ..." in stub + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{package}.models") + assert models.Input().required is None + output = models.Output(response_metadata_="modeled") + assert output.response_metadata_ == "modeled" + assert output.response_metadata is not None + + +def test_stub_and_runtime_use_the_same_member_names( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + package = "generated_member_names" + source_root = _generate_custom( + tmp_path, + { + "smithy": "2.0", + "shapes": { + "example#Leading": { + "type": "structure", + "members": {"_value": {"target": "smithy.api#String"}}, + }, + "example#Trailing": { + "type": "structure", + "members": {"value_": {"target": "smithy.api#String"}}, + }, + "example#Repeated": { + "type": "structure", + "members": {"some__value": {"target": "smithy.api#String"}}, + }, + "example#Digits": { + "type": "structure", + "members": { + "_1": {"target": "smithy.api#String"}, + "_2HTTPServer": {"target": "smithy.api#String"}, + }, + }, + "example#RuntimeMethods": { + "type": "structure", + "members": { + "serialize": {"target": "smithy.api#String"}, + "serializeMembers": {"target": "smithy.api#String"}, + "deserialize": {"target": "smithy.api#String"}, + "__init__": {"target": "smithy.api#String"}, + "__post_init__": {"target": "smithy.api#String"}, + "_smithy_default": {"target": "smithy.api#String"}, + "__repr__": {"target": "smithy.api#String"}, + }, + }, + "example#Keywords": { + "type": "enum", + "members": { + name: { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": name.lower()}, + } + for name in ("LIST", "CLASS") + }, + }, + "example#SunderString": { + "type": "enum", + "members": { + "_VALUE_": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "value"}, + }, + "__DUNDER__": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "dunder"}, + }, + }, + }, + "example#SunderInt": { + "type": "intEnum", + "members": { + "_VALUE_": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": 1}, + }, + "__DUNDER__": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": 2}, + }, + }, + }, + "example#EnumContainer": { + "type": "structure", + "members": { + "stringValue": {"target": "example#SunderString"}, + "intValue": {"target": "example#SunderInt"}, + }, + }, + }, + }, + package, + ) + stub = (source_root / package / "models.pyi").read_text() + assert "_value: str | None = ..." in stub + assert "value_: str | None = ..." in stub + assert "some__value: str | None = ..." in stub + assert "_1: str | None = ..." in stub + assert "_2_http_server: str | None = ..." in stub + assert "serialize_: str | None = ..." in stub + assert "serialize_members_: str | None = ..." in stub + assert "deserialize_: str | None = ..." in stub + assert "__init___: str | None = ..." in stub + assert "__post_init___: str | None = ..." in stub + assert "_smithy_default_: str | None = ..." in stub + assert "__repr___: str | None = ..." in stub + assert 'LIST_ = "list"' in stub + assert 'CLASS_ = "class"' in stub + assert ( + 'class SunderString(UnknownEnumMixin, StrEnum):\n _VALUE__ = "value"' in stub + ) + assert "class SunderInt(UnknownEnumMixin, IntEnum):\n _VALUE__ = 1" in stub + assert '__DUNDER___ = "dunder"' in stub + assert "__DUNDER___ = 2" in stub + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{package}.models") + assert models.Leading(_value="leading")._value == "leading" + assert models.Trailing(value_="trailing").value_ == "trailing" + assert models.Repeated(some__value="repeated").some__value == "repeated" + digits = models.Digits(_1="one", _2_http_server="two") + assert digits._1 == "one" + assert digits._2_http_server == "two" + methods = models.RuntimeMethods( + serialize_="serialize", + serialize_members_="members", + deserialize_="deserialize", + __init___="init", + __post_init___="post_init", + _smithy_default_="default", + __repr___="repr", + ) + assert callable(methods.serialize) + assert callable(methods.serialize_members) + assert callable(methods.deserialize) + assert methods.__init___ == "init" + assert methods.__post_init___ == "post_init" + assert methods._smithy_default_ == "default" + assert methods.__repr___ == "repr" + assert isinstance(methods._smithy_default(), models.RuntimeMethods) + assert "RuntimeMethods(" in repr(methods) + assert json.loads(JSONCodec().serialize(methods)) == { + "serialize": "serialize", + "serializeMembers": "members", + "deserialize": "deserialize", + "__init__": "init", + "__post_init__": "post_init", + "_smithy_default": "default", + "__repr__": "repr", + } + assert models.Keywords.LIST_ == "list" + assert models.Keywords.CLASS_ == "class" + assert models.SunderString._VALUE__ == "value" + assert models.SunderInt._VALUE__ == 1 + assert "__DUNDER___" in models.SunderString.__members__ + assert "__DUNDER___" in models.SunderInt.__members__ + value = models.EnumContainer( + string_value=models.SunderString.__DUNDER___, + int_value=models.SunderInt.__DUNDER___, + ) + encoded = JSONCodec().serialize(value) + decoded = JSONCodec().deserialize(encoded, models.EnumContainer) + assert decoded.string_value is models.SunderString.__DUNDER___ + assert decoded.int_value is models.SunderInt.__DUNDER___ + + +def test_unit_union_members_round_trip_as_empty_structures( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + package = "generated_unit_union" + source_root = _generate_custom( + tmp_path, + { + "smithy": "2.0", + "shapes": { + "example#Action": { + "type": "union", + "members": {"quit": {"target": "smithy.api#Unit"}}, + }, + "example#Container": { + "type": "structure", + "members": {"action": {"target": "example#Action"}}, + }, + }, + }, + package, + ) + runtime = (source_root / package / "models.py").read_text() + stub = (source_root / package / "models.pyi").read_text() + assert '_MODEL_NAMES = (*tuple(_MODEL_INDEX), "Unit")' in runtime + assert "class ActionQuit(_SerdeShape):\n value: Unit" in stub + assert "class Unit(_SerdeShape):" in stub + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{package}.models") + value = models.Container(action=models.ActionQuit(value=models.Unit())) + assert json.loads(JSONCodec().serialize(value)) == {"action": {"quit": {}}} + decoded = JSONCodec().deserialize(b'{"action":{"quit":{}}}', models.Container) + assert isinstance(decoded.action, models.ActionQuit) + assert isinstance(decoded.action.value, models.Unit) + assert "Unit" in models.__all__ + + +def test_modeled_unit_does_not_shadow_prelude_unit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + package = "generated_unit_collision" + source_root = _generate_custom( + tmp_path, + { + "smithy": "2.0", + "shapes": { + "example#Unit": { + "type": "structure", + "members": {"value": {"target": "smithy.api#String"}}, + }, + "example#Action": { + "type": "union", + "members": {"quit": {"target": "smithy.api#Unit"}}, + }, + "example#Container": { + "type": "structure", + "members": {"action": {"target": "example#Action"}}, + }, + }, + }, + package, + ) + stub = (source_root / package / "models.pyi").read_text() + assert "class Unit_(_SerdeShape):" in stub + assert "class Unit(_SerdeShape):" in stub + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{package}.models") + assert models.Unit_ is not models.Unit + decoded = JSONCodec().deserialize( + b'{"action":{"quit":{}}}', + models.Container, + ) + assert isinstance(decoded.action.value, models.Unit) + + +def test_generated_stub_passes_strict_pyright( + tmp_path: Path, +) -> None: + _, source_root = _generate(tmp_path) + (tmp_path / "check.py").write_text( + ( + f"from {_PACKAGE}.models import Input, Problem, Renamed, SelectionText, Tone\n\n" + "renamed = Renamed(value='value')\n" + "Input(required='required', renamed=renamed, " + "choice=SelectionText(value='x'), color=Tone.RED)\n" + "problem = Problem('bad')\n" + "message: str = problem.message\n" + ), + encoding="utf-8", + ) + (tmp_path / "pyrightconfig.json").write_text( + json.dumps( + { + "include": ["check.py"], + "extraPaths": [str(source_root)], + "typeCheckingMode": "strict", + } + ), + encoding="utf-8", + ) + result = _run_tool( + "pyright", + "--project", + str(tmp_path / "pyrightconfig.json"), + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_generated_models_pass_ruff(tmp_path: Path) -> None: + _, source_root = _generate(tmp_path) + models = source_root / _PACKAGE / "models.py" + stub = source_root / _PACKAGE / "models.pyi" + for command in (("check",), ("format", "--check")): + result = _run_tool( + "ruff", + *command, + str(models), + str(stub), + "--config", + str(Path(__file__).resolve().parents[4] / "pyproject.toml"), + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_stub_star_exports_and_private_member_names( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + package = "generated_private_names" + source_root = _generate_custom( + tmp_path, + { + "smithy": "2.0", + "shapes": { + "example#_2HTTPServer": { + "type": "structure", + "members": { + "__value": {"target": "smithy.api#String"}, + "__annotations__": {"target": "smithy.api#String"}, + }, + } + }, + }, + package, + ) + stub = (source_root / package / "models.pyi").read_text() + assert '"_2HTTPServer",' in stub + assert "__value___: str | None = ..." in stub + assert "__annotations___: str | None = ..." in stub + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{package}.models") + namespace: dict[str, object] = {} + exec(f"from {package}.models import *", namespace) + assert namespace["_2HTTPServer"] is models._2HTTPServer + value = models._2HTTPServer( + __value___="value", + __annotations___="annotations", + ) + assert value.__value___ == "value" + assert value.__annotations___ == "annotations" + + (tmp_path / "check.py").write_text( + ( + f"from {package}.models import *\n\n" + "value = _2HTTPServer(\n" + " __value___='value',\n" + " __annotations___='annotations',\n" + ")\n" + "member: str | None = value.__value___\n" + ), + encoding="utf-8", + ) + (tmp_path / "pyrightconfig.json").write_text( + json.dumps( + { + "include": ["check.py"], + "extraPaths": [str(source_root)], + "typeCheckingMode": "strict", + } + ), + encoding="utf-8", + ) + result = _run_tool( + "pyright", + "--project", + str(tmp_path / "pyrightconfig.json"), + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_model_output_is_deterministic(tmp_path: Path) -> None: + outputs: list[tuple[str, str]] = [] + for index in range(2): + _, source_root = _generate(tmp_path / str(index)) + package = source_root / _PACKAGE + outputs.append( + ( + (package / "models.py").read_text(), + (package / "models.pyi").read_text(), + ) + ) + assert outputs[0] == outputs[1] + + +def test_model_generation_without_service(tmp_path: Path) -> None: + context = _context( + { + "smithy": "2.0", + "shapes": { + "example#Standalone": { + "type": "structure", + "members": {"value": {"target": "smithy.api#String"}}, + } + }, + }, + service_id=None, + ) + SchemaGenerator().generate(context) + ModelGenerator().generate(context) + context.files.commit(tmp_path) + + runtime = (tmp_path / "src" / _PACKAGE / "models.py").read_text() + stub = (tmp_path / "src" / _PACKAGE / "models.pyi").read_text() + assert '"Standalone": "STANDALONE"' in runtime + assert "class Standalone(_SerdeShape):" in stub + + +def teardown_module() -> None: + for name in tuple(sys.modules): + if name == _PACKAGE or name.startswith(f"{_PACKAGE}."): + del sys.modules[name] diff --git a/packages/smithy-python/tests/unit/test_symbols.py b/packages/smithy-python/tests/unit/test_symbols.py index 4e0c77866..a1013999f 100644 --- a/packages/smithy-python/tests/unit/test_symbols.py +++ b/packages/smithy-python/tests/unit/test_symbols.py @@ -25,10 +25,13 @@ def model_with(shapes: dict[str, object]) -> Model: ("http_server", "HttpServer", "http_server", "HTTP_SERVER"), ("HTTP2Server", "HTTP2Server", "http2_server", "HTTP2_SERVER"), ("getURL2Value", "GetURL2Value", "get_url2_value", "GET_URL2_VALUE"), - ("__some__name__", "SomeName", "some_name", "SOME_NAME"), - ("__init__", "Init", "init", "INIT"), - ("_value", "Value", "value", "VALUE"), - ("class", "Class", "class_", "CLASS"), + ("__some__name__", "SomeName", "__some__name__", "__SOME__NAME___"), + ("__init__", "Init", "__init__", "__INIT___"), + ("__post_init__", "PostInit", "__post_init__", "__POST_INIT___"), + ("__value", "Value", "__value___", "__VALUE___"), + ("_value", "Value", "_value", "_VALUE"), + ("_VALUE_", "VALUE", "_value_", "_VALUE__"), + ("class", "Class", "class_", "CLASS_"), ("None", "None_", "none", "NONE"), ("match", "Match", "match", "MATCH"), ], @@ -59,6 +62,21 @@ def test_names(source: str, declaration: str, field: str, constant: str) -> None assert model.get_shape(f"example#Choice${source}").get_trait("enumValue") == "wire" +def test_service_error_is_reserved_for_the_synthetic_error_base() -> None: + model = model_with( + { + "example#ServiceError": {"type": "structure"}, + "example#Service": { + "type": "union", + "members": {"error": {"target": "smithy.api#String"}}, + }, + } + ) + symbols = SymbolProvider(model, select_shapes(model), package="example.client") + assert symbols.declaration_name("example#ServiceError") == "ServiceError_" + assert symbols.union_variant_name("example#Service$error") == "Service_Error" + + @pytest.mark.parametrize( ("kind", "name", "module"), [ @@ -99,7 +117,9 @@ def test_type_references(kind: str, name: str, module: str) -> None: if module != "example.client.models" and kind not in ("enum", "intEnum"): prelude_name = kind[0].upper() + kind[1:] assert symbols.type_reference(f"smithy.api#{prelude_name}") == ref - assert symbols.type_reference("smithy.api#Unit") == TypeReference("None") + assert symbols.type_reference("smithy.api#Unit") == TypeReference( + "Unit", "example.client.models" + ) def test_streaming_union_has_a_schema_name_without_a_model_symbol() -> None: @@ -403,7 +423,8 @@ def test_collection_only_cycles(mutual: bool) -> None: @pytest.mark.parametrize("kind", ["structure", "union", "enum", "intEnum"]) @pytest.mark.parametrize( - "names", [("getURL", "get_url"), ("class", "class_"), ("__init__", "init")] + "names", + [("getURL", "get_url"), ("class", "class_"), ("HTTPServer", "http_server")], ) def test_member_collisions(kind: str, names: tuple[str, str]) -> None: from smithy_python.model import ModelError @@ -420,7 +441,7 @@ def test_member_collisions(kind: str, names: tuple[str, str]) -> None: SymbolProvider(model, select_shapes(model), package="pkg") for name in names: assert f"example#Container${name}" in str(error.value) - result = "CLASS" if kind in ("enum", "intEnum") else "class_" + result = "CLASS_" if kind in ("enum", "intEnum") else "class_" if names[0] == "class": assert repr(result) in str(error.value) @@ -786,6 +807,54 @@ def test_reserved_member_names_still_collide() -> None: assert "'serialize_'" in str(error.value) +def test_enum_sunder_escaping_still_detects_collisions() -> None: + from smithy_python.model import ModelError + + model = model_with( + { + "example#Values": { + "type": "enum", + "members": { + "_VALUE_": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "first"}, + }, + "_VALUE__": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "second"}, + }, + }, + } + } + ) + with pytest.raises(ModelError, match="_VALUE__"): + SymbolProvider(model, select_shapes(model), package="pkg") + + +def test_enum_dunder_escaping_still_detects_collisions() -> None: + from smithy_python.model import ModelError + + model = model_with( + { + "example#Values": { + "type": "enum", + "members": { + "__VALUE__": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "first"}, + }, + "__VALUE___": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "second"}, + }, + }, + } + } + ) + with pytest.raises(ModelError, match="__VALUE___"): + SymbolProvider(model, select_shapes(model), package="pkg") + + def test_type_references_are_cached() -> None: from smithy_python.model import ModelError diff --git a/packages/smithy-python/tests/unit/test_writer.py b/packages/smithy-python/tests/unit/test_writer.py index 78852e0cc..c7df26470 100644 --- a/packages/smithy-python/tests/unit/test_writer.py +++ b/packages/smithy-python/tests/unit/test_writer.py @@ -13,7 +13,7 @@ from typing import get_type_hints import pytest -from smithy_python.dependencies import Dependency, DependencySet +from smithy_python.dependencies import SMITHY_CORE, Dependency, DependencySet from smithy_python.exceptions import CodegenError from smithy_python.symbols import TypeReference from smithy_python.writer import PythonWriter @@ -220,7 +220,7 @@ def test_class_body_references_load( ")", ) source = writer.render() - assert "from dataclasses import field as _dataclasses_field" in source + assert "field as _dataclasses_field" in source assert "default_factory=_builtins.list" in source module = load_written_module(writer, tmp_path, monkeypatch) node: type = module.Node @@ -600,7 +600,7 @@ def test_builtin_names_cover_running_interpreter() -> None: writer.line("value: ", TypeReference(name, "external")) source = writer.render() for name in names: - assert f"from external import {name} as _external_{name}\n" in source + assert f" {name} as _external_{name},\n" in source @pytest.mark.parametrize("module", ["written_models", "\uff57ritten_models"]) @@ -789,3 +789,28 @@ def test_import_identity_and_order() -> None: assert prefix.index("from datetime") < prefix.index("from decimal") prefixes.append(prefix) assert prefixes[0] == prefixes[1] + + +def test_imports_are_grouped_and_combined() -> None: + writer = PythonWriter("package.models") + writer.line("first: ", TypeReference("StrEnum", "enum")) + writer.line("second: ", TypeReference("IntEnum", "enum")) + writer.line( + "third: ", + TypeReference( + "Document", + "smithy_core.documents", + dependencies=(SMITHY_CORE,), + ), + ) + writer.line("fourth: ", TypeReference("Thing", "._private")) + + assert writer.render().startswith( + "from __future__ import annotations\n" + "\n" + "from enum import IntEnum, StrEnum\n" + "\n" + "from smithy_core.documents import Document\n" + "\n" + "from ._private import Thing\n" + ) diff --git a/pyproject.toml b/pyproject.toml index 8c2cbe851..cb3a1372e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ lint = [ ] typing = [ "pyright>=1.1.400", + "typing-extensions>=4.14.0", ] changelog = [ "jinja2>=3.1.6" diff --git a/scripts/api-snapshot/api.json b/scripts/api-snapshot/api.json index 5382d5c2c..b84b7ef58 100644 --- a/scripts/api-snapshot/api.json +++ b/scripts/api-snapshot/api.json @@ -1662,6 +1662,9 @@ "load_model": "rex-function", "to_json": "rex-function" }, + "smithy_python.models": { + "ModelGenerator": "class" + }, "smithy_python.schemas": { "SchemaGenerator": "class", "SchemaPlan": "class" diff --git a/scripts/codegen-output-py/generate.py b/scripts/codegen-output-py/generate.py index 54ea35f82..4306ffda4 100644 --- a/scripts/codegen-output-py/generate.py +++ b/scripts/codegen-output-py/generate.py @@ -1,5 +1,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +# pyright: reportPrivateUsage=false """Regenerate the committed Python codegen snapshot.""" import hashlib @@ -12,6 +13,7 @@ from smithy_python.files import FileManifest from smithy_python.generation import GenerationContext, GenerationPlan, TypesSettings from smithy_python.model import load_model +from smithy_python.models import ModelGenerator, _model_reserved_member_names from smithy_python.schemas import SchemaGenerator from smithy_python.selection import select_type_shapes from smithy_python.symbols import SymbolProvider @@ -35,7 +37,12 @@ def main() -> None: package_version="1.0.0", package_description="Example Python codegen output", ) - symbols = SymbolProvider(model, selection, package=settings.package) + symbols = SymbolProvider( + model, + selection, + package=settings.package, + reserved_member_names=_model_reserved_member_names(model), + ) dependencies = DependencySet() context = GenerationContext( model=model, @@ -47,6 +54,7 @@ def main() -> None: files=FileManifest(dependencies), ) SchemaGenerator().generate(context) + ModelGenerator().generate(context) context.files.commit(_OUTPUT) _format_output() @@ -57,7 +65,7 @@ def _format_output() -> None: args = [sys.executable, "-m", "ruff", command] if command == "check": args.append("--fix") - subprocess.run( + subprocess.run( # noqa: S603 [*args, str(_OUTPUT), "--config", str(config)], check=True, ) diff --git a/uv.lock b/uv.lock index f07a64d38..4cbf599d5 100644 --- a/uv.lock +++ b/uv.lock @@ -792,6 +792,7 @@ test = [ ] typing = [ { name = "pyright" }, + { name = "typing-extensions" }, ] [package.metadata] @@ -805,7 +806,10 @@ test = [ { name = "pytest-asyncio", specifier = ">=0.25.3" }, { name = "pytest-cov", specifier = ">=6.0.0" }, ] -typing = [{ name = "pyright", specifier = ">=1.1.400" }] +typing = [ + { name = "pyright", specifier = ">=1.1.400" }, + { name = "typing-extensions", specifier = ">=4.14.0" }, +] [[package]] name = "smithy-test" @@ -823,11 +827,11 @@ requires-dist = [{ name = "smithy-core", editable = "packages/smithy-core" }] [[package]] name = "typing-extensions" -version = "4.13.2" +version = "4.16.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f6/37/23083fcd6e35492953e8d2aaaa68b860eb422b34627b13f2ce3eb6106061/typing_extensions-4.13.2.tar.gz", hash = "sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef", size = 106967, upload-time = "2025-04-10T14:19:05.416Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/54/b1ae86c0973cc6f0210b53d508ca3641fb6d0c56823f288d108bc7ab3cc8/typing_extensions-4.13.2-py3-none-any.whl", hash = "sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c", size = 45806, upload-time = "2025-04-10T14:19:03.967Z" }, + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, ] [[package]] From ec1c6fdf62a76e3e43643a693ac9e7ebc38ad320 Mon Sep 17 00:00:00 2001 From: Sean O'Brien <60306702+stobrien89@users.noreply.github.com> Date: Sun, 25 Oct 2026 15:01:33 -0400 Subject: [PATCH 2/6] test streaming blob model generation --- .../smithy-python/tests/unit/test_models.py | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/packages/smithy-python/tests/unit/test_models.py b/packages/smithy-python/tests/unit/test_models.py index cd9f752ce..1192f205d 100644 --- a/packages/smithy-python/tests/unit/test_models.py +++ b/packages/smithy-python/tests/unit/test_models.py @@ -101,6 +101,7 @@ def _model() -> dict[str, object]: "renamed": {"target": "example#Original"}, "choice": {"target": "example#Choice"}, "color": {"target": "example#Color"}, + "stream": {"target": "example#Stream"}, }, }, "example#Output": { @@ -175,6 +176,10 @@ def _model() -> dict[str, object]: } }, }, + "example#Stream": { + "type": "blob", + "traits": {"smithy.api#streaming": {}}, + }, }, } @@ -264,10 +269,12 @@ def test_model_generator_emits_runtime_and_stub(tmp_path: Path) -> None: assert "def __dir__() -> list[str]:" in runtime assert "class Input(_SerdeShape):" in stub + assert "from smithy_core.aio.interfaces import StreamingBlob" in stub assert service_error_doc in stub assert '"Input docs."' in stub assert "required: str | None = ..." in stub assert "list_: str | None = ..." in stub + assert "stream: StreamingBlob | None = ..." in stub assert "class Problem(ServiceError, _SerdeShape):" in stub assert "message:" not in stub assert "fault_: str | None = ..." in stub @@ -321,6 +328,36 @@ def get_input(_: int) -> type[Any]: ) assert input_.list_ == "value" assert input_.renamed is value + + stream = b"stream" + stream_input = models.Input(stream=stream) + assert stream_input.stream is stream + + class StreamSerializer: + value: object | None = None + + def write_data_stream(self, schema: Any, value: object) -> None: + self.value = value + + serializer = StreamSerializer() + stream_input.serialize_members(serializer) + assert serializer.value is stream + + class StreamDeserializer: + def read_struct(self, schema: Any, consumer: Any) -> None: + member = next( + member + for member in schema.members_by_index + if member.expect_member_name() == "stream" + ) + consumer(member, self) + + def read_data_stream(self, schema: Any) -> bytes: + return stream + + decoded_stream = models.Input.deserialize(StreamDeserializer()) + assert decoded_stream.stream is stream + decoded = JSONCodec().deserialize( ( b'{"required":"yes","renamed":{"value":"hello"},' @@ -689,7 +726,7 @@ def test_generated_stub_passes_strict_pyright( f"from {_PACKAGE}.models import Input, Problem, Renamed, SelectionText, Tone\n\n" "renamed = Renamed(value='value')\n" "Input(required='required', renamed=renamed, " - "choice=SelectionText(value='x'), color=Tone.RED)\n" + "choice=SelectionText(value='x'), color=Tone.RED, stream=b'payload')\n" "problem = Problem('bad')\n" "message: str = problem.message\n" ), From 2cbfa44ee8041edfdb0437154fa6b1d7d010e251 Mon Sep 17 00:00:00 2001 From: Sean O'Brien <60306702+stobrien89@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:31:42 -0400 Subject: [PATCH 3/6] support event stream unions in model generation --- .../types/.smithy-python-manifest.json | 6 +-- .../src/example_types/_private/schemas.py | 40 +++++++++++++------ .../types/src/example_types/models.py | 4 ++ .../types/src/example_types/models.pyi | 26 ++++++++++++ designs/codegen/symbols.md | 5 ++- .../src/smithy_python/generation.py | 4 +- .../src/smithy_python/selection.py | 6 +-- .../src/smithy_python/symbols.py | 6 +-- .../tests/unit/test_data_selection.py | 10 ++--- .../tests/unit/test_generation.py | 10 ++++- .../smithy-python/tests/unit/test_models.py | 19 +++++++++ .../smithy-python/tests/unit/test_symbols.py | 25 +++++++----- scripts/codegen-output-py/model.json | 14 +++++++ 13 files changed, 125 insertions(+), 50 deletions(-) diff --git a/codegen-output-py/types/.smithy-python-manifest.json b/codegen-output-py/types/.smithy-python-manifest.json index f5426b5de..9ab9e0cb3 100644 --- a/codegen-output-py/types/.smithy-python-manifest.json +++ b/codegen-output-py/types/.smithy-python-manifest.json @@ -1,8 +1,8 @@ { "files": { - "src/example_types/_private/schemas.py": "d0f35e8979c4aa9d49a1d9c0c8efeb860420bc0ceab3ba3edcc9488c053e9110", - "src/example_types/models.py": "6042e2e6454b40fef929d96cb7608ae8c591ac7205577f14337435718bb7c077", - "src/example_types/models.pyi": "bb3d59e91999a62438fce52fe3be551c63f998565e4f9640e4fb1e275b972b14" + "src/example_types/_private/schemas.py": "1d708d275049f3f4a86b343c3f38104ef4a7e5bf749a27cd1a490c95c8f9db59", + "src/example_types/models.py": "51aadd47e9971f2b44fabd47e81e4b04d7156aa61b64454b39f4b6d47603ab45", + "src/example_types/models.pyi": "a841db3c47698f1cd5236a73ed4b9d4a40fcab3fbcef0ee7b4957e6a9df74952" }, "version": 1 } diff --git a/codegen-output-py/types/src/example_types/_private/schemas.py b/codegen-output-py/types/src/example_types/_private/schemas.py index 1192d7ce8..91264ec48 100644 --- a/codegen-output-py/types/src/example_types/_private/schemas.py +++ b/codegen-output-py/types/src/example_types/_private/schemas.py @@ -32,13 +32,16 @@ _s26 = "choice" _s27 = "status" _s28 = "code" -_s29 = "GetExampleOutput" -_s30 = "smithy.api#output" -_s31 = "ExampleError" -_s32 = "smithy.api#error" -_s33 = "client" -_s34 = "message" -_s35 = "retryAfterSeconds" +_s29 = "EventStream" +_s30 = "smithy.api#streaming" +_s31 = "GetExampleOutput" +_s32 = "smithy.api#output" +_s33 = "events" +_s34 = "ExampleError" +_s35 = "smithy.api#error" +_s36 = "client" +_s37 = "message" +_s38 = "retryAfterSeconds" GET_EXAMPLE_INPUT: StaticSchema = ( 18, @@ -145,29 +148,40 @@ ), ) -GET_EXAMPLE_OUTPUT: StaticSchema = ( - 18, +EVENT_STREAM: StaticSchema = ( + 19, n0, _s29, {_s30: None}, + (n0,), + ((RENAMED_EXAMPLE, 0),), +) + +GET_EXAMPLE_OUTPUT: StaticSchema = ( + 18, + n0, + _s31, + {_s32: None}, ( n0, _s7, + _s33, ), ( (RENAMED_EXAMPLE, 0), (EXAMPLE_VALUE, 0), + (EVENT_STREAM, 0), ), ) EXAMPLE_ERROR: StaticSchema = ( 18, n0, - _s31, - {_s32: _s33}, + _s34, + {_s35: _s36}, ( - _s34, - _s35, + _s37, + _s38, ), ( (3, 0), diff --git a/codegen-output-py/types/src/example_types/models.py b/codegen-output-py/types/src/example_types/models.py index aacb742b3..6803f9606 100644 --- a/codegen-output-py/types/src/example_types/models.py +++ b/codegen-output-py/types/src/example_types/models.py @@ -25,6 +25,9 @@ class ServiceError(ModeledError): "ExampleValueNumber": "EXAMPLE_VALUE", "ExampleValueEmpty": "EXAMPLE_VALUE", "ExampleValueUnknown": "EXAMPLE_VALUE", + "EventStream": "EVENT_STREAM", + "EventStreamExample": "EVENT_STREAM", + "EventStreamUnknown": "EVENT_STREAM", "Status": "STATUS", "Code": "CODE", } @@ -51,6 +54,7 @@ class ServiceError(ModeledError): shape_names=frozenset( { "Code", + "EventStream", "ExampleError", "ExampleValue", "GetExampleInput", diff --git a/codegen-output-py/types/src/example_types/models.pyi b/codegen-output-py/types/src/example_types/models.pyi index a4ba94f5e..616a2aaef 100644 --- a/codegen-output-py/types/src/example_types/models.pyi +++ b/codegen-output-py/types/src/example_types/models.pyi @@ -13,6 +13,9 @@ from smithy_core.types import UnknownEnumMixin __all__ = [ "Code", + "EventStream", + "EventStreamExample", + "EventStreamUnknown", "ExampleError", "ExampleValue", "ExampleValueEmpty", @@ -53,6 +56,7 @@ class GetExampleOutput(_SerdeShape): example: RenamedExample | None = ... value: ExampleValue | None = ... + events: EventStream | None = ... response_metadata: ResponseMetadata = ... """ Metadata about the response that produced this output. @@ -122,6 +126,28 @@ ExampleValue = ( class _ExampleValueDeserializer: def deserialize(self, deserializer: ShapeDeserializer) -> ExampleValue: ... +@dataclass +class EventStreamExample(_SerdeShape): + value: RenamedExample + +@dataclass +class EventStreamUnknown(_SerdeShape): + """ + Represents an unknown variant. + + If you receive this value, you will need to update your library to + receive the parsed value. + + This value may not be deliberately sent. + """ + + tag: str + +EventStream = EventStreamExample | EventStreamUnknown + +class _EventStreamDeserializer: + def deserialize(self, deserializer: ShapeDeserializer) -> EventStream: ... + class Status(UnknownEnumMixin, StrEnum): ACTIVE = "active" INACTIVE = "inactive" diff --git a/designs/codegen/symbols.md b/designs/codegen/symbols.md index 33daae01d..4c202e834 100644 --- a/designs/codegen/symbols.md +++ b/designs/codegen/symbols.md @@ -194,8 +194,9 @@ collections, such as two lists containing each other, raises `ModelError` with the IDs in the cycle. A failed lookup does not affect later lookups. Streaming blobs resolve to `smithy_core.aio.interfaces.StreamingBlob`, including -when nested in collections. Event-stream unions retain schema names for -protocol integration and do not produce model declarations. +when nested in collections. Event-stream unions produce the same model +declarations, variant classes, unknown class, and type references as other +unions. Their streaming behavior is handled by the client and protocol layers. ## Inputs and errors diff --git a/packages/smithy-python/src/smithy_python/generation.py b/packages/smithy-python/src/smithy_python/generation.py index 9d2b4ece7..ce75e1e19 100644 --- a/packages/smithy-python/src/smithy_python/generation.py +++ b/packages/smithy-python/src/smithy_python/generation.py @@ -81,9 +81,7 @@ def from_shapes( for shape in shapes ) for shape in shapes: - if shape.type not in _model.DECLARATION_TYPES or shape.has_trait( - "streaming" - ): + if shape.type not in _model.DECLARATION_TYPES: continue model_shapes.append(shape) declaration = symbols.declaration_name(shape.id) diff --git a/packages/smithy-python/src/smithy_python/selection.py b/packages/smithy-python/src/smithy_python/selection.py index 62779f1b0..6b554be33 100644 --- a/packages/smithy-python/src/smithy_python/selection.py +++ b/packages/smithy-python/src/smithy_python/selection.py @@ -197,11 +197,7 @@ def select_type_shapes( shapes = tuple(shape for shape in model.iter_shapes() if shape.id in closure) _check_names( - tuple( - shape - for shape in shapes - if shape.type in DECLARATION_TYPES and not shape.has_trait("streaming") - ), + tuple(shape for shape in shapes if shape.type in DECLARATION_TYPES), service=service, ) return Selection(service=service, shapes=shapes, excluded_count=excluded_count) diff --git a/packages/smithy-python/src/smithy_python/symbols.py b/packages/smithy-python/src/smithy_python/symbols.py index 3dfa2be3b..dbc7df1e4 100644 --- a/packages/smithy-python/src/smithy_python/symbols.py +++ b/packages/smithy-python/src/smithy_python/symbols.py @@ -79,7 +79,7 @@ def __init__( self._shape_names: frozenset[str] = frozenset() declarations: dict[str, ShapeId] = {} for shape in selection.shapes: - if shape.type not in DECLARATION_TYPES or shape.has_trait("streaming"): + if shape.type not in DECLARATION_TYPES: continue self._check_collision( declarations, self.declaration_name(shape.id), shape.id @@ -93,7 +93,7 @@ def __init__( self._check_collision(members, self.member_name(member.id), member.id) self._shape_names = frozenset((*declarations, *self._reserved_declarations)) for shape in selection.shapes: - if shape.type is not ShapeType.UNION or shape.has_trait("streaming"): + if shape.type is not ShapeType.UNION: continue for member in shape.members.values(): self._check_collision( @@ -273,8 +273,6 @@ def _shape(self, shape_id: ShapeId | str) -> Shape: in (ShapeType.SERVICE, ShapeType.OPERATION, ShapeType.RESOURCE) ): raise ModelError(f"No data symbol for {shape.id} ({shape.type})") - if shape.type is ShapeType.UNION and shape.has_trait("streaming"): - raise ModelError(f"Unsupported streaming {shape.type}: {shape.id}") return shape diff --git a/packages/smithy-python/tests/unit/test_data_selection.py b/packages/smithy-python/tests/unit/test_data_selection.py index 1439e9c8d..fe9c748fb 100644 --- a/packages/smithy-python/tests/unit/test_data_selection.py +++ b/packages/smithy-python/tests/unit/test_data_selection.py @@ -404,7 +404,7 @@ def test_type_selection_checks_only_selected_declaration_names() -> None: select_type_shapes(model) -def test_type_selection_excludes_streaming_unions_from_name_checks() -> None: +def test_type_selection_checks_streaming_union_names() -> None: model = _model( { "a#Events": { @@ -415,12 +415,8 @@ def test_type_selection_excludes_streaming_unions_from_name_checks() -> None: } ) - result = select_type_shapes(model) - - assert result.shapes == ( - model.get_shape("a#Events"), - model.get_shape("b#Events"), - ) + with pytest.raises(ModelError, match="a#Events, b#Events"): + select_type_shapes(model) def test_type_selection_uses_service_renames_for_name_conflicts() -> None: diff --git a/packages/smithy-python/tests/unit/test_generation.py b/packages/smithy-python/tests/unit/test_generation.py index 2b001d0ab..7be37ca29 100644 --- a/packages/smithy-python/tests/unit/test_generation.py +++ b/packages/smithy-python/tests/unit/test_generation.py @@ -116,7 +116,7 @@ def test_generation_plan_orders_schemas_and_plans_names() -> None: assert plan.schemas.strings["example"] == "n0" -def test_generation_plan_keeps_streaming_schemas_out_of_model_declarations() -> None: +def test_generation_plan_includes_streaming_union_models() -> None: model = load_model( { "smithy": "2.0", @@ -124,6 +124,7 @@ def test_generation_plan_keeps_streaming_schemas_out_of_model_declarations() -> "example#Events": { "type": "union", "traits": {"smithy.api#streaming": {}}, + "members": {"message": {"target": "smithy.api#String"}}, } }, } @@ -133,4 +134,9 @@ def test_generation_plan_keeps_streaming_schemas_out_of_model_declarations() -> plan = GenerationPlan.from_selection(model, selection, symbols) shape_id = model.get_shape("example#Events").id assert plan.schemas.names[shape_id] == "EVENTS" - assert shape_id not in plan.models.declaration_names + assert plan.models.declaration_names[shape_id] == "Events" + assert ( + plan.models.union_variant_names[model.get_shape("example#Events$message").id] + == "EventsMessage" + ) + assert plan.models.union_unknown_names[shape_id] == "EventsUnknown" diff --git a/packages/smithy-python/tests/unit/test_models.py b/packages/smithy-python/tests/unit/test_models.py index 1192f205d..4719dbc9d 100644 --- a/packages/smithy-python/tests/unit/test_models.py +++ b/packages/smithy-python/tests/unit/test_models.py @@ -102,6 +102,7 @@ def _model() -> dict[str, object]: "choice": {"target": "example#Choice"}, "color": {"target": "example#Color"}, "stream": {"target": "example#Stream"}, + "events": {"target": "example#Events"}, }, }, "example#Output": { @@ -180,6 +181,11 @@ def _model() -> dict[str, object]: "type": "blob", "traits": {"smithy.api#streaming": {}}, }, + "example#Events": { + "type": "union", + "traits": {"smithy.api#streaming": {}}, + "members": {"item": {"target": "example#Original"}}, + }, }, } @@ -233,6 +239,9 @@ def test_model_plan_precomputes_registry_metadata() -> None: "SelectionUnknown", "Tone", "Code", + "Events", + "EventsItem", + "EventsUnknown", ) assert plan.shape_names == { "Input", @@ -242,6 +251,7 @@ def test_model_plan_precomputes_registry_metadata() -> None: "Selection", "Tone", "Code", + "Events", } @@ -275,6 +285,10 @@ def test_model_generator_emits_runtime_and_stub(tmp_path: Path) -> None: assert "required: str | None = ..." in stub assert "list_: str | None = ..." in stub assert "stream: StreamingBlob | None = ..." in stub + assert "events: Events | None = ..." in stub + assert "class EventsItem(_SerdeShape):" in stub + assert "Events = EventsItem | EventsUnknown" in stub + assert "class _EventsDeserializer:" in stub assert "class Problem(ServiceError, _SerdeShape):" in stub assert "message:" not in stub assert "fault_: str | None = ..." in stub @@ -333,6 +347,11 @@ def get_input(_: int) -> type[Any]: stream_input = models.Input(stream=stream) assert stream_input.stream is stream + event = models.EventsItem(value=value) + event_input = models.Input(events=event) + assert event_input.events is event + assert models._EventsDeserializer.__name__ == "_EventsDeserializer" + class StreamSerializer: value: object | None = None diff --git a/packages/smithy-python/tests/unit/test_symbols.py b/packages/smithy-python/tests/unit/test_symbols.py index a1013999f..3fb2e4a4a 100644 --- a/packages/smithy-python/tests/unit/test_symbols.py +++ b/packages/smithy-python/tests/unit/test_symbols.py @@ -122,8 +122,8 @@ def test_type_references(kind: str, name: str, module: str) -> None: ) -def test_streaming_union_has_a_schema_name_without_a_model_symbol() -> None: - from smithy_python.model import ModelError +def test_streaming_union_has_schema_and_model_symbols() -> None: + from smithy_python.symbols import TypeReference model = model_with( { @@ -135,8 +135,10 @@ def test_streaming_union_has_a_schema_name_without_a_model_symbol() -> None: ) symbols = SymbolProvider(model, select_shapes(model), package="example") assert symbols.schema_name("example#Events") == "EVENTS" - with pytest.raises(ModelError, match="Unsupported streaming union"): - symbols.declaration_name("example#Events") + assert symbols.declaration_name("example#Events") == "Events" + assert symbols.type_reference("example#Events") == TypeReference( + "Events", "example.models" + ) @pytest.mark.parametrize("kind,value_type", [("enum", "str"), ("intEnum", "int")]) @@ -570,8 +572,8 @@ def test_streaming_blob_type_reference() -> None: ) -def test_streaming_union_rejected_when_resolved() -> None: - from smithy_python.model import ModelError +def test_streaming_union_type_reference() -> None: + from smithy_python.symbols import TypeReference model = model_with( { @@ -583,11 +585,12 @@ def test_streaming_union_rejected_when_resolved() -> None: } ) symbols = SymbolProvider(model, select_shapes(model), package="pkg") - for shape_id in ("example#Stream", "example#Streams"): - with pytest.raises(ModelError, match=r"Unsupported streaming.*example#Stream"): - symbols.type_reference(shape_id) - with pytest.raises(ModelError, match="Unsupported streaming"): - symbols.declaration_name("example#Stream") + stream = TypeReference("Stream", "pkg.models") + assert symbols.declaration_name("example#Stream") == "Stream" + assert symbols.type_reference("example#Stream") == stream + assert symbols.type_reference("example#Streams") == TypeReference( + "list", "builtins", (stream,) + ) def test_request_boundaries() -> None: diff --git a/scripts/codegen-output-py/model.json b/scripts/codegen-output-py/model.json index 6d4110a28..f0c2a2097 100644 --- a/scripts/codegen-output-py/model.json +++ b/scripts/codegen-output-py/model.json @@ -50,6 +50,9 @@ }, "value": { "target": "example#Value" + }, + "events": { + "target": "example#EventStream" } } }, @@ -133,6 +136,17 @@ } } }, + "example#EventStream": { + "type": "union", + "traits": { + "smithy.api#streaming": {} + }, + "members": { + "example": { + "target": "example#Example" + } + } + }, "example#Status": { "type": "enum", "members": { From 005fe7c5b8986a294de2502cae1017c2e9944374 Mon Sep 17 00:00:00 2001 From: Sean O'Brien <60306702+stobrien89@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:39:23 -0400 Subject: [PATCH 4/6] exclude eventstream members from generated structs --- .../types/.smithy-python-manifest.json | 2 +- .../types/src/example_types/models.pyi | 1 - .../src/smithy_core/_dynamic_models.py | 23 +++++- .../tests/unit/test_dynamic_models.py | 75 +++++++++++++++++++ .../smithy-python/src/smithy_python/models.py | 10 ++- .../smithy-python/tests/unit/test_models.py | 20 ++++- .../src/smithy_test/_private/conformance.py | 14 +++- .../tests/unit/test_conformance.py | 65 ++++++++++++++++ 8 files changed, 201 insertions(+), 9 deletions(-) create mode 100644 packages/smithy-test/tests/unit/test_conformance.py diff --git a/codegen-output-py/types/.smithy-python-manifest.json b/codegen-output-py/types/.smithy-python-manifest.json index 9ab9e0cb3..d0ae825ed 100644 --- a/codegen-output-py/types/.smithy-python-manifest.json +++ b/codegen-output-py/types/.smithy-python-manifest.json @@ -2,7 +2,7 @@ "files": { "src/example_types/_private/schemas.py": "1d708d275049f3f4a86b343c3f38104ef4a7e5bf749a27cd1a490c95c8f9db59", "src/example_types/models.py": "51aadd47e9971f2b44fabd47e81e4b04d7156aa61b64454b39f4b6d47603ab45", - "src/example_types/models.pyi": "a841db3c47698f1cd5236a73ed4b9d4a40fcab3fbcef0ee7b4957e6a9df74952" + "src/example_types/models.pyi": "31c14804662d43c11fc6c61d0a49244e5e606b65aadb774df913f6a26e1b135a" }, "version": 1 } diff --git a/codegen-output-py/types/src/example_types/models.pyi b/codegen-output-py/types/src/example_types/models.pyi index 616a2aaef..0d72eca2a 100644 --- a/codegen-output-py/types/src/example_types/models.pyi +++ b/codegen-output-py/types/src/example_types/models.pyi @@ -56,7 +56,6 @@ class GetExampleOutput(_SerdeShape): example: RenamedExample | None = ... value: ExampleValue | None = ... - events: EventStream | None = ... response_metadata: ResponseMetadata = ... """ Metadata about the response that produced this output. diff --git a/packages/smithy-core/src/smithy_core/_dynamic_models.py b/packages/smithy-core/src/smithy_core/_dynamic_models.py index 1ee10c042..446c3fdc7 100644 --- a/packages/smithy-core/src/smithy_core/_dynamic_models.py +++ b/packages/smithy-core/src/smithy_core/_dynamic_models.py @@ -301,6 +301,8 @@ def serialize_members( ) -> None: field_names = self._field_names(schema) for i, member in enumerate(schema.members_by_index): + if self._is_event_stream_member(member): + continue value = getattr(instance, field_names[i]) if value is None: continue @@ -316,8 +318,11 @@ def deserialize_kwargs( def consume(member_schema: Schema, de: ShapeDeserializer) -> None: idx = member_schema.expect_member_index() - _, reader = by_index[idx].get_extension(self._codec) - kwargs[field_names[idx]] = reader(de, by_index[idx]) + member = by_index[idx] + if self._is_event_stream_member(member): + return + _, reader = member.get_extension(self._codec) + kwargs[field_names[idx]] = reader(de, member) deserializer.read_struct(schema, consumer=consume) if not self._is_input(schema): @@ -333,6 +338,8 @@ def _error_correct( ) -> None: """Fill @required members a non-compliant server omitted (Smithy client error correction).""" for i, member in enumerate(by_index): + if self._is_event_stream_member(member): + continue fname = field_names[i] if fname in kwargs: continue @@ -362,6 +369,8 @@ def _build_struct(self, name: str, schema: Schema) -> type: field_names = self._field_names(schema) fields: list[Any] = [] for i, member in enumerate(schema.members_by_index): + if self._is_event_stream_member(member): + continue if is_error and field_names[i] == "message": if member.get_trait(SensitiveTrait) is not None: fields.append( @@ -657,6 +666,16 @@ def _is_streaming(self, member: Schema) -> bool: return False return target.get_trait(StreamingTrait) is not None + def _is_event_stream_member(self, member: Schema) -> bool: + try: + target = member.expect_member_target() + except Exception: + return False + return ( + target.shape_type is ShapeType.UNION + and target.get_trait(StreamingTrait) is not None + ) + def _resolve_member_codec(self, member: Schema) -> tuple[_Writer, _Reader]: st = member.shape_type if st == ShapeType.BLOB and self._is_streaming(member): diff --git a/packages/smithy-core/tests/unit/test_dynamic_models.py b/packages/smithy-core/tests/unit/test_dynamic_models.py index 6f342ca03..6ca783d3a 100644 --- a/packages/smithy-core/tests/unit/test_dynamic_models.py +++ b/packages/smithy-core/tests/unit/test_dynamic_models.py @@ -37,6 +37,18 @@ def __call__(self) -> ModeledError: ... def deserialize(self, deserializer: ShapeDeserializer) -> ModeledError: ... +class _Request(Protocol): + name: str + + def serialize_members(self, serializer: ShapeSerializer) -> None: ... + + +class _RequestClass(Protocol): + def __call__(self, *, name: str) -> _Request: ... + + def deserialize(self, deserializer: ShapeDeserializer) -> _Request: ... + + def _struct(name: str, members: tuple[tuple[str, object], ...]) -> StaticSchema: return ( ShapeType.STRUCTURE.value, @@ -78,6 +90,17 @@ def _union(name: str, members: tuple[tuple[str, object], ...]) -> StaticSchema: ) +def _event_stream(name: str, members: tuple[tuple[str, object], ...]) -> StaticSchema: + return ( + ShapeType.UNION.value, + NS, + name, + {"smithy.api#streaming": {}}, + tuple(m[0] for m in members), + tuple((m[1], 0) for m in members), + ) + + _ENUM_VALUE = "smithy.api#enumValue" @@ -154,6 +177,58 @@ def test_supplied_index_matches_scanning_fallback() -> None: assert _fields(scanned) == _fields(supplied) +def test_event_stream_member_is_not_a_model_field_or_serde_member() -> None: + _schema_compact._cache.clear() + events = _event_stream("Events", (("message", ShapeType.STRING.value),)) + request = _struct( + "Request", + ( + ("name", ShapeType.STRING.value), + ("events", lambda: events), + ), + ) + registry = ModelRegistry( + _module(EVENTS=events, REQUEST=request), + index={ + "Events": "EVENTS", + "EventsMessage": "EVENTS", + "EventsUnknown": "EVENTS", + "Request": "REQUEST", + }, + shape_names={"Events", "Request"}, + ) + request_type = registry.build("Request") + request_cls = cast(_RequestClass, request_type) + assert _fields(request_type) == ["name"] + assert registry.build("EventsMessage").__name__ == "EventsMessage" + + class _Serializer: + def __init__(self) -> None: + self.values: list[object] = [] + + def write_string(self, schema: Schema, value: object) -> None: + self.values.append(value) + + value = request_cls(name="test") + serializer = _Serializer() + value.serialize_members(cast(ShapeSerializer, serializer)) + assert serializer.values == ["test"] + + class _Deserializer: + def read_struct(self, schema: Schema, consumer: Any) -> None: + for member in schema.members_by_index: + consumer(member, self) + + def read_string(self, schema: Schema) -> str: + if schema.id.name == "events": + pytest.fail("event stream member should not be deserialized") + return "test" + + decoded = request_cls.deserialize(cast(ShapeDeserializer, _Deserializer())) + assert decoded.name == "test" + assert not hasattr(decoded, "events") + + def test_supplied_index_preserves_distinct_same_named_structures() -> None: first = ( ShapeType.STRUCTURE.value, diff --git a/packages/smithy-python/src/smithy_python/models.py b/packages/smithy-python/src/smithy_python/models.py index 14ac548dc..0e02263a0 100644 --- a/packages/smithy-python/src/smithy_python/models.py +++ b/packages/smithy-python/src/smithy_python/models.py @@ -240,7 +240,8 @@ def _write_structure(self, shape: _model.Shape) -> None: members = tuple( member for member in shape.members.values() - if not error or member.name.lower() not in _ERROR_MESSAGE_NAMES + if not _is_event_stream_member(self.model, member) + and (not error or member.name.lower() not in _ERROR_MESSAGE_NAMES) ) carries_metadata = error or shape.id in self.outputs _write_docstring( @@ -420,12 +421,19 @@ def _documentation(shape: _model.Shape) -> str | None: return value if isinstance(value, str) else None +def _is_event_stream_member(model: _model.Model, member: _model.MemberShape) -> bool: + target = model.get_target(member) + return target.type is _model.ShapeType.UNION and target.has_trait("streaming") + + def _validate_member_names(context: GenerationContext[TypesSettings]) -> None: reserved = _model_reserved_member_names(context.model) for shape in context.plan.models.shapes: names: set[str] = set() shape_reserved = reserved(shape) for member in shape.members.values(): + if _is_event_stream_member(context.model, member): + continue name = context.symbols.member_name(member.id) if name in shape_reserved: raise _exceptions.CodegenError( diff --git a/packages/smithy-python/tests/unit/test_models.py b/packages/smithy-python/tests/unit/test_models.py index 4719dbc9d..2d93ae8d6 100644 --- a/packages/smithy-python/tests/unit/test_models.py +++ b/packages/smithy-python/tests/unit/test_models.py @@ -15,6 +15,7 @@ from typing import Any import pytest +from smithy_core._schema_compact import hydrate from smithy_core.types import UnknownEnumMixin from smithy_json import JSONCodec from smithy_python.dependencies import SMITHY_CORE, DependencySet @@ -285,7 +286,7 @@ def test_model_generator_emits_runtime_and_stub(tmp_path: Path) -> None: assert "required: str | None = ..." in stub assert "list_: str | None = ..." in stub assert "stream: StreamingBlob | None = ..." in stub - assert "events: Events | None = ..." in stub + assert "events: Events | None = ..." not in stub assert "class EventsItem(_SerdeShape):" in stub assert "Events = EventsItem | EventsUnknown" in stub assert "class _EventsDeserializer:" in stub @@ -348,10 +349,14 @@ def get_input(_: int) -> type[Any]: assert stream_input.stream is stream event = models.EventsItem(value=value) - event_input = models.Input(events=event) - assert event_input.events is event + with pytest.raises(TypeError): + models.Input(events=event) assert models._EventsDeserializer.__name__ == "_EventsDeserializer" + schemas = importlib.import_module(f"{_PACKAGE}._private.schemas") + input_schema = hydrate(schemas.INPUT) + assert input_schema.members["events"].expect_member_target().id.name == "Events" + class StreamSerializer: value: object | None = None @@ -377,6 +382,15 @@ def read_data_stream(self, schema: Any) -> bytes: decoded_stream = models.Input.deserialize(StreamDeserializer()) assert decoded_stream.stream is stream + class EventStreamDeserializer: + def read_struct(self, schema: Any, consumer: Any) -> None: + if schema.id.name != "Input": + pytest.fail("event stream member should not be deserialized") + consumer(schema.members["events"], self) + + decoded_event_stream = models.Input.deserialize(EventStreamDeserializer()) + assert not hasattr(decoded_event_stream, "events") + decoded = JSONCodec().deserialize( ( b'{"required":"yes","renamed":{"value":"hello"},' diff --git a/packages/smithy-test/src/smithy_test/_private/conformance.py b/packages/smithy-test/src/smithy_test/_private/conformance.py index 89c1067e2..16761d133 100644 --- a/packages/smithy-test/src/smithy_test/_private/conformance.py +++ b/packages/smithy-test/src/smithy_test/_private/conformance.py @@ -20,7 +20,7 @@ from smithy_core._schema_compact import hydrate from smithy_core.shapes import ShapeType -from smithy_core.traits import DefaultTrait, RequiredTrait +from smithy_core.traits import DefaultTrait, RequiredTrait, StreamingTrait _SHAPE_CONSTS = ("STRUCTURE", "UNION", "ENUM", "INT_ENUM") @@ -91,6 +91,8 @@ def _assert_fields_match_schema(name: str, cls: type, schema: Any) -> None: raise ConformanceError(f"{name}: built class is not a dataclass") field_names = {f.name for f in dataclasses.fields(cls)} for member in schema.members_by_index: + if _is_event_stream_member(member): + continue # a runtime field must exist for every modeled member (name derived by the # same ported namer the registry uses) from smithy_core._model_naming import member_field_name @@ -129,6 +131,8 @@ def _build_instance(cls: type, schema: Any, models: ModuleType) -> Any: kwargs: dict[str, Any] = {} is_error = _is_error(schema) for member in schema.members_by_index: + if _is_event_stream_member(member): + continue required = member.get_trait(RequiredTrait) is not None has_default = member.get_trait(DefaultTrait) is not None if not required or has_default or _is_input(schema): @@ -230,3 +234,11 @@ def _is_input(schema: Any) -> bool: from smithy_core.shapes import ShapeID return schema.get_trait(ShapeID("smithy.api#input")) is not None + + +def _is_event_stream_member(member: Any) -> bool: + target = member.expect_member_target() + return ( + target.shape_type is ShapeType.UNION + and target.get_trait(StreamingTrait) is not None + ) diff --git a/packages/smithy-test/tests/unit/test_conformance.py b/packages/smithy-test/tests/unit/test_conformance.py new file mode 100644 index 000000000..5794819bb --- /dev/null +++ b/packages/smithy-test/tests/unit/test_conformance.py @@ -0,0 +1,65 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from dataclasses import dataclass +from pathlib import Path +from types import ModuleType + +from smithy_core._schema_compact import StaticSchema +from smithy_core.shapes import ShapeType +from smithy_test import verify_models + +_REQUIRED: dict[str, object] = {"smithy.api#required": {}} +_STREAMING: dict[str, object] = {"smithy.api#streaming": {}} + + +def test_event_stream_members_are_not_model_fields(tmp_path: Path) -> None: + events: StaticSchema = ( + ShapeType.UNION.value, + "example", + "Events", + _STREAMING, + ("message",), + ((ShapeType.STRING.value, 0),), + ) + input_schema: StaticSchema = ( + ShapeType.STRUCTURE.value, + "example", + "PublishEventsInput", + 0, + ("channel", "events"), + ( + (ShapeType.STRING.value, _REQUIRED), + (lambda: events, _REQUIRED), + ), + ) + schemas = ModuleType("example._private.schemas") + schemas.EVENTS = events # type: ignore[attr-defined] + schemas.PUBLISH_EVENTS_INPUT = input_schema # type: ignore[attr-defined] + + @dataclass + class PublishEventsInput: + channel: str + + models = ModuleType("example.models") + models.PublishEventsInput = PublishEventsInput # type: ignore[attr-defined] + + class _Codec: + def serialize(self, value: PublishEventsInput) -> bytes: + assert value.channel == "x" + return b"" + + def deserialize( + self, body: bytes, cls: type[PublishEventsInput] + ) -> PublishEventsInput: + return cls(channel="x") + + stub = tmp_path / "models.pyi" + stub.write_text( + "from dataclasses import dataclass\n" + "@dataclass\n" + "class PublishEventsInput:\n" + " channel: str\n" + ) + + assert verify_models(models, schemas, codec=_Codec(), pyi_path=str(stub)) == 1 From d83c5f638c6028a3a07d2407744581dbeaf7e49a Mon Sep 17 00:00:00 2001 From: Sean O'Brien <60306702+stobrien89@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:59:24 -0400 Subject: [PATCH 5/6] filter event stream members from generated stubs --- .../src/my_handwritten_model/models.pyi | 4 ---- .../python/codegen/generators/ModelStubGenerator.java | 7 +++++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/codegen-output/my-handwritten-model/src/my_handwritten_model/models.pyi b/codegen-output/my-handwritten-model/src/my_handwritten_model/models.pyi index 48e7e79d9..ce53aaa6e 100644 --- a/codegen-output/my-handwritten-model/src/my_handwritten_model/models.pyi +++ b/codegen-output/my-handwritten-model/src/my_handwritten_model/models.pyi @@ -88,7 +88,6 @@ class PublishEventsInput(_SerdeShape): """Dataclass for PublishEventsInput structure.""" channel: str | None = ... - events: PublishEventStream | None = ... @dataclass(kw_only=True) class PublishEventsOutput(_SerdeShape): @@ -188,7 +187,6 @@ class SubscribeToEventsOutput(_SerdeShape): """Dataclass for SubscribeToEventsOutput structure.""" subscription_id: str | None = ... - events: SubscribeEventStream | None = ... response_metadata: ResponseMetadata = ... """ Metadata about the response that produced this output. @@ -261,14 +259,12 @@ class TradeEventStreamInput(_SerdeShape): """Dataclass for TradeEventStreamInput structure.""" session_id: str | None = ... - events: TradeEvents | None = ... @dataclass(kw_only=True) class TradeEventStreamOutput(_SerdeShape): """Dataclass for TradeEventStreamOutput structure.""" greeting: str | None = ... - events: TradeEvents | None = ... response_metadata: ResponseMetadata = ... """ Metadata about the response that produced this output. diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ModelStubGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ModelStubGenerator.java index 7dcec3cc0..41bc0e69f 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ModelStubGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ModelStubGenerator.java @@ -19,6 +19,7 @@ import software.amazon.smithy.model.traits.DocumentationTrait; import software.amazon.smithy.model.traits.EnumValueTrait; import software.amazon.smithy.model.traits.ErrorTrait; +import software.amazon.smithy.model.traits.StreamingTrait; import software.amazon.smithy.python.codegen.CodegenUtils; import software.amazon.smithy.python.codegen.GenerationContext; import software.amazon.smithy.python.codegen.PythonSettings; @@ -116,6 +117,7 @@ private void writeStruct(PythonWriter writer, StructureShape shape) { var members = shape.members() .stream() .filter(m -> !CodegenUtils.isErrorMessage(model, m)) + .filter(m -> !isEventStreamMember(m)) .toList(); if (members.isEmpty() && !carriesResponseMetadata(shape)) { return; @@ -162,6 +164,11 @@ private boolean carriesResponseMetadata(Shape shape) { return shape.hasTrait(ErrorTrait.class) || OperationIndex.of(model).isOutputStructure(shape); } + private boolean isEventStreamMember(MemberShape member) { + var target = model.expectShape(member.getTarget()); + return target.isUnionShape() && target.hasTrait(StreamingTrait.class); + } + private void writeUnion(PythonWriter writer, UnionShape shape) { var parentSymbol = symbolProvider.toSymbol(shape); var parentName = parentSymbol.getName(); From 52d3dc658749483c1e0092e509ad3fb599931873 Mon Sep 17 00:00:00 2001 From: Sean O'Brien <60306702+stobrien89@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:54:26 -0400 Subject: [PATCH 6/6] filter event streams from generated typecheck fixtures --- .../rest-json-1/src/restjson/models.pyi | 12 ---------- .../codegen/test/PythonCodegenTest.java | 4 ++++ .../smithy/python/codegen/CodegenUtils.java | 13 ++++++++++ .../generators/ConformanceTestGenerator.java | 3 ++- .../generators/ModelStubGenerator.java | 8 +------ .../generators/StructureGenerator.java | 24 ++++++------------- 6 files changed, 27 insertions(+), 37 deletions(-) diff --git a/codegen-output/rest-json-1/src/restjson/models.pyi b/codegen-output/rest-json-1/src/restjson/models.pyi index 93ac87e64..029a5f51e 100644 --- a/codegen-output/rest-json-1/src/restjson/models.pyi +++ b/codegen-output/rest-json-1/src/restjson/models.pyi @@ -275,13 +275,10 @@ class DocumentTypeOutput(_SerdeShape): class DuplexStreamInput(_SerdeShape): """Dataclass for DuplexStreamInput structure.""" - stream: EventStream | None = ... - @dataclass(kw_only=True) class DuplexStreamOutput(_SerdeShape): """Dataclass for DuplexStreamOutput structure.""" - stream: EventStream | None = ... response_metadata: ResponseMetadata = ... """ Metadata about the response that produced this output. Use this to @@ -294,8 +291,6 @@ class DuplexStreamOutput(_SerdeShape): class DuplexStreamWithDistinctStreamsInput(_SerdeShape): """Dataclass for DuplexStreamWithDistinctStreamsInput structure.""" - stream: EventStream | None = ... - @dataclass(kw_only=True) class DuplexStreamWithDistinctStreamsOutput(_SerdeShape): """Dataclass for DuplexStreamWithDistinctStreamsOutput structure.""" @@ -314,14 +309,12 @@ class DuplexStreamWithInitialMessagesInput(_SerdeShape): """Dataclass for DuplexStreamWithInitialMessagesInput structure.""" initial_request_member: str | None = ... - stream: EventStream | None = ... @dataclass(kw_only=True) class DuplexStreamWithInitialMessagesOutput(_SerdeShape): """Dataclass for DuplexStreamWithInitialMessagesOutput structure.""" initial_response_member: str - stream: EventStream | None = ... response_metadata: ResponseMetadata = ... """ Metadata about the response that produced this output. Use this to @@ -962,8 +955,6 @@ class InputAndOutputWithHeadersOutput(_SerdeShape): class InputStreamInput(_SerdeShape): """Dataclass for InputStreamInput structure.""" - stream: EventStream | None = ... - @dataclass(kw_only=True) class InputStreamOutput(_SerdeShape): """Dataclass for InputStreamOutput structure.""" @@ -981,7 +972,6 @@ class InputStreamWithInitialRequestInput(_SerdeShape): """Dataclass for InputStreamWithInitialRequestInput structure.""" initial_request_member: str | None = ... - stream: EventStream | None = ... @dataclass(kw_only=True) class InputStreamWithInitialRequestOutput(_SerdeShape): @@ -2131,7 +2121,6 @@ class OutputStreamInput(_SerdeShape): class OutputStreamOutput(_SerdeShape): """Dataclass for OutputStreamOutput structure.""" - stream: EventStream | None = ... response_metadata: ResponseMetadata = ... """ Metadata about the response that produced this output. Use this to @@ -2149,7 +2138,6 @@ class OutputStreamWithInitialResponseOutput(_SerdeShape): """Dataclass for OutputStreamWithInitialResponseOutput structure.""" initial_response_member: str - stream: EventStream | None = ... response_metadata: ResponseMetadata = ... """ Metadata about the response that produced this output. Use this to diff --git a/codegen/core/src/it/java/software/amazon/smithy/python/codegen/test/PythonCodegenTest.java b/codegen/core/src/it/java/software/amazon/smithy/python/codegen/test/PythonCodegenTest.java index 65bec161b..f92fbefde 100644 --- a/codegen/core/src/it/java/software/amazon/smithy/python/codegen/test/PythonCodegenTest.java +++ b/codegen/core/src/it/java/software/amazon/smithy/python/codegen/test/PythonCodegenTest.java @@ -62,6 +62,10 @@ public void testCodegen(@TempDir Path tempDir) throws IOException { assertTrue(models.contains("response_metadata: ResponseMetadata")); assertTrue(models.contains("response_metadata_:")); + // Event streams are passed through the stream API, not model constructors. + var typecheck = Files.readString(tempDir.resolve("tests/_typecheck.py")); + assertFalse(typecheck.contains("stream=_")); + var schemas = Files.readString(tempDir.resolve("src/weather/_private/schemas.py")); assertTrue(schemas.contains("example.weather#fakeProtocol")); assertTrue(schemas.contains("aws.protocols#restJson1")); diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java index cf1c66444..20ce77cac 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java @@ -34,6 +34,7 @@ import software.amazon.smithy.model.shapes.Shape; import software.amazon.smithy.model.traits.DefaultTrait; import software.amazon.smithy.model.traits.ErrorTrait; +import software.amazon.smithy.model.traits.StreamingTrait; import software.amazon.smithy.model.traits.TimestampFormatTrait; import software.amazon.smithy.model.traits.TimestampFormatTrait.Format; import software.amazon.smithy.python.codegen.writer.PythonWriter; @@ -193,6 +194,18 @@ public static boolean isErrorMessage(Model model, MemberShape shape) { && model.expectShape(shape.getContainer()).hasTrait(ErrorTrait.class); } + /** + * Determines whether a member targets an event stream. + * + * @param model The whole service model. + * @param member The member to check. + * @return Returns whether the member targets a streaming union. + */ + public static boolean isEventStreamMember(Model model, MemberShape member) { + var target = model.expectShape(member.getTarget()); + return target.isUnionShape() && target.hasTrait(StreamingTrait.class); + } + /** * Determines whether a member is required in the generated Python constructor — that is, * neither nullable nor carrying a default value. diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ConformanceTestGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ConformanceTestGenerator.java index a827332dd..f399e204f 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ConformanceTestGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ConformanceTestGenerator.java @@ -121,7 +121,8 @@ private void writeStructUsage( if (!operationIndex.isInputStructure(shape)) { for (var member : shape.members()) { if (CodegenUtils.isRequiredMember(index, member) - && !CodegenUtils.isErrorMessage(model, member)) { + && !CodegenUtils.isErrorMessage(model, member) + && !CodegenUtils.isEventStreamMember(model, member)) { var memberName = symbolProvider.toMemberName(member); if (args.length() > 0) { args.append(", "); diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ModelStubGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ModelStubGenerator.java index 41bc0e69f..1f63795c8 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ModelStubGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ModelStubGenerator.java @@ -19,7 +19,6 @@ import software.amazon.smithy.model.traits.DocumentationTrait; import software.amazon.smithy.model.traits.EnumValueTrait; import software.amazon.smithy.model.traits.ErrorTrait; -import software.amazon.smithy.model.traits.StreamingTrait; import software.amazon.smithy.python.codegen.CodegenUtils; import software.amazon.smithy.python.codegen.GenerationContext; import software.amazon.smithy.python.codegen.PythonSettings; @@ -117,7 +116,7 @@ private void writeStruct(PythonWriter writer, StructureShape shape) { var members = shape.members() .stream() .filter(m -> !CodegenUtils.isErrorMessage(model, m)) - .filter(m -> !isEventStreamMember(m)) + .filter(m -> !CodegenUtils.isEventStreamMember(model, m)) .toList(); if (members.isEmpty() && !carriesResponseMetadata(shape)) { return; @@ -164,11 +163,6 @@ private boolean carriesResponseMetadata(Shape shape) { return shape.hasTrait(ErrorTrait.class) || OperationIndex.of(model).isOutputStructure(shape); } - private boolean isEventStreamMember(MemberShape member) { - var target = model.expectShape(member.getTarget()); - return target.isUnionShape() && target.hasTrait(StreamingTrait.class); - } - private void writeUnion(PythonWriter writer, UnionShape shape) { var parentSymbol = symbolProvider.toSymbol(shape); var parentName = parentSymbol.getName(); diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java index d0f7e6dd9..76eabd7a2 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java @@ -29,7 +29,6 @@ import software.amazon.smithy.model.traits.ErrorTrait; import software.amazon.smithy.model.traits.RetryableTrait; import software.amazon.smithy.model.traits.SensitiveTrait; -import software.amazon.smithy.model.traits.StreamingTrait; import software.amazon.smithy.python.codegen.CodegenUtils; import software.amazon.smithy.python.codegen.GenerationContext; import software.amazon.smithy.python.codegen.PythonSettings; @@ -302,21 +301,18 @@ private void writeMemberDocs(MemberShape member) { private List filterPropertyMembers(List members) { if (!shape.hasTrait(ErrorTrait.class)) { - return members.stream().filter(this::filterEventStreamMember).toList(); + return members.stream() + .filter(member -> !CodegenUtils.isEventStreamMember(model, member)) + .toList(); } // We replace modeled message members with a static `message` member. Serialization // and deserialization will handle assigning them properly. return members.stream() .filter(member -> !isErrorMessage(model, member)) - .filter(this::filterEventStreamMember) + .filter(member -> !CodegenUtils.isEventStreamMember(model, member)) .collect(Collectors.toList()); } - private boolean filterEventStreamMember(MemberShape member) { - var target = model.expectShape(member.getTarget()); - return !(target.isUnionShape() && target.hasTrait(StreamingTrait.class)); - } - private String getDefaultValue(PythonWriter writer, MemberShape member) { // The default value is defined in the model is a exposed as generic // json, so we need to convert it to the proper type based on the target. @@ -412,10 +408,7 @@ def serialize(self, serializer: ${shapeSerializer:T}): private List filterMembers() { return shape.members() .stream() - .filter(member -> { - var target = model.expectShape(member.getTarget()); - return !(target.hasTrait(StreamingTrait.class) && target.isUnionShape()); - }) + .filter(member -> !CodegenUtils.isEventStreamMember(model, member)) .toList(); } @@ -441,10 +434,7 @@ private void generateDeserializeMethod() { // index tuple would be an unused local. boolean hasMembers = shape.members() .stream() - .anyMatch(m -> { - var t = model.expectShape(m.getTarget()); - return !(t.hasTrait(StreamingTrait.class) && t.isUnionShape()); - }); + .anyMatch(member -> !CodegenUtils.isEventStreamMember(model, member)); writer.putContext("hasMembers", hasMembers); // TODO: either formalize deserialize_kwargs or remove it when http serde is converted @@ -558,7 +548,7 @@ private void deserializeMembers(Collection members) { for (MemberShape member : members) { index++; var target = model.expectShape(member.getTarget()); - if (target.hasTrait(StreamingTrait.class) && target.isUnionShape()) { + if (CodegenUtils.isEventStreamMember(model, member)) { continue; } writer.write("""