diff --git a/codegen-output-py/types/.smithy-python-manifest.json b/codegen-output-py/types/.smithy-python-manifest.json index 9c7015360..d0ae825ed 100644 --- a/codegen-output-py/types/.smithy-python-manifest.json +++ b/codegen-output-py/types/.smithy-python-manifest.json @@ -1,6 +1,8 @@ { "files": { - "src/example_types/_private/schemas.py": "d0f35e8979c4aa9d49a1d9c0c8efeb860420bc0ceab3ba3edcc9488c053e9110" + "src/example_types/_private/schemas.py": "1d708d275049f3f4a86b343c3f38104ef4a7e5bf749a27cd1a490c95c8f9db59", + "src/example_types/models.py": "51aadd47e9971f2b44fabd47e81e4b04d7156aa61b64454b39f4b6d47603ab45", + "src/example_types/models.pyi": "31c14804662d43c11fc6c61d0a49244e5e606b65aadb774df913f6a26e1b135a" }, "version": 1 } diff --git a/codegen-output-py/types/src/example_types/_private/schemas.py b/codegen-output-py/types/src/example_types/_private/schemas.py index 1192d7ce8..91264ec48 100644 --- a/codegen-output-py/types/src/example_types/_private/schemas.py +++ b/codegen-output-py/types/src/example_types/_private/schemas.py @@ -32,13 +32,16 @@ _s26 = "choice" _s27 = "status" _s28 = "code" -_s29 = "GetExampleOutput" -_s30 = "smithy.api#output" -_s31 = "ExampleError" -_s32 = "smithy.api#error" -_s33 = "client" -_s34 = "message" -_s35 = "retryAfterSeconds" +_s29 = "EventStream" +_s30 = "smithy.api#streaming" +_s31 = "GetExampleOutput" +_s32 = "smithy.api#output" +_s33 = "events" +_s34 = "ExampleError" +_s35 = "smithy.api#error" +_s36 = "client" +_s37 = "message" +_s38 = "retryAfterSeconds" GET_EXAMPLE_INPUT: StaticSchema = ( 18, @@ -145,29 +148,40 @@ ), ) -GET_EXAMPLE_OUTPUT: StaticSchema = ( - 18, +EVENT_STREAM: StaticSchema = ( + 19, n0, _s29, {_s30: None}, + (n0,), + ((RENAMED_EXAMPLE, 0),), +) + +GET_EXAMPLE_OUTPUT: StaticSchema = ( + 18, + n0, + _s31, + {_s32: None}, ( n0, _s7, + _s33, ), ( (RENAMED_EXAMPLE, 0), (EXAMPLE_VALUE, 0), + (EVENT_STREAM, 0), ), ) EXAMPLE_ERROR: StaticSchema = ( 18, n0, - _s31, - {_s32: _s33}, + _s34, + {_s35: _s36}, ( - _s34, - _s35, + _s37, + _s38, ), ( (3, 0), diff --git a/codegen-output-py/types/src/example_types/models.py b/codegen-output-py/types/src/example_types/models.py new file mode 100644 index 000000000..6803f9606 --- /dev/null +++ b/codegen-output-py/types/src/example_types/models.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +from smithy_core._dynamic_models import ModelRegistry +from smithy_core.exceptions import ModeledError + +from ._private import schemas + + +class ServiceError(ModeledError): + """ + Base error for all errors in the service. + + Some exceptions do not extend from this class, including synthetic, + implicit, and shared exception types. + """ + + +_MODEL_INDEX = { + "GetExampleInput": "GET_EXAMPLE_INPUT", + "GetExampleOutput": "GET_EXAMPLE_OUTPUT", + "ExampleError": "EXAMPLE_ERROR", + "RenamedExample": "RENAMED_EXAMPLE", + "ExampleValue": "EXAMPLE_VALUE", + "ExampleValueText": "EXAMPLE_VALUE", + "ExampleValueNumber": "EXAMPLE_VALUE", + "ExampleValueEmpty": "EXAMPLE_VALUE", + "ExampleValueUnknown": "EXAMPLE_VALUE", + "EventStream": "EVENT_STREAM", + "EventStreamExample": "EVENT_STREAM", + "EventStreamUnknown": "EVENT_STREAM", + "Status": "STATUS", + "Code": "CODE", +} + +_RENAMES = { + "RenamedExample": "RENAMED_EXAMPLE", + "ExampleValue": "EXAMPLE_VALUE", +} + +_TARGET_NAMES = { + "example#Example": "RenamedExample", + "example#Value": "ExampleValue", +} + +_MODEL_NAMES = (*tuple(_MODEL_INDEX), "Unit") +__all__ = ("ServiceError", *_MODEL_NAMES) + + +_registry = ModelRegistry( + schemas, + renames=_RENAMES, + index=_MODEL_INDEX, + target_names=_TARGET_NAMES, + shape_names=frozenset( + { + "Code", + "EventStream", + "ExampleError", + "ExampleValue", + "GetExampleInput", + "GetExampleOutput", + "RenamedExample", + "Status", + "Unit", + } + ), + service_error="ServiceError", +) +del schemas, ModelRegistry, ModeledError + + +def __getattr__(name: str): + value = _registry.build(name) + globals()[name] = value + return value + + +def __dir__() -> list[str]: + return sorted({*globals(), *__all__}) diff --git a/codegen-output-py/types/src/example_types/models.pyi b/codegen-output-py/types/src/example_types/models.pyi new file mode 100644 index 000000000..0d72eca2a --- /dev/null +++ b/codegen-output-py/types/src/example_types/models.pyi @@ -0,0 +1,160 @@ +from __future__ import annotations + +from dataclasses import dataclass +from enum import IntEnum, StrEnum +from typing import Self + +from smithy_core.deserializers import ShapeDeserializer +from smithy_core.documents import Document +from smithy_core.exceptions import ModeledError +from smithy_core.response import ResponseMetadata +from smithy_core.serializers import ShapeSerializer +from smithy_core.types import UnknownEnumMixin + +__all__ = [ + "Code", + "EventStream", + "EventStreamExample", + "EventStreamUnknown", + "ExampleError", + "ExampleValue", + "ExampleValueEmpty", + "ExampleValueNumber", + "ExampleValueText", + "ExampleValueUnknown", + "GetExampleInput", + "GetExampleOutput", + "RenamedExample", + "ServiceError", + "Status", + "Unit", +] + +class _SerdeShape: + def serialize(self, serializer: ShapeSerializer) -> None: ... + def serialize_members(self, serializer: ShapeSerializer) -> None: ... + @classmethod + def deserialize(cls, deserializer: ShapeDeserializer) -> Self: ... + +class ServiceError(ModeledError): + """ + Base error for all errors in the service. + + Some exceptions do not extend from this class, including synthetic, + implicit, and shared exception types. + """ + +@dataclass(kw_only=True) +class GetExampleInput(_SerdeShape): + "Input for retrieving an example." + + id: str | None = ... + +@dataclass(kw_only=True) +class GetExampleOutput(_SerdeShape): + "Dataclass for GetExampleOutput structure." + + example: RenamedExample | None = ... + value: ExampleValue | None = ... + response_metadata: ResponseMetadata = ... + """ + Metadata about the response that produced this output. + + Use this to recover the request identifiers a service's support team + needs in order to investigate a call. Members of the metadata are + individually optional. + """ + +@dataclass(kw_only=True) +class ExampleError(ServiceError, _SerdeShape): + "An example modeled error." + + retry_after_seconds: int | None = ... + response_metadata: ResponseMetadata = ... + """ + Metadata about the response that produced this output. + + Use this to recover the request identifiers a service's support team + needs in order to investigate a call. Members of the metadata are + individually optional. + """ + +@dataclass(kw_only=True) +class RenamedExample(_SerdeShape): + "A representative generated structure." + + name: str + "The example name." + count: int | None = ... + tags: list[str] | None = ... + attributes: dict[str, Document] | None = ... + choice: ExampleValue | None = ... + status: str | None = ... + code: int | None = ... + +@dataclass +class ExampleValueText(_SerdeShape): + value: str + +@dataclass +class ExampleValueNumber(_SerdeShape): + value: int + +@dataclass +class ExampleValueEmpty(_SerdeShape): + value: Unit + +@dataclass +class ExampleValueUnknown(_SerdeShape): + """ + Represents an unknown variant. + + If you receive this value, you will need to update your library to + receive the parsed value. + + This value may not be deliberately sent. + """ + + tag: str + +ExampleValue = ( + ExampleValueText | ExampleValueNumber | ExampleValueEmpty | ExampleValueUnknown +) +"A representative generated union." + +class _ExampleValueDeserializer: + def deserialize(self, deserializer: ShapeDeserializer) -> ExampleValue: ... + +@dataclass +class EventStreamExample(_SerdeShape): + value: RenamedExample + +@dataclass +class EventStreamUnknown(_SerdeShape): + """ + Represents an unknown variant. + + If you receive this value, you will need to update your library to + receive the parsed value. + + This value may not be deliberately sent. + """ + + tag: str + +EventStream = EventStreamExample | EventStreamUnknown + +class _EventStreamDeserializer: + def deserialize(self, deserializer: ShapeDeserializer) -> EventStream: ... + +class Status(UnknownEnumMixin, StrEnum): + ACTIVE = "active" + INACTIVE = "inactive" + +class Code(UnknownEnumMixin, IntEnum): + OK = 200 + NOT_FOUND = 404 + +@dataclass(kw_only=True) +class Unit(_SerdeShape): + pass diff --git a/codegen-output/my-handwritten-model/src/my_handwritten_model/models.pyi b/codegen-output/my-handwritten-model/src/my_handwritten_model/models.pyi index 48e7e79d9..ce53aaa6e 100644 --- a/codegen-output/my-handwritten-model/src/my_handwritten_model/models.pyi +++ b/codegen-output/my-handwritten-model/src/my_handwritten_model/models.pyi @@ -88,7 +88,6 @@ class PublishEventsInput(_SerdeShape): """Dataclass for PublishEventsInput structure.""" channel: str | None = ... - events: PublishEventStream | None = ... @dataclass(kw_only=True) class PublishEventsOutput(_SerdeShape): @@ -188,7 +187,6 @@ class SubscribeToEventsOutput(_SerdeShape): """Dataclass for SubscribeToEventsOutput structure.""" subscription_id: str | None = ... - events: SubscribeEventStream | None = ... response_metadata: ResponseMetadata = ... """ Metadata about the response that produced this output. @@ -261,14 +259,12 @@ class TradeEventStreamInput(_SerdeShape): """Dataclass for TradeEventStreamInput structure.""" session_id: str | None = ... - events: TradeEvents | None = ... @dataclass(kw_only=True) class TradeEventStreamOutput(_SerdeShape): """Dataclass for TradeEventStreamOutput structure.""" greeting: str | None = ... - events: TradeEvents | None = ... response_metadata: ResponseMetadata = ... """ Metadata about the response that produced this output. diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSymbolProvider.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSymbolProvider.java index 62b5b12e4..13e5883fc 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSymbolProvider.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSymbolProvider.java @@ -147,6 +147,9 @@ public String toMemberName(MemberShape shape) { } var memberName = escaper.escapeMemberName(CaseUtils.toSnakeCase(shape.getMemberName())); + if (memberName.startsWith("__") && !memberName.endsWith("__")) { + memberName += "___"; + } // Escape words that are only reserved for error members. if (shape.hasTrait(ErrorTrait.class)) { @@ -170,6 +173,16 @@ && carriesResponseMetadata(container)) { } if (container.isEnumShape() || container.isIntEnumShape()) { + if (memberName.startsWith("__") + && memberName.endsWith("__") + && !memberName.endsWith("___")) { + memberName += "_"; + } else if (memberName.startsWith("_") + && !memberName.startsWith("__") + && memberName.endsWith("_") + && !memberName.endsWith("__")) { + memberName += "_"; + } memberName = memberName.toUpperCase(Locale.ENGLISH); } return memberName; diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ModelStubGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ModelStubGenerator.java index 7dcec3cc0..41bc0e69f 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ModelStubGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ModelStubGenerator.java @@ -19,6 +19,7 @@ import software.amazon.smithy.model.traits.DocumentationTrait; import software.amazon.smithy.model.traits.EnumValueTrait; import software.amazon.smithy.model.traits.ErrorTrait; +import software.amazon.smithy.model.traits.StreamingTrait; import software.amazon.smithy.python.codegen.CodegenUtils; import software.amazon.smithy.python.codegen.GenerationContext; import software.amazon.smithy.python.codegen.PythonSettings; @@ -116,6 +117,7 @@ private void writeStruct(PythonWriter writer, StructureShape shape) { var members = shape.members() .stream() .filter(m -> !CodegenUtils.isErrorMessage(model, m)) + .filter(m -> !isEventStreamMember(m)) .toList(); if (members.isEmpty() && !carriesResponseMetadata(shape)) { return; @@ -162,6 +164,11 @@ private boolean carriesResponseMetadata(Shape shape) { return shape.hasTrait(ErrorTrait.class) || OperationIndex.of(model).isOutputStructure(shape); } + private boolean isEventStreamMember(MemberShape member) { + var target = model.expectShape(member.getTarget()); + return target.isUnionShape() && target.hasTrait(StreamingTrait.class); + } + private void writeUnion(PythonWriter writer, UnionShape shape) { var parentSymbol = symbolProvider.toSymbol(shape); var parentName = parentSymbol.getName(); diff --git a/codegen/core/src/main/resources/software/amazon/smithy/python/codegen/reserved-member-names.txt b/codegen/core/src/main/resources/software/amazon/smithy/python/codegen/reserved-member-names.txt index 1831fb341..b18e8e1a8 100644 --- a/codegen/core/src/main/resources/software/amazon/smithy/python/codegen/reserved-member-names.txt +++ b/codegen/core/src/main/resources/software/amazon/smithy/python/codegen/reserved-member-names.txt @@ -81,3 +81,17 @@ str # that are likely imported in the generated code (e.g. datetime) # We only escape the types we use. datetime + +# Generated model runtime methods. +serialize +serialize_members +deserialize + +# Dataclass-generated methods and hooks. +__init__ +__post_init__ +__repr__ +__annotations__ + +# Generated model runtime methods. +_smithy_default diff --git a/codegen/core/src/test/java/software/amazon/smithy/python/codegen/PythonSymbolProviderTest.java b/codegen/core/src/test/java/software/amazon/smithy/python/codegen/PythonSymbolProviderTest.java index 0102a560c..3941b92e2 100644 --- a/codegen/core/src/test/java/software/amazon/smithy/python/codegen/PythonSymbolProviderTest.java +++ b/codegen/core/src/test/java/software/amazon/smithy/python/codegen/PythonSymbolProviderTest.java @@ -137,6 +137,68 @@ public void testUnrelatedMembersOnOutputsAndErrorsAreUnaffected() { assertEquals("retry_after", memberName(provider, model, "ThingError$retryAfter")); } + @Test + public void testDataclassHooksAreEscaped() { + Model model = loadModel(""" + $version: "2" + namespace smithy.example + + service TestService { + version: "2024-01-01" + operations: [GetThing] + } + + operation GetThing { + input: GetThingInput + } + + structure GetThingInput { + __init__: String + __post_init__: String + _smithy_default: String + __repr__: String + __annotations__: String + __value: String + } + """); + PythonSymbolProvider provider = createProvider(model); + + assertEquals("__init___", memberName(provider, model, "GetThingInput$__init__")); + assertEquals("__post_init___", memberName(provider, model, "GetThingInput$__post_init__")); + assertEquals("_smithy_default_", memberName(provider, model, "GetThingInput$_smithy_default")); + assertEquals("__repr___", memberName(provider, model, "GetThingInput$__repr__")); + assertEquals("__annotations___", memberName(provider, model, "GetThingInput$__annotations__")); + assertEquals("__value___", memberName(provider, model, "GetThingInput$__value")); + } + + @Test + public void testEnumSunderNamesAreEscaped() { + Model model = loadModel(""" + $version: "2" + namespace smithy.example + + service TestService { + version: "2024-01-01" + } + + enum StringValues { + _VALUE_ = "value" + __DUNDER__ = "dunder" + } + + intEnum IntValues { + _VALUE_ = 1 + __DUNDER__ = 2 + } + """); + PythonSymbolProvider provider = createProvider(model); + + assertEquals("_VALUE__", memberName(provider, model, "StringValues$_VALUE_")); + assertEquals("_VALUE__", memberName(provider, model, "IntValues$_VALUE_")); + assertEquals("__DUNDER___", memberName(provider, model, "StringValues$__DUNDER__")); + assertEquals("__DUNDER___", memberName(provider, model, "IntValues$__DUNDER__")); + } + private static final String RESPONSE_METADATA_MODEL = """ $version: "2" namespace smithy.example diff --git a/designs/codegen/symbols.md b/designs/codegen/symbols.md index 33daae01d..4c202e834 100644 --- a/designs/codegen/symbols.md +++ b/designs/codegen/symbols.md @@ -194,8 +194,9 @@ collections, such as two lists containing each other, raises `ModelError` with the IDs in the cycle. A failed lookup does not affect later lookups. Streaming blobs resolve to `smithy_core.aio.interfaces.StreamingBlob`, including -when nested in collections. Event-stream unions retain schema names for -protocol integration and do not produce model declarations. +when nested in collections. Event-stream unions produce the same model +declarations, variant classes, unknown class, and type references as other +unions. Their streaming behavior is handled by the client and protocol layers. ## Inputs and errors diff --git a/packages/smithy-core/.changes/next-release/smithy-core-enhancement-model-target-names.json b/packages/smithy-core/.changes/next-release/smithy-core-enhancement-model-target-names.json new file mode 100644 index 000000000..723b0ddcb --- /dev/null +++ b/packages/smithy-core/.changes/next-release/smithy-core-enhancement-model-target-names.json @@ -0,0 +1,4 @@ +{ + "type": "enhancement", + "description": "Added optional full shape ID target mappings to `ModelRegistry` so generators can resolve renamed and same-named cross-namespace model targets without ambiguous local-name lookups." +} diff --git a/packages/smithy-core/src/smithy_core/_dynamic_models.py b/packages/smithy-core/src/smithy_core/_dynamic_models.py index 028323346..446c3fdc7 100644 --- a/packages/smithy-core/src/smithy_core/_dynamic_models.py +++ b/packages/smithy-core/src/smithy_core/_dynamic_models.py @@ -18,7 +18,7 @@ import datetime from base64 import b64decode -from collections.abc import Collection +from collections.abc import Collection, Mapping from dataclasses import field as _field from dataclasses import make_dataclass from decimal import Decimal @@ -30,7 +30,7 @@ from ._model_naming import member_field_name, union_unknown_name, variant_class_name from ._schema_compact import hydrate from .deserializers import ShapeDeserializer -from .exceptions import ModeledError +from .exceptions import ModeledError, SerializationError from .schemas import Schema, SchemaExtension from .serializers import ShapeSerializer from .shapes import ShapeID, ShapeType @@ -151,14 +151,15 @@ class ModelRegistry: def __init__( self, schemas_module: ModuleType, - renames: dict[str, str] | None = None, + renames: Mapping[str, str] | None = None, *, - index: dict[str, str] | None = None, + index: Mapping[str, str] | None = None, + target_names: Mapping[str, str] | None = None, shape_names: Collection[str] | None = None, service_error: str | None = None, ) -> None: self._schemas = schemas_module - self._renames = renames or {} + self._renames = dict(renames or {}) self._classes: dict[str, type] = {} self._union_schemas: dict[str, Schema] = {} # Reentrant because build() recurses: unions build their variants, and a @@ -169,13 +170,22 @@ def __init__( # A codegen-supplied index (name -> schema-const, variant/unknown names already # disambiguated) lets the registry resolve a build() without hydrating every # schema in the module just to map names; the scanning fallback hydrates all. - self._index: dict[str, str] | None = index + self._index = dict(index) if index is not None else None + # Full shape ID -> generated public model name. Full IDs keep member-target + # resolution unambiguous when service renames distinguish same-named shapes + # from different namespaces. + self._target_names = dict(target_names or {}) + self._renamed_schemas = { + schema_name: model_name for model_name, schema_name in self._renames.items() + } self._shape_names: set[str] = set(shape_names) if shape_names else set() # Generated clients name their per-service error base (a ModeledError subclass # emitted into the models module). Modeled errors extend it so `except # ServiceError` catches them; a hand-built registry omits it and falls back to # ModeledError. self._service_error_name = service_error + if service_error is not None: + self._shape_names.add(service_error) self._codec = SchemaExtension(self._resolve_member_codec) self._field_names_ext = SchemaExtension(self._compute_field_names) @@ -230,7 +240,7 @@ def build(self, name: str) -> type: # Event-stream union deserializer helper: _Deserializer().deserialize(d). if name.startswith("_") and name.endswith("Deserializer"): union_name = name[1 : -len("Deserializer")] - if union_name in index: + if union_name in index or union_name in self._renames: built = self._build_union_deserializer(union_name) self._classes[name] = built return built @@ -253,14 +263,26 @@ def build(self, name: str) -> type: return built raise AttributeError(name) schema = hydrate(getattr(self._schemas, const)) + canonical = self._renamed_schemas.get(const) if schema.shape_type == ShapeType.UNION: - self._build_union(schema.id.name, schema) - return self._classes[name] + self._build_union(canonical or schema.id.name, schema) + built = self._classes.get(name) + if built is None and name == schema.id.name: + built = self._classes[canonical or schema.id.name] + if built is None: + raise AttributeError(name) + self._classes[name] = built + return built + canonical = canonical or name + if cached := self._classes.get(canonical): + self._classes[name] = cached + return cached if schema.shape_type in (ShapeType.ENUM, ShapeType.INT_ENUM): - built = self._build_enum(name, schema) + built = self._build_enum(canonical, schema) else: - built = self._build_struct(name, schema) + built = self._build_struct(canonical, schema) built.__module__ = module + self._classes[canonical] = built self._classes[name] = built return built @@ -279,6 +301,8 @@ def serialize_members( ) -> None: field_names = self._field_names(schema) for i, member in enumerate(schema.members_by_index): + if self._is_event_stream_member(member): + continue value = getattr(instance, field_names[i]) if value is None: continue @@ -294,8 +318,11 @@ def deserialize_kwargs( def consume(member_schema: Schema, de: ShapeDeserializer) -> None: idx = member_schema.expect_member_index() - _, reader = by_index[idx].get_extension(self._codec) - kwargs[field_names[idx]] = reader(de, by_index[idx]) + member = by_index[idx] + if self._is_event_stream_member(member): + return + _, reader = member.get_extension(self._codec) + kwargs[field_names[idx]] = reader(de, member) deserializer.read_struct(schema, consumer=consume) if not self._is_input(schema): @@ -311,6 +338,8 @@ def _error_correct( ) -> None: """Fill @required members a non-compliant server omitted (Smithy client error correction).""" for i, member in enumerate(by_index): + if self._is_event_stream_member(member): + continue fname = field_names[i] if fname in kwargs: continue @@ -323,15 +352,16 @@ def _error_correct( def _correction_default(self, member: Schema) -> Any: st = member.shape_type if st == ShapeType.STRUCTURE: - target = member.expect_member_target().id.name + target = self._target_name(member.expect_member_target()) return self.build(target)._smithy_default() # type: ignore[attr-defined] if st == ShapeType.UNION: - target = member.expect_member_target().id.name - return self.build(union_unknown_name(target, self._shape_names))(tag="") + target = self._target_name(member.expect_member_target()) + return self.build(self._union_unknown_name(target))(tag="") return _CORRECTION_SCALARS[st]() def _build_struct(self, name: str, schema: Schema) -> type: - is_error = schema.get_trait(ErrorTrait) is not None + error_trait = schema.get_trait(ErrorTrait) + is_error = error_trait is not None # Per Smithy @input structure constraints, every member of an input # structure is effectively optional regardless of @required; only # non-input structures make a @required, non-@default member mandatory. @@ -339,11 +369,27 @@ def _build_struct(self, name: str, schema: Schema) -> type: field_names = self._field_names(schema) fields: list[Any] = [] for i, member in enumerate(schema.members_by_index): + if self._is_event_stream_member(member): + continue + if is_error and field_names[i] == "message": + if member.get_trait(SensitiveTrait) is not None: + fields.append( + ( + "message", + str, + _field(default="", kw_only=False, repr=False), + ) + ) + continue spec = self._field_spec(member, input_nullable) if spec is _MANDATORY: fields.append((field_names[i], _py_type(member))) else: fields.append((field_names[i], _py_type(member), spec)) + if error_trait is not None: + fields.append( + ("fault", str | None, _field(default=error_trait.fault.value)) + ) bases: tuple[type, ...] = (self._error_base(),) if is_error else () # Operation outputs carry a non-modeled response_metadata attribute, excluded # from eq and repr. Errors inherit it from ModeledError instead. @@ -456,16 +502,32 @@ def _default_factory(self, member: Schema, trait: DefaultTrait) -> Any: return None def _build_union(self, name: str, schema: Schema) -> None: + if name in self._union_schemas: + return module = self._models_module_name for member in schema.members_by_index: - vname = variant_class_name( - name, member.expect_member_name(), self._shape_names - ) + vname = self._union_variant_name(name, member.expect_member_name()) variant = self._build_variant(vname, schema, member.expect_member_index()) variant.__module__ = module self._classes[vname] = variant - unknown = union_unknown_name(name, self._shape_names) - unknown_cls = make_dataclass(unknown, [("tag", str)]) + unknown = self._union_unknown_name(name) + + def reject_serialization(self: Any, serializer: ShapeSerializer) -> None: + raise SerializationError("Unknown union variants may not be serialized.") + + @classmethod # type: ignore[misc] + def deserialize(cls: type, deserializer: ShapeDeserializer) -> Any: + raise NotImplementedError + + unknown_cls = make_dataclass( + unknown, + [("tag", str)], + namespace={ + "serialize": reject_serialization, + "serialize_members": reject_serialization, + "deserialize": deserialize, + }, + ) unknown_cls.__module__ = module self._classes[unknown] = unknown_cls self._union_schemas[name] = schema @@ -474,6 +536,55 @@ def _build_union(self, name: str, schema: Schema) -> None: marker = make_dataclass(name, []) marker.__module__ = module self._classes[name] = marker + original = schema.id.name + index = self._ensure_index() + const = self._renames.get(name, index.get(name)) + if ( + original != name + and original not in self._renames + and const is not None + and index.get(original) == const + ): + self._classes.setdefault(original, marker) + self._union_schemas.setdefault(original, schema) + for member in schema.members_by_index: + member_name = member.expect_member_name() + canonical_variant = self._union_variant_name(name, member_name) + original_variant = self._union_variant_name(original, member_name) + if ( + original_variant not in self._renames + and index.get(original_variant, const) == const + ): + self._classes.setdefault( + original_variant, self._classes[canonical_variant] + ) + original_unknown = self._union_unknown_name(original) + if ( + original_unknown not in self._renames + and index.get(original_unknown, const) == const + ): + self._classes.setdefault(original_unknown, unknown_cls) + + def _union_variant_name(self, container: str, member: str) -> str: + capitalized = member[:1].upper() + member[1:] + return self._union_synthetic_name(container, capitalized) + + def _union_unknown_name(self, container: str) -> str: + return self._union_synthetic_name(container, "Unknown") + + def _union_synthetic_name(self, container: str, suffix: str) -> str: + base = container + suffix + disambiguated = container + "_" + suffix + index = self._ensure_index() + const = self._renames.get(container, index.get(container)) + if const is not None: + for candidate in (base, disambiguated): + if self._renames.get(candidate, index.get(candidate)) == const: + return candidate + owner = self._renames.get(base, index.get(base)) + if owner is not None and owner != const: + return disambiguated + return disambiguated if base in self._shape_names else base def _build_variant(self, vname: str, union_schema: Schema, index: int) -> type: registry = self @@ -539,15 +650,13 @@ def consume(member_schema: Schema, de: ShapeDeserializer) -> None: member = by_index[member_schema.expect_member_index()] _, reader = member.get_extension(self._codec) variant = self.build( - variant_class_name(name, member.expect_member_name(), self._shape_names) + self._union_variant_name(name, member.expect_member_name()) ) result[0] = variant(value=reader(de, member)) deserializer.read_struct(schema, consumer=consume) if result[0] is None: - result[0] = self.build(union_unknown_name(name, self._shape_names))( - tag="unknown" - ) + result[0] = self.build(self._union_unknown_name(name))(tag="unknown") return result[0] def _is_streaming(self, member: Schema) -> bool: @@ -557,6 +666,16 @@ def _is_streaming(self, member: Schema) -> bool: return False return target.get_trait(StreamingTrait) is not None + def _is_event_stream_member(self, member: Schema) -> bool: + try: + target = member.expect_member_target() + except Exception: + return False + return ( + target.shape_type is ShapeType.UNION + and target.get_trait(StreamingTrait) is not None + ) + def _resolve_member_codec(self, member: Schema) -> tuple[_Writer, _Reader]: st = member.shape_type if st == ShapeType.BLOB and self._is_streaming(member): @@ -566,7 +685,7 @@ def _resolve_member_codec(self, member: Schema) -> tuple[_Writer, _Reader]: # coerce the raw primitive back through the enum class (matching the eager # EnumGenerator's `EnumClass(read_string(...))`). is_int = st == ShapeType.INT_ENUM - enum_cls = self.build(member.expect_member_target().id.name) + enum_cls = self.build(self._target_name(member.expect_member_target())) write_name = "write_integer" if is_int else "write_string" read_name = "read_integer" if is_int else "read_string" @@ -581,7 +700,7 @@ def re(d: ShapeDeserializer, m: Schema) -> Any: if scalar is not None: return scalar if st == ShapeType.STRUCTURE: - target = member.expect_member_target().id.name + target = self._target_name(member.expect_member_target()) def wr(s: ShapeSerializer, m: Schema, v: Any) -> None: s.write_struct(m, v) @@ -597,7 +716,7 @@ def rr(d: ShapeDeserializer, m: Schema) -> Any: return wr, rr if st == ShapeType.UNION: - target = member.expect_member_target().id.name + target = self._target_name(member.expect_member_target()) self.build(target) # ensure union schema is registered def wu(s: ShapeSerializer, m: Schema, v: Any) -> None: @@ -619,6 +738,17 @@ def ru(d: ShapeDeserializer, m: Schema) -> Any: ) raise NotImplementedError(f"no codec for shape type {st}") + def _target_name(self, schema: Schema) -> str: + if generated := self._target_names.get(str(schema.id)): + return generated + index = self._ensure_index() + const = index.get(schema.id.name) + return ( + self._renamed_schemas.get(const, schema.id.name) + if const is not None + else schema.id.name + ) + def _list_codec(self, elem: Schema, sparse: bool) -> tuple[_Writer, _Reader]: def w(s: ShapeSerializer, m: Schema, v: list[Any]) -> None: ew, _ = elem.get_extension(self._codec) diff --git a/packages/smithy-core/src/smithy_core/_model_naming.py b/packages/smithy-core/src/smithy_core/_model_naming.py index 05245e2e8..1fb098e3f 100644 --- a/packages/smithy-core/src/smithy_core/_model_naming.py +++ b/packages/smithy-core/src/smithy_core/_model_naming.py @@ -18,7 +18,8 @@ """and as assert async await break class continue def del elif else except finally for from global if import in is lambda nonlocal not or pass raise return try while with yield bool bytes bytearray dict float int list str - datetime""".split() + datetime serialize serialize_members deserialize __init__ __post_init__ + _smithy_default __repr__ __annotations__""".split() ) # reserved-error-member-names.txt _RESERVED_ERROR_MEMBERS = frozenset( @@ -54,14 +55,24 @@ def member_field_name( return "message" name = to_snake_case(wire_name) - if name in _RESERVED_MEMBERS: + if name.startswith("__") and not name.endswith("__"): + name += "___" + elif name in _RESERVED_MEMBERS: name += "_" if is_error and name in _RESERVED_ERROR_MEMBERS: name += "_" if name == _RESPONSE_METADATA_MEMBER and carries_response_metadata: name += "_" if is_enum: - name = name.upper() + name = _escape_enum_underscores(name).upper() + return name + + +def _escape_enum_underscores(name: str) -> str: + if name.startswith("__"): + return name + "_" if name.endswith("__") and not name.endswith("___") else name + if name.startswith("_") and name.endswith("_") and not name.endswith("__"): + return name + "_" return name diff --git a/packages/smithy-core/tests/unit/test_dynamic_models.py b/packages/smithy-core/tests/unit/test_dynamic_models.py index 60d4363dc..6ca783d3a 100644 --- a/packages/smithy-core/tests/unit/test_dynamic_models.py +++ b/packages/smithy-core/tests/unit/test_dynamic_models.py @@ -3,19 +3,52 @@ # pyright: reportPrivateUsage=false from enum import IntEnum, StrEnum from types import ModuleType -from typing import Any, cast +from typing import Any, Protocol, cast +import pytest from smithy_core import _schema_compact from smithy_core._dynamic_models import ModelRegistry from smithy_core._schema_compact import StaticSchema from smithy_core.deserializers import ShapeDeserializer -from smithy_core.exceptions import ModeledError +from smithy_core.exceptions import ModeledError, SerializationError +from smithy_core.schemas import Schema +from smithy_core.serializers import ShapeSerializer from smithy_core.shapes import ShapeType from smithy_core.types import UnknownEnumMixin NS = "ns.example" +class _UnknownVariant(Protocol): + def serialize(self, serializer: ShapeSerializer) -> None: ... + + def serialize_members(self, serializer: ShapeSerializer) -> None: ... + + +class _UnknownVariantClass(Protocol): + def __call__(self, *, tag: str) -> _UnknownVariant: ... + + def deserialize(self, deserializer: ShapeDeserializer) -> _UnknownVariant: ... + + +class _ModeledErrorClass(Protocol): + def __call__(self) -> ModeledError: ... + + def deserialize(self, deserializer: ShapeDeserializer) -> ModeledError: ... + + +class _Request(Protocol): + name: str + + def serialize_members(self, serializer: ShapeSerializer) -> None: ... + + +class _RequestClass(Protocol): + def __call__(self, *, name: str) -> _Request: ... + + def deserialize(self, deserializer: ShapeDeserializer) -> _Request: ... + + def _struct(name: str, members: tuple[tuple[str, object], ...]) -> StaticSchema: return ( ShapeType.STRUCTURE.value, @@ -30,12 +63,17 @@ def _struct(name: str, members: tuple[tuple[str, object], ...]) -> StaticSchema: _ERROR_TRAIT = "smithy.api#error" -def _error_struct(name: str, members: tuple[tuple[str, object], ...]) -> StaticSchema: +def _error_struct( + name: str, + members: tuple[tuple[str, object], ...], + *, + fault: str = "client", +) -> StaticSchema: return ( ShapeType.STRUCTURE.value, NS, name, - {_ERROR_TRAIT: "client"}, + {_ERROR_TRAIT: fault}, tuple(m[0] for m in members), tuple((m[1], 0) for m in members), ) @@ -52,6 +90,17 @@ def _union(name: str, members: tuple[tuple[str, object], ...]) -> StaticSchema: ) +def _event_stream(name: str, members: tuple[tuple[str, object], ...]) -> StaticSchema: + return ( + ShapeType.UNION.value, + NS, + name, + {"smithy.api#streaming": {}}, + tuple(m[0] for m in members), + tuple((m[1], 0) for m in members), + ) + + _ENUM_VALUE = "smithy.api#enumValue" @@ -128,6 +177,301 @@ def test_supplied_index_matches_scanning_fallback() -> None: assert _fields(scanned) == _fields(supplied) +def test_event_stream_member_is_not_a_model_field_or_serde_member() -> None: + _schema_compact._cache.clear() + events = _event_stream("Events", (("message", ShapeType.STRING.value),)) + request = _struct( + "Request", + ( + ("name", ShapeType.STRING.value), + ("events", lambda: events), + ), + ) + registry = ModelRegistry( + _module(EVENTS=events, REQUEST=request), + index={ + "Events": "EVENTS", + "EventsMessage": "EVENTS", + "EventsUnknown": "EVENTS", + "Request": "REQUEST", + }, + shape_names={"Events", "Request"}, + ) + request_type = registry.build("Request") + request_cls = cast(_RequestClass, request_type) + assert _fields(request_type) == ["name"] + assert registry.build("EventsMessage").__name__ == "EventsMessage" + + class _Serializer: + def __init__(self) -> None: + self.values: list[object] = [] + + def write_string(self, schema: Schema, value: object) -> None: + self.values.append(value) + + value = request_cls(name="test") + serializer = _Serializer() + value.serialize_members(cast(ShapeSerializer, serializer)) + assert serializer.values == ["test"] + + class _Deserializer: + def read_struct(self, schema: Schema, consumer: Any) -> None: + for member in schema.members_by_index: + consumer(member, self) + + def read_string(self, schema: Schema) -> str: + if schema.id.name == "events": + pytest.fail("event stream member should not be deserialized") + return "test" + + decoded = request_cls.deserialize(cast(ShapeDeserializer, _Deserializer())) + assert decoded.name == "test" + assert not hasattr(decoded, "events") + + +def test_supplied_index_preserves_distinct_same_named_structures() -> None: + first = ( + ShapeType.STRUCTURE.value, + "a.example", + "Thing", + 0, + ("first",), + ((ShapeType.STRING.value, 0),), + ) + second = ( + ShapeType.STRUCTURE.value, + "b.example", + "Thing", + 0, + ("second",), + ((ShapeType.STRING.value, 0),), + ) + registry = ModelRegistry( + _module(FIRST=first, SECOND=second), + index={"FirstThing": "FIRST", "SecondThing": "SECOND"}, + ) + + first_cls = registry.build("FirstThing") + second_cls = registry.build("SecondThing") + assert first_cls is not second_cls + assert first_cls.__name__ == "FirstThing" + assert second_cls.__name__ == "SecondThing" + assert _fields(first_cls) == ["first"] + assert _fields(second_cls) == ["second"] + + +def test_registry_copies_supplied_name_maps() -> None: + renames = {"Alias": "FOO"} + index = {"Alias": "FOO"} + registry = ModelRegistry( + _module(FOO=_struct("Foo", ())), + renames=renames, + index=index, + ) + renames.clear() + index.clear() + + assert registry.build("Alias").__name__ == "Alias" + + +def test_alias_and_canonical_struct_lookups_share_class_identity() -> None: + registry = ModelRegistry( + _module(FOO=_struct("Foo", ())), + renames={"Alias": "FOO"}, + ) + + alias = registry.build("Alias") + assert registry.build("Foo") is alias + assert registry.build("Alias") is alias + + +def test_alias_and_canonical_union_lookups_share_class_identity() -> None: + registry = ModelRegistry( + _module(FOO=_union("Foo", (("value", ShapeType.STRING.value),))), + renames={"Alias": "FOO"}, + ) + + alias = registry.build("Alias") + variant = registry.build("AliasValue") + unknown = registry.build("AliasUnknown") + assert registry.build("Foo") is alias + assert registry.build("Alias") is alias + assert registry.build("AliasValue") is variant + assert registry.build("FooValue") is variant + assert registry.build("FooUnknown") is unknown + + class _De: + def read_struct(self, schema: Schema, consumer: Any) -> None: + consumer(schema.members_by_index[0], self) + + def read_string(self, schema: Schema) -> str: + return "value" + + alias_deserializer = registry.build("_AliasDeserializer")() + original_deserializer = registry.build("_FooDeserializer")() + alias_value = alias_deserializer.deserialize(cast(ShapeDeserializer, _De())) + original_value = original_deserializer.deserialize(cast(ShapeDeserializer, _De())) + assert type(alias_value) is variant + assert type(original_value) is variant + assert alias_value.value == original_value.value == "value" + + +def test_renamed_union_does_not_replace_same_named_shape() -> None: + renamed = ( + ShapeType.UNION.value, + "a.example", + "Thing", + 0, + ("value",), + ((ShapeType.STRING.value, 0),), + ) + other = ( + ShapeType.STRUCTURE.value, + "b.example", + "Thing", + 0, + ("other",), + ((ShapeType.STRING.value, 0),), + ) + registry = ModelRegistry( + _module(A_OTHER=other, Z_RENAMED=renamed), + renames={"FirstThing": "Z_RENAMED"}, + ) + + first = registry.build("FirstThing") + thing = registry.build("Thing") + assert first is not thing + assert _fields(thing) == ["other"] + + +def test_supplied_union_names_override_original_shape_names() -> None: + union = ( + ShapeType.UNION.value, + "a.example", + "Foo", + 0, + ("item",), + ((ShapeType.STRING.value, 0),), + ) + item = ( + ShapeType.STRUCTURE.value, + "b.example", + "OldItem", + 0, + ("item",), + ((ShapeType.STRING.value, 0),), + ) + unknown = ( + ShapeType.STRUCTURE.value, + "b.example", + "OldUnknown", + 0, + ("unknown",), + ((ShapeType.STRING.value, 0),), + ) + registry = ModelRegistry( + _module(FOO=union, ITEM=item, UNKNOWN=unknown), + renames={ + "Choice": "FOO", + "ChoiceItem": "ITEM", + "ChoiceUnknown": "UNKNOWN", + }, + index={ + "Foo": "FOO", + "Choice_Item": "FOO", + "Choice_Unknown": "FOO", + "OldItem": "ITEM", + "OldUnknown": "UNKNOWN", + }, + shape_names={"Foo", "OldItem", "OldUnknown"}, + ) + + item_cls = registry.build("ChoiceItem") + unknown_cls = registry.build("ChoiceUnknown") + registry.build("Choice") + assert registry.build("Choice_Item") is not item_cls + assert registry.build("Choice_Unknown") is not unknown_cls + assert registry.build("ChoiceItem") is item_cls + assert registry.build("ChoiceUnknown") is unknown_cls + + +def test_renamed_union_aliases_do_not_replace_renamed_models() -> None: + registry = ModelRegistry( + _module( + FOO=_union("Foo", (("value", ShapeType.STRING.value),)), + VALUE=_struct("ValueModel", (("value", ShapeType.STRING.value),)), + UNKNOWN=_struct("UnknownModel", (("value", ShapeType.STRING.value),)), + ), + renames={ + "Alias": "FOO", + "FooValue": "VALUE", + "FooUnknown": "UNKNOWN", + }, + ) + + registry.build("Alias") + assert _fields(registry.build("FooValue")) == ["value"] + assert _fields(registry.build("FooUnknown")) == ["value"] + assert registry.build("FooValue") is not registry.build("AliasValue") + assert registry.build("FooUnknown") is not registry.build("AliasUnknown") + + +def test_renamed_union_original_alias_does_not_replace_renamed_model() -> None: + registry = ModelRegistry( + _module( + FOO=_union("Foo", (("value", ShapeType.STRING.value),)), + MODEL=_struct("Model", (("value", ShapeType.STRING.value),)), + ), + renames={"Alias": "FOO", "Foo": "MODEL"}, + ) + + registry.build("Alias") + assert _fields(registry.build("Foo")) == ["value"] + assert registry.build("Foo") is not registry.build("Alias") + + +def test_target_names_resolve_renames_by_full_shape_id() -> None: + _schema_compact._cache.clear() + first = ( + ShapeType.STRUCTURE.value, + "a.example", + "Thing", + 0, + ) + second = ( + ShapeType.STRUCTURE.value, + "b.example", + "Thing", + 0, + ) + container = _struct("Container", (("value", lambda: second),)) + registry = ModelRegistry( + _module(FIRST=first, SECOND=second, CONTAINER=container), + renames={"FirstThing": "FIRST", "SecondThing": "SECOND"}, + index={ + "FirstThing": "FIRST", + "SecondThing": "SECOND", + "Container": "CONTAINER", + }, + target_names={ + "a.example#Thing": "FirstThing", + "b.example#Thing": "SecondThing", + f"{NS}#Container": "Container", + }, + shape_names={"FirstThing", "SecondThing", "Container"}, + ) + schema = _schema_compact.hydrate(container) + member = schema.members_by_index[0] + _, reader = member.get_extension(registry._codec) + + class _De: + def read_struct(self, schema: object, consumer: object) -> None: + pass + + result = reader(cast(ShapeDeserializer, _De()), member) + assert type(result).__name__ == "SecondThing" + + def test_supplied_index_disambiguated_union_variant() -> None: # Union FooWarning.message collides with struct FooWarningMessage; codegen emits # the "_"-disambiguated variant name in the index. The real struct must still @@ -220,6 +564,20 @@ def test_modeled_error_falls_back_to_modeled_error_without_service_error() -> No assert cls.__mro__[1] is ModeledError +def test_modeled_error_uses_modeled_fault() -> None: + _schema_compact._cache.clear() + mod = _module(BOOM=_error_struct("Boom", (), fault="server")) + registry = ModelRegistry(mod, index={"Boom": "BOOM"}, shape_names={"Boom"}) + cls = cast(_ModeledErrorClass, registry.build("Boom")) + + class _De: + def read_struct(self, schema: Schema, consumer: Any) -> None: + pass + + assert cls().fault == "server" + assert cls.deserialize(cast(ShapeDeserializer, _De())).fault == "server" + + def test_enum_member_deserialize_coerces_to_enum_class() -> None: # A member whose target is an enum must deserialize to the enum class, not the # raw wire string (the eager generator wrapped it as EnumClass(read_string(...))). @@ -265,6 +623,22 @@ def test_union_classes_report_models_module() -> None: assert registry.build(name).__module__ == "ns.example.models" +def test_unknown_union_variant_rejects_serde() -> None: + _schema_compact._cache.clear() + mod = _module(FOO=_union("Foo", (("a", ShapeType.STRING.value),))) + registry = ModelRegistry(mod, index={"Foo": "FOO"}, shape_names={"Foo"}) + registry.build("Foo") + unknown = cast(_UnknownVariantClass, registry.build("FooUnknown")) + value = unknown(tag="future") + + with pytest.raises(SerializationError): + value.serialize(cast(ShapeSerializer, object())) + with pytest.raises(SerializationError): + value.serialize_members(cast(ShapeSerializer, object())) + with pytest.raises(NotImplementedError): + unknown.deserialize(cast(ShapeDeserializer, object())) + + def test_concurrent_build_returns_one_class() -> None: import threading diff --git a/packages/smithy-core/tests/unit/test_model_naming.py b/packages/smithy-core/tests/unit/test_model_naming.py index d47d7e495..d0198c905 100644 --- a/packages/smithy-core/tests/unit/test_model_naming.py +++ b/packages/smithy-core/tests/unit/test_model_naming.py @@ -1,6 +1,89 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from smithy_core._model_naming import union_unknown_name, variant_class_name +from smithy_core._model_naming import ( + member_field_name, + to_snake_case, + union_unknown_name, + variant_class_name, +) + + +def test_snake_case_preserves_prefix_for_leading_digit() -> None: + assert to_snake_case("_1") == "_1" + assert to_snake_case("_2HTTPServer") == "_2_http_server" + + +def test_snake_case_preserves_existing_underscores() -> None: + assert to_snake_case("_value") == "_value" + assert to_snake_case("value_") == "value_" + assert to_snake_case("some__value") == "some__value" + + +def test_enum_names_escape_before_uppercasing() -> None: + assert ( + member_field_name( + "CLASS", + is_error=False, + carries_response_metadata=False, + is_enum=True, + ) + == "CLASS_" + ) + + +def test_enum_sunder_names_are_escaped() -> None: + assert ( + member_field_name( + "_VALUE_", + is_error=False, + carries_response_metadata=False, + is_enum=True, + ) + == "_VALUE__" + ) + + +def test_enum_dunder_names_are_escaped() -> None: + assert ( + member_field_name( + "__VALUE__", + is_error=False, + carries_response_metadata=False, + is_enum=True, + ) + == "__VALUE___" + ) + + +def test_runtime_method_names_are_reserved() -> None: + for name in ( + "serialize", + "serializeMembers", + "deserialize", + "__init__", + "__post_init__", + "_smithy_default", + "__repr__", + "__annotations__", + ): + assert member_field_name( + name, + is_error=False, + carries_response_metadata=False, + is_enum=False, + ).endswith("_") + + +def test_double_leading_underscore_names_avoid_name_mangling() -> None: + assert ( + member_field_name( + "__value", + is_error=False, + carries_response_metadata=False, + is_enum=False, + ) + == "__value___" + ) def test_variant_class_name_default() -> None: diff --git a/packages/smithy-python/.changes/next-release/smithy-python-feature-model-generation.json b/packages/smithy-python/.changes/next-release/smithy-python-feature-model-generation.json new file mode 100644 index 000000000..d5c28cdf5 --- /dev/null +++ b/packages/smithy-python/.changes/next-release/smithy-python-feature-model-generation.json @@ -0,0 +1,4 @@ +{ + "type": "feature", + "description": "Added native generation of lazy runtime model modules and complete static model stubs for standalone types packages." +} diff --git a/packages/smithy-python/src/smithy_python/files.py b/packages/smithy-python/src/smithy_python/files.py index 736f71821..79fa17069 100644 --- a/packages/smithy-python/src/smithy_python/files.py +++ b/packages/smithy-python/src/smithy_python/files.py @@ -53,7 +53,11 @@ def python_module(self, path: str, module: str) -> _writer.PythonWriter: raise _exceptions.CodegenError( f"Generated path {path!r} is already a text file" ) - writer = _writer.PythonWriter(module, dependencies=self._dependencies) + writer = _writer.PythonWriter( + module, + dependencies=self._dependencies, + stub=path.endswith(".pyi"), + ) self._files[path] = _PythonFile(module, writer) return writer diff --git a/packages/smithy-python/src/smithy_python/generation.py b/packages/smithy-python/src/smithy_python/generation.py index 3bec23e70..ce75e1e19 100644 --- a/packages/smithy-python/src/smithy_python/generation.py +++ b/packages/smithy-python/src/smithy_python/generation.py @@ -25,6 +25,9 @@ "TypesSettings", ] +_SERVICE_ERROR = "ServiceError" +_UNIT = _model.ShapeId("smithy.api", "Unit") + @dataclass(frozen=True, slots=True) class ArtifactSettings: @@ -47,10 +50,15 @@ class TypesSettings(ArtifactSettings): class ModelPlan: """Derived semantic data used to emit lazy model declarations.""" + shapes: tuple[_model.Shape, ...] declaration_names: Mapping[_model.ShapeId, str] renames: Mapping[str, str] union_variant_names: Mapping[_model.ShapeId, str] union_unknown_names: Mapping[_model.ShapeId, str] + registry_index: Mapping[str, str] + target_names: Mapping[str, str] + shape_names: frozenset[str] + includes_unit: bool @classmethod def from_shapes( @@ -60,31 +68,53 @@ def from_shapes( schema_names: Mapping[_model.ShapeId, str], ) -> ModelPlan: """Plan declaration and union names in stable shape order.""" + model_shapes: list[_model.Shape] = [] declarations: dict[_model.ShapeId, str] = {} variants: dict[_model.ShapeId, str] = {} unknowns: dict[_model.ShapeId, str] = {} + index: dict[str, str] = {} + renames: dict[str, str] = {} + target_names: dict[str, str] = {} + includes_unit = any( + shape.type not in (_model.ShapeType.ENUM, _model.ShapeType.INT_ENUM) + and any(member.target == _UNIT for member in shape.members.values()) + for shape in shapes + ) for shape in shapes: - if shape.type not in _model.DECLARATION_TYPES or shape.has_trait( - "streaming" - ): + if shape.type not in _model.DECLARATION_TYPES: continue - declarations[shape.id] = symbols.declaration_name(shape.id) + model_shapes.append(shape) + declaration = symbols.declaration_name(shape.id) + schema = schema_names[shape.id] + declarations[shape.id] = declaration + index[declaration] = schema + if declaration != shape.id.name: + renames[declaration] = schema + target_names[str(shape.id)] = declaration if shape.type is _model.ShapeType.UNION: for member in shape.members.values(): - variants[member.id] = symbols.union_variant_name(member.id) - unknowns[shape.id] = symbols.union_unknown_name(shape.id) + variant = symbols.union_variant_name(member.id) + variants[member.id] = variant + index[variant] = schema + unknown = symbols.union_unknown_name(shape.id) + unknowns[shape.id] = unknown + index[unknown] = schema return cls( + shapes=tuple(model_shapes), declaration_names=MappingProxyType(declarations), - renames=MappingProxyType( - { - declaration: schema_names[shape_id] - for shape_id, declaration in declarations.items() - if declaration != shape_id.name - } - ), + renames=MappingProxyType(renames), union_variant_names=MappingProxyType(variants), union_unknown_names=MappingProxyType(unknowns), + registry_index=MappingProxyType(index), + target_names=MappingProxyType(target_names), + shape_names=frozenset( + ( + *declarations.values(), + *(("Unit",) if includes_unit else ()), + ) + ), + includes_unit=includes_unit, ) diff --git a/packages/smithy-python/src/smithy_python/models.py b/packages/smithy-python/src/smithy_python/models.py new file mode 100644 index 000000000..0e02263a0 --- /dev/null +++ b/packages/smithy-python/src/smithy_python/models.py @@ -0,0 +1,465 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Emission of lazy runtime models and their static typing surface.""" + +from __future__ import annotations + +from collections.abc import Callable, Collection, Mapping +from dataclasses import replace +from json import dumps +from textwrap import wrap +from typing import TYPE_CHECKING, Final + +from . import dependencies as _dependencies +from . import exceptions as _exceptions +from . import model as _model +from . import symbols as _symbols + +if TYPE_CHECKING: + from .generation import GenerationContext, ModelPlan, TypesSettings + from .writer import PythonWriter + +__all__ = ["ModelGenerator"] + + +class ModelGenerator: + """Emit lazy runtime models and the complete static model stub.""" + + def generate(self, context: GenerationContext[TypesSettings]) -> None: + """Register ``models.py`` and ``models.pyi`` with the file manifest.""" + _validate_member_names(context) + package = context.settings.package + directory = package.replace(".", "/") + _write_runtime( + context.files.python_module( + f"src/{directory}/models.py", f"{package}.models" + ), + context.plan.models, + ) + _StubEmitter( + context.files.python_module( + f"src/{directory}/models.pyi", f"{package}.models" + ), + context, + ).write() + + +def _core_reference(name: str, module: str) -> _symbols.TypeReference: + return _symbols.TypeReference( + name, f"smithy_core.{module}", dependencies=(_dependencies.SMITHY_CORE,) + ) + + +_MODEL_REGISTRY: Final = _core_reference("ModelRegistry", "_dynamic_models") +_MODELED_ERROR: Final = _core_reference("ModeledError", "exceptions") +_SHAPE_SERIALIZER: Final = _core_reference("ShapeSerializer", "serializers") +_SHAPE_DESERIALIZER: Final = _core_reference("ShapeDeserializer", "deserializers") +_UNKNOWN_ENUM_MIXIN: Final = _core_reference("UnknownEnumMixin", "types") +_RESPONSE_METADATA: Final = _core_reference("ResponseMetadata", "response") +_DATACLASS: Final = _symbols.TypeReference("dataclass", "dataclasses") +_SELF: Final = _symbols.TypeReference("Self", "typing") +_STR_ENUM: Final = _symbols.TypeReference("StrEnum", "enum") +_INT_ENUM: Final = _symbols.TypeReference("IntEnum", "enum") +_SCHEMAS: Final = _symbols.TypeReference("schemas", "._private") + +_ERROR_MESSAGE_NAMES: Final = frozenset({"errormessage", "error_message", "message"}) +_DOCSTRING_WIDTH: Final = 72 +_MODEL_MEMBER_NAMES: Final = frozenset( + { + "bool", + "bytes", + "bytearray", + "dict", + "float", + "int", + "list", + "str", + "datetime", + "serialize", + "serialize_members", + "deserialize", + "__init__", + "__post_init__", + "_smithy_default", + "__repr__", + "__annotations__", + } +) +_ERROR_MEMBER_NAMES: Final = frozenset( + {"is_retry_safe", "retry_after", "is_throttling_error", "fault"} +) +_RESPONSE_METADATA_DOC: Final = ( + "Metadata about the response that produced this output.\n\n" + "Use this to recover the request identifiers a service's support team needs " + "in order to investigate a call. Members of the metadata are individually " + "optional." +) +_SERVICE_ERROR_DOC: Final = ( + "Base error for all errors in the service.\n\n" + "Some exceptions do not extend from this class, including synthetic, implicit, " + "and shared exception types." +) +_UNKNOWN_UNION_DOC: Final = ( + "Represents an unknown variant.\n\n" + "If you receive this value, you will need to update your library to receive " + "the parsed value.\n\n" + "This value may not be deliberately sent." +) + + +def _write_runtime(writer: PythonWriter, plan: ModelPlan) -> None: + exports = _model_exports(plan) + writer.line("class ServiceError(", _MODELED_ERROR, "):") + with writer.indent(): + _write_docstring(writer, _SERVICE_ERROR_DOC) + writer.line() + writer.line() + _write_mapping(writer, "_MODEL_INDEX", plan.registry_index) + writer.line() + _write_mapping(writer, "_RENAMES", plan.renames) + writer.line() + _write_mapping(writer, "_TARGET_NAMES", plan.target_names) + writer.line() + additional = exports[len(plan.registry_index) + 1 :] + if additional: + writer.line( + "_MODEL_NAMES = (*tuple(_MODEL_INDEX), ", + ", ".join(map(_quote, additional)), + ")", + ) + else: + writer.line("_MODEL_NAMES = tuple(_MODEL_INDEX)") + writer.line("__all__ = (", _quote(exports[0]), ", *_MODEL_NAMES)") + writer.line() + writer.line() + writer.line("_registry = ", _MODEL_REGISTRY, "(") + with writer.indent(): + writer.line(_SCHEMAS, ",") + writer.line("renames=_RENAMES,") + writer.line("index=_MODEL_INDEX,") + writer.line("target_names=_TARGET_NAMES,") + _write_string_set(writer, "shape_names", plan.shape_names, suffix=",") + writer.line('service_error="ServiceError",') + writer.line(")") + writer.line("del ", _SCHEMAS, ", ", _MODEL_REGISTRY, ", ", _MODELED_ERROR) + writer.line() + writer.line() + writer.line("def __getattr__(name: str):") + with writer.indent(): + writer.line("value = _registry.build(name)") + writer.line("globals()[name] = value") + writer.line("return value") + writer.line() + writer.line() + writer.line("def __dir__() -> list[str]:") + with writer.indent(): + writer.line("return sorted({*globals(), *__all__})") + + +class _StubEmitter: + def __init__( + self, + writer: PythonWriter, + context: GenerationContext[TypesSettings], + ) -> None: + self.writer = writer + self.model = context.model + self.plan = context.plan.models + self.symbols = context.symbols + self.module = f"{context.settings.package}.models" + operations = tuple(context.model.iter_shapes(_model.OperationShape)) + self.inputs = {operation.input for operation in operations} | { + shape.id for shape in self.plan.shapes if shape.has_trait("input") + } + self.outputs = {operation.output for operation in operations} | { + shape.id for shape in self.plan.shapes if shape.has_trait("output") + } + + def write(self) -> None: + writer = self.writer + writer.line("__all__ = [") + with writer.indent(): + for name in sorted(_model_exports(self.plan)): + writer.line(_quote(name), ",") + writer.line("]") + writer.line() + writer.line("class _SerdeShape:") + with writer.indent(): + writer.line( + "def serialize(self, serializer: ", + _SHAPE_SERIALIZER, + ") -> None: ...", + ) + writer.line( + "def serialize_members(self, serializer: ", + _SHAPE_SERIALIZER, + ") -> None: ...", + ) + writer.line("@classmethod") + writer.line( + "def deserialize(cls, deserializer: ", + _SHAPE_DESERIALIZER, + ") -> ", + _SELF, + ": ...", + ) + writer.line() + writer.line("class ServiceError(", _MODELED_ERROR, "):") + with writer.indent(): + _write_docstring(writer, _SERVICE_ERROR_DOC) + + for shape in self.plan.shapes: + writer.line() + match shape.type: + case _model.ShapeType.STRUCTURE: + self._write_structure(shape) + case _model.ShapeType.UNION: + self._write_union(shape) + case _model.ShapeType.ENUM | _model.ShapeType.INT_ENUM: + self._write_enum(shape) + case _: + pass + if self.plan.includes_unit: + writer.line() + writer.line("@", _DATACLASS, "(kw_only=True)") + writer.line("class Unit(_SerdeShape):") + with writer.indent(): + writer.line("pass") + + def _write_structure(self, shape: _model.Shape) -> None: + writer = self.writer + name = self.plan.declaration_names[shape.id] + error = shape.has_trait("error") + writer.line("@", _DATACLASS, "(kw_only=True)") + writer.line( + "class ", + name, + "(ServiceError, _SerdeShape):" if error else "(_SerdeShape):", + ) + with writer.indent(): + members = tuple( + member + for member in shape.members.values() + if not _is_event_stream_member(self.model, member) + and (not error or member.name.lower() not in _ERROR_MESSAGE_NAMES) + ) + carries_metadata = error or shape.id in self.outputs + _write_docstring( + writer, + _documentation(shape) or f"Dataclass for {shape.id.name} structure.", + ) + if members or carries_metadata: + writer.line() + for member in members: + reference = self.symbols.type_reference(member.target) + has_default = member.has_trait("default") or self.model.get_target( + member + ).has_trait("default") + mandatory = ( + shape.id not in self.inputs + and member.has_trait("required") + and not has_default + ) + if not mandatory: + reference = replace(reference, nullable=True) + writer.line( + self.symbols.member_name(member.id), + ": ", + reference, + "" if mandatory else " = ...", + ) + if docs := _documentation(member): + _write_docstring(writer, docs) + if carries_metadata: + writer.line("response_metadata: ", _RESPONSE_METADATA, " = ...") + _write_docstring(writer, _RESPONSE_METADATA_DOC) + + def _write_union(self, shape: _model.Shape) -> None: + writer = self.writer + variants: list[_symbols.TypeReference] = [] + for member in shape.members.values(): + name = self.plan.union_variant_names[member.id] + variants.append(_symbols.TypeReference(name, self.module)) + writer.line("@", _DATACLASS) + writer.line("class ", name, "(_SerdeShape):") + with writer.indent(): + if docs := _documentation(member): + _write_docstring(writer, docs) + writer.line() + writer.line("value: ", self.symbols.type_reference(member.target)) + writer.line() + + unknown = self.plan.union_unknown_names[shape.id] + variants.append(_symbols.TypeReference(unknown, self.module)) + writer.line("@", _DATACLASS) + writer.line("class ", unknown, "(_SerdeShape):") + with writer.indent(): + _write_docstring(writer, _UNKNOWN_UNION_DOC) + writer.line() + writer.line("tag: str") + writer.line() + + name = self.plan.declaration_names[shape.id] + writer.line(name, " = ", *_join_types(variants)) + if docs := _documentation(shape): + _write_docstring(writer, docs) + writer.line() + writer.line("class _", name, "Deserializer:") + with writer.indent(): + writer.line( + "def deserialize(self, deserializer: ", + _SHAPE_DESERIALIZER, + ") -> ", + _symbols.TypeReference(name, self.module), + ": ...", + ) + + def _write_enum(self, shape: _model.Shape) -> None: + writer = self.writer + name = self.plan.declaration_names[shape.id] + base = _INT_ENUM if shape.type is _model.ShapeType.INT_ENUM else _STR_ENUM + writer.line("class ", name, "(", _UNKNOWN_ENUM_MIXIN, ", ", base, "):") + with writer.indent(): + docs = _documentation(shape) + if docs: + _write_docstring(writer, docs) + if shape.members: + writer.line() + for member in shape.members.values(): + value = member.get_trait("enumValue") + writer.line( + self.symbols.member_name(member.id), + " = ", + _literal(member.name if value is None else value), + ) + if docs := _documentation(member): + _write_docstring(writer, docs) + if not docs and not shape.members: + writer.line("pass") + + +def _model_exports(plan: ModelPlan) -> tuple[str, ...]: + exports = ("ServiceError", *plan.registry_index) + return (*exports, "Unit") if plan.includes_unit else exports + + +def _write_mapping(writer: PythonWriter, name: str, values: Mapping[str, str]) -> None: + if not values: + writer.line(name, " = {}") + return + writer.line(name, " = {") + with writer.indent(): + for key, value in values.items(): + writer.line(_quote(key), ": ", _quote(value), ",") + writer.line("}") + + +def _write_string_set( + writer: PythonWriter, + name: str, + values: Collection[str], + *, + suffix: str = "", +) -> None: + if not values: + writer.line(name, "=frozenset()", suffix) + return + writer.line(name, "=frozenset(") + with writer.indent(): + writer.line("{") + with writer.indent(): + for value in sorted(values): + writer.line(_quote(value), ",") + writer.line("}") + writer.line(")", suffix) + + +def _write_docstring(writer: PythonWriter, value: str) -> None: + lines = [ + wrapped + for line in value.split("\n") + for wrapped in ( + wrap( + line, + width=_DOCSTRING_WIDTH, + break_long_words=False, + break_on_hyphens=False, + ) + or [""] + ) + ] + if len(lines) > 1: + writer.line('"""') + for line in lines: + writer.line(line.replace("\\", "\\\\").replace('"""', '\\"""')) + writer.line('"""') + return + writer.line(_quote(lines[0])) + + +def _literal(value: object) -> str: + return _quote(value) if isinstance(value, str) else repr(value) + + +def _quote(value: str) -> str: + return dumps(value, ensure_ascii=False) + + +def _join_types( + types: Collection[_symbols.TypeReference], +) -> tuple[str | _symbols.TypeReference, ...]: + parts: list[str | _symbols.TypeReference] = [] + for reference in types: + if parts: + parts.append(" | ") + parts.append(reference) + return tuple(parts) + + +def _documentation(shape: _model.Shape) -> str | None: + value = shape.get_trait("documentation") + return value if isinstance(value, str) else None + + +def _is_event_stream_member(model: _model.Model, member: _model.MemberShape) -> bool: + target = model.get_target(member) + return target.type is _model.ShapeType.UNION and target.has_trait("streaming") + + +def _validate_member_names(context: GenerationContext[TypesSettings]) -> None: + reserved = _model_reserved_member_names(context.model) + for shape in context.plan.models.shapes: + names: set[str] = set() + shape_reserved = reserved(shape) + for member in shape.members.values(): + if _is_event_stream_member(context.model, member): + continue + name = context.symbols.member_name(member.id) + if name in shape_reserved: + raise _exceptions.CodegenError( + f"Symbol provider did not reserve model member name {name!r} " + f"for {member.id}" + ) + if name in names: + raise _exceptions.CodegenError( + f"Generated model member name collision {name!r} in {shape.id}" + ) + names.add(name) + + +def _model_reserved_member_names( + model: _model.Model, +) -> Callable[[_model.Shape], Collection[str]]: + outputs = { + operation.output for operation in model.iter_shapes(_model.OperationShape) + } | {shape.id for shape in model.iter_shapes() if shape.has_trait("output")} + + def reserved(shape: _model.Shape) -> Collection[str]: + names = set(_MODEL_MEMBER_NAMES) + if shape.has_trait("error"): + names.update(_ERROR_MEMBER_NAMES) + if shape.has_trait("error") or shape.id in outputs: + names.add("response_metadata") + return names + + return reserved diff --git a/packages/smithy-python/src/smithy_python/selection.py b/packages/smithy-python/src/smithy_python/selection.py index 62779f1b0..6b554be33 100644 --- a/packages/smithy-python/src/smithy_python/selection.py +++ b/packages/smithy-python/src/smithy_python/selection.py @@ -197,11 +197,7 @@ def select_type_shapes( shapes = tuple(shape for shape in model.iter_shapes() if shape.id in closure) _check_names( - tuple( - shape - for shape in shapes - if shape.type in DECLARATION_TYPES and not shape.has_trait("streaming") - ), + tuple(shape for shape in shapes if shape.type in DECLARATION_TYPES), service=service, ) return Selection(service=service, shapes=shapes, excluded_count=excluded_count) diff --git a/packages/smithy-python/src/smithy_python/symbols.py b/packages/smithy-python/src/smithy_python/symbols.py index 1fe1d417e..dbc7df1e4 100644 --- a/packages/smithy-python/src/smithy_python/symbols.py +++ b/packages/smithy-python/src/smithy_python/symbols.py @@ -69,10 +69,17 @@ def __init__( self._union_variant_names: dict[ShapeId, str] = {} self._union_unknown_names: dict[ShapeId, str] = {} self._selected = frozenset(shape.id for shape in selection.shapes) + self._reserved_declarations = _RESERVED_DECLARATIONS + if any( + shape.type not in _ENUMS + and any(member.target == _UNIT for member in shape.members.values()) + for shape in selection.shapes + ): + self._reserved_declarations |= {"Unit"} self._shape_names: frozenset[str] = frozenset() declarations: dict[str, ShapeId] = {} for shape in selection.shapes: - if shape.type not in DECLARATION_TYPES or shape.has_trait("streaming"): + if shape.type not in DECLARATION_TYPES: continue self._check_collision( declarations, self.declaration_name(shape.id), shape.id @@ -84,9 +91,9 @@ def __init__( members: dict[str, ShapeId] = {} for member in shape.members.values(): self._check_collision(members, self.member_name(member.id), member.id) - self._shape_names = frozenset(declarations) + self._shape_names = frozenset((*declarations, *self._reserved_declarations)) for shape in selection.shapes: - if shape.type is not ShapeType.UNION or shape.has_trait("streaming"): + if shape.type is not ShapeType.UNION: continue for member in shape.members.values(): self._check_collision( @@ -106,7 +113,7 @@ def declaration_name(self, shape_id: ShapeId | str) -> str: raise ModelError(f"No generated declaration for {shape.id}") service = self._selection.service name = service.rename.get(shape.id, shape.id.name) if service else shape.id.name - result = _name(name, pascal=True) + result = _name(name, pascal=True, reserved=self._reserved_declarations) if not re.fullmatch(r"[A-Za-z0-9_]+", name) or not result.isidentifier(): raise ModelError( f"Invalid Python declaration name {result!r} for {shape.id}" @@ -211,7 +218,7 @@ def type_reference(self, shape_id: ShapeId | str) -> TypeReference: pending.append((current, True)) pending.extend((target, False) for target in reversed(targets)) elif current == ShapeId("smithy.api", "Unit"): - resolved[current] = TypeReference("None") + resolved[current] = TypeReference("Unit", f"{self._package}.models") elif shape.type in (ShapeType.STRUCTURE, ShapeType.UNION): resolved[current] = TypeReference( self.declaration_name(current), f"{self._package}.models" @@ -237,6 +244,7 @@ def member_name(self, shape_id: ShapeId | str) -> str: return _name( member.name, constant=container.type in _ENUMS, + preserve_underscores=True, reserved=self._reserved_member_names.get(container.id, ()), ) @@ -265,8 +273,6 @@ def _shape(self, shape_id: ShapeId | str) -> Shape: in (ShapeType.SERVICE, ShapeType.OPERATION, ShapeType.RESOURCE) ): raise ModelError(f"No data symbol for {shape.id} ({shape.type})") - if shape.type is ShapeType.UNION and shape.has_trait("streaming"): - raise ModelError(f"Unsupported streaming {shape.type}: {shape.id}") return shape @@ -279,6 +285,8 @@ def _shape(self, shape_id: ShapeId | str) -> Shape: "not or pass raise return try while with yield".split() ) _ENUMS = frozenset({ShapeType.ENUM, ShapeType.INT_ENUM}) +_RESERVED_DECLARATIONS = frozenset({"ServiceError"}) +_UNIT = ShapeId("smithy.api", "Unit") _ACRONYM_BOUNDARY = re.compile(r"([A-Z]+)([A-Z][a-z])") _WORD_BOUNDARY = re.compile(r"([a-z0-9])([A-Z])") @@ -321,6 +329,7 @@ def _name( *, pascal: bool = False, constant: bool = False, + preserve_underscores: bool = False, reserved: Collection[str] = (), ) -> str: if pascal: @@ -328,16 +337,34 @@ def _name( else: value = _ACRONYM_BOUNDARY.sub(r"\1_\2", value) value = _WORD_BOUNDARY.sub(r"\1_\2", value) - result = "_".join(word.lower() for word in value.split("_") if word) - if constant: - result = result.upper() + result = ( + value.lower() + if preserve_underscores + else "_".join(word.lower() for word in value.split("_") if word) + ) if result[:1].isdigit(): result = "_" + result - if result in _KEYWORDS or result in reserved: + if result.startswith("__") and not result.endswith("__"): + result += "___" + elif ( + result in _KEYWORDS + or result in reserved + or (constant and result.upper() in reserved) + ): result += "_" + if constant: + result = _escape_enum_underscores(result).upper() return result +def _escape_enum_underscores(name: str) -> str: + if name.startswith("__"): + return name + "_" if name.endswith("__") and not name.endswith("___") else name + if name.startswith("_") and name.endswith("_") and not name.endswith("__"): + return name + "_" + return name + + def _plan_schema_names(selection: Selection) -> dict[ShapeId, str]: names: dict[ShapeId, str] = {} used: set[str] = set() diff --git a/packages/smithy-python/src/smithy_python/writer.py b/packages/smithy-python/src/smithy_python/writer.py index d91ffdf2e..efcf45e4b 100644 --- a/packages/smithy-python/src/smithy_python/writer.py +++ b/packages/smithy-python/src/smithy_python/writer.py @@ -58,11 +58,16 @@ class PythonWriter: """Write trusted Python lines, retaining type references until rendering.""" def __init__( - self, module: str, *, dependencies: _dependencies.DependencySet | None = None + self, + module: str, + *, + dependencies: _dependencies.DependencySet | None = None, + stub: bool = False, ) -> None: _validate_module(module, relative=False) self._module = _binding(module) self._dependencies = dependencies + self._stub = stub self._depth = 0 self._lines: list[tuple[int, tuple[str | TypeReference, ...]]] = [] @@ -91,9 +96,10 @@ def render(self) -> str: names, imports = self._plan_imports(*self._names()) header = "from __future__ import annotations\n" if imports: - header += "\n" + "\n".join(item.statement() for item in imports) + "\n" + header += "\n" + _render_imports(imports) + "\n" body = self._body(lambda reference: self._annotation(reference, names)) - return header + ("\n\n" + "\n".join(body) + "\n" if body else "") + separator = "\n" if self._stub else "\n\n" + return header + (separator + "\n".join(body) + "\n" if body else "") def _body(self, reference: Callable[[TypeReference], str]) -> list[str]: body: list[str] = [] @@ -139,6 +145,7 @@ def _plan_imports( self, module_names: frozenset[str], shadows: list[frozenset[str]] ) -> tuple[dict[tuple[str | None, str], str], list[_Import]]: identities: set[tuple[str | None, str]] = set() + third_party_roots: set[str] = set() # The local names visible where each identity is referenced. References # in one scope share a set, so keep the distinct sets, not their union. local_names: dict[tuple[str | None, str], set[frozenset[str]]] = {} @@ -153,6 +160,8 @@ def _plan_imports( if ref.module is not None: _validate_identifier(ref.name, "reference name") _validate_module(ref.module, relative=True) + if ref.dependencies and not ref.module.startswith("."): + third_party_roots.add(_module_root(ref.module)) identities.add((ref.module, ref.name)) key = _identity(ref.module, ref.name) local_names.setdefault(key, set()).add(shadow) @@ -195,7 +204,17 @@ def _plan_imports( else name ) names[(module, name)] = alias - imports.append((_Import(module, name, alias), scoped)) + imports.append( + ( + _Import( + module, + name, + alias, + third_party=_module_root(module) in third_party_roots, + ), + scoped, + ) + ) qualified: set[frozenset[str]] = set() for key, (module, name) in representatives.items(): module_binding, name_binding = key @@ -211,7 +230,9 @@ def _plan_imports( elif module_binding == self._module: names[(module, name)] = name if qualified: - imports.append((_Import("builtins", None, "_builtins"), qualified)) + imports.append( + (_Import("builtins", None, "_builtins", third_party=False), qualified) + ) # Check the complete plan, including unaliased imports. No binding gets # priority merely because its reference was encountered first. @@ -273,12 +294,7 @@ class _Import: module: str name: str | None alias: str - - def statement(self) -> str: - if self.name is None: - return f"import {self.module} as {self.alias}" - suffix = f" as {self.alias}" if self.alias != self.name else "" - return f"from {self.module} import {self.name}{suffix}" + third_party: bool @dataclass(eq=False, slots=True) @@ -319,11 +335,50 @@ def _binding(name: str) -> str: return normalize("NFKC", name) +def _render_imports(imports: list[_Import]) -> str: + sections: dict[int, dict[str, list[_Import]]] = {} + for item in imports: + section = 2 if item.module.startswith(".") else 1 if item.third_party else 0 + sections.setdefault(section, {}).setdefault(item.module, []).append(item) + + rendered: list[str] = [] + for section in sorted(sections): + lines: list[str] = [] + for module, items in sections[section].items(): + for item in items: + if item.name is None: + lines.append(f"import {module} as {item.alias}") + named = [item for item in items if item.name is not None] + if named: + lines.extend(_render_from_import(module, named)) + rendered.append("\n".join(lines)) + return "\n\n".join(rendered) + + +def _render_from_import(module: str, imports: list[_Import]) -> list[str]: + names: list[str] = [] + for item in imports: + if item.name is None: + continue + names.append( + item.name if item.alias == item.name else f"{item.name} as {item.alias}" + ) + prefix = f"from {module} import " + joined = ", ".join(names) + if len(prefix) + len(joined) <= 88: + return [prefix + joined] + return [prefix + "(", *(f" {name}," for name in names), ")"] + + def _identity(module: str | None, name: str) -> tuple[str | None, str]: """Return a reference's module and name as Python binds them.""" return (None if module is None else _binding(module), _binding(name)) +def _module_root(module: str) -> str: + return _binding(module.lstrip(".").split(".", 1)[0]) + + def _validate_identifier(value: str, kind: str) -> None: """Require a Python identifier that is not a hard keyword.""" if not value.isidentifier() or keyword.iskeyword(_binding(value)): diff --git a/packages/smithy-python/tests/unit/test_data_selection.py b/packages/smithy-python/tests/unit/test_data_selection.py index 1439e9c8d..fe9c748fb 100644 --- a/packages/smithy-python/tests/unit/test_data_selection.py +++ b/packages/smithy-python/tests/unit/test_data_selection.py @@ -404,7 +404,7 @@ def test_type_selection_checks_only_selected_declaration_names() -> None: select_type_shapes(model) -def test_type_selection_excludes_streaming_unions_from_name_checks() -> None: +def test_type_selection_checks_streaming_union_names() -> None: model = _model( { "a#Events": { @@ -415,12 +415,8 @@ def test_type_selection_excludes_streaming_unions_from_name_checks() -> None: } ) - result = select_type_shapes(model) - - assert result.shapes == ( - model.get_shape("a#Events"), - model.get_shape("b#Events"), - ) + with pytest.raises(ModelError, match="a#Events, b#Events"): + select_type_shapes(model) def test_type_selection_uses_service_renames_for_name_conflicts() -> None: diff --git a/packages/smithy-python/tests/unit/test_generation.py b/packages/smithy-python/tests/unit/test_generation.py index 2b001d0ab..7be37ca29 100644 --- a/packages/smithy-python/tests/unit/test_generation.py +++ b/packages/smithy-python/tests/unit/test_generation.py @@ -116,7 +116,7 @@ def test_generation_plan_orders_schemas_and_plans_names() -> None: assert plan.schemas.strings["example"] == "n0" -def test_generation_plan_keeps_streaming_schemas_out_of_model_declarations() -> None: +def test_generation_plan_includes_streaming_union_models() -> None: model = load_model( { "smithy": "2.0", @@ -124,6 +124,7 @@ def test_generation_plan_keeps_streaming_schemas_out_of_model_declarations() -> "example#Events": { "type": "union", "traits": {"smithy.api#streaming": {}}, + "members": {"message": {"target": "smithy.api#String"}}, } }, } @@ -133,4 +134,9 @@ def test_generation_plan_keeps_streaming_schemas_out_of_model_declarations() -> plan = GenerationPlan.from_selection(model, selection, symbols) shape_id = model.get_shape("example#Events").id assert plan.schemas.names[shape_id] == "EVENTS" - assert shape_id not in plan.models.declaration_names + assert plan.models.declaration_names[shape_id] == "Events" + assert ( + plan.models.union_variant_names[model.get_shape("example#Events$message").id] + == "EventsMessage" + ) + assert plan.models.union_unknown_names[shape_id] == "EventsUnknown" diff --git a/packages/smithy-python/tests/unit/test_models.py b/packages/smithy-python/tests/unit/test_models.py new file mode 100644 index 000000000..2d93ae8d6 --- /dev/null +++ b/packages/smithy-python/tests/unit/test_models.py @@ -0,0 +1,911 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# pyright: reportPrivateUsage=false + +from __future__ import annotations + +import importlib +import json +import pickle +import subprocess +import sys +from collections.abc import Mapping +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Any + +import pytest +from smithy_core._schema_compact import hydrate +from smithy_core.types import UnknownEnumMixin +from smithy_json import JSONCodec +from smithy_python.dependencies import SMITHY_CORE, DependencySet +from smithy_python.files import FileManifest +from smithy_python.generation import GenerationContext, GenerationPlan, TypesSettings +from smithy_python.model import load_model +from smithy_python.models import ModelGenerator, _model_reserved_member_names +from smithy_python.schemas import SchemaGenerator +from smithy_python.selection import select_type_shapes +from smithy_python.symbols import SymbolProvider + +_PACKAGE = "generated_types" + + +def _run_tool(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "-m", *args], + capture_output=True, + text=True, + check=False, + ) + + +def _context( + model_data: Mapping[str, object], + *, + service_id: str | None = "example#Service", + package: str = _PACKAGE, +) -> GenerationContext[TypesSettings]: + model = load_model(model_data) + selection = select_type_shapes( + model, + service_id=service_id, + generate_inputs_and_outputs=True, + ) + settings = TypesSettings(package, "1.0.0", None) + symbols = SymbolProvider( + model, + selection, + package=settings.package, + reserved_member_names=_model_reserved_member_names(model), + ) + dependencies = DependencySet() + return GenerationContext( + model=model, + selection=selection, + settings=settings, + symbols=symbols, + plan=GenerationPlan.from_selection(model, selection, symbols), + dependencies=dependencies, + files=FileManifest(dependencies), + ) + + +def _model() -> dict[str, object]: + return { + "smithy": "2.0", + "shapes": { + "example#Service": { + "type": "service", + "version": "1", + "operations": [{"target": "example#Operation"}], + "rename": { + "example#Original": "Renamed", + "example#Choice": "Selection", + "example#Color": "Tone", + }, + }, + "example#Operation": { + "type": "operation", + "input": {"target": "example#Input"}, + "output": {"target": "example#Output"}, + "errors": [{"target": "example#Problem"}], + }, + "example#Input": { + "type": "structure", + "traits": {"smithy.api#documentation": "Input docs."}, + "members": { + "required": { + "target": "smithy.api#String", + "traits": {"smithy.api#required": {}}, + }, + "list": {"target": "smithy.api#String"}, + "renamed": {"target": "example#Original"}, + "choice": {"target": "example#Choice"}, + "color": {"target": "example#Color"}, + "stream": {"target": "example#Stream"}, + "events": {"target": "example#Events"}, + }, + }, + "example#Output": { + "type": "structure", + "members": { + "responseMetadata": {"target": "smithy.api#String"}, + "code": {"target": "example#Code"}, + }, + }, + "example#Problem": { + "type": "structure", + "traits": { + "smithy.api#error": "client", + "smithy.api#documentation": "Problem docs.", + }, + "members": { + "message": { + "target": "smithy.api#String", + "traits": {"smithy.api#sensitive": {}}, + }, + "fault": {"target": "smithy.api#String"}, + "detail": {"target": "example#Original"}, + }, + }, + "example#Original": { + "type": "structure", + "traits": {"smithy.api#documentation": "Renamed structure."}, + "members": { + "value": { + "target": "smithy.api#String", + "traits": { + "smithy.api#required": {}, + "smithy.api#documentation": "Value docs.", + }, + }, + "count": { + "target": "smithy.api#PrimitiveInteger", + "traits": {"smithy.api#required": {}}, + }, + }, + }, + "example#Choice": { + "type": "union", + "traits": {"smithy.api#documentation": "Choice docs."}, + "members": { + "text": { + "target": "smithy.api#String", + "traits": {"smithy.api#documentation": "Text docs."}, + }, + "item": {"target": "example#Original"}, + }, + }, + "example#Color": { + "type": "enum", + "traits": {"smithy.api#documentation": "Color docs."}, + "members": { + "RED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "red", + "smithy.api#documentation": "Red docs.", + }, + } + }, + }, + "example#Code": { + "type": "intEnum", + "members": { + "OK": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": 200}, + } + }, + }, + "example#Stream": { + "type": "blob", + "traits": {"smithy.api#streaming": {}}, + }, + "example#Events": { + "type": "union", + "traits": {"smithy.api#streaming": {}}, + "members": {"item": {"target": "example#Original"}}, + }, + }, + } + + +def _generate(tmp_path: Path) -> tuple[GenerationContext[TypesSettings], Path]: + context = _context(_model()) + SchemaGenerator().generate(context) + ModelGenerator().generate(context) + context.files.text_file(f"src/{_PACKAGE}/__init__.py", "") + context.files.text_file(f"src/{_PACKAGE}/_private/__init__.py", "") + context.files.commit(tmp_path) + return context, tmp_path / "src" + + +def _generate_custom( + tmp_path: Path, + model: Mapping[str, object], + package: str, +) -> Path: + context = _context(model, service_id=None, package=package) + SchemaGenerator().generate(context) + ModelGenerator().generate(context) + context.files.text_file(f"src/{package}/__init__.py", "") + context.files.text_file(f"src/{package}/_private/__init__.py", "") + context.files.commit(tmp_path) + return tmp_path / "src" + + +def test_model_plan_precomputes_registry_metadata() -> None: + plan = _context(_model()).plan.models + + assert plan.renames == { + "Renamed": "RENAMED", + "Selection": "SELECTION", + "Tone": "TONE", + } + assert "Original" not in plan.registry_index + assert plan.registry_index["SelectionText"] == "SELECTION" + assert plan.registry_index["SelectionUnknown"] == "SELECTION" + assert plan.target_names["example#Original"] == "Renamed" + assert plan.target_names["example#Choice"] == "Selection" + assert plan.target_names["example#Color"] == "Tone" + assert tuple(plan.registry_index) == ( + "Input", + "Output", + "Problem", + "Renamed", + "Selection", + "SelectionText", + "SelectionItem", + "SelectionUnknown", + "Tone", + "Code", + "Events", + "EventsItem", + "EventsUnknown", + ) + assert plan.shape_names == { + "Input", + "Output", + "Problem", + "Renamed", + "Selection", + "Tone", + "Code", + "Events", + } + + +def test_model_generator_emits_runtime_and_stub(tmp_path: Path) -> None: + context, source_root = _generate(tmp_path) + + runtime = (source_root / _PACKAGE / "models.py").read_text() + stub = (source_root / _PACKAGE / "models.pyi").read_text() + assert "from smithy_core._dynamic_models import ModelRegistry" in runtime + service_error_doc = ( + "class ServiceError(ModeledError):\n" + ' """\n' + " Base error for all errors in the service.\n" + "\n" + " Some exceptions do not extend from this class, including synthetic,\n" + " implicit, and shared exception types.\n" + ' """' + ) + assert service_error_doc in runtime + assert '"Renamed": "RENAMED"' in runtime + assert '"Original": "RENAMED"' not in runtime + assert "_MODEL_NAMES = tuple(_MODEL_INDEX)" in runtime + assert '__all__ = ("ServiceError", *_MODEL_NAMES)' in runtime + assert "globals()[name] = value" in runtime + assert "def __dir__() -> list[str]:" in runtime + + assert "class Input(_SerdeShape):" in stub + assert "from smithy_core.aio.interfaces import StreamingBlob" in stub + assert service_error_doc in stub + assert '"Input docs."' in stub + assert "required: str | None = ..." in stub + assert "list_: str | None = ..." in stub + assert "stream: StreamingBlob | None = ..." in stub + assert "events: Events | None = ..." not in stub + assert "class EventsItem(_SerdeShape):" in stub + assert "Events = EventsItem | EventsUnknown" in stub + assert "class _EventsDeserializer:" in stub + assert "class Problem(ServiceError, _SerdeShape):" in stub + assert "message:" not in stub + assert "fault_: str | None = ..." in stub + assert "class Renamed(_SerdeShape):" in stub + assert '"Value docs."' in stub + assert "count: int | None = ..." in stub + assert "class SelectionText(_SerdeShape):" in stub + assert "Selection = SelectionText | SelectionItem | SelectionUnknown" in stub + assert "class Tone(UnknownEnumMixin, StrEnum):" in stub + assert 'RED = "red"' in stub + assert "class Code(UnknownEnumMixin, IntEnum):" in stub + assert "OK = 200" in stub + assert "response_metadata_: str | None = ..." in stub + assert "response_metadata: ResponseMetadata = ..." in stub + assert SMITHY_CORE in context.dependencies + + +def test_generated_models_are_lazy_and_runtime_compatible( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _, source_root = _generate(tmp_path) + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{_PACKAGE}.models") + + assert "Renamed" not in vars(models) + assert "Renamed" in dir(models) + renamed = models.Renamed + assert vars(models)["Renamed"] is renamed + assert models.Renamed is renamed + assert renamed.__module__ == f"{_PACKAGE}.models" + value = renamed(value="hello") + assert value.value == "hello" + assert value.count == 0 + assert pickle.loads(pickle.dumps(value)) == value + + def get_input(_: int) -> type[Any]: + return models.Input + + with ThreadPoolExecutor(max_workers=8) as executor: + inputs = tuple(executor.map(get_input, range(32))) + assert len({id(model) for model in inputs}) == 1 + input_ = inputs[0]( + required="yes", + list_="value", + renamed=value, + choice=models.SelectionText(value="selected"), + color=models.Tone.RED, + ) + assert input_.list_ == "value" + assert input_.renamed is value + + stream = b"stream" + stream_input = models.Input(stream=stream) + assert stream_input.stream is stream + + event = models.EventsItem(value=value) + with pytest.raises(TypeError): + models.Input(events=event) + assert models._EventsDeserializer.__name__ == "_EventsDeserializer" + + schemas = importlib.import_module(f"{_PACKAGE}._private.schemas") + input_schema = hydrate(schemas.INPUT) + assert input_schema.members["events"].expect_member_target().id.name == "Events" + + class StreamSerializer: + value: object | None = None + + def write_data_stream(self, schema: Any, value: object) -> None: + self.value = value + + serializer = StreamSerializer() + stream_input.serialize_members(serializer) + assert serializer.value is stream + + class StreamDeserializer: + def read_struct(self, schema: Any, consumer: Any) -> None: + member = next( + member + for member in schema.members_by_index + if member.expect_member_name() == "stream" + ) + consumer(member, self) + + def read_data_stream(self, schema: Any) -> bytes: + return stream + + decoded_stream = models.Input.deserialize(StreamDeserializer()) + assert decoded_stream.stream is stream + + class EventStreamDeserializer: + def read_struct(self, schema: Any, consumer: Any) -> None: + if schema.id.name != "Input": + pytest.fail("event stream member should not be deserialized") + consumer(schema.members["events"], self) + + decoded_event_stream = models.Input.deserialize(EventStreamDeserializer()) + assert not hasattr(decoded_event_stream, "events") + + decoded = JSONCodec().deserialize( + ( + b'{"required":"yes","renamed":{"value":"hello"},' + b'"choice":{"text":"selected"},"color":"red"}' + ), + inputs[0], + ) + assert type(decoded.renamed) is renamed + assert decoded.renamed.value == "hello" + assert type(decoded.choice) is models.SelectionText + assert decoded.choice.value == "selected" + assert decoded.color is models.Tone.RED + assert json.loads(JSONCodec().serialize(input_)) == { + "required": "yes", + "list": "value", + "renamed": {"value": "hello", "count": 0}, + "choice": {"text": "selected"}, + "color": "red", + } + + assert issubclass(models.Problem, models.ServiceError) + problem = models.Problem("bad", fault_="client", detail=value) + assert problem.message == "bad" + assert problem.fault_ == "client" + assert "bad" not in repr(problem) + assert models.Problem().message == "" + + assert models.Tone.RED == "red" + assert issubclass(models.Tone, UnknownEnumMixin) + assert models.Code.OK == 200 + assert models.SelectionText(value="text").value == "text" + assert models.SelectionItem(value=value).value is value + assert models.SelectionUnknown(tag="future").tag == "future" + + namespace: dict[str, object] = {} + exec(f"from {_PACKAGE}.models import *", namespace) + assert tuple(name for name in models.__all__ if name not in namespace) == () + assert "Original" not in models.__all__ + with pytest.raises(AttributeError): + models.Original + + +def test_generated_model_names_do_not_shadow_runtime_helpers( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + package = "generated_collisions" + source_root = _generate_custom( + tmp_path, + { + "smithy": "2.0", + "shapes": { + f"example#{name}": {"type": "structure"} + for name in ("ServiceError", "ModelRegistry", "ModeledError") + }, + }, + package, + ) + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{package}.models") + + assert models.ServiceError_.__name__ == "ServiceError_" + assert models.ModelRegistry.__name__ == "ModelRegistry" + assert models.ModeledError.__name__ == "ModeledError" + assert models.ServiceError is not models.ServiceError_ + assert {"ServiceError_", "ModelRegistry", "ModeledError"} <= set(models.__all__) + + +def test_explicit_input_and_output_traits_match_runtime_and_stub( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + package = "generated_io_traits" + source_root = _generate_custom( + tmp_path, + { + "smithy": "2.0", + "shapes": { + "example#Input": { + "type": "structure", + "traits": {"smithy.api#input": {}}, + "members": { + "required": { + "target": "smithy.api#String", + "traits": {"smithy.api#required": {}}, + } + }, + }, + "example#Output": { + "type": "structure", + "traits": {"smithy.api#output": {}}, + "members": {"responseMetadata": {"target": "smithy.api#String"}}, + }, + }, + }, + package, + ) + stub = (source_root / package / "models.pyi").read_text() + assert "required: str | None = ..." in stub + assert "response_metadata_: str | None = ..." in stub + assert "response_metadata: ResponseMetadata = ..." in stub + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{package}.models") + assert models.Input().required is None + output = models.Output(response_metadata_="modeled") + assert output.response_metadata_ == "modeled" + assert output.response_metadata is not None + + +def test_stub_and_runtime_use_the_same_member_names( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + package = "generated_member_names" + source_root = _generate_custom( + tmp_path, + { + "smithy": "2.0", + "shapes": { + "example#Leading": { + "type": "structure", + "members": {"_value": {"target": "smithy.api#String"}}, + }, + "example#Trailing": { + "type": "structure", + "members": {"value_": {"target": "smithy.api#String"}}, + }, + "example#Repeated": { + "type": "structure", + "members": {"some__value": {"target": "smithy.api#String"}}, + }, + "example#Digits": { + "type": "structure", + "members": { + "_1": {"target": "smithy.api#String"}, + "_2HTTPServer": {"target": "smithy.api#String"}, + }, + }, + "example#RuntimeMethods": { + "type": "structure", + "members": { + "serialize": {"target": "smithy.api#String"}, + "serializeMembers": {"target": "smithy.api#String"}, + "deserialize": {"target": "smithy.api#String"}, + "__init__": {"target": "smithy.api#String"}, + "__post_init__": {"target": "smithy.api#String"}, + "_smithy_default": {"target": "smithy.api#String"}, + "__repr__": {"target": "smithy.api#String"}, + }, + }, + "example#Keywords": { + "type": "enum", + "members": { + name: { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": name.lower()}, + } + for name in ("LIST", "CLASS") + }, + }, + "example#SunderString": { + "type": "enum", + "members": { + "_VALUE_": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "value"}, + }, + "__DUNDER__": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "dunder"}, + }, + }, + }, + "example#SunderInt": { + "type": "intEnum", + "members": { + "_VALUE_": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": 1}, + }, + "__DUNDER__": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": 2}, + }, + }, + }, + "example#EnumContainer": { + "type": "structure", + "members": { + "stringValue": {"target": "example#SunderString"}, + "intValue": {"target": "example#SunderInt"}, + }, + }, + }, + }, + package, + ) + stub = (source_root / package / "models.pyi").read_text() + assert "_value: str | None = ..." in stub + assert "value_: str | None = ..." in stub + assert "some__value: str | None = ..." in stub + assert "_1: str | None = ..." in stub + assert "_2_http_server: str | None = ..." in stub + assert "serialize_: str | None = ..." in stub + assert "serialize_members_: str | None = ..." in stub + assert "deserialize_: str | None = ..." in stub + assert "__init___: str | None = ..." in stub + assert "__post_init___: str | None = ..." in stub + assert "_smithy_default_: str | None = ..." in stub + assert "__repr___: str | None = ..." in stub + assert 'LIST_ = "list"' in stub + assert 'CLASS_ = "class"' in stub + assert ( + 'class SunderString(UnknownEnumMixin, StrEnum):\n _VALUE__ = "value"' in stub + ) + assert "class SunderInt(UnknownEnumMixin, IntEnum):\n _VALUE__ = 1" in stub + assert '__DUNDER___ = "dunder"' in stub + assert "__DUNDER___ = 2" in stub + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{package}.models") + assert models.Leading(_value="leading")._value == "leading" + assert models.Trailing(value_="trailing").value_ == "trailing" + assert models.Repeated(some__value="repeated").some__value == "repeated" + digits = models.Digits(_1="one", _2_http_server="two") + assert digits._1 == "one" + assert digits._2_http_server == "two" + methods = models.RuntimeMethods( + serialize_="serialize", + serialize_members_="members", + deserialize_="deserialize", + __init___="init", + __post_init___="post_init", + _smithy_default_="default", + __repr___="repr", + ) + assert callable(methods.serialize) + assert callable(methods.serialize_members) + assert callable(methods.deserialize) + assert methods.__init___ == "init" + assert methods.__post_init___ == "post_init" + assert methods._smithy_default_ == "default" + assert methods.__repr___ == "repr" + assert isinstance(methods._smithy_default(), models.RuntimeMethods) + assert "RuntimeMethods(" in repr(methods) + assert json.loads(JSONCodec().serialize(methods)) == { + "serialize": "serialize", + "serializeMembers": "members", + "deserialize": "deserialize", + "__init__": "init", + "__post_init__": "post_init", + "_smithy_default": "default", + "__repr__": "repr", + } + assert models.Keywords.LIST_ == "list" + assert models.Keywords.CLASS_ == "class" + assert models.SunderString._VALUE__ == "value" + assert models.SunderInt._VALUE__ == 1 + assert "__DUNDER___" in models.SunderString.__members__ + assert "__DUNDER___" in models.SunderInt.__members__ + value = models.EnumContainer( + string_value=models.SunderString.__DUNDER___, + int_value=models.SunderInt.__DUNDER___, + ) + encoded = JSONCodec().serialize(value) + decoded = JSONCodec().deserialize(encoded, models.EnumContainer) + assert decoded.string_value is models.SunderString.__DUNDER___ + assert decoded.int_value is models.SunderInt.__DUNDER___ + + +def test_unit_union_members_round_trip_as_empty_structures( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + package = "generated_unit_union" + source_root = _generate_custom( + tmp_path, + { + "smithy": "2.0", + "shapes": { + "example#Action": { + "type": "union", + "members": {"quit": {"target": "smithy.api#Unit"}}, + }, + "example#Container": { + "type": "structure", + "members": {"action": {"target": "example#Action"}}, + }, + }, + }, + package, + ) + runtime = (source_root / package / "models.py").read_text() + stub = (source_root / package / "models.pyi").read_text() + assert '_MODEL_NAMES = (*tuple(_MODEL_INDEX), "Unit")' in runtime + assert "class ActionQuit(_SerdeShape):\n value: Unit" in stub + assert "class Unit(_SerdeShape):" in stub + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{package}.models") + value = models.Container(action=models.ActionQuit(value=models.Unit())) + assert json.loads(JSONCodec().serialize(value)) == {"action": {"quit": {}}} + decoded = JSONCodec().deserialize(b'{"action":{"quit":{}}}', models.Container) + assert isinstance(decoded.action, models.ActionQuit) + assert isinstance(decoded.action.value, models.Unit) + assert "Unit" in models.__all__ + + +def test_modeled_unit_does_not_shadow_prelude_unit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + package = "generated_unit_collision" + source_root = _generate_custom( + tmp_path, + { + "smithy": "2.0", + "shapes": { + "example#Unit": { + "type": "structure", + "members": {"value": {"target": "smithy.api#String"}}, + }, + "example#Action": { + "type": "union", + "members": {"quit": {"target": "smithy.api#Unit"}}, + }, + "example#Container": { + "type": "structure", + "members": {"action": {"target": "example#Action"}}, + }, + }, + }, + package, + ) + stub = (source_root / package / "models.pyi").read_text() + assert "class Unit_(_SerdeShape):" in stub + assert "class Unit(_SerdeShape):" in stub + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{package}.models") + assert models.Unit_ is not models.Unit + decoded = JSONCodec().deserialize( + b'{"action":{"quit":{}}}', + models.Container, + ) + assert isinstance(decoded.action.value, models.Unit) + + +def test_generated_stub_passes_strict_pyright( + tmp_path: Path, +) -> None: + _, source_root = _generate(tmp_path) + (tmp_path / "check.py").write_text( + ( + f"from {_PACKAGE}.models import Input, Problem, Renamed, SelectionText, Tone\n\n" + "renamed = Renamed(value='value')\n" + "Input(required='required', renamed=renamed, " + "choice=SelectionText(value='x'), color=Tone.RED, stream=b'payload')\n" + "problem = Problem('bad')\n" + "message: str = problem.message\n" + ), + encoding="utf-8", + ) + (tmp_path / "pyrightconfig.json").write_text( + json.dumps( + { + "include": ["check.py"], + "extraPaths": [str(source_root)], + "typeCheckingMode": "strict", + } + ), + encoding="utf-8", + ) + result = _run_tool( + "pyright", + "--project", + str(tmp_path / "pyrightconfig.json"), + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_generated_models_pass_ruff(tmp_path: Path) -> None: + _, source_root = _generate(tmp_path) + models = source_root / _PACKAGE / "models.py" + stub = source_root / _PACKAGE / "models.pyi" + for command in (("check",), ("format", "--check")): + result = _run_tool( + "ruff", + *command, + str(models), + str(stub), + "--config", + str(Path(__file__).resolve().parents[4] / "pyproject.toml"), + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_stub_star_exports_and_private_member_names( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + package = "generated_private_names" + source_root = _generate_custom( + tmp_path, + { + "smithy": "2.0", + "shapes": { + "example#_2HTTPServer": { + "type": "structure", + "members": { + "__value": {"target": "smithy.api#String"}, + "__annotations__": {"target": "smithy.api#String"}, + }, + } + }, + }, + package, + ) + stub = (source_root / package / "models.pyi").read_text() + assert '"_2HTTPServer",' in stub + assert "__value___: str | None = ..." in stub + assert "__annotations___: str | None = ..." in stub + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{package}.models") + namespace: dict[str, object] = {} + exec(f"from {package}.models import *", namespace) + assert namespace["_2HTTPServer"] is models._2HTTPServer + value = models._2HTTPServer( + __value___="value", + __annotations___="annotations", + ) + assert value.__value___ == "value" + assert value.__annotations___ == "annotations" + + (tmp_path / "check.py").write_text( + ( + f"from {package}.models import *\n\n" + "value = _2HTTPServer(\n" + " __value___='value',\n" + " __annotations___='annotations',\n" + ")\n" + "member: str | None = value.__value___\n" + ), + encoding="utf-8", + ) + (tmp_path / "pyrightconfig.json").write_text( + json.dumps( + { + "include": ["check.py"], + "extraPaths": [str(source_root)], + "typeCheckingMode": "strict", + } + ), + encoding="utf-8", + ) + result = _run_tool( + "pyright", + "--project", + str(tmp_path / "pyrightconfig.json"), + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_model_output_is_deterministic(tmp_path: Path) -> None: + outputs: list[tuple[str, str]] = [] + for index in range(2): + _, source_root = _generate(tmp_path / str(index)) + package = source_root / _PACKAGE + outputs.append( + ( + (package / "models.py").read_text(), + (package / "models.pyi").read_text(), + ) + ) + assert outputs[0] == outputs[1] + + +def test_model_generation_without_service(tmp_path: Path) -> None: + context = _context( + { + "smithy": "2.0", + "shapes": { + "example#Standalone": { + "type": "structure", + "members": {"value": {"target": "smithy.api#String"}}, + } + }, + }, + service_id=None, + ) + SchemaGenerator().generate(context) + ModelGenerator().generate(context) + context.files.commit(tmp_path) + + runtime = (tmp_path / "src" / _PACKAGE / "models.py").read_text() + stub = (tmp_path / "src" / _PACKAGE / "models.pyi").read_text() + assert '"Standalone": "STANDALONE"' in runtime + assert "class Standalone(_SerdeShape):" in stub + + +def teardown_module() -> None: + for name in tuple(sys.modules): + if name == _PACKAGE or name.startswith(f"{_PACKAGE}."): + del sys.modules[name] diff --git a/packages/smithy-python/tests/unit/test_symbols.py b/packages/smithy-python/tests/unit/test_symbols.py index 4e0c77866..3fb2e4a4a 100644 --- a/packages/smithy-python/tests/unit/test_symbols.py +++ b/packages/smithy-python/tests/unit/test_symbols.py @@ -25,10 +25,13 @@ def model_with(shapes: dict[str, object]) -> Model: ("http_server", "HttpServer", "http_server", "HTTP_SERVER"), ("HTTP2Server", "HTTP2Server", "http2_server", "HTTP2_SERVER"), ("getURL2Value", "GetURL2Value", "get_url2_value", "GET_URL2_VALUE"), - ("__some__name__", "SomeName", "some_name", "SOME_NAME"), - ("__init__", "Init", "init", "INIT"), - ("_value", "Value", "value", "VALUE"), - ("class", "Class", "class_", "CLASS"), + ("__some__name__", "SomeName", "__some__name__", "__SOME__NAME___"), + ("__init__", "Init", "__init__", "__INIT___"), + ("__post_init__", "PostInit", "__post_init__", "__POST_INIT___"), + ("__value", "Value", "__value___", "__VALUE___"), + ("_value", "Value", "_value", "_VALUE"), + ("_VALUE_", "VALUE", "_value_", "_VALUE__"), + ("class", "Class", "class_", "CLASS_"), ("None", "None_", "none", "NONE"), ("match", "Match", "match", "MATCH"), ], @@ -59,6 +62,21 @@ def test_names(source: str, declaration: str, field: str, constant: str) -> None assert model.get_shape(f"example#Choice${source}").get_trait("enumValue") == "wire" +def test_service_error_is_reserved_for_the_synthetic_error_base() -> None: + model = model_with( + { + "example#ServiceError": {"type": "structure"}, + "example#Service": { + "type": "union", + "members": {"error": {"target": "smithy.api#String"}}, + }, + } + ) + symbols = SymbolProvider(model, select_shapes(model), package="example.client") + assert symbols.declaration_name("example#ServiceError") == "ServiceError_" + assert symbols.union_variant_name("example#Service$error") == "Service_Error" + + @pytest.mark.parametrize( ("kind", "name", "module"), [ @@ -99,11 +117,13 @@ def test_type_references(kind: str, name: str, module: str) -> None: if module != "example.client.models" and kind not in ("enum", "intEnum"): prelude_name = kind[0].upper() + kind[1:] assert symbols.type_reference(f"smithy.api#{prelude_name}") == ref - assert symbols.type_reference("smithy.api#Unit") == TypeReference("None") + assert symbols.type_reference("smithy.api#Unit") == TypeReference( + "Unit", "example.client.models" + ) -def test_streaming_union_has_a_schema_name_without_a_model_symbol() -> None: - from smithy_python.model import ModelError +def test_streaming_union_has_schema_and_model_symbols() -> None: + from smithy_python.symbols import TypeReference model = model_with( { @@ -115,8 +135,10 @@ def test_streaming_union_has_a_schema_name_without_a_model_symbol() -> None: ) symbols = SymbolProvider(model, select_shapes(model), package="example") assert symbols.schema_name("example#Events") == "EVENTS" - with pytest.raises(ModelError, match="Unsupported streaming union"): - symbols.declaration_name("example#Events") + assert symbols.declaration_name("example#Events") == "Events" + assert symbols.type_reference("example#Events") == TypeReference( + "Events", "example.models" + ) @pytest.mark.parametrize("kind,value_type", [("enum", "str"), ("intEnum", "int")]) @@ -403,7 +425,8 @@ def test_collection_only_cycles(mutual: bool) -> None: @pytest.mark.parametrize("kind", ["structure", "union", "enum", "intEnum"]) @pytest.mark.parametrize( - "names", [("getURL", "get_url"), ("class", "class_"), ("__init__", "init")] + "names", + [("getURL", "get_url"), ("class", "class_"), ("HTTPServer", "http_server")], ) def test_member_collisions(kind: str, names: tuple[str, str]) -> None: from smithy_python.model import ModelError @@ -420,7 +443,7 @@ def test_member_collisions(kind: str, names: tuple[str, str]) -> None: SymbolProvider(model, select_shapes(model), package="pkg") for name in names: assert f"example#Container${name}" in str(error.value) - result = "CLASS" if kind in ("enum", "intEnum") else "class_" + result = "CLASS_" if kind in ("enum", "intEnum") else "class_" if names[0] == "class": assert repr(result) in str(error.value) @@ -549,8 +572,8 @@ def test_streaming_blob_type_reference() -> None: ) -def test_streaming_union_rejected_when_resolved() -> None: - from smithy_python.model import ModelError +def test_streaming_union_type_reference() -> None: + from smithy_python.symbols import TypeReference model = model_with( { @@ -562,11 +585,12 @@ def test_streaming_union_rejected_when_resolved() -> None: } ) symbols = SymbolProvider(model, select_shapes(model), package="pkg") - for shape_id in ("example#Stream", "example#Streams"): - with pytest.raises(ModelError, match=r"Unsupported streaming.*example#Stream"): - symbols.type_reference(shape_id) - with pytest.raises(ModelError, match="Unsupported streaming"): - symbols.declaration_name("example#Stream") + stream = TypeReference("Stream", "pkg.models") + assert symbols.declaration_name("example#Stream") == "Stream" + assert symbols.type_reference("example#Stream") == stream + assert symbols.type_reference("example#Streams") == TypeReference( + "list", "builtins", (stream,) + ) def test_request_boundaries() -> None: @@ -786,6 +810,54 @@ def test_reserved_member_names_still_collide() -> None: assert "'serialize_'" in str(error.value) +def test_enum_sunder_escaping_still_detects_collisions() -> None: + from smithy_python.model import ModelError + + model = model_with( + { + "example#Values": { + "type": "enum", + "members": { + "_VALUE_": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "first"}, + }, + "_VALUE__": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "second"}, + }, + }, + } + } + ) + with pytest.raises(ModelError, match="_VALUE__"): + SymbolProvider(model, select_shapes(model), package="pkg") + + +def test_enum_dunder_escaping_still_detects_collisions() -> None: + from smithy_python.model import ModelError + + model = model_with( + { + "example#Values": { + "type": "enum", + "members": { + "__VALUE__": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "first"}, + }, + "__VALUE___": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "second"}, + }, + }, + } + } + ) + with pytest.raises(ModelError, match="__VALUE___"): + SymbolProvider(model, select_shapes(model), package="pkg") + + def test_type_references_are_cached() -> None: from smithy_python.model import ModelError diff --git a/packages/smithy-python/tests/unit/test_writer.py b/packages/smithy-python/tests/unit/test_writer.py index 78852e0cc..c7df26470 100644 --- a/packages/smithy-python/tests/unit/test_writer.py +++ b/packages/smithy-python/tests/unit/test_writer.py @@ -13,7 +13,7 @@ from typing import get_type_hints import pytest -from smithy_python.dependencies import Dependency, DependencySet +from smithy_python.dependencies import SMITHY_CORE, Dependency, DependencySet from smithy_python.exceptions import CodegenError from smithy_python.symbols import TypeReference from smithy_python.writer import PythonWriter @@ -220,7 +220,7 @@ def test_class_body_references_load( ")", ) source = writer.render() - assert "from dataclasses import field as _dataclasses_field" in source + assert "field as _dataclasses_field" in source assert "default_factory=_builtins.list" in source module = load_written_module(writer, tmp_path, monkeypatch) node: type = module.Node @@ -600,7 +600,7 @@ def test_builtin_names_cover_running_interpreter() -> None: writer.line("value: ", TypeReference(name, "external")) source = writer.render() for name in names: - assert f"from external import {name} as _external_{name}\n" in source + assert f" {name} as _external_{name},\n" in source @pytest.mark.parametrize("module", ["written_models", "\uff57ritten_models"]) @@ -789,3 +789,28 @@ def test_import_identity_and_order() -> None: assert prefix.index("from datetime") < prefix.index("from decimal") prefixes.append(prefix) assert prefixes[0] == prefixes[1] + + +def test_imports_are_grouped_and_combined() -> None: + writer = PythonWriter("package.models") + writer.line("first: ", TypeReference("StrEnum", "enum")) + writer.line("second: ", TypeReference("IntEnum", "enum")) + writer.line( + "third: ", + TypeReference( + "Document", + "smithy_core.documents", + dependencies=(SMITHY_CORE,), + ), + ) + writer.line("fourth: ", TypeReference("Thing", "._private")) + + assert writer.render().startswith( + "from __future__ import annotations\n" + "\n" + "from enum import IntEnum, StrEnum\n" + "\n" + "from smithy_core.documents import Document\n" + "\n" + "from ._private import Thing\n" + ) diff --git a/packages/smithy-test/src/smithy_test/_private/conformance.py b/packages/smithy-test/src/smithy_test/_private/conformance.py index 89c1067e2..16761d133 100644 --- a/packages/smithy-test/src/smithy_test/_private/conformance.py +++ b/packages/smithy-test/src/smithy_test/_private/conformance.py @@ -20,7 +20,7 @@ from smithy_core._schema_compact import hydrate from smithy_core.shapes import ShapeType -from smithy_core.traits import DefaultTrait, RequiredTrait +from smithy_core.traits import DefaultTrait, RequiredTrait, StreamingTrait _SHAPE_CONSTS = ("STRUCTURE", "UNION", "ENUM", "INT_ENUM") @@ -91,6 +91,8 @@ def _assert_fields_match_schema(name: str, cls: type, schema: Any) -> None: raise ConformanceError(f"{name}: built class is not a dataclass") field_names = {f.name for f in dataclasses.fields(cls)} for member in schema.members_by_index: + if _is_event_stream_member(member): + continue # a runtime field must exist for every modeled member (name derived by the # same ported namer the registry uses) from smithy_core._model_naming import member_field_name @@ -129,6 +131,8 @@ def _build_instance(cls: type, schema: Any, models: ModuleType) -> Any: kwargs: dict[str, Any] = {} is_error = _is_error(schema) for member in schema.members_by_index: + if _is_event_stream_member(member): + continue required = member.get_trait(RequiredTrait) is not None has_default = member.get_trait(DefaultTrait) is not None if not required or has_default or _is_input(schema): @@ -230,3 +234,11 @@ def _is_input(schema: Any) -> bool: from smithy_core.shapes import ShapeID return schema.get_trait(ShapeID("smithy.api#input")) is not None + + +def _is_event_stream_member(member: Any) -> bool: + target = member.expect_member_target() + return ( + target.shape_type is ShapeType.UNION + and target.get_trait(StreamingTrait) is not None + ) diff --git a/packages/smithy-test/tests/unit/test_conformance.py b/packages/smithy-test/tests/unit/test_conformance.py new file mode 100644 index 000000000..5794819bb --- /dev/null +++ b/packages/smithy-test/tests/unit/test_conformance.py @@ -0,0 +1,65 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from dataclasses import dataclass +from pathlib import Path +from types import ModuleType + +from smithy_core._schema_compact import StaticSchema +from smithy_core.shapes import ShapeType +from smithy_test import verify_models + +_REQUIRED: dict[str, object] = {"smithy.api#required": {}} +_STREAMING: dict[str, object] = {"smithy.api#streaming": {}} + + +def test_event_stream_members_are_not_model_fields(tmp_path: Path) -> None: + events: StaticSchema = ( + ShapeType.UNION.value, + "example", + "Events", + _STREAMING, + ("message",), + ((ShapeType.STRING.value, 0),), + ) + input_schema: StaticSchema = ( + ShapeType.STRUCTURE.value, + "example", + "PublishEventsInput", + 0, + ("channel", "events"), + ( + (ShapeType.STRING.value, _REQUIRED), + (lambda: events, _REQUIRED), + ), + ) + schemas = ModuleType("example._private.schemas") + schemas.EVENTS = events # type: ignore[attr-defined] + schemas.PUBLISH_EVENTS_INPUT = input_schema # type: ignore[attr-defined] + + @dataclass + class PublishEventsInput: + channel: str + + models = ModuleType("example.models") + models.PublishEventsInput = PublishEventsInput # type: ignore[attr-defined] + + class _Codec: + def serialize(self, value: PublishEventsInput) -> bytes: + assert value.channel == "x" + return b"" + + def deserialize( + self, body: bytes, cls: type[PublishEventsInput] + ) -> PublishEventsInput: + return cls(channel="x") + + stub = tmp_path / "models.pyi" + stub.write_text( + "from dataclasses import dataclass\n" + "@dataclass\n" + "class PublishEventsInput:\n" + " channel: str\n" + ) + + assert verify_models(models, schemas, codec=_Codec(), pyi_path=str(stub)) == 1 diff --git a/pyproject.toml b/pyproject.toml index 8c2cbe851..cb3a1372e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ lint = [ ] typing = [ "pyright>=1.1.400", + "typing-extensions>=4.14.0", ] changelog = [ "jinja2>=3.1.6" diff --git a/scripts/api-snapshot/api.json b/scripts/api-snapshot/api.json index 5382d5c2c..b84b7ef58 100644 --- a/scripts/api-snapshot/api.json +++ b/scripts/api-snapshot/api.json @@ -1662,6 +1662,9 @@ "load_model": "rex-function", "to_json": "rex-function" }, + "smithy_python.models": { + "ModelGenerator": "class" + }, "smithy_python.schemas": { "SchemaGenerator": "class", "SchemaPlan": "class" diff --git a/scripts/codegen-output-py/generate.py b/scripts/codegen-output-py/generate.py index 54ea35f82..4306ffda4 100644 --- a/scripts/codegen-output-py/generate.py +++ b/scripts/codegen-output-py/generate.py @@ -1,5 +1,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +# pyright: reportPrivateUsage=false """Regenerate the committed Python codegen snapshot.""" import hashlib @@ -12,6 +13,7 @@ from smithy_python.files import FileManifest from smithy_python.generation import GenerationContext, GenerationPlan, TypesSettings from smithy_python.model import load_model +from smithy_python.models import ModelGenerator, _model_reserved_member_names from smithy_python.schemas import SchemaGenerator from smithy_python.selection import select_type_shapes from smithy_python.symbols import SymbolProvider @@ -35,7 +37,12 @@ def main() -> None: package_version="1.0.0", package_description="Example Python codegen output", ) - symbols = SymbolProvider(model, selection, package=settings.package) + symbols = SymbolProvider( + model, + selection, + package=settings.package, + reserved_member_names=_model_reserved_member_names(model), + ) dependencies = DependencySet() context = GenerationContext( model=model, @@ -47,6 +54,7 @@ def main() -> None: files=FileManifest(dependencies), ) SchemaGenerator().generate(context) + ModelGenerator().generate(context) context.files.commit(_OUTPUT) _format_output() @@ -57,7 +65,7 @@ def _format_output() -> None: args = [sys.executable, "-m", "ruff", command] if command == "check": args.append("--fix") - subprocess.run( + subprocess.run( # noqa: S603 [*args, str(_OUTPUT), "--config", str(config)], check=True, ) diff --git a/scripts/codegen-output-py/model.json b/scripts/codegen-output-py/model.json index 6d4110a28..f0c2a2097 100644 --- a/scripts/codegen-output-py/model.json +++ b/scripts/codegen-output-py/model.json @@ -50,6 +50,9 @@ }, "value": { "target": "example#Value" + }, + "events": { + "target": "example#EventStream" } } }, @@ -133,6 +136,17 @@ } } }, + "example#EventStream": { + "type": "union", + "traits": { + "smithy.api#streaming": {} + }, + "members": { + "example": { + "target": "example#Example" + } + } + }, "example#Status": { "type": "enum", "members": { diff --git a/uv.lock b/uv.lock index f07a64d38..4cbf599d5 100644 --- a/uv.lock +++ b/uv.lock @@ -792,6 +792,7 @@ test = [ ] typing = [ { name = "pyright" }, + { name = "typing-extensions" }, ] [package.metadata] @@ -805,7 +806,10 @@ test = [ { name = "pytest-asyncio", specifier = ">=0.25.3" }, { name = "pytest-cov", specifier = ">=6.0.0" }, ] -typing = [{ name = "pyright", specifier = ">=1.1.400" }] +typing = [ + { name = "pyright", specifier = ">=1.1.400" }, + { name = "typing-extensions", specifier = ">=4.14.0" }, +] [[package]] name = "smithy-test" @@ -823,11 +827,11 @@ requires-dist = [{ name = "smithy-core", editable = "packages/smithy-core" }] [[package]] name = "typing-extensions" -version = "4.13.2" +version = "4.16.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f6/37/23083fcd6e35492953e8d2aaaa68b860eb422b34627b13f2ce3eb6106061/typing_extensions-4.13.2.tar.gz", hash = "sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef", size = 106967, upload-time = "2025-04-10T14:19:05.416Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/54/b1ae86c0973cc6f0210b53d508ca3641fb6d0c56823f288d108bc7ab3cc8/typing_extensions-4.13.2-py3-none-any.whl", hash = "sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c", size = 45806, upload-time = "2025-04-10T14:19:03.967Z" }, + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, ] [[package]]