diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80cb8a5d4..f7c543387 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,7 @@ jobs: run: | make build-py - - name: Check native Python codegen output + - name: Check Python codegen output if: ${{ matrix.python-version == '3.12' }} run: | make generate-codegen-output-py diff --git a/codegen-output-py/types/.smithy-python-manifest.json b/codegen-output-py/types/.smithy-python-manifest.json index 9c7015360..cf68a3188 100644 --- a/codegen-output-py/types/.smithy-python-manifest.json +++ b/codegen-output-py/types/.smithy-python-manifest.json @@ -1,6 +1,13 @@ { "files": { - "src/example_types/_private/schemas.py": "d0f35e8979c4aa9d49a1d9c0c8efeb860420bc0ceab3ba3edcc9488c053e9110" + "README.md": "b070865ebb1dc46192ea8a45f0961e1b0f812dfccd069537cfc04311b3327660", + "pyproject.toml": "c419d5bcb7e49329ae50cab3b1a766446d3d14d40a422328788f93e8b2bbeb2b", + "src/example_types/__init__.py": "6d153749d98df8adfe6264fb74d642917cbf2d6922b9211719dee35483dab943", + "src/example_types/_private/__init__.py": "5384bfdb2df380b6557cc7a71d16891415bccaa87699406e236f752c6415389f", + "src/example_types/_private/schemas.py": "d0f35e8979c4aa9d49a1d9c0c8efeb860420bc0ceab3ba3edcc9488c053e9110", + "src/example_types/models.py": "6042e2e6454b40fef929d96cb7608ae8c591ac7205577f14337435718bb7c077", + "src/example_types/models.pyi": "bb3d59e91999a62438fce52fe3be551c63f998565e4f9640e4fb1e275b972b14", + "src/example_types/py.typed": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, "version": 1 } diff --git a/codegen-output-py/types/README.md b/codegen-output-py/types/README.md new file mode 100644 index 000000000..8e6571146 --- /dev/null +++ b/codegen-output-py/types/README.md @@ -0,0 +1,5 @@ + + +## example_types Types Package + +Example Python codegen output diff --git a/codegen-output-py/types/pyproject.toml b/codegen-output-py/types/pyproject.toml new file mode 100644 index 000000000..5df7d40fb --- /dev/null +++ b/codegen-output-py/types/pyproject.toml @@ -0,0 +1,43 @@ +# Code generated by smithy-python-codegen DO NOT EDIT. + +[project] +name = "example_types" +version = "1.0.0" +description = "Example Python codegen output" +readme = "README.md" +requires-python = ">=3.12" +keywords = ["smithy", "example_types"] +license = {text = "Apache-2.0"} +dependencies = [ + "smithy-core~=0.10.0", +] + +[build-system] +requires = [ + "hatchling", +] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = [ + "src/example_types", +] + +[tool.pyright] +typeCheckingMode = "strict" +reportPrivateUsage = false +reportUnusedFunction = false +reportUnusedVariable = false +reportUnnecessaryComparison = false +reportUnusedClass = false +enableExperimentalFeatures = true +exclude = ["**/models.py"] + +[tool.ruff] +target-version = "py312" + +[tool.ruff.lint] +ignore = ["F841"] + +[tool.ruff.format] +skip-magic-trailing-comma = true diff --git a/codegen-output-py/types/src/example_types/__init__.py b/codegen-output-py/types/src/example_types/__init__.py new file mode 100644 index 000000000..f81c022be --- /dev/null +++ b/codegen-output-py/types/src/example_types/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__version__: str = "1.0.0" diff --git a/codegen-output-py/types/src/example_types/_private/__init__.py b/codegen-output-py/types/src/example_types/_private/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/codegen-output-py/types/src/example_types/_private/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/codegen-output-py/types/src/example_types/models.py b/codegen-output-py/types/src/example_types/models.py new file mode 100644 index 000000000..aacb742b3 --- /dev/null +++ b/codegen-output-py/types/src/example_types/models.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +from smithy_core._dynamic_models import ModelRegistry +from smithy_core.exceptions import ModeledError + +from ._private import schemas + + +class ServiceError(ModeledError): + """ + Base error for all errors in the service. + + Some exceptions do not extend from this class, including synthetic, + implicit, and shared exception types. + """ + + +_MODEL_INDEX = { + "GetExampleInput": "GET_EXAMPLE_INPUT", + "GetExampleOutput": "GET_EXAMPLE_OUTPUT", + "ExampleError": "EXAMPLE_ERROR", + "RenamedExample": "RENAMED_EXAMPLE", + "ExampleValue": "EXAMPLE_VALUE", + "ExampleValueText": "EXAMPLE_VALUE", + "ExampleValueNumber": "EXAMPLE_VALUE", + "ExampleValueEmpty": "EXAMPLE_VALUE", + "ExampleValueUnknown": "EXAMPLE_VALUE", + "Status": "STATUS", + "Code": "CODE", +} + +_RENAMES = { + "RenamedExample": "RENAMED_EXAMPLE", + "ExampleValue": "EXAMPLE_VALUE", +} + +_TARGET_NAMES = { + "example#Example": "RenamedExample", + "example#Value": "ExampleValue", +} + +_MODEL_NAMES = (*tuple(_MODEL_INDEX), "Unit") +__all__ = ("ServiceError", *_MODEL_NAMES) + + +_registry = ModelRegistry( + schemas, + renames=_RENAMES, + index=_MODEL_INDEX, + target_names=_TARGET_NAMES, + shape_names=frozenset( + { + "Code", + "ExampleError", + "ExampleValue", + "GetExampleInput", + "GetExampleOutput", + "RenamedExample", + "Status", + "Unit", + } + ), + service_error="ServiceError", +) +del schemas, ModelRegistry, ModeledError + + +def __getattr__(name: str): + value = _registry.build(name) + globals()[name] = value + return value + + +def __dir__() -> list[str]: + return sorted({*globals(), *__all__}) diff --git a/codegen-output-py/types/src/example_types/models.pyi b/codegen-output-py/types/src/example_types/models.pyi new file mode 100644 index 000000000..a4ba94f5e --- /dev/null +++ b/codegen-output-py/types/src/example_types/models.pyi @@ -0,0 +1,135 @@ +from __future__ import annotations + +from dataclasses import dataclass +from enum import IntEnum, StrEnum +from typing import Self + +from smithy_core.deserializers import ShapeDeserializer +from smithy_core.documents import Document +from smithy_core.exceptions import ModeledError +from smithy_core.response import ResponseMetadata +from smithy_core.serializers import ShapeSerializer +from smithy_core.types import UnknownEnumMixin + +__all__ = [ + "Code", + "ExampleError", + "ExampleValue", + "ExampleValueEmpty", + "ExampleValueNumber", + "ExampleValueText", + "ExampleValueUnknown", + "GetExampleInput", + "GetExampleOutput", + "RenamedExample", + "ServiceError", + "Status", + "Unit", +] + +class _SerdeShape: + def serialize(self, serializer: ShapeSerializer) -> None: ... + def serialize_members(self, serializer: ShapeSerializer) -> None: ... + @classmethod + def deserialize(cls, deserializer: ShapeDeserializer) -> Self: ... + +class ServiceError(ModeledError): + """ + Base error for all errors in the service. + + Some exceptions do not extend from this class, including synthetic, + implicit, and shared exception types. + """ + +@dataclass(kw_only=True) +class GetExampleInput(_SerdeShape): + "Input for retrieving an example." + + id: str | None = ... + +@dataclass(kw_only=True) +class GetExampleOutput(_SerdeShape): + "Dataclass for GetExampleOutput structure." + + example: RenamedExample | None = ... + value: ExampleValue | None = ... + response_metadata: ResponseMetadata = ... + """ + Metadata about the response that produced this output. + + Use this to recover the request identifiers a service's support team + needs in order to investigate a call. Members of the metadata are + individually optional. + """ + +@dataclass(kw_only=True) +class ExampleError(ServiceError, _SerdeShape): + "An example modeled error." + + retry_after_seconds: int | None = ... + response_metadata: ResponseMetadata = ... + """ + Metadata about the response that produced this output. + + Use this to recover the request identifiers a service's support team + needs in order to investigate a call. Members of the metadata are + individually optional. + """ + +@dataclass(kw_only=True) +class RenamedExample(_SerdeShape): + "A representative generated structure." + + name: str + "The example name." + count: int | None = ... + tags: list[str] | None = ... + attributes: dict[str, Document] | None = ... + choice: ExampleValue | None = ... + status: str | None = ... + code: int | None = ... + +@dataclass +class ExampleValueText(_SerdeShape): + value: str + +@dataclass +class ExampleValueNumber(_SerdeShape): + value: int + +@dataclass +class ExampleValueEmpty(_SerdeShape): + value: Unit + +@dataclass +class ExampleValueUnknown(_SerdeShape): + """ + Represents an unknown variant. + + If you receive this value, you will need to update your library to + receive the parsed value. + + This value may not be deliberately sent. + """ + + tag: str + +ExampleValue = ( + ExampleValueText | ExampleValueNumber | ExampleValueEmpty | ExampleValueUnknown +) +"A representative generated union." + +class _ExampleValueDeserializer: + def deserialize(self, deserializer: ShapeDeserializer) -> ExampleValue: ... + +class Status(UnknownEnumMixin, StrEnum): + ACTIVE = "active" + INACTIVE = "inactive" + +class Code(UnknownEnumMixin, IntEnum): + OK = 200 + NOT_FOUND = 404 + +@dataclass(kw_only=True) +class Unit(_SerdeShape): + pass diff --git a/codegen-output-py/types/src/example_types/py.typed b/codegen-output-py/types/src/example_types/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSymbolProvider.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSymbolProvider.java index 62b5b12e4..13e5883fc 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSymbolProvider.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSymbolProvider.java @@ -147,6 +147,9 @@ public String toMemberName(MemberShape shape) { } var memberName = escaper.escapeMemberName(CaseUtils.toSnakeCase(shape.getMemberName())); + if (memberName.startsWith("__") && !memberName.endsWith("__")) { + memberName += "___"; + } // Escape words that are only reserved for error members. if (shape.hasTrait(ErrorTrait.class)) { @@ -170,6 +173,16 @@ && carriesResponseMetadata(container)) { } if (container.isEnumShape() || container.isIntEnumShape()) { + if (memberName.startsWith("__") + && memberName.endsWith("__") + && !memberName.endsWith("___")) { + memberName += "_"; + } else if (memberName.startsWith("_") + && !memberName.startsWith("__") + && memberName.endsWith("_") + && !memberName.endsWith("__")) { + memberName += "_"; + } memberName = memberName.toUpperCase(Locale.ENGLISH); } return memberName; diff --git a/codegen/core/src/main/resources/software/amazon/smithy/python/codegen/reserved-member-names.txt b/codegen/core/src/main/resources/software/amazon/smithy/python/codegen/reserved-member-names.txt index 1831fb341..b18e8e1a8 100644 --- a/codegen/core/src/main/resources/software/amazon/smithy/python/codegen/reserved-member-names.txt +++ b/codegen/core/src/main/resources/software/amazon/smithy/python/codegen/reserved-member-names.txt @@ -81,3 +81,17 @@ str # that are likely imported in the generated code (e.g. datetime) # We only escape the types we use. datetime + +# Generated model runtime methods. +serialize +serialize_members +deserialize + +# Dataclass-generated methods and hooks. +__init__ +__post_init__ +__repr__ +__annotations__ + +# Generated model runtime methods. +_smithy_default diff --git a/codegen/core/src/test/java/software/amazon/smithy/python/codegen/PythonSymbolProviderTest.java b/codegen/core/src/test/java/software/amazon/smithy/python/codegen/PythonSymbolProviderTest.java index 0102a560c..3941b92e2 100644 --- a/codegen/core/src/test/java/software/amazon/smithy/python/codegen/PythonSymbolProviderTest.java +++ b/codegen/core/src/test/java/software/amazon/smithy/python/codegen/PythonSymbolProviderTest.java @@ -137,6 +137,68 @@ public void testUnrelatedMembersOnOutputsAndErrorsAreUnaffected() { assertEquals("retry_after", memberName(provider, model, "ThingError$retryAfter")); } + @Test + public void testDataclassHooksAreEscaped() { + Model model = loadModel(""" + $version: "2" + namespace smithy.example + + service TestService { + version: "2024-01-01" + operations: [GetThing] + } + + operation GetThing { + input: GetThingInput + } + + structure GetThingInput { + __init__: String + __post_init__: String + _smithy_default: String + __repr__: String + __annotations__: String + __value: String + } + """); + PythonSymbolProvider provider = createProvider(model); + + assertEquals("__init___", memberName(provider, model, "GetThingInput$__init__")); + assertEquals("__post_init___", memberName(provider, model, "GetThingInput$__post_init__")); + assertEquals("_smithy_default_", memberName(provider, model, "GetThingInput$_smithy_default")); + assertEquals("__repr___", memberName(provider, model, "GetThingInput$__repr__")); + assertEquals("__annotations___", memberName(provider, model, "GetThingInput$__annotations__")); + assertEquals("__value___", memberName(provider, model, "GetThingInput$__value")); + } + + @Test + public void testEnumSunderNamesAreEscaped() { + Model model = loadModel(""" + $version: "2" + namespace smithy.example + + service TestService { + version: "2024-01-01" + } + + enum StringValues { + _VALUE_ = "value" + __DUNDER__ = "dunder" + } + + intEnum IntValues { + _VALUE_ = 1 + __DUNDER__ = 2 + } + """); + PythonSymbolProvider provider = createProvider(model); + + assertEquals("_VALUE__", memberName(provider, model, "StringValues$_VALUE_")); + assertEquals("_VALUE__", memberName(provider, model, "IntValues$_VALUE_")); + assertEquals("__DUNDER___", memberName(provider, model, "StringValues$__DUNDER__")); + assertEquals("__DUNDER___", memberName(provider, model, "IntValues$__DUNDER__")); + } + private static final String RESPONSE_METADATA_MODEL = """ $version: "2" namespace smithy.example diff --git a/designs/codegen/index.md b/designs/codegen/index.md index 5455a37b9..81d968097 100644 --- a/designs/codegen/index.md +++ b/designs/codegen/index.md @@ -65,3 +65,4 @@ behavior of generated packages. * [Python Symbols](symbols.md) * [Python Writer](writer.md) * [Standalone Types Generation](types.md) +* [Code Generation Partitioning](partitioning.md) diff --git a/designs/codegen/partitioning.md b/designs/codegen/partitioning.md new file mode 100644 index 000000000..e4f0c8266 --- /dev/null +++ b/designs/codegen/partitioning.md @@ -0,0 +1,163 @@ +# Code Generation Partitioning + +## Abstract + +Large standalone types packages split compact schemas and model stubs across +private modules. Stable facade modules preserve generated imports while +reducing generator memory and static analysis time. + +## Motivation + +An EC2 types package contains 4,065 selected shapes. Emitting its schemas and +model declarations as two large files makes each file the unit of work for +rendering, staging, parsing, and static analysis. + +`FileManifest` releases each rendered file after staging it, so the largest file +sets the lower bound for source memory during publication. Pyright analyzes the +model stub as one large declaration file. Lazy schema hydration and model +construction reduce runtime work but do not reduce these file-level costs. + +Private partitions bound the work associated with one generated file. Stable +facades preserve generated imports and lazy runtime behavior while tools process +the smaller partition files. + +## Settings + +`TypesSettings` owns both partition targets: + +```python +@dataclass(frozen=True, slots=True) +class TypesSettings(ArtifactSettings): + selector: Selector = field(default_factory=Selector.identity) + generate_inputs_and_outputs: bool = False + schema_partition_size: int | None = 250 + stub_partition_size: int | None = 250 +``` + +The CLI uses these defaults. Python callers may set either target to a positive +integer or `None`. A value of `None` emits that surface as one file. + +Partitioning starts when the number of planned items exceeds its target. A +package with 250 schema shapes and 250 model declarations keeps the single-file +layout. + +## Output Layout + +Partitioned packages retain the stable module paths used by unpartitioned +packages. + +| Surface | Facade | Partitions | +| --- | --- | --- | +| Compact schemas | `._private.schemas` | `._private.schemas_N` | +| Model stubs | `.models.pyi` | `._private.models_N.pyi` | +| Runtime models | `.models` | Single module | + +### Schema File Layout + +```text +src// +├── models.py +└── _private/ + ├── __init__.py + ├── schemas.py + ├── schemas_0.py + ├── schemas_1.py + └── schemas_N.py +``` + +`models.py` imports the `schemas.py` facade. The facade maps schema constants to +partition indexes and loads the matching `schemas_N.py` module on first access. +Each partition contains compact schema tuples. + +### Stub File Layout + +```text +src// +├── models.py +├── models.pyi +└── _private/ + ├── __init__.py + ├── models_0.pyi + ├── models_1.pyi + └── models_N.pyi +``` + +`models.pyi` is the public typing facade. It imports declarations from each +private stub partition. `models.py` remains the runtime module. + +`FileManifest` registers the facade and partition files before publication. +Its ownership rules remove stale, unmodified partition files after a model +shrinks or a target changes. + +Files under `._private` remain private implementation details. + +## Schema Partitions + +`SchemaPlan` provides schema shapes in deterministic emission order and groups +each recursive strongly connected component. A recursive group stays in one +partition so its schemas can use direct local references. + +The schema partitioner: + +1. Packs recursive groups in plan order using the configured shape target. +2. Uses the resulting partition count for the final layout. +3. Estimates the emitted source size of each group. +4. Selects contiguous boundaries that balance estimated source size. + +The estimate includes shape and member identifiers, schema references, and +trait values. It uses plan metadata instead of rendering schemas a second time. +The target determines the partition count. Final partitions may exceed the +target to balance source size, and recursive groups remain indivisible. + +The facade maps each schema constant to a partition index. Its `__getattr__` +imports the selected partition, reads the constant, and caches the result in +the facade module. `__dir__` includes every planned schema constant. + +Partition modules route cross-partition references through the facade. This +keeps dependencies lazy and avoids direct import cycles between partitions. +Hydration remains lazy through `smithy_core._schema_compact`. + +## Stub Partitions + +`ModelPlan` provides model declarations in deterministic order. The stub +partitioner writes contiguous groups up to the configured target. + +The public `models.pyi` facade imports every declaration from its private stub +partition and defines the same `__all__` as `models.py`. Private stubs reference +generated declarations through the public `models` module. This keeps their +type references independent of private partition boundaries. + +Runtime `models.py` remains one lazy registry module. Runtime model construction +and class identity stay unchanged. + +## Compatibility + +Partitioning preserves: + +* Public model module paths and runtime schema lookups. +* Model names, constructor signatures, and type annotations. +* Public model `__all__`, explicit imports, star imports, and `dir()` results. +* Lazy schema hydration and model construction. +* Schema identity, recursive references, and serde behavior. + +Partition indexes follow generation-plan order. Model changes can move +declarations between private files while the public facades remain stable. + +## Validation + +Tests cover packages below and above each target, recursive schema groups, +cross-partition references, facade caching, strict Pyright, Ruff, and +deterministic output. + +The final x86 EC2 benchmark compared unpartitioned output with 250-item schema +and stub partitions: + +| Metric | Unpartitioned | Partitioned | Change | +| --- | ---: | ---: | ---: | +| Generation time | 3.557 s | 3.555 s | -0.1% | +| Peak RSS | 264.5 MiB | 122.4 MiB | -53.7% | +| Peak Python allocation | 222.6 MiB | 84.7 MiB | -61.9% | +| Generated bytes | 4.78 MiB | 5.33 MiB | +11.6% | +| Model import | 50.6 ms | 40.4 ms | -20.2% | +| First model access | 12.3 ms | 22.3 ms | +10.0 ms | +| Strict Pyright | 81.98 s | 15.25 s | -81.4% | diff --git a/designs/codegen/types.md b/designs/codegen/types.md index 9f7135dc3..d91fa8669 100644 --- a/designs/codegen/types.md +++ b/designs/codegen/types.md @@ -100,11 +100,15 @@ class ArtifactSettings: class TypesSettings(ArtifactSettings): selector: Selector = field(default_factory=Selector.identity) generate_inputs_and_outputs: bool = False + schema_partition_size: int | None = 250 + stub_partition_size: int | None = 250 ``` The resolved service remains on `Selection`. It is not duplicated in settings. The CLI subcommand identifies the artifact, so settings do not duplicate that value. Artifact-specific settings extend the shared frozen dataclass. +Partition targets and their output layout are defined in +[Code Generation Partitioning](partitioning.md). The `types` command accepts: @@ -415,26 +419,12 @@ Package setup renders the dependency set collected during generation. ## Large Models -Schema and stub partitioning must not change public imports. The output API and -generation plan allow private content to move behind stable facade modules. +Large standalone types packages partition compact schemas and model stubs +behind stable facade modules. Schema partitions use source-size weighting and +keep recursive components together. Stub partitions use declaration counts. -smithy-java partitions schema classes after a model reaches 500 schema fields -and then targets about 50 KB of generated schema content per class. smithy-php -uses a centralized lazy schema registry. Python can combine those approaches: - -* Keep `._private.schemas` as the stable facade. -* Place compact schema tuples in private partition modules. -* Keep every recursive strongly connected component in one partition. -* Route schema constant access through generated indexes. -* Partition large type stubs behind the public `models.pyi` facade. - -Python partitioning thresholds must be based on Python measurements. Relevant inputs include writer render time, peak generator memory, generated import time, and type -checker time. - -Partitioning bounds the syntax tree analyzed by `PythonWriter` and prevents a -single EC2-sized source file from dominating codegen memory. Measurements -establish the partitioning thresholds. Facade modules preserve generated -imports. +[Code Generation Partitioning](partitioning.md) defines the output layout, +thresholds, and compatibility requirements. ## Performance diff --git a/packages/smithy-core/.changes/next-release/smithy-core-enhancement-model-target-names.json b/packages/smithy-core/.changes/next-release/smithy-core-enhancement-model-target-names.json new file mode 100644 index 000000000..723b0ddcb --- /dev/null +++ b/packages/smithy-core/.changes/next-release/smithy-core-enhancement-model-target-names.json @@ -0,0 +1,4 @@ +{ + "type": "enhancement", + "description": "Added optional full shape ID target mappings to `ModelRegistry` so generators can resolve renamed and same-named cross-namespace model targets without ambiguous local-name lookups." +} diff --git a/packages/smithy-core/src/smithy_core/_dynamic_models.py b/packages/smithy-core/src/smithy_core/_dynamic_models.py index 028323346..1ee10c042 100644 --- a/packages/smithy-core/src/smithy_core/_dynamic_models.py +++ b/packages/smithy-core/src/smithy_core/_dynamic_models.py @@ -18,7 +18,7 @@ import datetime from base64 import b64decode -from collections.abc import Collection +from collections.abc import Collection, Mapping from dataclasses import field as _field from dataclasses import make_dataclass from decimal import Decimal @@ -30,7 +30,7 @@ from ._model_naming import member_field_name, union_unknown_name, variant_class_name from ._schema_compact import hydrate from .deserializers import ShapeDeserializer -from .exceptions import ModeledError +from .exceptions import ModeledError, SerializationError from .schemas import Schema, SchemaExtension from .serializers import ShapeSerializer from .shapes import ShapeID, ShapeType @@ -151,14 +151,15 @@ class ModelRegistry: def __init__( self, schemas_module: ModuleType, - renames: dict[str, str] | None = None, + renames: Mapping[str, str] | None = None, *, - index: dict[str, str] | None = None, + index: Mapping[str, str] | None = None, + target_names: Mapping[str, str] | None = None, shape_names: Collection[str] | None = None, service_error: str | None = None, ) -> None: self._schemas = schemas_module - self._renames = renames or {} + self._renames = dict(renames or {}) self._classes: dict[str, type] = {} self._union_schemas: dict[str, Schema] = {} # Reentrant because build() recurses: unions build their variants, and a @@ -169,13 +170,22 @@ def __init__( # A codegen-supplied index (name -> schema-const, variant/unknown names already # disambiguated) lets the registry resolve a build() without hydrating every # schema in the module just to map names; the scanning fallback hydrates all. - self._index: dict[str, str] | None = index + self._index = dict(index) if index is not None else None + # Full shape ID -> generated public model name. Full IDs keep member-target + # resolution unambiguous when service renames distinguish same-named shapes + # from different namespaces. + self._target_names = dict(target_names or {}) + self._renamed_schemas = { + schema_name: model_name for model_name, schema_name in self._renames.items() + } self._shape_names: set[str] = set(shape_names) if shape_names else set() # Generated clients name their per-service error base (a ModeledError subclass # emitted into the models module). Modeled errors extend it so `except # ServiceError` catches them; a hand-built registry omits it and falls back to # ModeledError. self._service_error_name = service_error + if service_error is not None: + self._shape_names.add(service_error) self._codec = SchemaExtension(self._resolve_member_codec) self._field_names_ext = SchemaExtension(self._compute_field_names) @@ -230,7 +240,7 @@ def build(self, name: str) -> type: # Event-stream union deserializer helper: _Deserializer().deserialize(d). if name.startswith("_") and name.endswith("Deserializer"): union_name = name[1 : -len("Deserializer")] - if union_name in index: + if union_name in index or union_name in self._renames: built = self._build_union_deserializer(union_name) self._classes[name] = built return built @@ -253,14 +263,26 @@ def build(self, name: str) -> type: return built raise AttributeError(name) schema = hydrate(getattr(self._schemas, const)) + canonical = self._renamed_schemas.get(const) if schema.shape_type == ShapeType.UNION: - self._build_union(schema.id.name, schema) - return self._classes[name] + self._build_union(canonical or schema.id.name, schema) + built = self._classes.get(name) + if built is None and name == schema.id.name: + built = self._classes[canonical or schema.id.name] + if built is None: + raise AttributeError(name) + self._classes[name] = built + return built + canonical = canonical or name + if cached := self._classes.get(canonical): + self._classes[name] = cached + return cached if schema.shape_type in (ShapeType.ENUM, ShapeType.INT_ENUM): - built = self._build_enum(name, schema) + built = self._build_enum(canonical, schema) else: - built = self._build_struct(name, schema) + built = self._build_struct(canonical, schema) built.__module__ = module + self._classes[canonical] = built self._classes[name] = built return built @@ -323,15 +345,16 @@ def _error_correct( def _correction_default(self, member: Schema) -> Any: st = member.shape_type if st == ShapeType.STRUCTURE: - target = member.expect_member_target().id.name + target = self._target_name(member.expect_member_target()) return self.build(target)._smithy_default() # type: ignore[attr-defined] if st == ShapeType.UNION: - target = member.expect_member_target().id.name - return self.build(union_unknown_name(target, self._shape_names))(tag="") + target = self._target_name(member.expect_member_target()) + return self.build(self._union_unknown_name(target))(tag="") return _CORRECTION_SCALARS[st]() def _build_struct(self, name: str, schema: Schema) -> type: - is_error = schema.get_trait(ErrorTrait) is not None + error_trait = schema.get_trait(ErrorTrait) + is_error = error_trait is not None # Per Smithy @input structure constraints, every member of an input # structure is effectively optional regardless of @required; only # non-input structures make a @required, non-@default member mandatory. @@ -339,11 +362,25 @@ def _build_struct(self, name: str, schema: Schema) -> type: field_names = self._field_names(schema) fields: list[Any] = [] for i, member in enumerate(schema.members_by_index): + if is_error and field_names[i] == "message": + if member.get_trait(SensitiveTrait) is not None: + fields.append( + ( + "message", + str, + _field(default="", kw_only=False, repr=False), + ) + ) + continue spec = self._field_spec(member, input_nullable) if spec is _MANDATORY: fields.append((field_names[i], _py_type(member))) else: fields.append((field_names[i], _py_type(member), spec)) + if error_trait is not None: + fields.append( + ("fault", str | None, _field(default=error_trait.fault.value)) + ) bases: tuple[type, ...] = (self._error_base(),) if is_error else () # Operation outputs carry a non-modeled response_metadata attribute, excluded # from eq and repr. Errors inherit it from ModeledError instead. @@ -456,16 +493,32 @@ def _default_factory(self, member: Schema, trait: DefaultTrait) -> Any: return None def _build_union(self, name: str, schema: Schema) -> None: + if name in self._union_schemas: + return module = self._models_module_name for member in schema.members_by_index: - vname = variant_class_name( - name, member.expect_member_name(), self._shape_names - ) + vname = self._union_variant_name(name, member.expect_member_name()) variant = self._build_variant(vname, schema, member.expect_member_index()) variant.__module__ = module self._classes[vname] = variant - unknown = union_unknown_name(name, self._shape_names) - unknown_cls = make_dataclass(unknown, [("tag", str)]) + unknown = self._union_unknown_name(name) + + def reject_serialization(self: Any, serializer: ShapeSerializer) -> None: + raise SerializationError("Unknown union variants may not be serialized.") + + @classmethod # type: ignore[misc] + def deserialize(cls: type, deserializer: ShapeDeserializer) -> Any: + raise NotImplementedError + + unknown_cls = make_dataclass( + unknown, + [("tag", str)], + namespace={ + "serialize": reject_serialization, + "serialize_members": reject_serialization, + "deserialize": deserialize, + }, + ) unknown_cls.__module__ = module self._classes[unknown] = unknown_cls self._union_schemas[name] = schema @@ -474,6 +527,55 @@ def _build_union(self, name: str, schema: Schema) -> None: marker = make_dataclass(name, []) marker.__module__ = module self._classes[name] = marker + original = schema.id.name + index = self._ensure_index() + const = self._renames.get(name, index.get(name)) + if ( + original != name + and original not in self._renames + and const is not None + and index.get(original) == const + ): + self._classes.setdefault(original, marker) + self._union_schemas.setdefault(original, schema) + for member in schema.members_by_index: + member_name = member.expect_member_name() + canonical_variant = self._union_variant_name(name, member_name) + original_variant = self._union_variant_name(original, member_name) + if ( + original_variant not in self._renames + and index.get(original_variant, const) == const + ): + self._classes.setdefault( + original_variant, self._classes[canonical_variant] + ) + original_unknown = self._union_unknown_name(original) + if ( + original_unknown not in self._renames + and index.get(original_unknown, const) == const + ): + self._classes.setdefault(original_unknown, unknown_cls) + + def _union_variant_name(self, container: str, member: str) -> str: + capitalized = member[:1].upper() + member[1:] + return self._union_synthetic_name(container, capitalized) + + def _union_unknown_name(self, container: str) -> str: + return self._union_synthetic_name(container, "Unknown") + + def _union_synthetic_name(self, container: str, suffix: str) -> str: + base = container + suffix + disambiguated = container + "_" + suffix + index = self._ensure_index() + const = self._renames.get(container, index.get(container)) + if const is not None: + for candidate in (base, disambiguated): + if self._renames.get(candidate, index.get(candidate)) == const: + return candidate + owner = self._renames.get(base, index.get(base)) + if owner is not None and owner != const: + return disambiguated + return disambiguated if base in self._shape_names else base def _build_variant(self, vname: str, union_schema: Schema, index: int) -> type: registry = self @@ -539,15 +641,13 @@ def consume(member_schema: Schema, de: ShapeDeserializer) -> None: member = by_index[member_schema.expect_member_index()] _, reader = member.get_extension(self._codec) variant = self.build( - variant_class_name(name, member.expect_member_name(), self._shape_names) + self._union_variant_name(name, member.expect_member_name()) ) result[0] = variant(value=reader(de, member)) deserializer.read_struct(schema, consumer=consume) if result[0] is None: - result[0] = self.build(union_unknown_name(name, self._shape_names))( - tag="unknown" - ) + result[0] = self.build(self._union_unknown_name(name))(tag="unknown") return result[0] def _is_streaming(self, member: Schema) -> bool: @@ -566,7 +666,7 @@ def _resolve_member_codec(self, member: Schema) -> tuple[_Writer, _Reader]: # coerce the raw primitive back through the enum class (matching the eager # EnumGenerator's `EnumClass(read_string(...))`). is_int = st == ShapeType.INT_ENUM - enum_cls = self.build(member.expect_member_target().id.name) + enum_cls = self.build(self._target_name(member.expect_member_target())) write_name = "write_integer" if is_int else "write_string" read_name = "read_integer" if is_int else "read_string" @@ -581,7 +681,7 @@ def re(d: ShapeDeserializer, m: Schema) -> Any: if scalar is not None: return scalar if st == ShapeType.STRUCTURE: - target = member.expect_member_target().id.name + target = self._target_name(member.expect_member_target()) def wr(s: ShapeSerializer, m: Schema, v: Any) -> None: s.write_struct(m, v) @@ -597,7 +697,7 @@ def rr(d: ShapeDeserializer, m: Schema) -> Any: return wr, rr if st == ShapeType.UNION: - target = member.expect_member_target().id.name + target = self._target_name(member.expect_member_target()) self.build(target) # ensure union schema is registered def wu(s: ShapeSerializer, m: Schema, v: Any) -> None: @@ -619,6 +719,17 @@ def ru(d: ShapeDeserializer, m: Schema) -> Any: ) raise NotImplementedError(f"no codec for shape type {st}") + def _target_name(self, schema: Schema) -> str: + if generated := self._target_names.get(str(schema.id)): + return generated + index = self._ensure_index() + const = index.get(schema.id.name) + return ( + self._renamed_schemas.get(const, schema.id.name) + if const is not None + else schema.id.name + ) + def _list_codec(self, elem: Schema, sparse: bool) -> tuple[_Writer, _Reader]: def w(s: ShapeSerializer, m: Schema, v: list[Any]) -> None: ew, _ = elem.get_extension(self._codec) diff --git a/packages/smithy-core/src/smithy_core/_model_naming.py b/packages/smithy-core/src/smithy_core/_model_naming.py index 05245e2e8..1fb098e3f 100644 --- a/packages/smithy-core/src/smithy_core/_model_naming.py +++ b/packages/smithy-core/src/smithy_core/_model_naming.py @@ -18,7 +18,8 @@ """and as assert async await break class continue def del elif else except finally for from global if import in is lambda nonlocal not or pass raise return try while with yield bool bytes bytearray dict float int list str - datetime""".split() + datetime serialize serialize_members deserialize __init__ __post_init__ + _smithy_default __repr__ __annotations__""".split() ) # reserved-error-member-names.txt _RESERVED_ERROR_MEMBERS = frozenset( @@ -54,14 +55,24 @@ def member_field_name( return "message" name = to_snake_case(wire_name) - if name in _RESERVED_MEMBERS: + if name.startswith("__") and not name.endswith("__"): + name += "___" + elif name in _RESERVED_MEMBERS: name += "_" if is_error and name in _RESERVED_ERROR_MEMBERS: name += "_" if name == _RESPONSE_METADATA_MEMBER and carries_response_metadata: name += "_" if is_enum: - name = name.upper() + name = _escape_enum_underscores(name).upper() + return name + + +def _escape_enum_underscores(name: str) -> str: + if name.startswith("__"): + return name + "_" if name.endswith("__") and not name.endswith("___") else name + if name.startswith("_") and name.endswith("_") and not name.endswith("__"): + return name + "_" return name diff --git a/packages/smithy-core/tests/unit/test_dynamic_models.py b/packages/smithy-core/tests/unit/test_dynamic_models.py index 60d4363dc..6f342ca03 100644 --- a/packages/smithy-core/tests/unit/test_dynamic_models.py +++ b/packages/smithy-core/tests/unit/test_dynamic_models.py @@ -3,19 +3,40 @@ # pyright: reportPrivateUsage=false from enum import IntEnum, StrEnum from types import ModuleType -from typing import Any, cast +from typing import Any, Protocol, cast +import pytest from smithy_core import _schema_compact from smithy_core._dynamic_models import ModelRegistry from smithy_core._schema_compact import StaticSchema from smithy_core.deserializers import ShapeDeserializer -from smithy_core.exceptions import ModeledError +from smithy_core.exceptions import ModeledError, SerializationError +from smithy_core.schemas import Schema +from smithy_core.serializers import ShapeSerializer from smithy_core.shapes import ShapeType from smithy_core.types import UnknownEnumMixin NS = "ns.example" +class _UnknownVariant(Protocol): + def serialize(self, serializer: ShapeSerializer) -> None: ... + + def serialize_members(self, serializer: ShapeSerializer) -> None: ... + + +class _UnknownVariantClass(Protocol): + def __call__(self, *, tag: str) -> _UnknownVariant: ... + + def deserialize(self, deserializer: ShapeDeserializer) -> _UnknownVariant: ... + + +class _ModeledErrorClass(Protocol): + def __call__(self) -> ModeledError: ... + + def deserialize(self, deserializer: ShapeDeserializer) -> ModeledError: ... + + def _struct(name: str, members: tuple[tuple[str, object], ...]) -> StaticSchema: return ( ShapeType.STRUCTURE.value, @@ -30,12 +51,17 @@ def _struct(name: str, members: tuple[tuple[str, object], ...]) -> StaticSchema: _ERROR_TRAIT = "smithy.api#error" -def _error_struct(name: str, members: tuple[tuple[str, object], ...]) -> StaticSchema: +def _error_struct( + name: str, + members: tuple[tuple[str, object], ...], + *, + fault: str = "client", +) -> StaticSchema: return ( ShapeType.STRUCTURE.value, NS, name, - {_ERROR_TRAIT: "client"}, + {_ERROR_TRAIT: fault}, tuple(m[0] for m in members), tuple((m[1], 0) for m in members), ) @@ -128,6 +154,249 @@ def test_supplied_index_matches_scanning_fallback() -> None: assert _fields(scanned) == _fields(supplied) +def test_supplied_index_preserves_distinct_same_named_structures() -> None: + first = ( + ShapeType.STRUCTURE.value, + "a.example", + "Thing", + 0, + ("first",), + ((ShapeType.STRING.value, 0),), + ) + second = ( + ShapeType.STRUCTURE.value, + "b.example", + "Thing", + 0, + ("second",), + ((ShapeType.STRING.value, 0),), + ) + registry = ModelRegistry( + _module(FIRST=first, SECOND=second), + index={"FirstThing": "FIRST", "SecondThing": "SECOND"}, + ) + + first_cls = registry.build("FirstThing") + second_cls = registry.build("SecondThing") + assert first_cls is not second_cls + assert first_cls.__name__ == "FirstThing" + assert second_cls.__name__ == "SecondThing" + assert _fields(first_cls) == ["first"] + assert _fields(second_cls) == ["second"] + + +def test_registry_copies_supplied_name_maps() -> None: + renames = {"Alias": "FOO"} + index = {"Alias": "FOO"} + registry = ModelRegistry( + _module(FOO=_struct("Foo", ())), + renames=renames, + index=index, + ) + renames.clear() + index.clear() + + assert registry.build("Alias").__name__ == "Alias" + + +def test_alias_and_canonical_struct_lookups_share_class_identity() -> None: + registry = ModelRegistry( + _module(FOO=_struct("Foo", ())), + renames={"Alias": "FOO"}, + ) + + alias = registry.build("Alias") + assert registry.build("Foo") is alias + assert registry.build("Alias") is alias + + +def test_alias_and_canonical_union_lookups_share_class_identity() -> None: + registry = ModelRegistry( + _module(FOO=_union("Foo", (("value", ShapeType.STRING.value),))), + renames={"Alias": "FOO"}, + ) + + alias = registry.build("Alias") + variant = registry.build("AliasValue") + unknown = registry.build("AliasUnknown") + assert registry.build("Foo") is alias + assert registry.build("Alias") is alias + assert registry.build("AliasValue") is variant + assert registry.build("FooValue") is variant + assert registry.build("FooUnknown") is unknown + + class _De: + def read_struct(self, schema: Schema, consumer: Any) -> None: + consumer(schema.members_by_index[0], self) + + def read_string(self, schema: Schema) -> str: + return "value" + + alias_deserializer = registry.build("_AliasDeserializer")() + original_deserializer = registry.build("_FooDeserializer")() + alias_value = alias_deserializer.deserialize(cast(ShapeDeserializer, _De())) + original_value = original_deserializer.deserialize(cast(ShapeDeserializer, _De())) + assert type(alias_value) is variant + assert type(original_value) is variant + assert alias_value.value == original_value.value == "value" + + +def test_renamed_union_does_not_replace_same_named_shape() -> None: + renamed = ( + ShapeType.UNION.value, + "a.example", + "Thing", + 0, + ("value",), + ((ShapeType.STRING.value, 0),), + ) + other = ( + ShapeType.STRUCTURE.value, + "b.example", + "Thing", + 0, + ("other",), + ((ShapeType.STRING.value, 0),), + ) + registry = ModelRegistry( + _module(A_OTHER=other, Z_RENAMED=renamed), + renames={"FirstThing": "Z_RENAMED"}, + ) + + first = registry.build("FirstThing") + thing = registry.build("Thing") + assert first is not thing + assert _fields(thing) == ["other"] + + +def test_supplied_union_names_override_original_shape_names() -> None: + union = ( + ShapeType.UNION.value, + "a.example", + "Foo", + 0, + ("item",), + ((ShapeType.STRING.value, 0),), + ) + item = ( + ShapeType.STRUCTURE.value, + "b.example", + "OldItem", + 0, + ("item",), + ((ShapeType.STRING.value, 0),), + ) + unknown = ( + ShapeType.STRUCTURE.value, + "b.example", + "OldUnknown", + 0, + ("unknown",), + ((ShapeType.STRING.value, 0),), + ) + registry = ModelRegistry( + _module(FOO=union, ITEM=item, UNKNOWN=unknown), + renames={ + "Choice": "FOO", + "ChoiceItem": "ITEM", + "ChoiceUnknown": "UNKNOWN", + }, + index={ + "Foo": "FOO", + "Choice_Item": "FOO", + "Choice_Unknown": "FOO", + "OldItem": "ITEM", + "OldUnknown": "UNKNOWN", + }, + shape_names={"Foo", "OldItem", "OldUnknown"}, + ) + + item_cls = registry.build("ChoiceItem") + unknown_cls = registry.build("ChoiceUnknown") + registry.build("Choice") + assert registry.build("Choice_Item") is not item_cls + assert registry.build("Choice_Unknown") is not unknown_cls + assert registry.build("ChoiceItem") is item_cls + assert registry.build("ChoiceUnknown") is unknown_cls + + +def test_renamed_union_aliases_do_not_replace_renamed_models() -> None: + registry = ModelRegistry( + _module( + FOO=_union("Foo", (("value", ShapeType.STRING.value),)), + VALUE=_struct("ValueModel", (("value", ShapeType.STRING.value),)), + UNKNOWN=_struct("UnknownModel", (("value", ShapeType.STRING.value),)), + ), + renames={ + "Alias": "FOO", + "FooValue": "VALUE", + "FooUnknown": "UNKNOWN", + }, + ) + + registry.build("Alias") + assert _fields(registry.build("FooValue")) == ["value"] + assert _fields(registry.build("FooUnknown")) == ["value"] + assert registry.build("FooValue") is not registry.build("AliasValue") + assert registry.build("FooUnknown") is not registry.build("AliasUnknown") + + +def test_renamed_union_original_alias_does_not_replace_renamed_model() -> None: + registry = ModelRegistry( + _module( + FOO=_union("Foo", (("value", ShapeType.STRING.value),)), + MODEL=_struct("Model", (("value", ShapeType.STRING.value),)), + ), + renames={"Alias": "FOO", "Foo": "MODEL"}, + ) + + registry.build("Alias") + assert _fields(registry.build("Foo")) == ["value"] + assert registry.build("Foo") is not registry.build("Alias") + + +def test_target_names_resolve_renames_by_full_shape_id() -> None: + _schema_compact._cache.clear() + first = ( + ShapeType.STRUCTURE.value, + "a.example", + "Thing", + 0, + ) + second = ( + ShapeType.STRUCTURE.value, + "b.example", + "Thing", + 0, + ) + container = _struct("Container", (("value", lambda: second),)) + registry = ModelRegistry( + _module(FIRST=first, SECOND=second, CONTAINER=container), + renames={"FirstThing": "FIRST", "SecondThing": "SECOND"}, + index={ + "FirstThing": "FIRST", + "SecondThing": "SECOND", + "Container": "CONTAINER", + }, + target_names={ + "a.example#Thing": "FirstThing", + "b.example#Thing": "SecondThing", + f"{NS}#Container": "Container", + }, + shape_names={"FirstThing", "SecondThing", "Container"}, + ) + schema = _schema_compact.hydrate(container) + member = schema.members_by_index[0] + _, reader = member.get_extension(registry._codec) + + class _De: + def read_struct(self, schema: object, consumer: object) -> None: + pass + + result = reader(cast(ShapeDeserializer, _De()), member) + assert type(result).__name__ == "SecondThing" + + def test_supplied_index_disambiguated_union_variant() -> None: # Union FooWarning.message collides with struct FooWarningMessage; codegen emits # the "_"-disambiguated variant name in the index. The real struct must still @@ -220,6 +489,20 @@ def test_modeled_error_falls_back_to_modeled_error_without_service_error() -> No assert cls.__mro__[1] is ModeledError +def test_modeled_error_uses_modeled_fault() -> None: + _schema_compact._cache.clear() + mod = _module(BOOM=_error_struct("Boom", (), fault="server")) + registry = ModelRegistry(mod, index={"Boom": "BOOM"}, shape_names={"Boom"}) + cls = cast(_ModeledErrorClass, registry.build("Boom")) + + class _De: + def read_struct(self, schema: Schema, consumer: Any) -> None: + pass + + assert cls().fault == "server" + assert cls.deserialize(cast(ShapeDeserializer, _De())).fault == "server" + + def test_enum_member_deserialize_coerces_to_enum_class() -> None: # A member whose target is an enum must deserialize to the enum class, not the # raw wire string (the eager generator wrapped it as EnumClass(read_string(...))). @@ -265,6 +548,22 @@ def test_union_classes_report_models_module() -> None: assert registry.build(name).__module__ == "ns.example.models" +def test_unknown_union_variant_rejects_serde() -> None: + _schema_compact._cache.clear() + mod = _module(FOO=_union("Foo", (("a", ShapeType.STRING.value),))) + registry = ModelRegistry(mod, index={"Foo": "FOO"}, shape_names={"Foo"}) + registry.build("Foo") + unknown = cast(_UnknownVariantClass, registry.build("FooUnknown")) + value = unknown(tag="future") + + with pytest.raises(SerializationError): + value.serialize(cast(ShapeSerializer, object())) + with pytest.raises(SerializationError): + value.serialize_members(cast(ShapeSerializer, object())) + with pytest.raises(NotImplementedError): + unknown.deserialize(cast(ShapeDeserializer, object())) + + def test_concurrent_build_returns_one_class() -> None: import threading diff --git a/packages/smithy-core/tests/unit/test_model_naming.py b/packages/smithy-core/tests/unit/test_model_naming.py index d47d7e495..d0198c905 100644 --- a/packages/smithy-core/tests/unit/test_model_naming.py +++ b/packages/smithy-core/tests/unit/test_model_naming.py @@ -1,6 +1,89 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from smithy_core._model_naming import union_unknown_name, variant_class_name +from smithy_core._model_naming import ( + member_field_name, + to_snake_case, + union_unknown_name, + variant_class_name, +) + + +def test_snake_case_preserves_prefix_for_leading_digit() -> None: + assert to_snake_case("_1") == "_1" + assert to_snake_case("_2HTTPServer") == "_2_http_server" + + +def test_snake_case_preserves_existing_underscores() -> None: + assert to_snake_case("_value") == "_value" + assert to_snake_case("value_") == "value_" + assert to_snake_case("some__value") == "some__value" + + +def test_enum_names_escape_before_uppercasing() -> None: + assert ( + member_field_name( + "CLASS", + is_error=False, + carries_response_metadata=False, + is_enum=True, + ) + == "CLASS_" + ) + + +def test_enum_sunder_names_are_escaped() -> None: + assert ( + member_field_name( + "_VALUE_", + is_error=False, + carries_response_metadata=False, + is_enum=True, + ) + == "_VALUE__" + ) + + +def test_enum_dunder_names_are_escaped() -> None: + assert ( + member_field_name( + "__VALUE__", + is_error=False, + carries_response_metadata=False, + is_enum=True, + ) + == "__VALUE___" + ) + + +def test_runtime_method_names_are_reserved() -> None: + for name in ( + "serialize", + "serializeMembers", + "deserialize", + "__init__", + "__post_init__", + "_smithy_default", + "__repr__", + "__annotations__", + ): + assert member_field_name( + name, + is_error=False, + carries_response_metadata=False, + is_enum=False, + ).endswith("_") + + +def test_double_leading_underscore_names_avoid_name_mangling() -> None: + assert ( + member_field_name( + "__value", + is_error=False, + carries_response_metadata=False, + is_enum=False, + ) + == "__value___" + ) def test_variant_class_name_default() -> None: diff --git a/packages/smithy-python/.changes/next-release/smithy-python-feature-model-generation.json b/packages/smithy-python/.changes/next-release/smithy-python-feature-model-generation.json new file mode 100644 index 000000000..d5c28cdf5 --- /dev/null +++ b/packages/smithy-python/.changes/next-release/smithy-python-feature-model-generation.json @@ -0,0 +1,4 @@ +{ + "type": "feature", + "description": "Added native generation of lazy runtime model modules and complete static model stubs for standalone types packages." +} diff --git a/packages/smithy-python/README.md b/packages/smithy-python/README.md index 79db2d4bc..2392b8f02 100644 --- a/packages/smithy-python/README.md +++ b/packages/smithy-python/README.md @@ -1,19 +1,17 @@ # smithy-python -> [!WARNING] -> This package is an experimental scaffold. It does not generate code yet. The -> Java generator in the repository's `codegen` directory remains authoritative. - -`smithy-python` will provide Python-native code generation for Smithy models. -The initial command-line interface exposes the planned client and types generation -commands so that their top-level shape can be developed independently from the -generator implementation. +`smithy-python` provides Python-native code generation for Smithy models. +Standalone types generation is available; client generation remains under +development. ```console smithy-python generate client [OPTIONS] smithy-python generate types [OPTIONS] ``` -After validating their invocation options, both generation commands currently exit -with an error explaining that generation has not been implemented. The package is -included in workspace builds to validate its packaging and entry points. +`generate types` produces an installable package containing compact schemas, lazy +runtime models, and a complete static typing surface. Run +`smithy-python generate types --help` for artifact settings. + +`generate client` currently exits with an error explaining that client generation +has not been implemented. diff --git a/packages/smithy-python/pyproject.toml b/packages/smithy-python/pyproject.toml index cab33c833..2a3e2ec9e 100644 --- a/packages/smithy-python/pyproject.toml +++ b/packages/smithy-python/pyproject.toml @@ -26,7 +26,9 @@ classifiers = [ "Programming Language :: Python :: Free Threading :: 2 - Beta", "Topic :: Software Development :: Code Generators", ] -dependencies = [] +dependencies = [ + "packaging", +] [project.scripts] smithy-python = "smithy_python.cli:main" diff --git a/packages/smithy-python/src/smithy_python/cli.py b/packages/smithy-python/src/smithy_python/cli.py index a94d90e03..15f8012ab 100644 --- a/packages/smithy-python/src/smithy_python/cli.py +++ b/packages/smithy-python/src/smithy_python/cli.py @@ -11,7 +11,18 @@ from pathlib import Path from typing import BinaryIO, Final +from packaging.version import InvalidVersion, Version + from . import __version__ +from . import dependencies as _dependencies +from . import exceptions as _exceptions +from . import files as _files +from . import generation as _generation +from . import model as _model +from . import models as _models +from . import selection as _selection +from . import symbols as _symbols +from . import types as _types from .environment import PluginEnvironment from .exceptions import CodegenError, InvalidInvocationError from .model import load_model @@ -26,10 +37,8 @@ @dataclass(frozen=True, slots=True) class _Invocation: - artifact: str model_source: bytes output_dir: Path - environment: PluginEnvironment def main( @@ -48,24 +57,19 @@ def main( try: invocation = _resolve_invocation(args, environ=environ, stdin=stdin) - except InvalidInvocationError as error: - sys.stderr.write(f"smithy-python: error: {error}\n") - return 2 - except (CodegenError, OSError) as error: - sys.stderr.write(f"smithy-python: error: {error}\n") - return 1 - - try: model = load_model(invocation.model_source) - selection = select_shapes( - model, - service_id=args.service, - require_service=invocation.artifact == "client", - ) + if args.artifact == "types": + selection = _generate_types(args, invocation.output_dir, model) + else: + selection = select_shapes( + model, + service_id=args.service, + require_service=True, + ) except InvalidInvocationError as error: sys.stderr.write(f"smithy-python: error: {error}\n") return 2 - except CodegenError as error: + except (CodegenError, OSError) as error: sys.stderr.write(f"smithy-python: error: {error}\n") return 1 @@ -74,6 +78,8 @@ def main( f"smithy-python: excluded {selection.excluded_count} eligible data shape(s) " f"outside the closure of service {selection.service.id}\n" ) + if args.artifact == "types": + return 0 sys.stderr.write(_GENERATION_NOT_IMPLEMENTED.format(artifact=args.artifact)) return 1 @@ -93,11 +99,36 @@ def _create_parser() -> argparse.ArgumentParser: generate = commands.add_parser("generate", help="Generate Python source") artifacts = generate.add_subparsers(dest="artifact", required=True) common = _common_artifact_options() - for name, help_text in ( - ("client", "Generate a client package"), - ("types", "Generate a standalone types package"), - ): - artifacts.add_parser(name, help=help_text, parents=[common]) + artifacts.add_parser("client", help="Generate a client package", parents=[common]) + types = artifacts.add_parser( + "types", help="Generate a standalone types package", parents=[common] + ) + types.add_argument( + "--package", + required=True, + help="Import package and distribution name for the generated artifact.", + ) + types.add_argument( + "--package-version", + required=True, + type=_package_version, + help="Version for the generated package.", + ) + types.add_argument( + "--package-description", + help='Package description; defaults to " types package".', + ) + types.add_argument( + "--selector", + type=_parse_selector, + default=_selection.Selector.identity(), + help="Smithy selector used to narrow generated declaration roots.", + ) + types.add_argument( + "--generate-inputs-and-outputs", + action="store_true", + help="Include operation input and output structures as generation roots.", + ) return parser @@ -175,8 +206,68 @@ def _resolve_invocation( raise InvalidInvocationError("Expected a Smithy JSON AST model") return _Invocation( - artifact=args.artifact, model_source=model_source, output_dir=output_dir, - environment=environment, ) + + +def _generate_types( + args: argparse.Namespace, + output_dir: Path, + model: _model.Model, +) -> _selection.Selection: + description = args.package_description or f"{args.package} types package" + settings = _generation.TypesSettings( + package=args.package, + package_version=args.package_version, + package_description=description, + selector=args.selector, + generate_inputs_and_outputs=args.generate_inputs_and_outputs, + ) + selection = _selection.select_type_shapes( + model, + service_id=args.service, + selector=settings.selector, + generate_inputs_and_outputs=settings.generate_inputs_and_outputs, + ) + try: + symbols = _symbols.SymbolProvider( + model, + selection, + package=settings.package, + reserved_member_names=_models._model_reserved_member_names( # pyright: ignore[reportPrivateUsage] + model + ), + ) + except ValueError as error: + raise InvalidInvocationError(str(error)) from error + + dependencies = _dependencies.DependencySet() + context = _generation.GenerationContext( + model=model, + selection=selection, + settings=settings, + symbols=symbols, + plan=_generation.GenerationPlan.from_selection(model, selection, symbols), + dependencies=dependencies, + files=_files.FileManifest(dependencies), + ) + _types.TypesGenerator().generate(context) + context.files.commit(output_dir) + return selection + + +def _parse_selector(value: str) -> _selection.Selector: + try: + return _selection.Selector.parse(value) + except _exceptions.SelectorSyntaxError as error: + raise argparse.ArgumentTypeError(str(error)) from error + + +def _package_version(value: str) -> str: + if not value: + raise argparse.ArgumentTypeError("expected a non-empty value") + try: + return str(Version(value)) + except InvalidVersion as error: + raise argparse.ArgumentTypeError(str(error)) from error diff --git a/packages/smithy-python/src/smithy_python/files.py b/packages/smithy-python/src/smithy_python/files.py index 736f71821..79fa17069 100644 --- a/packages/smithy-python/src/smithy_python/files.py +++ b/packages/smithy-python/src/smithy_python/files.py @@ -53,7 +53,11 @@ def python_module(self, path: str, module: str) -> _writer.PythonWriter: raise _exceptions.CodegenError( f"Generated path {path!r} is already a text file" ) - writer = _writer.PythonWriter(module, dependencies=self._dependencies) + writer = _writer.PythonWriter( + module, + dependencies=self._dependencies, + stub=path.endswith(".pyi"), + ) self._files[path] = _PythonFile(module, writer) return writer diff --git a/packages/smithy-python/src/smithy_python/generation.py b/packages/smithy-python/src/smithy_python/generation.py index 3bec23e70..de9631ab4 100644 --- a/packages/smithy-python/src/smithy_python/generation.py +++ b/packages/smithy-python/src/smithy_python/generation.py @@ -25,6 +25,9 @@ "TypesSettings", ] +_SERVICE_ERROR = "ServiceError" +_UNIT = _model.ShapeId("smithy.api", "Unit") + @dataclass(frozen=True, slots=True) class ArtifactSettings: @@ -41,16 +44,30 @@ class TypesSettings(ArtifactSettings): selector: _selection.Selector = field(default_factory=_selection.Selector.identity) generate_inputs_and_outputs: bool = False + schema_partition_size: int | None = 250 + stub_partition_size: int | None = 250 + + def __post_init__(self) -> None: + if any( + size is not None and size < 1 + for size in (self.schema_partition_size, self.stub_partition_size) + ): + raise ValueError("Partition sizes must be positive") @dataclass(frozen=True, slots=True) class ModelPlan: """Derived semantic data used to emit lazy model declarations.""" + shapes: tuple[_model.Shape, ...] declaration_names: Mapping[_model.ShapeId, str] renames: Mapping[str, str] union_variant_names: Mapping[_model.ShapeId, str] union_unknown_names: Mapping[_model.ShapeId, str] + registry_index: Mapping[str, str] + target_names: Mapping[str, str] + shape_names: frozenset[str] + includes_unit: bool @classmethod def from_shapes( @@ -60,31 +77,55 @@ def from_shapes( schema_names: Mapping[_model.ShapeId, str], ) -> ModelPlan: """Plan declaration and union names in stable shape order.""" + model_shapes: list[_model.Shape] = [] declarations: dict[_model.ShapeId, str] = {} variants: dict[_model.ShapeId, str] = {} unknowns: dict[_model.ShapeId, str] = {} + index: dict[str, str] = {} + renames: dict[str, str] = {} + target_names: dict[str, str] = {} + includes_unit = any( + shape.type not in (_model.ShapeType.ENUM, _model.ShapeType.INT_ENUM) + and any(member.target == _UNIT for member in shape.members.values()) + for shape in shapes + ) for shape in shapes: if shape.type not in _model.DECLARATION_TYPES or shape.has_trait( "streaming" ): continue - declarations[shape.id] = symbols.declaration_name(shape.id) + model_shapes.append(shape) + declaration = symbols.declaration_name(shape.id) + schema = schema_names[shape.id] + declarations[shape.id] = declaration + index[declaration] = schema + if declaration != shape.id.name: + renames[declaration] = schema + target_names[str(shape.id)] = declaration if shape.type is _model.ShapeType.UNION: for member in shape.members.values(): - variants[member.id] = symbols.union_variant_name(member.id) - unknowns[shape.id] = symbols.union_unknown_name(shape.id) + variant = symbols.union_variant_name(member.id) + variants[member.id] = variant + index[variant] = schema + unknown = symbols.union_unknown_name(shape.id) + unknowns[shape.id] = unknown + index[unknown] = schema return cls( + shapes=tuple(model_shapes), declaration_names=MappingProxyType(declarations), - renames=MappingProxyType( - { - declaration: schema_names[shape_id] - for shape_id, declaration in declarations.items() - if declaration != shape_id.name - } - ), + renames=MappingProxyType(renames), union_variant_names=MappingProxyType(variants), union_unknown_names=MappingProxyType(unknowns), + registry_index=MappingProxyType(index), + target_names=MappingProxyType(target_names), + shape_names=frozenset( + ( + *declarations.values(), + *(("Unit",) if includes_unit else ()), + ) + ), + includes_unit=includes_unit, ) diff --git a/packages/smithy-python/src/smithy_python/models.py b/packages/smithy-python/src/smithy_python/models.py new file mode 100644 index 000000000..f795d7682 --- /dev/null +++ b/packages/smithy-python/src/smithy_python/models.py @@ -0,0 +1,557 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Emission of lazy runtime models and their static typing surface.""" + +from __future__ import annotations + +from collections.abc import Callable, Collection, Mapping +from dataclasses import replace +from json import dumps +from textwrap import wrap +from typing import TYPE_CHECKING, Final + +from . import dependencies as _dependencies +from . import exceptions as _exceptions +from . import model as _model +from . import symbols as _symbols + +if TYPE_CHECKING: + from .generation import GenerationContext, ModelPlan, TypesSettings + from .writer import PythonWriter + +__all__ = ["ModelGenerator"] + + +class ModelGenerator: + """Emit lazy runtime models and the complete static model stub.""" + + def generate(self, context: GenerationContext[TypesSettings]) -> None: + """Register ``models.py`` and ``models.pyi`` with the file manifest.""" + _validate_member_names(context) + package = context.settings.package + directory = package.replace(".", "/") + _write_runtime( + context.files.python_module( + f"src/{directory}/models.py", f"{package}.models" + ), + context.plan.models, + ) + facade = _StubEmitter( + context.files.python_module( + f"src/{directory}/models.pyi", f"{package}.models" + ), + context, + ) + shapes = context.plan.models.shapes + size = context.settings.stub_partition_size + if size is None or len(shapes) <= size: + facade.write() + return + + partitions = tuple( + shapes[index : index + size] for index in range(0, len(shapes), size) + ) + facade.write_facade(partitions) + for index, shapes in enumerate(partitions): + _StubEmitter( + context.files.python_module( + f"src/{directory}/_private/models_{index}.pyi", + f"{package}._private.models_{index}", + ), + context, + external_bases=True, + ).write_partition(shapes) + + +def _core_reference(name: str, module: str) -> _symbols.TypeReference: + return _symbols.TypeReference( + name, f"smithy_core.{module}", dependencies=(_dependencies.SMITHY_CORE,) + ) + + +_MODEL_REGISTRY: Final = _core_reference("ModelRegistry", "_dynamic_models") +_MODELED_ERROR: Final = _core_reference("ModeledError", "exceptions") +_SHAPE_SERIALIZER: Final = _core_reference("ShapeSerializer", "serializers") +_SHAPE_DESERIALIZER: Final = _core_reference("ShapeDeserializer", "deserializers") +_UNKNOWN_ENUM_MIXIN: Final = _core_reference("UnknownEnumMixin", "types") +_RESPONSE_METADATA: Final = _core_reference("ResponseMetadata", "response") +_DATACLASS: Final = _symbols.TypeReference("dataclass", "dataclasses") +_SELF: Final = _symbols.TypeReference("Self", "typing") +_STR_ENUM: Final = _symbols.TypeReference("StrEnum", "enum") +_INT_ENUM: Final = _symbols.TypeReference("IntEnum", "enum") +_SCHEMAS: Final = _symbols.TypeReference("schemas", "._private") + +_ERROR_MESSAGE_NAMES: Final = frozenset({"errormessage", "error_message", "message"}) +_DOCSTRING_WIDTH: Final = 72 +_MODEL_MEMBER_NAMES: Final = frozenset( + { + "bool", + "bytes", + "bytearray", + "dict", + "float", + "int", + "list", + "str", + "datetime", + "serialize", + "serialize_members", + "deserialize", + "__init__", + "__post_init__", + "_smithy_default", + "__repr__", + "__annotations__", + } +) +_ERROR_MEMBER_NAMES: Final = frozenset( + {"is_retry_safe", "retry_after", "is_throttling_error", "fault"} +) +_RESPONSE_METADATA_DOC: Final = ( + "Metadata about the response that produced this output.\n\n" + "Use this to recover the request identifiers a service's support team needs " + "in order to investigate a call. Members of the metadata are individually " + "optional." +) +_SERVICE_ERROR_DOC: Final = ( + "Base error for all errors in the service.\n\n" + "Some exceptions do not extend from this class, including synthetic, implicit, " + "and shared exception types." +) +_UNKNOWN_UNION_DOC: Final = ( + "Represents an unknown variant.\n\n" + "If you receive this value, you will need to update your library to receive " + "the parsed value.\n\n" + "This value may not be deliberately sent." +) + + +def _write_runtime(writer: PythonWriter, plan: ModelPlan) -> None: + exports = _model_exports(plan) + writer.line("class ServiceError(", _MODELED_ERROR, "):") + with writer.indent(): + _write_docstring(writer, _SERVICE_ERROR_DOC) + writer.line() + writer.line() + _write_mapping(writer, "_MODEL_INDEX", plan.registry_index) + writer.line() + _write_mapping(writer, "_RENAMES", plan.renames) + writer.line() + _write_mapping(writer, "_TARGET_NAMES", plan.target_names) + writer.line() + additional = exports[len(plan.registry_index) + 1 :] + if additional: + writer.line( + "_MODEL_NAMES = (*tuple(_MODEL_INDEX), ", + ", ".join(map(_quote, additional)), + ")", + ) + else: + writer.line("_MODEL_NAMES = tuple(_MODEL_INDEX)") + writer.line("__all__ = (", _quote(exports[0]), ", *_MODEL_NAMES)") + writer.line() + writer.line() + writer.line("_registry = ", _MODEL_REGISTRY, "(") + with writer.indent(): + writer.line(_SCHEMAS, ",") + writer.line("renames=_RENAMES,") + writer.line("index=_MODEL_INDEX,") + writer.line("target_names=_TARGET_NAMES,") + _write_string_set(writer, "shape_names", plan.shape_names, suffix=",") + writer.line('service_error="ServiceError",') + writer.line(")") + writer.line("del ", _SCHEMAS, ", ", _MODEL_REGISTRY, ", ", _MODELED_ERROR) + writer.line() + writer.line() + writer.line("def __getattr__(name: str):") + with writer.indent(): + writer.line("value = _registry.build(name)") + writer.line("globals()[name] = value") + writer.line("return value") + writer.line() + writer.line() + writer.line("def __dir__() -> list[str]:") + with writer.indent(): + writer.line("return sorted({*globals(), *__all__})") + + +class _StubEmitter: + def __init__( + self, + writer: PythonWriter, + context: GenerationContext[TypesSettings], + *, + external_bases: bool = False, + ) -> None: + self.writer = writer + self.model = context.model + self.plan = context.plan.models + self.symbols = context.symbols + self.module = f"{context.settings.package}.models" + self.external_bases = external_bases + self.serde_shape: str | _symbols.TypeReference = ( + _symbols.TypeReference("_SerdeShape", "..models") + if external_bases + else "_SerdeShape" + ) + self.service_error: str | _symbols.TypeReference = ( + _symbols.TypeReference("ServiceError", "..models") + if external_bases + else "ServiceError" + ) + operations = tuple(context.model.iter_shapes(_model.OperationShape)) + self.inputs = {operation.input for operation in operations} | { + shape.id for shape in self.plan.shapes if shape.has_trait("input") + } + self.outputs = {operation.output for operation in operations} | { + shape.id for shape in self.plan.shapes if shape.has_trait("output") + } + + def write(self) -> None: + """Write the complete unpartitioned public stub.""" + self._write_preamble() + self._write_shapes(self.plan.shapes) + self._write_unit() + + def write_facade(self, partitions: tuple[tuple[_model.Shape, ...], ...]) -> None: + """Write the public stub facade and re-export private declarations.""" + writer = self.writer + for index, shapes in enumerate(partitions): + writer.line(f"from ._private.models_{index} import (") + with writer.indent(): + for shape in shapes: + for name in _stub_shape_names(self.plan, shape): + writer.line(name, ",") + writer.line(")") + writer.line() + self._write_preamble() + self._write_unit() + + def write_partition(self, shapes: tuple[_model.Shape, ...]) -> None: + """Write declarations assigned to one private stub partition.""" + self._write_shapes(shapes) + + def _write_preamble(self) -> None: + writer = self.writer + writer.line("__all__ = [") + with writer.indent(): + for name in sorted(_model_exports(self.plan)): + writer.line(_quote(name), ",") + writer.line("]") + writer.line() + writer.line("class _SerdeShape:") + with writer.indent(): + writer.line( + "def serialize(self, serializer: ", + _SHAPE_SERIALIZER, + ") -> None: ...", + ) + writer.line( + "def serialize_members(self, serializer: ", + _SHAPE_SERIALIZER, + ") -> None: ...", + ) + writer.line("@classmethod") + writer.line( + "def deserialize(cls, deserializer: ", + _SHAPE_DESERIALIZER, + ") -> ", + _SELF, + ": ...", + ) + writer.line() + writer.line("class ServiceError(", _MODELED_ERROR, "):") + with writer.indent(): + _write_docstring(writer, _SERVICE_ERROR_DOC) + + def _write_shapes(self, shapes: Collection[_model.Shape]) -> None: + writer = self.writer + for shape in shapes: + writer.line() + match shape.type: + case _model.ShapeType.STRUCTURE: + self._write_structure(shape) + case _model.ShapeType.UNION: + self._write_union(shape) + case _model.ShapeType.ENUM | _model.ShapeType.INT_ENUM: + self._write_enum(shape) + case _: + pass + + def _write_unit(self) -> None: + writer = self.writer + if self.plan.includes_unit: + writer.line() + writer.line("@", _DATACLASS, "(kw_only=True)") + writer.line("class Unit(", self.serde_shape, "):") + with writer.indent(): + writer.line("pass") + + def _write_structure(self, shape: _model.Shape) -> None: + writer = self.writer + name = self.plan.declaration_names[shape.id] + error = shape.has_trait("error") + writer.line("@", _DATACLASS, "(kw_only=True)") + bases: tuple[str | _symbols.TypeReference, ...] = ( + (self.service_error, ", ", self.serde_shape) + if error + else (self.serde_shape,) + ) + writer.line( + "class ", + name, + "(", + *bases, + "):", + ) + with writer.indent(): + members = tuple( + member + for member in shape.members.values() + if not error or member.name.lower() not in _ERROR_MESSAGE_NAMES + ) + carries_metadata = error or shape.id in self.outputs + _write_docstring( + writer, + _documentation(shape) or f"Dataclass for {shape.id.name} structure.", + ) + if members or carries_metadata: + writer.line() + for member in members: + reference = self._type_reference(member.target) + has_default = member.has_trait("default") or self.model.get_target( + member + ).has_trait("default") + mandatory = ( + shape.id not in self.inputs + and member.has_trait("required") + and not has_default + ) + if not mandatory: + reference = replace(reference, nullable=True) + writer.line( + self.symbols.member_name(member.id), + ": ", + reference, + "" if mandatory else " = ...", + ) + if docs := _documentation(member): + _write_docstring(writer, docs) + if carries_metadata: + writer.line("response_metadata: ", _RESPONSE_METADATA, " = ...") + _write_docstring(writer, _RESPONSE_METADATA_DOC) + + def _write_union(self, shape: _model.Shape) -> None: + writer = self.writer + variants: list[_symbols.TypeReference] = [] + for member in shape.members.values(): + name = self.plan.union_variant_names[member.id] + variants.append(self._public_reference(name)) + writer.line("@", _DATACLASS) + writer.line("class ", name, "(", self.serde_shape, "):") + with writer.indent(): + if docs := _documentation(member): + _write_docstring(writer, docs) + writer.line() + writer.line("value: ", self._type_reference(member.target)) + writer.line() + + unknown = self.plan.union_unknown_names[shape.id] + variants.append(self._public_reference(unknown)) + writer.line("@", _DATACLASS) + writer.line("class ", unknown, "(", self.serde_shape, "):") + with writer.indent(): + _write_docstring(writer, _UNKNOWN_UNION_DOC) + writer.line() + writer.line("tag: str") + writer.line() + + name = self.plan.declaration_names[shape.id] + writer.line(name, " = ", *_join_types(variants)) + if docs := _documentation(shape): + _write_docstring(writer, docs) + writer.line() + writer.line("class _", name, "Deserializer:") + with writer.indent(): + writer.line( + "def deserialize(self, deserializer: ", + _SHAPE_DESERIALIZER, + ") -> ", + self._public_reference(name), + ": ...", + ) + + def _write_enum(self, shape: _model.Shape) -> None: + writer = self.writer + name = self.plan.declaration_names[shape.id] + base = _INT_ENUM if shape.type is _model.ShapeType.INT_ENUM else _STR_ENUM + writer.line("class ", name, "(", _UNKNOWN_ENUM_MIXIN, ", ", base, "):") + with writer.indent(): + docs = _documentation(shape) + if docs: + _write_docstring(writer, docs) + if shape.members: + writer.line() + for member in shape.members.values(): + value = member.get_trait("enumValue") + writer.line( + self.symbols.member_name(member.id), + " = ", + _literal(member.name if value is None else value), + ) + if docs := _documentation(member): + _write_docstring(writer, docs) + if not docs and not shape.members: + writer.line("pass") + + def _type_reference(self, shape_id: _model.ShapeId) -> _symbols.TypeReference: + return self._partition_reference(self.symbols.type_reference(shape_id)) + + def _public_reference(self, name: str) -> _symbols.TypeReference: + return _symbols.TypeReference( + name, "..models" if self.external_bases else self.module + ) + + def _partition_reference( + self, reference: _symbols.TypeReference + ) -> _symbols.TypeReference: + arguments = tuple(map(self._partition_reference, reference.arguments)) + module = ( + "..models" + if self.external_bases and reference.module == self.module + else reference.module + ) + if module == reference.module and arguments == reference.arguments: + return reference + return replace(reference, module=module, arguments=arguments) + + +def _model_exports(plan: ModelPlan) -> tuple[str, ...]: + exports = ("ServiceError", *plan.registry_index) + return (*exports, "Unit") if plan.includes_unit else exports + + +def _stub_shape_names(plan: ModelPlan, shape: _model.Shape) -> tuple[str, ...]: + declaration = plan.declaration_names[shape.id] + if shape.type is not _model.ShapeType.UNION: + return (declaration,) + return ( + *(plan.union_variant_names[member.id] for member in shape.members.values()), + plan.union_unknown_names[shape.id], + declaration, + ) + + +def _write_mapping(writer: PythonWriter, name: str, values: Mapping[str, str]) -> None: + if not values: + writer.line(name, " = {}") + return + writer.line(name, " = {") + with writer.indent(): + for key, value in values.items(): + writer.line(_quote(key), ": ", _quote(value), ",") + writer.line("}") + + +def _write_string_set( + writer: PythonWriter, + name: str, + values: Collection[str], + *, + suffix: str = "", +) -> None: + if not values: + writer.line(name, "=frozenset()", suffix) + return + writer.line(name, "=frozenset(") + with writer.indent(): + writer.line("{") + with writer.indent(): + for value in sorted(values): + writer.line(_quote(value), ",") + writer.line("}") + writer.line(")", suffix) + + +def _write_docstring(writer: PythonWriter, value: str) -> None: + lines = [ + wrapped + for line in value.split("\n") + for wrapped in ( + wrap( + line, + width=_DOCSTRING_WIDTH, + break_long_words=False, + break_on_hyphens=False, + ) + or [""] + ) + ] + if len(lines) > 1: + writer.line('"""') + for line in lines: + writer.line(line.replace("\\", "\\\\").replace('"""', '\\"""')) + writer.line('"""') + return + writer.line(_quote(lines[0])) + + +def _literal(value: object) -> str: + return _quote(value) if isinstance(value, str) else repr(value) + + +def _quote(value: str) -> str: + return dumps(value, ensure_ascii=False) + + +def _join_types( + types: Collection[_symbols.TypeReference], +) -> tuple[str | _symbols.TypeReference, ...]: + parts: list[str | _symbols.TypeReference] = [] + for reference in types: + if parts: + parts.append(" | ") + parts.append(reference) + return tuple(parts) + + +def _documentation(shape: _model.Shape) -> str | None: + value = shape.get_trait("documentation") + return value if isinstance(value, str) else None + + +def _validate_member_names(context: GenerationContext[TypesSettings]) -> None: + reserved = _model_reserved_member_names(context.model) + for shape in context.plan.models.shapes: + names: set[str] = set() + shape_reserved = reserved(shape) + for member in shape.members.values(): + name = context.symbols.member_name(member.id) + if name in shape_reserved: + raise _exceptions.CodegenError( + f"Symbol provider did not reserve model member name {name!r} " + f"for {member.id}" + ) + if name in names: + raise _exceptions.CodegenError( + f"Generated model member name collision {name!r} in {shape.id}" + ) + names.add(name) + + +def _model_reserved_member_names( + model: _model.Model, +) -> Callable[[_model.Shape], Collection[str]]: + outputs = { + operation.output for operation in model.iter_shapes(_model.OperationShape) + } | {shape.id for shape in model.iter_shapes() if shape.has_trait("output")} + + def reserved(shape: _model.Shape) -> Collection[str]: + names = set(_MODEL_MEMBER_NAMES) + if shape.has_trait("error"): + names.update(_ERROR_MEMBER_NAMES) + if shape.has_trait("error") or shape.id in outputs: + names.add("response_metadata") + return names + + return reserved diff --git a/packages/smithy-python/src/smithy_python/schemas.py b/packages/smithy-python/src/smithy_python/schemas.py index c47084e45..cf0764641 100644 --- a/packages/smithy-python/src/smithy_python/schemas.py +++ b/packages/smithy-python/src/smithy_python/schemas.py @@ -27,6 +27,7 @@ class SchemaPlan: """Derived semantic data used to emit compact schemas.""" shapes: tuple[_model.Shape, ...] + partition_groups: tuple[tuple[_model.Shape, ...], ...] names: Mapping[_model.ShapeId, str] strings: Mapping[str, str] traits: Mapping[_model.ShapeId, _TraitPlan] @@ -39,7 +40,7 @@ def from_shapes( symbols: _symbols.SymbolProvider, ) -> SchemaPlan: """Plan schema order, filtered traits, names, and interned strings.""" - emitted = _schema_order(shapes) + emitted, partition_groups = _schema_order(shapes) included = _included_traits(model) inputs = { operation.input for operation in model.iter_shapes(_model.OperationShape) @@ -69,6 +70,7 @@ def from_shapes( return cls( shapes=emitted, + partition_groups=partition_groups, names=MappingProxyType( {shape.id: symbols.schema_name(shape.id) for shape in emitted} ), @@ -83,18 +85,111 @@ class SchemaGenerator: def generate(self, context: GenerationContext[TypesSettings]) -> None: """Register ``._private.schemas`` with the file manifest.""" package = context.settings.package - path = f"src/{package.replace('.', '/')}/_private/schemas.py" - writer = context.files.python_module(path, f"{package}._private.schemas") + directory = package.replace(".", "/") plan = context.plan.schemas - for value, name in plan.strings.items(): + size = context.settings.schema_partition_size + if size is None or len(plan.shapes) <= size: + writer = context.files.python_module( + f"src/{directory}/_private/schemas.py", + f"{package}._private.schemas", + ) + _write_schema_module(writer, plan, plan.shapes) + return + + partitions = _schema_partitions(plan, size) + _write_schema_facade( + context.files.python_module( + f"src/{directory}/_private/schemas.py", + f"{package}._private.schemas", + ), + plan, + partitions, + ) + for index, shapes in enumerate(partitions): + _write_schema_module( + context.files.python_module( + f"src/{directory}/_private/schemas_{index}.py", + f"{package}._private.schemas_{index}", + ), + plan, + shapes, + facade="schemas", + ) + + +_IMPORT_MODULE: Final = _symbols.TypeReference("import_module", "importlib") + + +def _write_schema_facade( + writer: PythonWriter, + plan: SchemaPlan, + partitions: tuple[tuple[_model.Shape, ...], ...], +) -> None: + writer.line("_PARTITION_NAMES: tuple[tuple[str, ...], ...] = (") + with writer.indent(): + for shapes in partitions: + writer.line("(") + with writer.indent(): + for shape in shapes: + writer.line(repr(plan.names[shape.id]), ",") + writer.line("),") + writer.line(")") + writer.line("_PARTITIONS = {") + with writer.indent(): + writer.line("name: index") + writer.line("for index, names in enumerate(_PARTITION_NAMES)") + writer.line("for name in names") + writer.line("}") + writer.line("del _PARTITION_NAMES") + writer.line("def __getattr__(name: str):") + with writer.indent(): + writer.line("try:") + with writer.indent(): + writer.line("partition = _PARTITIONS[name]") + writer.line("except KeyError as error:") + with writer.indent(): + writer.line("raise AttributeError(name) from error") + writer.line( + "module = ", + _IMPORT_MODULE, + '(f".schemas_{partition}", __package__)', + ) + writer.line("value = getattr(module, name)") + writer.line("globals()[name] = value") + writer.line("return value") + writer.line() + writer.line() + writer.line("def __dir__() -> list[str]:") + with writer.indent(): + writer.line("return sorted({*globals(), *_PARTITIONS})") + + +def _write_schema_module( + writer: PythonWriter, + plan: SchemaPlan, + shapes: tuple[_model.Shape, ...], + *, + facade: str | None = None, +) -> None: + local = frozenset(shape.id for shape in shapes) + if facade is not None and any( + member.target in plan.names and member.target not in local + for shape in shapes + for member in shape.members.values() + ): + writer.line("from . import ", facade) + writer.line() + strings = _schema_strings(plan, shapes) + for value, name in plan.strings.items(): + if value in strings: writer.line(name, " = ", repr(value)) - if plan.strings: - writer.line() + if strings: + writer.line() - written: set[_model.ShapeId] = set() - for shape in plan.shapes: - _write_shape(writer, plan, shape, written) - written.add(shape.id) + written: set[_model.ShapeId] = set() + for shape in shapes: + _write_shape(writer, plan, shape, written, local=local, facade=facade) + written.add(shape.id) # This contract is intentionally local to the generator. It must match @@ -264,6 +359,9 @@ def _write_shape( plan: SchemaPlan, shape: _model.Shape, written: set[_model.ShapeId], + *, + local: Collection[_model.ShapeId], + facade: str | None, ) -> None: writer.line(plan.names[shape.id], ": ", _STATIC_SCHEMA, " = (") with writer.indent(): @@ -284,7 +382,13 @@ def _write_shape( for member in shape.members.values(): writer.line( "(", - _schema_ref(plan, member.target, written), + *_schema_ref( + plan, + member.target, + written, + local=local, + facade=facade, + ), ", ", *_trait_parts(plan, plan.traits[member.id]), "),", @@ -406,23 +510,36 @@ def _node_parts( def _schema_ref( - plan: SchemaPlan, target: _model.ShapeId, written: set[_model.ShapeId] -) -> str: + plan: SchemaPlan, + target: _model.ShapeId, + written: set[_model.ShapeId], + *, + local: Collection[_model.ShapeId], + facade: str | None, +) -> tuple[_SourcePart, ...]: if (prelude := _PRELUDE_INTS.get(target)) is not None: - return str(prelude) + return (str(prelude),) try: name = plan.names[target] except KeyError as error: raise _exceptions.CodegenError( f"Schema target {target} was not selected" ) from error - return name if target in written else f"lambda: {name}" + if target in local: + return (name if target in written else f"lambda: {name}",) + if facade is None: + raise _exceptions.CodegenError(f"Schema target {target} is not in this module") + return ("lambda: ", facade, ".", name) def _schema_order( shapes: tuple[_model.Shape, ...], -) -> tuple[_model.Shape, ...]: +) -> tuple[ + tuple[_model.Shape, ...], + tuple[tuple[_model.Shape, ...], ...], +]: by_id = {shape.id: shape for shape in shapes} + positions = {shape.id: index for index, shape in enumerate(shapes)} graph = { shape.id: tuple( member.target for member in shape.members.values() if member.target in by_id @@ -457,6 +574,7 @@ def _schema_order( pending.append((target, 0)) recursive: set[_model.ShapeId] = set() + recursive_components: dict[_model.ShapeId, tuple[_model.ShapeId, ...]] = {} visited.clear() for root in reversed(finished): if root in visited: @@ -473,11 +591,97 @@ def _schema_order( component_pending.append(target) if len(component) > 1 or root in graph[root]: recursive.update(component) + ordered_component = tuple(sorted(component, key=positions.__getitem__)) + for shape_id in component: + recursive_components[shape_id] = ordered_component - return ( + emitted = ( *(by_id[shape_id] for shape_id in finished if shape_id not in recursive), *(shape for shape in shapes if shape.id in recursive), ) + groups: list[tuple[_model.Shape, ...]] = [] + grouped: set[_model.ShapeId] = set() + for shape in emitted: + if shape.id in grouped: + continue + group_ids = recursive_components.get(shape.id, (shape.id,)) + groups.append(tuple(by_id[shape_id] for shape_id in group_ids)) + grouped.update(group_ids) + return emitted, tuple(groups) + + +def _schema_partitions( + plan: SchemaPlan, + size: int, +) -> tuple[tuple[_model.Shape, ...], ...]: + groups = plan.partition_groups + partitions: list[tuple[_model.Shape, ...]] = [] + current: list[_model.Shape] = [] + for group in groups: + if current and len(current) + len(group) > size: + partitions.append(tuple(current)) + current.clear() + current.extend(group) + if current: + partitions.append(tuple(current)) + partition_count = len(partitions) + if partition_count < 2: + return tuple(partitions) + + weights = tuple(_schema_group_weight(plan, group) for group in groups) + partitions.clear() + start = 0 + remaining_weight = sum(weights) + for remaining_parts in range(partition_count, 1, -1): + target = remaining_weight / remaining_parts + end = start + weight = 0 + last_end = len(groups) - remaining_parts + 1 + while end < last_end: + next_weight = weight + weights[end] + if weight and abs(target - weight) <= abs(target - next_weight): + break + weight = next_weight + end += 1 + partitions.append( + tuple(shape for group in groups[start:end] for shape in group) + ) + start = end + remaining_weight -= weight + partitions.append(tuple(shape for group in groups[start:] for shape in group)) + return tuple(partitions) + + +def _schema_group_weight(plan: SchemaPlan, group: tuple[_model.Shape, ...]) -> int: + declarations = sum( + len(plan.strings[value]) + len(repr(value)) + 4 + for value in _schema_strings(plan, group) + ) + return declarations + sum(_schema_shape_weight(plan, shape) for shape in group) + + +def _schema_shape_weight(plan: SchemaPlan, shape: _model.Shape) -> int: + """Estimate compact-schema source bytes without rendering shapes twice.""" + return ( + 60 + + len(shape.id.namespace) + + 2 * len(shape.id.name) + + _source_size(_trait_parts(plan, plan.traits[shape.id])) + + sum( + 40 + + 2 * len(member.name) + + len(plan.names.get(member.target, member.target.name)) + + _source_size(_trait_parts(plan, plan.traits[member.id])) + for member in shape.members.values() + ) + ) + + +def _source_size(parts: Collection[_SourcePart]) -> int: + return sum( + len(part.name if isinstance(part, _symbols.TypeReference) else part) + for part in parts + ) def _intern_traits(strings: _StringStore, traits: _TraitPlan) -> None: @@ -489,6 +693,28 @@ def _intern_traits(strings: _StringStore, traits: _TraitPlan) -> None: strings.intern(value) +def _schema_strings( + plan: SchemaPlan, shapes: Collection[_model.Shape] +) -> frozenset[str]: + values: set[str] = set() + for shape in shapes: + values.update((shape.id.namespace, shape.id.name)) + _collect_trait_strings(values, plan.traits[shape.id]) + for member in shape.members.values(): + values.add(member.name) + _collect_trait_strings(values, plan.traits[member.id]) + return frozenset(values) + + +def _collect_trait_strings(values: set[str], traits: _TraitPlan) -> None: + if isinstance(traits, int): + return + for trait_id, value in traits.items(): + values.add(str(trait_id)) + if isinstance(value, str): + values.add(value) + + class _StringStore: __slots__ = ("_allocated", "_next", "values") diff --git a/packages/smithy-python/src/smithy_python/symbols.py b/packages/smithy-python/src/smithy_python/symbols.py index 1fe1d417e..7256888f9 100644 --- a/packages/smithy-python/src/smithy_python/symbols.py +++ b/packages/smithy-python/src/smithy_python/symbols.py @@ -54,11 +54,13 @@ def __init__( package: str, reserved_member_names: Callable[[Shape], Collection[str]] | None = None, ) -> None: - if any( + if not _DISTRIBUTION_NAME.fullmatch(package) or any( not part.isidentifier() or unicodedata.normalize("NFKC", part) in _KEYWORDS for part in package.split(".") ): - raise ValueError(f"Invalid Python package name: {package!r}") + raise ValueError( + f"Invalid Python package or distribution name: {package!r}" + ) self._model = model self._selection = selection self._package = package @@ -69,6 +71,13 @@ def __init__( self._union_variant_names: dict[ShapeId, str] = {} self._union_unknown_names: dict[ShapeId, str] = {} self._selected = frozenset(shape.id for shape in selection.shapes) + self._reserved_declarations = _RESERVED_DECLARATIONS + if any( + shape.type not in _ENUMS + and any(member.target == _UNIT for member in shape.members.values()) + for shape in selection.shapes + ): + self._reserved_declarations |= {"Unit"} self._shape_names: frozenset[str] = frozenset() declarations: dict[str, ShapeId] = {} for shape in selection.shapes: @@ -84,7 +93,7 @@ def __init__( members: dict[str, ShapeId] = {} for member in shape.members.values(): self._check_collision(members, self.member_name(member.id), member.id) - self._shape_names = frozenset(declarations) + self._shape_names = frozenset((*declarations, *self._reserved_declarations)) for shape in selection.shapes: if shape.type is not ShapeType.UNION or shape.has_trait("streaming"): continue @@ -106,7 +115,7 @@ def declaration_name(self, shape_id: ShapeId | str) -> str: raise ModelError(f"No generated declaration for {shape.id}") service = self._selection.service name = service.rename.get(shape.id, shape.id.name) if service else shape.id.name - result = _name(name, pascal=True) + result = _name(name, pascal=True, reserved=self._reserved_declarations) if not re.fullmatch(r"[A-Za-z0-9_]+", name) or not result.isidentifier(): raise ModelError( f"Invalid Python declaration name {result!r} for {shape.id}" @@ -211,7 +220,7 @@ def type_reference(self, shape_id: ShapeId | str) -> TypeReference: pending.append((current, True)) pending.extend((target, False) for target in reversed(targets)) elif current == ShapeId("smithy.api", "Unit"): - resolved[current] = TypeReference("None") + resolved[current] = TypeReference("Unit", f"{self._package}.models") elif shape.type in (ShapeType.STRUCTURE, ShapeType.UNION): resolved[current] = TypeReference( self.declaration_name(current), f"{self._package}.models" @@ -237,6 +246,7 @@ def member_name(self, shape_id: ShapeId | str) -> str: return _name( member.name, constant=container.type in _ENUMS, + preserve_underscores=True, reserved=self._reserved_member_names.get(container.id, ()), ) @@ -279,6 +289,9 @@ def _shape(self, shape_id: ShapeId | str) -> Shape: "not or pass raise return try while with yield".split() ) _ENUMS = frozenset({ShapeType.ENUM, ShapeType.INT_ENUM}) +_RESERVED_DECLARATIONS = frozenset({"ServiceError"}) +_UNIT = ShapeId("smithy.api", "Unit") +_DISTRIBUTION_NAME = re.compile(r"[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?") _ACRONYM_BOUNDARY = re.compile(r"([A-Z]+)([A-Z][a-z])") _WORD_BOUNDARY = re.compile(r"([a-z0-9])([A-Z])") @@ -321,6 +334,7 @@ def _name( *, pascal: bool = False, constant: bool = False, + preserve_underscores: bool = False, reserved: Collection[str] = (), ) -> str: if pascal: @@ -328,16 +342,34 @@ def _name( else: value = _ACRONYM_BOUNDARY.sub(r"\1_\2", value) value = _WORD_BOUNDARY.sub(r"\1_\2", value) - result = "_".join(word.lower() for word in value.split("_") if word) - if constant: - result = result.upper() + result = ( + value.lower() + if preserve_underscores + else "_".join(word.lower() for word in value.split("_") if word) + ) if result[:1].isdigit(): result = "_" + result - if result in _KEYWORDS or result in reserved: + if result.startswith("__") and not result.endswith("__"): + result += "___" + elif ( + result in _KEYWORDS + or result in reserved + or (constant and result.upper() in reserved) + ): result += "_" + if constant: + result = _escape_enum_underscores(result).upper() return result +def _escape_enum_underscores(name: str) -> str: + if name.startswith("__"): + return name + "_" if name.endswith("__") and not name.endswith("___") else name + if name.startswith("_") and name.endswith("_") and not name.endswith("__"): + return name + "_" + return name + + def _plan_schema_names(selection: Selection) -> dict[ShapeId, str]: names: dict[ShapeId, str] = {} used: set[str] = set() diff --git a/packages/smithy-python/src/smithy_python/types.py b/packages/smithy-python/src/smithy_python/types.py new file mode 100644 index 000000000..ef2b43560 --- /dev/null +++ b/packages/smithy-python/src/smithy_python/types.py @@ -0,0 +1,134 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Generation of complete standalone types packages.""" + +from __future__ import annotations + +import json +from collections.abc import Iterable + +from . import dependencies as _dependencies +from . import generation as _generation +from . import models as _models +from . import schemas as _schemas + +__all__ = ["TypesGenerator"] + + +class TypesGenerator: + """Generate a standalone package containing schemas and lazy models.""" + + def generate( + self, context: _generation.GenerationContext[_generation.TypesSettings] + ) -> None: + """Register every file and dependency in the generation context.""" + _schemas.SchemaGenerator().generate(context) + _models.ModelGenerator().generate(context) + _write_package(context) + + +_GENERATED_COMMENT = "# Code generated by smithy-python-codegen DO NOT EDIT." +_GENERATED_MARKDOWN_COMMENT = ( + "" +) +_PYPROJECT_SUFFIX = """ +[tool.pyright] +typeCheckingMode = "strict" +reportPrivateUsage = false +reportUnusedFunction = false +reportUnusedVariable = false +reportUnnecessaryComparison = false +reportUnusedClass = false +enableExperimentalFeatures = true +exclude = ["**/models.py"] + +[tool.ruff] +target-version = "py312" + +[tool.ruff.lint] +ignore = ["F841"] + +[tool.ruff.format] +skip-magic-trailing-comma = true +""" + + +def _write_package( + context: _generation.GenerationContext[_generation.TypesSettings], +) -> None: + settings = context.settings + package = settings.package + directory = package.replace(".", "/") + description = settings.package_description or f"{package} types package" + + context.dependencies.add(_dependencies.HATCHLING) + context.files.text_file( + "pyproject.toml", + _render_pyproject(settings, context.dependencies, description), + ) + context.files.text_file( + "README.md", + f"{_GENERATED_MARKDOWN_COMMENT}\n\n" + f"## {package} Types Package\n\n{description}\n", + ) + + init = context.files.python_module(f"src/{directory}/__init__.py", package) + init.line("__version__: str = ", repr(settings.package_version)) + context.files.python_module( + f"src/{directory}/_private/__init__.py", f"{package}._private" + ) + context.files.text_file(f"src/{directory}/py.typed", "") + + +def _render_pyproject( + settings: _generation.TypesSettings, + dependencies: _dependencies.DependencySet, + description: str, +) -> str: + package_root = settings.package.split(".", 1)[0] + groups = "\n".join( + f"{group} = {_toml_array(map(_requirement, values))}" + for group in ("test", "docs") + if (values := dependencies.for_group(group)) + ) + if groups: + groups = f"\n\n[dependency-groups]\n{groups}" + + return f"""{_GENERATED_COMMENT} + +[project] +name = {_toml_string(settings.package)} +version = {_toml_string(settings.package_version)} +description = {_toml_string(description)} +readme = "README.md" +requires-python = ">=3.12" +keywords = ["smithy", {_toml_string(settings.package)}] +license = {{text = "Apache-2.0"}} +dependencies = {_toml_array(map(_requirement, dependencies.for_group("runtime")))}{groups} + +[build-system] +requires = {_toml_array(map(_requirement, dependencies.for_group("build")))} +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = {_toml_array((f"src/{package_root}",))} + +{_PYPROJECT_SUFFIX.lstrip()}""" + + +def _toml_array(values: Iterable[str]) -> str: + items = tuple(values) + return ( + "[\n" + "\n".join(f" {_toml_string(value)}," for value in items) + "\n]" + if items + else "[]" + ) + + +def _requirement(dependency: _dependencies.Dependency) -> str: + extras = f"[{','.join(dependency.extras)}]" if dependency.extras else "" + return f"{dependency.package}{extras}{dependency.specifier}" + + +def _toml_string(value: str) -> str: + return json.dumps(value, ensure_ascii=False).replace("\x7f", "\\u007f") diff --git a/packages/smithy-python/src/smithy_python/writer.py b/packages/smithy-python/src/smithy_python/writer.py index d91ffdf2e..efcf45e4b 100644 --- a/packages/smithy-python/src/smithy_python/writer.py +++ b/packages/smithy-python/src/smithy_python/writer.py @@ -58,11 +58,16 @@ class PythonWriter: """Write trusted Python lines, retaining type references until rendering.""" def __init__( - self, module: str, *, dependencies: _dependencies.DependencySet | None = None + self, + module: str, + *, + dependencies: _dependencies.DependencySet | None = None, + stub: bool = False, ) -> None: _validate_module(module, relative=False) self._module = _binding(module) self._dependencies = dependencies + self._stub = stub self._depth = 0 self._lines: list[tuple[int, tuple[str | TypeReference, ...]]] = [] @@ -91,9 +96,10 @@ def render(self) -> str: names, imports = self._plan_imports(*self._names()) header = "from __future__ import annotations\n" if imports: - header += "\n" + "\n".join(item.statement() for item in imports) + "\n" + header += "\n" + _render_imports(imports) + "\n" body = self._body(lambda reference: self._annotation(reference, names)) - return header + ("\n\n" + "\n".join(body) + "\n" if body else "") + separator = "\n" if self._stub else "\n\n" + return header + (separator + "\n".join(body) + "\n" if body else "") def _body(self, reference: Callable[[TypeReference], str]) -> list[str]: body: list[str] = [] @@ -139,6 +145,7 @@ def _plan_imports( self, module_names: frozenset[str], shadows: list[frozenset[str]] ) -> tuple[dict[tuple[str | None, str], str], list[_Import]]: identities: set[tuple[str | None, str]] = set() + third_party_roots: set[str] = set() # The local names visible where each identity is referenced. References # in one scope share a set, so keep the distinct sets, not their union. local_names: dict[tuple[str | None, str], set[frozenset[str]]] = {} @@ -153,6 +160,8 @@ def _plan_imports( if ref.module is not None: _validate_identifier(ref.name, "reference name") _validate_module(ref.module, relative=True) + if ref.dependencies and not ref.module.startswith("."): + third_party_roots.add(_module_root(ref.module)) identities.add((ref.module, ref.name)) key = _identity(ref.module, ref.name) local_names.setdefault(key, set()).add(shadow) @@ -195,7 +204,17 @@ def _plan_imports( else name ) names[(module, name)] = alias - imports.append((_Import(module, name, alias), scoped)) + imports.append( + ( + _Import( + module, + name, + alias, + third_party=_module_root(module) in third_party_roots, + ), + scoped, + ) + ) qualified: set[frozenset[str]] = set() for key, (module, name) in representatives.items(): module_binding, name_binding = key @@ -211,7 +230,9 @@ def _plan_imports( elif module_binding == self._module: names[(module, name)] = name if qualified: - imports.append((_Import("builtins", None, "_builtins"), qualified)) + imports.append( + (_Import("builtins", None, "_builtins", third_party=False), qualified) + ) # Check the complete plan, including unaliased imports. No binding gets # priority merely because its reference was encountered first. @@ -273,12 +294,7 @@ class _Import: module: str name: str | None alias: str - - def statement(self) -> str: - if self.name is None: - return f"import {self.module} as {self.alias}" - suffix = f" as {self.alias}" if self.alias != self.name else "" - return f"from {self.module} import {self.name}{suffix}" + third_party: bool @dataclass(eq=False, slots=True) @@ -319,11 +335,50 @@ def _binding(name: str) -> str: return normalize("NFKC", name) +def _render_imports(imports: list[_Import]) -> str: + sections: dict[int, dict[str, list[_Import]]] = {} + for item in imports: + section = 2 if item.module.startswith(".") else 1 if item.third_party else 0 + sections.setdefault(section, {}).setdefault(item.module, []).append(item) + + rendered: list[str] = [] + for section in sorted(sections): + lines: list[str] = [] + for module, items in sections[section].items(): + for item in items: + if item.name is None: + lines.append(f"import {module} as {item.alias}") + named = [item for item in items if item.name is not None] + if named: + lines.extend(_render_from_import(module, named)) + rendered.append("\n".join(lines)) + return "\n\n".join(rendered) + + +def _render_from_import(module: str, imports: list[_Import]) -> list[str]: + names: list[str] = [] + for item in imports: + if item.name is None: + continue + names.append( + item.name if item.alias == item.name else f"{item.name} as {item.alias}" + ) + prefix = f"from {module} import " + joined = ", ".join(names) + if len(prefix) + len(joined) <= 88: + return [prefix + joined] + return [prefix + "(", *(f" {name}," for name in names), ")"] + + def _identity(module: str | None, name: str) -> tuple[str | None, str]: """Return a reference's module and name as Python binds them.""" return (None if module is None else _binding(module), _binding(name)) +def _module_root(module: str) -> str: + return _binding(module.lstrip(".").split(".", 1)[0]) + + def _validate_identifier(value: str, kind: str) -> None: """Require a Python identifier that is not a hard keyword.""" if not value.isidentifier() or keyword.iskeyword(_binding(value)): diff --git a/packages/smithy-python/tests/unit/test_cli.py b/packages/smithy-python/tests/unit/test_cli.py index 6f08e6178..a0d554608 100644 --- a/packages/smithy-python/tests/unit/test_cli.py +++ b/packages/smithy-python/tests/unit/test_cli.py @@ -3,8 +3,11 @@ from __future__ import annotations +import importlib +import json import subprocess import sys +import tomllib from io import BytesIO from pathlib import Path @@ -20,6 +23,27 @@ def isatty(self) -> bool: return True +def _types_args( + output: Path, + *options: str, + package: str = "example_types", + version: str = "1.2.3", +) -> tuple[str, ...]: + args = ("generate", "types", "--output", str(output)) + return (*args, "--package", package, "--package-version", version, *options) + + +def _run_types( + output: Path, + *options: str, + source: bytes = VALID_MODEL.encode(), + package: str = "example_types", + version: str = "1.2.3", +) -> int: + args = _types_args(output, *options, package=package, version=version) + return main(args, environ={}, stdin=BytesIO(source)) + + @pytest.mark.parametrize( ("argv", "expected"), [ @@ -34,9 +58,7 @@ def test_information_commands( assert capsys.readouterr().out.startswith(expected) -@pytest.mark.parametrize("artifact", ["client", "types"]) -def test_generation_commands_are_explicitly_unavailable( - artifact: str, +def test_client_generation_is_explicitly_unavailable( tmp_path: Path, capsys: pytest.CaptureFixture[str], ) -> None: @@ -47,7 +69,7 @@ def test_generation_commands_are_explicitly_unavailable( main( ( "generate", - artifact, + "client", "--model", str(model), "--output", @@ -58,8 +80,147 @@ def test_generation_commands_are_explicitly_unavailable( == 1 ) assert capsys.readouterr().err == ( - f"smithy-python: error: {artifact} generation is not implemented yet\n" + "smithy-python: error: client generation is not implemented yet\n" + ) + + +def test_types_generation_writes_complete_package( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + model = tmp_path / "model.json" + model.write_text( + '{"smithy": "2.0", "shapes": {"example#Data": {"type": "structure"}}}' ) + output = tmp_path / "output" + + assert _run_types(output, "--model", str(model)) == 0 + assert capsys.readouterr().err == "" + assert { + path.relative_to(output).as_posix() + for path in output.rglob("*") + if path.is_file() + } == { + ".smithy-python-manifest.json", + "README.md", + "pyproject.toml", + "src/example_types/__init__.py", + "src/example_types/_private/__init__.py", + "src/example_types/_private/schemas.py", + "src/example_types/models.py", + "src/example_types/models.pyi", + "src/example_types/py.typed", + } + project = tomllib.loads((output / "pyproject.toml").read_text()) + assert project["project"] == { + "name": "example_types", + "version": "1.2.3", + "description": "example_types types package", + "readme": "README.md", + "requires-python": ">=3.12", + "keywords": ["smithy", "example_types"], + "license": {"text": "Apache-2.0"}, + "dependencies": ["smithy-core~=0.10.0"], + } + assert project["build-system"] == { + "requires": ["hatchling"], + "build-backend": "hatchling.build", + } + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(output / "src") + ) + package = importlib.import_module("example_types") + models = importlib.import_module("example_types.models") + assert package.__version__ == "1.2.3" + assert models.Data.__module__ == "example_types.models" + + +def test_types_generation_escapes_del_in_toml_metadata(tmp_path: Path) -> None: + output = tmp_path / "output" + description = "before\x7fafter" + + assert _run_types(output, "--package-description", description) == 0 + pyproject = (output / "pyproject.toml").read_text() + assert "\\u007f" in pyproject + assert tomllib.loads(pyproject)["project"]["description"] == description + + +def test_types_generation_applies_selector_and_input_output_settings( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + source = json.dumps( + { + "smithy": "2.0", + "shapes": { + "example#Operation": { + "type": "operation", + "input": {"target": "example#Input"}, + "output": {"target": "example#Output"}, + }, + "example#Input": {"type": "structure"}, + "example#Output": {"type": "structure"}, + "example#Selected": {"type": "structure"}, + "example#Excluded": {"type": "structure"}, + }, + } + ).encode() + selected = tmp_path / "selected" + assert ( + _run_types( + selected, + "--selector", + "[id|name=Selected]", + source=source, + ) + == 0 + ) + stub = (selected / "src/example_types/models.pyi").read_text() + assert "class Selected(" in stub + assert "class Excluded(" not in stub + assert "class Input(" not in stub + assert "class Output(" not in stub + + complete = tmp_path / "complete" + assert ( + _run_types( + complete, + "--generate-inputs-and-outputs", + source=source, + ) + == 0 + ) + stub = (complete / "src/example_types/models.pyi").read_text() + assert "class Input(" in stub + assert "class Output(" in stub + assert capsys.readouterr().err == "" + + +def test_types_generation_packages_dotted_modules_from_namespace_root( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + output = tmp_path / "output" + assert ( + _run_types( + output, + package="generated.nested_types", + version="1.0.0", + ) + == 0 + ) + project = tomllib.loads((output / "pyproject.toml").read_text()) + wheel = project["tool"]["hatch"]["build"]["targets"]["wheel"] + assert wheel["packages"] == ["src/generated"] + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(output / "src") + ) + package = importlib.import_module("generated.nested_types") + assert package.__version__ == "1.0.0" + assert capsys.readouterr().err == "" @pytest.mark.parametrize( @@ -127,26 +288,73 @@ def test_invalid_model_is_a_model_failure( model = tmp_path / "model.json" model.write_bytes(source) - assert ( - main( - ( - "generate", - "types", - "--model", - str(model), - "--output", - str(tmp_path / "output"), - ), - environ={}, - ) - == 1 - ) + assert _run_types(tmp_path / "output", "--model", str(model)) == 1 err = capsys.readouterr().err assert err.startswith("smithy-python: error: ") assert expected in err assert "not implemented" not in err +@pytest.mark.parametrize("option", ["--package", "--package-version"]) +def test_types_generation_requires_package_settings( + option: str, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + output = tmp_path / "output" + args = list(_types_args(output)) + index = args.index(option) + del args[index : index + 2] + + assert main(args, environ={}, stdin=BytesIO(VALID_MODEL.encode())) == 2 + assert f"the following arguments are required: {option}" in capsys.readouterr().err + assert not output.exists() + + +@pytest.mark.parametrize( + ("option", "value", "expected"), + [ + ( + "--package", + "invalid-package", + "Invalid Python package or distribution name", + ), + ("--package", "_example", "Invalid Python package or distribution name"), + ("--package", "example_", "Invalid Python package or distribution name"), + ("--package", "café", "Invalid Python package or distribution name"), + ("--package-version", "", "expected a non-empty value"), + ("--package-version", "not-a-version", "Invalid version"), + ("--selector", "[", "Expected an attribute path segment"), + ], +) +def test_types_generation_rejects_invalid_settings( + option: str, + value: str, + expected: str, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + output = tmp_path / "output" + args = list(_types_args(output)) + index = args.index(option) if option in args else len(args) + if option in args: + args[index + 1] = value + else: + args.extend((option, value)) + + assert main(args, environ={}, stdin=BytesIO(VALID_MODEL.encode())) == 2 + assert expected in capsys.readouterr().err + assert not output.exists() + + +def test_types_generation_normalizes_package_version(tmp_path: Path) -> None: + output = tmp_path / "output" + + assert _run_types(output, version="v1.2") == 0 + project = tomllib.loads((output / "pyproject.toml").read_text()) + assert project["project"]["version"] == "1.2" + + @pytest.mark.parametrize("option", ["--model", "--output"]) def test_run_plugin_rejects_direct_invocation_options( option: str, diff --git a/packages/smithy-python/tests/unit/test_generation.py b/packages/smithy-python/tests/unit/test_generation.py index 2b001d0ab..f4a3e6b29 100644 --- a/packages/smithy-python/tests/unit/test_generation.py +++ b/packages/smithy-python/tests/unit/test_generation.py @@ -33,6 +33,15 @@ def test_types_settings_default_to_identity_selection() -> None: ) assert settings.selector == Selector.identity() assert not settings.generate_inputs_and_outputs + assert settings.schema_partition_size == 250 + assert settings.stub_partition_size == 250 + + +def test_types_settings_require_positive_partition_size() -> None: + with pytest.raises(ValueError, match="Partition sizes must be positive"): + TypesSettings("example", "1.0.0", None, schema_partition_size=0) + with pytest.raises(ValueError, match="Partition sizes must be positive"): + TypesSettings("example", "1.0.0", None, stub_partition_size=0) def test_generation_context_shares_manifest_dependencies() -> None: diff --git a/packages/smithy-python/tests/unit/test_models.py b/packages/smithy-python/tests/unit/test_models.py new file mode 100644 index 000000000..ef5b09cc4 --- /dev/null +++ b/packages/smithy-python/tests/unit/test_models.py @@ -0,0 +1,955 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# pyright: reportPrivateUsage=false + +from __future__ import annotations + +import importlib +import json +import pickle +import subprocess +import sys +from collections.abc import Mapping +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Any + +import pytest +from smithy_core.types import UnknownEnumMixin +from smithy_json import JSONCodec +from smithy_python.dependencies import SMITHY_CORE, DependencySet +from smithy_python.files import FileManifest +from smithy_python.generation import GenerationContext, GenerationPlan, TypesSettings +from smithy_python.model import load_model +from smithy_python.models import ModelGenerator, _model_reserved_member_names +from smithy_python.schemas import SchemaGenerator +from smithy_python.selection import select_type_shapes +from smithy_python.symbols import SymbolProvider + +_PACKAGE = "generated_types" + + +def _run_tool(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "-m", *args], + capture_output=True, + text=True, + check=False, + ) + + +def _context( + model_data: Mapping[str, object], + *, + service_id: str | None = "example#Service", + package: str = _PACKAGE, + stub_partition_size: int | None = 250, +) -> GenerationContext[TypesSettings]: + model = load_model(model_data) + selection = select_type_shapes( + model, + service_id=service_id, + generate_inputs_and_outputs=True, + ) + settings = TypesSettings( + package, + "1.0.0", + None, + stub_partition_size=stub_partition_size, + ) + symbols = SymbolProvider( + model, + selection, + package=settings.package, + reserved_member_names=_model_reserved_member_names(model), + ) + dependencies = DependencySet() + return GenerationContext( + model=model, + selection=selection, + settings=settings, + symbols=symbols, + plan=GenerationPlan.from_selection(model, selection, symbols), + dependencies=dependencies, + files=FileManifest(dependencies), + ) + + +def _model() -> dict[str, object]: + return { + "smithy": "2.0", + "shapes": { + "example#Service": { + "type": "service", + "version": "1", + "operations": [{"target": "example#Operation"}], + "rename": { + "example#Original": "Renamed", + "example#Choice": "Selection", + "example#Color": "Tone", + }, + }, + "example#Operation": { + "type": "operation", + "input": {"target": "example#Input"}, + "output": {"target": "example#Output"}, + "errors": [{"target": "example#Problem"}], + }, + "example#Input": { + "type": "structure", + "traits": {"smithy.api#documentation": "Input docs."}, + "members": { + "required": { + "target": "smithy.api#String", + "traits": {"smithy.api#required": {}}, + }, + "list": {"target": "smithy.api#String"}, + "renamed": {"target": "example#Original"}, + "choice": {"target": "example#Choice"}, + "color": {"target": "example#Color"}, + "stream": {"target": "example#Stream"}, + }, + }, + "example#Output": { + "type": "structure", + "members": { + "responseMetadata": {"target": "smithy.api#String"}, + "code": {"target": "example#Code"}, + }, + }, + "example#Problem": { + "type": "structure", + "traits": { + "smithy.api#error": "client", + "smithy.api#documentation": "Problem docs.", + }, + "members": { + "message": { + "target": "smithy.api#String", + "traits": {"smithy.api#sensitive": {}}, + }, + "fault": {"target": "smithy.api#String"}, + "detail": {"target": "example#Original"}, + }, + }, + "example#Original": { + "type": "structure", + "traits": {"smithy.api#documentation": "Renamed structure."}, + "members": { + "value": { + "target": "smithy.api#String", + "traits": { + "smithy.api#required": {}, + "smithy.api#documentation": "Value docs.", + }, + }, + "count": { + "target": "smithy.api#PrimitiveInteger", + "traits": {"smithy.api#required": {}}, + }, + }, + }, + "example#Choice": { + "type": "union", + "traits": {"smithy.api#documentation": "Choice docs."}, + "members": { + "text": { + "target": "smithy.api#String", + "traits": {"smithy.api#documentation": "Text docs."}, + }, + "item": {"target": "example#Original"}, + }, + }, + "example#Color": { + "type": "enum", + "traits": {"smithy.api#documentation": "Color docs."}, + "members": { + "RED": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "red", + "smithy.api#documentation": "Red docs.", + }, + } + }, + }, + "example#Code": { + "type": "intEnum", + "members": { + "OK": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": 200}, + } + }, + }, + "example#Stream": { + "type": "blob", + "traits": {"smithy.api#streaming": {}}, + }, + }, + } + + +def _generate( + tmp_path: Path, + *, + stub_partition_size: int | None = 250, +) -> tuple[GenerationContext[TypesSettings], Path]: + context = _context( + _model(), + stub_partition_size=stub_partition_size, + ) + SchemaGenerator().generate(context) + ModelGenerator().generate(context) + context.files.text_file(f"src/{_PACKAGE}/__init__.py", "") + context.files.text_file(f"src/{_PACKAGE}/_private/__init__.py", "") + context.files.commit(tmp_path) + return context, tmp_path / "src" + + +def _generate_custom( + tmp_path: Path, + model: Mapping[str, object], + package: str, +) -> Path: + context = _context(model, service_id=None, package=package) + SchemaGenerator().generate(context) + ModelGenerator().generate(context) + context.files.text_file(f"src/{package}/__init__.py", "") + context.files.text_file(f"src/{package}/_private/__init__.py", "") + context.files.commit(tmp_path) + return tmp_path / "src" + + +def test_model_plan_precomputes_registry_metadata() -> None: + plan = _context(_model()).plan.models + + assert plan.renames == { + "Renamed": "RENAMED", + "Selection": "SELECTION", + "Tone": "TONE", + } + assert "Original" not in plan.registry_index + assert plan.registry_index["SelectionText"] == "SELECTION" + assert plan.registry_index["SelectionUnknown"] == "SELECTION" + assert plan.target_names["example#Original"] == "Renamed" + assert plan.target_names["example#Choice"] == "Selection" + assert plan.target_names["example#Color"] == "Tone" + assert tuple(plan.registry_index) == ( + "Input", + "Output", + "Problem", + "Renamed", + "Selection", + "SelectionText", + "SelectionItem", + "SelectionUnknown", + "Tone", + "Code", + ) + assert plan.shape_names == { + "Input", + "Output", + "Problem", + "Renamed", + "Selection", + "Tone", + "Code", + } + + +def test_model_generator_emits_runtime_and_stub(tmp_path: Path) -> None: + context, source_root = _generate(tmp_path) + + runtime = (source_root / _PACKAGE / "models.py").read_text() + stub = (source_root / _PACKAGE / "models.pyi").read_text() + assert "from smithy_core._dynamic_models import ModelRegistry" in runtime + service_error_doc = ( + "class ServiceError(ModeledError):\n" + ' """\n' + " Base error for all errors in the service.\n" + "\n" + " Some exceptions do not extend from this class, including synthetic,\n" + " implicit, and shared exception types.\n" + ' """' + ) + assert service_error_doc in runtime + assert '"Renamed": "RENAMED"' in runtime + assert '"Original": "RENAMED"' not in runtime + assert "_MODEL_NAMES = tuple(_MODEL_INDEX)" in runtime + assert '__all__ = ("ServiceError", *_MODEL_NAMES)' in runtime + assert "globals()[name] = value" in runtime + assert "def __dir__() -> list[str]:" in runtime + + assert "class Input(_SerdeShape):" in stub + assert "from smithy_core.aio.interfaces import StreamingBlob" in stub + assert service_error_doc in stub + assert '"Input docs."' in stub + assert "required: str | None = ..." in stub + assert "list_: str | None = ..." in stub + assert "stream: StreamingBlob | None = ..." in stub + assert "class Problem(ServiceError, _SerdeShape):" in stub + assert "message:" not in stub + assert "fault_: str | None = ..." in stub + assert "class Renamed(_SerdeShape):" in stub + assert '"Value docs."' in stub + assert "count: int | None = ..." in stub + assert "class SelectionText(_SerdeShape):" in stub + assert "Selection = SelectionText | SelectionItem | SelectionUnknown" in stub + assert "class Tone(UnknownEnumMixin, StrEnum):" in stub + assert 'RED = "red"' in stub + assert "class Code(UnknownEnumMixin, IntEnum):" in stub + assert "OK = 200" in stub + assert "response_metadata_: str | None = ..." in stub + assert "response_metadata: ResponseMetadata = ..." in stub + assert SMITHY_CORE in context.dependencies + + +def test_generated_models_are_lazy_and_runtime_compatible( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _, source_root = _generate(tmp_path) + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{_PACKAGE}.models") + + assert "Renamed" not in vars(models) + assert "Renamed" in dir(models) + renamed = models.Renamed + assert vars(models)["Renamed"] is renamed + assert models.Renamed is renamed + assert renamed.__module__ == f"{_PACKAGE}.models" + value = renamed(value="hello") + assert value.value == "hello" + assert value.count == 0 + assert pickle.loads(pickle.dumps(value)) == value + + def get_input(_: int) -> type[Any]: + return models.Input + + with ThreadPoolExecutor(max_workers=8) as executor: + inputs = tuple(executor.map(get_input, range(32))) + assert len({id(model) for model in inputs}) == 1 + input_ = inputs[0]( + required="yes", + list_="value", + renamed=value, + choice=models.SelectionText(value="selected"), + color=models.Tone.RED, + ) + assert input_.list_ == "value" + assert input_.renamed is value + + stream = b"stream" + stream_input = models.Input(stream=stream) + assert stream_input.stream is stream + + class StreamSerializer: + value: object | None = None + + def write_data_stream(self, schema: Any, value: object) -> None: + self.value = value + + serializer = StreamSerializer() + stream_input.serialize_members(serializer) + assert serializer.value is stream + + class StreamDeserializer: + def read_struct(self, schema: Any, consumer: Any) -> None: + member = next( + member + for member in schema.members_by_index + if member.expect_member_name() == "stream" + ) + consumer(member, self) + + def read_data_stream(self, schema: Any) -> bytes: + return stream + + decoded_stream = models.Input.deserialize(StreamDeserializer()) + assert decoded_stream.stream is stream + + decoded = JSONCodec().deserialize( + ( + b'{"required":"yes","renamed":{"value":"hello"},' + b'"choice":{"text":"selected"},"color":"red"}' + ), + inputs[0], + ) + assert type(decoded.renamed) is renamed + assert decoded.renamed.value == "hello" + assert type(decoded.choice) is models.SelectionText + assert decoded.choice.value == "selected" + assert decoded.color is models.Tone.RED + assert json.loads(JSONCodec().serialize(input_)) == { + "required": "yes", + "list": "value", + "renamed": {"value": "hello", "count": 0}, + "choice": {"text": "selected"}, + "color": "red", + } + + assert issubclass(models.Problem, models.ServiceError) + problem = models.Problem("bad", fault_="client", detail=value) + assert problem.message == "bad" + assert problem.fault_ == "client" + assert "bad" not in repr(problem) + assert models.Problem().message == "" + + assert models.Tone.RED == "red" + assert issubclass(models.Tone, UnknownEnumMixin) + assert models.Code.OK == 200 + assert models.SelectionText(value="text").value == "text" + assert models.SelectionItem(value=value).value is value + assert models.SelectionUnknown(tag="future").tag == "future" + + namespace: dict[str, object] = {} + exec(f"from {_PACKAGE}.models import *", namespace) + assert tuple(name for name in models.__all__ if name not in namespace) == () + assert "Original" not in models.__all__ + with pytest.raises(AttributeError): + models.Original + + +def test_partitioned_stub_preserves_public_models( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _, source_root = _generate( + tmp_path, + stub_partition_size=2, + ) + package = source_root / _PACKAGE + facade = (package / "models.pyi").read_text() + partitions = sorted((package / "_private").glob("models_*.pyi")) + assert len(partitions) == 4 + assert "class Input(" not in facade + assert "from ._private.models_0 import (" in facade + assert "Input," in facade + assert "Input as Input" not in facade + assert "_SelectionDeserializer" not in facade + assert any("class Input(" in path.read_text() for path in partitions) + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{_PACKAGE}.models") + assert models.Renamed(value="value").value == "value" + assert models.SelectionText(value="value").value == "value" + + generated = tuple( + str(path) + for path in sorted(package.rglob("*")) + if path.suffix in {".py", ".pyi"} + ) + result = _run_tool("ruff", "check", "--isolated", *generated) + assert result.returncode == 0, result.stdout + result.stderr + + +@pytest.mark.parametrize(("declaration_count", "partition_count"), [(250, 0), (251, 2)]) +def test_stub_partition_threshold( + tmp_path: Path, + declaration_count: int, + partition_count: int, +) -> None: + package = "generated_threshold" + context = _context( + { + "smithy": "2.0", + "shapes": { + f"example#Shape{index}": {"type": "structure"} + for index in range(declaration_count) + }, + }, + service_id=None, + package=package, + ) + ModelGenerator().generate(context) + context.files.commit(tmp_path) + + source = tmp_path / f"src/{package}" + assert len(tuple((source / "_private").glob("models_*.pyi"))) == partition_count + facade = (source / "models.pyi").read_text() + assert ("._private.models_0" in facade) is bool(partition_count) + + +def test_generated_model_names_do_not_shadow_runtime_helpers( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + package = "generated_collisions" + source_root = _generate_custom( + tmp_path, + { + "smithy": "2.0", + "shapes": { + f"example#{name}": {"type": "structure"} + for name in ("ServiceError", "ModelRegistry", "ModeledError") + }, + }, + package, + ) + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{package}.models") + + assert models.ServiceError_.__name__ == "ServiceError_" + assert models.ModelRegistry.__name__ == "ModelRegistry" + assert models.ModeledError.__name__ == "ModeledError" + assert models.ServiceError is not models.ServiceError_ + assert {"ServiceError_", "ModelRegistry", "ModeledError"} <= set(models.__all__) + + +def test_explicit_input_and_output_traits_match_runtime_and_stub( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + package = "generated_io_traits" + source_root = _generate_custom( + tmp_path, + { + "smithy": "2.0", + "shapes": { + "example#Input": { + "type": "structure", + "traits": {"smithy.api#input": {}}, + "members": { + "required": { + "target": "smithy.api#String", + "traits": {"smithy.api#required": {}}, + } + }, + }, + "example#Output": { + "type": "structure", + "traits": {"smithy.api#output": {}}, + "members": {"responseMetadata": {"target": "smithy.api#String"}}, + }, + }, + }, + package, + ) + stub = (source_root / package / "models.pyi").read_text() + assert "required: str | None = ..." in stub + assert "response_metadata_: str | None = ..." in stub + assert "response_metadata: ResponseMetadata = ..." in stub + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{package}.models") + assert models.Input().required is None + output = models.Output(response_metadata_="modeled") + assert output.response_metadata_ == "modeled" + assert output.response_metadata is not None + + +def test_stub_and_runtime_use_the_same_member_names( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + package = "generated_member_names" + source_root = _generate_custom( + tmp_path, + { + "smithy": "2.0", + "shapes": { + "example#Leading": { + "type": "structure", + "members": {"_value": {"target": "smithy.api#String"}}, + }, + "example#Trailing": { + "type": "structure", + "members": {"value_": {"target": "smithy.api#String"}}, + }, + "example#Repeated": { + "type": "structure", + "members": {"some__value": {"target": "smithy.api#String"}}, + }, + "example#Digits": { + "type": "structure", + "members": { + "_1": {"target": "smithy.api#String"}, + "_2HTTPServer": {"target": "smithy.api#String"}, + }, + }, + "example#RuntimeMethods": { + "type": "structure", + "members": { + "serialize": {"target": "smithy.api#String"}, + "serializeMembers": {"target": "smithy.api#String"}, + "deserialize": {"target": "smithy.api#String"}, + "__init__": {"target": "smithy.api#String"}, + "__post_init__": {"target": "smithy.api#String"}, + "_smithy_default": {"target": "smithy.api#String"}, + "__repr__": {"target": "smithy.api#String"}, + }, + }, + "example#Keywords": { + "type": "enum", + "members": { + name: { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": name.lower()}, + } + for name in ("LIST", "CLASS") + }, + }, + "example#SunderString": { + "type": "enum", + "members": { + "_VALUE_": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "value"}, + }, + "__DUNDER__": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "dunder"}, + }, + }, + }, + "example#SunderInt": { + "type": "intEnum", + "members": { + "_VALUE_": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": 1}, + }, + "__DUNDER__": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": 2}, + }, + }, + }, + "example#EnumContainer": { + "type": "structure", + "members": { + "stringValue": {"target": "example#SunderString"}, + "intValue": {"target": "example#SunderInt"}, + }, + }, + }, + }, + package, + ) + stub = (source_root / package / "models.pyi").read_text() + assert "_value: str | None = ..." in stub + assert "value_: str | None = ..." in stub + assert "some__value: str | None = ..." in stub + assert "_1: str | None = ..." in stub + assert "_2_http_server: str | None = ..." in stub + assert "serialize_: str | None = ..." in stub + assert "serialize_members_: str | None = ..." in stub + assert "deserialize_: str | None = ..." in stub + assert "__init___: str | None = ..." in stub + assert "__post_init___: str | None = ..." in stub + assert "_smithy_default_: str | None = ..." in stub + assert "__repr___: str | None = ..." in stub + assert 'LIST_ = "list"' in stub + assert 'CLASS_ = "class"' in stub + assert ( + 'class SunderString(UnknownEnumMixin, StrEnum):\n _VALUE__ = "value"' in stub + ) + assert "class SunderInt(UnknownEnumMixin, IntEnum):\n _VALUE__ = 1" in stub + assert '__DUNDER___ = "dunder"' in stub + assert "__DUNDER___ = 2" in stub + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{package}.models") + assert models.Leading(_value="leading")._value == "leading" + assert models.Trailing(value_="trailing").value_ == "trailing" + assert models.Repeated(some__value="repeated").some__value == "repeated" + digits = models.Digits(_1="one", _2_http_server="two") + assert digits._1 == "one" + assert digits._2_http_server == "two" + methods = models.RuntimeMethods( + serialize_="serialize", + serialize_members_="members", + deserialize_="deserialize", + __init___="init", + __post_init___="post_init", + _smithy_default_="default", + __repr___="repr", + ) + assert callable(methods.serialize) + assert callable(methods.serialize_members) + assert callable(methods.deserialize) + assert methods.__init___ == "init" + assert methods.__post_init___ == "post_init" + assert methods._smithy_default_ == "default" + assert methods.__repr___ == "repr" + assert isinstance(methods._smithy_default(), models.RuntimeMethods) + assert "RuntimeMethods(" in repr(methods) + assert json.loads(JSONCodec().serialize(methods)) == { + "serialize": "serialize", + "serializeMembers": "members", + "deserialize": "deserialize", + "__init__": "init", + "__post_init__": "post_init", + "_smithy_default": "default", + "__repr__": "repr", + } + assert models.Keywords.LIST_ == "list" + assert models.Keywords.CLASS_ == "class" + assert models.SunderString._VALUE__ == "value" + assert models.SunderInt._VALUE__ == 1 + assert "__DUNDER___" in models.SunderString.__members__ + assert "__DUNDER___" in models.SunderInt.__members__ + value = models.EnumContainer( + string_value=models.SunderString.__DUNDER___, + int_value=models.SunderInt.__DUNDER___, + ) + encoded = JSONCodec().serialize(value) + decoded = JSONCodec().deserialize(encoded, models.EnumContainer) + assert decoded.string_value is models.SunderString.__DUNDER___ + assert decoded.int_value is models.SunderInt.__DUNDER___ + + +def test_unit_union_members_round_trip_as_empty_structures( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + package = "generated_unit_union" + source_root = _generate_custom( + tmp_path, + { + "smithy": "2.0", + "shapes": { + "example#Action": { + "type": "union", + "members": {"quit": {"target": "smithy.api#Unit"}}, + }, + "example#Container": { + "type": "structure", + "members": {"action": {"target": "example#Action"}}, + }, + }, + }, + package, + ) + runtime = (source_root / package / "models.py").read_text() + stub = (source_root / package / "models.pyi").read_text() + assert '_MODEL_NAMES = (*tuple(_MODEL_INDEX), "Unit")' in runtime + assert "class ActionQuit(_SerdeShape):\n value: Unit" in stub + assert "class Unit(_SerdeShape):" in stub + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{package}.models") + value = models.Container(action=models.ActionQuit(value=models.Unit())) + assert json.loads(JSONCodec().serialize(value)) == {"action": {"quit": {}}} + decoded = JSONCodec().deserialize(b'{"action":{"quit":{}}}', models.Container) + assert isinstance(decoded.action, models.ActionQuit) + assert isinstance(decoded.action.value, models.Unit) + assert "Unit" in models.__all__ + + +def test_modeled_unit_does_not_shadow_prelude_unit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + package = "generated_unit_collision" + source_root = _generate_custom( + tmp_path, + { + "smithy": "2.0", + "shapes": { + "example#Unit": { + "type": "structure", + "members": {"value": {"target": "smithy.api#String"}}, + }, + "example#Action": { + "type": "union", + "members": {"quit": {"target": "smithy.api#Unit"}}, + }, + "example#Container": { + "type": "structure", + "members": {"action": {"target": "example#Action"}}, + }, + }, + }, + package, + ) + stub = (source_root / package / "models.pyi").read_text() + assert "class Unit_(_SerdeShape):" in stub + assert "class Unit(_SerdeShape):" in stub + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{package}.models") + assert models.Unit_ is not models.Unit + decoded = JSONCodec().deserialize( + b'{"action":{"quit":{}}}', + models.Container, + ) + assert isinstance(decoded.action.value, models.Unit) + + +@pytest.mark.parametrize("stub_partition_size", [None, 2]) +def test_generated_stub_passes_strict_pyright( + tmp_path: Path, + stub_partition_size: int | None, +) -> None: + _, source_root = _generate(tmp_path, stub_partition_size=stub_partition_size) + (tmp_path / "check.py").write_text( + ( + f"from {_PACKAGE}.models import Input, Problem, Renamed, SelectionText, Tone\n\n" + "renamed = Renamed(value='value')\n" + "Input(required='required', renamed=renamed, " + "choice=SelectionText(value='x'), color=Tone.RED, stream=b'payload')\n" + "problem = Problem('bad')\n" + "message: str = problem.message\n" + ), + encoding="utf-8", + ) + (tmp_path / "pyrightconfig.json").write_text( + json.dumps( + { + "include": ["check.py"], + "extraPaths": [str(source_root)], + "typeCheckingMode": "strict", + } + ), + encoding="utf-8", + ) + result = _run_tool( + "pyright", + "--project", + str(tmp_path / "pyrightconfig.json"), + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_generated_models_pass_ruff(tmp_path: Path) -> None: + _, source_root = _generate(tmp_path) + models = source_root / _PACKAGE / "models.py" + stub = source_root / _PACKAGE / "models.pyi" + for command in (("check",), ("format", "--check")): + result = _run_tool( + "ruff", + *command, + str(models), + str(stub), + "--config", + str(Path(__file__).resolve().parents[4] / "pyproject.toml"), + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_stub_star_exports_and_private_member_names( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + package = "generated_private_names" + source_root = _generate_custom( + tmp_path, + { + "smithy": "2.0", + "shapes": { + "example#_2HTTPServer": { + "type": "structure", + "members": { + "__value": {"target": "smithy.api#String"}, + "__annotations__": {"target": "smithy.api#String"}, + }, + } + }, + }, + package, + ) + stub = (source_root / package / "models.pyi").read_text() + assert '"_2HTTPServer",' in stub + assert "__value___: str | None = ..." in stub + assert "__annotations___: str | None = ..." in stub + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(source_root) + ) + models = importlib.import_module(f"{package}.models") + namespace: dict[str, object] = {} + exec(f"from {package}.models import *", namespace) + assert namespace["_2HTTPServer"] is models._2HTTPServer + value = models._2HTTPServer( + __value___="value", + __annotations___="annotations", + ) + assert value.__value___ == "value" + assert value.__annotations___ == "annotations" + + (tmp_path / "check.py").write_text( + ( + f"from {package}.models import *\n\n" + "value = _2HTTPServer(\n" + " __value___='value',\n" + " __annotations___='annotations',\n" + ")\n" + "member: str | None = value.__value___\n" + ), + encoding="utf-8", + ) + (tmp_path / "pyrightconfig.json").write_text( + json.dumps( + { + "include": ["check.py"], + "extraPaths": [str(source_root)], + "typeCheckingMode": "strict", + } + ), + encoding="utf-8", + ) + result = _run_tool( + "pyright", + "--project", + str(tmp_path / "pyrightconfig.json"), + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_model_output_is_deterministic(tmp_path: Path) -> None: + outputs: list[tuple[str, str]] = [] + for index in range(2): + _, source_root = _generate(tmp_path / str(index)) + package = source_root / _PACKAGE + outputs.append( + ( + (package / "models.py").read_text(), + (package / "models.pyi").read_text(), + ) + ) + assert outputs[0] == outputs[1] + + +def test_model_generation_without_service(tmp_path: Path) -> None: + context = _context( + { + "smithy": "2.0", + "shapes": { + "example#Standalone": { + "type": "structure", + "members": {"value": {"target": "smithy.api#String"}}, + } + }, + }, + service_id=None, + ) + SchemaGenerator().generate(context) + ModelGenerator().generate(context) + context.files.commit(tmp_path) + + runtime = (tmp_path / "src" / _PACKAGE / "models.py").read_text() + stub = (tmp_path / "src" / _PACKAGE / "models.pyi").read_text() + assert '"Standalone": "STANDALONE"' in runtime + assert "class Standalone(_SerdeShape):" in stub + + +def teardown_module() -> None: + for name in tuple(sys.modules): + if name == _PACKAGE or name.startswith(f"{_PACKAGE}."): + del sys.modules[name] diff --git a/packages/smithy-python/tests/unit/test_schemas.py b/packages/smithy-python/tests/unit/test_schemas.py index ca5bc72cb..3a8b81f06 100644 --- a/packages/smithy-python/tests/unit/test_schemas.py +++ b/packages/smithy-python/tests/unit/test_schemas.py @@ -4,7 +4,9 @@ from __future__ import annotations +import importlib import runpy +import sys from collections.abc import Mapping from pathlib import Path from types import MappingProxyType @@ -32,10 +34,19 @@ from smithy_python.symbols import SymbolProvider -def _context(model_data: Mapping[str, object]) -> GenerationContext[TypesSettings]: +def _context( + model_data: Mapping[str, object], + *, + schema_partition_size: int | None = 250, +) -> GenerationContext[TypesSettings]: model = load_model(model_data) selection = select_type_shapes(model, generate_inputs_and_outputs=True) - settings = TypesSettings("example.types", "1.0.0", None) + settings = TypesSettings( + "example.types", + "1.0.0", + None, + schema_partition_size=schema_partition_size, + ) symbols = SymbolProvider(model, selection, package=settings.package) dependencies = DependencySet() return GenerationContext( @@ -231,6 +242,158 @@ def test_schema_generator_emits_lazy_hydratable_schemas(tmp_path: Path) -> None: assert str(node.members_by_index[2].expect_member_target().id) == "example#Choice" +def test_schema_generator_partitions_behind_lazy_facade( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context = _context( + { + "smithy": "2.0", + "shapes": { + "example#Leaf": {"type": "string"}, + "example#Node": { + "type": "structure", + "members": {"leaf": {"target": "example#Leaf"}}, + }, + "example#CycleA": { + "type": "structure", + "members": {"b": {"target": "example#CycleB"}}, + }, + "example#CycleB": { + "type": "structure", + "members": {"a": {"target": "example#CycleA"}}, + }, + }, + }, + schema_partition_size=1, + ) + SchemaGenerator().generate(context) + context.files.text_file("src/example/types/__init__.py", "") + context.files.text_file("src/example/types/_private/__init__.py", "") + context.files.commit(tmp_path) + + package = tmp_path / "src/example/types/_private" + assert (package / "schemas.py").is_file() + assert (package / "schemas_0.py").is_file() + assert (package / "schemas_1.py").is_file() + facade_source = (package / "schemas.py").read_text() + assert "_PARTITION_NAMES: tuple[tuple[str, ...], ...]" in facade_source + assert "name: index" in facade_source + assert facade_source.count(".schemas_") == 1 + partitions = tuple(package.glob("schemas_*.py")) + cycle = next( + path.read_text() for path in partitions if "CYCLE_A:" in path.read_text() + ) + assert "CYCLE_B:" in cycle + + monkeypatch.syspath_prepend( # pyright: ignore[reportUnknownMemberType] + str(tmp_path / "src") + ) + facade = importlib.import_module("example.types._private.schemas") + assert "example.types._private.schemas_0" not in sys.modules + assert "example.types._private.schemas_1" not in sys.modules + + static_node = cast(tuple[Any, ...], facade.NODE) + assert facade.NODE is static_node + assert "NODE" in dir(facade) + assert "example.types._private.schemas_1" in sys.modules + assert "example.types._private.schemas_0" not in sys.modules + with pytest.raises(AttributeError): + facade.MISSING + node = _schema_compact.hydrate(static_node) + assert str(node.members["leaf"].expect_member_target().id) == "example#Leaf" + assert "example.types._private.schemas_0" in sys.modules + cycle_a = _schema_compact.hydrate(cast(tuple[Any, ...], facade.CYCLE_A)) + cycle_b = cycle_a.members["b"].expect_member_target() + assert cycle_b.members["a"].expect_member_target() is cycle_a + + +@pytest.mark.parametrize(("shape_count", "partition_count"), [(250, 0), (251, 2)]) +def test_schema_partition_threshold( + tmp_path: Path, + shape_count: int, + partition_count: int, +) -> None: + context = _context( + { + "smithy": "2.0", + "shapes": { + f"example#Shape{index}": {"type": "structure"} + for index in range(shape_count) + }, + }, + ) + SchemaGenerator().generate(context) + context.files.commit(tmp_path) + + private = tmp_path / "src/example/types/_private" + assert len(tuple(private.glob("schemas_*.py"))) == partition_count + facade = (private / "schemas.py").read_text() + assert ("_PARTITIONS" in facade) is bool(partition_count) + + +def test_schema_partitions_balance_emitted_size(tmp_path: Path) -> None: + context = _context( + { + "smithy": "2.0", + "shapes": { + "example#AHeavy": { + "type": "structure", + "members": { + f"member{index}": {"target": "smithy.api#String"} + for index in range(20) + }, + }, + "example#BTiny": {"type": "structure"}, + "example#CTiny": {"type": "structure"}, + "example#DTiny": {"type": "structure"}, + }, + }, + schema_partition_size=2, + ) + SchemaGenerator().generate(context) + context.files.commit(tmp_path) + private = tmp_path / "src/example/types/_private" + + first = (private / "schemas_0.py").read_text() + second = (private / "schemas_1.py").read_text() + assert "A_HEAVY:" in first + assert "B_TINY:" not in first + assert all(f"{name}:" in second for name in ("B_TINY", "C_TINY", "D_TINY")) + + +def test_schema_partition_weights_include_interned_strings(tmp_path: Path) -> None: + context = _context( + { + "smithy": "2.0", + "shapes": { + "example#AHeavy": { + "type": "structure", + "members": { + "value": { + "target": "smithy.api#String", + "traits": {"smithy.api#default": "x" * 10_000}, + } + }, + }, + "example#BTiny": {"type": "structure"}, + "example#CTiny": {"type": "structure"}, + "example#DTiny": {"type": "structure"}, + }, + }, + schema_partition_size=2, + ) + SchemaGenerator().generate(context) + context.files.commit(tmp_path) + private = tmp_path / "src/example/types/_private" + + first = (private / "schemas_0.py").read_text() + second = (private / "schemas_1.py").read_text() + assert "A_HEAVY:" in first + assert "B_TINY:" not in first + assert all(f"{name}:" in second for name in ("B_TINY", "C_TINY", "D_TINY")) + + def test_schema_generator_preserves_primitive_prelude_targets(tmp_path: Path) -> None: context = _context( { diff --git a/packages/smithy-python/tests/unit/test_selection.py b/packages/smithy-python/tests/unit/test_selection.py index a41eaf87e..b24bbeba0 100644 --- a/packages/smithy-python/tests/unit/test_selection.py +++ b/packages/smithy-python/tests/unit/test_selection.py @@ -10,6 +10,16 @@ import pytest from smithy_python.cli import main +_TYPES_OPTIONS = ("--package", "example_types", "--package-version", "1.0.0") + + +def _artifact_args(artifact: str, *options: str) -> list[str]: + args = ["generate", artifact] + if artifact == "types": + args.extend(_TYPES_OPTIONS) + args.extend(options) + return args + @pytest.mark.parametrize("artifact", ["client", "types"]) @pytest.mark.parametrize( @@ -53,19 +63,29 @@ def test_cli_service_selection( } ).encode() output = tmp_path / "output" - argv = ["generate", artifact, "--output", str(output)] + argv = _artifact_args(artifact, "--output", str(output)) if service is not None: argv.extend(("--service", service)) + if artifact == "types" and service in (None, "example#Second"): + expected = 0 + diagnostic = ( + "" + if service is None + else "excluded 1 eligible data shape(s) outside the closure" + ) assert main(argv, environ={}, stdin=BytesIO(source)) == expected captured = capsys.readouterr() assert captured.out == "" - assert diagnostic in captured.err + if diagnostic: + assert diagnostic in captured.err + else: + assert captured.err == "" assert "unrecognized arguments" not in captured.err if service is None: assert "example#Base" not in captured.err if expected == 2: assert "not implemented" not in captured.err - assert not output.exists() + assert output.exists() is (expected == 0) @pytest.mark.parametrize("artifact", ["client", "types"]) @@ -82,18 +102,22 @@ def test_cli_implicit_service( if has_service: shapes["example#Service"] = {"type": "service"} source = json.dumps({"smithy": "2.0", "shapes": shapes}).encode() - requires_service = artifact == "client" and not has_service - assert main( - ("generate", artifact), - environ={"SMITHY_PLUGIN_DIR": str(tmp_path / "output")}, - stdin=BytesIO(source), - ) == (2 if requires_service else 1) + argv = _artifact_args(artifact) + expected = 0 if artifact == "types" else 1 if has_service else 2 + assert ( + main( + argv, + environ={"SMITHY_PLUGIN_DIR": str(tmp_path / "output")}, + stdin=BytesIO(source), + ) + == expected + ) captured = capsys.readouterr() assert captured.out == "" - assert ("not implemented" in captured.err) is not requires_service - if requires_service: + assert ("not implemented" in captured.err) is (artifact == "client" and has_service) + if artifact == "client" and not has_service: assert "service" in captured.err - assert not (tmp_path / "output").exists() + assert (tmp_path / "output").exists() is (artifact == "types") @pytest.mark.parametrize("artifact", ["client", "types"]) @@ -124,19 +148,24 @@ def test_cli_reports_only_excluded_eligible_data( } ).encode() output = tmp_path / "output" - argv = ["generate", artifact, "--service", "example#Service"] + argv = _artifact_args(artifact, "--service", "example#Service") environ = {"SMITHY_PLUGIN_DIR": str(output)} if plugin else {} if not plugin: argv.extend(("--output", str(output))) - assert main(argv, environ=environ, stdin=BytesIO(source)) == 1 + assert main(argv, environ=environ, stdin=BytesIO(source)) == ( + 0 if artifact == "types" else 1 + ) captured = capsys.readouterr() assert captured.out == "" - assert captured.err == ( - "smithy-python: excluded 1 eligible data shape(s) " - "outside the closure of service example#Service\n" - f"smithy-python: error: {artifact} generation is not implemented yet\n" - ) - assert not output.exists() + expected = "" + if artifact == "client": + expected = ( + "smithy-python: excluded 1 eligible data shape(s) " + "outside the closure of service example#Service\n" + "smithy-python: error: client generation is not implemented yet\n" + ) + assert captured.err == expected + assert output.exists() is (artifact == "types") def test_cli_conflicts_are_model_errors( @@ -146,7 +175,7 @@ def test_cli_conflicts_are_model_errors( { "smithy": "2.0", "shapes": { - shape_id: {"type": "string"} + shape_id: {"type": "structure"} for shape_id in ("a#Name", "b#name", "c#NAME", "a#Other", "b#other") }, } @@ -154,7 +183,7 @@ def test_cli_conflicts_are_model_errors( output = tmp_path / "output" assert ( main( - ("generate", "types", "--output", str(output)), + _artifact_args("types", "--output", str(output)), environ={}, stdin=BytesIO(source), ) diff --git a/packages/smithy-python/tests/unit/test_symbols.py b/packages/smithy-python/tests/unit/test_symbols.py index 4e0c77866..62f9e82f1 100644 --- a/packages/smithy-python/tests/unit/test_symbols.py +++ b/packages/smithy-python/tests/unit/test_symbols.py @@ -25,10 +25,13 @@ def model_with(shapes: dict[str, object]) -> Model: ("http_server", "HttpServer", "http_server", "HTTP_SERVER"), ("HTTP2Server", "HTTP2Server", "http2_server", "HTTP2_SERVER"), ("getURL2Value", "GetURL2Value", "get_url2_value", "GET_URL2_VALUE"), - ("__some__name__", "SomeName", "some_name", "SOME_NAME"), - ("__init__", "Init", "init", "INIT"), - ("_value", "Value", "value", "VALUE"), - ("class", "Class", "class_", "CLASS"), + ("__some__name__", "SomeName", "__some__name__", "__SOME__NAME___"), + ("__init__", "Init", "__init__", "__INIT___"), + ("__post_init__", "PostInit", "__post_init__", "__POST_INIT___"), + ("__value", "Value", "__value___", "__VALUE___"), + ("_value", "Value", "_value", "_VALUE"), + ("_VALUE_", "VALUE", "_value_", "_VALUE__"), + ("class", "Class", "class_", "CLASS_"), ("None", "None_", "none", "NONE"), ("match", "Match", "match", "MATCH"), ], @@ -59,6 +62,21 @@ def test_names(source: str, declaration: str, field: str, constant: str) -> None assert model.get_shape(f"example#Choice${source}").get_trait("enumValue") == "wire" +def test_service_error_is_reserved_for_the_synthetic_error_base() -> None: + model = model_with( + { + "example#ServiceError": {"type": "structure"}, + "example#Service": { + "type": "union", + "members": {"error": {"target": "smithy.api#String"}}, + }, + } + ) + symbols = SymbolProvider(model, select_shapes(model), package="example.client") + assert symbols.declaration_name("example#ServiceError") == "ServiceError_" + assert symbols.union_variant_name("example#Service$error") == "Service_Error" + + @pytest.mark.parametrize( ("kind", "name", "module"), [ @@ -99,7 +117,9 @@ def test_type_references(kind: str, name: str, module: str) -> None: if module != "example.client.models" and kind not in ("enum", "intEnum"): prelude_name = kind[0].upper() + kind[1:] assert symbols.type_reference(f"smithy.api#{prelude_name}") == ref - assert symbols.type_reference("smithy.api#Unit") == TypeReference("None") + assert symbols.type_reference("smithy.api#Unit") == TypeReference( + "Unit", "example.client.models" + ) def test_streaming_union_has_a_schema_name_without_a_model_symbol() -> None: @@ -403,7 +423,8 @@ def test_collection_only_cycles(mutual: bool) -> None: @pytest.mark.parametrize("kind", ["structure", "union", "enum", "intEnum"]) @pytest.mark.parametrize( - "names", [("getURL", "get_url"), ("class", "class_"), ("__init__", "init")] + "names", + [("getURL", "get_url"), ("class", "class_"), ("HTTPServer", "http_server")], ) def test_member_collisions(kind: str, names: tuple[str, str]) -> None: from smithy_python.model import ModelError @@ -420,7 +441,7 @@ def test_member_collisions(kind: str, names: tuple[str, str]) -> None: SymbolProvider(model, select_shapes(model), package="pkg") for name in names: assert f"example#Container${name}" in str(error.value) - result = "CLASS" if kind in ("enum", "intEnum") else "class_" + result = "CLASS_" if kind in ("enum", "intEnum") else "class_" if names[0] == "class": assert repr(result) in str(error.value) @@ -491,6 +512,9 @@ def test_only_emitted_declarations_collide() -> None: "pkg.class", "None", "pkg/child", + "_example", + "example_", + "café", "pkg.\uff43\uff4c\uff41\uff53\uff53", ], ) @@ -786,6 +810,54 @@ def test_reserved_member_names_still_collide() -> None: assert "'serialize_'" in str(error.value) +def test_enum_sunder_escaping_still_detects_collisions() -> None: + from smithy_python.model import ModelError + + model = model_with( + { + "example#Values": { + "type": "enum", + "members": { + "_VALUE_": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "first"}, + }, + "_VALUE__": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "second"}, + }, + }, + } + } + ) + with pytest.raises(ModelError, match="_VALUE__"): + SymbolProvider(model, select_shapes(model), package="pkg") + + +def test_enum_dunder_escaping_still_detects_collisions() -> None: + from smithy_python.model import ModelError + + model = model_with( + { + "example#Values": { + "type": "enum", + "members": { + "__VALUE__": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "first"}, + }, + "__VALUE___": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "second"}, + }, + }, + } + } + ) + with pytest.raises(ModelError, match="__VALUE___"): + SymbolProvider(model, select_shapes(model), package="pkg") + + def test_type_references_are_cached() -> None: from smithy_python.model import ModelError diff --git a/packages/smithy-python/tests/unit/test_writer.py b/packages/smithy-python/tests/unit/test_writer.py index 78852e0cc..c7df26470 100644 --- a/packages/smithy-python/tests/unit/test_writer.py +++ b/packages/smithy-python/tests/unit/test_writer.py @@ -13,7 +13,7 @@ from typing import get_type_hints import pytest -from smithy_python.dependencies import Dependency, DependencySet +from smithy_python.dependencies import SMITHY_CORE, Dependency, DependencySet from smithy_python.exceptions import CodegenError from smithy_python.symbols import TypeReference from smithy_python.writer import PythonWriter @@ -220,7 +220,7 @@ def test_class_body_references_load( ")", ) source = writer.render() - assert "from dataclasses import field as _dataclasses_field" in source + assert "field as _dataclasses_field" in source assert "default_factory=_builtins.list" in source module = load_written_module(writer, tmp_path, monkeypatch) node: type = module.Node @@ -600,7 +600,7 @@ def test_builtin_names_cover_running_interpreter() -> None: writer.line("value: ", TypeReference(name, "external")) source = writer.render() for name in names: - assert f"from external import {name} as _external_{name}\n" in source + assert f" {name} as _external_{name},\n" in source @pytest.mark.parametrize("module", ["written_models", "\uff57ritten_models"]) @@ -789,3 +789,28 @@ def test_import_identity_and_order() -> None: assert prefix.index("from datetime") < prefix.index("from decimal") prefixes.append(prefix) assert prefixes[0] == prefixes[1] + + +def test_imports_are_grouped_and_combined() -> None: + writer = PythonWriter("package.models") + writer.line("first: ", TypeReference("StrEnum", "enum")) + writer.line("second: ", TypeReference("IntEnum", "enum")) + writer.line( + "third: ", + TypeReference( + "Document", + "smithy_core.documents", + dependencies=(SMITHY_CORE,), + ), + ) + writer.line("fourth: ", TypeReference("Thing", "._private")) + + assert writer.render().startswith( + "from __future__ import annotations\n" + "\n" + "from enum import IntEnum, StrEnum\n" + "\n" + "from smithy_core.documents import Document\n" + "\n" + "from ._private import Thing\n" + ) diff --git a/pyproject.toml b/pyproject.toml index 8c2cbe851..cb3a1372e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ lint = [ ] typing = [ "pyright>=1.1.400", + "typing-extensions>=4.14.0", ] changelog = [ "jinja2>=3.1.6" diff --git a/scripts/api-snapshot/api.json b/scripts/api-snapshot/api.json index 8c13bf091..71bbeb680 100644 --- a/scripts/api-snapshot/api.json +++ b/scripts/api-snapshot/api.json @@ -1550,6 +1550,9 @@ "load_model": "rex-function", "to_json": "rex-function" }, + "smithy_python.models": { + "ModelGenerator": "class" + }, "smithy_python.schemas": { "SchemaGenerator": "class", "SchemaPlan": "class" @@ -1585,6 +1588,9 @@ "SymbolProvider": "class", "TypeReference": "class" }, + "smithy_python.types": { + "TypesGenerator": "class" + }, "smithy_python.writer": { "CodegenError": "rex-exception", "PythonWriter": "class", diff --git a/scripts/codegen-output-py/generate.py b/scripts/codegen-output-py/generate.py index 54ea35f82..a341e9a95 100644 --- a/scripts/codegen-output-py/generate.py +++ b/scripts/codegen-output-py/generate.py @@ -1,5 +1,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +# pyright: reportPrivateUsage=false """Regenerate the committed Python codegen snapshot.""" import hashlib @@ -8,13 +9,7 @@ import sys from pathlib import Path -from smithy_python.dependencies import DependencySet -from smithy_python.files import FileManifest -from smithy_python.generation import GenerationContext, GenerationPlan, TypesSettings -from smithy_python.model import load_model -from smithy_python.schemas import SchemaGenerator -from smithy_python.selection import select_type_shapes -from smithy_python.symbols import SymbolProvider +from smithy_python.cli import main as generate _ROOT = Path(__file__).resolve().parents[2] _MODEL = Path(__file__).with_name("model.json") @@ -24,30 +19,28 @@ def main() -> None: - model = load_model(_MODEL.read_bytes()) - selection = select_type_shapes( - model, - service_id=_SERVICE, - generate_inputs_and_outputs=True, + result = generate( + ( + "generate", + "types", + "--model", + str(_MODEL), + "--output", + str(_OUTPUT), + "--service", + _SERVICE, + "--package", + _PACKAGE, + "--package-version", + "1.0.0", + "--package-description", + "Example Python codegen output", + "--generate-inputs-and-outputs", + ), + environ={}, ) - settings = TypesSettings( - package=_PACKAGE, - package_version="1.0.0", - package_description="Example Python codegen output", - ) - symbols = SymbolProvider(model, selection, package=settings.package) - dependencies = DependencySet() - context = GenerationContext( - model=model, - selection=selection, - settings=settings, - symbols=symbols, - plan=GenerationPlan.from_selection(model, selection, symbols), - dependencies=dependencies, - files=FileManifest(dependencies), - ) - SchemaGenerator().generate(context) - context.files.commit(_OUTPUT) + if result: + raise SystemExit(result) _format_output() @@ -57,7 +50,7 @@ def _format_output() -> None: args = [sys.executable, "-m", "ruff", command] if command == "check": args.append("--fix") - subprocess.run( + subprocess.run( # noqa: S603 [*args, str(_OUTPUT), "--config", str(config)], check=True, ) diff --git a/uv.lock b/uv.lock index f07a64d38..f7f327293 100644 --- a/uv.lock +++ b/uv.lock @@ -771,6 +771,12 @@ requires-dist = [{ name = "smithy-core", editable = "packages/smithy-core" }] [[package]] name = "smithy-python" source = { editable = "packages/smithy-python" } +dependencies = [ + { name = "packaging" }, +] + +[package.metadata] +requires-dist = [{ name = "packaging" }] [[package]] name = "smithy-python-workspace" @@ -792,6 +798,7 @@ test = [ ] typing = [ { name = "pyright" }, + { name = "typing-extensions" }, ] [package.metadata] @@ -805,7 +812,10 @@ test = [ { name = "pytest-asyncio", specifier = ">=0.25.3" }, { name = "pytest-cov", specifier = ">=6.0.0" }, ] -typing = [{ name = "pyright", specifier = ">=1.1.400" }] +typing = [ + { name = "pyright", specifier = ">=1.1.400" }, + { name = "typing-extensions", specifier = ">=4.14.0" }, +] [[package]] name = "smithy-test" @@ -823,11 +833,11 @@ requires-dist = [{ name = "smithy-core", editable = "packages/smithy-core" }] [[package]] name = "typing-extensions" -version = "4.13.2" +version = "4.16.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f6/37/23083fcd6e35492953e8d2aaaa68b860eb422b34627b13f2ce3eb6106061/typing_extensions-4.13.2.tar.gz", hash = "sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef", size = 106967, upload-time = "2025-04-10T14:19:05.416Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/54/b1ae86c0973cc6f0210b53d508ca3641fb6d0c56823f288d108bc7ab3cc8/typing_extensions-4.13.2-py3-none-any.whl", hash = "sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c", size = 45806, upload-time = "2025-04-10T14:19:03.967Z" }, + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, ] [[package]]