From 13e11e32c28201b4eb2c022c43b4b470e3ed563e Mon Sep 17 00:00:00 2001 From: trim21 Date: Wed, 2 Sep 2026 00:48:59 +0800 Subject: [PATCH] feat: add cgen subcommand to mypyc for building without setuptools --- mypyc/__main__.py | 13 +- mypyc/build.py | 381 +++++++++++++++++++++++++++++++++------- mypyc/cgen.py | 159 +++++++++++++++++ mypyc/test/test_cgen.py | 158 +++++++++++++++++ 4 files changed, 643 insertions(+), 68 deletions(-) create mode 100644 mypyc/cgen.py create mode 100644 mypyc/test/test_cgen.py diff --git a/mypyc/__main__.py b/mypyc/__main__.py index 9b3973710efac..406f5b332d616 100644 --- a/mypyc/__main__.py +++ b/mypyc/__main__.py @@ -5,9 +5,12 @@ $ mypyc foo.py [...] $ python3 -c 'import foo' # Uses compiled 'foo' + $ python3 -m mypyc cgen [--target-dir DIR] SOURCE... # See mypyc.cgen -This is just a thin wrapper that generates a setup.py file that uses -mypycify, suitable for prototyping and testing. +The default invocation is just a thin wrapper that generates a setup.py +file that uses mypycify, suitable for prototyping and testing. The 'cgen' +subcommand instead emits C code and JSON build metadata for use with +build systems other than setuptools. """ from __future__ import annotations @@ -37,6 +40,12 @@ def main() -> None: + if len(sys.argv) > 1 and sys.argv[1] == "cgen": + from mypyc.cgen import main as cgen_main + + cgen_main(sys.argv[2:]) + return + build_dir = "build" # can this be overridden?? try: os.mkdir(build_dir) diff --git a/mypyc/build.py b/mypyc/build.py index 8c6eabead17c9..da3b708d2b183 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -20,15 +20,17 @@ from __future__ import annotations +import copy import hashlib import os.path import re import sys +import sysconfig import time from collections.abc import Iterable +from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, NamedTuple, NoReturn, cast -import mypyc.build_setup # noqa: F401 from mypy.build import BuildSource from mypy.errors import CompileError from mypy.fscache import FileSystemCache @@ -37,7 +39,7 @@ from mypy.util import write_junit_xml from mypyc.annotate import generate_annotated_html from mypyc.codegen import emitmodule -from mypyc.common import IS_FREE_THREADED, RUNTIME_C_FILES, shared_lib_name +from mypyc.common import EXT_SUFFIX, IS_FREE_THREADED, RUNTIME_C_FILES, shared_lib_name from mypyc.errors import Errors from mypyc.ir.deps import SourceDep from mypyc.ir.pprint import format_modules @@ -152,14 +154,15 @@ class ModDesc(NamedTuple): Extension: TypeAlias = _setuptools_Extension | _distutils_Extension -if sys.version_info >= (3, 12): - # From setuptools' monkeypatch - from distutils import ccompiler, sysconfig # type: ignore[import-not-found] -else: - from distutils import ccompiler, sysconfig - def get_extension() -> type[Extension]: + # The build_setup monkeypatch (per-file compile flags) must be active + # before any extension is compiled. It requires distutils, which on + # Python 3.12+ is only available through setuptools, so this is lazy: + # build paths that don't construct setuptools extensions (cgen) don't + # need setuptools at all. + import mypyc.build_setup # noqa: F401 + # We can work with either setuptools or distutils, and pick setuptools # if it has been imported. use_setuptools = "setuptools" in sys.modules @@ -180,6 +183,7 @@ def get_extension() -> type[Extension]: def setup_mypycify_vars() -> None: """Rewrite a bunch of config vars in pretty dubious ways.""" # There has to be a better approach to this. + from distutils import sysconfig # The vars can contain ints but we only work with str ones vars = cast(dict[str, str], sysconfig.get_config_vars()) @@ -451,7 +455,8 @@ def write_file(path: str, contents: str) -> None: except OSError: old_contents = None if old_contents != encoded_contents: - os.makedirs(os.path.dirname(path), exist_ok=True) + if os.path.dirname(path): + os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "wb") as g: g.write(encoded_contents) @@ -672,22 +677,55 @@ def resolve_cfile_deps( if candidate in resolved: break resolved.add(candidate) - # Recurse only into headers. Some lib-rt sources are pulled in as `#include "init.c"` etc.; - # those do not resolve under target_dir so they get filtered out before we would try to scan - # them, but the .h guard is a cheap belt-and-braces. - if candidate.endswith(".h"): - try: - with open(candidate, encoding="utf-8") as f: - header_contents = f.read() - except OSError: - header_contents = "" - sub_dir = os.path.dirname(candidate) - for sub_angled, sub in _extract_includes(header_contents): - worklist.append((sub_dir, sub_angled, sub)) + # Recurse into any resolved file: generated headers include each other, and lib-rt + # sources may be pulled in as `#include "init.c"` once the runtime has been copied + # into target_dir. + try: + with open(candidate, encoding="utf-8") as f: + header_contents = f.read() + except OSError: + continue + sub_dir = os.path.dirname(candidate) + for sub_angled, sub in _extract_includes(header_contents): + worklist.append((sub_dir, sub_angled, sub)) break return resolved +def _copy_runtime_files( + compiler_options: CompilerOptions, source_deps: list[SourceDep] +) -> tuple[list[str], list[str]]: + """Copy the runtime library and source dep files into target_dir. + + Returns (runtime_c_files, extra_include_dirs). If the runtime library is + directly #included instead (CompilerOptions.include_runtime_files), + nothing is copied. + """ + build_dir = compiler_options.target_dir + shared_cfilenames = [] + include_dirs = set() + if not compiler_options.include_runtime_files: + files_to_copy = list(RUNTIME_C_FILES) + for source_dep in source_deps: + files_to_copy.append(source_dep.path) + files_to_copy.append(source_dep.get_header()) + include_dirs.update(source_dep.include_dirs) + + if compiler_options.depends_on_librt_internal: + files_to_copy.append("internal/librt_internal_api.h") + files_to_copy.append("internal/librt_internal_api.c") + include_dirs.add("internal") + + for name in files_to_copy: + rt_file = os.path.join(build_dir, name) + with open(os.path.join(include_dir(), name), encoding="utf-8") as f: + write_file(rt_file, f.read()) + if name.endswith(".c"): + shared_cfilenames.append(rt_file) + + return shared_cfilenames, [os.path.join(include_dir(), dir) for dir in include_dirs] + + def mypyc_build( paths: list[str], compiler_options: CompilerOptions, @@ -755,14 +793,18 @@ def mypyc_build( # For fully-cached groups ctext is empty; read the on-disk .c so the dep resolver # can walk its transitive header chain and populate Extension.depends. Otherwise, # cross-group export-table header changes (e.g. a new class shifting struct offsets) - # won't trigger a recompile of this cached consumer's .o. + # won't trigger a recompile of this cached consumer's .o. If the read fails, fall + # back to no deps (missing a rebuild trigger is better than crashing). if not ctext and os.path.exists(cfile): try: with open(cfile, encoding="utf-8") as _f: ctext = _f.read() except OSError: - pass - per_cfile_deps.append((cfile, get_header_deps([(cfile, ctext)]))) + ctext = "" + if ctext: + per_cfile_deps.append((cfile, get_header_deps([(cfile, ctext)]))) + else: + per_cfile_deps.append((cfile, [])) pending.append(per_cfile_deps) # Second pass: assemble each group's .c filenames and resolve transitive deps now that every group's @@ -782,11 +824,235 @@ def mypyc_build( return groups, group_cfilenames, source_deps +@dataclass(frozen=True) +class GroupSpec: + """A group of modules compiled into one C extension unit. + + group_name is None only for a standalone group containing a single + top-level module (no shared library). Otherwise it is the group name + used for the shared library module (via shared_lib_name) and for the + names of the generated C files. + + modules are the dotted Python module names in the group. + + c_files are the generated .c files of the group (absolute paths), + excluding runtime files and shims. depends are the header files the + generated code includes (absolute paths), for build systems that + track rebuilds. + + shims maps the extension module name of each shim (e.g. "pkg.__init__" + for a package's __init__.py) to its generated .c file. It is empty + unless a shared library is used. + """ + + group_name: str | None + modules: list[str] + c_files: list[str] + depends: list[str] + shims: dict[str, str] = field(default_factory=dict) + + +@dataclass(frozen=True) +class GeneratedC: + """The result of generate_c_sources. + + All generated C files (group sources, runtime library, shims) have been + written into target_dir by the time this is returned. compiler_options + is the (adjusted) options the generation ran with. + """ + + target_dir: str + compiler_options: CompilerOptions + groups: list[GroupSpec] + # Runtime library .c files copied into target_dir. Empty if the runtime + # is directly #included instead (CompilerOptions.include_runtime_files). + runtime_c_files: list[str] + # Additional include directories (under mypyc/lib-rt) required to compile + # the generated code, e.g. for source deps of primitive operations. + extra_include_dirs: list[str] + + +@dataclass(frozen=True) +class ExtensionSpec: + """A backend-neutral description of one C extension module to build. + + module is the full dotted name of the extension module; for a package's + __init__ this follows the setuptools convention of "pkg.__init__". + out_path is where the built file must end up (relative to the root of + the installation target, e.g. site-packages), including the platform + extension suffix. + + cflags are compiler flags required by the generated code; link_args are + the linker flags needed to produce a loadable extension module on this + platform. Backends are additionally responsible for the platform's + standard Python extension linking (e.g. on Windows, linking against + the interpreter's import library). + """ + + module: str + out_path: str + sources: list[str] + include_dirs: list[str] + depends: list[str] + cflags: list[str] + link_args: list[str] + + +def _extension_output_path(module: str) -> str: + return os.path.join(*module.split(".")) + EXT_SUFFIX + + +def _platform_link_args() -> list[str]: + """Link flags needed to produce a loadable extension module on this platform. + + Derived from the linker command Python itself was configured with, minus + the compiler invocation (e.g. '-bundle -undefined dynamic_lookup' on macOS, + '-shared' on Linux). Empty when the platform doesn't define one. + """ + ldshared = str(sysconfig.get_config_var("LDSHARED") or "") + if not ldshared: + return [] + return ldshared.split()[1:] + + +def generate_c_sources( + paths: list[str], + compiler_options: CompilerOptions, + *, + separate: bool | list[tuple[list[str], str | None]] | None = None, + only_compile_paths: Iterable[str] | None = None, +) -> GeneratedC: + """Run mypyc's frontend and C generation, without any build system. + + This drives the mypy cgen CLI: it type checks the sources, generates C + code, and writes everything (group sources, runtime library, shims) into + compiler_options.target_dir. Use extension_build_specs on the result to + get buildable units for a build system of the caller's choice. + + The 'separate' argument controls how modules are assigned to compilation + groups, with the same meaning as in mypycify. If None (the default), + CompilerOptions.separate is used as-is; passing a value overrides it. + + No caching is performed: callers are responsible for deciding when to + regenerate (e.g. by hashing the input sources themselves). + """ + options = copy.copy(compiler_options) + if separate is not None: + options.separate = separate is not False + options.global_opts = not options.separate + + groups, group_cfilenames, source_deps = mypyc_build( + paths, + only_compile_paths=only_compile_paths, + compiler_options=options, + separate=separate if separate is not None else options.separate, + ) + assert len(groups) == len(group_cfilenames) + + shared_cfilenames, extra_include_dirs = _copy_runtime_files(options, source_deps) + + group_specs = [] + for (group_sources, lib_name), (cfilenames, deps) in zip(groups, group_cfilenames): + shims = {} + if lib_name is not None: + for source in group_sources: + shim = generate_c_extension_shim( + source.module, source.module.split(".")[-1], options.target_dir, lib_name + ) + ext_module = source.module + if is_package_source(source): + ext_module += ".__init__" + shims[ext_module] = shim + group_specs.append( + GroupSpec( + group_name=lib_name, + modules=[source.module for source in group_sources], + c_files=cfilenames, + depends=deps, + shims=shims, + ) + ) + + return GeneratedC( + target_dir=options.target_dir, + compiler_options=options, + groups=group_specs, + runtime_c_files=shared_cfilenames, + extra_include_dirs=extra_include_dirs, + ) + + +def extension_build_specs( + result: GeneratedC, *, opt_level: str | None = None, debug_level: str | None = None +) -> list[ExtensionSpec]: + """Turn the output of generate_c_sources into buildable extension specs. + + Each spec describes one extension module: what to compile, with which + include directories and flags, and where the built file must be placed. + The result is pure data — building it is up to the caller's build system. + + opt_level and debug_level have the same meaning as in mypycify. If None + (the default), no optimization or debug flags are emitted and the + caller's build system controls them. + """ + options = result.compiler_options + cflags = get_cflags( + opt_level=opt_level, + debug_level=debug_level, + multi_file=options.multi_file, + experimental_features=options.experimental_features, + log_trace=options.log_trace, + ) + link_args = _platform_link_args() + + specs = [] + for group in result.groups: + if group.group_name is not None: + lib_module = shared_lib_name(group.group_name) + specs.append( + ExtensionSpec( + module=lib_module, + out_path=_extension_output_path(lib_module), + sources=group.c_files + result.runtime_c_files, + include_dirs=[include_dir(), result.target_dir] + result.extra_include_dirs, + depends=group.depends, + cflags=cflags, + link_args=link_args, + ) + ) + for ext_module, shim_file in group.shims.items(): + specs.append( + ExtensionSpec( + module=ext_module, + out_path=_extension_output_path(ext_module), + sources=[shim_file], + include_dirs=[], + depends=[], + cflags=cflags, + link_args=link_args, + ) + ) + else: + assert len(group.modules) == 1 + specs.append( + ExtensionSpec( + module=group.modules[0], + out_path=_extension_output_path(group.modules[0]), + sources=group.c_files + result.runtime_c_files, + include_dirs=[include_dir()] + result.extra_include_dirs, + depends=group.depends, + cflags=cflags, + link_args=link_args, + ) + ) + return specs + + def get_cflags( *, compiler_type: str | None = None, - opt_level: str = "3", - debug_level: str = "1", + opt_level: str | None = None, + debug_level: str | None = None, multi_file: bool = False, experimental_features: bool = False, log_trace: bool = False, @@ -795,8 +1061,10 @@ def get_cflags( Args: compiler_type: Compiler type, e.g. "unix" or "msvc". If None, detected automatically. - opt_level: Optimization level as string ("0", "1", "2", or "3"). - debug_level: Debug level as string ("0", "1", "2", or "3"). + opt_level: Optimization level as string ("0", "1", "2", or "3"), or None to not + emit any optimization flags (the caller's build system controls them). + debug_level: Debug level as string ("0", "1", "2", or "3"), or None to not + emit any debug flags. multi_file: Whether multi-file compilation mode is enabled. experimental_features: Whether experimental features are enabled. log_trace: Whether trace logging is enabled. @@ -805,15 +1073,17 @@ def get_cflags( List of compiler flags. """ if compiler_type is None: - compiler: Any = ccompiler.new_compiler() - sysconfig.customize_compiler(compiler) - compiler_type = compiler.compiler_type + # This mirrors what distutils would pick, without needing distutils + # (which isn't available without setuptools on Python 3.12+). + compiler_type = "msvc" if sys.platform == "win32" else "unix" cflags: list[str] = [] if compiler_type == "unix": + if opt_level is not None: + cflags.append(f"-O{opt_level}") + if debug_level is not None: + cflags.append(f"-g{debug_level}") cflags += [ - f"-O{opt_level}", - f"-g{debug_level}", "-Werror", "-Wno-unused-function", "-Wno-unused-label", @@ -843,19 +1113,17 @@ def get_cflags( elif compiler_type == "msvc": # msvc doesn't have levels, '/O2' is full and '/Od' is disable if opt_level == "0": - opt_level = "d" cflags.append("/UNDEBUG") + cflags.append("/Od") elif opt_level in ("1", "2", "3"): - opt_level = "2" + cflags.append("/O2") if debug_level == "0": - debug_level = "NONE" + cflags.append("/DEBUG:NONE") elif debug_level == "1": - debug_level = "FASTLINK" + cflags.append("/DEBUG:FASTLINK") elif debug_level in ("2", "3"): - debug_level = "FULL" + cflags.append("/DEBUG:FULL") cflags += [ - f"/O{opt_level}", - f"/DEBUG:{debug_level}", "/wd4102", # unreferenced label "/wd4101", # unreferenced local variable "/wd4146", # negating unsigned int @@ -983,6 +1251,9 @@ def mypycify( # Mess around with setuptools and actually get the thing built setup_mypycify_vars() + # On 3.12+ this comes from setuptools' bundled distutils + from distutils import ccompiler, sysconfig + # Create a compiler object so we can make decisions based on what # compiler is being used. typeshed is missing some attributes on the # compiler object so we give it type Any @@ -1000,34 +1271,12 @@ def mypycify( log_trace=log_trace, ) - # If configured to (defaults to yes in multi-file mode), copy the - # runtime library in. Otherwise it just gets #included to save on - # compiler invocations. - shared_cfilenames = [] - include_dirs = set() - if not compiler_options.include_runtime_files: - # Collect all files to copy: runtime files + conditional source files - files_to_copy = list(RUNTIME_C_FILES) - for source_dep in source_deps: - files_to_copy.append(source_dep.path) - files_to_copy.append(source_dep.get_header()) - include_dirs.update(source_dep.include_dirs) - - if compiler_options.depends_on_librt_internal: - files_to_copy.append("internal/librt_internal_api.h") - files_to_copy.append("internal/librt_internal_api.c") - include_dirs.add("internal") - - # Copy all files - for name in files_to_copy: - rt_file = os.path.join(build_dir, name) - with open(os.path.join(include_dir(), name), encoding="utf-8") as f: - write_file(rt_file, f.read()) - if name.endswith(".c"): - shared_cfilenames.append(rt_file) + # If configured to (defaults to yes in multi-file mode), the runtime + # library gets directly #included; otherwise it's copied in and linked + # separately to save on compiler invocations. + shared_cfilenames, extra_include_dirs = _copy_runtime_files(compiler_options, source_deps) extensions = [] - extra_include_dirs = [os.path.join(include_dir(), dir) for dir in include_dirs] for (group_sources, lib_name), (cfilenames, deps) in zip(groups, group_cfilenames): if lib_name: extensions.extend( diff --git a/mypyc/cgen.py b/mypyc/cgen.py new file mode 100644 index 0000000000000..78424041a3326 --- /dev/null +++ b/mypyc/cgen.py @@ -0,0 +1,159 @@ +"""Generate mypyc C code and machine-readable build metadata, without setuptools. + +Usage: + + $ python -m mypyc cgen [--target-dir DIR] [--output-file FILE] SOURCE... + +This runs mypyc's frontend and C generation and writes all generated C +files (group sources, runtime library, shims) into --target-dir. It then +emits a JSON description of the extensions to build, either to +--output-file or to stdout. The JSON is meant to be consumed by a build +system (meson, cmake, ...) which performs the actual C compilation itself. + +The JSON schema is versioned and currently looks like: + + { + "schema_version": 1, + "target_dir": "", + "extensions": [ + { + "module": "...", # full dotted extension module name + "out_path": "...", # built file path, relative to the + # installation root (e.g. site-packages), + # including the platform extension suffix + "sources": [...], # .c files to compile, relative to target_dir + "include_dirs": [...], # absolute paths + "depends": [...], # headers to trigger rebuilds, relative to target_dir + "cflags": [...], # required flags (warnings, feature macros); no + # optimization or debug levels — those are up + # to the caller's build system + "link_args": [...] # may be empty; the caller is always responsible + # for the platform's standard extension linking, + # including the Python.h include directory + }, + ... + ] + } +""" + +from __future__ import annotations + +import argparse +import dataclasses +import json +import os.path + +from mypyc.build import ( + ExtensionSpec, + GeneratedC, + extension_build_specs, + generate_c_sources, + write_file, +) +from mypyc.options import CompilerOptions + +SCHEMA_VERSION = 1 + + +@dataclasses.dataclass(frozen=True) +class CgenArgs: + sources: list[str] + target_dir: str = "build/mypyc" + output_file: str | None = None + multi_file: bool = False + separate: bool = False + + +@dataclasses.dataclass(frozen=True) +class ExtensionInfo: + """One extension to build, as described in the module docstring's JSON schema.""" + + module: str + out_path: str + sources: list[str] + include_dirs: list[str] + depends: list[str] + cflags: list[str] + link_args: list[str] + + +@dataclasses.dataclass(frozen=True) +class BuildInfo: + schema_version: int + target_dir: str + extensions: list[ExtensionInfo] + + +def build_info(result: GeneratedC, specs: list[ExtensionSpec]) -> BuildInfo: + target_dir = os.path.abspath(result.target_dir) + return BuildInfo( + schema_version=SCHEMA_VERSION, + target_dir=target_dir, + extensions=[ + ExtensionInfo( + module=spec.module, + out_path=spec.out_path, + sources=[os.path.relpath(source, target_dir) for source in spec.sources], + include_dirs=spec.include_dirs, + depends=[os.path.relpath(dep, target_dir) for dep in spec.depends], + cflags=spec.cflags, + link_args=spec.link_args, + ) + for spec in specs + ], + ) + + +def parse_args(argv: list[str] | None = None) -> CgenArgs: + parser = argparse.ArgumentParser( + prog="mypyc cgen", + description="Generate mypyc C code and JSON build metadata for external build systems", + ) + parser.add_argument("sources", nargs="+", metavar="SOURCE", help="files to compile") + parser.add_argument( + "--target-dir", + default="build/mypyc", + help="directory to write generated C files to (default: build/mypyc)", + ) + parser.add_argument( + "--output-file", + default=None, + help="write the JSON build info to this file instead of stdout", + ) + parser.add_argument( + "--multi-file", action="store_true", help="compile each module into its own C source file" + ) + parser.add_argument( + "--separate", action="store_true", help="place each module in its own extension module" + ) + args = parser.parse_args(argv) + return CgenArgs( + sources=args.sources, + target_dir=args.target_dir, + output_file=args.output_file, + multi_file=args.multi_file, + separate=args.separate, + ) + + +def main(argv: list[str] | None = None) -> None: + args = parse_args(argv) + + options = CompilerOptions( + target_dir=args.target_dir, multi_file=args.multi_file, separate=args.separate + ) + result = generate_c_sources(args.sources, options) + # The emitted cflags intentionally exclude optimization and debug levels; + # the caller's build system controls those. + specs = extension_build_specs(result) + info = build_info(result, specs) + + text = json.dumps(dataclasses.asdict(info), indent=2) + "\n" + if args.output_file: + write_file(args.output_file, text) + else: + print(text, end="") + + +if __name__ == "__main__": + main() diff --git a/mypyc/test/test_cgen.py b/mypyc/test/test_cgen.py new file mode 100644 index 0000000000000..ddc6b970a0910 --- /dev/null +++ b/mypyc/test/test_cgen.py @@ -0,0 +1,158 @@ +"""End-to-end tests for the mypyc cgen CLI. + +These spawn real C compilations, so they are slow-ish; they verify that the +JSON build info is actually sufficient to build working extension modules +with a build system other than setuptools. +""" + +from __future__ import annotations + +import importlib.util +import json +import os +import os.path +import subprocess +import sys +import sysconfig +import tempfile +import unittest +from typing import Any + +from mypyc.common import EXT_SUFFIX + +base_path = os.path.join(os.path.dirname(__file__), "..", "..") + +# Injected via sitecustomize to make importing setuptools/distutils fail, +# proving that cgen doesn't need them. +_BLOCK_SITEPACKAGES = """\ +import sys + +_BLOCKED = {"setuptools", "distutils"} + + +class _Blocker: + def find_spec(self, fullname, path=None, target=None): # type: ignore[no-untyped-def] + if fullname.split(".")[0] in _BLOCKED: + raise ImportError(f"{fullname} is blocked for this test") + return None + + +sys.meta_path.insert(0, _Blocker()) +""" + + +def run_cgen(cwd: str, *args: str) -> dict[str, Any]: + env = os.environ.copy() + env["PYTHONPATH"] = base_path + os.pathsep + env.get("PYTHONPATH", "") + proc = subprocess.run( + [sys.executable, "-m", "mypyc", "cgen", *args], capture_output=True, cwd=cwd, env=env + ) + assert proc.returncode == 0, proc.stderr.decode() + info: dict[str, Any] = json.loads(proc.stdout) + return info + + +def compile_extension(target_dir: str, ext: dict[str, Any], out_path: str) -> None: + # Build in a single compiler invocation, like distutils does for simple + # unix-style extensions. The caller is responsible for the Python.h + # include directory. Windows needs a different link setup. + ldshared = sysconfig.get_config_var("LDSHARED") + assert ldshared + cmd = ldshared.split() + (sysconfig.get_config_var("CFLAGS") or "").split() + cmd += ext["cflags"] + cmd += ["-I" + sysconfig.get_path("include"), "-I" + sysconfig.get_path("platinclude")] + cmd += ["-I" + d for d in ext["include_dirs"]] + cmd += [os.path.join(target_dir, source) for source in ext["sources"]] + cmd += ["-o", out_path] + subprocess.run(cmd, check=True) + + +def import_compiled(path: str, name: str) -> Any: + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +class TestCgen(unittest.TestCase): + def test_works_without_setuptools(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + with open(os.path.join(tmp, "a.py"), "w") as f: + f.write("def f(x: int) -> int:\n return x + 1\n") + with tempfile.TemporaryDirectory() as blocker_dir: + with open(os.path.join(blocker_dir, "sitecustomize.py"), "w") as f: + f.write(_BLOCK_SITEPACKAGES) + env = os.environ.copy() + env["PYTHONPATH"] = blocker_dir + os.pathsep + base_path + proc = subprocess.run( + [sys.executable, "-m", "mypyc", "cgen", "--output-file", "info.json", "a.py"], + capture_output=True, + cwd=tmp, + env=env, + ) + assert proc.returncode == 0, proc.stderr.decode() + with open(os.path.join(tmp, "info.json")) as f: + info: dict[str, Any] = json.load(f) + assert info["schema_version"] == 1 + assert [ext["module"] for ext in info["extensions"]] == ["a"] + + def test_single_module_json_and_build(self) -> None: + if sys.platform == "win32": + self.skipTest("requires a unix-style compiler") + with tempfile.TemporaryDirectory() as tmp: + with open(os.path.join(tmp, "a.py"), "w") as f: + f.write("def f(x: int) -> int:\n return x + 1\n") + info = run_cgen(tmp, "--target-dir", os.path.join("build", "mypyc"), "a.py") + + assert info["schema_version"] == 1 + target = info["target_dir"] + assert os.path.isabs(target) + exts = info["extensions"] + assert isinstance(exts, list) and len(exts) == 1 + ext = exts[0] + assert ext["module"] == "a" + assert ext["out_path"] == "a" + EXT_SUFFIX + assert ext["cflags"] + # Optimization/debug levels are the caller's business. + assert not any(c.startswith(("-O", "-g")) for c in ext["cflags"]) + assert all(os.path.isfile(os.path.join(target, s)) for s in ext["sources"]) + assert all(os.path.isfile(os.path.join(target, d)) for d in ext["depends"]) + assert all(os.path.isdir(d) for d in ext["include_dirs"]) + + out_path = os.path.join(target, ext["out_path"]) + compile_extension(target, ext, out_path) + mod = import_compiled(out_path, "a") + assert mod.f(1) == 2 + + def test_package_shared_lib_and_shims_build(self) -> None: + if sys.platform == "win32": + self.skipTest("requires a unix-style compiler") + with tempfile.TemporaryDirectory() as tmp: + pkg = os.path.join(tmp, "pkg") + os.makedirs(pkg) + with open(os.path.join(pkg, "__init__.py"), "w") as f: + f.write("from pkg.mod import add\n") + with open(os.path.join(pkg, "mod.py"), "w") as f: + f.write("def add(x: int, y: int) -> int:\n return x + y\n") + + info = run_cgen(tmp, "pkg") + exts = {ext["module"]: ext for ext in info["extensions"]} + lib_exts = [m for m in exts if m.endswith("__mypyc")] + assert len(lib_exts) == 1 + assert {"pkg.__init__", "pkg.mod"} <= set(exts) + + target = info["target_dir"] + # Built files go relative to the installation root, which here is + # the source tree itself. + for ext in exts.values(): + compile_extension(target, ext, os.path.join(tmp, ext["out_path"])) + + script = "import pkg\nassert pkg.add(1, 2) == 3\nprint('ok')\n" + proc = subprocess.run([sys.executable, "-c", script], cwd=tmp, capture_output=True) + assert proc.returncode == 0, proc.stderr.decode() + assert proc.stdout.strip() == b"ok" + + +if __name__ == "__main__": + unittest.main()