diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml index a68e2e6281..2161706cd0 100644 --- a/.github/workflows/dist.yml +++ b/.github/workflows/dist.yml @@ -34,15 +34,32 @@ jobs: # Github Actions doesn't support pairing matrix values together, let's improvise # https://github.com/github/feedback/discussions/7835#discussioncomment-1769026 buildplat: - - [ubuntu-latest, "manylinux_x86_64", "cp3*-manylinux_x86_64"] - - [ubuntu-latest, "manylinux_aarch64", "cp3*-manylinux_aarch64"] - - [ubuntu-latest, "manylinux_ppc64le", "cp3*-manylinux_ppc64le"] - - [ubuntu-latest, "manylinux_s390x", "cp3*-manylinux_s390x"] - - [ubuntu-latest, "manylinux_i686", "cp3*-manylinux_i686"] - - [windows-2022, "win_amd6", "cp3*-win_amd64"] - - [windows-2022, "win32", "cp3*-win32"] - - [windows-11-arm, "win_arm64", "cp3*-win_arm64"] - - [macos-14, "macos", "cp*-macosx_*"] + # [runner, platform, identifier, cibuildwheel-build-selector] + # abi3 builds - single build per platform, covers GIL-enabled Python 3.11+ + - [ubuntu-latest, "manylinux_x86_64", "cp311-abi3-manylinux_x86_64", "cp311-manylinux_x86_64"] + - [ubuntu-latest, "manylinux_aarch64", "cp311-abi3-manylinux_aarch64", "cp311-manylinux_aarch64"] + - [ubuntu-latest, "manylinux_ppc64le", "cp311-manylinux_ppc64le", "cp311-manylinux_ppc64le"] + - [ubuntu-latest, "manylinux_s390x", "cp311-manylinux_s390x", "cp311-manylinux_s390x"] + - [ubuntu-latest, "manylinux_i686", "cp311-manylinux_i686", "cp311-manylinux_i686"] + - [windows-2022, "win_amd64", "cp311-abi3-win_amd64", "cp311-win_amd64"] + - [windows-2022, "win32", "cp311-abi3-win32", "cp311-win32"] + - [windows-11-arm, "win_arm64", "cp311-abi3-win_arm64", "cp311-win_arm64"] + - [macos-14, "macos", "cp311-abi3-macosx", "cp311-macosx_*"] + # Per-version builds for 3.10 + - [ubuntu-latest, "manylinux_x86_64", "cp310-manylinux_x86_64", "cp310-manylinux_x86_64"] + - [ubuntu-latest, "manylinux_aarch64", "cp310-manylinux_aarch64", "cp310-manylinux_aarch64"] + - [ubuntu-latest, "manylinux_ppc64le", "cp310-manylinux_ppc64le", "cp310-manylinux_ppc64le"] + - [ubuntu-latest, "manylinux_s390x", "cp310-manylinux_s390x", "cp310-manylinux_s390x"] + - [ubuntu-latest, "manylinux_i686", "cp310-manylinux_i686", "cp310-manylinux_i686"] + - [windows-2022, "win_amd64", "cp310-win_amd64", "cp310-win_amd64"] + - [windows-2022, "win32", "cp310-win32", "cp310-win32"] + - [windows-11-arm, "win_arm64", "cp310-win_arm64", "cp310-win_arm64"] + - [macos-14, "macos", "cp310-macosx", "cp310-macosx_*"] + # Free-threaded 3.14t (per-version, no stable ABI) + - [ubuntu-latest, "manylinux_x86_64", "cp314t-manylinux_x86_64", "cp314t-manylinux_x86_64"] + - [ubuntu-latest, "manylinux_aarch64", "cp314t-manylinux_aarch64", "cp314t-manylinux_aarch64"] + - [windows-2022, "win_amd64", "cp314t-win_amd64", "cp314t-win_amd64"] + - [macos-14, "macos", "cp314t-macosx"] steps: - name: Checkout pymongo @@ -77,7 +94,18 @@ jobs: - name: Build wheels env: - CIBW_BUILD: ${{ matrix.buildplat[2] }} + # Abi3 rows build under the plain per-version interpreter selector + # (matrix.buildplat[3], e.g. "cp311-manylinux_x86_64"); the abi3 tag + # comes from py_limited_api with PYMONGO_BUILD_ABI3 set below. + CIBW_BUILD: ${{ matrix.buildplat[3] }} + # macos-14 runners are Apple Silicon; build arm64 so the wheel's + # arch matches delocate's --require-archs. + CIBW_ARCHS: ${{ matrix.buildplat[1] == 'manylinux_aarch64' && 'aarch64' || matrix.buildplat[1] == 'manylinux_ppc64le' && 'ppc64le' || matrix.buildplat[1] == 'manylinux_s390x' && 's390x' || matrix.buildplat[1] == 'manylinux_i686' && 'i686' || matrix.buildplat[1] == 'macos' && 'arm64' || '' }} + # abi3 builds set PYMONGO_BUILD_ABI3 to select the limited-API code + # path and produce a cp311-abi3 wheel. cp310 and cp314t builds leave + # it unset: 3.10 cannot (from a 3.10 runner) and 3.14t must not use + # abi3 (no stable ABI for free-threaded Python). + CIBW_ENVIRONMENT: ${{ startsWith(matrix.buildplat[2], 'cp311-abi3') && 'PYMONGO_BUILD_ABI3=1' || '' }} # Strip any in-place C extensions from previous builds # so they don't leak into the next wheel. CIBW_BEFORE_BUILD: python ./tools/clean.py @@ -86,18 +114,25 @@ jobs: - name: Assert all versions in wheelhouse if: ${{ ! startsWith(matrix.buildplat[1], 'macos') }} run: | - ls wheelhouse/*cp39*.whl - ls wheelhouse/*cp310*.whl - ls wheelhouse/*cp311*.whl - ls wheelhouse/*cp312*.whl - ls wheelhouse/*cp313*.whl - ls wheelhouse/*cp314*.whl - # Free-threading builds: - ls wheelhouse/*cp314t*.whl + # abi3 wheels (one per platform, covers GIL-enabled 3.11-3.14+) + if [[ "${{ matrix.buildplat[2] }}" == "cp311-abi3-"* ]]; then + ls wheelhouse/*abi3*.whl + # Per-version cp311 wheels (archs without abi3 support) + elif [[ "${{ matrix.buildplat[2] }}" == "cp311-"* ]]; then + ls wheelhouse/*cp311*.whl + # Per-version cp310 wheels + elif [[ "${{ matrix.buildplat[2] }}" == "cp310-"* ]]; then + ls wheelhouse/*cp310*.whl + # Free-threaded cp314t wheels + elif [[ "${{ matrix.buildplat[2] }}" == "cp314t-"* ]]; then + ls wheelhouse/*cp314t*.whl + fi - uses: actions/upload-artifact@v7 with: - name: wheel-${{ matrix.buildplat[1] }} + # Unique per matrix row - multiple rows share a platform (manylinux_x86_64 + # builds cp311-abi3, cp310, cp314t), and same-name uploads overwrite. + name: wheel-${{ matrix.buildplat[2] }} path: ./wheelhouse/*.whl if-no-files-found: error @@ -114,7 +149,7 @@ jobs: - uses: actions/setup-python@v7.0.0 with: # Build sdist on lowest supported Python - python-version: "3.9" + python-version: "3.10" - name: Build SDist run: | @@ -149,3 +184,98 @@ jobs: with: name: all-dist-${{ github.run_id }} path: "./*" + + test_abi3_compat: + name: Test abi3 wheel on CPython ${{ matrix.python-version }} + runs-on: ubuntu-latest + needs: build_wheels + strategy: + fail-fast: false + matrix: + # The cp311-abi3 wheel claims support for every GIL-enabled Python + # >= 3.11. Its decode path relies on CPython's frozen PyDateTime_CAPI + # (re-declared in bson/_cbsonmodule.c), so we must run the BSON suite + # against every version we ship it for, not just the build Python. + python-version: ["3.11", "3.12", "3.13", "3.14"] + steps: + - name: Checkout pymongo + uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + ref: ${{ inputs.ref }} + + - uses: actions/setup-python@v7.0.0 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + + - name: Download abi3 wheel + uses: actions/download-artifact@v8 + with: + # Pin to this run - the wheel is rebuilt for every commit, so a + # digest check against a prior run's artifact fails spurious. + run-id: ${{ github.run_id }} + name: wheel-cp311-abi3-manylinux_x86_64 + path: wheelhouse + digest-mismatch: warn + + - name: Install abi3 wheel and run BSON suite + run: | + python -m pip install pytest wheelhouse/*cp311-abi3*.whl + # Leave the source tree so the freshly-installed wheel (with the C + # extension) is imported, not the checkout's pure-Python fallback. + cd "$RUNNER_TEMP" + python "$GITHUB_WORKSPACE/tools/fail_if_no_c.py" + # The repo pyproject pytest config needs pytest-asyncio and the + # live-server test markers (--strict-config, asyncio fixture scope); + # these two sync BSON files run fine under an empty config. + printf '[pytest]\n' > pytest.ini + python -m pytest -c "$RUNNER_TEMP/pytest.ini" --import-mode=importlib "$GITHUB_WORKSPACE/test/test_bson.py" "$GITHUB_WORKSPACE/test/test_bson_corpus.py" -q + + test_abi3_next_release: + name: Test abi3 wheel on the next CPython pre-release + runs-on: ubuntu-latest + needs: build_wheels + # Best-effort backstop for the non-stable-ABI PyDateTime_CAPI usage: if + # the next CPython (e.g. 3.15 dev/beta) resolves or changes that layout, + # we want to catch it during its beta phase. allow-prereleases + a "-dev" + # specifier tracks the in-development version; the job is non-blocking so + # a version that has not yet been published on the setup-python mirror + # does not fail the release. + continue-on-error: true + strategy: + fail-fast: false + matrix: + python-version: ["3.15-dev"] + steps: + - name: Checkout pymongo + uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + ref: ${{ inputs.ref }} + + - uses: actions/setup-python@v7.0.0 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + + - name: Download abi3 wheel + uses: actions/download-artifact@v8 + with: + run-id: ${{ github.run_id }} + name: wheel-cp311-abi3-manylinux_x86_64 + path: wheelhouse + digest-mismatch: warn + + - name: Install abi3 wheel and run BSON suite + run: | + python -m pip install pytest wheelhouse/*cp311-abi3*.whl + # Leave the source tree so the freshly-installed wheel (with the C + # extension) is imported, not the checkout's pure-Python fallback. + cd "$RUNNER_TEMP" + python "$GITHUB_WORKSPACE/tools/fail_if_no_c.py" + # The repo pyproject pytest config needs pytest-asyncio and the + # live-server test markers (--strict-config, asyncio fixture scope); + # these two sync BSON files run fine under an empty config. + printf '[pytest]\n' > pytest.ini + python -m pytest -c "$RUNNER_TEMP/pytest.ini" --import-mode=importlib "$GITHUB_WORKSPACE/test/test_bson.py" "$GITHUB_WORKSPACE/test/test_bson_corpus.py" -q diff --git a/_setup.py b/_setup.py index f8536b3371..51f0545a6d 100644 --- a/_setup.py +++ b/_setup.py @@ -105,21 +105,39 @@ def build_extension(self, ext): ) +use_abi3 = bool(os.environ.get("PYMONGO_BUILD_ABI3")) and sys.version_info >= (3, 11) + +_cbson_sources = ["bson/_cbsonmodule.c", "bson/buffer.c"] +_cmessage_sources = [ + "pymongo/_cmessagemodule.c", + "bson/_cbsonmodule.c", + "bson/buffer.c", +] + +_abi3_kwargs = ( + { + "py_limited_api": "3.11", + # Modern setuptools infers the .abi3.so suffix from py_limited_api + # but does not always inject the Py_LIMITED_API define; set it + # explicitly so the limited-API code paths are selected at compile. + "define_macros": [("Py_LIMITED_API", "0x030B0000")], + } + if use_abi3 + else {} +) + ext_modules = [ Extension( "bson._cbson", include_dirs=["bson"], - sources=["bson/_cbsonmodule.c", "bson/time64.c", "bson/buffer.c"], + sources=_cbson_sources, + **_abi3_kwargs, ), Extension( "pymongo._cmessage", include_dirs=["bson"], - sources=[ - "pymongo/_cmessagemodule.c", - "bson/_cbsonmodule.c", - "bson/time64.c", - "bson/buffer.c", - ], + sources=_cmessage_sources, + **_abi3_kwargs, ), ] diff --git a/bson/_cbsonmodule.c b/bson/_cbsonmodule.c index d383cee308..a4c4f6408a 100644 --- a/bson/_cbsonmodule.c +++ b/bson/_cbsonmodule.c @@ -22,14 +22,110 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" +#ifndef Py_LIMITED_API #include "datetime.h" +#endif + +/* The limited C API (Py_LIMITED_API >= 0x030B0000) deliberately omits the + * libc headers from Python.h, so include the ones this module uses. */ +#ifdef Py_LIMITED_API +#include +#include +#include +#include + +/* PEP 590 vectorcall (PyObject_Vectorcall / PyObject_VectorcallMethod) is not + * part of the stable ABI and is not exported by CPython's import library on + * Windows, so a limited-API build must call through the stable PyObject_Call* + * family. These helpers build a positional-args tuple; building the tuple is + * the portability cost. */ +static PyObject* _limited_vectorcall(PyObject* callable, + PyObject* const* args, Py_ssize_t nargs) { + PyObject* tup = PyTuple_New(nargs); + if (!tup) { + return NULL; + } + for (Py_ssize_t i = 0; i < nargs; i++) { + PyObject* a = args[i]; + Py_INCREF(a); + /* PyTuple_SetItem steals the reference; i is always in range. */ + PyTuple_SetItem(tup, i, a); + } + PyObject* result = PyObject_Call(callable, tup, NULL); + Py_DECREF(tup); + return result; +} + +static PyObject* _limited_vectorcall_method(PyObject* name, + PyObject* const* args, Py_ssize_t nargs) { + /* PyObject_VectorcallMethod calls the method `name` on args[0] with the + * remaining args[1..nargs-1]. */ + PyObject* method = PyObject_GetAttr(args[0], name); + if (!method) { + return NULL; + } + PyObject* result = _limited_vectorcall(method, args + 1, nargs - 1); + Py_DECREF(method); + return result; +} +#endif + + +#ifdef Py_LIMITED_API +#define PYMONGO_VECTORCALL(callable, args, nargs, kwnames) \ + _limited_vectorcall((callable), (args), (nargs)) +#define PYMONGO_VECTORCALL_METHOD(name, args, nargs, kwnames) \ + _limited_vectorcall_method((name), (args), (nargs)) +#else +#define PYMONGO_VECTORCALL(callable, args, nargs, kwnames) \ + PyObject_Vectorcall((callable), (args), (nargs), (kwnames)) +#define PYMONGO_VECTORCALL_METHOD(name, args, nargs, kwnames) \ + PyObject_VectorcallMethod((name), (args), (nargs), (kwnames)) +#endif #include "buffer.h" -#include "time64.h" #define _CBSON_MODULE #include "_cbsonmodule.h" +#ifdef Py_LIMITED_API +/* CPython's datetime C API, re-declared so a limited-API (abi3) build can use + * it. Include/datetime.h is entirely guarded by `#ifndef Py_LIMITED_API`, so + * these symbols are not declared under the limited API. The PyDateTime_CAPI + * struct and the "datetime.datetime_CAPI" capsule that carries it have been + * stable since Python 2.4; numpy and pandas consume this same capsule (via + * PyDateTime_IMPORT) in their non-limited builds, so it is a capsule-stability + * precedent rather than an abi3 one. Because the layout is frozen and fetched + * by name at runtime (no link-time symbol in the .so), an abi3 build can + * re-declare it and call DateTime_FromDateAndTime directly, avoiding the + * 7-arg constructor and its seven PyLong allocations. This is a non-stable-ABI + * usage relying on a capsule that has not changed in ~20 years; if CPython + * ever changes it, the abi3 wheel is exercised against the new version in CI + * during that release's beta phase, so a portability regression is caught. */ +typedef struct { + PyTypeObject *DateType; + PyTypeObject *DateTimeType; + PyTypeObject *TimeType; + PyTypeObject *DeltaType; + PyTypeObject *TZInfoType; + PyObject *TimeZone_UTC; + PyObject *(*Date_FromDate)(int, int, int, PyTypeObject*); + PyObject *(*DateTime_FromDateAndTime)( + int, int, int, int, int, int, int, PyObject*, PyTypeObject*); + PyObject *(*Time_FromTime)(int, int, int, int, PyObject*, PyTypeObject*); + PyObject *(*Delta_FromDelta)(int, int, int, int, PyTypeObject*); + PyObject *(*TimeZone_FromTimeZone)(PyObject*, PyObject*); + PyObject *(*DateTime_FromTimestamp)(PyObject*, PyObject*, PyObject*); + PyObject *(*Date_FromTimestamp)(PyObject*, PyObject*); + PyObject *(*DateTime_FromDateAndTimeAndFold)( + int, int, int, int, int, int, int, PyObject*, int, PyTypeObject*); + PyObject *(*Time_FromTimeAndFold)( + int, int, int, int, PyObject*, int, PyTypeObject*); +} _PyDateTime_CAPI; + +#define _PYDATETIME_CAPSULE_NAME "datetime.datetime_CAPI" +#endif + /* New module state and initialization code. * See the module-initialization-and-state * section in the following doc: @@ -48,7 +144,11 @@ struct module_state { PyObject* MinKey; PyObject* MaxKey; PyObject* UTC; +#ifdef Py_LIMITED_API + PyObject* REType; +#else PyTypeObject* REType; +#endif PyObject* BSONInt64; PyObject* Decimal128; PyObject* Mapping; @@ -82,6 +182,14 @@ struct module_state { PyObject* _from_uuid_str; PyObject* _as_uuid_str; PyObject* _from_bid_str; +#ifdef Py_LIMITED_API + PyObject* datetime_type; + PyObject* timedelta_type; + _PyDateTime_CAPI* datetime_capi; + PyObject* _days_str; + PyObject* _seconds_str; + PyObject* _microseconds_delta_str; +#endif int64_t min_millis; int64_t max_millis; Py_ssize_t raw_bson_view_threshold; @@ -112,6 +220,75 @@ struct module_state { #define DATETIME_AUTO 4 #define PYTHON_3_12 0x030C0000 +/* Convert a days-since-epoch value to a civil (y, m, d) tuple. Used by both the + * limited and non-limited datetime decode paths, replacing the loop-based + * time64 library with O(1) closed-form arithmetic (Hinnant). */ +static void civil_from_days(int64_t z, int *y, int *m, int *d) { + z += 719468; + int64_t era = (z >= 0 ? z : z - 146096) / 146097; + int64_t doe = z - era * 146097; + int64_t yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; + int64_t yy = yoe + era * 400; + int64_t doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + int64_t mp = (5 * doy + 2) / 153; + int64_t dd = doy - (153 * mp + 2) / 5 + 1; + int64_t mm = mp + (mp < 10 ? 3 : -9); + *y = (int)(yy + (mm <= 2)); + *m = (int)mm; + *d = (int)dd; +} + +/* Inverse of civil_from_days: days since 1970-01-01 for a civil date. */ +static int64_t days_from_civil(int64_t y, int m, int d) { + y -= m <= 2; + int64_t era = (y >= 0 ? y : y - 399) / 400; + int64_t yoe = y - era * 400; + int64_t doy = (153 * (m + (m > 2 ? -3 : 9)) + 2) / 5 + d - 1; + int64_t doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + return era * 146097 + doe - 719468; +} + +#ifdef Py_LIMITED_API +/* Mirror of CPython's datetime object field layout (see _PyDateTime_CAPI). + * Include/datetime.h defines _PyTZINFO_HEAD as PyObject_HEAD + a Py_hash_t + * hashcode + a hastzinfo flag; both the naive (_PyDateTime_BaseDateTime) and + * aware (PyDateTime_DateTime) layouts place data[] immediately after that + * head, so reading data[0..9] is layout-agnostic. These read the same bytes + * as PyDateTime_GET_YEAR / PyDateTime_DATE_GET_HOUR, so datetimes are encoded + * from their raw fields with no getattr calls. */ +typedef struct { + PyObject_HEAD + Py_hash_t hashcode; + char hastzinfo; + unsigned char data[10]; +} _PyDateTime_Fields; + +#define _DT_YEAR(o) ((((_PyDateTime_Fields*)(o))->data[0] << 8) | ((_PyDateTime_Fields*)(o))->data[1]) +#define _DT_MONTH(o) ((_PyDateTime_Fields*)(o))->data[2] +#define _DT_DAY(o) ((_PyDateTime_Fields*)(o))->data[3] +#define _DT_HOUR(o) ((_PyDateTime_Fields*)(o))->data[4] +#define _DT_MINUTE(o) ((_PyDateTime_Fields*)(o))->data[5] +#define _DT_SECOND(o) ((_PyDateTime_Fields*)(o))->data[6] +#define _DT_MICROSECOND(o) (((((_PyDateTime_Fields*)(o))->data[7] << 16) | (((_PyDateTime_Fields*)(o))->data[8] << 8)) | ((_PyDateTime_Fields*)(o))->data[9]) +#endif + +/* Check for an exact type first, then fall back to the subclass check. + * + * In the limited API the PyXxx_Check subclass macros expand through + * PyType_FastSubclass -> PyType_HasFeature -> PyType_GetFlags(), which is a + * function call, whereas Py_IS_TYPE is a direct pointer compare. Exact + * built-in types are far more common than subclasses, so short-circuiting on + * the exact match avoids the out-of-line call and is byte-identical in + * behavior. The non-limited build already inlines the subclass check, so it + * is left unchanged. */ +#ifdef Py_LIMITED_API +#define _CBSON_EXACT_OR_SUBCLASS(obj, typetok, typeobj) \ + (Py_IS_TYPE((obj), (typeobj)) || Py##typetok##_Check((obj))) +#else +#define _CBSON_EXACT_OR_SUBCLASS(obj, typetok, typeobj) \ + (Py##typetok##_Check((obj))) +#endif + /* Converts integer to its string representation in decimal notation. */ extern int cbson_long_long_to_str(long long num, char* str, size_t size) { // Buffer should fit 64-bit signed integer @@ -251,11 +428,6 @@ static int _write_element_to_buffer(PyObject* self, buffer_t buffer, */ static int write_raw_doc(buffer_t buffer, PyObject* raw, PyObject* _raw); -/* Get a read-only buffer view of a bytes-like object. - * Returns 1 on success or 0 on failure with an exception set. - */ -static int _get_buffer(PyObject *exporter, Py_buffer *view); - #if PY_VERSION_HEX >= PYTHON_3_12 /* Transfer traceback from old_exc to new_exc. * Steals reference to old_exc. */ @@ -318,6 +490,88 @@ static void _rewrap_as_invalid_bson(void) { } /* Date stuff */ +#ifdef Py_LIMITED_API +static PyObject* datetime_from_millis(PyObject* self, long long millis) { + struct module_state *state = GETSTATE(self); + if (!state) { + return NULL; + } + PyObject* datetime = NULL; + int diff = (int)(((millis % 1000) + 1000) % 1000); + int microseconds = diff * 1000; + int64_t seconds = (millis - diff) / 1000; + /* Handle negative seconds with floor division */ + int64_t days = seconds / 86400; + int64_t sec_of_day = seconds - days * 86400; + if (sec_of_day < 0) { sec_of_day += 86400; days -= 1; } + int y, mo, d; + civil_from_days(days, &y, &mo, &d); + int hh = (int)(sec_of_day / 3600); + int mm_ = (int)((sec_of_day % 3600) / 60); + int ss = (int)(sec_of_day % 60); + + /* Construct the datetime directly through CPython's frozen datetime C API + * instead of a 7-arg constructor call. This avoids allocating seven PyLongs + * per value and the type-call dispatch; see the _PyDateTime_CAPI comment. */ + datetime = state->datetime_capi->DateTime_FromDateAndTime( + y, mo, d, hh, mm_, ss, microseconds, + Py_None, state->datetime_capi->DateTimeType); + if(!datetime) { + #if PY_VERSION_HEX >= PYTHON_3_12 + PyObject *exc = PyErr_GetRaisedException(); + + /* Only add additional error message on ValueError exceptions. */ + if (exc && PyErr_GivenExceptionMatches(exc, PyExc_ValueError)) { + PyObject* err_msg = PyObject_Str(exc); + if (err_msg) { + PyObject* appendage = PyUnicode_FromString(" (Consider Using CodecOptions(datetime_conversion=DATETIME_AUTO) or MongoClient(datetime_conversion='DATETIME_AUTO')). See: https://www.mongodb.com/docs/languages/python/pymongo-driver/current/data-formats/dates-and-times/#handling-out-of-range-datetimes"); + if (appendage) { + PyObject* msg = PyUnicode_Concat(err_msg, appendage); + if (msg) { + PyObject* new_exc = PyObject_CallOneArg(PyExc_ValueError, msg); + if (new_exc) { + exc = _transfer_traceback(exc, new_exc); + } + Py_DECREF(msg); + } + } + Py_XDECREF(appendage); + } + Py_XDECREF(err_msg); + } + /* Steals reference to exc. */ + PyErr_SetRaisedException(exc); + #else + /* Calling _error clears the error state, so fetch it first.*/ + PyObject *etype = NULL, *evalue = NULL, *etrace = NULL; + PyErr_Fetch(&etype, &evalue, &etrace); + + /* Only add additional error message on ValueError exceptions. */ + if (PyErr_GivenExceptionMatches(etype, PyExc_ValueError)) { + if (evalue) { + PyObject* err_msg = PyObject_Str(evalue); + if (err_msg) { + PyObject* appendage = PyUnicode_FromString(" (Consider Using CodecOptions(datetime_conversion=DATETIME_AUTO) or MongoClient(datetime_conversion='DATETIME_AUTO')). See: https://www.mongodb.com/docs/languages/python/pymongo-driver/current/data-formats/dates-and-times/#handling-out-of-range-datetimes"); + if (appendage) { + PyObject* msg = PyUnicode_Concat(err_msg, appendage); + if (msg) { + Py_DECREF(evalue); + evalue = msg; + } + } + Py_XDECREF(appendage); + } + Py_XDECREF(err_msg); + } + PyErr_NormalizeException(&etype, &evalue, &etrace); + } + /* Steals references to args. */ + PyErr_Restore(etype, evalue, etrace); + #endif + } + return datetime; +} +#else static PyObject* datetime_from_millis(long long millis) { /* To encode a datetime instance like datetime(9999, 12, 31, 23, 59, 59, 999999) * we follow these steps: @@ -350,17 +604,18 @@ static PyObject* datetime_from_millis(long long millis) { PyObject* datetime = NULL; int diff = (int)(((millis % 1000) + 1000) % 1000); int microseconds = diff * 1000; - Time64_T seconds = (millis - diff) / 1000; - struct TM timeinfo; - cbson_gmtime64_r(&seconds, &timeinfo); - - datetime = PyDateTime_FromDateAndTime(timeinfo.tm_year + 1900, - timeinfo.tm_mon + 1, - timeinfo.tm_mday, - timeinfo.tm_hour, - timeinfo.tm_min, - timeinfo.tm_sec, - microseconds); + int64_t seconds = (millis - diff) / 1000; + /* Handle negative seconds with floor division */ + int64_t days = seconds / 86400; + int64_t sec_of_day = seconds - days * 86400; + if (sec_of_day < 0) { sec_of_day += 86400; days -= 1; } + int y, mo, d; + civil_from_days(days, &y, &mo, &d); + int hh = (int)(sec_of_day / 3600); + int mm_ = (int)((sec_of_day % 3600) / 60); + int ss = (int)(sec_of_day % 60); + + datetime = PyDateTime_FromDateAndTime(y, mo, d, hh, mm_, ss, microseconds); if(!datetime) { #if PY_VERSION_HEX >= PYTHON_3_12 PyObject *exc = PyErr_GetRaisedException(); @@ -416,22 +671,44 @@ static PyObject* datetime_from_millis(long long millis) { } return datetime; } - -static long long millis_from_datetime(PyObject* datetime) { - struct TM timeinfo; +#endif /* Py_LIMITED_API */ + +#ifdef Py_LIMITED_API +static long long millis_from_datetime(struct module_state* state, PyObject* datetime) { + /* Read the datetime's raw fields directly (see _PyDateTime_Fields); this + * mirrors the PyDateTime_GET_YEAR / PyDateTime_DATE_GET_HOUR family and + * avoids the per-field getattr calls the limited API otherwise forces. + * days_from_civil() does the calendar math in O(1). */ + long long y = _DT_YEAR(datetime); + long long mo = _DT_MONTH(datetime); + long long d = _DT_DAY(datetime); + long long hh = _DT_HOUR(datetime); + long long mm_ = _DT_MINUTE(datetime); + long long ss = _DT_SECOND(datetime); + long long us = _DT_MICROSECOND(datetime); long long millis; - timeinfo.tm_year = PyDateTime_GET_YEAR(datetime) - 1900; - timeinfo.tm_mon = PyDateTime_GET_MONTH(datetime) - 1; - timeinfo.tm_mday = PyDateTime_GET_DAY(datetime); - timeinfo.tm_hour = PyDateTime_DATE_GET_HOUR(datetime); - timeinfo.tm_min = PyDateTime_DATE_GET_MINUTE(datetime); - timeinfo.tm_sec = PyDateTime_DATE_GET_SECOND(datetime); - - millis = cbson_timegm64(&timeinfo) * 1000; - millis += PyDateTime_DATE_GET_MICROSECOND(datetime) / 1000; + millis = days_from_civil(y, (int)mo, (int)d) * 86400000LL + + hh * 3600000LL + mm_ * 60000LL + ss * 1000LL + us / 1000; return millis; } +#else +static long long millis_from_datetime(PyObject* datetime) { + /* Read the datetime's raw fields via the PyDateTime_GET_* macros and do the + * calendar math with O(1) days_from_civil instead of the loop-based time64 + * library; see the shared civil_from_days/days_from_civil helpers. */ + long long y = PyDateTime_GET_YEAR(datetime); + long long mo = PyDateTime_GET_MONTH(datetime); + long long d = PyDateTime_GET_DAY(datetime); + long long hh = PyDateTime_DATE_GET_HOUR(datetime); + long long mm_ = PyDateTime_DATE_GET_MINUTE(datetime); + long long ss = PyDateTime_DATE_GET_SECOND(datetime); + long long us = PyDateTime_DATE_GET_MICROSECOND(datetime); + + return days_from_civil(y, (int)mo, (int)d) * 86400000LL + + hh * 3600000LL + mm_ * 60000LL + ss * 1000LL + us / 1000; +} +#endif /* Py_LIMITED_API */ /* Extended-range datetime, returns a DatetimeMS object with millis */ static PyObject* datetime_ms_from_millis(PyObject* self, long long millis){ @@ -448,7 +725,7 @@ static PyObject* datetime_ms_from_millis(PyObject* self, long long millis){ return NULL; } PyObject* args[1] = {ll_millis}; - dt = PyObject_Vectorcall(state->DatetimeMS, args, 1, NULL); + dt = PYMONGO_VECTORCALL(state->DatetimeMS, args, 1, NULL); Py_DECREF(ll_millis); return dt; } @@ -494,12 +771,35 @@ static PyObject* decode_datetime(PyObject* self, long long millis, const codec_o int64_t max_millis_offset = 0; if (options->tz_aware && options->tzinfo && options->tzinfo != Py_None) { PyObject* utcoffset_args[2] = {options->tzinfo, state->min_datetime}; - PyObject* utcoffset = PyObject_VectorcallMethod( + PyObject* utcoffset = PYMONGO_VECTORCALL_METHOD( state->_utcoffset_str, utcoffset_args, 2, NULL); if (utcoffset == NULL) { return 0; } if (utcoffset != Py_None) { +#ifdef Py_LIMITED_API + if (!Py_IS_TYPE(utcoffset, (PyTypeObject*)state->timedelta_type)) { + PyObject* BSONError = _error("BSONError"); + if (BSONError) { + PyErr_SetString(BSONError, "tzinfo.utcoffset() did not return a datetime.timedelta"); + Py_DECREF(BSONError); + } + Py_DECREF(utcoffset); + return 0; + } + PyObject* days_obj = PyObject_GetAttr(utcoffset, state->_days_str); + PyObject* seconds_obj = PyObject_GetAttr(utcoffset, state->_seconds_str); + PyObject* micros_obj = PyObject_GetAttr(utcoffset, state->_microseconds_delta_str); + if (!days_obj || !seconds_obj || !micros_obj) { + Py_XDECREF(days_obj); Py_XDECREF(seconds_obj); Py_XDECREF(micros_obj); + Py_DECREF(utcoffset); + return 0; + } + min_millis_offset = (PyLong_AsLong(days_obj) * (int64_t)86400 + + PyLong_AsLong(seconds_obj)) * (int64_t)1000 + + (PyLong_AsLong(micros_obj) / 1000); + Py_DECREF(days_obj); Py_DECREF(seconds_obj); Py_DECREF(micros_obj); +#else if (!PyDelta_Check(utcoffset)) { PyObject* BSONError = _error("BSONError"); if (BSONError) { @@ -512,15 +812,39 @@ static PyObject* decode_datetime(PyObject* self, long long millis, const codec_o min_millis_offset = (PyDateTime_DELTA_GET_DAYS(utcoffset) * (int64_t)86400 + PyDateTime_DELTA_GET_SECONDS(utcoffset)) * (int64_t)1000 + (PyDateTime_DELTA_GET_MICROSECONDS(utcoffset) / 1000); +#endif } Py_DECREF(utcoffset); utcoffset_args[1] = state->max_datetime; - utcoffset = PyObject_VectorcallMethod( + utcoffset = PYMONGO_VECTORCALL_METHOD( state->_utcoffset_str, utcoffset_args, 2, NULL); if (utcoffset == NULL) { return 0; } if (utcoffset != Py_None) { +#ifdef Py_LIMITED_API + if (!Py_IS_TYPE(utcoffset, (PyTypeObject*)state->timedelta_type)) { + PyObject* BSONError = _error("BSONError"); + if (BSONError) { + PyErr_SetString(BSONError, "tzinfo.utcoffset() did not return a datetime.timedelta"); + Py_DECREF(BSONError); + } + Py_DECREF(utcoffset); + return 0; + } + PyObject* days_obj = PyObject_GetAttr(utcoffset, state->_days_str); + PyObject* seconds_obj = PyObject_GetAttr(utcoffset, state->_seconds_str); + PyObject* micros_obj = PyObject_GetAttr(utcoffset, state->_microseconds_delta_str); + if (!days_obj || !seconds_obj || !micros_obj) { + Py_XDECREF(days_obj); Py_XDECREF(seconds_obj); Py_XDECREF(micros_obj); + Py_DECREF(utcoffset); + return 0; + } + max_millis_offset = (PyLong_AsLong(days_obj) * (int64_t)86400 + + PyLong_AsLong(seconds_obj)) * (int64_t)1000 + + (PyLong_AsLong(micros_obj) / 1000); + Py_DECREF(days_obj); Py_DECREF(seconds_obj); Py_DECREF(micros_obj); +#else if (!PyDelta_Check(utcoffset)) { PyObject* BSONError = _error("BSONError"); if (BSONError) { @@ -533,6 +857,7 @@ static PyObject* decode_datetime(PyObject* self, long long millis, const codec_o max_millis_offset = (PyDateTime_DELTA_GET_DAYS(utcoffset) * (int64_t)86400 + PyDateTime_DELTA_GET_SECONDS(utcoffset)) * (int64_t)1000 + (PyDateTime_DELTA_GET_MICROSECONDS(utcoffset) / 1000); +#endif } Py_DECREF(utcoffset); } @@ -558,7 +883,11 @@ static PyObject* decode_datetime(PyObject* self, long long millis, const codec_o } } +#ifdef Py_LIMITED_API + naive = datetime_from_millis(self, millis); +#else naive = datetime_from_millis(millis); +#endif if (!naive) { goto invalid; } @@ -578,7 +907,7 @@ static PyObject* decode_datetime(PyObject* self, long long millis, const codec_o /* convert to local time */ if (options->tzinfo != Py_None) { PyObject* astimezone_args[2] = {value, options->tzinfo}; - PyObject* temp = PyObject_VectorcallMethod( + PyObject* temp = PYMONGO_VECTORCALL_METHOD( state->_astimezone_str, astimezone_args, 2, NULL); Py_DECREF(value); value = temp; @@ -621,37 +950,33 @@ void buffer_write_int32_at_position(buffer_t buffer, static int write_unicode(buffer_t buffer, PyObject* py_string) { int size; - const char* data; - PyObject* encoded = PyUnicode_AsUTF8String(py_string); - if (!encoded) { + /* PyUnicode_AsUTF8AndSize returns a pointer to the string's cached UTF-8 + * buffer, avoiding the temporary bytes object that PyUnicode_AsUTF8String + * allocates. The returned pointer is valid for the lifetime of py_string, + * which the caller keeps alive for the duration of this call. */ + Py_ssize_t ssize; + const char* data = PyUnicode_AsUTF8AndSize(py_string, &ssize); + if (!data) { return 0; } - data = PyBytes_AS_STRING(encoded); - if (!data) - goto unicodefail; - if ((size = _downcast_and_check(PyBytes_GET_SIZE(encoded), 1)) == -1) - goto unicodefail; + if ((size = _downcast_and_check(ssize, 1)) == -1) + return 0; if (!buffer_write_int32(buffer, (int32_t)size)) - goto unicodefail; + return 0; if (!buffer_write_bytes(buffer, data, size)) - goto unicodefail; + return 0; - Py_DECREF(encoded); return 1; - -unicodefail: - Py_DECREF(encoded); - return 0; } /* returns 0 on failure */ static int write_string(buffer_t buffer, PyObject* py_string) { int size; const char* data; - if (PyUnicode_Check(py_string)){ + if (_CBSON_EXACT_OR_SUBCLASS(py_string, Unicode, &PyUnicode_Type)){ return write_unicode(buffer, py_string); } data = PyBytes_AsString(py_string); @@ -792,15 +1117,20 @@ static int _load_python_objects(PyObject* module) { } PyObject* compile_args[1] = {empty_string}; - compiled = PyObject_Vectorcall(re_compile, compile_args, 1, NULL); + compiled = PYMONGO_VECTORCALL(re_compile, compile_args, 1, NULL); Py_DECREF(re_compile); if (compiled == NULL) { state->REType = NULL; Py_DECREF(empty_string); return 1; } +#ifdef Py_LIMITED_API + state->REType = (PyObject*)Py_TYPE(compiled); + Py_INCREF(state->REType); +#else Py_INCREF(Py_TYPE(compiled)); state->REType = Py_TYPE(compiled); +#endif Py_DECREF(empty_string); Py_DECREF(compiled); return 0; @@ -836,11 +1166,17 @@ static long _type_marker(PyObject* object, PyObject* _type_marker_str) { * or method. In some cases "value" could be a subtype of something * we know how to serialize. Make a best effort to encode these types. */ - if (type_marker && PyLong_CheckExact(type_marker)) { - type = PyLong_AsLong(type_marker); - Py_DECREF(type_marker); - } else { - Py_XDECREF(type_marker); + if (type_marker) { +#ifdef Py_LIMITED_API + if (Py_IS_TYPE(type_marker, &PyLong_Type)) { +#else + if (PyLong_CheckExact(type_marker)) { +#endif + type = PyLong_AsLong(type_marker); + Py_DECREF(type_marker); + } else { + Py_XDECREF(type_marker); + } } return type; @@ -1131,7 +1467,7 @@ static int _write_element_to_buffer(PyObject* self, buffer_t buffer, * of PyObject_HasAttr/PyObject_GetAttr calls for the most common cases. */ if (PyUnicode_CheckExact(value) || PyLong_CheckExact(value) || PyFloat_CheckExact(value) || - PyBool_Check(value) || PyDict_CheckExact(value) || PyList_CheckExact(value) || + _CBSON_EXACT_OR_SUBCLASS(value, Bool, &PyBool_Type) || PyDict_CheckExact(value) || PyList_CheckExact(value) || PyTuple_CheckExact(value) || PyBytes_CheckExact(value) || value == Py_None) { type = 0; } else { @@ -1350,7 +1686,7 @@ static int _write_element_to_buffer(PyObject* self, buffer_t buffer, { /* DBRef */ PyObject* as_doc_args[1] = {value}; - PyObject* as_doc = PyObject_VectorcallMethod( + PyObject* as_doc = PYMONGO_VECTORCALL_METHOD( state->_as_doc_str, as_doc_args, 1, NULL); if (!as_doc) { return 0; @@ -1388,16 +1724,157 @@ static int _write_element_to_buffer(PyObject* self, buffer_t buffer, /* No _type_marker attribute or not one of our types. */ - if (PyBool_Check(value)) { + PyTypeObject* vtype = Py_TYPE(value); + + /* Fast exact-type dispatch. Py_TYPE(value) is a single deref available in + * both the limited and non-limited builds; comparing it against the built-in + * type objects (and the cached datetime/regex types) is a cheap pointer + * compare, whereas the subclass checks (PyXxx_Check) expand to an out-of-line + * PyType_GetFlags() call under the limited API. Subclasses fall through to the + * checks below, reusing the same handlers. */ + if (vtype == &PyUnicode_Type) goto handle_unicode; + if (vtype == &PyDict_Type) goto handle_dict; + if (vtype == &PyBool_Type) goto handle_bool; + if (vtype == &PyLong_Type) goto handle_long; + if (vtype == &PyFloat_Type) goto handle_float; + if (value == Py_None) goto handle_none; + if (vtype == &PyList_Type || vtype == &PyTuple_Type) goto handle_list; + if (vtype == &PyBytes_Type) goto handle_bytes; +#ifdef Py_LIMITED_API + if (vtype == (PyTypeObject*)state->datetime_type) goto handle_datetime; +#else + if (vtype == (PyTypeObject*)PyDateTimeAPI->DateTimeType) goto handle_datetime; +#endif + if (vtype == (PyTypeObject*)state->REType) goto handle_regex; + + /* Subclass fallback (an exact built-in did not match). */ + if (PyBool_Check(value)) goto handle_bool; + if (PyLong_Check(value)) goto handle_long; + if (PyFloat_Check(value)) goto handle_float; + if (PyDict_Check(value)) goto handle_dict; + if (PyList_Check(value) || PyTuple_Check(value)) goto handle_list; + if (PyBytes_Check(value)) goto handle_bytes; + if (PyUnicode_Check(value)) goto handle_unicode; +#ifdef Py_LIMITED_API + if (Py_IS_TYPE(value, (PyTypeObject*)state->datetime_type) || + PyObject_IsInstance(value, state->datetime_type)) goto handle_datetime; +#else + if (PyDateTime_Check(value)) goto handle_datetime; +#endif +#ifdef Py_LIMITED_API + if (Py_IS_TYPE(value, (PyTypeObject*)state->REType) || + PyObject_IsInstance(value, state->REType)) goto handle_regex; +#else + if (PyObject_TypeCheck(value, state->REType)) goto handle_regex; +#endif + if (PyObject_IsInstance(value, state->Mapping)) { + /* PyObject_IsInstance returns -1 on error */ + if (PyErr_Occurred()) { + return 0; + } + *(pymongo_buffer_get_buffer(buffer) + type_byte) = 0x03; + return write_dict(self, buffer, value, check_keys, options, 0); + } else if (PyObject_IsInstance(value, state->UUID)) { + PyObject* binary_value = NULL; + PyObject *uuid_rep_obj = NULL; + int result; + + /* PyObject_IsInstance returns -1 on error */ + if (PyErr_Occurred()) { + return 0; + } + + if (!(uuid_rep_obj = PyLong_FromLong(options->uuid_rep))) { + return 0; + } + PyObject* from_uuid_args[3] = {state->Binary, value, uuid_rep_obj}; + binary_value = PYMONGO_VECTORCALL_METHOD( + state->_from_uuid_str, from_uuid_args, 3, NULL); + Py_DECREF(uuid_rep_obj); + + if (binary_value == NULL) { + return 0; + } + + result = _write_element_to_buffer(self, buffer, + type_byte, binary_value, + check_keys, options, + in_custom_call, + in_fallback_call); + Py_DECREF(binary_value); + return result; + } + +handle_fallback: + /* Try a custom encoder if one is provided and we have not already + * attempted to use a type encoder. */ + if (!in_custom_call && !options->type_registry.is_encoder_empty) { + PyObject* value_type = NULL; + PyObject* converter = NULL; + value_type = PyObject_Type(value); + if (value_type == NULL) { + return 0; + } + converter = PyDict_GetItem(options->type_registry.encoder_map, value_type); + Py_XDECREF(value_type); + if (converter != NULL) { + /* Transform types that have a registered converter. + * A new reference is created upon transformation. */ + PyObject* converter_args[1] = {value}; + new_value = PYMONGO_VECTORCALL(converter, converter_args, 1, NULL); + if (new_value == NULL) { + return 0; + } + retval = write_element_to_buffer(self, buffer, type_byte, new_value, + check_keys, options, 1, 0); + Py_XDECREF(new_value); + return retval; + } + } + + /* Try the fallback encoder if one is provided and we have not already + * attempted to use the fallback encoder. */ + if (!in_fallback_call && options->type_registry.has_fallback_encoder) { + PyObject* fallback_args[1] = {value}; + new_value = PYMONGO_VECTORCALL( + options->type_registry.fallback_encoder, fallback_args, 1, NULL); + if (new_value == NULL) { + // propagate any exception raised by the callback + return 0; + } + retval = write_element_to_buffer(self, buffer, type_byte, new_value, + check_keys, options, 0, 1); + Py_XDECREF(new_value); + return retval; + } + + /* We can't determine value's type. Fail. */ + _set_cannot_encode(value); + return 0; + +handle_unicode: + { + *(pymongo_buffer_get_buffer(buffer) + type_byte) = 0x02; + return write_unicode(buffer, value); + } +handle_dict: + { + *(pymongo_buffer_get_buffer(buffer) + type_byte) = 0x03; + return write_dict(self, buffer, value, check_keys, options, 0); + } +handle_bool: + { const char c = (value == Py_True) ? 0x01 : 0x00; *(pymongo_buffer_get_buffer(buffer) + type_byte) = 0x08; return buffer_write_bytes(buffer, &c, 1); } - else if (PyLong_Check(value)) { +handle_long: + { const long long long_long_value = PyLong_AsLongLong(value); if (long_long_value == -1 && PyErr_Occurred()) { /* Ignore error and give the fallback_encoder a chance. */ PyErr_Clear(); + goto handle_fallback; } else if (-2147483648LL <= long_long_value && long_long_value <= 2147483647LL) { *(pymongo_buffer_get_buffer(buffer) + type_byte) = 0x10; return buffer_write_int32(buffer, (int32_t)long_long_value); @@ -1405,23 +1882,27 @@ static int _write_element_to_buffer(PyObject* self, buffer_t buffer, *(pymongo_buffer_get_buffer(buffer) + type_byte) = 0x12; return buffer_write_int64(buffer, (int64_t)long_long_value); } - } else if (PyFloat_Check(value)) { + } +handle_float: + { const double d = PyFloat_AsDouble(value); *(pymongo_buffer_get_buffer(buffer) + type_byte) = 0x01; return buffer_write_double(buffer, d); - } else if (value == Py_None) { + } +handle_none: + { *(pymongo_buffer_get_buffer(buffer) + type_byte) = 0x0A; return 1; - } else if (PyDict_Check(value)) { - *(pymongo_buffer_get_buffer(buffer) + type_byte) = 0x03; - return write_dict(self, buffer, value, check_keys, options, 0); - } else if ((is_list = PyList_Check(value)) || PyTuple_Check(value)) { + } +handle_list: + { Py_ssize_t items, i; int start_position, length_location, length; char zero = 0; + is_list = PyList_Check(value); *(pymongo_buffer_get_buffer(buffer) + type_byte) = 0x04; start_position = pymongo_buffer_get_position(buffer); @@ -1460,9 +1941,9 @@ static int _write_element_to_buffer(PyObject* self, buffer_t buffer, return 0; } if (is_list) { - item_value = PyList_GET_ITEM(value, i); + item_value = PyList_GetItem(value, i); } else { - item_value = PyTuple_GET_ITEM(value, i); + item_value = PyTuple_GetItem(value, i); } if (!item_value) { return 0; @@ -1482,14 +1963,16 @@ static int _write_element_to_buffer(PyObject* self, buffer_t buffer, buffer_write_int32_at_position( buffer, length_location, (int32_t)length); return 1; - /* Python3 special case. Store bytes as BSON binary subtype 0. */ - } else if (PyBytes_Check(value)) { + } +handle_bytes: + { char subtype = 0; int size; - const char* data = PyBytes_AS_STRING(value); - if (!data) + Py_ssize_t ssize; + char* data; + if (PyBytes_AsStringAndSize(value, &data, &ssize) < 0) return 0; - if ((size = _downcast_and_check(PyBytes_GET_SIZE(value), 0)) == -1) + if ((size = _downcast_and_check(ssize, 0)) == -1) return 0; *(pymongo_buffer_get_buffer(buffer) + type_byte) = 0x05; if (!buffer_write_int32(buffer, (int32_t)size)) { @@ -1502,117 +1985,66 @@ static int _write_element_to_buffer(PyObject* self, buffer_t buffer, return 0; } return 1; - } else if (PyUnicode_Check(value)) { - *(pymongo_buffer_get_buffer(buffer) + type_byte) = 0x02; - return write_unicode(buffer, value); - } else if (PyDateTime_Check(value)) { + } +handle_datetime: + { long long millis; PyObject* utcoffset_args[1] = {value}; - PyObject* utcoffset = PyObject_VectorcallMethod( + PyObject* utcoffset = PYMONGO_VECTORCALL_METHOD( state->_utcoffset_str, utcoffset_args, 1, NULL); if (utcoffset == NULL) return 0; if (utcoffset != Py_None) { - PyObject* result = PyNumber_Subtract(value, utcoffset); - if (!result) { - Py_DECREF(utcoffset); - return 0; +#ifndef Py_LIMITED_API + /* Read the offset timedelta's fields directly instead of building a + * second aware datetime with PyNumber_Subtract. The offset is whole + * milliseconds for every real timezone, so subtracting in millis is + * byte-identical to value - utcoffset(); guard on a non-sub-ms + * offset to keep the uncommon sub-millisecond case exact. */ + if (PyDelta_Check(utcoffset) && + PyDateTime_DELTA_GET_MICROSECONDS(utcoffset) % 1000 == 0) { + long long offset_ms = + PyDateTime_DELTA_GET_DAYS(utcoffset) * 86400000LL + + PyDateTime_DELTA_GET_SECONDS(utcoffset) * 1000LL + + PyDateTime_DELTA_GET_MICROSECONDS(utcoffset) / 1000LL; + millis = millis_from_datetime(value) - offset_ms; + } else +#endif + { + PyObject* result = PyNumber_Subtract(value, utcoffset); + if (!result) { + Py_DECREF(utcoffset); + return 0; + } +#ifdef Py_LIMITED_API + millis = millis_from_datetime(state, result); +#else + millis = millis_from_datetime(result); +#endif + Py_DECREF(result); } - millis = millis_from_datetime(result); - Py_DECREF(result); } else { +#ifdef Py_LIMITED_API + millis = millis_from_datetime(state, value); +#else millis = millis_from_datetime(value); +#endif + } + if (millis == -1 && PyErr_Occurred()) { + Py_DECREF(utcoffset); + return 0; } Py_DECREF(utcoffset); *(pymongo_buffer_get_buffer(buffer) + type_byte) = 0x09; return buffer_write_int64(buffer, (int64_t)millis); - } else if (PyObject_TypeCheck(value, state->REType)) { - return _write_regex_to_buffer(buffer, type_byte, value, state->_flags_str, state->_pattern_str); - } else if (PyObject_IsInstance(value, state->Mapping)) { - /* PyObject_IsInstance returns -1 on error */ - if (PyErr_Occurred()) { - return 0; - } - *(pymongo_buffer_get_buffer(buffer) + type_byte) = 0x03; - return write_dict(self, buffer, value, check_keys, options, 0); - } else if (PyObject_IsInstance(value, state->UUID)) { - PyObject* binary_value = NULL; - PyObject *uuid_rep_obj = NULL; - int result; - - /* PyObject_IsInstance returns -1 on error */ - if (PyErr_Occurred()) { - return 0; - } - - if (!(uuid_rep_obj = PyLong_FromLong(options->uuid_rep))) { - return 0; - } - PyObject* from_uuid_args[3] = {state->Binary, value, uuid_rep_obj}; - binary_value = PyObject_VectorcallMethod( - state->_from_uuid_str, from_uuid_args, 3, NULL); - Py_DECREF(uuid_rep_obj); - - if (binary_value == NULL) { - return 0; - } - - result = _write_element_to_buffer(self, buffer, - type_byte, binary_value, - check_keys, options, - in_custom_call, - in_fallback_call); - Py_DECREF(binary_value); - return result; - } - - /* Try a custom encoder if one is provided and we have not already - * attempted to use a type encoder. */ - if (!in_custom_call && !options->type_registry.is_encoder_empty) { - PyObject* value_type = NULL; - PyObject* converter = NULL; - value_type = PyObject_Type(value); - if (value_type == NULL) { - return 0; - } - converter = PyDict_GetItem(options->type_registry.encoder_map, value_type); - Py_XDECREF(value_type); - if (converter != NULL) { - /* Transform types that have a registered converter. - * A new reference is created upon transformation. */ - PyObject* converter_args[1] = {value}; - new_value = PyObject_Vectorcall(converter, converter_args, 1, NULL); - if (new_value == NULL) { - return 0; - } - retval = write_element_to_buffer(self, buffer, type_byte, new_value, - check_keys, options, 1, 0); - Py_XDECREF(new_value); - return retval; - } } - - /* Try the fallback encoder if one is provided and we have not already - * attempted to use the fallback encoder. */ - if (!in_fallback_call && options->type_registry.has_fallback_encoder) { - PyObject* fallback_args[1] = {value}; - new_value = PyObject_Vectorcall( - options->type_registry.fallback_encoder, fallback_args, 1, NULL); - if (new_value == NULL) { - // propagate any exception raised by the callback - return 0; - } - retval = write_element_to_buffer(self, buffer, type_byte, new_value, - check_keys, options, 0, 1); - Py_XDECREF(new_value); - return retval; +handle_regex: + { + return _write_regex_to_buffer(buffer, type_byte, value, state->_flags_str, state->_pattern_str); } - - /* We can't determine value's type. Fail. */ - _set_cannot_encode(value); - return 0; } + static int check_key_name(const char* name, int name_length) { if (name_length > 0 && name[0] == '$') { @@ -1681,20 +2113,18 @@ int decode_and_write_pair(PyObject* self, buffer_t buffer, unsigned char check_keys, const codec_options_t* options, unsigned char top_level) { - PyObject* encoded; const char* data; int size; - if (PyUnicode_Check(key)) { - encoded = PyUnicode_AsUTF8String(key); - if (!encoded) { + Py_ssize_t ssize; + if (_CBSON_EXACT_OR_SUBCLASS(key, Unicode, &PyUnicode_Type)) { + /* PyUnicode_AsUTF8AndSize returns a pointer to the key's cached UTF-8 + * buffer, avoiding the temporary bytes object allocation that + * PyUnicode_AsUTF8String performs for the key of every element. */ + data = PyUnicode_AsUTF8AndSize(key, &ssize); + if (!data) { return 0; } - if (!(data = PyBytes_AS_STRING(encoded))) { - Py_DECREF(encoded); - return 0; - } - if ((size = _downcast_and_check(PyBytes_GET_SIZE(encoded), 1)) == -1) { - Py_DECREF(encoded); + if ((size = _downcast_and_check(ssize, 1)) == -1) { return 0; } if (strlen(data) != (size_t)(size - 1)) { @@ -1704,7 +2134,6 @@ int decode_and_write_pair(PyObject* self, buffer_t buffer, "Key names must not contain the NULL byte"); Py_DECREF(InvalidDocument); } - Py_DECREF(encoded); return 0; } } else { @@ -1734,11 +2163,9 @@ int decode_and_write_pair(PyObject* self, buffer_t buffer, /* If top_level is True, don't allow writing _id here - it was already written. */ if (!write_pair(self, buffer, data, size - 1, value, check_keys, options, !top_level)) { - Py_DECREF(encoded); return 0; } - Py_DECREF(encoded); return 1; } @@ -1746,8 +2173,13 @@ int decode_and_write_pair(PyObject* self, buffer_t buffer, /* Write a RawBSONDocument to the buffer. * Returns the number of bytes written or 0 on failure. */ +/* Get a read-only buffer view of a bytes-like object. + * Returns 1 on success or 0 on failure with an exception set. + */ +static int _get_buffer(PyObject *exporter, Py_buffer *view); + static int write_raw_doc(buffer_t buffer, PyObject* raw, PyObject* _raw_str) { - char* data; + char* bytes; Py_ssize_t len; int len_int; int bytes_written = 0; @@ -1761,26 +2193,26 @@ static int write_raw_doc(buffer_t buffer, PyObject* raw, PyObject* _raw_str) { if (PyBytes_Check(bytes_obj)) { /* The common case: raw is bytes. */ - data = PyBytes_AS_STRING(bytes_obj); - len = PyBytes_GET_SIZE(bytes_obj); + if (PyBytes_AsStringAndSize(bytes_obj, &bytes, &len) < 0) { + goto fail; + } } else if (PyMemoryView_Check(bytes_obj)) { /* raw may also be a memoryview of the decode buffer. */ if (!_get_buffer(bytes_obj, &view)) { goto fail; } - data = (char*)view.buf; + bytes = (char*)view.buf; len = view.len; } else { - PyErr_Format(PyExc_TypeError, - "RawBSONDocument.raw must be bytes or memoryview, not %.200s", - Py_TYPE(bytes_obj)->tp_name); + PyErr_SetString(PyExc_TypeError, + "RawBSONDocument.raw must be bytes or memoryview"); goto fail; } len_int = _downcast_and_check(len, 0); if (-1 == len_int) { goto fail; } - if (!buffer_write_bytes(buffer, data, len_int)) { + if (!buffer_write_bytes(buffer, bytes, len_int)) { goto fail; } bytes_written = len_int; @@ -1811,7 +2243,7 @@ void handle_invalid_doc_error(PyObject* dict) { if (PyErr_GivenExceptionMatches(exc, InvalidDocument)) { msg = PyObject_Str(exc); if (msg) { - const char *msg_utf8 = PyUnicode_AsUTF8(msg); + const char *msg_utf8 = PyUnicode_AsUTF8AndSize(msg, NULL); if (msg_utf8 == NULL) { goto cleanup; } @@ -1821,7 +2253,7 @@ void handle_invalid_doc_error(PyObject* dict) { } /* Add doc to the error instance as a property. */ PyObject* exc_args[2] = {new_msg, dict}; - PyObject* new_exc = PyObject_Vectorcall(InvalidDocument, exc_args, 2, NULL); + PyObject* new_exc = PYMONGO_VECTORCALL(InvalidDocument, exc_args, 2, NULL); if (new_exc) { exc = _transfer_traceback(exc, new_exc); } @@ -1845,7 +2277,7 @@ void handle_invalid_doc_error(PyObject* dict) { if (evalue && PyErr_GivenExceptionMatches(etype, InvalidDocument)) { msg = PyObject_Str(evalue); if (msg) { - const char * msg_utf8 = PyUnicode_AsUTF8(msg); + const char * msg_utf8 = PyUnicode_AsUTF8AndSize(msg, NULL); if (msg_utf8 == NULL) { goto cleanup; } @@ -1855,7 +2287,7 @@ void handle_invalid_doc_error(PyObject* dict) { } // Add doc to the error instance as a property. PyObject* exc_args[2] = {new_msg, dict}; - new_evalue = PyObject_Vectorcall(InvalidDocument, exc_args, 2, NULL); + new_evalue = PYMONGO_VECTORCALL(InvalidDocument, exc_args, 2, NULL); Py_DECREF(evalue); Py_DECREF(etype); etype = InvalidDocument; @@ -1891,7 +2323,7 @@ int write_dict(PyObject* self, buffer_t buffer, int length_location; struct module_state *state = GETSTATE(self); long type_marker; - int is_dict = PyDict_Check(dict); + int is_dict = _CBSON_EXACT_OR_SUBCLASS(dict, Dict, &PyDict_Type); if (!state) { return 0; } @@ -2140,7 +2572,7 @@ static PyObject *_dbref_hook(PyObject* self, PyObject* value) { } PyObject* dbref_args[4] = {ref, id, database, value}; - ret = PyObject_Vectorcall(state->DBRef, dbref_args, 4, NULL); + ret = PYMONGO_VECTORCALL(state->DBRef, dbref_args, 4, NULL); Py_DECREF(value); } else { ret = value; @@ -2361,7 +2793,7 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, goto uuiderror; } PyObject* binary_args[2] = {data, subtype_obj}; - binary_value = PyObject_Vectorcall(state->Binary, binary_args, 2, NULL); + binary_value = PYMONGO_VECTORCALL(state->Binary, binary_args, 2, NULL); Py_DECREF(subtype_obj); if (binary_value == NULL) { goto uuiderror; @@ -2378,7 +2810,7 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, goto uuiderror; } PyObject* as_uuid_args[2] = {binary_value, uuid_rep_obj}; - value = PyObject_VectorcallMethod( + value = PYMONGO_VECTORCALL_METHOD( state->_as_uuid_str, as_uuid_args, 2, NULL); Py_DECREF(uuid_rep_obj); } @@ -2399,7 +2831,7 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, goto invalid; } PyObject* binary_args[2] = {data, st}; - value = PyObject_Vectorcall(state->Binary, binary_args, 2, NULL); + value = PYMONGO_VECTORCALL(state->Binary, binary_args, 2, NULL); Py_DECREF(st); Py_DECREF(data); if (!value) { @@ -2425,7 +2857,7 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, goto invalid; } PyObject* oid_args[1] = {oid_bytes}; - value = PyObject_Vectorcall(state->ObjectId, oid_args, 1, NULL); + value = PYMONGO_VECTORCALL(state->ObjectId, oid_args, 1, NULL); Py_DECREF(oid_bytes); *position += 12; break; @@ -2511,7 +2943,7 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, goto invalid; } PyObject* regex_args[2] = {pattern, flags_obj}; - value = PyObject_Vectorcall(state->Regex, regex_args, 2, NULL); + value = PYMONGO_VECTORCALL(state->Regex, regex_args, 2, NULL); Py_DECREF(flags_obj); Py_DECREF(pattern); break; @@ -2551,7 +2983,7 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, goto invalid; } PyObject* oid_args[1] = {oid_bytes}; - id = PyObject_Vectorcall(state->ObjectId, oid_args, 1, NULL); + id = PYMONGO_VECTORCALL(state->ObjectId, oid_args, 1, NULL); Py_DECREF(oid_bytes); if (!id) { Py_DECREF(collection); @@ -2559,7 +2991,7 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, } *position += 12; PyObject* dbref_args[2] = {collection, id}; - value = PyObject_Vectorcall(state->DBRef, dbref_args, 2, NULL); + value = PYMONGO_VECTORCALL(state->DBRef, dbref_args, 2, NULL); Py_DECREF(collection); Py_DECREF(id); break; @@ -2590,7 +3022,7 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, } *position += value_length; PyObject* code_args[1] = {code}; - value = PyObject_Vectorcall(state->Code, code_args, 1, NULL); + value = PYMONGO_VECTORCALL(state->Code, code_args, 1, NULL); Py_DECREF(code); break; } @@ -2657,7 +3089,7 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, *position += scope_size; PyObject* code_scope_args[2] = {code, scope}; - value = PyObject_Vectorcall(state->Code, code_scope_args, 2, NULL); + value = PYMONGO_VECTORCALL(state->Code, code_scope_args, 2, NULL); Py_DECREF(code); Py_DECREF(scope); break; @@ -2697,7 +3129,7 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, goto invalid; } PyObject* ts_args[2] = {time_obj, inc_obj}; - value = PyObject_Vectorcall(state->Timestamp, ts_args, 2, NULL); + value = PYMONGO_VECTORCALL(state->Timestamp, ts_args, 2, NULL); Py_DECREF(time_obj); Py_DECREF(inc_obj); *position += 8; @@ -2716,7 +3148,7 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, goto invalid; } PyObject* int64_args[1] = {ll_obj}; - value = PyObject_Vectorcall(state->BSONInt64, int64_args, 1, NULL); + value = PYMONGO_VECTORCALL(state->BSONInt64, int64_args, 1, NULL); Py_DECREF(ll_obj); *position += 8; break; @@ -2731,7 +3163,7 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, goto invalid; } PyObject* dec128_args[2] = {state->Decimal128, _bytes_obj}; - value = PyObject_VectorcallMethod( + value = PYMONGO_VECTORCALL_METHOD( state->_from_bid_str, dec128_args, 2, NULL); Py_DECREF(_bytes_obj); *position += 16; @@ -2739,12 +3171,12 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, } case 255: { - value = PyObject_Vectorcall(state->MinKey, NULL, 0, NULL); + value = PYMONGO_VECTORCALL(state->MinKey, NULL, 0, NULL); break; } case 127: { - value = PyObject_Vectorcall(state->MaxKey, NULL, 0, NULL); + value = PYMONGO_VECTORCALL(state->MaxKey, NULL, 0, NULL); break; } default: @@ -2797,7 +3229,7 @@ static PyObject* get_value(PyObject* self, PyObject* name, const char* buffer, converter = PyDict_GetItem(options->type_registry.decoder_map, value_type); if (converter != NULL) { PyObject* converter_args[1] = {value}; - PyObject* new_value = PyObject_Vectorcall(converter, converter_args, 1, NULL); + PyObject* new_value = PYMONGO_VECTORCALL(converter, converter_args, 1, NULL); Py_DECREF(value_type); Py_DECREF(value); return new_value; @@ -2867,6 +3299,7 @@ static int _element_to_dict(PyObject* self, const char* string, } static PyObject* _cbson_element_to_dict(PyObject* self, PyObject* args) { + /* TODO: Support buffer protocol */ char* string; PyObject* bson; PyObject* options_obj = NULL; @@ -2877,7 +3310,7 @@ static PyObject* _cbson_element_to_dict(PyObject* self, PyObject* args) { int raw_array = 0; PyObject* name; PyObject* value; - PyObject* result_tuple = NULL; + PyObject* result_tuple; if (!(PyArg_ParseTuple(args, "OIIOp", &bson, &position, &max, &options_obj, &raw_array) && @@ -2887,23 +3320,26 @@ static PyObject* _cbson_element_to_dict(PyObject* self, PyObject* args) { if (!PyBytes_Check(bson)) { PyErr_SetString(PyExc_TypeError, "argument to _element_to_dict must be a bytes object"); - goto done; + return NULL; + } + Py_ssize_t bson_len; + if (PyBytes_AsStringAndSize(bson, &string, &bson_len) < 0) { + return NULL; } - string = PyBytes_AS_STRING(bson); options.buffer_owner = bson; new_position = _element_to_dict(self, string, position, max, &options, raw_array, &name, &value); if (new_position < 0) { - goto done; + return NULL; } result_tuple = Py_BuildValue("NNi", name, value, new_position); if (!result_tuple) { Py_DECREF(name); Py_DECREF(value); + return NULL; } -done: destroy_codec_options(&options); return result_tuple; } @@ -2971,12 +3407,19 @@ static PyObject* elements_to_dict(PyObject* self, const char* string, if (options->is_raw_bson) { PyObject* bson_bytes; PyObject* buffer_owner = options->buffer_owner; - int owner_is_bytes = buffer_owner && PyBytes_Check(buffer_owner); + const char* owner_data = NULL; + Py_ssize_t owner_size = 0; + int owner_is_bytes = 0; + if (buffer_owner && PyBytes_Check(buffer_owner)) { + if (PyBytes_AsStringAndSize(buffer_owner, (char**)&owner_data, &owner_size) < 0) { + return NULL; + } + owner_is_bytes = 1; + } Py_ssize_t offset = 0; if (owner_is_bytes) { - offset = string - PyBytes_AS_STRING(buffer_owner); - if (offset < 0 || - offset + (Py_ssize_t)max > PyBytes_GET_SIZE(buffer_owner)) { + offset = string - owner_data; + if (offset < 0 || offset + (Py_ssize_t)max > owner_size) { PyObject* InvalidBSON = _error("InvalidBSON"); if (InvalidBSON) { PyErr_SetString(InvalidBSON, "invalid buffer offset"); @@ -2985,24 +3428,18 @@ static PyObject* elements_to_dict(PyObject* self, const char* string, return NULL; } } - if (owner_is_bytes && offset == 0 && - (Py_ssize_t)max == PyBytes_GET_SIZE(buffer_owner)) { - /* The document spans the entire buffer, pass the buffer - * itself through. */ + if (owner_is_bytes && offset == 0 && (Py_ssize_t)max == owner_size) { + /* The document spans the entire buffer, pass the buffer itself through. */ bson_bytes = buffer_owner; Py_INCREF(bson_bytes); } else if (owner_is_bytes && (Py_ssize_t)max >= GETSTATE(self)->raw_bson_view_threshold) { - /* Zero-copy: pass a read-only slice of the buffer instead of a - * bytes copy. Only immutable (bytes) buffers may be sliced this - * way, mutable buffers must be copied so - * the caller can't mutate the document after decoding. */ + /* Zero-copy: pass a read-only slice of the buffer instead of a bytes copy. */ PyObject* top_view = PyMemoryView_FromObject(buffer_owner); if (!top_view) { return NULL; } - bson_bytes = PySequence_GetSlice(top_view, offset, - offset + (Py_ssize_t)max); + bson_bytes = PySequence_GetSlice(top_view, offset, offset + (Py_ssize_t)max); Py_DECREF(top_view); } else { bson_bytes = PyBytes_FromStringAndSize(string, max); @@ -3011,7 +3448,7 @@ static PyObject* elements_to_dict(PyObject* self, const char* string, return NULL; } PyObject* raw_args[2] = {bson_bytes, options->options_obj}; - result = PyObject_Vectorcall(options->document_class, raw_args, 2, NULL); + result = PYMONGO_VECTORCALL(options->document_class, raw_args, 2, NULL); Py_DECREF(bson_bytes); return result; } @@ -3046,8 +3483,6 @@ static int _get_buffer(PyObject *exporter, Py_buffer *view) { return 0; } -/* Return 1 if any document in a stream of BSON documents is at least - * `threshold` bytes. Malformed lengths return 0. */ static int _contains_view_eligible_doc(const char* data, Py_ssize_t len, Py_ssize_t threshold) { Py_ssize_t position = 0; @@ -3205,7 +3640,6 @@ static PyObject* _cbson_bson_to_dict(PyObject* self, PyObject* args) { } if (!_get_buffer(bson, &view)) { - Py_DECREF(bson); destroy_codec_options(&options); return result; } @@ -3506,14 +3940,14 @@ static PyObject* _cbson_array_of_documents_to_buffer(PyObject* self, PyObject* a static PyMethodDef _CBSONMethods[] = { {"_dict_to_bson", _cbson_dict_to_bson, METH_VARARGS, "convert a dictionary to a string containing its BSON representation."}, + {"_raw_to_dict", _cbson_raw_to_dict, METH_VARARGS, + "Decode all elements of a raw BSON document",}, {"_bson_to_dict", _cbson_bson_to_dict, METH_VARARGS, "convert a BSON string to a SON object."}, {"_decode_all", _cbson_decode_all, METH_VARARGS, "convert binary data to a sequence of documents."}, {"_element_to_dict", _cbson_element_to_dict, METH_VARARGS, "Decode a single key, value pair."}, - {"_raw_to_dict", _cbson_raw_to_dict, METH_VARARGS, - "Decode all elements of a raw BSON document into a result mapping."}, {"_array_of_documents_to_buffer", _cbson_array_of_documents_to_buffer, METH_VARARGS, "Convert raw array of documents to a stream of BSON documents"}, {"_test_long_long_to_str", _test_long_long_to_str, METH_VARARGS, "Test conversion of extreme and common Py_ssize_t values to str."}, {NULL, NULL, 0, NULL} @@ -3565,6 +3999,13 @@ static int _cbson_traverse(PyObject *m, visitproc visit, void *arg) { Py_VISIT(state->max_datetime); Py_VISIT(state->replace_args); Py_VISIT(state->replace_kwargs); +#ifdef Py_LIMITED_API + Py_VISIT(state->datetime_type); + Py_VISIT(state->timedelta_type); + Py_VISIT(state->_days_str); + Py_VISIT(state->_seconds_str); + Py_VISIT(state->_microseconds_delta_str); +#endif return 0; } @@ -3613,6 +4054,13 @@ static int _cbson_clear(PyObject *m) { Py_CLEAR(state->max_datetime); Py_CLEAR(state->replace_args); Py_CLEAR(state->replace_kwargs); +#ifdef Py_LIMITED_API + Py_CLEAR(state->datetime_type); + Py_CLEAR(state->timedelta_type); + Py_CLEAR(state->_days_str); + Py_CLEAR(state->_seconds_str); + Py_CLEAR(state->_microseconds_delta_str); +#endif return 0; } @@ -3625,10 +4073,42 @@ _cbson_exec(PyObject *m) PyObject *c_api_object; static void *_cbson_API[_cbson_API_POINTER_COUNT]; +#ifdef Py_LIMITED_API + /* Import datetime types for limited API compatibility */ + struct module_state *state = GETSTATE(m); + if (!state) { + INITERROR; + } + PyObject* datetime_module = PyImport_ImportModule("datetime"); + if (!datetime_module) { + INITERROR; + } + state->datetime_type = PyObject_GetAttrString(datetime_module, "datetime"); + state->timedelta_type = PyObject_GetAttrString(datetime_module, "timedelta"); + Py_DECREF(datetime_module); + if (!state->datetime_type || !state->timedelta_type) { + INITERROR; + } + + /* Fetch the frozen datetime C API (see the _PyDateTime_CAPI comment). */ + state->datetime_capi = + (_PyDateTime_CAPI*)PyCapsule_Import(_PYDATETIME_CAPSULE_NAME, 0); + if (!state->datetime_capi) { + INITERROR; + } + + /* Intern attribute name strings */ + if (!((state->_days_str = PyUnicode_InternFromString("days")) && + (state->_seconds_str = PyUnicode_InternFromString("seconds")) && + (state->_microseconds_delta_str = PyUnicode_InternFromString("microseconds")))) { + INITERROR; + } +#else PyDateTime_IMPORT; if (PyDateTimeAPI == NULL) { INITERROR; } +#endif /* Export C API */ _cbson_API[_cbson_buffer_write_bytes_INDEX] = (void *) buffer_write_bytes; @@ -3676,7 +4156,7 @@ static PyModuleDef_Slot _cbson_slots[] = { #if defined(Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED) {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED}, #endif -#if PY_VERSION_HEX >= 0x030D0000 +#if defined(Py_mod_gil) {Py_mod_gil, Py_MOD_GIL_NOT_USED}, #endif {0, NULL}, diff --git a/bson/time64.c b/bson/time64.c deleted file mode 100644 index a21fbb90bd..0000000000 --- a/bson/time64.c +++ /dev/null @@ -1,781 +0,0 @@ -/* - -Copyright (c) 2007-2010 Michael G Schwern - -This software originally derived from Paul Sheer's pivotal_gmtime_r.c. - -The MIT License: - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - -*/ - -/* - -Programmers who have available to them 64-bit time values as a 'long -long' type can use cbson_localtime64_r() and cbson_gmtime64_r() which correctly -converts the time even on 32-bit systems. Whether you have 64-bit time -values will depend on the operating system. - -cbson_localtime64_r() is a 64-bit equivalent of localtime_r(). - -cbson_gmtime64_r() is a 64-bit equivalent of gmtime_r(). - -*/ - -#ifdef _MSC_VER - #define _CRT_SECURE_NO_WARNINGS -#endif - -/* Including Python.h fixes issues with interpreters built with -std=c99. */ -#define PY_SSIZE_T_CLEAN -#include "Python.h" - -#include -#include "time64.h" -#include "time64_limits.h" - - -/* Spec says except for stftime() and the _r() functions, these - all return static memory. Stabbings! */ -static struct TM Static_Return_Date; - -static const int days_in_month[2][12] = { - {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}, - {31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}, -}; - -static const int julian_days_by_month[2][12] = { - {0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334}, - {0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335}, -}; - -static const int length_of_year[2] = { 365, 366 }; - -/* Some numbers relating to the gregorian cycle */ -static const Year years_in_gregorian_cycle = 400; -#define days_in_gregorian_cycle ((365 * 400) + 100 - 4 + 1) -static const Time64_T seconds_in_gregorian_cycle = days_in_gregorian_cycle * 60LL * 60LL * 24LL; - -/* Year range we can trust the time functions with */ -#define MAX_SAFE_YEAR 2037 -#define MIN_SAFE_YEAR 1971 - -/* 28 year Julian calendar cycle */ -#define SOLAR_CYCLE_LENGTH 28 - -/* Year cycle from MAX_SAFE_YEAR down. */ -static const int safe_years_high[SOLAR_CYCLE_LENGTH] = { - 2016, 2017, 2018, 2019, - 2020, 2021, 2022, 2023, - 2024, 2025, 2026, 2027, - 2028, 2029, 2030, 2031, - 2032, 2033, 2034, 2035, - 2036, 2037, 2010, 2011, - 2012, 2013, 2014, 2015 -}; - -/* Year cycle from MIN_SAFE_YEAR up */ -static const int safe_years_low[SOLAR_CYCLE_LENGTH] = { - 1996, 1997, 1998, 1971, - 1972, 1973, 1974, 1975, - 1976, 1977, 1978, 1979, - 1980, 1981, 1982, 1983, - 1984, 1985, 1986, 1987, - 1988, 1989, 1990, 1991, - 1992, 1993, 1994, 1995, -}; - -/* Let's assume people are going to be looking for dates in the future. - Let's provide some cheats so you can skip ahead. - This has a 4x speed boost when near 2008. -*/ -/* Number of days since epoch on Jan 1st, 2008 GMT */ -#define CHEAT_DAYS (1199145600 / 24 / 60 / 60) -#define CHEAT_YEARS 108 - -#define IS_LEAP(n) ((!(((n) + 1900) % 400) || (!(((n) + 1900) % 4) && (((n) + 1900) % 100))) != 0) -#define _TIME64_WRAP(a,b,m) ((a) = ((a) < 0 ) ? ((b)--, (a) + (m)) : (a)) - -#ifdef USE_SYSTEM_LOCALTIME -# define SHOULD_USE_SYSTEM_LOCALTIME(a) ( \ - (a) <= SYSTEM_LOCALTIME_MAX && \ - (a) >= SYSTEM_LOCALTIME_MIN \ -) -#else -# define SHOULD_USE_SYSTEM_LOCALTIME(a) (0) -#endif - -#ifdef USE_SYSTEM_GMTIME -# define SHOULD_USE_SYSTEM_GMTIME(a) ( \ - (a) <= SYSTEM_GMTIME_MAX && \ - (a) >= SYSTEM_GMTIME_MIN \ -) -#else -# define SHOULD_USE_SYSTEM_GMTIME(a) (0) -#endif - -/* Multi varadic macros are a C99 thing, alas */ -#ifdef TIME_64_DEBUG -# define TIME64_TRACE(format) (fprintf(stderr, format)) -# define TIME64_TRACE1(format, var1) (fprintf(stderr, format, var1)) -# define TIME64_TRACE2(format, var1, var2) (fprintf(stderr, format, var1, var2)) -# define TIME64_TRACE3(format, var1, var2, var3) (fprintf(stderr, format, var1, var2, var3)) -#else -# define TIME64_TRACE(format) ((void)0) -# define TIME64_TRACE1(format, var1) ((void)0) -# define TIME64_TRACE2(format, var1, var2) ((void)0) -# define TIME64_TRACE3(format, var1, var2, var3) ((void)0) -#endif - - -static int is_exception_century(Year year) -{ - int is_exception = ((year % 100 == 0) && !(year % 400 == 0)); - TIME64_TRACE1("# is_exception_century: %s\n", is_exception ? "yes" : "no"); - - return(is_exception); -} - - -/* Compare two dates. - The result is like cmp. - Ignores things like gmtoffset and dst -*/ -int cbson_cmp_date( const struct TM* left, const struct tm* right ) { - if( left->tm_year > right->tm_year ) - return 1; - else if( left->tm_year < right->tm_year ) - return -1; - - if( left->tm_mon > right->tm_mon ) - return 1; - else if( left->tm_mon < right->tm_mon ) - return -1; - - if( left->tm_mday > right->tm_mday ) - return 1; - else if( left->tm_mday < right->tm_mday ) - return -1; - - if( left->tm_hour > right->tm_hour ) - return 1; - else if( left->tm_hour < right->tm_hour ) - return -1; - - if( left->tm_min > right->tm_min ) - return 1; - else if( left->tm_min < right->tm_min ) - return -1; - - if( left->tm_sec > right->tm_sec ) - return 1; - else if( left->tm_sec < right->tm_sec ) - return -1; - - return 0; -} - - -/* Check if a date is safely inside a range. - The intention is to check if its a few days inside. -*/ -int cbson_date_in_safe_range( const struct TM* date, const struct tm* min, const struct tm* max ) { - if( cbson_cmp_date(date, min) == -1 ) - return 0; - - if( cbson_cmp_date(date, max) == 1 ) - return 0; - - return 1; -} - - -/* timegm() is not in the C or POSIX spec, but it is such a useful - extension I would be remiss in leaving it out. Also I need it - for cbson_localtime64() -*/ -Time64_T cbson_timegm64(const struct TM *date) { - Time64_T days = 0; - Time64_T seconds = 0; - Year year; - Year orig_year = (Year)date->tm_year; - int cycles = 0; - - if( orig_year > 100 ) { - cycles = (int)((orig_year - 100) / 400); - orig_year -= cycles * 400; - days += (Time64_T)cycles * days_in_gregorian_cycle; - } - else if( orig_year < -300 ) { - cycles = (int)((orig_year - 100) / 400); - orig_year -= cycles * 400; - days += (Time64_T)cycles * days_in_gregorian_cycle; - } - TIME64_TRACE3("# timegm/ cycles: %d, days: %lld, orig_year: %lld\n", cycles, days, orig_year); - - if( orig_year > 70 ) { - year = 70; - while( year < orig_year ) { - days += length_of_year[IS_LEAP(year)]; - year++; - } - } - else if ( orig_year < 70 ) { - year = 69; - do { - days -= length_of_year[IS_LEAP(year)]; - year--; - } while( year >= orig_year ); - } - - days += julian_days_by_month[IS_LEAP(orig_year)][date->tm_mon]; - days += date->tm_mday - 1; - - seconds = days * 60 * 60 * 24; - - seconds += date->tm_hour * 60 * 60; - seconds += date->tm_min * 60; - seconds += date->tm_sec; - - return(seconds); -} - - -#ifndef NDEBUG -static int check_tm(struct TM *tm) -{ - /* Don't forget leap seconds */ - assert(tm->tm_sec >= 0); - assert(tm->tm_sec <= 61); - - assert(tm->tm_min >= 0); - assert(tm->tm_min <= 59); - - assert(tm->tm_hour >= 0); - assert(tm->tm_hour <= 23); - - assert(tm->tm_mday >= 1); - assert(tm->tm_mday <= days_in_month[IS_LEAP(tm->tm_year)][tm->tm_mon]); - - assert(tm->tm_mon >= 0); - assert(tm->tm_mon <= 11); - - assert(tm->tm_wday >= 0); - assert(tm->tm_wday <= 6); - - assert(tm->tm_yday >= 0); - assert(tm->tm_yday <= length_of_year[IS_LEAP(tm->tm_year)]); - -#ifdef HAS_TM_TM_GMTOFF - assert(tm->tm_gmtoff >= -24 * 60 * 60); - assert(tm->tm_gmtoff <= 24 * 60 * 60); -#endif - - return 1; -} -#endif - - -/* The exceptional centuries without leap years cause the cycle to - shift by 16 -*/ -static Year cycle_offset(Year year) -{ - const Year start_year = 2000; - Year year_diff = year - start_year; - Year exceptions; - - if( year > start_year ) - year_diff--; - - exceptions = year_diff / 100; - exceptions -= year_diff / 400; - - TIME64_TRACE3("# year: %lld, exceptions: %lld, year_diff: %lld\n", - year, exceptions, year_diff); - - return exceptions * 16; -} - -/* For a given year after 2038, pick the latest possible matching - year in the 28 year calendar cycle. - - A matching year... - 1) Starts on the same day of the week. - 2) Has the same leap year status. - - This is so the calendars match up. - - Also the previous year must match. When doing Jan 1st you might - wind up on Dec 31st the previous year when doing a -UTC time zone. - - Finally, the next year must have the same start day of week. This - is for Dec 31st with a +UTC time zone. - It doesn't need the same leap year status since we only care about - January 1st. -*/ -static int safe_year(const Year year) -{ - int safe_year = 0; - Year year_cycle; - - if( year >= MIN_SAFE_YEAR && year <= MAX_SAFE_YEAR ) { - return (int)year; - } - - year_cycle = year + cycle_offset(year); - - /* safe_years_low is off from safe_years_high by 8 years */ - if( year < MIN_SAFE_YEAR ) - year_cycle -= 8; - - /* Change non-leap xx00 years to an equivalent */ - if( is_exception_century(year) ) - year_cycle += 11; - - /* Also xx01 years, since the previous year will be wrong */ - if( is_exception_century(year - 1) ) - year_cycle += 17; - - year_cycle %= SOLAR_CYCLE_LENGTH; - if( year_cycle < 0 ) - year_cycle = SOLAR_CYCLE_LENGTH + year_cycle; - - assert( year_cycle >= 0 ); - assert( year_cycle < SOLAR_CYCLE_LENGTH ); - if( year < MIN_SAFE_YEAR ) - safe_year = safe_years_low[year_cycle]; - else if( year > MAX_SAFE_YEAR ) - safe_year = safe_years_high[year_cycle]; - else - assert(0); - - TIME64_TRACE3("# year: %lld, year_cycle: %lld, safe_year: %d\n", - year, year_cycle, safe_year); - - assert(safe_year <= MAX_SAFE_YEAR && safe_year >= MIN_SAFE_YEAR); - - return safe_year; -} - - -void pymongo_copy_tm_to_TM64(const struct tm *src, struct TM *dest) { - if( src == NULL ) { - memset(dest, 0, sizeof(*dest)); - } - else { -# ifdef USE_TM64 - dest->tm_sec = src->tm_sec; - dest->tm_min = src->tm_min; - dest->tm_hour = src->tm_hour; - dest->tm_mday = src->tm_mday; - dest->tm_mon = src->tm_mon; - dest->tm_year = (Year)src->tm_year; - dest->tm_wday = src->tm_wday; - dest->tm_yday = src->tm_yday; - dest->tm_isdst = src->tm_isdst; - -# ifdef HAS_TM_TM_GMTOFF - dest->tm_gmtoff = src->tm_gmtoff; -# endif - -# ifdef HAS_TM_TM_ZONE - dest->tm_zone = src->tm_zone; -# endif - -# else - /* They're the same type */ - memcpy(dest, src, sizeof(*dest)); -# endif - } -} - - -void cbson_copy_TM64_to_tm(const struct TM *src, struct tm *dest) { - if( src == NULL ) { - memset(dest, 0, sizeof(*dest)); - } - else { -# ifdef USE_TM64 - dest->tm_sec = src->tm_sec; - dest->tm_min = src->tm_min; - dest->tm_hour = src->tm_hour; - dest->tm_mday = src->tm_mday; - dest->tm_mon = src->tm_mon; - dest->tm_year = (int)src->tm_year; - dest->tm_wday = src->tm_wday; - dest->tm_yday = src->tm_yday; - dest->tm_isdst = src->tm_isdst; - -# ifdef HAS_TM_TM_GMTOFF - dest->tm_gmtoff = src->tm_gmtoff; -# endif - -# ifdef HAS_TM_TM_ZONE - dest->tm_zone = src->tm_zone; -# endif - -# else - /* They're the same type */ - memcpy(dest, src, sizeof(*dest)); -# endif - } -} - - -/* Simulate localtime_r() to the best of our ability */ -struct tm * cbson_fake_localtime_r(const time_t *time, struct tm *result) { - const struct tm *static_result = localtime(time); - - assert(result != NULL); - - if( static_result == NULL ) { - memset(result, 0, sizeof(*result)); - return NULL; - } - else { - memcpy(result, static_result, sizeof(*result)); - return result; - } -} - - -/* Simulate gmtime_r() to the best of our ability */ -struct tm * cbson_fake_gmtime_r(const time_t *time, struct tm *result) { - const struct tm *static_result = gmtime(time); - - assert(result != NULL); - - if( static_result == NULL ) { - memset(result, 0, sizeof(*result)); - return NULL; - } - else { - memcpy(result, static_result, sizeof(*result)); - return result; - } -} - - -static Time64_T seconds_between_years(Year left_year, Year right_year) { - int increment = (left_year > right_year) ? 1 : -1; - Time64_T seconds = 0; - int cycles; - - if( left_year > 2400 ) { - cycles = (int)((left_year - 2400) / 400); - left_year -= cycles * 400; - seconds += cycles * seconds_in_gregorian_cycle; - } - else if( left_year < 1600 ) { - cycles = (int)((left_year - 1600) / 400); - left_year += cycles * 400; - seconds += cycles * seconds_in_gregorian_cycle; - } - - while( left_year != right_year ) { - seconds += length_of_year[IS_LEAP(right_year - 1900)] * 60 * 60 * 24; - right_year += increment; - } - - return seconds * increment; -} - - -Time64_T cbson_mktime64(const struct TM *input_date) { - struct tm safe_date; - struct TM date; - Time64_T time; - Year year = input_date->tm_year + 1900; - - if( cbson_date_in_safe_range(input_date, &SYSTEM_MKTIME_MIN, &SYSTEM_MKTIME_MAX) ) - { - cbson_copy_TM64_to_tm(input_date, &safe_date); - return (Time64_T)mktime(&safe_date); - } - - /* Have to make the year safe in date else it won't fit in safe_date */ - date = *input_date; - date.tm_year = safe_year(year) - 1900; - cbson_copy_TM64_to_tm(&date, &safe_date); - - time = (Time64_T)mktime(&safe_date); - - time += seconds_between_years(year, (Year)(safe_date.tm_year + 1900)); - - return time; -} - - -/* Because I think mktime() is a crappy name */ -Time64_T timelocal64(const struct TM *date) { - return cbson_mktime64(date); -} - - -struct TM *cbson_gmtime64_r (const Time64_T *in_time, struct TM *p) -{ - int v_tm_sec, v_tm_min, v_tm_hour, v_tm_mon, v_tm_wday; - Time64_T v_tm_tday; - int leap; - Time64_T m; - Time64_T time = *in_time; - Year year = 70; - int cycles = 0; - - assert(p != NULL); - -#ifdef USE_SYSTEM_GMTIME - /* Use the system gmtime() if time_t is small enough */ - if( SHOULD_USE_SYSTEM_GMTIME(*in_time) ) { - time_t safe_time = (time_t)*in_time; - struct tm safe_date; - GMTIME_R(&safe_time, &safe_date); - - pymongo_copy_tm_to_TM64(&safe_date, p); - assert(check_tm(p)); - - return p; - } -#endif - -#ifdef HAS_TM_TM_GMTOFF - p->tm_gmtoff = 0; -#endif - p->tm_isdst = 0; - -#ifdef HAS_TM_TM_ZONE - p->tm_zone = "UTC"; -#endif - - v_tm_sec = (int)(time % 60); - time /= 60; - v_tm_min = (int)(time % 60); - time /= 60; - v_tm_hour = (int)(time % 24); - time /= 24; - v_tm_tday = time; - - _TIME64_WRAP (v_tm_sec, v_tm_min, 60); - _TIME64_WRAP (v_tm_min, v_tm_hour, 60); - _TIME64_WRAP (v_tm_hour, v_tm_tday, 24); - - v_tm_wday = (int)((v_tm_tday + 4) % 7); - if (v_tm_wday < 0) - v_tm_wday += 7; - m = v_tm_tday; - - if (m >= CHEAT_DAYS) { - year = CHEAT_YEARS; - m -= CHEAT_DAYS; - } - - if (m >= 0) { - /* Gregorian cycles, this is huge optimization for distant times */ - cycles = (int)(m / (Time64_T) days_in_gregorian_cycle); - if( cycles ) { - m -= (cycles * (Time64_T) days_in_gregorian_cycle); - year += (cycles * years_in_gregorian_cycle); - } - - /* Years */ - leap = IS_LEAP (year); - while (m >= (Time64_T) length_of_year[leap]) { - m -= (Time64_T) length_of_year[leap]; - year++; - leap = IS_LEAP (year); - } - - /* Months */ - v_tm_mon = 0; - while (m >= (Time64_T) days_in_month[leap][v_tm_mon]) { - m -= (Time64_T) days_in_month[leap][v_tm_mon]; - v_tm_mon++; - } - } else { - year--; - - /* Gregorian cycles */ - cycles = (int)((m / (Time64_T) days_in_gregorian_cycle) + 1); - if( cycles ) { - m -= (cycles * (Time64_T) days_in_gregorian_cycle); - year += (cycles * years_in_gregorian_cycle); - } - - /* Years */ - leap = IS_LEAP (year); - while (m < (Time64_T) -length_of_year[leap]) { - m += (Time64_T) length_of_year[leap]; - year--; - leap = IS_LEAP (year); - } - - /* Months */ - v_tm_mon = 11; - while (m < (Time64_T) -days_in_month[leap][v_tm_mon]) { - m += (Time64_T) days_in_month[leap][v_tm_mon]; - v_tm_mon--; - } - m += (Time64_T) days_in_month[leap][v_tm_mon]; - } - - p->tm_year = (int)year; - if( p->tm_year != year ) { -#ifdef EOVERFLOW - errno = EOVERFLOW; -#endif - return NULL; - } - - /* At this point m is less than a year so casting to an int is safe */ - p->tm_mday = (int) m + 1; - p->tm_yday = julian_days_by_month[leap][v_tm_mon] + (int)m; - p->tm_sec = v_tm_sec; - p->tm_min = v_tm_min; - p->tm_hour = v_tm_hour; - p->tm_mon = v_tm_mon; - p->tm_wday = v_tm_wday; - - assert(check_tm(p)); - - return p; -} - - -struct TM *cbson_localtime64_r (const Time64_T *time, struct TM *local_tm) -{ - time_t safe_time; - struct tm safe_date; - struct TM gm_tm; - Year orig_year; - int month_diff; - - assert(local_tm != NULL); - -#ifdef USE_SYSTEM_LOCALTIME - /* Use the system localtime() if time_t is small enough */ - if( SHOULD_USE_SYSTEM_LOCALTIME(*time) ) { - safe_time = (time_t)*time; - - TIME64_TRACE1("Using system localtime for %lld\n", *time); - - LOCALTIME_R(&safe_time, &safe_date); - - pymongo_copy_tm_to_TM64(&safe_date, local_tm); - assert(check_tm(local_tm)); - - return local_tm; - } -#endif - - if( cbson_gmtime64_r(time, &gm_tm) == NULL ) { - TIME64_TRACE1("cbson_gmtime64_r returned null for %lld\n", *time); - return NULL; - } - - orig_year = gm_tm.tm_year; - - if (gm_tm.tm_year > (2037 - 1900) || - gm_tm.tm_year < (1970 - 1900) - ) - { - TIME64_TRACE1("Mapping tm_year %lld to safe_year\n", (Year)gm_tm.tm_year); - gm_tm.tm_year = safe_year((Year)(gm_tm.tm_year + 1900)) - 1900; - } - - safe_time = (time_t)cbson_timegm64(&gm_tm); - if( LOCALTIME_R(&safe_time, &safe_date) == NULL ) { - TIME64_TRACE1("localtime_r(%d) returned NULL\n", (int)safe_time); - return NULL; - } - - pymongo_copy_tm_to_TM64(&safe_date, local_tm); - - local_tm->tm_year = (int)orig_year; - if( local_tm->tm_year != orig_year ) { - TIME64_TRACE2("tm_year overflow: tm_year %lld, orig_year %lld\n", - (Year)local_tm->tm_year, (Year)orig_year); - -#ifdef EOVERFLOW - errno = EOVERFLOW; -#endif - return NULL; - } - - - month_diff = local_tm->tm_mon - gm_tm.tm_mon; - - /* When localtime is Dec 31st previous year and - gmtime is Jan 1st next year. - */ - if( month_diff == 11 ) { - local_tm->tm_year--; - } - - /* When localtime is Jan 1st, next year and - gmtime is Dec 31st, previous year. - */ - if( month_diff == -11 ) { - local_tm->tm_year++; - } - - /* GMT is Jan 1st, xx01 year, but localtime is still Dec 31st - in a non-leap xx00. There is one point in the cycle - we can't account for which the safe xx00 year is a leap - year. So we need to correct for Dec 31st coming out as - the 366th day of the year. - */ - if( !IS_LEAP(local_tm->tm_year) && local_tm->tm_yday == 365 ) - local_tm->tm_yday--; - - assert(check_tm(local_tm)); - - return local_tm; -} - - -int cbson_valid_tm_wday( const struct TM* date ) { - if( 0 <= date->tm_wday && date->tm_wday <= 6 ) - return 1; - else - return 0; -} - -int cbson_valid_tm_mon( const struct TM* date ) { - if( 0 <= date->tm_mon && date->tm_mon <= 11 ) - return 1; - else - return 0; -} - - -/* Non-thread safe versions of the above */ -struct TM *cbson_localtime64(const Time64_T *time) { -#ifdef _MSC_VER - _tzset(); -#else - tzset(); -#endif - return cbson_localtime64_r(time, &Static_Return_Date); -} - -struct TM *cbson_gmtime64(const Time64_T *time) { - return cbson_gmtime64_r(time, &Static_Return_Date); -} diff --git a/bson/time64.h b/bson/time64.h deleted file mode 100644 index 6321eb307e..0000000000 --- a/bson/time64.h +++ /dev/null @@ -1,67 +0,0 @@ -#ifndef TIME64_H -# define TIME64_H - -#include -#include "time64_config.h" - -/* Set our custom types */ -typedef INT_64_T Int64; -typedef Int64 Time64_T; -typedef Int64 Year; - - -/* A copy of the tm struct but with a 64 bit year */ -struct TM64 { - int tm_sec; - int tm_min; - int tm_hour; - int tm_mday; - int tm_mon; - Year tm_year; - int tm_wday; - int tm_yday; - int tm_isdst; - -#ifdef HAS_TM_TM_GMTOFF - long tm_gmtoff; -#endif - -#ifdef HAS_TM_TM_ZONE - char *tm_zone; -#endif -}; - - -/* Decide which tm struct to use */ -#ifdef USE_TM64 -#define TM TM64 -#else -#define TM tm -#endif - - -/* Declare public functions */ -struct TM *cbson_gmtime64_r (const Time64_T *, struct TM *); -struct TM *cbson_localtime64_r (const Time64_T *, struct TM *); -struct TM *cbson_gmtime64 (const Time64_T *); -struct TM *cbson_localtime64 (const Time64_T *); - -Time64_T cbson_timegm64 (const struct TM *); -Time64_T cbson_mktime64 (const struct TM *); -Time64_T timelocal64 (const struct TM *); - - -/* Not everyone has gm/localtime_r(), provide a replacement */ -#ifdef HAS_LOCALTIME_R -# define LOCALTIME_R(clock, result) localtime_r(clock, result) -#else -# define LOCALTIME_R(clock, result) cbson_fake_localtime_r(clock, result) -#endif -#ifdef HAS_GMTIME_R -# define GMTIME_R(clock, result) gmtime_r(clock, result) -#else -# define GMTIME_R(clock, result) cbson_fake_gmtime_r(clock, result) -#endif - - -#endif diff --git a/bson/time64_config.h b/bson/time64_config.h deleted file mode 100644 index 9d4c111c95..0000000000 --- a/bson/time64_config.h +++ /dev/null @@ -1,78 +0,0 @@ -/* Configuration - ------------- - Define as appropriate for your system. - Sensible defaults provided. -*/ - - -#ifndef TIME64_CONFIG_H -# define TIME64_CONFIG_H - -/* Debugging - TIME_64_DEBUG - Define if you want debugging messages -*/ -/* #define TIME_64_DEBUG */ - - -/* INT_64_T - A 64 bit integer type to use to store time and others. - Must be defined. -*/ -#define INT_64_T long long - - -/* USE_TM64 - Should we use a 64 bit safe replacement for tm? This will - let you go past year 2 billion but the struct will be incompatible - with tm. Conversion functions will be provided. -*/ -/* #define USE_TM64 */ - - -/* Availability of system functions. - - HAS_GMTIME_R - Define if your system has gmtime_r() - - HAS_LOCALTIME_R - Define if your system has localtime_r() - - HAS_TIMEGM - Define if your system has timegm(), a GNU extension. -*/ -#if !defined(WIN32) && !defined(_MSC_VER) -#define HAS_GMTIME_R -#define HAS_LOCALTIME_R -#endif -/* #define HAS_TIMEGM */ - - -/* Details of non-standard tm struct elements. - - HAS_TM_TM_GMTOFF - True if your tm struct has a "tm_gmtoff" element. - A BSD extension. - - HAS_TM_TM_ZONE - True if your tm struct has a "tm_zone" element. - A BSD extension. -*/ -/* #define HAS_TM_TM_GMTOFF */ -/* #define HAS_TM_TM_ZONE */ - - -/* USE_SYSTEM_LOCALTIME - USE_SYSTEM_GMTIME - USE_SYSTEM_MKTIME - USE_SYSTEM_TIMEGM - Should we use the system functions if the time is inside their range? - Your system localtime() is probably more accurate, but our gmtime() is - fast and safe. -*/ -#define USE_SYSTEM_LOCALTIME -/* #define USE_SYSTEM_GMTIME */ -#define USE_SYSTEM_MKTIME -/* #define USE_SYSTEM_TIMEGM */ - -#endif /* TIME64_CONFIG_H */ diff --git a/bson/time64_limits.h b/bson/time64_limits.h deleted file mode 100644 index 1d30607bae..0000000000 --- a/bson/time64_limits.h +++ /dev/null @@ -1,95 +0,0 @@ -/* - Maximum and minimum inputs your system's respective time functions - can correctly handle. time64.h will use your system functions if - the input falls inside these ranges and corresponding USE_SYSTEM_* - constant is defined. -*/ - -#ifndef TIME64_LIMITS_H -#define TIME64_LIMITS_H - -/* Max/min for localtime() */ -#define SYSTEM_LOCALTIME_MAX 2147483647 -#define SYSTEM_LOCALTIME_MIN -2147483647-1 - -/* Max/min for gmtime() */ -#define SYSTEM_GMTIME_MAX 2147483647 -#define SYSTEM_GMTIME_MIN -2147483647-1 - -/* Max/min for mktime() */ -static const struct tm SYSTEM_MKTIME_MAX = { - 7, - 14, - 19, - 18, - 0, - 138, - 1, - 17, - 0 -#ifdef HAS_TM_TM_GMTOFF - ,-28800 -#endif -#ifdef HAS_TM_TM_ZONE - ,"PST" -#endif -}; - -static const struct tm SYSTEM_MKTIME_MIN = { - 52, - 45, - 12, - 13, - 11, - 1, - 5, - 346, - 0 -#ifdef HAS_TM_TM_GMTOFF - ,-28800 -#endif -#ifdef HAS_TM_TM_ZONE - ,"PST" -#endif -}; - -/* Max/min for timegm() */ -#ifdef HAS_TIMEGM -static const struct tm SYSTEM_TIMEGM_MAX = { - 7, - 14, - 3, - 19, - 0, - 138, - 2, - 18, - 0 - #ifdef HAS_TM_TM_GMTOFF - ,0 - #endif - #ifdef HAS_TM_TM_ZONE - ,"UTC" - #endif -}; - -static const struct tm SYSTEM_TIMEGM_MIN = { - 52, - 45, - 20, - 13, - 11, - 1, - 5, - 346, - 0 - #ifdef HAS_TM_TM_GMTOFF - ,0 - #endif - #ifdef HAS_TM_TM_ZONE - ,"UTC" - #endif -}; -#endif /* HAS_TIMEGM */ - -#endif /* TIME64_LIMITS_H */ diff --git a/hatch_build.py b/hatch_build.py index 47a72c1303..d75fbf5a48 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -25,6 +25,17 @@ def initialize(self, version, build_data): # Ensure wheel is marked as binary and contains the binary files. build_data["infer_tag"] = True build_data["pure_python"] = False + if os.environ.get("PYMONGO_BUILD_ABI3"): + from packaging.tags import sys_tags + + # abi3 wheels support Python 3.11+ (with the GIL). Build the + # cp311-abi3- tag from the current platform. + tag = next( + t + for t in sys_tags() + if "manylinux" not in t.platform and "musllinux" not in t.platform + ) + build_data["tag"] = f"cp311-abi3-{tag.platform}" if os.name == "nt": patt = ".pyd" else: diff --git a/pymongo/_cmessagemodule.c b/pymongo/_cmessagemodule.c index 1597652d69..aeffb9e5bd 100644 --- a/pymongo/_cmessagemodule.c +++ b/pymongo/_cmessagemodule.c @@ -23,6 +23,15 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" +/* The limited C API (Py_LIMITED_API >= 0x030B0000) omits the libc headers + * from Python.h; include the ones this module uses. */ +#ifdef Py_LIMITED_API +#include +#include +#include +#include +#endif + #include "_cbsonmodule.h" #include "buffer.h" diff --git a/test/performance/bench_abi3.py b/test/performance/bench_abi3.py new file mode 100644 index 0000000000..bfb55e6507 --- /dev/null +++ b/test/performance/bench_abi3.py @@ -0,0 +1,262 @@ +# Copyright 2009-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Measure the throughput difference between abi3 and non-abi3 `_cbson`. + +PyMongo's C extensions normally use the CPython C API directly. The limited +API (abi3) replaces the C datetime calls with Python-level attribute access, +which costs some encode/decode throughput. This script measures that cost on +the `extended_bson` fixtures so we can decide whether the regression is +acceptable (< 5%). + +Workflow +-------- +1. Build a baseline (non-abi3) wheel and an abi3 wheel from the same tree. +2. Create two virtualenvs, one per wheel. +3. Point ``TEST_PATH`` at the ``extended_bson`` directory that contains + ``flat_bson.json``, ``deep_bson.json`` and ``full_bson.json``. +4. Run ``--measure`` in each environment: + + python test/performance/bench_abi3.py --measure --output abi3.json + +5. Compare the two result files: + + python test/performance/bench_abi3.py --compare \ + --baseline baseline.json --abi3 abi3.json + +``--compare`` exits non-zero when the overall regression exceeds +``REGRESSION_LIMIT`` (5%). + +Run this on a dedicated perf host (an Evergreen spawn host) because the +fixtures are small enough that machine load, CPU throttling, and other +background work dominate the measurement otherwise. +""" + +from __future__ import annotations + +import argparse +import glob +import json +import os +import statistics +import sys +import time +from typing import Any + +from bson import decode, encode, json_util + +REGRESSION_LIMIT = 0.05 + +# Each run holds ``target_time`` seconds of wall clock so the median is +# stable regardless of how fast the machine is. +TARGET_TIME = 1.0 +REPEATS = 7 + + +def load_documents(test_path: str) -> dict[str, Any]: + """Load the three BSON fixtures as Python documents.""" + docs: dict[str, Any] = {} + for name in ("flat_bson.json", "deep_bson.json", "full_bson.json"): + path = os.path.join(test_path, name) + if not os.path.exists(path): + raise FileNotFoundError(f"missing fixture: {path}") + with open(path) as f: + docs[name] = json_util.loads(f.read()) + return docs + + +def calibrate(fn, target_time: float) -> int: + """Return an iteration count that takes at least ``target_time`` seconds.""" + start = time.perf_counter() + count = 1 + while time.perf_counter() - start < target_time: + for _ in range(count): + fn() + count *= 2 + return max(count // 2, 1) + + +def per_call_seconds(fn, iterations: int, repeats: int) -> float: + """Return the median seconds per operation over ``repeats`` untimed runs.""" + samples = [] + for _ in range(repeats): + start = time.perf_counter() + for _ in range(iterations): + fn() + samples.append((time.perf_counter() - start) / iterations) + return statistics.median(samples) + + +def bench(documents: dict[str, Any], target_time: float, repeats: int) -> dict[str, float]: + """Measure encode and decode throughput for each fixture in MB/s.""" + prepared = {name: {"doc": doc, "encoded": encode(doc)} for name, doc in documents.items()} + results: dict[str, float] = {} + for name, data in prepared.items(): + size = len(data["encoded"]) + enc_time = per_call_seconds( + lambda: encode(data["doc"]), + calibrate(lambda: encode(data["doc"]), target_time), + repeats, + ) + dec_time = per_call_seconds( + lambda: decode(data["encoded"]), + calibrate(lambda: decode(data["encoded"]), target_time), + repeats, + ) + results[f"{name}.encode"] = size / (enc_time * 1024 * 1024) + results[f"{name}.decode"] = size / (dec_time * 1024 * 1024) + return results + + +def evergreen_results(results: dict[str, float]) -> list[dict[str, Any]]: + """Build Evergreen perf-harness result entries from measured MB/s.""" + entries = [] + for key, mb_per_sec in results.items(): + fixture, op = key.rsplit(".", 1) + entries.append( + { + "info": { + "test_name": f"{fixture.replace('.json', '')}-{op}", + "args": {"source": "abi3"}, + }, + "metrics": [ + { + "name": "megabytes_per_sec", + "type": "MEDIAN", + "value": mb_per_sec, + "metadata": { + "improvement_direction": "up", + "measurement_unit": "megabytes_per_second", + }, + } + ], + } + ) + return entries + + +def print_report(results: dict[str, float], baseline: dict[str, float]) -> float: + """Print the comparison table and return the overall regression.""" + print(f"\n{'fixture':<34}{'baseline MB/s':>14}{'abi3 MB/s':>12}{'regression':>12}") + print("-" * 72) + total_base = 0.0 + total_abi3 = 0.0 + for key in sorted(results): + base = baseline[key] + abi3 = results[key] + reg = (abi3 - base) / base if base else float("nan") + total_base += base + total_abi3 += abi3 + flag = " <-- OVER LIMIT" if reg < -REGRESSION_LIMIT else "" + print(f"{key:<34}{base:>14.2f}{abi3:>12.2f}{reg:>+12.2%}{flag}") + overall = (total_abi3 - total_base) / total_base if total_base else float("nan") + print("-" * 72) + print(f"{'Overall (sum of MB/s)':<34}{total_base:>14.2f}{total_abi3:>12.2f}{overall:>+12.2%}") + return overall + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--test-path", + default=os.environ.get("TEST_PATH"), + help="directory containing flat/deep/full_bson.json (default: $TEST_PATH)", + ) + parser.add_argument( + "--target-time", + type=float, + default=TARGET_TIME, + help=f"seconds of wall clock per measurement (default: {TARGET_TIME})", + ) + parser.add_argument("--repeats", type=int, default=REPEATS) + parser.add_argument( + "--pin", + action="store_true", + help="pin to the available CPU set (Unix only)", + ) + + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--measure", action="store_true", help="benchmark and write a results file") + mode.add_argument("--compare", action="store_true", help="compare two results files") + + parser.add_argument("--output", help="results file to write when using --measure") + parser.add_argument( + "--evergreen", + action="store_true", + help="write Evergreen perf-harness JSON to $OUTPUT_FILE (or --output)", + ) + parser.add_argument("--baseline", help="baseline results file when using --compare") + parser.add_argument("--abi3", help="abi3 results file when using --compare") + args = parser.parse_args() + + if args.pin and hasattr(os, "sched_setaffinity"): + try: + os.sched_setaffinity(0, {0}) + except (OSError, ValueError): + pass + + if args.measure: + if not args.test_path: + print("error: --measure requires --test-path or TEST_PATH", file=sys.stderr) + return 2 + print(f"C extension in use: {__import__('bson').has_c()}") + if not __import__("bson").has_c(): + print("error: bson C extension not installed", file=sys.stderr) + return 2 + test_path = args.test_path + for candidate in (test_path, os.path.join(test_path, "extended_bson")): + if glob.glob(os.path.join(candidate, "flat_bson.json")): + test_path = candidate + break + documents = load_documents(test_path) + results = bench(documents, args.target_time, args.repeats) + + # A single run can't compare abi3 to baseline; it only emits the + # measurements. The compare step decides pass/fail. + if args.evergreen: + evergreen_path = args.output or os.environ.get("OUTPUT_FILE") + if not evergreen_path: + print("error: --evergreen requires --output or OUTPUT_FILE", file=sys.stderr) + return 2 + with open(evergreen_path, "w") as f: + json.dump(evergreen_results(results), f, indent=4) + print(f"wrote Evergreen results to {evergreen_path}") + return 0 + + if not args.output: + print("error: --measure requires --output", file=sys.stderr) + return 2 + with open(args.output, "w") as f: + json.dump(results, f, indent=2, sort_keys=True) + print(f"wrote {args.output}: {len(results)} measurements") + return 0 + + if not args.baseline or not args.abi3: + print("error: --compare requires --baseline and --abi3", file=sys.stderr) + return 2 + with open(args.baseline) as f: + baseline = json.load(f) + with open(args.abi3) as f: + abi3 = json.load(f) + overall = print_report(abi3, baseline) + ok = overall >= -REGRESSION_LIMIT + print( + f"\n{'PASS' if ok else 'FAIL'}: overall regression {overall:+.2%} " + f"(limit -{REGRESSION_LIMIT:.0%})" + ) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main())