From 1861a65e6a4e0fddea8d52b4fef046e7222e988e Mon Sep 17 00:00:00 2001 From: "Kirill Lukonin (Evil Wireless Man)" Date: Mon, 31 Aug 2026 22:49:25 +0300 Subject: [PATCH 1/6] unix-ffi/re: Free the memory allocated by PCRE2. Every call to search() allocated a match data block with pcre2_match_data_create_from_pattern() and never freed it again, leaking a few kilobytes per call, on the no-match path as well. Free it once the offsets have been copied out of it. The module level functions compile a pattern that the caller never gets to see, and that was leaked as well. Free it when the call is done; the match object that is returned does not refer to it. Note that a pattern returned by re.compile() still has to be kept alive by the caller and cannot be released automatically, because MicroPython does not run __del__ on instances of Python classes. Signed-off-by: Kirill Lukonin (Evil Wireless Man) --- unix-ffi/re/re.py | 63 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 50 insertions(+), 13 deletions(-) diff --git a/unix-ffi/re/re.py b/unix-ffi/re/re.py index bf108686a..0391d1b64 100644 --- a/unix-ffi/re/re.py +++ b/unix-ffi/re/re.py @@ -22,6 +22,12 @@ # PCRE2_SIZE *pcre2_get_ovector_pointer(pcre2_match_data *match_data); pcre2_get_ovector_pointer = pcre2.func("p", "pcre2_get_ovector_pointer_8", "p") +# void pcre2_code_free(pcre2_code *code); +pcre2_code_free = pcre2.func("v", "pcre2_code_free_8", "p") + +# void pcre2_match_data_free(pcre2_match_data *match_data); +pcre2_match_data_free = pcre2.func("v", "pcre2_match_data_free_8", "p") + # pcre2_match_data *pcre2_match_data_create_from_pattern(const pcre2_code *code, # pcre2_general_context *gcontext); pcre2_match_data_create_from_pattern = pcre2.func( @@ -86,20 +92,32 @@ class PCREPattern: def __init__(self, compiled_ptn): self.obj = compiled_ptn + def _free(self): + # MicroPython does not run __del__ on instances of Python classes, so + # the compiled pattern cannot be released by the garbage collector and + # has to be freed explicitly. + if self.obj is not None: + pcre2_code_free(self.obj) + self.obj = None + def search(self, s, pos=0, endpos=-1, _flags=0): assert endpos == -1, "pos: %d, endpos: %d" % (pos, endpos) buf = array.array("i", [0]) pcre2_pattern_info(self.obj, PCRE2_INFO_CAPTURECOUNT, buf) cap_count = buf[0] match_data = pcre2_match_data_create_from_pattern(self.obj, None) - num = pcre2_match(self.obj, s, len(s), pos, _flags, match_data, None) - if num == -1: - # No match - return None - ov_ptr = pcre2_get_ovector_pointer(match_data) - # pcre2_get_ovector_pointer return PCRE2_SIZE - ov_buf = uctypes.bytearray_at(ov_ptr, PCRE2_SIZE_SIZE * (cap_count + 1) * 2) - ov = array.array(PCRE2_SIZE_TYPE, ov_buf) + try: + num = pcre2_match(self.obj, s, len(s), pos, _flags, match_data, None) + if num == -1: + # No match + return None + ov_ptr = pcre2_get_ovector_pointer(match_data) + # pcre2_get_ovector_pointer return PCRE2_SIZE. The offsets are + # copied out here, because the match data is freed below. + ov_buf = uctypes.bytearray_at(ov_ptr, PCRE2_SIZE_SIZE * (cap_count + 1) * 2) + ov = array.array(PCRE2_SIZE_TYPE, ov_buf) + finally: + pcre2_match_data_free(match_data) # We don't care how many matching subexpressions we got, we # care only about total # of capturing ones (including empty) return PCREMatch(s, cap_count + 1, ov) @@ -174,29 +192,48 @@ def compile(pattern, flags=0): return PCREPattern(regex) +# The functions below compile a pattern that is not visible to the caller, so +# they must free it again. The match objects they return do not refer to it. + + def search(pattern, string, flags=0): r = compile(pattern, flags) - return r.search(string) + try: + return r.search(string) + finally: + r._free() def match(pattern, string, flags=0): r = compile(pattern, flags | PCRE2_ANCHORED) - return r.search(string) + try: + return r.search(string) + finally: + r._free() def sub(pattern, repl, s, count=0, flags=0): r = compile(pattern, flags) - return r.sub(repl, s, count) + try: + return r.sub(repl, s, count) + finally: + r._free() def split(pattern, s, maxsplit=0, flags=0): r = compile(pattern, flags) - return r.split(s, maxsplit) + try: + return r.split(s, maxsplit) + finally: + r._free() def findall(pattern, s, flags=0): r = compile(pattern, flags) - return r.findall(s) + try: + return r.findall(s) + finally: + r._free() def escape(s): From 98705727a7a7d8d9223d324b52bcb924cd9231ff Mon Sep 17 00:00:00 2001 From: "Kirill Lukonin (Evil Wireless Man)" Date: Mon, 31 Aug 2026 22:49:25 +0300 Subject: [PATCH 2/6] unix-ffi/re: Fix the output buffers passed to pcre2_compile(). The error code and the error offset were passed as bytes(4). Such objects are immutable, and the error offset is a PCRE2_SIZE, which is 8 bytes on a 64-bit target, so a failing compile wrote 4 bytes past the end of the buffer. Use writable arrays of the right size instead, and report the values in the assertion. Signed-off-by: Kirill Lukonin (Evil Wireless Man) --- unix-ffi/re/re.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/unix-ffi/re/re.py b/unix-ffi/re/re.py index 0391d1b64..95f389540 100644 --- a/unix-ffi/re/re.py +++ b/unix-ffi/re/re.py @@ -185,10 +185,12 @@ def findall(self, s): def compile(pattern, flags=0): - errcode = bytes(4) - erroffset = bytes(4) + # These are output arguments and must be writable and of the size that + # pcre2_compile() writes: int for the error code, PCRE2_SIZE for the offset. + errcode = array.array("i", [0]) + erroffset = array.array(PCRE2_SIZE_TYPE, [0]) regex = pcre2_compile(pattern, PCRE2_ZERO_TERMINATED, flags, errcode, erroffset, None) - assert regex + assert regex, "error %d compiling regex at offset %d" % (errcode[0], erroffset[0]) return PCREPattern(regex) From df9ba1367be59bba1c949599a007d499f74325ed Mon Sep 17 00:00:00 2001 From: "Kirill Lukonin (Evil Wireless Man)" Date: Mon, 31 Aug 2026 22:49:25 +0300 Subject: [PATCH 3/6] unix-ffi/re: Add a test for the PCRE2 memory leaks. The test measures the resident set size around a few thousand calls and fails if it keeps growing. It covers every entry point that makes PCRE2 allocate: matching with a compiled pattern, the module level functions, and compiling itself, including a pattern that fails to compile. Without the preceding fixes it reports between 4.6 and 17.8 kilobytes of growth per call, depending on the entry point. Signed-off-by: Kirill Lukonin (Evil Wireless Man) --- tools/ci.sh | 1 + unix-ffi/re/test_re_leak.py | 99 +++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 unix-ffi/re/test_re_leak.py diff --git a/tools/ci.sh b/tools/ci.sh index 2fc6fe90c..905d359b5 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -78,6 +78,7 @@ function ci_package_tests_run { unix-ffi/gettext/test_gettext.py \ unix-ffi/pwd/test_getpwnam.py \ unix-ffi/re/test_re.py \ + unix-ffi/re/test_re_leak.py \ unix-ffi/sqlite3/test_sqlite3.py \ unix-ffi/sqlite3/test_sqlite3_2.py \ unix-ffi/sqlite3/test_sqlite3_3.py \ diff --git a/unix-ffi/re/test_re_leak.py b/unix-ffi/re/test_re_leak.py new file mode 100644 index 000000000..d87b8cbd0 --- /dev/null +++ b/unix-ffi/re/test_re_leak.py @@ -0,0 +1,99 @@ +# Regression test for the memory that PCRE2 allocates behind this module: the +# match data of every match, and every pattern compiled by the module level +# functions, have to be freed again. Otherwise each call leaks a few +# kilobytes. +# +# A pattern returned by re.compile() and kept by the caller is not covered +# here. MicroPython does not run __del__ on instances of Python classes, so +# such a pattern can only be released explicitly. + +import gc +import re + + +def rss(): + # Resident set size in KiB, from the second field of /proc/self/statm. + with open("/proc/self/statm") as f: + return int(f.read().split()[1]) * 4096 // 1024 + + +try: + rss() +except OSError: + # No /proc, so memory use cannot be measured here. + raise SystemExit + + +N = 4000 +LIMIT = 256 # KiB + + +def check_no_leak(name, fn): + # Run the calls once to let the MicroPython heap grow to its steady state, + # so that only the memory allocated by PCRE2 is measured afterwards. + for _ in range(N): + fn() + gc.collect() + before = rss() + for _ in range(N): + fn() + gc.collect() + growth = rss() - before + assert growth < LIMIT, "%s leaks %d KiB per %d calls (%d bytes per call)" % ( + name, + growth, + N, + growth * 1024 // N, + ) + + +text = "He was carefully disguised but captured quickly by police." +p = re.compile("a(b)c") + +# Matching with a compiled pattern. +check_no_leak("Pattern.search() with a match", lambda: p.search("xxabcxx")) +check_no_leak("Pattern.search() without a match", lambda: p.search("xxxxxxx")) +check_no_leak("Pattern.match()", lambda: p.match("abcxx")) +check_no_leak("Pattern.sub()", lambda: p.sub("z", "xxabcxx")) +check_no_leak("Pattern.split()", lambda: p.split("xxabcxx")) +check_no_leak("Pattern.findall()", lambda: p.findall("xxabcxx abc")) + +# The module level functions, which compile a pattern of their own. +check_no_leak("re.search()", lambda: re.search("a(b)c", "xxabcxx")) +check_no_leak("re.match()", lambda: re.match("a(b)c", "abcxx")) +check_no_leak("re.sub()", lambda: re.sub("a", "z", "caaab")) +check_no_leak("re.split()", lambda: re.split(r"\W+", "Words, words, words.")) +check_no_leak("re.findall()", lambda: re.findall(r"(\w+)ly", text)) + + +# Compiling, including the path that does not produce a usable pattern. +def compile_and_free(): + re.compile("a(b)c")._free() + + +def free_twice(): + r = re.compile("a(b)c") + r._free() + r._free() + + +def failed_compile(): + try: + re.compile("(") + except AssertionError: + pass + + +check_no_leak("re.compile() and _free()", compile_and_free) +check_no_leak("_free() called twice", free_twice) +check_no_leak("re.compile() of a bad pattern", failed_compile) + + +# A pattern with several groups needs a larger match data block. +def many_groups(): + r = re.compile(r"(\w+)(\s+)(\w+)(\s+)(\w+)") + assert r.search("one two three").groups() == ("one", " ", "two", " ", "three") + r._free() + + +check_no_leak("pattern with several groups", many_groups) From 77b3e8e5a082801edf17d1c362440c7bd7e86f8d Mon Sep 17 00:00:00 2001 From: "Kirill Lukonin (Evil Wireless Man)" Date: Mon, 31 Aug 2026 23:25:09 +0300 Subject: [PATCH 4/6] unix-ffi/re: Cache the compiled patterns. Every call to re.search(), and to the functions next to it, compiled the pattern it was given. Keep the compiled patterns in a small cache instead, the way CPython does, so that using the same pattern again does not compile it a second time. compile() returns the cached pattern as well, so re.compile(p) is re.compile(p), as it is in CPython. Matching against a repeated pattern gets about twice as fast, compiling one about eight times. Because MicroPython cannot release a compiled pattern by itself, the cache also decides what is kept: a cached pattern stays for the lifetime of the program, and a pattern that this module compiled for its own use is freed again afterwards. The cache owns what it holds and never evicts it. A pattern that is still in use, by the caller or by a call further up the stack, must not be freed underneath it, which a replacement callback passed to sub() can otherwise trigger. The cache is bounded instead: once it is full, further patterns are compiled and, where this module owns them, freed again after use. Signed-off-by: Kirill Lukonin (Evil Wireless Man) --- unix-ffi/re/re.py | 66 +++++++++++++++++++++++++++++-------- unix-ffi/re/test_re_leak.py | 50 ++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 13 deletions(-) diff --git a/unix-ffi/re/re.py b/unix-ffi/re/re.py index 95f389540..0c4a0ad34 100644 --- a/unix-ffi/re/re.py +++ b/unix-ffi/re/re.py @@ -91,12 +91,18 @@ def span(self, n=0): class PCREPattern: def __init__(self, compiled_ptn): self.obj = compiled_ptn + self.key = None # set while this pattern is held by the cache def _free(self): # MicroPython does not run __del__ on instances of Python classes, so # the compiled pattern cannot be released by the garbage collector and # has to be freed explicitly. if self.obj is not None: + if self.key is not None: + # Drop the pattern from the cache first, so that nothing hands + # out a pointer that is about to become invalid. + del _cache[self.key] + self.key = None pcre2_code_free(self.obj) self.obj = None @@ -184,7 +190,7 @@ def findall(self, s): start = end -def compile(pattern, flags=0): +def _compile(pattern, flags): # These are output arguments and must be writable and of the size that # pcre2_compile() writes: int for the error code, PCRE2_SIZE for the offset. errcode = array.array("i", [0]) @@ -194,48 +200,82 @@ def compile(pattern, flags=0): return PCREPattern(regex) -# The functions below compile a pattern that is not visible to the caller, so -# they must free it again. The match objects they return do not refer to it. +# Compiled patterns are cached, the way CPython does it, so that using the same +# pattern again does not compile it a second time. compile() returns the +# cached pattern, so re.compile(p) is re.compile(p), as in CPython. +# +# The cache owns the patterns it holds and never evicts them. A pattern that +# is still being used, either by the caller or by a call further up the stack, +# must not be freed underneath it; a replacement callback passed to sub() can +# otherwise trigger exactly that. The cache is bounded instead: once it is +# full, further patterns are compiled and, where this module owns them, freed +# again after use. +_MAXCACHE = 32 +_cache = {} + + +def _cached(pattern, flags): + # Return the compiled pattern, and whether the caller has to free it. + key = (pattern, flags) + r = _cache.get(key) + if r is not None: + return r, False + r = _compile(pattern, flags) + if len(_cache) < _MAXCACHE: + _cache[key] = r + r.key = key + return r, False + return r, True + + +def compile(pattern, flags=0): + # The pattern belongs to the caller, so it is never freed here. + return _cached(pattern, flags)[0] def search(pattern, string, flags=0): - r = compile(pattern, flags) + r, owned = _cached(pattern, flags) try: return r.search(string) finally: - r._free() + if owned: + r._free() def match(pattern, string, flags=0): - r = compile(pattern, flags | PCRE2_ANCHORED) + r, owned = _cached(pattern, flags | PCRE2_ANCHORED) try: return r.search(string) finally: - r._free() + if owned: + r._free() def sub(pattern, repl, s, count=0, flags=0): - r = compile(pattern, flags) + r, owned = _cached(pattern, flags) try: return r.sub(repl, s, count) finally: - r._free() + if owned: + r._free() def split(pattern, s, maxsplit=0, flags=0): - r = compile(pattern, flags) + r, owned = _cached(pattern, flags) try: return r.split(s, maxsplit) finally: - r._free() + if owned: + r._free() def findall(pattern, s, flags=0): - r = compile(pattern, flags) + r, owned = _cached(pattern, flags) try: return r.findall(s) finally: - r._free() + if owned: + r._free() def escape(s): diff --git a/unix-ffi/re/test_re_leak.py b/unix-ffi/re/test_re_leak.py index d87b8cbd0..a78d83c47 100644 --- a/unix-ffi/re/test_re_leak.py +++ b/unix-ffi/re/test_re_leak.py @@ -6,6 +6,10 @@ # A pattern returned by re.compile() and kept by the caller is not covered # here. MicroPython does not run __del__ on instances of Python classes, so # such a pattern can only be released explicitly. +# +# The bounded cache that the module level functions keep is covered: it must +# not grow past its limit, and the patterns that do not fit into it must be +# freed again. import gc import re @@ -21,6 +25,7 @@ def rss(): rss() except OSError: # No /proc, so memory use cannot be measured here. + print("SKIP") raise SystemExit @@ -97,3 +102,48 @@ def many_groups(): check_no_leak("pattern with several groups", many_groups) + + +# compile() returns the cached pattern, the way CPython does, so compiling the +# same pattern again does not allocate. +assert re.compile("a(b)c") is re.compile("a(b)c") +check_no_leak("re.compile() with the same pattern", lambda: re.compile("a(b)c")) + +# _free() drops the pattern from the cache, so that nothing afterwards hands +# out a pointer to memory that has been released. +r = re.compile("zz(y)") +r._free() +assert re.search("zz(y)", "xxzzyxx").group(0) == "zzy" + + +# The module level functions cache the patterns they compile. That cache must +# stay bounded, and a pattern that does not fit into it has to be freed again. +counter = [0] + + +def distinct_patterns(): + counter[0] += 1 + re.search("a%dc" % counter[0], "xxabcxx") + + +# Push far more distinct patterns through the cache than it can hold: it has +# to stop growing. +for _ in range(re._MAXCACHE * 4): + distinct_patterns() +assert len(re._cache) <= re._MAXCACHE, len(re._cache) + +check_no_leak("re.search() with distinct patterns", distinct_patterns) +assert len(re._cache) <= re._MAXCACHE, len(re._cache) + + +# A replacement callback runs while sub() is still using its own pattern, and +# may push further patterns through the cache. The pattern that is in use must +# survive that. +def reentrant_repl(m): + counter[0] += 1 + re.search("z%dz" % counter[0], "nothing here") + return "z" + + +check_no_leak("re.sub() with a reentrant callback", lambda: re.sub("a", reentrant_repl, "caaab")) +assert len(re._cache) <= re._MAXCACHE, len(re._cache) From 08d5c9078752f7b33418dd8f162c2a505b69df74 Mon Sep 17 00:00:00 2001 From: "Kirill Lukonin (Evil Wireless Man)" Date: Sun, 13 Sep 2026 14:36:23 +0300 Subject: [PATCH 5/6] unix-ffi/re: Pass plain bytes objects to pcre2_compile(). The error code and the error offset were turned into array.array() objects on the assumption that a bytes object cannot be written to. ffi passes the buffer of either one straight to the C function, so the array buys nothing and only makes the constructor larger at every call. Use bytes again, and read the values back out of them with int.from_bytes(). What was wrong is the size of the error offset. It is a PCRE2_SIZE, which is 8 bytes on a 64-bit target, so a failing compile wrote past the end of the 4 bytes it was given. Size it from PCRE2_SIZE_SIZE, which is already derived from uctypes.ULONG for this purpose. Signed-off-by: Kirill Lukonin (Evil Wireless Man) --- unix-ffi/re/re.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/unix-ffi/re/re.py b/unix-ffi/re/re.py index 0c4a0ad34..5f67ac16b 100644 --- a/unix-ffi/re/re.py +++ b/unix-ffi/re/re.py @@ -191,12 +191,15 @@ def findall(self, s): def _compile(pattern, flags): - # These are output arguments and must be writable and of the size that - # pcre2_compile() writes: int for the error code, PCRE2_SIZE for the offset. - errcode = array.array("i", [0]) - erroffset = array.array(PCRE2_SIZE_TYPE, [0]) + # These are output arguments and must be of the size that pcre2_compile() + # writes: int for the error code, PCRE2_SIZE for the offset. + errcode = bytes(4) + erroffset = bytes(PCRE2_SIZE_SIZE) regex = pcre2_compile(pattern, PCRE2_ZERO_TERMINATED, flags, errcode, erroffset, None) - assert regex, "error %d compiling regex at offset %d" % (errcode[0], erroffset[0]) + assert regex, "compile error %d at %d" % ( + int.from_bytes(errcode, sys.byteorder), + int.from_bytes(erroffset, sys.byteorder), + ) return PCREPattern(regex) From 7b52877ee3d7640da7f4f55b3e905130b7b0dbf2 Mon Sep 17 00:00:00 2001 From: "Kirill Lukonin (Evil Wireless Man)" Date: Sun, 13 Sep 2026 14:36:30 +0300 Subject: [PATCH 6/6] unix-ffi/re: Free the compiled patterns with weakref.finalize(). A compiled pattern was released by an explicit _free(), so the module had to keep track of who owned each one: the cache marked the patterns it held, the module level functions freed the ones that did not fit into it, and compile() freed nothing at all because that pattern belongs to the caller. The cache could not evict either, because a pattern that is still in use, by the caller or by a replacement callback further up the stack, must not be freed underneath it. weakref.finalize() does all of this instead: a pattern is freed once nothing refers to it any more. The ownership tracking goes away, the module level functions go back to the two lines they were before, and the cache becomes a plain optimisation that evicts the way CPython does. A pattern returned by re.compile() no longer has to be kept alive by the caller to avoid leaking it. This needs a build with MICROPY_PY_WEAKREF enabled, which the unix port does not do by default yet. Signed-off-by: Kirill Lukonin (Evil Wireless Man) --- unix-ffi/re/manifest.py | 2 +- unix-ffi/re/re.py | 92 +++++++++++-------------------------- unix-ffi/re/test_re_leak.py | 66 +++++++++++--------------- 3 files changed, 55 insertions(+), 105 deletions(-) diff --git a/unix-ffi/re/manifest.py b/unix-ffi/re/manifest.py index d7f6ce98e..7268f7b12 100644 --- a/unix-ffi/re/manifest.py +++ b/unix-ffi/re/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.2.6") +metadata(version="0.3.0") # Originally written by Paul Sokolovsky. diff --git a/unix-ffi/re/re.py b/unix-ffi/re/re.py index 5f67ac16b..958f81be8 100644 --- a/unix-ffi/re/re.py +++ b/unix-ffi/re/re.py @@ -2,6 +2,7 @@ import ffilib import array import uctypes +import weakref pcre2 = ffilib.open("libpcre2-8") @@ -91,20 +92,10 @@ def span(self, n=0): class PCREPattern: def __init__(self, compiled_ptn): self.obj = compiled_ptn - self.key = None # set while this pattern is held by the cache - - def _free(self): - # MicroPython does not run __del__ on instances of Python classes, so - # the compiled pattern cannot be released by the garbage collector and - # has to be freed explicitly. - if self.obj is not None: - if self.key is not None: - # Drop the pattern from the cache first, so that nothing hands - # out a pointer that is about to become invalid. - del _cache[self.key] - self.key = None - pcre2_code_free(self.obj) - self.obj = None + # The compiled pattern lives in memory that PCRE2 owns and that the + # garbage collector knows nothing about, so release it once this object + # is collected. + weakref.finalize(self, pcre2_code_free, compiled_ptn) def search(self, s, pos=0, endpos=-1, _flags=0): assert endpos == -1, "pos: %d, endpos: %d" % (pos, endpos) @@ -205,80 +196,49 @@ def _compile(pattern, flags): # Compiled patterns are cached, the way CPython does it, so that using the same # pattern again does not compile it a second time. compile() returns the -# cached pattern, so re.compile(p) is re.compile(p), as in CPython. -# -# The cache owns the patterns it holds and never evicts them. A pattern that -# is still being used, either by the caller or by a call further up the stack, -# must not be freed underneath it; a replacement callback passed to sub() can -# otherwise trigger exactly that. The cache is bounded instead: once it is -# full, further patterns are compiled and, where this module owns them, freed -# again after use. +# cached pattern, so re.compile(p) is re.compile(p), as in CPython. A pattern +# that is dropped from the cache is freed by the garbage collector once nothing +# refers to it any more. _MAXCACHE = 32 _cache = {} -def _cached(pattern, flags): - # Return the compiled pattern, and whether the caller has to free it. +def compile(pattern, flags=0): key = (pattern, flags) r = _cache.get(key) - if r is not None: - return r, False - r = _compile(pattern, flags) - if len(_cache) < _MAXCACHE: + if r is None: + r = _compile(pattern, flags) + if len(_cache) >= _MAXCACHE: + # Drop the whole cache, the way CPython does, instead of keeping + # track of which entry was used last. + _cache.clear() _cache[key] = r - r.key = key - return r, False - return r, True - - -def compile(pattern, flags=0): - # The pattern belongs to the caller, so it is never freed here. - return _cached(pattern, flags)[0] + return r def search(pattern, string, flags=0): - r, owned = _cached(pattern, flags) - try: - return r.search(string) - finally: - if owned: - r._free() + r = compile(pattern, flags) + return r.search(string) def match(pattern, string, flags=0): - r, owned = _cached(pattern, flags | PCRE2_ANCHORED) - try: - return r.search(string) - finally: - if owned: - r._free() + r = compile(pattern, flags | PCRE2_ANCHORED) + return r.search(string) def sub(pattern, repl, s, count=0, flags=0): - r, owned = _cached(pattern, flags) - try: - return r.sub(repl, s, count) - finally: - if owned: - r._free() + r = compile(pattern, flags) + return r.sub(repl, s, count) def split(pattern, s, maxsplit=0, flags=0): - r, owned = _cached(pattern, flags) - try: - return r.split(s, maxsplit) - finally: - if owned: - r._free() + r = compile(pattern, flags) + return r.split(s, maxsplit) def findall(pattern, s, flags=0): - r, owned = _cached(pattern, flags) - try: - return r.findall(s) - finally: - if owned: - r._free() + r = compile(pattern, flags) + return r.findall(s) def escape(s): diff --git a/unix-ffi/re/test_re_leak.py b/unix-ffi/re/test_re_leak.py index a78d83c47..582a41421 100644 --- a/unix-ffi/re/test_re_leak.py +++ b/unix-ffi/re/test_re_leak.py @@ -1,15 +1,14 @@ # Regression test for the memory that PCRE2 allocates behind this module: the -# match data of every match, and every pattern compiled by the module level -# functions, have to be freed again. Otherwise each call leaks a few -# kilobytes. +# match data of every match, and every compiled pattern, have to be freed +# again. Otherwise each call leaks a few kilobytes. # -# A pattern returned by re.compile() and kept by the caller is not covered -# here. MicroPython does not run __del__ on instances of Python classes, so -# such a pattern can only be released explicitly. +# The match data is freed by the call that created it. A compiled pattern is +# freed by the garbage collector, through the weakref.finalize() that the +# pattern registers for itself, so a pattern that is no longer reachable does +# not have to be released by hand. # -# The bounded cache that the module level functions keep is covered: it must -# not grow past its limit, and the patterns that do not fit into it must be -# freed again. +# The bounded cache that compile() keeps is covered as well: it must not grow +# past its limit, and the patterns that it drops have to be freed. import gc import re @@ -71,17 +70,16 @@ def check_no_leak(name, fn): check_no_leak("re.findall()", lambda: re.findall(r"(\w+)ly", text)) -# Compiling, including the path that does not produce a usable pattern. -def compile_and_free(): - re.compile("a(b)c")._free() +# A pattern with several groups needs a larger match data block. +def many_groups(): + r = re.compile(r"(\w+)(\s+)(\w+)(\s+)(\w+)") + assert r.search("one two three").groups() == ("one", " ", "two", " ", "three") -def free_twice(): - r = re.compile("a(b)c") - r._free() - r._free() +check_no_leak("pattern with several groups", many_groups) +# The path that does not produce a usable pattern must not leak either. def failed_compile(): try: re.compile("(") @@ -89,35 +87,17 @@ def failed_compile(): pass -check_no_leak("re.compile() and _free()", compile_and_free) -check_no_leak("_free() called twice", free_twice) check_no_leak("re.compile() of a bad pattern", failed_compile) -# A pattern with several groups needs a larger match data block. -def many_groups(): - r = re.compile(r"(\w+)(\s+)(\w+)(\s+)(\w+)") - assert r.search("one two three").groups() == ("one", " ", "two", " ", "three") - r._free() - - -check_no_leak("pattern with several groups", many_groups) - - # compile() returns the cached pattern, the way CPython does, so compiling the # same pattern again does not allocate. assert re.compile("a(b)c") is re.compile("a(b)c") check_no_leak("re.compile() with the same pattern", lambda: re.compile("a(b)c")) -# _free() drops the pattern from the cache, so that nothing afterwards hands -# out a pointer to memory that has been released. -r = re.compile("zz(y)") -r._free() -assert re.search("zz(y)", "xxzzyxx").group(0) == "zzy" - -# The module level functions cache the patterns they compile. That cache must -# stay bounded, and a pattern that does not fit into it has to be freed again. +# The cache has to stay bounded, and a pattern that it drops has to be freed by +# the garbage collector. counter = [0] @@ -136,9 +116,19 @@ def distinct_patterns(): assert len(re._cache) <= re._MAXCACHE, len(re._cache) + +def compile_distinct(): + counter[0] += 1 + re.compile("b%dc" % counter[0]) + + +check_no_leak("re.compile() with distinct patterns", compile_distinct) +assert len(re._cache) <= re._MAXCACHE, len(re._cache) + + # A replacement callback runs while sub() is still using its own pattern, and -# may push further patterns through the cache. The pattern that is in use must -# survive that. +# may push further patterns through the cache. The reference that sub() holds +# has to keep that pattern alive across the callback. def reentrant_repl(m): counter[0] += 1 re.search("z%dz" % counter[0], "nothing here")