From c5d4e0c78e2cf8e91e833bcc105ad36ff349adbc Mon Sep 17 00:00:00 2001 From: Jax Calder Date: Wed, 28 Oct 2026 14:00:41 +0000 Subject: [PATCH 1/4] deflate: a compression method of the DMFC files, by the dmcomp library Dmod_Compression_* has a second method, "deflate" (raw DEFLATE, RFC 1951), by dmcomp (choco-technologies/dmcomp, vendored in lib/third-party/dmcomp like FastLZ is). The modules of the dmod-boot firmware are 172 kB with it, and 215 kB with FastLZ. * The decoder is 1.7 kB of code. Its tables (~3 kB) come from the heap (dmcomp_inflate_ws), not from the stack: a module is loaded from whatever thread asks for it, some of them have 4 kB of the stack. * The encoder (Dmod_Compression_Pack) allocates ~330 kB with malloc and is for the host tools. DMOD_USE_DMCOMP_ENCODER is ON when DMOD_USE_STDLIB is, so a firmware (dmod-boot has no stdlib) gets the decoder only, with no change in its configuration: +2.6 kB. * DMOD_USE_DMCOMP (ON) switches the method. * fastlz stays the first method (the default of todmfc and of DMOD_COMPRESSION_METHOD): a loader built before this cannot read deflate. todmfc's default level depends on the method: 2 for fastlz, 9 for deflate. Tests: Dmod_Compression_* with deflate on every level, against fastlz, damaged and cut data, no memory for the decoder (the tables are given back); DMFC -> DMF with both methods. The SDK installs the library with its license. Checked in the build of the Docker image (two passes, examples, tools install), a real module built as DMOD_MODE=DMOD_MODULE with DMOD_COMPRESSION_METHOD=deflate, and the dmod-boot firmware. Co-Authored-By: Claude Sonnet 5.5 --- CMakeLists.txt | 3 + cmake/runtime/CMakeLists.txt | 4 + dmod-cfg.cmake | 13 + dmod-cfg.mk | 4 + dmod-config.h.in | 2 + dmod-defaults.cmake | 14 + dmod-defaults.mk | 14 + docs/cmake-functions.md | 2 +- docs/development-package.md | 2 +- docs/tools-installation.md | 9 +- examples/system/dif_test/Makefile | 3 + examples/system/dif_test/dmod-cfg.mk | 2 + examples/system/dmod_loader/Makefile | 3 + examples/system/dmod_loader/dmod-cfg.mk | 2 + lib/third-party/CMakeLists.txt | 3 +- lib/third-party/Makefile | 2 +- lib/third-party/dmcomp/CMakeLists.txt | 26 + lib/third-party/dmcomp/LICENSE.MIT | 21 + lib/third-party/dmcomp/Makefile | 35 + lib/third-party/dmcomp/README.md | 11 + lib/third-party/dmcomp/dmcomp.h | 118 +++ lib/third-party/dmcomp/dmcomp_deflate.c | 781 ++++++++++++++++++ lib/third-party/dmcomp/dmcomp_inflate.c | 571 +++++++++++++ lib/third-party/dmcomp/dmcomp_strerror.c | 19 + lib/third-party/dmcomp/dmcomp_tables.h | 52 ++ paths.cmake | 3 + scripts/install-sdk.cmake | 6 + scripts/system-tool.mk | 3 + src/system/CMakeLists.txt | 1 + src/system/Makefile | 4 +- src/system/if/dmod_if_comp.c | 57 +- templates/system/dmod-cfg.cmake | 13 + templates/system/dmod-cfg.mk | 4 + tests/CMakeLists.txt | 2 +- tests/system/if/CMakeLists.txt | 1 + tests/system/if/tests_dmod_compression.cpp | 186 +++++ tests/system/public/CMakeLists.txt | 1 + .../system/public/tests_dmod_dmfc_deflate.cpp | 76 ++ tools/system/todmd/dmod-cfg.mk | 2 + tools/system/todmfc/README.md | 11 +- tools/system/todmfc/dmod-cfg.mk | 2 + tools/system/todmfc/main.c | 7 +- tools/system/todmm/dmod-cfg.mk | 2 + 43 files changed, 2081 insertions(+), 16 deletions(-) create mode 100644 lib/third-party/dmcomp/CMakeLists.txt create mode 100644 lib/third-party/dmcomp/LICENSE.MIT create mode 100644 lib/third-party/dmcomp/Makefile create mode 100644 lib/third-party/dmcomp/README.md create mode 100644 lib/third-party/dmcomp/dmcomp.h create mode 100644 lib/third-party/dmcomp/dmcomp_deflate.c create mode 100644 lib/third-party/dmcomp/dmcomp_inflate.c create mode 100644 lib/third-party/dmcomp/dmcomp_strerror.c create mode 100644 lib/third-party/dmcomp/dmcomp_tables.h create mode 100644 tests/system/if/tests_dmod_compression.cpp create mode 100644 tests/system/public/tests_dmod_dmfc_deflate.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index e7ca3aa4..2368ceb6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -224,6 +224,9 @@ target_link_libraries(dmod INTERFACE dmod_inc dmod_common) if (DMOD_USE_FASTLZ) target_link_libraries(dmod INTERFACE dmod_fastlz) endif() +if (DMOD_USE_DMCOMP) + target_link_libraries(dmod INTERFACE dmod_dmcomp) +endif() if (DMOD_SYSTEM) target_link_libraries(dmod INTERFACE dmod_system) elseif (DMOD_MODULE) diff --git a/cmake/runtime/CMakeLists.txt b/cmake/runtime/CMakeLists.txt index 8b4672e5..576c3cef 100644 --- a/cmake/runtime/CMakeLists.txt +++ b/cmake/runtime/CMakeLists.txt @@ -27,3 +27,7 @@ if(DMOD_USE_FASTLZ) add_subdirectory("${DMOD_DIR}/lib/third-party/FastLZ" fastlz) target_link_libraries(dmod INTERFACE dmod_fastlz) endif() +if(DMOD_USE_DMCOMP) + add_subdirectory("${DMOD_DIR}/lib/third-party/dmcomp" dmcomp) + target_link_libraries(dmod INTERFACE dmod_dmcomp) +endif() diff --git a/dmod-cfg.cmake b/dmod-cfg.cmake index f2056759..2e854c7e 100644 --- a/dmod-cfg.cmake +++ b/dmod-cfg.cmake @@ -37,6 +37,19 @@ set(DMOD_USE_TIME ON CACHE BOOL "Enable to use time.h for system time functions # Use FastLZ compression library set(DMOD_USE_FASTLZ ON CACHE BOOL "Enable to use the FastLZ compression library") +# Use the dmcomp library: the "deflate" compression method (the decoder) +set(DMOD_USE_DMCOMP ON CACHE BOOL "Enable to use the dmcomp library (the deflate compression method)") + +# Build the deflate encoder (Dmod_Compression_Pack) of the dmcomp library: for the host tools. It allocates its work memory +# (~330 kB) with malloc, so by default it is built when DMOD_USE_STDLIB is (a firmware has no malloc of the standard +# library and nothing to compress for - it gets the decoder only) +if(DMOD_USE_STDLIB) + set(DMOD_USE_DMCOMP_ENCODER_DEFAULT ON) +else() + set(DMOD_USE_DMCOMP_ENCODER_DEFAULT OFF) +endif() +set(DMOD_USE_DMCOMP_ENCODER ${DMOD_USE_DMCOMP_ENCODER_DEFAULT} CACHE BOOL "Enable to build the deflate encoder of the dmcomp library (it needs malloc and ~330 kB of the heap)") + # Maximum number of modules set(DMOD_MAX_MODULES 30 CACHE STRING "Maximum number of modules") diff --git a/dmod-cfg.mk b/dmod-cfg.mk index 4d4128b9..3925cab9 100644 --- a/dmod-cfg.mk +++ b/dmod-cfg.mk @@ -41,6 +41,10 @@ DMOD_USE_TIME=ON # Use FastLZ compression library DMOD_USE_FASTLZ=ON +# Use the dmcomp library (the deflate compression method) and its encoder (for the host tools) +DMOD_USE_DMCOMP=ON +DMOD_USE_DMCOMP_ENCODER=ON + # Maximum number of modules DMOD_MAX_MODULES=30 diff --git a/dmod-config.h.in b/dmod-config.h.in index fb443204..c4a5bdb5 100644 --- a/dmod-config.h.in +++ b/dmod-config.h.in @@ -22,6 +22,8 @@ #define DMOD_USE_TERMIOS @DMOD_USE_TERMIOS@ #define DMOD_USE_TIME @DMOD_USE_TIME@ #define DMOD_USE_FASTLZ @DMOD_USE_FASTLZ@ +#define DMOD_USE_DMCOMP @DMOD_USE_DMCOMP@ +#define DMOD_USE_DMCOMP_ENCODER @DMOD_USE_DMCOMP_ENCODER@ #define DMOD_IMPLEMENT_PRINTF @DMOD_IMPLEMENT_PRINTF@ #define DMOD_IMPLEMENT_SCANF @DMOD_IMPLEMENT_SCANF@ #define DMOD_MODE "@DMOD_MODE@" diff --git a/dmod-defaults.cmake b/dmod-defaults.cmake index 3adb3aec..f1c5ae7a 100644 --- a/dmod-defaults.cmake +++ b/dmod-defaults.cmake @@ -68,6 +68,20 @@ if (NOT DEFINED DMOD_USE_FASTLZ) set(DMOD_USE_FASTLZ ON) endif() +if (NOT DEFINED DMOD_USE_DMCOMP) + set(DMOD_USE_DMCOMP ON) +endif() + +if (NOT DEFINED DMOD_USE_DMCOMP_ENCODER) + # The encoder allocates its work memory (~330 kB) with malloc: it is for the host. Without the standard library + # (a firmware) there is no malloc, and nothing to compress for - only the decoder is built. + if (DMOD_USE_STDLIB) + set(DMOD_USE_DMCOMP_ENCODER ON) + else() + set(DMOD_USE_DMCOMP_ENCODER OFF) + endif() +endif() + if (NOT DEFINED DMOD_MAX_MODULES) set(DMOD_MAX_MODULES 30) endif() diff --git a/dmod-defaults.mk b/dmod-defaults.mk index 88a1b0d7..4633f1ce 100644 --- a/dmod-defaults.mk +++ b/dmod-defaults.mk @@ -73,6 +73,20 @@ ifndef DMOD_USE_FASTLZ DMOD_USE_FASTLZ = ON endif +ifndef DMOD_USE_DMCOMP + DMOD_USE_DMCOMP = ON +endif + +ifndef DMOD_USE_DMCOMP_ENCODER + # The encoder allocates its work memory (~330 kB) with malloc: it is for the host. Without the standard library + # (a firmware) there is no malloc, and nothing to compress for - only the decoder is built. + ifeq ($(DMOD_USE_STDLIB),ON) + DMOD_USE_DMCOMP_ENCODER = ON + else + DMOD_USE_DMCOMP_ENCODER = OFF + endif +endif + ifndef DMOD_MAX_MODULES DMOD_MAX_MODULES = 30 endif diff --git a/docs/cmake-functions.md b/docs/cmake-functions.md index 8078d7aa..2231fdb7 100644 --- a/docs/cmake-functions.md +++ b/docs/cmake-functions.md @@ -334,7 +334,7 @@ The following variables should be set before calling `dmod_add_executable` or `d - `DMOD_MANUAL_LOAD` - Manual load flag (default: OFF) - `DMOD_MAL_IMPLS` - List of MAL (Module Abstraction Layer) interfaces implemented - `DMOD_DIF_IMPLS` - List of DIF (Device Interface) implementations -- `DMOD_COMPRESSION_METHOD` - Compression method for DMFC files (default: "fastlz") +- `DMOD_COMPRESSION_METHOD` - Compression method for DMFC files: "fastlz" (default) or "deflate" (~20 % smaller; the loader needs the `deflate` decoder, see `DMOD_USE_DMCOMP`) - `DMOD_ASSETS_PATHS` - List of directories with the sources of a module's assets (views, images, fonts). When set, they are converted into `DMOD_VIEWS_DIR` (default: `/views`) by the `dmod_assets` target, which the module depends on - with tools `dmf-get` downloads at configure time, only for the kinds of files found: - `*.dmvs` views - `todmv` - `.dmv`; - images (`DMOD_IMAGE_EXTENSIONS`, default `png jpg jpeg jpe jfif bmp qoi gif tif tiff svg`) - `todmvi` with the `dmimg` decoder of each extension (`dmimg_png`, `dmimg_jpeg` for `.jpg`, ...; `DMOD_IMAGE_DECODERS` overrides the list) - `.dmvi`, next to the views, so a view finds them by name. An optional `.ini` next to an image (`logo.png.ini`) sets its conversion: `format` (`auto`, `rgb565`, `rgb565a8`, `argb8888`, `i8`, `a8`, `a4`), `size` (`WIDTHxHEIGHT`, scaled down to fit), `compression` (`fastlz`, `none`), `dither` (`on`/`off`), `blur` (pixels: a Gaussian blur of that standard deviation, after scaling), `crop` (`X,Y,W,H`: only that of it, after scaling and blurring - what a box shows), `radius` (pixels: its corners rounded after `crop`, transparent outside them - an image in a rounded box). Keys before any section apply to every output; each `[section]` is an output `
.dmvi` of its own (e.g. `[logo]` and `[logo-mask]` from one image). diff --git a/docs/development-package.md b/docs/development-package.md index a70e68c9..57ca305c 100644 --- a/docs/development-package.md +++ b/docs/development-package.md @@ -37,7 +37,7 @@ and compiled runtime objects are created in the consumer's build directory; the installed SDK remains read-only. Existing module API declaration rules still apply; see the [CMake function reference](cmake-functions.md). -Set DMOD options before `find_package`, for example `DMOD_USE_FASTLZ`, +Set DMOD options before `find_package`, for example `DMOD_USE_FASTLZ`, `DMOD_USE_DMCOMP` (the `deflate` method) and `DMOD_USE_DMCOMP_ENCODER`, `DMOD_DMF_DIR`, `DMOD_DMFC_DIR`, `DMOD_REPO_DIR` and `DMOD_ARCH`. Set per-module options such as `DMOD_AUTHOR_NAME`, `DMOD_STACK_SIZE`, `DMOD_MAL_IMPLS` and `DMOD_DMR_PATH` before calling `dmod_add_library` or `dmod_add_executable`. diff --git a/docs/tools-installation.md b/docs/tools-installation.md index 92747216..a21349de 100644 --- a/docs/tools-installation.md +++ b/docs/tools-installation.md @@ -220,7 +220,8 @@ Full documentation: [todmm-tool.md](todmm-tool.md) todmfc input.dmf output.dmfc # Specify compression method and level -todmfc input.dmf output.dmfc fastlz 3 +todmfc input.dmf output.dmfc fastlz 1 +todmfc input.dmf output.dmfc deflate 9 # ~20 % smaller than fastlz # Display available compression methods todmfc --help @@ -229,8 +230,10 @@ todmfc --help **Parameters:** - `path/to/file.dmf` - Input DMF file - `path/to/output.dmfc` - Output DMFC file -- `[compression_method]` - Optional compression method (default: fastlz) -- `[level]` - Optional compression level (default: 2) +- `[compression_method]` - Optional compression method: `fastlz` (default) or `deflate` (RFC 1951, from the + [dmcomp](../lib/third-party/dmcomp) library). A loader has to know the method: the `deflate` decoder is 1.7 kB of code + and needs a work memory of ~3 kB from the heap, so keep `fastlz` for the modules that older firmware has to load +- `[level]` - Optional compression level: `fastlz` 1 or 2 (default: 2), `deflate` 0 (stored) to 9 (default: 9) ### todmp diff --git a/examples/system/dif_test/Makefile b/examples/system/dif_test/Makefile index a821e8ac..8627a446 100644 --- a/examples/system/dif_test/Makefile +++ b/examples/system/dif_test/Makefile @@ -35,6 +35,9 @@ DEFINITIONS = DMOD_CURRENT_ALLOCATOR="dif_test" ifeq ($(DMOD_USE_FASTLZ),ON) DMOD_LIBS += dmod_fastlz endif +ifeq ($(DMOD_USE_DMCOMP),ON) + DMOD_LIBS += dmod_dmcomp +endif # ----------------------------------------------------------------------------- # List of objects diff --git a/examples/system/dif_test/dmod-cfg.mk b/examples/system/dif_test/dmod-cfg.mk index 962e9869..bb4c147a 100644 --- a/examples/system/dif_test/dmod-cfg.mk +++ b/examples/system/dif_test/dmod-cfg.mk @@ -12,6 +12,8 @@ DMOD_USE_STDIO=ON DMOD_USE_ASSERT=ON DMOD_USE_PTHREAD=ON DMOD_USE_FASTLZ=ON +DMOD_USE_DMCOMP=ON +DMOD_USE_DMCOMP_ENCODER=ON DMOD_USE_MMAN=ON DMOD_USE_ALIGNED_ALLOC=ON DMOD_USE_ALIGNED_MALLOC_MOCK=OFF diff --git a/examples/system/dmod_loader/Makefile b/examples/system/dmod_loader/Makefile index 40777536..3f79712e 100644 --- a/examples/system/dmod_loader/Makefile +++ b/examples/system/dmod_loader/Makefile @@ -33,6 +33,9 @@ DEFINITIONS = DMOD_CURRENT_ALLOCATOR="dmod_loader" ifeq ($(DMOD_USE_FASTLZ),ON) DMOD_LIBS += dmod_fastlz endif +ifeq ($(DMOD_USE_DMCOMP),ON) + DMOD_LIBS += dmod_dmcomp +endif # ----------------------------------------------------------------------------- # List of objects diff --git a/examples/system/dmod_loader/dmod-cfg.mk b/examples/system/dmod_loader/dmod-cfg.mk index 962e9869..bb4c147a 100644 --- a/examples/system/dmod_loader/dmod-cfg.mk +++ b/examples/system/dmod_loader/dmod-cfg.mk @@ -12,6 +12,8 @@ DMOD_USE_STDIO=ON DMOD_USE_ASSERT=ON DMOD_USE_PTHREAD=ON DMOD_USE_FASTLZ=ON +DMOD_USE_DMCOMP=ON +DMOD_USE_DMCOMP_ENCODER=ON DMOD_USE_MMAN=ON DMOD_USE_ALIGNED_ALLOC=ON DMOD_USE_ALIGNED_MALLOC_MOCK=OFF diff --git a/lib/third-party/CMakeLists.txt b/lib/third-party/CMakeLists.txt index 1ef320a5..f10f6ded 100644 --- a/lib/third-party/CMakeLists.txt +++ b/lib/third-party/CMakeLists.txt @@ -1 +1,2 @@ -add_subdirectory(FastLZ) \ No newline at end of file +add_subdirectory(FastLZ) +add_subdirectory(dmcomp) \ No newline at end of file diff --git a/lib/third-party/Makefile b/lib/third-party/Makefile index 6b6c20a7..d3e0c3fb 100644 --- a/lib/third-party/Makefile +++ b/lib/third-party/Makefile @@ -6,7 +6,7 @@ DMOD_DIR=../.. # ----------------------------------------------------------------------------- # Subdirectories # ----------------------------------------------------------------------------- -SUBDIRS = FastLZ +SUBDIRS = FastLZ dmcomp SYSTEM_SUBDIRS = MODULE_SUBDIRS = diff --git a/lib/third-party/dmcomp/CMakeLists.txt b/lib/third-party/dmcomp/CMakeLists.txt new file mode 100644 index 00000000..df6287e1 --- /dev/null +++ b/lib/third-party/dmcomp/CMakeLists.txt @@ -0,0 +1,26 @@ +set(MODULE_NAME dmod_dmcomp) + +# The decoder is always there; the encoder (it allocates ~330 kB and is ~10 kB of code) is for the host: +# a firmware that only loads the modules sets DMOD_USE_DMCOMP_ENCODER to OFF +add_library(${MODULE_NAME} STATIC + dmcomp_inflate.c + dmcomp_strerror.c +) +if(DMOD_USE_DMCOMP_ENCODER) + target_sources(${MODULE_NAME} PRIVATE dmcomp_deflate.c) +else() + target_compile_definitions(${MODULE_NAME} PUBLIC DMCOMP_NO_ENCODER) +endif() + +target_include_directories(${MODULE_NAME} + PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR} +) + +target_compile_options(${MODULE_NAME} PRIVATE + -Os + -ffunction-sections + -fdata-sections +) + +create_library_makefile(${MODULE_NAME}) diff --git a/lib/third-party/dmcomp/LICENSE.MIT b/lib/third-party/dmcomp/LICENSE.MIT new file mode 100644 index 00000000..7e7d6605 --- /dev/null +++ b/lib/third-party/dmcomp/LICENSE.MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Choco-Technologies + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/lib/third-party/dmcomp/Makefile b/lib/third-party/dmcomp/Makefile new file mode 100644 index 00000000..fae87d24 --- /dev/null +++ b/lib/third-party/dmcomp/Makefile @@ -0,0 +1,35 @@ +# ############################################################################## +# Makefile for the dmod_dmcomp module +# +# This Makefile is used to build the dmod_dmcomp module library +# +# DONT EDIT THIS FILE - it is automatically generated. +# Edit the scripts/Makefile-lib.in file instead. +# +# ############################################################################## +ifeq ($(DMOD_DIR),) + DMOD_DIR = ../../../ +endif + +# +# Name of the module +# +DMOD_LIB_NAME=libdmod_dmcomp.a +DMOD_SOURCES=dmcomp_inflate.c\ + dmcomp_strerror.c\ + dmcomp_deflate.c +DMOD_INC_DIRS = $(DMOD_DIR)/lib/third-party/dmcomp +DMOD_LIBS = +DMOD_GEN_HEADERS_IN = +DMOD_DEFINITIONS = DMOD_CURRENT_ALLOCATOR="dmod_dmcomp" + +# ----------------------------------------------------------------------------- +# Initialization of paths +# ----------------------------------------------------------------------------- +include $(DMOD_DIR)/paths.mk + +# ----------------------------------------------------------------------------- +# Including the template for the library +# ----------------------------------------------------------------------------- +DMOD_LIB_OBJS_DIR = $(DMOD_OBJS_DIR)/dmod_dmcomp +include $(DMOD_SLIB_FILE_PATH) diff --git a/lib/third-party/dmcomp/README.md b/lib/third-party/dmcomp/README.md new file mode 100644 index 00000000..ccad41b6 --- /dev/null +++ b/lib/third-party/dmcomp/README.md @@ -0,0 +1,11 @@ +# dmcomp (third-party) + +DEFLATE (RFC 1951) coder and decoder of [choco-technologies/dmcomp](https://github.com/choco-technologies/dmcomp), +commit `7314bf2` (`feat/inflate-workspace`). MIT licensed, see `LICENSE.MIT`. + +It is the `deflate` method of `Dmod_Compression_*` (`src/system/if/dmod_if_comp.c`): the DMFC files of a module that +`todmfc` makes with `deflate` are ~20 % smaller than the `fastlz` ones. The decoder is 1.7 kB of code on Cortex-M, the +encoder is for the host (`DMOD_USE_DMCOMP_ENCODER`). + +**Do not edit the files here** – change them in dmcomp and copy them again (the tests, the sanitizers and the +size checks of the CI are there). diff --git a/lib/third-party/dmcomp/dmcomp.h b/lib/third-party/dmcomp/dmcomp.h new file mode 100644 index 00000000..bd71f83a --- /dev/null +++ b/lib/third-party/dmcomp/dmcomp.h @@ -0,0 +1,118 @@ +/** + * @file dmcomp.h + * @brief DMOD compression - DEFLATE (RFC 1951) coder and decoder + * + * A raw DEFLATE stream (no zlib or gzip wrapper: the sizes travel beside the data, e.g. in the DMFC header) is + * decoded and encoded memory to memory. Both directions live in this library, so the host tools and the target see + * the very same implementation: + * + * - the decoder (dmcomp_inflate) is small, needs no heap and no libc beyond //; its + * state is a few kB on the stack (see DMCOMP_LIT_FAST_BITS / DMCOMP_DIST_FAST_BITS) and the output buffer is the + * window - nothing is copied around; + * - the encoder (dmcomp_deflate) is meant for the host: it allocates ~330 kB for the hash chains. Build the library + * with DMCOMP_NO_ENCODER (CMake: -DDMCOMP_BUILD_ENCODER=OFF) to leave it out. + * + * The streams are plain DEFLATE: zlib/gzip tools read what dmcomp_deflate writes (inflateInit2 with -15) and + * dmcomp_inflate reads what they write. + */ +#ifndef DMCOMP_H +#define DMCOMP_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** Version of the library */ +#define DMCOMP_VERSION_MAJOR 0 +#define DMCOMP_VERSION_MINOR 1 + +/** Results: 0 is success, errors are negative */ +typedef enum +{ + DMCOMP_OK = 0, //!< Success + DMCOMP_ERR_PARAM = -1, //!< Invalid argument (NULL pointer, size too big for the coder) + DMCOMP_ERR_DATA = -2, //!< The compressed data is not valid DEFLATE + DMCOMP_ERR_TRUNCATED = -3, //!< The compressed data ends before the stream does + DMCOMP_ERR_OVERFLOW = -4, //!< The output buffer is too small + DMCOMP_ERR_MEMORY = -5, //!< The encoder could not allocate its work memory +} dmcomp_result_t; + +/** + * @brief Decompress a raw DEFLATE stream (the tables on the stack: ~3 kB, see dmcomp_inflate_ws() for the other way) + * + * Every block type is accepted (stored, fixed and dynamic Huffman codes). The data after the final block is ignored. + * Corrupted data is never trusted: nothing is read beyond @p src + @p src_size and nothing is written beyond + * @p dst + @p dst_capacity; on an error the content of @p dst is undefined. + * + * @param src Compressed data + * @param src_size Size of the compressed data + * @param dst Output buffer (the whole decompressed data is kept in it - it is the window) + * @param dst_capacity Size of the output buffer + * @param out_size Receives the number of bytes written (may not be NULL) + * + * @return DMCOMP_OK or an error (DMCOMP_ERR_DATA, DMCOMP_ERR_TRUNCATED, DMCOMP_ERR_OVERFLOW, DMCOMP_ERR_PARAM) + */ +int dmcomp_inflate( const void* src, size_t src_size, void* dst, size_t dst_capacity, size_t* out_size ); + +/** + * @brief Size of the work memory of dmcomp_inflate_ws() + * + * The tables of the Huffman codes: ~2.9 kB (see DMCOMP_LIT_FAST_BITS and DMCOMP_DIST_FAST_BITS in dmcomp_inflate.c). + */ +size_t dmcomp_inflate_workspace_size( void ); + +/** + * @brief dmcomp_inflate() with the work memory given by the caller + * + * The same decoder, but its tables are not on the stack: for a thread with a small one (the decoding may be called from + * anywhere a module is loaded), give it memory from the heap, or a buffer of your own that is not used at the time. + * + * @param workspace dmcomp_inflate_workspace_size() bytes, aligned to 2 bytes (what malloc returns is); not used after the call + * + * Other parameters and results: see dmcomp_inflate(). DMCOMP_ERR_PARAM also for a NULL or odd workspace. + */ +int dmcomp_inflate_ws( const void* src, size_t src_size, void* dst, size_t dst_capacity, size_t* out_size, + void* workspace ); + +/** Name of a result, for logs. Never NULL. */ +const char* dmcomp_strerror( int result ); + +#ifndef DMCOMP_NO_ENCODER + +/** Level of compression that dmcomp_deflate uses for a level out of range */ +#define DMCOMP_LEVEL_DEFAULT 6 + +/** + * @brief Upper bound of the size of the compressed data + * + * dmcomp_deflate never writes more than this (incompressible data is stored), so a buffer of this size cannot overflow. + */ +size_t dmcomp_deflate_bound( size_t src_size ); + +/** + * @brief Compress data to a raw DEFLATE stream + * + * Distances up to 32767 bytes are used. The result is the smallest of stored, fixed and dynamic Huffman codes for + * every block. + * + * @param src Data to compress + * @param src_size Size of the data (up to ~4 GB) + * @param dst Output buffer + * @param dst_capacity Size of the output buffer (dmcomp_deflate_bound() is always enough) + * @param out_size Receives the size of the compressed data (may not be NULL) + * @param level 0: stored, 1 (fastest) .. 9 (smallest); out of range: DMCOMP_LEVEL_DEFAULT (negative) or 9 + * + * @return DMCOMP_OK or an error (DMCOMP_ERR_OVERFLOW, DMCOMP_ERR_MEMORY, DMCOMP_ERR_PARAM) + */ +int dmcomp_deflate( const void* src, size_t src_size, void* dst, size_t dst_capacity, size_t* out_size, int level ); + +#endif /* DMCOMP_NO_ENCODER */ + +#ifdef __cplusplus +} +#endif + +#endif /* DMCOMP_H */ diff --git a/lib/third-party/dmcomp/dmcomp_deflate.c b/lib/third-party/dmcomp/dmcomp_deflate.c new file mode 100644 index 00000000..d6d2c1c2 --- /dev/null +++ b/lib/third-party/dmcomp/dmcomp_deflate.c @@ -0,0 +1,781 @@ +/** + * @file dmcomp_deflate.c + * @brief DEFLATE encoder (RFC 1951), memory to memory + * + * LZ77 with hash chains over the whole input (it is in memory, so there is no sliding window to manage), optionally + * with lazy matching, then every block of up to DMCOMP_BLOCK_TOKENS tokens is written as the smallest of a stored, + * a fixed-code and a dynamic-code block. The code lengths are length limited Huffman codes. + * + * It is a tool for the host: ~330 kB of work memory. Everything the format needs is here, nothing of the decoder is. + */ +#include +#include +#include "dmcomp.h" +#include "dmcomp_tables.h" + +#ifndef DMCOMP_MALLOC +# define DMCOMP_MALLOC malloc +# define DMCOMP_FREE free +#endif + +#define WINDOW_SIZE 32768u +#define WINDOW_MASK (WINDOW_SIZE - 1u) +#define MAX_DISTANCE (WINDOW_SIZE - 1u) // 32768 would do for the format; this keeps the chains simple +#define HASH_BITS 15 +#define BLOCK_TOKENS 16384u +#define TOO_FAR 4096u // A match of 3 bytes this far is not worth its bits +#define STORED_MAX 65535u +#define MAX_INPUT 0xFFFF0000u // Positions are 32 bit + +typedef struct +{ + uint16_t good; //!< A previous match this long: search a quarter of the chain + uint16_t lazy; //!< A match this long is taken at once (lazy matching only) + uint16_t nice; //!< A match this long ends the search + uint16_t chain; //!< Candidates looked at per position + uint8_t lazy_on; //!< Look whether the next position has a longer match +} level_t; + +static const level_t g_levels[10] = +{ + { 0, 0, 0, 0, 0 }, + { 4, 4, 8, 4, 0 }, + { 4, 5, 16, 8, 0 }, + { 4, 6, 32, 32, 0 }, + { 4, 4, 16, 16, 1 }, + { 8, 16, 32, 32, 1 }, + { 8, 16, 128, 128, 1 }, + { 8, 32, 128, 256, 1 }, + { 32, 128, 258, 1024, 1 }, + { 32, 258, 258, 4096, 1 }, +}; + +// ===================================================================================================================== +// Bit writer +// ===================================================================================================================== + +typedef struct +{ + uint8_t* start; + uint8_t* p; + uint8_t* end; + uint32_t acc; //!< Bits not written yet, the first one lowest + unsigned n; //!< Their number (always < 8 between the calls) + int overflow; //!< The output buffer was too small - the rest is counted, not written +} bitwriter_t; + +static void put( bitwriter_t* w, uint32_t value, unsigned count ) +{ + w->acc |= value << w->n; + w->n += count; + while( w->n >= 8 ) + { + if( w->p < w->end ) + { + *w->p++ = (uint8_t)w->acc; + } + else + { + w->overflow = 1; + } + w->acc >>= 8; + w->n -= 8; + } +} + +static void put_stored( bitwriter_t* w, const uint8_t* data, size_t length, int final ) +{ + do + { + size_t chunk = length > STORED_MAX ? STORED_MAX : length; + + length -= chunk; + put( w, (final && length == 0) ? 1u : 0u, 1 ); + put( w, 0, 2 ); + put( w, 0, (8 - w->n) & 7u ); // To the next byte + put( w, (uint32_t)chunk, 16 ); + put( w, (uint32_t)(~chunk & 0xFFFFu), 16 ); + if( (size_t)(w->end - w->p) >= chunk ) + { + if( chunk > 0 ) + { + memcpy( w->p, data, chunk ); + } + w->p += chunk; + } + else + { + w->overflow = 1; + } + if( chunk > 0 ) + { + data += chunk; + } + } + while( length > 0 ); +} + +// ===================================================================================================================== +// Huffman codes +// ===================================================================================================================== + +static unsigned reverse_bits( unsigned code, unsigned length ) +{ + unsigned reversed = 0; + while( length-- > 0 ) + { + reversed = (reversed << 1) | (code & 1u); + code >>= 1; + } + return reversed; +} + +/** Canonical codes of the given lengths, ready to be put into the stream (first bit lowest) */ +static void make_codes( const uint8_t* lengths, unsigned symbols, uint16_t* codes ) +{ + unsigned count[DMCOMP_MAX_BITS + 1] = { 0 }; + unsigned next[DMCOMP_MAX_BITS + 2]; + unsigned i, length, code = 0; + + for( i = 0; i < symbols; i++ ) + { + count[lengths[i]]++; + } + count[0] = 0; + for( length = 1; length <= DMCOMP_MAX_BITS; length++ ) + { + code = (code + count[length - 1]) << 1; + next[length] = code; + } + for( i = 0; i < symbols; i++ ) + { + codes[i] = lengths[i] ? (uint16_t)reverse_bits( next[lengths[i]]++, lengths[i] ) : 0; + } +} + +/** + * Code lengths of the symbols of the given frequencies, none longer than @p limit. + * + * Huffman's tree is built by merging the two lightest nodes (leaves sorted once, the merged nodes come out in + * order, so two queues are enough). A tree deeper than the limit is cut the way zlib does it: the leaves below the + * limit move up and make room by pushing a shallower leaf down. At least two symbols get a code (a stream with one + * code is valid but some decoders do not take it). + */ +static void build_lengths( const uint32_t* freq, unsigned symbols, unsigned limit, uint8_t* lengths ) +{ + uint32_t weight[2 * DMCOMP_LITLEN_SYMBOLS]; + uint16_t parent[2 * DMCOMP_LITLEN_SYMBOLS]; + uint16_t depth[2 * DMCOMP_LITLEN_SYMBOLS]; + uint32_t f[DMCOMP_LITLEN_SYMBOLS]; + uint16_t order[DMCOMP_LITLEN_SYMBOLS]; // The used symbols, the rarest first + unsigned bl_count[DMCOMP_MAX_BITS + 1] = { 0 }; + unsigned i, j, used = 0, overflow = 0; + + for( i = 0; i < symbols; i++ ) + { + f[i] = freq[i]; + lengths[i] = 0; + if( f[i] != 0 ) + { + used++; + } + } + for( i = 0; used < 2 && i < symbols; i++ ) + { + if( f[i] == 0 ) + { + f[i] = 1; + used++; + } + } + + // Insertion sort by frequency (then by value): there are at most 286 symbols + used = 0; + for( i = 0; i < symbols; i++ ) + { + if( f[i] == 0 ) + { + continue; + } + for( j = used; j > 0 && f[order[j - 1]] > f[i]; j-- ) + { + order[j] = order[j - 1]; + } + order[j] = (uint16_t)i; + used++; + } + + // The leaves are nodes 0..used-1, the merged nodes follow + { + unsigned leaf = 0, node = used, created = used; + unsigned step; + + for( i = 0; i < used; i++ ) + { + weight[i] = f[order[i]]; + } + for( step = 0; step + 1 < used; step++ ) + { + unsigned pick[2]; + for( j = 0; j < 2; j++ ) + { + if( leaf < used && (node >= created || weight[leaf] <= weight[node]) ) + { + pick[j] = leaf++; + } + else + { + pick[j] = node++; + } + } + weight[created] = weight[pick[0]] + weight[pick[1]]; + parent[pick[0]] = (uint16_t)created; + parent[pick[1]] = (uint16_t)created; + created++; + } + + // The root is the last node; every parent comes after its children. A node deeper than the limit is cut to it + // (its children are measured from there) - every node that is counts as overflow, the inner ones too: that is + // what the correction below is made of + depth[created - 1] = 0; + for( i = created - 1; i-- > 0; ) + { + unsigned d = depth[parent[i]] + 1u; + if( d > limit ) + { + d = limit; + overflow++; + } + depth[i] = (uint16_t)d; + } + for( i = 0; i < used; i++ ) + { + bl_count[depth[i]]++; + } + } + + while( overflow > 0 ) + { + unsigned bits = limit - 1; + while( bl_count[bits] == 0 ) + { + bits--; + } + bl_count[bits]--; // A leaf moves down... + bl_count[bits + 1] += 2; // ...and takes one of the leaves that were cut as its brother + bl_count[limit]--; + overflow = overflow > 2 ? overflow - 2 : 0; + } + + // The rarest symbols get the longest codes + { + unsigned k = 0, length; + for( length = limit; length >= 1; length-- ) + { + for( i = 0; i < bl_count[length]; i++ ) + { + lengths[order[k++]] = (uint8_t)length; + } + } + } +} + +/** + * Run-length codes the lengths of the codes (the alphabet of RFC 1951, 3.2.7): 0..15 as they are, 16 repeats the + * previous one 3-6 times, 17 and 18 repeat zero 3-10 and 11-138 times. + * + * @return number of symbols; symbols[] and extras[] (the value of the extra bits) are filled + */ +static unsigned rle_lengths( const uint8_t* lengths, unsigned total, uint8_t* symbols, uint8_t* extras ) +{ + unsigned i = 0, k = 0; + + while( i < total ) + { + uint8_t value = lengths[i]; + unsigned run = 1; + + while( i + run < total && lengths[i + run] == value ) + { + run++; + } + i += run; + + if( value == 0 ) + { + while( run >= 11 ) + { + unsigned r = run > 138 ? 138 : run; + symbols[k] = 18; + extras[k++] = (uint8_t)(r - 11); + run -= r; + } + if( run >= 3 ) + { + symbols[k] = 17; + extras[k++] = (uint8_t)(run - 3); + run = 0; + } + while( run-- > 0 ) + { + symbols[k] = 0; + extras[k++] = 0; + } + } + else + { + symbols[k] = value; + extras[k++] = 0; + run--; + while( run >= 3 ) + { + unsigned r = run > 6 ? 6 : run; + symbols[k] = 16; + extras[k++] = (uint8_t)(r - 3); + run -= r; + } + while( run-- > 0 ) + { + symbols[k] = value; + extras[k++] = 0; + } + } + } + return k; +} + +// ===================================================================================================================== +// Encoder state +// ===================================================================================================================== + +typedef struct +{ + const uint8_t* in; + size_t n; + const level_t* cfg; + bitwriter_t w; + + size_t ins; //!< Positions below it are in the hash chains + size_t cov; //!< The input covered by the tokens so far (the written blocks included) + size_t blk_start; //!< Where the block being collected starts + unsigned ntok; + uint32_t lit_freq[DMCOMP_LITLEN_SYMBOLS]; + uint32_t dist_freq[DMCOMP_DIST_SYMBOLS]; + uint8_t lcode[DMCOMP_MAX_MATCH + 1]; //!< Length -> length code (0..28) + uint8_t dcode[512]; //!< Distance - 1 -> distance code, see DIST_CODE + uint32_t head[1u << HASH_BITS]; //!< Newest position (+1) of every hash + uint32_t prev[WINDOW_SIZE]; //!< Older position (+1) with the same hash, by position + uint32_t tokens[BLOCK_TOKENS]; //!< A literal: its value; a match: (length << 16) | distance +} encoder_t; + +#define DIST_CODE( e, d ) ((e)->dcode[(d) < 256u ? (d) : 256u + ((d) >> 7)]) + +static void init_code_tables( encoder_t* e ) +{ + unsigned c, v; + + for( c = 0; c < 28; c++ ) + { + for( v = dmcomp_len_base[c]; v < dmcomp_len_base[c] + (1u << dmcomp_len_extra[c]); v++ ) + { + e->lcode[v] = (uint8_t)c; + } + } + e->lcode[258] = 28; // 258 has a code of its own, the last of code 27's range is not used + + for( c = 0; c < DMCOMP_DIST_SYMBOLS; c++ ) + { + for( v = dmcomp_dist_base[c] - 1u; v < dmcomp_dist_base[c] - 1u + (1u << dmcomp_dist_extra[c]); v++ ) + { + e->dcode[v < 256u ? v : 256u + (v >> 7)] = (uint8_t)c; + } + } +} + +// ===================================================================================================================== +// Matching +// ===================================================================================================================== + +static unsigned hash3( const uint8_t* p ) +{ + uint32_t v = (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16); + return (v * 2654435761u) >> (32 - HASH_BITS); +} + +/** Puts the positions below @p upto into the hash chains */ +static void insert_upto( encoder_t* e, size_t upto ) +{ + while( e->ins < upto ) + { + size_t pos = e->ins++; + if( pos + 2 < e->n ) + { + unsigned h = hash3( e->in + pos ); + e->prev[pos & WINDOW_MASK] = e->head[h]; + e->head[h] = (uint32_t)pos + 1; + } + } +} + +/** + * Longest match for the data at @p pos, only if it is longer than @p better_than (and at least 3 bytes). The + * positions below @p pos have to be in the chains, @p pos itself not yet. + * + * @return the length, or 0 + */ +static unsigned find_match( encoder_t* e, size_t pos, unsigned better_than, unsigned chain, unsigned* distance ) +{ + size_t avail = e->n - pos; + const uint8_t* cur = e->in + pos; + unsigned max_len = avail > DMCOMP_MAX_MATCH ? DMCOMP_MAX_MATCH : (unsigned)avail; + unsigned best = better_than < DMCOMP_MIN_MATCH - 1 ? DMCOMP_MIN_MATCH - 1 : better_than; + unsigned best_distance = 0; + uint32_t candidate; + + if( avail < DMCOMP_MIN_MATCH || best >= max_len ) + { + return 0; + } + + candidate = e->head[hash3( cur )]; + while( candidate != 0 && chain-- > 0 ) + { + size_t cand_pos = candidate - 1; + size_t d = pos - cand_pos; + const uint8_t* m = e->in + cand_pos; + uint32_t next; + + if( cand_pos >= pos || d > MAX_DISTANCE ) + { + break; + } + if( m[best] == cur[best] && m[0] == cur[0] ) + { + unsigned l = 0; + while( l < max_len && m[l] == cur[l] ) + { + l++; + } + if( l > best ) + { + best = l; + best_distance = (unsigned)d; + if( l >= e->cfg->nice || l >= max_len ) + { + break; + } + } + } + next = e->prev[cand_pos & WINDOW_MASK]; + if( next >= candidate ) + { + break; // The chain goes back in time, always + } + candidate = next; + } + + if( best_distance == 0 || (best == DMCOMP_MIN_MATCH && best_distance > TOO_FAR) ) + { + return 0; + } + *distance = best_distance; + return best; +} + +// ===================================================================================================================== +// Blocks +// ===================================================================================================================== + +static uint64_t data_bits( const encoder_t* e, const uint8_t* lit_len, const uint8_t* dist_len ) +{ + uint64_t bits = 0; + unsigned s; + + for( s = 0; s < DMCOMP_LITLEN_SYMBOLS; s++ ) + { + bits += (uint64_t)e->lit_freq[s] * lit_len[s]; + } + for( s = 257; s < DMCOMP_LITLEN_SYMBOLS; s++ ) + { + bits += (uint64_t)e->lit_freq[s] * dmcomp_len_extra[s - 257]; + } + for( s = 0; s < DMCOMP_DIST_SYMBOLS; s++ ) + { + bits += (uint64_t)e->dist_freq[s] * (dist_len[s] + dmcomp_dist_extra[s]); + } + return bits; +} + +static void put_tokens( encoder_t* e, const uint16_t* lit_code, const uint8_t* lit_len, const uint16_t* dist_code, + const uint8_t* dist_len ) +{ + unsigned i; + + for( i = 0; i < e->ntok; i++ ) + { + uint32_t t = e->tokens[i]; + if( (t >> 16) == 0 ) + { + put( &e->w, lit_code[t], lit_len[t] ); + } + else + { + unsigned length = t >> 16; + unsigned distance = t & 0xFFFFu; + unsigned lc = e->lcode[length]; + unsigned dc = DIST_CODE( e, distance - 1u ); + + put( &e->w, lit_code[257 + lc], lit_len[257 + lc] ); + if( dmcomp_len_extra[lc] != 0 ) + { + put( &e->w, length - dmcomp_len_base[lc], dmcomp_len_extra[lc] ); + } + put( &e->w, dist_code[dc], dist_len[dc] ); + if( dmcomp_dist_extra[dc] != 0 ) + { + put( &e->w, distance - dmcomp_dist_base[dc], dmcomp_dist_extra[dc] ); + } + } + } + put( &e->w, lit_code[256], lit_len[256] ); +} + +/** Writes the collected tokens as one block - the smallest of the three kinds - and starts the next one */ +static void flush_block( encoder_t* e, int final ) +{ + uint8_t lit_len[DMCOMP_LITLEN_SYMBOLS], dist_len[DMCOMP_DIST_SYMBOLS]; + uint8_t fixed_lit_len[288], fixed_dist_len[DMCOMP_DIST_SYMBOLS]; + uint8_t all_len[DMCOMP_LITLEN_SYMBOLS + DMCOMP_DIST_SYMBOLS]; + uint8_t rle_sym[DMCOMP_LITLEN_SYMBOLS + DMCOMP_DIST_SYMBOLS], rle_extra[DMCOMP_LITLEN_SYMBOLS + DMCOMP_DIST_SYMBOLS]; + uint32_t clen_freq[DMCOMP_CLEN_SYMBOLS] = { 0 }; + uint8_t clen_len[DMCOMP_CLEN_SYMBOLS]; + unsigned hlit, hdist, hclen, nrle, i; + uint64_t dynamic_bits, fixed_bits, stored_bits; + size_t span = e->cov - e->blk_start; + + e->lit_freq[256] = 1; // The end of the block + + // --- Dynamic codes + build_lengths( e->lit_freq, DMCOMP_LITLEN_SYMBOLS, DMCOMP_MAX_BITS, lit_len ); + build_lengths( e->dist_freq, DMCOMP_DIST_SYMBOLS, DMCOMP_MAX_BITS, dist_len ); + for( hlit = DMCOMP_LITLEN_SYMBOLS; hlit > 257 && lit_len[hlit - 1] == 0; hlit-- ) {} + for( hdist = DMCOMP_DIST_SYMBOLS; hdist > 1 && dist_len[hdist - 1] == 0; hdist-- ) {} + memcpy( all_len, lit_len, hlit ); + memcpy( all_len + hlit, dist_len, hdist ); + nrle = rle_lengths( all_len, hlit + hdist, rle_sym, rle_extra ); + for( i = 0; i < nrle; i++ ) + { + clen_freq[rle_sym[i]]++; + } + build_lengths( clen_freq, DMCOMP_CLEN_SYMBOLS, DMCOMP_MAX_CLEN_BITS, clen_len ); + for( hclen = DMCOMP_CLEN_SYMBOLS; hclen > 4 && clen_len[dmcomp_clen_order[hclen - 1]] == 0; hclen-- ) {} + + dynamic_bits = 3 + 14 + 3 * (uint64_t)hclen + data_bits( e, lit_len, dist_len ); + for( i = 0; i < nrle; i++ ) + { + dynamic_bits += clen_len[rle_sym[i]] + (rle_sym[i] == 16 ? 2 : rle_sym[i] == 17 ? 3 : rle_sym[i] == 18 ? 7 : 0); + } + + // --- Fixed codes + for( i = 0; i < 144; i++ ) fixed_lit_len[i] = 8; + for( ; i < 256; i++ ) fixed_lit_len[i] = 9; + for( ; i < 280; i++ ) fixed_lit_len[i] = 7; + for( ; i < 288; i++ ) fixed_lit_len[i] = 8; + for( i = 0; i < DMCOMP_DIST_SYMBOLS; i++ ) fixed_dist_len[i] = 5; + fixed_bits = 3 + data_bits( e, fixed_lit_len, fixed_dist_len ); + + // --- Stored: 3 bits and the padding in front of the first chunk, 4 bytes in front of every chunk + { + uint64_t chunks = span == 0 ? 1 : (span + STORED_MAX - 1) / STORED_MAX; + unsigned pad = (8 - ((e->w.n + 3) & 7u)) & 7u; + stored_bits = 3 + pad + 32 * chunks + 8 * (chunks - 1) + 8 * (uint64_t)span; + } + + if( stored_bits < fixed_bits && stored_bits < dynamic_bits ) + { + put_stored( &e->w, e->in + e->blk_start, span, final ); // Writes the BFINAL bit itself + } + else if( fixed_bits <= dynamic_bits ) + { + uint16_t lit_code[288], dist_code[DMCOMP_DIST_SYMBOLS]; + + make_codes( fixed_lit_len, 288, lit_code ); + make_codes( fixed_dist_len, DMCOMP_DIST_SYMBOLS, dist_code ); + put( &e->w, final ? 1u : 0u, 1 ); + put( &e->w, 1, 2 ); + put_tokens( e, lit_code, fixed_lit_len, dist_code, fixed_dist_len ); + } + else + { + uint16_t lit_code[DMCOMP_LITLEN_SYMBOLS], dist_code[DMCOMP_DIST_SYMBOLS], clen_code[DMCOMP_CLEN_SYMBOLS]; + + make_codes( lit_len, DMCOMP_LITLEN_SYMBOLS, lit_code ); + make_codes( dist_len, DMCOMP_DIST_SYMBOLS, dist_code ); + make_codes( clen_len, DMCOMP_CLEN_SYMBOLS, clen_code ); + + put( &e->w, final ? 1u : 0u, 1 ); + put( &e->w, 2, 2 ); + put( &e->w, hlit - 257, 5 ); + put( &e->w, hdist - 1, 5 ); + put( &e->w, hclen - 4, 4 ); + for( i = 0; i < hclen; i++ ) + { + put( &e->w, clen_len[dmcomp_clen_order[i]], 3 ); + } + for( i = 0; i < nrle; i++ ) + { + put( &e->w, clen_code[rle_sym[i]], clen_len[rle_sym[i]] ); + if( rle_sym[i] == 16 ) put( &e->w, rle_extra[i], 2 ); + else if( rle_sym[i] == 17 ) put( &e->w, rle_extra[i], 3 ); + else if( rle_sym[i] == 18 ) put( &e->w, rle_extra[i], 7 ); + } + put_tokens( e, lit_code, lit_len, dist_code, dist_len ); + } + + memset( e->lit_freq, 0, sizeof(e->lit_freq) ); + memset( e->dist_freq, 0, sizeof(e->dist_freq) ); + e->ntok = 0; + e->blk_start = e->cov; +} + +static void emit_literal( encoder_t* e, uint8_t value ) +{ + e->tokens[e->ntok++] = value; + e->lit_freq[value]++; + e->cov += 1; +} + +static void emit_match( encoder_t* e, unsigned length, unsigned distance ) +{ + e->tokens[e->ntok++] = ((uint32_t)length << 16) | distance; + e->lit_freq[257 + e->lcode[length]]++; + e->dist_freq[DIST_CODE( e, distance - 1u )]++; + e->cov += length; +} + +static void compress_blocks( encoder_t* e ) +{ + const level_t* cfg = e->cfg; + size_t i = 0; + unsigned length = 0, distance = 0; + int have_match = 0; // 'length' and 'distance' are the match at i (found while looking ahead) + + while( i < e->n ) + { + if( !have_match ) + { + insert_upto( e, i ); + length = find_match( e, i, 0, cfg->chain, &distance ); + insert_upto( e, i + 1 ); + } + have_match = 0; + + if( length >= DMCOMP_MIN_MATCH ) + { + if( cfg->lazy_on && length < cfg->lazy && i + 1 < e->n ) + { + // A longer match one byte later is worth a literal now + unsigned next_distance = 0; + unsigned chain = length >= cfg->good ? cfg->chain >> 2 : cfg->chain; + unsigned next_length = find_match( e, i + 1, length, chain, &next_distance ); + + insert_upto( e, i + 2 ); + if( next_length > length ) + { + emit_literal( e, e->in[i] ); + i++; + length = next_length; + distance = next_distance; + have_match = 1; + if( e->ntok >= BLOCK_TOKENS ) + { + flush_block( e, 0 ); + } + continue; + } + } + emit_match( e, length, distance ); + insert_upto( e, i + length ); + i += length; + } + else + { + emit_literal( e, e->in[i] ); + i++; + } + + if( e->ntok >= BLOCK_TOKENS ) + { + flush_block( e, 0 ); + } + } + flush_block( e, 1 ); +} + +// ===================================================================================================================== +// Public API +// ===================================================================================================================== + +size_t dmcomp_deflate_bound( size_t src_size ) +{ + // The worst case is a stored block: 5 bytes (the header, the padding, LEN and NLEN) per 64 kB, and a block is at + // least BLOCK_TOKENS bytes long unless it is the last one + return src_size + 6 * (src_size / BLOCK_TOKENS + 2) + 16; +} + +int dmcomp_deflate( const void* src, size_t src_size, void* dst, size_t dst_capacity, size_t* out_size, int level ) +{ + encoder_t* e; + int result; + + if( out_size == NULL || (src == NULL && src_size != 0) || (dst == NULL && dst_capacity != 0) || + src_size > MAX_INPUT ) + { + return DMCOMP_ERR_PARAM; + } + *out_size = 0; + + if( level < 0 ) + { + level = DMCOMP_LEVEL_DEFAULT; + } + else if( level > 9 ) + { + level = 9; + } + + e = (encoder_t*)DMCOMP_MALLOC( sizeof(encoder_t) ); + if( e == NULL ) + { + return DMCOMP_ERR_MEMORY; + } + memset( e, 0, offsetof( encoder_t, prev ) ); // The chains (prev) and the tokens are written before they are read + e->in = (const uint8_t*)src; + e->n = src_size; + e->cfg = &g_levels[level]; + e->w.start = (uint8_t*)dst; + e->w.p = (uint8_t*)dst; + e->w.end = dst_capacity != 0 ? (uint8_t*)dst + dst_capacity : (uint8_t*)dst; // NULL is not moved + init_code_tables( e ); + + if( level == 0 ) + { + put_stored( &e->w, e->in, e->n, 1 ); + } + else + { + compress_blocks( e ); + } + if( e->w.n > 0 ) + { + put( &e->w, 0, 8 - e->w.n ); + } + + result = e->w.overflow ? DMCOMP_ERR_OVERFLOW : DMCOMP_OK; + if( result == DMCOMP_OK ) + { + *out_size = (size_t)(e->w.p - e->w.start); + } + DMCOMP_FREE( e ); + return result; +} diff --git a/lib/third-party/dmcomp/dmcomp_inflate.c b/lib/third-party/dmcomp/dmcomp_inflate.c new file mode 100644 index 00000000..90b37ca3 --- /dev/null +++ b/lib/third-party/dmcomp/dmcomp_inflate.c @@ -0,0 +1,571 @@ +/** + * @file dmcomp_inflate.c + * @brief DEFLATE decoder (RFC 1951), memory to memory + * + * Design: the output buffer is the window, the state is a few kB on the stack, there is no heap and no global state + * (reentrant). Huffman codes are decoded with a table of the first DMCOMP_*_FAST_BITS bits of the stream; the rare + * longer codes are walked canonically, bit by bit. The input is never read beyond its end: the missing bytes read as + * zeros, and the decoder notices that it used them (DMCOMP_ERR_TRUNCATED) before it acts on them. + */ +#include +#include "dmcomp.h" +#include "dmcomp_tables.h" + +/** Bits of the table of literal/length codes: 2^bits * 2 bytes of stack. 9 covers almost every code of real data. */ +#ifndef DMCOMP_LIT_FAST_BITS +# define DMCOMP_LIT_FAST_BITS 9 +#endif + +/** Bits of the table of distance codes: 2^bits * 2 bytes of stack */ +#ifndef DMCOMP_DIST_FAST_BITS +# define DMCOMP_DIST_FAST_BITS 7 +#endif + +#define CLEN_FAST_BITS DMCOMP_MAX_CLEN_BITS // The whole alphabet of code lengths is in the table +#define LITLEN_TABLE_SIZE 288 // Fixed codes have 288 symbols (286 and 287 are invalid) +#define DIST_TABLE_SIZE 32 // Fixed codes have 32 symbols (30 and 31 are invalid) + +// ===================================================================================================================== +// Bit reader +// ===================================================================================================================== + +typedef struct +{ + const uint8_t* p; //!< Next byte of the input + const uint8_t* end; //!< End of the input + uint32_t bits; //!< Bits not used yet (the next one is the lowest) + unsigned n; //!< Number of valid bits in 'bits' + unsigned virt; //!< Number of the bits that were added past the end of the input (they are the highest) +} bitreader_t; + +/** Fills the buffer to 25..32 bits. Past the end of the input zeros are added (and counted in 'virt'). */ +static void refill( bitreader_t* b ) +{ + while( b->n <= 24 ) + { + uint32_t byte = 0; + if( b->p < b->end ) + { + byte = *b->p++; + } + else + { + b->virt += 8; + } + b->bits |= byte << b->n; + b->n += 8; + } +} + +/** True when a bit that does not exist was used - the input is too short */ +static int overrun( const bitreader_t* b ) +{ + return b->n < b->virt; +} + +/** Takes @p count bits (0..16); the buffer must have been filled */ +static uint32_t take( bitreader_t* b, unsigned count ) +{ + uint32_t value = b->bits & ((1u << count) - 1u); + b->bits >>= count; + b->n -= count; + return value; +} + +// ===================================================================================================================== +// Huffman codes +// ===================================================================================================================== + +/** + * A code is described by three arrays: + * - fast[] : indexed by the next 'fast_bits' bits of the stream: (symbol << 4) | length of the code; 0: the code is + * longer than the table (or does not exist) + * - count[] : the number of codes of every length + * - sym[] : the symbols sorted by the length of their codes, then by the value + */ +typedef struct +{ + uint16_t lit_fast[1u << DMCOMP_LIT_FAST_BITS]; + uint16_t lit_count[DMCOMP_MAX_BITS + 1]; + uint16_t lit_sym[LITLEN_TABLE_SIZE]; + uint16_t dist_fast[1u << DMCOMP_DIST_FAST_BITS]; + uint16_t dist_count[DMCOMP_MAX_BITS + 1]; + uint16_t dist_sym[DIST_TABLE_SIZE]; + + // The header of a dynamic block (the lengths of the codes, and the code of the lengths) + uint8_t lengths[LITLEN_TABLE_SIZE + DIST_TABLE_SIZE]; + uint8_t clen[DMCOMP_CLEN_SYMBOLS]; + uint16_t cfast[1u << CLEN_FAST_BITS]; + uint16_t ccount[DMCOMP_MAX_BITS + 1]; + uint16_t csym[DMCOMP_CLEN_SYMBOLS]; +} workspace_t; + +static unsigned reverse_bits( unsigned code, unsigned length ) +{ + unsigned reversed = 0; + while( length-- > 0 ) + { + reversed = (reversed << 1) | (code & 1u); + code >>= 1; + } + return reversed; +} + +/** + * Builds the decoding tables of a code described by the lengths of its symbols. + * + * The set of codes has to be complete (it may not leave any bit sequence unused), with one exception that zlib allows + * too: a single code of one bit. No codes at all is accepted, every decoding fails then. + */ +static int huff_build( uint16_t* fast, unsigned fast_bits, uint16_t* count, uint16_t* sym, const uint8_t* lengths, + unsigned symbols ) +{ + uint16_t offsets[DMCOMP_MAX_BITS + 2]; + unsigned i, length, used; + int left = 1; + + for( length = 0; length <= DMCOMP_MAX_BITS; length++ ) + { + count[length] = 0; + } + for( i = 0; i < symbols; i++ ) + { + count[lengths[i]]++; + } + used = symbols - count[0]; + + for( i = 0; i < (1u << fast_bits); i++ ) + { + fast[i] = 0; + } + if( used == 0 ) + { + return DMCOMP_OK; + } + + for( length = 1; length <= DMCOMP_MAX_BITS; length++ ) + { + left <<= 1; + left -= count[length]; + if( left < 0 ) + { + return DMCOMP_ERR_DATA; // Over-subscribed + } + } + if( left > 0 && !(used == 1 && count[1] == 1) ) + { + return DMCOMP_ERR_DATA; // Incomplete + } + + offsets[1] = 0; + for( length = 1; length < DMCOMP_MAX_BITS; length++ ) + { + offsets[length + 1] = (uint16_t)(offsets[length] + count[length]); + } + for( i = 0; i < symbols; i++ ) + { + if( lengths[i] != 0 ) + { + sym[offsets[lengths[i]]++] = (uint16_t)i; + } + } + + // The canonical codes are given out in the order of 'sym'; the stream has them with the first bit lowest + unsigned code = 0, index = 0; + for( length = 1; length <= DMCOMP_MAX_BITS; length++ ) + { + for( i = 0; i < count[length]; i++, index++, code++ ) + { + if( length <= fast_bits ) + { + unsigned j; + uint16_t entry = (uint16_t)((sym[index] << 4) | length); + for( j = reverse_bits( code, length ); j < (1u << fast_bits); j += (1u << length) ) + { + fast[j] = entry; + } + } + } + code <<= 1; + } + return DMCOMP_OK; +} + +/** Decodes one symbol; the buffer must have been filled. Returns the symbol or -1 for a bit sequence of no code. */ +static int huff_decode( bitreader_t* b, const uint16_t* fast, unsigned fast_bits, const uint16_t* count, + const uint16_t* sym ) +{ + uint16_t entry = fast[b->bits & ((1u << fast_bits) - 1u)]; + if( entry != 0 ) + { + take( b, entry & 15u ); + return entry >> 4; + } + + // A code longer than the table (or no code): walk it canonically, one bit at a time + int code = 0, first = 0, index = 0; + uint32_t bits = b->bits; + unsigned length; + for( length = 1; length <= DMCOMP_MAX_BITS; length++ ) + { + int c; + code |= (int)(bits & 1u); + bits >>= 1; + c = count[length]; + if( code - c < first ) + { + take( b, length ); + return sym[index + (code - first)]; + } + index += c; + first += c; + first <<= 1; + code <<= 1; + } + return -1; +} + +// ===================================================================================================================== +// Blocks +// ===================================================================================================================== + +static int build_fixed( workspace_t* t ) +{ + uint8_t* lengths = t->lengths; + unsigned i; + for( i = 0; i < 144; i++ ) lengths[i] = 8; + for( ; i < 256; i++ ) lengths[i] = 9; + for( ; i < 280; i++ ) lengths[i] = 7; + for( ; i < 288; i++ ) lengths[i] = 8; + int r = huff_build( t->lit_fast, DMCOMP_LIT_FAST_BITS, t->lit_count, t->lit_sym, lengths, LITLEN_TABLE_SIZE ); + if( r != DMCOMP_OK ) + { + return r; + } + for( i = 0; i < DIST_TABLE_SIZE; i++ ) lengths[i] = 5; + return huff_build( t->dist_fast, DMCOMP_DIST_FAST_BITS, t->dist_count, t->dist_sym, lengths, DIST_TABLE_SIZE ); +} + +/** Reads the header of a dynamic block (RFC 1951, 3.2.7) and builds its tables */ +static int build_dynamic( bitreader_t* b, workspace_t* t ) +{ + uint8_t* lengths = t->lengths; + uint8_t* clen = t->clen; + uint16_t* cfast = t->cfast; + uint16_t* ccount = t->ccount; + uint16_t* csym = t->csym; + unsigned hlit, hdist, hclen, i, total; + int r; + + for( i = 0; i < DMCOMP_CLEN_SYMBOLS; i++ ) + { + clen[i] = 0; + } + + refill( b ); + hlit = take( b, 5 ) + 257; + hdist = take( b, 5 ) + 1; + hclen = take( b, 4 ) + 4; + if( overrun( b ) ) + { + return DMCOMP_ERR_TRUNCATED; + } + if( hlit > DMCOMP_LITLEN_SYMBOLS || hdist > DMCOMP_DIST_SYMBOLS ) + { + return DMCOMP_ERR_DATA; + } + + for( i = 0; i < hclen; i++ ) + { + refill( b ); + clen[dmcomp_clen_order[i]] = (uint8_t)take( b, 3 ); + } + if( overrun( b ) ) + { + return DMCOMP_ERR_TRUNCATED; + } + r = huff_build( cfast, CLEN_FAST_BITS, ccount, csym, clen, DMCOMP_CLEN_SYMBOLS ); + if( r != DMCOMP_OK ) + { + return r; + } + + // The lengths of the literal/length and distance codes are one run-length coded sequence + total = hlit + hdist; + for( i = 0; i < total; ) + { + int s; + unsigned repeat; + uint8_t value; + + refill( b ); + s = huff_decode( b, cfast, CLEN_FAST_BITS, ccount, csym ); + if( s < 0 ) + { + return overrun( b ) ? DMCOMP_ERR_TRUNCATED : DMCOMP_ERR_DATA; + } + if( s < 16 ) + { + lengths[i++] = (uint8_t)s; + continue; + } + if( s == 16 ) + { + if( i == 0 ) + { + return DMCOMP_ERR_DATA; + } + value = lengths[i - 1]; + repeat = 3 + take( b, 2 ); + } + else if( s == 17 ) + { + value = 0; + repeat = 3 + take( b, 3 ); + } + else + { + value = 0; + repeat = 11 + take( b, 7 ); + } + if( overrun( b ) ) + { + return DMCOMP_ERR_TRUNCATED; + } + if( i + repeat > total ) + { + return DMCOMP_ERR_DATA; + } + while( repeat-- > 0 ) + { + lengths[i++] = value; + } + } + if( lengths[256] == 0 ) + { + return DMCOMP_ERR_DATA; // No end of block code + } + + r = huff_build( t->lit_fast, DMCOMP_LIT_FAST_BITS, t->lit_count, t->lit_sym, lengths, hlit ); + if( r != DMCOMP_OK ) + { + return r; + } + return huff_build( t->dist_fast, DMCOMP_DIST_FAST_BITS, t->dist_count, t->dist_sym, lengths + hlit, hdist ); +} + +/** Decodes the symbols of a Huffman coded block up to its end of block code */ +static int inflate_codes( bitreader_t* b, const workspace_t* t, uint8_t* dst, uint8_t** pout, uint8_t* end ) +{ + uint8_t* out = *pout; + + for( ;; ) + { + int s; + + if( overrun( b ) ) + { + return DMCOMP_ERR_TRUNCATED; + } + refill( b ); + s = huff_decode( b, t->lit_fast, DMCOMP_LIT_FAST_BITS, t->lit_count, t->lit_sym ); + if( s < 0 ) + { + return overrun( b ) ? DMCOMP_ERR_TRUNCATED : DMCOMP_ERR_DATA; + } + if( s < 256 ) + { + if( out == end ) + { + return DMCOMP_ERR_OVERFLOW; + } + *out++ = (uint8_t)s; + continue; + } + if( s == 256 ) + { + break; + } + + // A match: length, then distance (at most 15 + 5 and 15 + 13 bits, so the buffer is filled in between) + unsigned length, dist; + int d; + + s -= 257; + if( s >= 29 ) + { + return DMCOMP_ERR_DATA; // Length codes 286 and 287 are not used + } + length = dmcomp_len_base[s] + take( b, dmcomp_len_extra[s] ); + + refill( b ); + d = huff_decode( b, t->dist_fast, DMCOMP_DIST_FAST_BITS, t->dist_count, t->dist_sym ); + if( d < 0 ) + { + return overrun( b ) ? DMCOMP_ERR_TRUNCATED : DMCOMP_ERR_DATA; + } + if( d >= DMCOMP_DIST_SYMBOLS ) + { + return DMCOMP_ERR_DATA; // Distance codes 30 and 31 are not used + } + refill( b ); + dist = dmcomp_dist_base[d] + take( b, dmcomp_dist_extra[d] ); + + if( overrun( b ) ) + { + return DMCOMP_ERR_TRUNCATED; + } + if( dist > (size_t)(out - dst) ) + { + return DMCOMP_ERR_DATA; // Before the start of the output + } + if( length > (size_t)(end - out) ) + { + return DMCOMP_ERR_OVERFLOW; + } + { + // The source may overlap the destination (dist < length repeats a pattern), so no memcpy + const uint8_t* from = out - dist; + while( length-- > 0 ) + { + *out++ = *from++; + } + } + } + + *pout = out; + return DMCOMP_OK; +} + +/** Copies a stored block (RFC 1951, 3.2.4) */ +static int inflate_stored( bitreader_t* b, uint8_t** pout, uint8_t* end ) +{ + unsigned length, inverted; + + // The rest of the byte is skipped + take( b, b->n & 7u ); + refill( b ); + length = take( b, 16 ); + inverted = take( b, 16 ); + if( overrun( b ) ) + { + return DMCOMP_ERR_TRUNCATED; + } + if( length != (~inverted & 0xFFFFu) ) + { + return DMCOMP_ERR_DATA; + } + + // The whole bytes in the buffer go back to the input + b->p -= (b->n - b->virt) / 8; + b->bits = 0; + b->n = 0; + b->virt = 0; + + if( length > (size_t)(b->end - b->p) ) + { + return DMCOMP_ERR_TRUNCATED; + } + if( length > (size_t)(end - *pout) ) + { + return DMCOMP_ERR_OVERFLOW; + } + if( length > 0 ) + { + memcpy( *pout, b->p, length ); + } + *pout += length; + b->p += length; + return DMCOMP_OK; +} + +// ===================================================================================================================== +// Public API +// ===================================================================================================================== + +size_t dmcomp_inflate_workspace_size( void ) +{ + return sizeof(workspace_t); +} + +int dmcomp_inflate_ws( const void* src, size_t src_size, void* dst, size_t dst_capacity, size_t* out_size, + void* workspace ) +{ + bitreader_t b; + uint8_t* out = (uint8_t*)dst; + uint8_t* end; + int last; + workspace_t* tables = (workspace_t*)workspace; + + // Checked before any pointer is moved: a NULL pointer is only valid for an empty buffer, and is not added to + if( out_size == NULL || (src == NULL && src_size != 0) || (dst == NULL && dst_capacity != 0) || + workspace == NULL || ((uintptr_t)workspace & (sizeof(uint16_t) - 1u)) != 0 ) + { + return DMCOMP_ERR_PARAM; + } + *out_size = 0; + + end = dst_capacity != 0 ? out + dst_capacity : out; + b.p = (const uint8_t*)src; + b.end = src_size != 0 ? b.p + src_size : b.p; + b.bits = 0; + b.n = 0; + b.virt = 0; + + do + { + unsigned type; + int r; + + refill( &b ); + last = (int)take( &b, 1 ); + type = take( &b, 2 ); + if( overrun( &b ) ) + { + return DMCOMP_ERR_TRUNCATED; + } + + if( type == 0 ) + { + r = inflate_stored( &b, &out, end ); + } + else + { + if( type == 1 ) + { + r = build_fixed( tables ); + } + else if( type == 2 ) + { + r = build_dynamic( &b, tables ); + } + else + { + return DMCOMP_ERR_DATA; // Reserved + } + if( r == DMCOMP_OK ) + { + r = inflate_codes( &b, tables, (uint8_t*)dst, &out, end ); + } + } + if( r != DMCOMP_OK ) + { + return r; + } + } + while( !last ); + + if( overrun( &b ) ) + { + return DMCOMP_ERR_TRUNCATED; + } + *out_size = (size_t)(out - (uint8_t*)dst); + return DMCOMP_OK; +} + +int dmcomp_inflate( const void* src, size_t src_size, void* dst, size_t dst_capacity, size_t* out_size ) +{ + workspace_t workspace; // A few kB of the stack: dmcomp_inflate_ws() puts it where the caller wants + + return dmcomp_inflate_ws( src, src_size, dst, dst_capacity, out_size, &workspace ); +} diff --git a/lib/third-party/dmcomp/dmcomp_strerror.c b/lib/third-party/dmcomp/dmcomp_strerror.c new file mode 100644 index 00000000..3c2d5d09 --- /dev/null +++ b/lib/third-party/dmcomp/dmcomp_strerror.c @@ -0,0 +1,19 @@ +/** + * @file dmcomp_strerror.c + * @brief Names of the results (a file of its own: a firmware that does not log them does not carry the strings) + */ +#include "dmcomp.h" + +const char* dmcomp_strerror( int result ) +{ + switch( result ) + { + case DMCOMP_OK: return "ok"; + case DMCOMP_ERR_PARAM: return "invalid argument"; + case DMCOMP_ERR_DATA: return "invalid compressed data"; + case DMCOMP_ERR_TRUNCATED: return "compressed data truncated"; + case DMCOMP_ERR_OVERFLOW: return "output buffer too small"; + case DMCOMP_ERR_MEMORY: return "out of memory"; + default: return "unknown error"; + } +} diff --git a/lib/third-party/dmcomp/dmcomp_tables.h b/lib/third-party/dmcomp/dmcomp_tables.h new file mode 100644 index 00000000..22ba870e --- /dev/null +++ b/lib/third-party/dmcomp/dmcomp_tables.h @@ -0,0 +1,52 @@ +/** + * @file dmcomp_tables.h + * @brief Constants of the DEFLATE format shared by the decoder and the encoder (RFC 1951, 3.2.5 - 3.2.7) + */ +#ifndef DMCOMP_TABLES_H +#define DMCOMP_TABLES_H + +#include + +#define DMCOMP_MAX_BITS 15 //!< Longest Huffman code of literals/lengths and distances +#define DMCOMP_MAX_CLEN_BITS 7 //!< Longest code of the code length alphabet +#define DMCOMP_LITLEN_SYMBOLS 286 //!< Literals 0..255, end of block 256, length codes 257..285 +#define DMCOMP_DIST_SYMBOLS 30 +#define DMCOMP_CLEN_SYMBOLS 19 +#define DMCOMP_MIN_MATCH 3 +#define DMCOMP_MAX_MATCH 258 + +/** Base length of the length codes 257..285 (index: code - 257) */ +static const uint16_t dmcomp_len_base[29] = +{ + 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, + 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258 +}; + +/** Extra bits of the length codes */ +static const uint8_t dmcomp_len_extra[29] = +{ + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, + 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0 +}; + +/** Base distance of the distance codes */ +static const uint16_t dmcomp_dist_base[30] = +{ + 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, + 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577 +}; + +/** Extra bits of the distance codes */ +static const uint8_t dmcomp_dist_extra[30] = +{ + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, + 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13 +}; + +/** Order in which the lengths of the code length alphabet are stored in a dynamic block header */ +static const uint8_t dmcomp_clen_order[DMCOMP_CLEN_SYMBOLS] = +{ + 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 +}; + +#endif /* DMCOMP_TABLES_H */ diff --git a/paths.cmake b/paths.cmake index 38ae3b17..adc7cb13 100644 --- a/paths.cmake +++ b/paths.cmake @@ -147,6 +147,9 @@ macro(dmod_setup_external_module) if (DMOD_USE_FASTLZ) target_link_libraries(dmod INTERFACE dmod_fastlz) endif() + if (DMOD_USE_DMCOMP) + target_link_libraries(dmod INTERFACE dmod_dmcomp) + endif() if (DMOD_SYSTEM) target_link_libraries(dmod INTERFACE dmod_system) elseif (DMOD_MODULE) diff --git a/scripts/install-sdk.cmake b/scripts/install-sdk.cmake index c158e124..0f149485 100644 --- a/scripts/install-sdk.cmake +++ b/scripts/install-sdk.cmake @@ -29,6 +29,10 @@ install(DIRECTORY "${DMOD_DIR}/lib/third-party/FastLZ" DESTINATION "${DMOD_SDK_INSTALL_DIR}/lib/third-party" COMPONENT development FILES_MATCHING PATTERN "*.c" PATTERN "*.h" PATTERN "CMakeLists.txt" PATTERN "LICENSE.MIT") +install(DIRECTORY "${DMOD_DIR}/lib/third-party/dmcomp" + DESTINATION "${DMOD_SDK_INSTALL_DIR}/lib/third-party" COMPONENT development + FILES_MATCHING PATTERN "*.c" PATTERN "*.h" PATTERN "CMakeLists.txt" + PATTERN "LICENSE.MIT") install(DIRECTORY "${DMOD_DIR}/scripts/" DESTINATION "${DMOD_SDK_INSTALL_DIR}/scripts" COMPONENT development FILES_MATCHING PATTERN "CMakeLists.txt" PATTERN "*.in" PATTERN "*.ld" @@ -44,5 +48,7 @@ install(FILES "${DMOD_DIR}/license.md" DESTINATION share/doc/dmod-dev RENAME copyright COMPONENT development) install(FILES "${DMOD_DIR}/lib/third-party/FastLZ/LICENSE.MIT" DESTINATION share/doc/dmod-dev RENAME copyright.FastLZ COMPONENT development) +install(FILES "${DMOD_DIR}/lib/third-party/dmcomp/LICENSE.MIT" + DESTINATION share/doc/dmod-dev RENAME copyright.dmcomp COMPONENT development) install(FILES "${DMOD_DIR}/docs/development-package.md" DESTINATION share/doc/dmod-dev COMPONENT development) diff --git a/scripts/system-tool.mk b/scripts/system-tool.mk index e7786845..2261c7ee 100644 --- a/scripts/system-tool.mk +++ b/scripts/system-tool.mk @@ -28,6 +28,9 @@ OBJ_OUTPUT_DIR = $(DMOD_OBJS_DIR)/$(PROJECT_NAME) ifeq ($(DMOD_USE_FASTLZ),ON) DMOD_LIBS += dmod_fastlz endif +ifeq ($(DMOD_USE_DMCOMP),ON) + DMOD_LIBS += dmod_dmcomp +endif # ----------------------------------------------------------------------------- # List of objects diff --git a/src/system/CMakeLists.txt b/src/system/CMakeLists.txt index e48fe771..e410f76f 100644 --- a/src/system/CMakeLists.txt +++ b/src/system/CMakeLists.txt @@ -62,6 +62,7 @@ target_link_libraries(${MODULE_NAME} dmod_inc dmod_common dmod_fastlz + dmod_dmcomp ) if (DMOD_USE_PTHREAD) diff --git a/src/system/Makefile b/src/system/Makefile index fe086b88..228c82e8 100644 --- a/src/system/Makefile +++ b/src/system/Makefile @@ -49,10 +49,12 @@ DMOD_INC_DIRS = $(DMOD_DIR)/src/system\ $(DMOD_DIR)/inc\ $(DMOD_BUILD_DIR)\ $(DMOD_DIR)/src/common\ - $(DMOD_DIR)/lib/third-party/FastLZ + $(DMOD_DIR)/lib/third-party/FastLZ\ + $(DMOD_DIR)/lib/third-party/dmcomp DMOD_LIBS = dmod_inc\ dmod_common\ dmod_fastlz\ + dmod_dmcomp\ pthread DMOD_GEN_HEADERS_IN = DMOD_DEFINITIONS = DMOD_CURRENT_ALLOCATOR="dmod_system" diff --git a/src/system/if/dmod_if_comp.c b/src/system/if/dmod_if_comp.c index d959459c..b197d289 100644 --- a/src/system/if/dmod_if_comp.c +++ b/src/system/if/dmod_if_comp.c @@ -35,13 +35,19 @@ #if DMOD_USE_FASTLZ # include "fastlz.h" #endif +#if DMOD_USE_DMCOMP +# include "dmcomp.h" +#endif //============================================================================== // GLOBAL VARIABLES //============================================================================== static const char* SupportedCompressions[] = { #if DMOD_USE_FASTLZ - "fastlz", + "fastlz", // The first one is the default of todmfc: it has to stay fastlz (the loaders that were built before deflate do not read the other) + #endif + #if DMOD_USE_DMCOMP + "deflate", // Raw DEFLATE (RFC 1951) by dmcomp: ~20 % smaller than fastlz, 1.7 kB of the decoder #endif }; static const size_t SupportedCompressionsCount = sizeof(SupportedCompressions) / sizeof(SupportedCompressions[0]); @@ -73,9 +79,15 @@ DMOD_INPUT_WEAK_API_DECLARATION(Dmod, 1.0, size_t, _Compression_GetMaxSize, ( co max_output_size = SrcSize + (SrcSize / 16) + 64 + 3; } #endif + #if DMOD_USE_DMCOMP && DMOD_USE_DMCOMP_ENCODER + else if( strcmp( Name, "deflate" ) == 0 ) + { + max_output_size = dmcomp_deflate_bound( SrcSize ); + } + #endif else { - DMOD_LOG_ERROR("Compression algorithm '%s' is not supported\n", Name); + DMOD_LOG_ERROR("Compression algorithm '%s' is not supported (or this build cannot compress with it)\n", Name); } return max_output_size; @@ -107,9 +119,21 @@ DMOD_INPUT_WEAK_API_DECLARATION(Dmod, 1.0, size_t, _Compression_Pack, ( const ch Ret = fastlz_compress_level(Level, Src, SrcSize, Dest ); } #endif + #if DMOD_USE_DMCOMP && DMOD_USE_DMCOMP_ENCODER + else if( strcmp( Name, "deflate" ) == 0 ) + { + // Level 0 (stored) .. 9 (the smallest); dmcomp takes a level out of the range as its default + int result = dmcomp_deflate( Src, SrcSize, Dest, DestSize, &Ret, Level ); + if( result != DMCOMP_OK ) + { + DMOD_LOG_ERROR("Cannot compress with deflate: %s\n", dmcomp_strerror( result )); + Ret = 0; + } + } + #endif else { - DMOD_LOG_ERROR("Compression algorithm '%s' is not supported\n", Name); + DMOD_LOG_ERROR("Compression algorithm '%s' is not supported (or this build cannot compress with it)\n", Name); } return Ret; @@ -140,7 +164,34 @@ DMOD_INPUT_WEAK_API_DECLARATION(Dmod, 1.0, size_t, _Compression_Unpack, ( const Ret = fastlz_decompress( Src, SrcSize, Dest, DestSize ); } #endif + #if DMOD_USE_DMCOMP + #if DMOD_USE_FASTLZ + else + #endif + if( strcmp( Name, "deflate" ) == 0 ) + { + // The tables of the decoder (~3 kB) come from the heap, not from the stack: a module is loaded from whatever + // thread asks for it, and some of them have 4 kB of the stack + void* workspace = Dmod_Malloc( dmcomp_inflate_workspace_size() ); + if( workspace == NULL ) + { + DMOD_LOG_ERROR("Cannot decompress with deflate: out of memory\n"); + } + else + { + int result = dmcomp_inflate_ws( Src, SrcSize, Dest, DestSize, &Ret, workspace ); + if( result != DMCOMP_OK ) + { + DMOD_LOG_ERROR("Cannot decompress with deflate: %s\n", dmcomp_strerror( result )); + Ret = 0; + } + Dmod_Free( workspace ); + } + } + #endif + #if DMOD_USE_FASTLZ || DMOD_USE_DMCOMP else + #endif { DMOD_LOG_ERROR("Compression algorithm '%s' is not supported\n", Name); } diff --git a/templates/system/dmod-cfg.cmake b/templates/system/dmod-cfg.cmake index 646d6cf6..226e22d4 100644 --- a/templates/system/dmod-cfg.cmake +++ b/templates/system/dmod-cfg.cmake @@ -31,6 +31,19 @@ set(DMOD_USE_REALLOC ON CACHE BOOL "Enable to use the realloc function") # Use FastLZ compression library set(DMOD_USE_FASTLZ ON CACHE BOOL "Enable to use the FastLZ compression library") +# Use the dmcomp library: the "deflate" compression method (the decoder) +set(DMOD_USE_DMCOMP ON CACHE BOOL "Enable to use the dmcomp library (the deflate compression method)") + +# Build the deflate encoder (Dmod_Compression_Pack) of the dmcomp library: for the host tools. It allocates its work memory +# (~330 kB) with malloc, so by default it is built when DMOD_USE_STDLIB is (a firmware has no malloc of the standard +# library and nothing to compress for - it gets the decoder only) +if(DMOD_USE_STDLIB) + set(DMOD_USE_DMCOMP_ENCODER_DEFAULT ON) +else() + set(DMOD_USE_DMCOMP_ENCODER_DEFAULT OFF) +endif() +set(DMOD_USE_DMCOMP_ENCODER ${DMOD_USE_DMCOMP_ENCODER_DEFAULT} CACHE BOOL "Enable to build the deflate encoder of the dmcomp library (it needs malloc and ~330 kB of the heap)") + # Maximum number of modules set(DMOD_MAX_MODULES 30 CACHE STRING "Maximum number of modules") diff --git a/templates/system/dmod-cfg.mk b/templates/system/dmod-cfg.mk index a36b0bdb..c449b4b6 100644 --- a/templates/system/dmod-cfg.mk +++ b/templates/system/dmod-cfg.mk @@ -38,6 +38,10 @@ DMOD_USE_REALLOC=ON # Use FastLZ compression library DMOD_USE_FASTLZ=ON +# Use the dmcomp library (the deflate compression method) and its encoder (for the host tools) +DMOD_USE_DMCOMP=ON +DMOD_USE_DMCOMP_ENCODER=ON + # Maximum number of modules DMOD_MAX_MODULES=30 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 97de2c8e..5cc3a85b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -38,7 +38,7 @@ foreach(test_src ${TEST_SOURCES}) get_filename_component(test_name ${test_src} NAME_WE) add_executable(${test_name} ${test_src} ut_main.cpp) dmod_set_current_allocator(${test_name}) - target_link_libraries(${test_name} gtest gtest_main gmock dmod_system dmod_common dmod_inc dmod_fastlz dmod_version dmod_manifest dmod_dependencies dmod_resource) + target_link_libraries(${test_name} gtest gtest_main gmock dmod_system dmod_common dmod_inc dmod_fastlz dmod_dmcomp dmod_version dmod_manifest dmod_dependencies dmod_resource) add_test(NAME ${test_name} COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${test_name}) list(APPEND TEST_FILES ${test_name}) diff --git a/tests/system/if/CMakeLists.txt b/tests/system/if/CMakeLists.txt index 4d9427c7..9fd4ef9f 100644 --- a/tests/system/if/CMakeLists.txt +++ b/tests/system/if/CMakeLists.txt @@ -18,5 +18,6 @@ set(TEST_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/tests_dmod_str_merge.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tests_dmod_module_exists.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tests_dmod_reboot.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests_dmod_compression.cpp PARENT_SCOPE ) diff --git a/tests/system/if/tests_dmod_compression.cpp b/tests/system/if/tests_dmod_compression.cpp new file mode 100644 index 00000000..039ae6a6 --- /dev/null +++ b/tests/system/if/tests_dmod_compression.cpp @@ -0,0 +1,186 @@ +#define DMOD_PRIVATE +#include +#include +#include +#include +#include +#include "dmod_sal.h" + +// The heap of this test: counts what the compression takes and gives back, and can refuse to give anything +static int g_LiveAllocations = 0; +static bool g_FailAllocations = false; + +extern "C" +{ + void* Dmod_MallocEx(size_t Size, const char* ModuleName) + { + (void)ModuleName; + if( g_FailAllocations ) + { + return nullptr; + } + void* ptr = malloc(Size); + if( ptr != nullptr ) + { + g_LiveAllocations++; + } + return ptr; + } + + void Dmod_FreeEx(void* ptr, bool Concatenate) + { + (void)Concatenate; + if( ptr != nullptr ) + { + g_LiveAllocations--; + free(ptr); + } + } +} + +class DmodCompressionTest : public ::testing::Test +{ +protected: + void SetUp() override + { + g_LiveAllocations = 0; + g_FailAllocations = false; + } + + // Something that compresses like a module does: structured, repetitive, with some noise + static std::vector Sample(size_t size) + { + static const char* words[] = { "module", "loader", "flash", "driver", "device", "memory", "thread", "mount" }; + std::vector data; + uint32_t state = 12345; + + while( data.size() < size ) + { + state = state * 1103515245u + 12345u; + const char* word = words[(state >> 16) % 8]; + data.insert(data.end(), word, word + strlen(word)); + data.push_back((state >> 8) % 11 == 0 ? '\n' : ' '); + if( (state >> 20) % 5 == 0 ) + { + data.push_back((uint8_t)(state >> 3)); + } + } + data.resize(size); + return data; + } + + static size_t Pack(const char* name, int level, const std::vector& data, std::vector& packed) + { + size_t max = Dmod_Compression_GetMaxSize(name, level, data.size()); + packed.assign(max, 0); + size_t size = Dmod_Compression_Pack(name, level, packed.data(), packed.size(), data.data(), data.size()); + packed.resize(size); + return size; + } +}; + +TEST_F(DmodCompressionTest, SupportedMethods) +{ + EXPECT_TRUE(Dmod_Compression_IsSupported("fastlz")); + EXPECT_TRUE(Dmod_Compression_IsSupported("deflate")); + EXPECT_FALSE(Dmod_Compression_IsSupported("lzma")); + + // fastlz stays the first: it is the default of todmfc, and what the loaders without deflate can read + ASSERT_NE(Dmod_Compression_GetNextSupported(nullptr), nullptr); + EXPECT_STREQ(Dmod_Compression_GetNextSupported(nullptr), "fastlz"); + EXPECT_STREQ(Dmod_Compression_GetNextSupported("fastlz"), "deflate"); + EXPECT_EQ(Dmod_Compression_GetNextSupported("deflate"), nullptr); +} + +TEST_F(DmodCompressionTest, DeflateRoundTripOnEveryLevel) +{ + const std::vector data = Sample(20000); + + for( int level = 0; level <= 9; level++ ) + { + std::vector packed; + size_t max = Dmod_Compression_GetMaxSize("deflate", level, data.size()); + + ASSERT_GE(max, data.size()) << "level " << level; + ASSERT_GT(Pack("deflate", level, data, packed), 0u) << "level " << level; + EXPECT_LE(packed.size(), max); + if( level > 0 ) + { + EXPECT_LT(packed.size(), data.size() / 2) << "level " << level; + } + + std::vector unpacked(data.size()); + EXPECT_EQ(Dmod_Compression_Unpack("deflate", unpacked.data(), unpacked.size(), packed.data(), packed.size()), + data.size()) << "level " << level; + EXPECT_EQ(unpacked, data) << "level " << level; + } + // The tables of the decoder come from the heap and go back to it + EXPECT_EQ(g_LiveAllocations, 0); +} + +TEST_F(DmodCompressionTest, DeflateIsSmallerThanFastlz) +{ + const std::vector data = Sample(60000); + std::vector deflate, fastlz; + + ASSERT_GT(Pack("deflate", 9, data, deflate), 0u); + ASSERT_GT(Pack("fastlz", 2, data, fastlz), 0u); + EXPECT_LT(deflate.size(), fastlz.size()); +} + +TEST_F(DmodCompressionTest, FastlzStillWorks) +{ + const std::vector data = Sample(10000); + std::vector packed; + + ASSERT_GT(Pack("fastlz", 2, data, packed), 0u); + std::vector unpacked(data.size()); + EXPECT_EQ(Dmod_Compression_Unpack("fastlz", unpacked.data(), unpacked.size(), packed.data(), packed.size()), data.size()); + EXPECT_EQ(unpacked, data); +} + +TEST_F(DmodCompressionTest, DeflateRefusesWhatIsNotDeflate) +{ + const std::vector data = Sample(5000); + std::vector packed; + std::vector out(data.size()); + + ASSERT_GT(Pack("deflate", 6, data, packed), 0u); + + // Garbage + std::vector garbage(300); + for( size_t i = 0; i < garbage.size(); i++ ) garbage[i] = (uint8_t)(i * 7 + 3); + EXPECT_EQ(Dmod_Compression_Unpack("deflate", out.data(), out.size(), garbage.data(), garbage.size()), 0u); + + // Cut short + EXPECT_EQ(Dmod_Compression_Unpack("deflate", out.data(), out.size(), packed.data(), packed.size() / 2), 0u); + + // An output buffer that is too small + EXPECT_EQ(Dmod_Compression_Unpack("deflate", out.data(), out.size() - 1, packed.data(), packed.size()), 0u); + + EXPECT_EQ(g_LiveAllocations, 0); +} + +TEST_F(DmodCompressionTest, DeflateDecoderWithoutMemory) +{ + const std::vector data = Sample(5000); + std::vector packed; + std::vector out(data.size()); + + ASSERT_GT(Pack("deflate", 6, data, packed), 0u); + + g_FailAllocations = true; + EXPECT_EQ(Dmod_Compression_Unpack("deflate", out.data(), out.size(), packed.data(), packed.size()), 0u); + g_FailAllocations = false; + EXPECT_EQ(Dmod_Compression_Unpack("deflate", out.data(), out.size(), packed.data(), packed.size()), data.size()); +} + +TEST_F(DmodCompressionTest, NothingToCompress) +{ + uint8_t out[64]; + uint8_t in = 0; + + EXPECT_EQ(Dmod_Compression_Pack("deflate", 6, out, sizeof(out), &in, 0), 0u); + EXPECT_EQ(Dmod_Compression_Unpack("deflate", out, sizeof(out), &in, 0), 0u); + EXPECT_EQ(Dmod_Compression_GetMaxSize("deflate", 6, 0), 0u); +} diff --git a/tests/system/public/CMakeLists.txt b/tests/system/public/CMakeLists.txt index 3644572b..adcae860 100644 --- a/tests/system/public/CMakeLists.txt +++ b/tests/system/public/CMakeLists.txt @@ -1,6 +1,7 @@ set(TEST_SOURCES ${TEST_SOURCES} ${CMAKE_CURRENT_SOURCE_DIR}/tests_dmod_dmfc_api.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests_dmod_dmfc_deflate.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tests_dmod_getname.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tests_dmod_arch.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tests_dmod_findmatch.cpp diff --git a/tests/system/public/tests_dmod_dmfc_deflate.cpp b/tests/system/public/tests_dmod_dmfc_deflate.cpp new file mode 100644 index 00000000..93113896 --- /dev/null +++ b/tests/system/public/tests_dmod_dmfc_deflate.cpp @@ -0,0 +1,76 @@ +#define DMOD_PRIVATE +#include +#include +#include +#include "dmod.h" +#include "dmod_defs.h" +#include "dmod_types.h" + +// DMF -> DMFC -> DMF with every compression method. A DMF for the conversion only has to start with the module header +// (the signature and the name); the rest is the module's code, which this is not - but it is as compressible. +static std::vector FakeModule(size_t size) +{ + std::vector dmf(size); + uint32_t state = 777; + + for( size_t i = 0; i < size; i++ ) + { + state = state * 1664525u + 1013904223u; + // Words of a few kinds of values, like the instructions and the tables of a real module + dmf[i] = (i % 4 == 3) ? 0xE0 : (uint8_t)((state >> 24) % 24); + } + Dmod_ModuleHeader_t* header = reinterpret_cast(dmf.data()); + memset(header, 0, sizeof(*header)); + header->Signature = DMOD_HEADER_SIGNATURE; + strncpy(header->Name, "fakemodule", sizeof(header->Name) - 1); + return dmf; +} + +class DmodDmfcMethodsTest : public ::testing::TestWithParam +{ +}; + +TEST_P(DmodDmfcMethodsTest, ToDmfcAndBack) +{ + const char* method = GetParam(); + const std::vector dmf = FakeModule(12000); + void* dmfc = nullptr; + size_t dmfcSize = 0; + + ASSERT_TRUE(Dmod_ToDMFC(method, 2, dmf.data(), dmf.size(), &dmfc, &dmfcSize)); + ASSERT_NE(dmfc, nullptr); + EXPECT_LT(dmfcSize, dmf.size()); + EXPECT_TRUE(Dmod_IsDMFC(dmfc, dmfcSize)); + + // The method is named in the header: the loader decompresses with it + const Dmod_DmfcHeader_t* header = static_cast(dmfc); + EXPECT_STREQ(header->Compression, method); + EXPECT_EQ(header->OriginalSize, dmf.size()); + + void* back = nullptr; + size_t backSize = 0; + ASSERT_TRUE(Dmod_FromDMFC(dmfc, dmfcSize, &back, &backSize)); + ASSERT_EQ(backSize, dmf.size()); + EXPECT_EQ(memcmp(back, dmf.data(), dmf.size()), 0); + + Dmod_Free(back); + Dmod_Free(dmfc); +} + +// A DMFC with the compressed data cut short: deflate notices (the stream ends before its last block), so the loader +// does not get a half of a module. (fastlz gives what it has decoded, and Dmod_FromDMFC only warns about the size.) +TEST(DmodDmfcDeflateTest, DamagedDmfcIsRefused) +{ + const std::vector dmf = FakeModule(12000); + void* dmfc = nullptr; + size_t dmfcSize = 0; + + ASSERT_TRUE(Dmod_ToDMFC("deflate", 2, dmf.data(), dmf.size(), &dmfc, &dmfcSize)); + + void* back = nullptr; + size_t backSize = 0; + EXPECT_FALSE(Dmod_FromDMFC(dmfc, dmfcSize - dmfcSize / 3, &back, &backSize)); + Dmod_Free(dmfc); +} + +INSTANTIATE_TEST_SUITE_P(Methods, DmodDmfcMethodsTest, ::testing::Values("fastlz", "deflate")); diff --git a/tools/system/todmd/dmod-cfg.mk b/tools/system/todmd/dmod-cfg.mk index 6b9df1f4..cd6e22f2 100644 --- a/tools/system/todmd/dmod-cfg.mk +++ b/tools/system/todmd/dmod-cfg.mk @@ -12,6 +12,8 @@ DMOD_USE_STDIO=ON DMOD_USE_ASSERT=ON DMOD_USE_PTHREAD=ON DMOD_USE_FASTLZ=ON +DMOD_USE_DMCOMP=ON +DMOD_USE_DMCOMP_ENCODER=ON DMOD_USE_MMAN=ON DMOD_USE_ALIGNED_ALLOC=ON DMOD_USE_ALIGNED_MALLOC_MOCK=OFF diff --git a/tools/system/todmfc/README.md b/tools/system/todmfc/README.md index 518f7f6b..44c6a9b7 100644 --- a/tools/system/todmfc/README.md +++ b/tools/system/todmfc/README.md @@ -7,7 +7,7 @@ ## Features - **Compression of DMF files**: Converts standard DMF files to compressed DMFC format -- **Multiple compression methods**: Supports various compression algorithms (currently: fastlz) +- **Multiple compression methods**: Supports various compression algorithms (currently: fastlz and deflate) - **Configurable compression levels**: Allows tuning between compression ratio and speed - **Size comparison**: Displays original and compressed file sizes with compression ratio - **Simple interface**: Easy-to-use command-line tool @@ -22,6 +22,13 @@ todmfc path/to/file.dmf path/to/output.dmfc This will compress the DMF file using the default compression method (fastlz) and level (2). +The `deflate` method (RFC 1951, [dmcomp](../../../lib/third-party/dmcomp)) makes the files ~20 % smaller than `fastlz`; its decoder is 1.7 kB of code. A loader has to know it, so the modules that older firmware has to load stay `fastlz`: + +```bash +todmfc path/to/file.dmf path/to/output.dmfc deflate # level 9 +todmfc path/to/file.dmf path/to/output.dmfc deflate 6 +``` + ### Advanced Usage ```bash @@ -32,7 +39,7 @@ todmfc path/to/file.dmf path/to/output.dmfc [compression_method] [level] - `path/to/file.dmf` - Input DMF file to compress - `path/to/output.dmfc` - Output DMFC file path - `[compression_method]` - (Optional) Compression algorithm to use (default: fastlz) -- `[level]` - (Optional) Compression level (default: 2) +- `[level]` - (Optional) Compression level: fastlz 1 or 2 (default: 2), deflate 0 (stored) .. 9 (default: 9) ### Options diff --git a/tools/system/todmfc/dmod-cfg.mk b/tools/system/todmfc/dmod-cfg.mk index b5fedbc1..ceffbf63 100644 --- a/tools/system/todmfc/dmod-cfg.mk +++ b/tools/system/todmfc/dmod-cfg.mk @@ -12,6 +12,8 @@ DMOD_USE_STDIO=ON DMOD_USE_ASSERT=ON DMOD_USE_PTHREAD=ON DMOD_USE_FASTLZ=ON +DMOD_USE_DMCOMP=ON +DMOD_USE_DMCOMP_ENCODER=ON DMOD_USE_MMAN=ON DMOD_USE_ALIGNED_ALLOC=ON DMOD_USE_ALIGNED_MALLOC_MOCK=OFF diff --git a/tools/system/todmfc/main.c b/tools/system/todmfc/main.c index 62e54229..2c5dfeb6 100644 --- a/tools/system/todmfc/main.c +++ b/tools/system/todmfc/main.c @@ -43,7 +43,9 @@ void PrintHelp( const char* AppName ) printf(" -v, --version Print version information\n"); printf(" [compression_method] (default: %s) Compression method method to be used. ", Dmod_Compression_GetNextSupported(NULL)); PrintSupportedCompressionMethods(); - printf(" [level] (default: 2) Compression level - depends on the [compression_method]\n"); + printf(" [level] Compression level - depends on the [compression_method]:\n"); + printf(" fastlz: 1 (faster) or 2 (smaller, the default)\n"); + printf(" deflate: 0 (stored) .. 9 (the smallest, the default)\n"); } // ----------------------------------------- @@ -96,7 +98,8 @@ int main( int argc, char *argv[] ) } } - int level = 2; + // The default level depends on the method: fastlz has two (1, 2), deflate ten (0 - 9) + int level = strcmp( compression, "deflate" ) == 0 ? 9 : 2; if( argc >= 5 ) { level = atoi( argv[4] ); diff --git a/tools/system/todmm/dmod-cfg.mk b/tools/system/todmm/dmod-cfg.mk index 6b9df1f4..cd6e22f2 100644 --- a/tools/system/todmm/dmod-cfg.mk +++ b/tools/system/todmm/dmod-cfg.mk @@ -12,6 +12,8 @@ DMOD_USE_STDIO=ON DMOD_USE_ASSERT=ON DMOD_USE_PTHREAD=ON DMOD_USE_FASTLZ=ON +DMOD_USE_DMCOMP=ON +DMOD_USE_DMCOMP_ENCODER=ON DMOD_USE_MMAN=ON DMOD_USE_ALIGNED_ALLOC=ON DMOD_USE_ALIGNED_MALLOC_MOCK=OFF From 31cc4e23b173daea1b2ac8a5b9e99371c416edb7 Mon Sep 17 00:00:00 2001 From: Jax Calder Date: Wed, 28 Oct 2026 14:00:41 +0000 Subject: [PATCH 2/4] Firmware SAL: Dmod_FirmwareReady, Dmod_WritePersistency, Dmod_ReadPersistency A platform with more than one firmware image (banks, with a boot loader that picks one) needs two things from the firmware, and the firmware needs them from the platform: * Dmod_FirmwareReady(): the firmware says that it has started properly. A firmware that is on trial stays only if it does; otherwise the platform falls back to the previous one after a reset. What "properly" means is the firmware's own decision (a console, the network, a service ...), so it is the firmware that asks. * Dmod_WritePersistency(Id, Data, Size) / Dmod_ReadPersistency(Id, Buffer, BufferSize): data that survives a reset and an update of the firmware (the settings, the secrets), kept apart from the images. An item is found by its Id; writing 0 bytes removes it; the read returns the size of the whole item (ask with a NULL buffer for the size). Like Dmod_Reboot, these are implemented by the platform; the default (weak) implementations return -ENOSYS and do nothing. Added now, together with the deflate method, so that the tools image has to be built once. Co-Authored-By: Claude Sonnet 5.5 --- inc/dmod_sal.h | 66 +++++++++++++++++++++++++ src/system/CMakeLists.txt | 1 + src/system/Makefile | 1 + src/system/if/dmod_if_firmware.c | 29 +++++++++++ tests/system/if/CMakeLists.txt | 1 + tests/system/if/tests_dmod_firmware.cpp | 34 +++++++++++++ 6 files changed, 132 insertions(+) create mode 100644 src/system/if/dmod_if_firmware.c create mode 100644 tests/system/if/tests_dmod_firmware.cpp diff --git a/inc/dmod_sal.h b/inc/dmod_sal.h index 7097502d..1e9f1560 100644 --- a/inc/dmod_sal.h +++ b/inc/dmod_sal.h @@ -677,6 +677,72 @@ DMOD_BUILTIN_API(Dmod, 1.0, int, _Reboot, ( void ) ); //! @} +/** + * @defgroup DMOD_SAL_FIRMWARE Firmware Interface + * @ingroup DMOD_SAL + * + * What the running firmware tells the platform, and what the platform keeps for it between the runs. + * + * A platform that has more than one firmware image (banks) starts a new one on trial: it stays only if it says that it + * runs properly (Dmod_FirmwareReady()); if it does not, the platform goes back to the previous one after a reset. The + * persistent data (Dmod_WritePersistency(), Dmod_ReadPersistency()) is kept apart from the images, so that an update of + * the firmware - or a firmware that did not start - does not touch it. + * + * All of the functions are implemented by the platform (the boot loader of the board, for instance); the default + * (weak) implementations of the host and of the platforms without them return -ENOSYS and have no side effects. + * @{ + */ + +/** + * @brief Tell the platform that the running firmware has started properly and may stay. + * + * What "properly" means is up to the firmware, so it is the firmware that calls this - when the console is there, the + * file system is mounted, the network is up, or whatever its project needs to be able to be updated again. A platform + * with a firmware on trial keeps it from then on; calling it again, or on a firmware that is not on trial, is harmless. + * Until it is called a platform may restart the system (a watchdog) and fall back to the previous firmware. + * + * @return 0 on success, a negative errno value otherwise (-ENOSYS: the platform has nothing to confirm) + */ +DMOD_BUILTIN_API(Dmod, 1.0, int, _FirmwareReady, ( void ) ); + +/** + * @brief Store data that survives a reset and an update of the firmware. + * + * The item @p Id is replaced with the @p Size bytes of @p Data. @p Data may be NULL only when @p Size is 0, which + * removes the item. The platform decides the sizes, and the number of the items, that it can keep (-ENOSPC when it + * cannot). + * + * Writing may erase and program a non-volatile memory: it can take long (hundreds of milliseconds for a sector of a + * flash) and, where the code runs from the same memory, stop the whole system meanwhile. Do not do it often, nor from + * a time critical thread; the data is for the settings, not for the logs. + * + * @param Id Identifier of the item, chosen by the caller (the same one reads it back) + * @param Data The data + * @param Size Size of the data in bytes + * + * @return 0 on success, a negative errno value otherwise (-EINVAL: wrong arguments, -ENOSPC: no room, + * -EIO: the memory failed, -ENOSYS: the platform keeps nothing) + */ +DMOD_BUILTIN_API(Dmod, 1.0, int, _WritePersistency, ( uint32_t Id, const void* Data, size_t Size ) ); + +/** + * @brief Read data stored with Dmod_WritePersistency(). + * + * Copies at most @p BufferSize bytes of the item @p Id. The result is the size of the whole item, which may be more + * than @p BufferSize (then the rest was not copied): call it with a NULL @p Buffer and 0 as @p BufferSize to learn the + * size first. + * + * @param Id Identifier of the item + * @param Buffer Where to copy the data; may be NULL when @p BufferSize is 0 + * @param BufferSize Size of the buffer in bytes + * + * @return Size of the item in bytes (0 or more), or a negative errno value (-ENOENT: there is no such item, + * -EINVAL: wrong arguments, -EIO: the memory failed, -ENOSYS: the platform keeps nothing) + */ +DMOD_BUILTIN_API(Dmod, 1.0, int, _ReadPersistency, ( uint32_t Id, void* Buffer, size_t BufferSize ) ); + +//! @} + #ifdef __cplusplus } #endif diff --git a/src/system/CMakeLists.txt b/src/system/CMakeLists.txt index e410f76f..56e53ece 100644 --- a/src/system/CMakeLists.txt +++ b/src/system/CMakeLists.txt @@ -30,6 +30,7 @@ set(SYSTEM_SOURCES if/dmod_if_input.c if/dmod_if_proc.c if/dmod_if_power.c + if/dmod_if_firmware.c if/dmod_if_time.c Dmod.cpp ) diff --git a/src/system/Makefile b/src/system/Makefile index 228c82e8..a4e1f62d 100644 --- a/src/system/Makefile +++ b/src/system/Makefile @@ -42,6 +42,7 @@ DMOD_SOURCES=private/dmod_vars.c\ if/dmod_if_input.c\ if/dmod_if_proc.c\ if/dmod_if_power.c\ + if/dmod_if_firmware.c\ if/dmod_if_time.c\ Dmod.cpp\ if/dmod_prf.c diff --git a/src/system/if/dmod_if_firmware.c b/src/system/if/dmod_if_firmware.c new file mode 100644 index 00000000..9faea7f4 --- /dev/null +++ b/src/system/if/dmod_if_firmware.c @@ -0,0 +1,29 @@ +/** + * @brief Platform-overridable firmware interface: the confirmation of a firmware on trial and the persistent data. + * + * The platform with a boot loader that keeps more than one firmware (see DMOD_SAL_FIRMWARE in dmod_sal.h) overrides + * these; everywhere else (the host included) there is nothing to confirm and nothing is kept. + */ +#include "dmod.h" +#include + +DMOD_INPUT_WEAK_API_DECLARATION(Dmod, 1.0, int, _FirmwareReady, ( void )) +{ + return -ENOSYS; +} + +DMOD_INPUT_WEAK_API_DECLARATION(Dmod, 1.0, int, _WritePersistency, ( uint32_t Id, const void* Data, size_t Size )) +{ + (void)Id; + (void)Data; + (void)Size; + return -ENOSYS; +} + +DMOD_INPUT_WEAK_API_DECLARATION(Dmod, 1.0, int, _ReadPersistency, ( uint32_t Id, void* Buffer, size_t BufferSize )) +{ + (void)Id; + (void)Buffer; + (void)BufferSize; + return -ENOSYS; +} diff --git a/tests/system/if/CMakeLists.txt b/tests/system/if/CMakeLists.txt index 9fd4ef9f..e03362ff 100644 --- a/tests/system/if/CMakeLists.txt +++ b/tests/system/if/CMakeLists.txt @@ -18,6 +18,7 @@ set(TEST_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/tests_dmod_str_merge.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tests_dmod_module_exists.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tests_dmod_reboot.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests_dmod_firmware.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tests_dmod_compression.cpp PARENT_SCOPE ) diff --git a/tests/system/if/tests_dmod_firmware.cpp b/tests/system/if/tests_dmod_firmware.cpp new file mode 100644 index 00000000..104fc460 --- /dev/null +++ b/tests/system/if/tests_dmod_firmware.cpp @@ -0,0 +1,34 @@ +#define DMOD_PRIVATE +#include +#include +#include "dmod_sal.h" + +// The host has no boot loader and no persistent memory: the default (weak) implementations say so and do nothing. +// A platform overrides them (dmod-boot does, with the selector in the first sector of the flash). + +TEST(DmodFirmwareTest, FirmwareReadyIsUnsupportedByDefault) +{ + EXPECT_EQ(Dmod_FirmwareReady(), -ENOSYS); + // Repeated confirmations must be harmless as well. + EXPECT_EQ(Dmod_FirmwareReady(), -ENOSYS); +} + +TEST(DmodFirmwareTest, WritePersistencyIsUnsupportedByDefault) +{ + const char data[] = "settings"; + + EXPECT_EQ(Dmod_WritePersistency(1, data, sizeof(data)), -ENOSYS); + // Removing an item is a write of nothing + EXPECT_EQ(Dmod_WritePersistency(1, nullptr, 0), -ENOSYS); +} + +TEST(DmodFirmwareTest, ReadPersistencyIsUnsupportedByDefault) +{ + char buffer[16] = { 'x' }; + + EXPECT_EQ(Dmod_ReadPersistency(1, buffer, sizeof(buffer)), -ENOSYS); + // Asking for the size only + EXPECT_EQ(Dmod_ReadPersistency(1, nullptr, 0), -ENOSYS); + // Nothing was copied + EXPECT_EQ(buffer[0], 'x'); +} From 7bea18f142e144a6e011857b8bb6eb1e540029b5 Mon Sep 17 00:00:00 2001 From: Jax Calder Date: Wed, 28 Oct 2026 14:29:00 +0000 Subject: [PATCH 3/4] dmcomp is a git submodule, not a copy lib/third-party/dmcomp is the submodule choco-technologies/dmcomp (7314bf2) instead of copied files, so the library has one place to be changed and tested. What builds it is lib/third-party/dmod_dmcomp (the CMakeLists.txt, and the Makefile that it writes; there are no sources in it): the same library dmod_dmcomp as before, for the CMake and the Make based builds. The checkouts of the CI workflows take the submodules, the script that builds the Docker image initializes them (the Dockerfile copies the directory), and the SDK installs the sources of the submodule where the build glue finds them. Without the submodule the configuration says what to do (git submodule update --init --recursive). Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 8 + .github/workflows/debian-package.yml | 2 + .gitmodules | 3 + Docker/build.sh | 10 + cmake/runtime/CMakeLists.txt | 2 +- lib/third-party/CMakeLists.txt | 2 +- lib/third-party/Makefile | 2 +- lib/third-party/dmcomp | 1 + lib/third-party/dmcomp/CMakeLists.txt | 26 - lib/third-party/dmcomp/LICENSE.MIT | 21 - lib/third-party/dmcomp/README.md | 11 - lib/third-party/dmcomp/dmcomp.h | 118 --- lib/third-party/dmcomp/dmcomp_deflate.c | 781 ------------------ lib/third-party/dmcomp/dmcomp_inflate.c | 571 ------------- lib/third-party/dmcomp/dmcomp_strerror.c | 19 - lib/third-party/dmcomp/dmcomp_tables.h | 52 -- lib/third-party/dmod_dmcomp/CMakeLists.txt | 36 + .../{dmcomp => dmod_dmcomp}/Makefile | 9 +- lib/third-party/dmod_dmcomp/README.md | 8 + scripts/install-sdk.cmake | 10 +- src/system/Makefile | 2 +- 21 files changed, 83 insertions(+), 1611 deletions(-) create mode 100644 .gitmodules create mode 160000 lib/third-party/dmcomp delete mode 100644 lib/third-party/dmcomp/CMakeLists.txt delete mode 100644 lib/third-party/dmcomp/LICENSE.MIT delete mode 100644 lib/third-party/dmcomp/README.md delete mode 100644 lib/third-party/dmcomp/dmcomp.h delete mode 100644 lib/third-party/dmcomp/dmcomp_deflate.c delete mode 100644 lib/third-party/dmcomp/dmcomp_inflate.c delete mode 100644 lib/third-party/dmcomp/dmcomp_strerror.c delete mode 100644 lib/third-party/dmcomp/dmcomp_tables.h create mode 100644 lib/third-party/dmod_dmcomp/CMakeLists.txt rename lib/third-party/{dmcomp => dmod_dmcomp}/Makefile (81%) create mode 100644 lib/third-party/dmod_dmcomp/README.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eaae1b5f..9d35848b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + with: + submodules: recursive - name: Clean build directory run: rm -rf build @@ -190,6 +192,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + with: + submodules: recursive - name: Clean build directory run: rm -rf build @@ -281,6 +285,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + with: + submodules: recursive - name: Build dmod in MODULE mode run: | @@ -331,6 +337,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + with: + submodules: recursive - name: Build dmod with Make in MODULE mode run: make DMOD_MODE=DMOD_MODULE diff --git a/.github/workflows/debian-package.yml b/.github/workflows/debian-package.yml index b29d89fc..79265045 100644 --- a/.github/workflows/debian-package.yml +++ b/.github/workflows/debian-package.yml @@ -14,6 +14,8 @@ jobs: examples: ['OFF', 'ON'] steps: - uses: actions/checkout@v4 + with: + submodules: recursive - name: Install build dependencies run: sudo apt-get update && sudo apt-get install -y build-essential cmake dpkg-dev libcurl4-openssl-dev unzip bubblewrap python3 fonts-dejavu-core - name: Bootstrap host tools diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..49212efb --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "lib/third-party/dmcomp"] + path = lib/third-party/dmcomp + url = https://github.com/choco-technologies/dmcomp.git diff --git a/Docker/build.sh b/Docker/build.sh index 79cef54f..a3285a62 100755 --- a/Docker/build.sh +++ b/Docker/build.sh @@ -103,6 +103,15 @@ function isTargetSelected() [[ ",$TARGETS," == *",$1,"* ]] } +# +# The image is built from this directory (COPY . in the Dockerfile): the submodules (lib/third-party/dmcomp) have to be +# in it +# +function updateSubmodules() +{ + git -C "$ROOT_DIR" submodule update --init --recursive +} + # # Builds the environment image # @@ -152,6 +161,7 @@ function buildOpencode() # MAIN # prepareScript "$@" +updateSubmodules isTargetSelected env && buildEnv isTargetSelected dmod && build isTargetSelected claude && buildClaude diff --git a/cmake/runtime/CMakeLists.txt b/cmake/runtime/CMakeLists.txt index 576c3cef..38f34d0f 100644 --- a/cmake/runtime/CMakeLists.txt +++ b/cmake/runtime/CMakeLists.txt @@ -28,6 +28,6 @@ if(DMOD_USE_FASTLZ) target_link_libraries(dmod INTERFACE dmod_fastlz) endif() if(DMOD_USE_DMCOMP) - add_subdirectory("${DMOD_DIR}/lib/third-party/dmcomp" dmcomp) + add_subdirectory("${DMOD_DIR}/lib/third-party/dmod_dmcomp" dmcomp) target_link_libraries(dmod INTERFACE dmod_dmcomp) endif() diff --git a/lib/third-party/CMakeLists.txt b/lib/third-party/CMakeLists.txt index f10f6ded..32a22cdb 100644 --- a/lib/third-party/CMakeLists.txt +++ b/lib/third-party/CMakeLists.txt @@ -1,2 +1,2 @@ add_subdirectory(FastLZ) -add_subdirectory(dmcomp) \ No newline at end of file +add_subdirectory(dmod_dmcomp) \ No newline at end of file diff --git a/lib/third-party/Makefile b/lib/third-party/Makefile index d3e0c3fb..b79e1a93 100644 --- a/lib/third-party/Makefile +++ b/lib/third-party/Makefile @@ -6,7 +6,7 @@ DMOD_DIR=../.. # ----------------------------------------------------------------------------- # Subdirectories # ----------------------------------------------------------------------------- -SUBDIRS = FastLZ dmcomp +SUBDIRS = FastLZ dmod_dmcomp SYSTEM_SUBDIRS = MODULE_SUBDIRS = diff --git a/lib/third-party/dmcomp b/lib/third-party/dmcomp new file mode 160000 index 00000000..7314bf2e --- /dev/null +++ b/lib/third-party/dmcomp @@ -0,0 +1 @@ +Subproject commit 7314bf2e1d3becff64d5e0a8148ce3d86432d1d9 diff --git a/lib/third-party/dmcomp/CMakeLists.txt b/lib/third-party/dmcomp/CMakeLists.txt deleted file mode 100644 index df6287e1..00000000 --- a/lib/third-party/dmcomp/CMakeLists.txt +++ /dev/null @@ -1,26 +0,0 @@ -set(MODULE_NAME dmod_dmcomp) - -# The decoder is always there; the encoder (it allocates ~330 kB and is ~10 kB of code) is for the host: -# a firmware that only loads the modules sets DMOD_USE_DMCOMP_ENCODER to OFF -add_library(${MODULE_NAME} STATIC - dmcomp_inflate.c - dmcomp_strerror.c -) -if(DMOD_USE_DMCOMP_ENCODER) - target_sources(${MODULE_NAME} PRIVATE dmcomp_deflate.c) -else() - target_compile_definitions(${MODULE_NAME} PUBLIC DMCOMP_NO_ENCODER) -endif() - -target_include_directories(${MODULE_NAME} - PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR} -) - -target_compile_options(${MODULE_NAME} PRIVATE - -Os - -ffunction-sections - -fdata-sections -) - -create_library_makefile(${MODULE_NAME}) diff --git a/lib/third-party/dmcomp/LICENSE.MIT b/lib/third-party/dmcomp/LICENSE.MIT deleted file mode 100644 index 7e7d6605..00000000 --- a/lib/third-party/dmcomp/LICENSE.MIT +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2026 Choco-Technologies - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/lib/third-party/dmcomp/README.md b/lib/third-party/dmcomp/README.md deleted file mode 100644 index ccad41b6..00000000 --- a/lib/third-party/dmcomp/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# dmcomp (third-party) - -DEFLATE (RFC 1951) coder and decoder of [choco-technologies/dmcomp](https://github.com/choco-technologies/dmcomp), -commit `7314bf2` (`feat/inflate-workspace`). MIT licensed, see `LICENSE.MIT`. - -It is the `deflate` method of `Dmod_Compression_*` (`src/system/if/dmod_if_comp.c`): the DMFC files of a module that -`todmfc` makes with `deflate` are ~20 % smaller than the `fastlz` ones. The decoder is 1.7 kB of code on Cortex-M, the -encoder is for the host (`DMOD_USE_DMCOMP_ENCODER`). - -**Do not edit the files here** – change them in dmcomp and copy them again (the tests, the sanitizers and the -size checks of the CI are there). diff --git a/lib/third-party/dmcomp/dmcomp.h b/lib/third-party/dmcomp/dmcomp.h deleted file mode 100644 index bd71f83a..00000000 --- a/lib/third-party/dmcomp/dmcomp.h +++ /dev/null @@ -1,118 +0,0 @@ -/** - * @file dmcomp.h - * @brief DMOD compression - DEFLATE (RFC 1951) coder and decoder - * - * A raw DEFLATE stream (no zlib or gzip wrapper: the sizes travel beside the data, e.g. in the DMFC header) is - * decoded and encoded memory to memory. Both directions live in this library, so the host tools and the target see - * the very same implementation: - * - * - the decoder (dmcomp_inflate) is small, needs no heap and no libc beyond //; its - * state is a few kB on the stack (see DMCOMP_LIT_FAST_BITS / DMCOMP_DIST_FAST_BITS) and the output buffer is the - * window - nothing is copied around; - * - the encoder (dmcomp_deflate) is meant for the host: it allocates ~330 kB for the hash chains. Build the library - * with DMCOMP_NO_ENCODER (CMake: -DDMCOMP_BUILD_ENCODER=OFF) to leave it out. - * - * The streams are plain DEFLATE: zlib/gzip tools read what dmcomp_deflate writes (inflateInit2 with -15) and - * dmcomp_inflate reads what they write. - */ -#ifndef DMCOMP_H -#define DMCOMP_H - -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/** Version of the library */ -#define DMCOMP_VERSION_MAJOR 0 -#define DMCOMP_VERSION_MINOR 1 - -/** Results: 0 is success, errors are negative */ -typedef enum -{ - DMCOMP_OK = 0, //!< Success - DMCOMP_ERR_PARAM = -1, //!< Invalid argument (NULL pointer, size too big for the coder) - DMCOMP_ERR_DATA = -2, //!< The compressed data is not valid DEFLATE - DMCOMP_ERR_TRUNCATED = -3, //!< The compressed data ends before the stream does - DMCOMP_ERR_OVERFLOW = -4, //!< The output buffer is too small - DMCOMP_ERR_MEMORY = -5, //!< The encoder could not allocate its work memory -} dmcomp_result_t; - -/** - * @brief Decompress a raw DEFLATE stream (the tables on the stack: ~3 kB, see dmcomp_inflate_ws() for the other way) - * - * Every block type is accepted (stored, fixed and dynamic Huffman codes). The data after the final block is ignored. - * Corrupted data is never trusted: nothing is read beyond @p src + @p src_size and nothing is written beyond - * @p dst + @p dst_capacity; on an error the content of @p dst is undefined. - * - * @param src Compressed data - * @param src_size Size of the compressed data - * @param dst Output buffer (the whole decompressed data is kept in it - it is the window) - * @param dst_capacity Size of the output buffer - * @param out_size Receives the number of bytes written (may not be NULL) - * - * @return DMCOMP_OK or an error (DMCOMP_ERR_DATA, DMCOMP_ERR_TRUNCATED, DMCOMP_ERR_OVERFLOW, DMCOMP_ERR_PARAM) - */ -int dmcomp_inflate( const void* src, size_t src_size, void* dst, size_t dst_capacity, size_t* out_size ); - -/** - * @brief Size of the work memory of dmcomp_inflate_ws() - * - * The tables of the Huffman codes: ~2.9 kB (see DMCOMP_LIT_FAST_BITS and DMCOMP_DIST_FAST_BITS in dmcomp_inflate.c). - */ -size_t dmcomp_inflate_workspace_size( void ); - -/** - * @brief dmcomp_inflate() with the work memory given by the caller - * - * The same decoder, but its tables are not on the stack: for a thread with a small one (the decoding may be called from - * anywhere a module is loaded), give it memory from the heap, or a buffer of your own that is not used at the time. - * - * @param workspace dmcomp_inflate_workspace_size() bytes, aligned to 2 bytes (what malloc returns is); not used after the call - * - * Other parameters and results: see dmcomp_inflate(). DMCOMP_ERR_PARAM also for a NULL or odd workspace. - */ -int dmcomp_inflate_ws( const void* src, size_t src_size, void* dst, size_t dst_capacity, size_t* out_size, - void* workspace ); - -/** Name of a result, for logs. Never NULL. */ -const char* dmcomp_strerror( int result ); - -#ifndef DMCOMP_NO_ENCODER - -/** Level of compression that dmcomp_deflate uses for a level out of range */ -#define DMCOMP_LEVEL_DEFAULT 6 - -/** - * @brief Upper bound of the size of the compressed data - * - * dmcomp_deflate never writes more than this (incompressible data is stored), so a buffer of this size cannot overflow. - */ -size_t dmcomp_deflate_bound( size_t src_size ); - -/** - * @brief Compress data to a raw DEFLATE stream - * - * Distances up to 32767 bytes are used. The result is the smallest of stored, fixed and dynamic Huffman codes for - * every block. - * - * @param src Data to compress - * @param src_size Size of the data (up to ~4 GB) - * @param dst Output buffer - * @param dst_capacity Size of the output buffer (dmcomp_deflate_bound() is always enough) - * @param out_size Receives the size of the compressed data (may not be NULL) - * @param level 0: stored, 1 (fastest) .. 9 (smallest); out of range: DMCOMP_LEVEL_DEFAULT (negative) or 9 - * - * @return DMCOMP_OK or an error (DMCOMP_ERR_OVERFLOW, DMCOMP_ERR_MEMORY, DMCOMP_ERR_PARAM) - */ -int dmcomp_deflate( const void* src, size_t src_size, void* dst, size_t dst_capacity, size_t* out_size, int level ); - -#endif /* DMCOMP_NO_ENCODER */ - -#ifdef __cplusplus -} -#endif - -#endif /* DMCOMP_H */ diff --git a/lib/third-party/dmcomp/dmcomp_deflate.c b/lib/third-party/dmcomp/dmcomp_deflate.c deleted file mode 100644 index d6d2c1c2..00000000 --- a/lib/third-party/dmcomp/dmcomp_deflate.c +++ /dev/null @@ -1,781 +0,0 @@ -/** - * @file dmcomp_deflate.c - * @brief DEFLATE encoder (RFC 1951), memory to memory - * - * LZ77 with hash chains over the whole input (it is in memory, so there is no sliding window to manage), optionally - * with lazy matching, then every block of up to DMCOMP_BLOCK_TOKENS tokens is written as the smallest of a stored, - * a fixed-code and a dynamic-code block. The code lengths are length limited Huffman codes. - * - * It is a tool for the host: ~330 kB of work memory. Everything the format needs is here, nothing of the decoder is. - */ -#include -#include -#include "dmcomp.h" -#include "dmcomp_tables.h" - -#ifndef DMCOMP_MALLOC -# define DMCOMP_MALLOC malloc -# define DMCOMP_FREE free -#endif - -#define WINDOW_SIZE 32768u -#define WINDOW_MASK (WINDOW_SIZE - 1u) -#define MAX_DISTANCE (WINDOW_SIZE - 1u) // 32768 would do for the format; this keeps the chains simple -#define HASH_BITS 15 -#define BLOCK_TOKENS 16384u -#define TOO_FAR 4096u // A match of 3 bytes this far is not worth its bits -#define STORED_MAX 65535u -#define MAX_INPUT 0xFFFF0000u // Positions are 32 bit - -typedef struct -{ - uint16_t good; //!< A previous match this long: search a quarter of the chain - uint16_t lazy; //!< A match this long is taken at once (lazy matching only) - uint16_t nice; //!< A match this long ends the search - uint16_t chain; //!< Candidates looked at per position - uint8_t lazy_on; //!< Look whether the next position has a longer match -} level_t; - -static const level_t g_levels[10] = -{ - { 0, 0, 0, 0, 0 }, - { 4, 4, 8, 4, 0 }, - { 4, 5, 16, 8, 0 }, - { 4, 6, 32, 32, 0 }, - { 4, 4, 16, 16, 1 }, - { 8, 16, 32, 32, 1 }, - { 8, 16, 128, 128, 1 }, - { 8, 32, 128, 256, 1 }, - { 32, 128, 258, 1024, 1 }, - { 32, 258, 258, 4096, 1 }, -}; - -// ===================================================================================================================== -// Bit writer -// ===================================================================================================================== - -typedef struct -{ - uint8_t* start; - uint8_t* p; - uint8_t* end; - uint32_t acc; //!< Bits not written yet, the first one lowest - unsigned n; //!< Their number (always < 8 between the calls) - int overflow; //!< The output buffer was too small - the rest is counted, not written -} bitwriter_t; - -static void put( bitwriter_t* w, uint32_t value, unsigned count ) -{ - w->acc |= value << w->n; - w->n += count; - while( w->n >= 8 ) - { - if( w->p < w->end ) - { - *w->p++ = (uint8_t)w->acc; - } - else - { - w->overflow = 1; - } - w->acc >>= 8; - w->n -= 8; - } -} - -static void put_stored( bitwriter_t* w, const uint8_t* data, size_t length, int final ) -{ - do - { - size_t chunk = length > STORED_MAX ? STORED_MAX : length; - - length -= chunk; - put( w, (final && length == 0) ? 1u : 0u, 1 ); - put( w, 0, 2 ); - put( w, 0, (8 - w->n) & 7u ); // To the next byte - put( w, (uint32_t)chunk, 16 ); - put( w, (uint32_t)(~chunk & 0xFFFFu), 16 ); - if( (size_t)(w->end - w->p) >= chunk ) - { - if( chunk > 0 ) - { - memcpy( w->p, data, chunk ); - } - w->p += chunk; - } - else - { - w->overflow = 1; - } - if( chunk > 0 ) - { - data += chunk; - } - } - while( length > 0 ); -} - -// ===================================================================================================================== -// Huffman codes -// ===================================================================================================================== - -static unsigned reverse_bits( unsigned code, unsigned length ) -{ - unsigned reversed = 0; - while( length-- > 0 ) - { - reversed = (reversed << 1) | (code & 1u); - code >>= 1; - } - return reversed; -} - -/** Canonical codes of the given lengths, ready to be put into the stream (first bit lowest) */ -static void make_codes( const uint8_t* lengths, unsigned symbols, uint16_t* codes ) -{ - unsigned count[DMCOMP_MAX_BITS + 1] = { 0 }; - unsigned next[DMCOMP_MAX_BITS + 2]; - unsigned i, length, code = 0; - - for( i = 0; i < symbols; i++ ) - { - count[lengths[i]]++; - } - count[0] = 0; - for( length = 1; length <= DMCOMP_MAX_BITS; length++ ) - { - code = (code + count[length - 1]) << 1; - next[length] = code; - } - for( i = 0; i < symbols; i++ ) - { - codes[i] = lengths[i] ? (uint16_t)reverse_bits( next[lengths[i]]++, lengths[i] ) : 0; - } -} - -/** - * Code lengths of the symbols of the given frequencies, none longer than @p limit. - * - * Huffman's tree is built by merging the two lightest nodes (leaves sorted once, the merged nodes come out in - * order, so two queues are enough). A tree deeper than the limit is cut the way zlib does it: the leaves below the - * limit move up and make room by pushing a shallower leaf down. At least two symbols get a code (a stream with one - * code is valid but some decoders do not take it). - */ -static void build_lengths( const uint32_t* freq, unsigned symbols, unsigned limit, uint8_t* lengths ) -{ - uint32_t weight[2 * DMCOMP_LITLEN_SYMBOLS]; - uint16_t parent[2 * DMCOMP_LITLEN_SYMBOLS]; - uint16_t depth[2 * DMCOMP_LITLEN_SYMBOLS]; - uint32_t f[DMCOMP_LITLEN_SYMBOLS]; - uint16_t order[DMCOMP_LITLEN_SYMBOLS]; // The used symbols, the rarest first - unsigned bl_count[DMCOMP_MAX_BITS + 1] = { 0 }; - unsigned i, j, used = 0, overflow = 0; - - for( i = 0; i < symbols; i++ ) - { - f[i] = freq[i]; - lengths[i] = 0; - if( f[i] != 0 ) - { - used++; - } - } - for( i = 0; used < 2 && i < symbols; i++ ) - { - if( f[i] == 0 ) - { - f[i] = 1; - used++; - } - } - - // Insertion sort by frequency (then by value): there are at most 286 symbols - used = 0; - for( i = 0; i < symbols; i++ ) - { - if( f[i] == 0 ) - { - continue; - } - for( j = used; j > 0 && f[order[j - 1]] > f[i]; j-- ) - { - order[j] = order[j - 1]; - } - order[j] = (uint16_t)i; - used++; - } - - // The leaves are nodes 0..used-1, the merged nodes follow - { - unsigned leaf = 0, node = used, created = used; - unsigned step; - - for( i = 0; i < used; i++ ) - { - weight[i] = f[order[i]]; - } - for( step = 0; step + 1 < used; step++ ) - { - unsigned pick[2]; - for( j = 0; j < 2; j++ ) - { - if( leaf < used && (node >= created || weight[leaf] <= weight[node]) ) - { - pick[j] = leaf++; - } - else - { - pick[j] = node++; - } - } - weight[created] = weight[pick[0]] + weight[pick[1]]; - parent[pick[0]] = (uint16_t)created; - parent[pick[1]] = (uint16_t)created; - created++; - } - - // The root is the last node; every parent comes after its children. A node deeper than the limit is cut to it - // (its children are measured from there) - every node that is counts as overflow, the inner ones too: that is - // what the correction below is made of - depth[created - 1] = 0; - for( i = created - 1; i-- > 0; ) - { - unsigned d = depth[parent[i]] + 1u; - if( d > limit ) - { - d = limit; - overflow++; - } - depth[i] = (uint16_t)d; - } - for( i = 0; i < used; i++ ) - { - bl_count[depth[i]]++; - } - } - - while( overflow > 0 ) - { - unsigned bits = limit - 1; - while( bl_count[bits] == 0 ) - { - bits--; - } - bl_count[bits]--; // A leaf moves down... - bl_count[bits + 1] += 2; // ...and takes one of the leaves that were cut as its brother - bl_count[limit]--; - overflow = overflow > 2 ? overflow - 2 : 0; - } - - // The rarest symbols get the longest codes - { - unsigned k = 0, length; - for( length = limit; length >= 1; length-- ) - { - for( i = 0; i < bl_count[length]; i++ ) - { - lengths[order[k++]] = (uint8_t)length; - } - } - } -} - -/** - * Run-length codes the lengths of the codes (the alphabet of RFC 1951, 3.2.7): 0..15 as they are, 16 repeats the - * previous one 3-6 times, 17 and 18 repeat zero 3-10 and 11-138 times. - * - * @return number of symbols; symbols[] and extras[] (the value of the extra bits) are filled - */ -static unsigned rle_lengths( const uint8_t* lengths, unsigned total, uint8_t* symbols, uint8_t* extras ) -{ - unsigned i = 0, k = 0; - - while( i < total ) - { - uint8_t value = lengths[i]; - unsigned run = 1; - - while( i + run < total && lengths[i + run] == value ) - { - run++; - } - i += run; - - if( value == 0 ) - { - while( run >= 11 ) - { - unsigned r = run > 138 ? 138 : run; - symbols[k] = 18; - extras[k++] = (uint8_t)(r - 11); - run -= r; - } - if( run >= 3 ) - { - symbols[k] = 17; - extras[k++] = (uint8_t)(run - 3); - run = 0; - } - while( run-- > 0 ) - { - symbols[k] = 0; - extras[k++] = 0; - } - } - else - { - symbols[k] = value; - extras[k++] = 0; - run--; - while( run >= 3 ) - { - unsigned r = run > 6 ? 6 : run; - symbols[k] = 16; - extras[k++] = (uint8_t)(r - 3); - run -= r; - } - while( run-- > 0 ) - { - symbols[k] = value; - extras[k++] = 0; - } - } - } - return k; -} - -// ===================================================================================================================== -// Encoder state -// ===================================================================================================================== - -typedef struct -{ - const uint8_t* in; - size_t n; - const level_t* cfg; - bitwriter_t w; - - size_t ins; //!< Positions below it are in the hash chains - size_t cov; //!< The input covered by the tokens so far (the written blocks included) - size_t blk_start; //!< Where the block being collected starts - unsigned ntok; - uint32_t lit_freq[DMCOMP_LITLEN_SYMBOLS]; - uint32_t dist_freq[DMCOMP_DIST_SYMBOLS]; - uint8_t lcode[DMCOMP_MAX_MATCH + 1]; //!< Length -> length code (0..28) - uint8_t dcode[512]; //!< Distance - 1 -> distance code, see DIST_CODE - uint32_t head[1u << HASH_BITS]; //!< Newest position (+1) of every hash - uint32_t prev[WINDOW_SIZE]; //!< Older position (+1) with the same hash, by position - uint32_t tokens[BLOCK_TOKENS]; //!< A literal: its value; a match: (length << 16) | distance -} encoder_t; - -#define DIST_CODE( e, d ) ((e)->dcode[(d) < 256u ? (d) : 256u + ((d) >> 7)]) - -static void init_code_tables( encoder_t* e ) -{ - unsigned c, v; - - for( c = 0; c < 28; c++ ) - { - for( v = dmcomp_len_base[c]; v < dmcomp_len_base[c] + (1u << dmcomp_len_extra[c]); v++ ) - { - e->lcode[v] = (uint8_t)c; - } - } - e->lcode[258] = 28; // 258 has a code of its own, the last of code 27's range is not used - - for( c = 0; c < DMCOMP_DIST_SYMBOLS; c++ ) - { - for( v = dmcomp_dist_base[c] - 1u; v < dmcomp_dist_base[c] - 1u + (1u << dmcomp_dist_extra[c]); v++ ) - { - e->dcode[v < 256u ? v : 256u + (v >> 7)] = (uint8_t)c; - } - } -} - -// ===================================================================================================================== -// Matching -// ===================================================================================================================== - -static unsigned hash3( const uint8_t* p ) -{ - uint32_t v = (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16); - return (v * 2654435761u) >> (32 - HASH_BITS); -} - -/** Puts the positions below @p upto into the hash chains */ -static void insert_upto( encoder_t* e, size_t upto ) -{ - while( e->ins < upto ) - { - size_t pos = e->ins++; - if( pos + 2 < e->n ) - { - unsigned h = hash3( e->in + pos ); - e->prev[pos & WINDOW_MASK] = e->head[h]; - e->head[h] = (uint32_t)pos + 1; - } - } -} - -/** - * Longest match for the data at @p pos, only if it is longer than @p better_than (and at least 3 bytes). The - * positions below @p pos have to be in the chains, @p pos itself not yet. - * - * @return the length, or 0 - */ -static unsigned find_match( encoder_t* e, size_t pos, unsigned better_than, unsigned chain, unsigned* distance ) -{ - size_t avail = e->n - pos; - const uint8_t* cur = e->in + pos; - unsigned max_len = avail > DMCOMP_MAX_MATCH ? DMCOMP_MAX_MATCH : (unsigned)avail; - unsigned best = better_than < DMCOMP_MIN_MATCH - 1 ? DMCOMP_MIN_MATCH - 1 : better_than; - unsigned best_distance = 0; - uint32_t candidate; - - if( avail < DMCOMP_MIN_MATCH || best >= max_len ) - { - return 0; - } - - candidate = e->head[hash3( cur )]; - while( candidate != 0 && chain-- > 0 ) - { - size_t cand_pos = candidate - 1; - size_t d = pos - cand_pos; - const uint8_t* m = e->in + cand_pos; - uint32_t next; - - if( cand_pos >= pos || d > MAX_DISTANCE ) - { - break; - } - if( m[best] == cur[best] && m[0] == cur[0] ) - { - unsigned l = 0; - while( l < max_len && m[l] == cur[l] ) - { - l++; - } - if( l > best ) - { - best = l; - best_distance = (unsigned)d; - if( l >= e->cfg->nice || l >= max_len ) - { - break; - } - } - } - next = e->prev[cand_pos & WINDOW_MASK]; - if( next >= candidate ) - { - break; // The chain goes back in time, always - } - candidate = next; - } - - if( best_distance == 0 || (best == DMCOMP_MIN_MATCH && best_distance > TOO_FAR) ) - { - return 0; - } - *distance = best_distance; - return best; -} - -// ===================================================================================================================== -// Blocks -// ===================================================================================================================== - -static uint64_t data_bits( const encoder_t* e, const uint8_t* lit_len, const uint8_t* dist_len ) -{ - uint64_t bits = 0; - unsigned s; - - for( s = 0; s < DMCOMP_LITLEN_SYMBOLS; s++ ) - { - bits += (uint64_t)e->lit_freq[s] * lit_len[s]; - } - for( s = 257; s < DMCOMP_LITLEN_SYMBOLS; s++ ) - { - bits += (uint64_t)e->lit_freq[s] * dmcomp_len_extra[s - 257]; - } - for( s = 0; s < DMCOMP_DIST_SYMBOLS; s++ ) - { - bits += (uint64_t)e->dist_freq[s] * (dist_len[s] + dmcomp_dist_extra[s]); - } - return bits; -} - -static void put_tokens( encoder_t* e, const uint16_t* lit_code, const uint8_t* lit_len, const uint16_t* dist_code, - const uint8_t* dist_len ) -{ - unsigned i; - - for( i = 0; i < e->ntok; i++ ) - { - uint32_t t = e->tokens[i]; - if( (t >> 16) == 0 ) - { - put( &e->w, lit_code[t], lit_len[t] ); - } - else - { - unsigned length = t >> 16; - unsigned distance = t & 0xFFFFu; - unsigned lc = e->lcode[length]; - unsigned dc = DIST_CODE( e, distance - 1u ); - - put( &e->w, lit_code[257 + lc], lit_len[257 + lc] ); - if( dmcomp_len_extra[lc] != 0 ) - { - put( &e->w, length - dmcomp_len_base[lc], dmcomp_len_extra[lc] ); - } - put( &e->w, dist_code[dc], dist_len[dc] ); - if( dmcomp_dist_extra[dc] != 0 ) - { - put( &e->w, distance - dmcomp_dist_base[dc], dmcomp_dist_extra[dc] ); - } - } - } - put( &e->w, lit_code[256], lit_len[256] ); -} - -/** Writes the collected tokens as one block - the smallest of the three kinds - and starts the next one */ -static void flush_block( encoder_t* e, int final ) -{ - uint8_t lit_len[DMCOMP_LITLEN_SYMBOLS], dist_len[DMCOMP_DIST_SYMBOLS]; - uint8_t fixed_lit_len[288], fixed_dist_len[DMCOMP_DIST_SYMBOLS]; - uint8_t all_len[DMCOMP_LITLEN_SYMBOLS + DMCOMP_DIST_SYMBOLS]; - uint8_t rle_sym[DMCOMP_LITLEN_SYMBOLS + DMCOMP_DIST_SYMBOLS], rle_extra[DMCOMP_LITLEN_SYMBOLS + DMCOMP_DIST_SYMBOLS]; - uint32_t clen_freq[DMCOMP_CLEN_SYMBOLS] = { 0 }; - uint8_t clen_len[DMCOMP_CLEN_SYMBOLS]; - unsigned hlit, hdist, hclen, nrle, i; - uint64_t dynamic_bits, fixed_bits, stored_bits; - size_t span = e->cov - e->blk_start; - - e->lit_freq[256] = 1; // The end of the block - - // --- Dynamic codes - build_lengths( e->lit_freq, DMCOMP_LITLEN_SYMBOLS, DMCOMP_MAX_BITS, lit_len ); - build_lengths( e->dist_freq, DMCOMP_DIST_SYMBOLS, DMCOMP_MAX_BITS, dist_len ); - for( hlit = DMCOMP_LITLEN_SYMBOLS; hlit > 257 && lit_len[hlit - 1] == 0; hlit-- ) {} - for( hdist = DMCOMP_DIST_SYMBOLS; hdist > 1 && dist_len[hdist - 1] == 0; hdist-- ) {} - memcpy( all_len, lit_len, hlit ); - memcpy( all_len + hlit, dist_len, hdist ); - nrle = rle_lengths( all_len, hlit + hdist, rle_sym, rle_extra ); - for( i = 0; i < nrle; i++ ) - { - clen_freq[rle_sym[i]]++; - } - build_lengths( clen_freq, DMCOMP_CLEN_SYMBOLS, DMCOMP_MAX_CLEN_BITS, clen_len ); - for( hclen = DMCOMP_CLEN_SYMBOLS; hclen > 4 && clen_len[dmcomp_clen_order[hclen - 1]] == 0; hclen-- ) {} - - dynamic_bits = 3 + 14 + 3 * (uint64_t)hclen + data_bits( e, lit_len, dist_len ); - for( i = 0; i < nrle; i++ ) - { - dynamic_bits += clen_len[rle_sym[i]] + (rle_sym[i] == 16 ? 2 : rle_sym[i] == 17 ? 3 : rle_sym[i] == 18 ? 7 : 0); - } - - // --- Fixed codes - for( i = 0; i < 144; i++ ) fixed_lit_len[i] = 8; - for( ; i < 256; i++ ) fixed_lit_len[i] = 9; - for( ; i < 280; i++ ) fixed_lit_len[i] = 7; - for( ; i < 288; i++ ) fixed_lit_len[i] = 8; - for( i = 0; i < DMCOMP_DIST_SYMBOLS; i++ ) fixed_dist_len[i] = 5; - fixed_bits = 3 + data_bits( e, fixed_lit_len, fixed_dist_len ); - - // --- Stored: 3 bits and the padding in front of the first chunk, 4 bytes in front of every chunk - { - uint64_t chunks = span == 0 ? 1 : (span + STORED_MAX - 1) / STORED_MAX; - unsigned pad = (8 - ((e->w.n + 3) & 7u)) & 7u; - stored_bits = 3 + pad + 32 * chunks + 8 * (chunks - 1) + 8 * (uint64_t)span; - } - - if( stored_bits < fixed_bits && stored_bits < dynamic_bits ) - { - put_stored( &e->w, e->in + e->blk_start, span, final ); // Writes the BFINAL bit itself - } - else if( fixed_bits <= dynamic_bits ) - { - uint16_t lit_code[288], dist_code[DMCOMP_DIST_SYMBOLS]; - - make_codes( fixed_lit_len, 288, lit_code ); - make_codes( fixed_dist_len, DMCOMP_DIST_SYMBOLS, dist_code ); - put( &e->w, final ? 1u : 0u, 1 ); - put( &e->w, 1, 2 ); - put_tokens( e, lit_code, fixed_lit_len, dist_code, fixed_dist_len ); - } - else - { - uint16_t lit_code[DMCOMP_LITLEN_SYMBOLS], dist_code[DMCOMP_DIST_SYMBOLS], clen_code[DMCOMP_CLEN_SYMBOLS]; - - make_codes( lit_len, DMCOMP_LITLEN_SYMBOLS, lit_code ); - make_codes( dist_len, DMCOMP_DIST_SYMBOLS, dist_code ); - make_codes( clen_len, DMCOMP_CLEN_SYMBOLS, clen_code ); - - put( &e->w, final ? 1u : 0u, 1 ); - put( &e->w, 2, 2 ); - put( &e->w, hlit - 257, 5 ); - put( &e->w, hdist - 1, 5 ); - put( &e->w, hclen - 4, 4 ); - for( i = 0; i < hclen; i++ ) - { - put( &e->w, clen_len[dmcomp_clen_order[i]], 3 ); - } - for( i = 0; i < nrle; i++ ) - { - put( &e->w, clen_code[rle_sym[i]], clen_len[rle_sym[i]] ); - if( rle_sym[i] == 16 ) put( &e->w, rle_extra[i], 2 ); - else if( rle_sym[i] == 17 ) put( &e->w, rle_extra[i], 3 ); - else if( rle_sym[i] == 18 ) put( &e->w, rle_extra[i], 7 ); - } - put_tokens( e, lit_code, lit_len, dist_code, dist_len ); - } - - memset( e->lit_freq, 0, sizeof(e->lit_freq) ); - memset( e->dist_freq, 0, sizeof(e->dist_freq) ); - e->ntok = 0; - e->blk_start = e->cov; -} - -static void emit_literal( encoder_t* e, uint8_t value ) -{ - e->tokens[e->ntok++] = value; - e->lit_freq[value]++; - e->cov += 1; -} - -static void emit_match( encoder_t* e, unsigned length, unsigned distance ) -{ - e->tokens[e->ntok++] = ((uint32_t)length << 16) | distance; - e->lit_freq[257 + e->lcode[length]]++; - e->dist_freq[DIST_CODE( e, distance - 1u )]++; - e->cov += length; -} - -static void compress_blocks( encoder_t* e ) -{ - const level_t* cfg = e->cfg; - size_t i = 0; - unsigned length = 0, distance = 0; - int have_match = 0; // 'length' and 'distance' are the match at i (found while looking ahead) - - while( i < e->n ) - { - if( !have_match ) - { - insert_upto( e, i ); - length = find_match( e, i, 0, cfg->chain, &distance ); - insert_upto( e, i + 1 ); - } - have_match = 0; - - if( length >= DMCOMP_MIN_MATCH ) - { - if( cfg->lazy_on && length < cfg->lazy && i + 1 < e->n ) - { - // A longer match one byte later is worth a literal now - unsigned next_distance = 0; - unsigned chain = length >= cfg->good ? cfg->chain >> 2 : cfg->chain; - unsigned next_length = find_match( e, i + 1, length, chain, &next_distance ); - - insert_upto( e, i + 2 ); - if( next_length > length ) - { - emit_literal( e, e->in[i] ); - i++; - length = next_length; - distance = next_distance; - have_match = 1; - if( e->ntok >= BLOCK_TOKENS ) - { - flush_block( e, 0 ); - } - continue; - } - } - emit_match( e, length, distance ); - insert_upto( e, i + length ); - i += length; - } - else - { - emit_literal( e, e->in[i] ); - i++; - } - - if( e->ntok >= BLOCK_TOKENS ) - { - flush_block( e, 0 ); - } - } - flush_block( e, 1 ); -} - -// ===================================================================================================================== -// Public API -// ===================================================================================================================== - -size_t dmcomp_deflate_bound( size_t src_size ) -{ - // The worst case is a stored block: 5 bytes (the header, the padding, LEN and NLEN) per 64 kB, and a block is at - // least BLOCK_TOKENS bytes long unless it is the last one - return src_size + 6 * (src_size / BLOCK_TOKENS + 2) + 16; -} - -int dmcomp_deflate( const void* src, size_t src_size, void* dst, size_t dst_capacity, size_t* out_size, int level ) -{ - encoder_t* e; - int result; - - if( out_size == NULL || (src == NULL && src_size != 0) || (dst == NULL && dst_capacity != 0) || - src_size > MAX_INPUT ) - { - return DMCOMP_ERR_PARAM; - } - *out_size = 0; - - if( level < 0 ) - { - level = DMCOMP_LEVEL_DEFAULT; - } - else if( level > 9 ) - { - level = 9; - } - - e = (encoder_t*)DMCOMP_MALLOC( sizeof(encoder_t) ); - if( e == NULL ) - { - return DMCOMP_ERR_MEMORY; - } - memset( e, 0, offsetof( encoder_t, prev ) ); // The chains (prev) and the tokens are written before they are read - e->in = (const uint8_t*)src; - e->n = src_size; - e->cfg = &g_levels[level]; - e->w.start = (uint8_t*)dst; - e->w.p = (uint8_t*)dst; - e->w.end = dst_capacity != 0 ? (uint8_t*)dst + dst_capacity : (uint8_t*)dst; // NULL is not moved - init_code_tables( e ); - - if( level == 0 ) - { - put_stored( &e->w, e->in, e->n, 1 ); - } - else - { - compress_blocks( e ); - } - if( e->w.n > 0 ) - { - put( &e->w, 0, 8 - e->w.n ); - } - - result = e->w.overflow ? DMCOMP_ERR_OVERFLOW : DMCOMP_OK; - if( result == DMCOMP_OK ) - { - *out_size = (size_t)(e->w.p - e->w.start); - } - DMCOMP_FREE( e ); - return result; -} diff --git a/lib/third-party/dmcomp/dmcomp_inflate.c b/lib/third-party/dmcomp/dmcomp_inflate.c deleted file mode 100644 index 90b37ca3..00000000 --- a/lib/third-party/dmcomp/dmcomp_inflate.c +++ /dev/null @@ -1,571 +0,0 @@ -/** - * @file dmcomp_inflate.c - * @brief DEFLATE decoder (RFC 1951), memory to memory - * - * Design: the output buffer is the window, the state is a few kB on the stack, there is no heap and no global state - * (reentrant). Huffman codes are decoded with a table of the first DMCOMP_*_FAST_BITS bits of the stream; the rare - * longer codes are walked canonically, bit by bit. The input is never read beyond its end: the missing bytes read as - * zeros, and the decoder notices that it used them (DMCOMP_ERR_TRUNCATED) before it acts on them. - */ -#include -#include "dmcomp.h" -#include "dmcomp_tables.h" - -/** Bits of the table of literal/length codes: 2^bits * 2 bytes of stack. 9 covers almost every code of real data. */ -#ifndef DMCOMP_LIT_FAST_BITS -# define DMCOMP_LIT_FAST_BITS 9 -#endif - -/** Bits of the table of distance codes: 2^bits * 2 bytes of stack */ -#ifndef DMCOMP_DIST_FAST_BITS -# define DMCOMP_DIST_FAST_BITS 7 -#endif - -#define CLEN_FAST_BITS DMCOMP_MAX_CLEN_BITS // The whole alphabet of code lengths is in the table -#define LITLEN_TABLE_SIZE 288 // Fixed codes have 288 symbols (286 and 287 are invalid) -#define DIST_TABLE_SIZE 32 // Fixed codes have 32 symbols (30 and 31 are invalid) - -// ===================================================================================================================== -// Bit reader -// ===================================================================================================================== - -typedef struct -{ - const uint8_t* p; //!< Next byte of the input - const uint8_t* end; //!< End of the input - uint32_t bits; //!< Bits not used yet (the next one is the lowest) - unsigned n; //!< Number of valid bits in 'bits' - unsigned virt; //!< Number of the bits that were added past the end of the input (they are the highest) -} bitreader_t; - -/** Fills the buffer to 25..32 bits. Past the end of the input zeros are added (and counted in 'virt'). */ -static void refill( bitreader_t* b ) -{ - while( b->n <= 24 ) - { - uint32_t byte = 0; - if( b->p < b->end ) - { - byte = *b->p++; - } - else - { - b->virt += 8; - } - b->bits |= byte << b->n; - b->n += 8; - } -} - -/** True when a bit that does not exist was used - the input is too short */ -static int overrun( const bitreader_t* b ) -{ - return b->n < b->virt; -} - -/** Takes @p count bits (0..16); the buffer must have been filled */ -static uint32_t take( bitreader_t* b, unsigned count ) -{ - uint32_t value = b->bits & ((1u << count) - 1u); - b->bits >>= count; - b->n -= count; - return value; -} - -// ===================================================================================================================== -// Huffman codes -// ===================================================================================================================== - -/** - * A code is described by three arrays: - * - fast[] : indexed by the next 'fast_bits' bits of the stream: (symbol << 4) | length of the code; 0: the code is - * longer than the table (or does not exist) - * - count[] : the number of codes of every length - * - sym[] : the symbols sorted by the length of their codes, then by the value - */ -typedef struct -{ - uint16_t lit_fast[1u << DMCOMP_LIT_FAST_BITS]; - uint16_t lit_count[DMCOMP_MAX_BITS + 1]; - uint16_t lit_sym[LITLEN_TABLE_SIZE]; - uint16_t dist_fast[1u << DMCOMP_DIST_FAST_BITS]; - uint16_t dist_count[DMCOMP_MAX_BITS + 1]; - uint16_t dist_sym[DIST_TABLE_SIZE]; - - // The header of a dynamic block (the lengths of the codes, and the code of the lengths) - uint8_t lengths[LITLEN_TABLE_SIZE + DIST_TABLE_SIZE]; - uint8_t clen[DMCOMP_CLEN_SYMBOLS]; - uint16_t cfast[1u << CLEN_FAST_BITS]; - uint16_t ccount[DMCOMP_MAX_BITS + 1]; - uint16_t csym[DMCOMP_CLEN_SYMBOLS]; -} workspace_t; - -static unsigned reverse_bits( unsigned code, unsigned length ) -{ - unsigned reversed = 0; - while( length-- > 0 ) - { - reversed = (reversed << 1) | (code & 1u); - code >>= 1; - } - return reversed; -} - -/** - * Builds the decoding tables of a code described by the lengths of its symbols. - * - * The set of codes has to be complete (it may not leave any bit sequence unused), with one exception that zlib allows - * too: a single code of one bit. No codes at all is accepted, every decoding fails then. - */ -static int huff_build( uint16_t* fast, unsigned fast_bits, uint16_t* count, uint16_t* sym, const uint8_t* lengths, - unsigned symbols ) -{ - uint16_t offsets[DMCOMP_MAX_BITS + 2]; - unsigned i, length, used; - int left = 1; - - for( length = 0; length <= DMCOMP_MAX_BITS; length++ ) - { - count[length] = 0; - } - for( i = 0; i < symbols; i++ ) - { - count[lengths[i]]++; - } - used = symbols - count[0]; - - for( i = 0; i < (1u << fast_bits); i++ ) - { - fast[i] = 0; - } - if( used == 0 ) - { - return DMCOMP_OK; - } - - for( length = 1; length <= DMCOMP_MAX_BITS; length++ ) - { - left <<= 1; - left -= count[length]; - if( left < 0 ) - { - return DMCOMP_ERR_DATA; // Over-subscribed - } - } - if( left > 0 && !(used == 1 && count[1] == 1) ) - { - return DMCOMP_ERR_DATA; // Incomplete - } - - offsets[1] = 0; - for( length = 1; length < DMCOMP_MAX_BITS; length++ ) - { - offsets[length + 1] = (uint16_t)(offsets[length] + count[length]); - } - for( i = 0; i < symbols; i++ ) - { - if( lengths[i] != 0 ) - { - sym[offsets[lengths[i]]++] = (uint16_t)i; - } - } - - // The canonical codes are given out in the order of 'sym'; the stream has them with the first bit lowest - unsigned code = 0, index = 0; - for( length = 1; length <= DMCOMP_MAX_BITS; length++ ) - { - for( i = 0; i < count[length]; i++, index++, code++ ) - { - if( length <= fast_bits ) - { - unsigned j; - uint16_t entry = (uint16_t)((sym[index] << 4) | length); - for( j = reverse_bits( code, length ); j < (1u << fast_bits); j += (1u << length) ) - { - fast[j] = entry; - } - } - } - code <<= 1; - } - return DMCOMP_OK; -} - -/** Decodes one symbol; the buffer must have been filled. Returns the symbol or -1 for a bit sequence of no code. */ -static int huff_decode( bitreader_t* b, const uint16_t* fast, unsigned fast_bits, const uint16_t* count, - const uint16_t* sym ) -{ - uint16_t entry = fast[b->bits & ((1u << fast_bits) - 1u)]; - if( entry != 0 ) - { - take( b, entry & 15u ); - return entry >> 4; - } - - // A code longer than the table (or no code): walk it canonically, one bit at a time - int code = 0, first = 0, index = 0; - uint32_t bits = b->bits; - unsigned length; - for( length = 1; length <= DMCOMP_MAX_BITS; length++ ) - { - int c; - code |= (int)(bits & 1u); - bits >>= 1; - c = count[length]; - if( code - c < first ) - { - take( b, length ); - return sym[index + (code - first)]; - } - index += c; - first += c; - first <<= 1; - code <<= 1; - } - return -1; -} - -// ===================================================================================================================== -// Blocks -// ===================================================================================================================== - -static int build_fixed( workspace_t* t ) -{ - uint8_t* lengths = t->lengths; - unsigned i; - for( i = 0; i < 144; i++ ) lengths[i] = 8; - for( ; i < 256; i++ ) lengths[i] = 9; - for( ; i < 280; i++ ) lengths[i] = 7; - for( ; i < 288; i++ ) lengths[i] = 8; - int r = huff_build( t->lit_fast, DMCOMP_LIT_FAST_BITS, t->lit_count, t->lit_sym, lengths, LITLEN_TABLE_SIZE ); - if( r != DMCOMP_OK ) - { - return r; - } - for( i = 0; i < DIST_TABLE_SIZE; i++ ) lengths[i] = 5; - return huff_build( t->dist_fast, DMCOMP_DIST_FAST_BITS, t->dist_count, t->dist_sym, lengths, DIST_TABLE_SIZE ); -} - -/** Reads the header of a dynamic block (RFC 1951, 3.2.7) and builds its tables */ -static int build_dynamic( bitreader_t* b, workspace_t* t ) -{ - uint8_t* lengths = t->lengths; - uint8_t* clen = t->clen; - uint16_t* cfast = t->cfast; - uint16_t* ccount = t->ccount; - uint16_t* csym = t->csym; - unsigned hlit, hdist, hclen, i, total; - int r; - - for( i = 0; i < DMCOMP_CLEN_SYMBOLS; i++ ) - { - clen[i] = 0; - } - - refill( b ); - hlit = take( b, 5 ) + 257; - hdist = take( b, 5 ) + 1; - hclen = take( b, 4 ) + 4; - if( overrun( b ) ) - { - return DMCOMP_ERR_TRUNCATED; - } - if( hlit > DMCOMP_LITLEN_SYMBOLS || hdist > DMCOMP_DIST_SYMBOLS ) - { - return DMCOMP_ERR_DATA; - } - - for( i = 0; i < hclen; i++ ) - { - refill( b ); - clen[dmcomp_clen_order[i]] = (uint8_t)take( b, 3 ); - } - if( overrun( b ) ) - { - return DMCOMP_ERR_TRUNCATED; - } - r = huff_build( cfast, CLEN_FAST_BITS, ccount, csym, clen, DMCOMP_CLEN_SYMBOLS ); - if( r != DMCOMP_OK ) - { - return r; - } - - // The lengths of the literal/length and distance codes are one run-length coded sequence - total = hlit + hdist; - for( i = 0; i < total; ) - { - int s; - unsigned repeat; - uint8_t value; - - refill( b ); - s = huff_decode( b, cfast, CLEN_FAST_BITS, ccount, csym ); - if( s < 0 ) - { - return overrun( b ) ? DMCOMP_ERR_TRUNCATED : DMCOMP_ERR_DATA; - } - if( s < 16 ) - { - lengths[i++] = (uint8_t)s; - continue; - } - if( s == 16 ) - { - if( i == 0 ) - { - return DMCOMP_ERR_DATA; - } - value = lengths[i - 1]; - repeat = 3 + take( b, 2 ); - } - else if( s == 17 ) - { - value = 0; - repeat = 3 + take( b, 3 ); - } - else - { - value = 0; - repeat = 11 + take( b, 7 ); - } - if( overrun( b ) ) - { - return DMCOMP_ERR_TRUNCATED; - } - if( i + repeat > total ) - { - return DMCOMP_ERR_DATA; - } - while( repeat-- > 0 ) - { - lengths[i++] = value; - } - } - if( lengths[256] == 0 ) - { - return DMCOMP_ERR_DATA; // No end of block code - } - - r = huff_build( t->lit_fast, DMCOMP_LIT_FAST_BITS, t->lit_count, t->lit_sym, lengths, hlit ); - if( r != DMCOMP_OK ) - { - return r; - } - return huff_build( t->dist_fast, DMCOMP_DIST_FAST_BITS, t->dist_count, t->dist_sym, lengths + hlit, hdist ); -} - -/** Decodes the symbols of a Huffman coded block up to its end of block code */ -static int inflate_codes( bitreader_t* b, const workspace_t* t, uint8_t* dst, uint8_t** pout, uint8_t* end ) -{ - uint8_t* out = *pout; - - for( ;; ) - { - int s; - - if( overrun( b ) ) - { - return DMCOMP_ERR_TRUNCATED; - } - refill( b ); - s = huff_decode( b, t->lit_fast, DMCOMP_LIT_FAST_BITS, t->lit_count, t->lit_sym ); - if( s < 0 ) - { - return overrun( b ) ? DMCOMP_ERR_TRUNCATED : DMCOMP_ERR_DATA; - } - if( s < 256 ) - { - if( out == end ) - { - return DMCOMP_ERR_OVERFLOW; - } - *out++ = (uint8_t)s; - continue; - } - if( s == 256 ) - { - break; - } - - // A match: length, then distance (at most 15 + 5 and 15 + 13 bits, so the buffer is filled in between) - unsigned length, dist; - int d; - - s -= 257; - if( s >= 29 ) - { - return DMCOMP_ERR_DATA; // Length codes 286 and 287 are not used - } - length = dmcomp_len_base[s] + take( b, dmcomp_len_extra[s] ); - - refill( b ); - d = huff_decode( b, t->dist_fast, DMCOMP_DIST_FAST_BITS, t->dist_count, t->dist_sym ); - if( d < 0 ) - { - return overrun( b ) ? DMCOMP_ERR_TRUNCATED : DMCOMP_ERR_DATA; - } - if( d >= DMCOMP_DIST_SYMBOLS ) - { - return DMCOMP_ERR_DATA; // Distance codes 30 and 31 are not used - } - refill( b ); - dist = dmcomp_dist_base[d] + take( b, dmcomp_dist_extra[d] ); - - if( overrun( b ) ) - { - return DMCOMP_ERR_TRUNCATED; - } - if( dist > (size_t)(out - dst) ) - { - return DMCOMP_ERR_DATA; // Before the start of the output - } - if( length > (size_t)(end - out) ) - { - return DMCOMP_ERR_OVERFLOW; - } - { - // The source may overlap the destination (dist < length repeats a pattern), so no memcpy - const uint8_t* from = out - dist; - while( length-- > 0 ) - { - *out++ = *from++; - } - } - } - - *pout = out; - return DMCOMP_OK; -} - -/** Copies a stored block (RFC 1951, 3.2.4) */ -static int inflate_stored( bitreader_t* b, uint8_t** pout, uint8_t* end ) -{ - unsigned length, inverted; - - // The rest of the byte is skipped - take( b, b->n & 7u ); - refill( b ); - length = take( b, 16 ); - inverted = take( b, 16 ); - if( overrun( b ) ) - { - return DMCOMP_ERR_TRUNCATED; - } - if( length != (~inverted & 0xFFFFu) ) - { - return DMCOMP_ERR_DATA; - } - - // The whole bytes in the buffer go back to the input - b->p -= (b->n - b->virt) / 8; - b->bits = 0; - b->n = 0; - b->virt = 0; - - if( length > (size_t)(b->end - b->p) ) - { - return DMCOMP_ERR_TRUNCATED; - } - if( length > (size_t)(end - *pout) ) - { - return DMCOMP_ERR_OVERFLOW; - } - if( length > 0 ) - { - memcpy( *pout, b->p, length ); - } - *pout += length; - b->p += length; - return DMCOMP_OK; -} - -// ===================================================================================================================== -// Public API -// ===================================================================================================================== - -size_t dmcomp_inflate_workspace_size( void ) -{ - return sizeof(workspace_t); -} - -int dmcomp_inflate_ws( const void* src, size_t src_size, void* dst, size_t dst_capacity, size_t* out_size, - void* workspace ) -{ - bitreader_t b; - uint8_t* out = (uint8_t*)dst; - uint8_t* end; - int last; - workspace_t* tables = (workspace_t*)workspace; - - // Checked before any pointer is moved: a NULL pointer is only valid for an empty buffer, and is not added to - if( out_size == NULL || (src == NULL && src_size != 0) || (dst == NULL && dst_capacity != 0) || - workspace == NULL || ((uintptr_t)workspace & (sizeof(uint16_t) - 1u)) != 0 ) - { - return DMCOMP_ERR_PARAM; - } - *out_size = 0; - - end = dst_capacity != 0 ? out + dst_capacity : out; - b.p = (const uint8_t*)src; - b.end = src_size != 0 ? b.p + src_size : b.p; - b.bits = 0; - b.n = 0; - b.virt = 0; - - do - { - unsigned type; - int r; - - refill( &b ); - last = (int)take( &b, 1 ); - type = take( &b, 2 ); - if( overrun( &b ) ) - { - return DMCOMP_ERR_TRUNCATED; - } - - if( type == 0 ) - { - r = inflate_stored( &b, &out, end ); - } - else - { - if( type == 1 ) - { - r = build_fixed( tables ); - } - else if( type == 2 ) - { - r = build_dynamic( &b, tables ); - } - else - { - return DMCOMP_ERR_DATA; // Reserved - } - if( r == DMCOMP_OK ) - { - r = inflate_codes( &b, tables, (uint8_t*)dst, &out, end ); - } - } - if( r != DMCOMP_OK ) - { - return r; - } - } - while( !last ); - - if( overrun( &b ) ) - { - return DMCOMP_ERR_TRUNCATED; - } - *out_size = (size_t)(out - (uint8_t*)dst); - return DMCOMP_OK; -} - -int dmcomp_inflate( const void* src, size_t src_size, void* dst, size_t dst_capacity, size_t* out_size ) -{ - workspace_t workspace; // A few kB of the stack: dmcomp_inflate_ws() puts it where the caller wants - - return dmcomp_inflate_ws( src, src_size, dst, dst_capacity, out_size, &workspace ); -} diff --git a/lib/third-party/dmcomp/dmcomp_strerror.c b/lib/third-party/dmcomp/dmcomp_strerror.c deleted file mode 100644 index 3c2d5d09..00000000 --- a/lib/third-party/dmcomp/dmcomp_strerror.c +++ /dev/null @@ -1,19 +0,0 @@ -/** - * @file dmcomp_strerror.c - * @brief Names of the results (a file of its own: a firmware that does not log them does not carry the strings) - */ -#include "dmcomp.h" - -const char* dmcomp_strerror( int result ) -{ - switch( result ) - { - case DMCOMP_OK: return "ok"; - case DMCOMP_ERR_PARAM: return "invalid argument"; - case DMCOMP_ERR_DATA: return "invalid compressed data"; - case DMCOMP_ERR_TRUNCATED: return "compressed data truncated"; - case DMCOMP_ERR_OVERFLOW: return "output buffer too small"; - case DMCOMP_ERR_MEMORY: return "out of memory"; - default: return "unknown error"; - } -} diff --git a/lib/third-party/dmcomp/dmcomp_tables.h b/lib/third-party/dmcomp/dmcomp_tables.h deleted file mode 100644 index 22ba870e..00000000 --- a/lib/third-party/dmcomp/dmcomp_tables.h +++ /dev/null @@ -1,52 +0,0 @@ -/** - * @file dmcomp_tables.h - * @brief Constants of the DEFLATE format shared by the decoder and the encoder (RFC 1951, 3.2.5 - 3.2.7) - */ -#ifndef DMCOMP_TABLES_H -#define DMCOMP_TABLES_H - -#include - -#define DMCOMP_MAX_BITS 15 //!< Longest Huffman code of literals/lengths and distances -#define DMCOMP_MAX_CLEN_BITS 7 //!< Longest code of the code length alphabet -#define DMCOMP_LITLEN_SYMBOLS 286 //!< Literals 0..255, end of block 256, length codes 257..285 -#define DMCOMP_DIST_SYMBOLS 30 -#define DMCOMP_CLEN_SYMBOLS 19 -#define DMCOMP_MIN_MATCH 3 -#define DMCOMP_MAX_MATCH 258 - -/** Base length of the length codes 257..285 (index: code - 257) */ -static const uint16_t dmcomp_len_base[29] = -{ - 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, - 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258 -}; - -/** Extra bits of the length codes */ -static const uint8_t dmcomp_len_extra[29] = -{ - 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, - 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0 -}; - -/** Base distance of the distance codes */ -static const uint16_t dmcomp_dist_base[30] = -{ - 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, - 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577 -}; - -/** Extra bits of the distance codes */ -static const uint8_t dmcomp_dist_extra[30] = -{ - 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, - 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13 -}; - -/** Order in which the lengths of the code length alphabet are stored in a dynamic block header */ -static const uint8_t dmcomp_clen_order[DMCOMP_CLEN_SYMBOLS] = -{ - 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 -}; - -#endif /* DMCOMP_TABLES_H */ diff --git a/lib/third-party/dmod_dmcomp/CMakeLists.txt b/lib/third-party/dmod_dmcomp/CMakeLists.txt new file mode 100644 index 00000000..b8c0ef40 --- /dev/null +++ b/lib/third-party/dmod_dmcomp/CMakeLists.txt @@ -0,0 +1,36 @@ +set(MODULE_NAME dmod_dmcomp) + +# The sources are the submodule choco-technologies/dmcomp (lib/third-party/dmcomp): this only builds them the way DMOD +# builds its libraries (and writes the Makefile of the library, which the Make based builds use) +set(DMCOMP_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../dmcomp") +if(NOT EXISTS "${DMCOMP_DIR}/include/dmcomp.h") + message(FATAL_ERROR "The dmcomp library is missing: ${DMCOMP_DIR} is empty. " + "It is a git submodule - run 'git submodule update --init --recursive'.") +endif() + +# The decoder is always there; the encoder (it allocates ~330 kB with malloc and is ~10 kB of code) is for the host: +# DMOD_USE_DMCOMP_ENCODER is OFF in a configuration without the standard library, a firmware that only loads modules +add_library(${MODULE_NAME} STATIC + ../dmcomp/src/dmcomp_inflate.c + ../dmcomp/src/dmcomp_strerror.c +) +if(DMOD_USE_DMCOMP_ENCODER) + target_sources(${MODULE_NAME} PRIVATE ../dmcomp/src/dmcomp_deflate.c) +else() + target_compile_definitions(${MODULE_NAME} PUBLIC DMCOMP_NO_ENCODER) +endif() + +target_include_directories(${MODULE_NAME} + PUBLIC + ${DMCOMP_DIR}/include + PRIVATE + ${DMCOMP_DIR}/src +) + +target_compile_options(${MODULE_NAME} PRIVATE + -Os + -ffunction-sections + -fdata-sections +) + +create_library_makefile(${MODULE_NAME}) diff --git a/lib/third-party/dmcomp/Makefile b/lib/third-party/dmod_dmcomp/Makefile similarity index 81% rename from lib/third-party/dmcomp/Makefile rename to lib/third-party/dmod_dmcomp/Makefile index fae87d24..46508b4d 100644 --- a/lib/third-party/dmcomp/Makefile +++ b/lib/third-party/dmod_dmcomp/Makefile @@ -15,10 +15,11 @@ endif # Name of the module # DMOD_LIB_NAME=libdmod_dmcomp.a -DMOD_SOURCES=dmcomp_inflate.c\ - dmcomp_strerror.c\ - dmcomp_deflate.c -DMOD_INC_DIRS = $(DMOD_DIR)/lib/third-party/dmcomp +DMOD_SOURCES=../dmcomp/src/dmcomp_inflate.c\ + ../dmcomp/src/dmcomp_strerror.c\ + ../dmcomp/src/dmcomp_deflate.c +DMOD_INC_DIRS = $(DMOD_DIR)/lib/third-party/dmod_dmcomp/../dmcomp/include\ + $(DMOD_DIR)/lib/third-party/dmod_dmcomp/../dmcomp/src DMOD_LIBS = DMOD_GEN_HEADERS_IN = DMOD_DEFINITIONS = DMOD_CURRENT_ALLOCATOR="dmod_dmcomp" diff --git a/lib/third-party/dmod_dmcomp/README.md b/lib/third-party/dmod_dmcomp/README.md new file mode 100644 index 00000000..634ca5ae --- /dev/null +++ b/lib/third-party/dmod_dmcomp/README.md @@ -0,0 +1,8 @@ +# dmod_dmcomp + +The `dmod_dmcomp` library of DMOD: the DEFLATE coder and decoder of +[choco-technologies/dmcomp](https://github.com/choco-technologies/dmcomp), which is the git submodule +`lib/third-party/dmcomp`. There are no sources here, only the build (`CMakeLists.txt`, and the `Makefile` that it writes). + +It is the `deflate` method of `Dmod_Compression_*` (`src/system/if/dmod_if_comp.c`). To change the library, change dmcomp +(its tests, sanitizers and size checks are there) and move the submodule to the new commit. diff --git a/scripts/install-sdk.cmake b/scripts/install-sdk.cmake index 0f149485..17a6c881 100644 --- a/scripts/install-sdk.cmake +++ b/scripts/install-sdk.cmake @@ -29,10 +29,12 @@ install(DIRECTORY "${DMOD_DIR}/lib/third-party/FastLZ" DESTINATION "${DMOD_SDK_INSTALL_DIR}/lib/third-party" COMPONENT development FILES_MATCHING PATTERN "*.c" PATTERN "*.h" PATTERN "CMakeLists.txt" PATTERN "LICENSE.MIT") -install(DIRECTORY "${DMOD_DIR}/lib/third-party/dmcomp" +install(DIRECTORY "${DMOD_DIR}/lib/third-party/dmod_dmcomp" DESTINATION "${DMOD_SDK_INSTALL_DIR}/lib/third-party" COMPONENT development - FILES_MATCHING PATTERN "*.c" PATTERN "*.h" PATTERN "CMakeLists.txt" - PATTERN "LICENSE.MIT") + FILES_MATCHING PATTERN "CMakeLists.txt") +install(DIRECTORY "${DMOD_DIR}/lib/third-party/dmcomp/src" "${DMOD_DIR}/lib/third-party/dmcomp/include" + DESTINATION "${DMOD_SDK_INSTALL_DIR}/lib/third-party/dmcomp" COMPONENT development + FILES_MATCHING PATTERN "*.c" PATTERN "*.h") install(DIRECTORY "${DMOD_DIR}/scripts/" DESTINATION "${DMOD_SDK_INSTALL_DIR}/scripts" COMPONENT development FILES_MATCHING PATTERN "CMakeLists.txt" PATTERN "*.in" PATTERN "*.ld" @@ -48,7 +50,7 @@ install(FILES "${DMOD_DIR}/license.md" DESTINATION share/doc/dmod-dev RENAME copyright COMPONENT development) install(FILES "${DMOD_DIR}/lib/third-party/FastLZ/LICENSE.MIT" DESTINATION share/doc/dmod-dev RENAME copyright.FastLZ COMPONENT development) -install(FILES "${DMOD_DIR}/lib/third-party/dmcomp/LICENSE.MIT" +install(FILES "${DMOD_DIR}/lib/third-party/dmcomp/LICENSE" DESTINATION share/doc/dmod-dev RENAME copyright.dmcomp COMPONENT development) install(FILES "${DMOD_DIR}/docs/development-package.md" DESTINATION share/doc/dmod-dev COMPONENT development) diff --git a/src/system/Makefile b/src/system/Makefile index a4e1f62d..94fb5d88 100644 --- a/src/system/Makefile +++ b/src/system/Makefile @@ -51,7 +51,7 @@ DMOD_INC_DIRS = $(DMOD_DIR)/src/system\ $(DMOD_BUILD_DIR)\ $(DMOD_DIR)/src/common\ $(DMOD_DIR)/lib/third-party/FastLZ\ - $(DMOD_DIR)/lib/third-party/dmcomp + $(DMOD_DIR)/lib/third-party/dmod_dmcomp/../dmcomp/include DMOD_LIBS = dmod_inc\ dmod_common\ dmod_fastlz\ From f48d7cfb677a93b37df2caff0d0240ef4999c781 Mon Sep 17 00:00:00 2001 From: Jax Calder Date: Wed, 28 Oct 2026 14:41:45 +0000 Subject: [PATCH 4/4] dmcomp: the submodule on main (0126ec1) dmcomp#2 (dmcomp_inflate_ws) is merged; the submodule was on its commit 7314bf2. The merge commit has the same tree, so nothing changes in the build. Co-Authored-By: Claude Sonnet 5.5 --- lib/third-party/dmcomp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/third-party/dmcomp b/lib/third-party/dmcomp index 7314bf2e..0126ec1c 160000 --- a/lib/third-party/dmcomp +++ b/lib/third-party/dmcomp @@ -1 +1 @@ -Subproject commit 7314bf2e1d3becff64d5e0a8148ce3d86432d1d9 +Subproject commit 0126ec1c8a3d1c4a683356335e5f663f6b6535f4