diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eaae1b5f..9d35848b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + with: + submodules: recursive - name: Clean build directory run: rm -rf build @@ -190,6 +192,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + with: + submodules: recursive - name: Clean build directory run: rm -rf build @@ -281,6 +285,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + with: + submodules: recursive - name: Build dmod in MODULE mode run: | @@ -331,6 +337,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + with: + submodules: recursive - name: Build dmod with Make in MODULE mode run: make DMOD_MODE=DMOD_MODULE diff --git a/.github/workflows/debian-package.yml b/.github/workflows/debian-package.yml index b29d89fc..79265045 100644 --- a/.github/workflows/debian-package.yml +++ b/.github/workflows/debian-package.yml @@ -14,6 +14,8 @@ jobs: examples: ['OFF', 'ON'] steps: - uses: actions/checkout@v4 + with: + submodules: recursive - name: Install build dependencies run: sudo apt-get update && sudo apt-get install -y build-essential cmake dpkg-dev libcurl4-openssl-dev unzip bubblewrap python3 fonts-dejavu-core - name: Bootstrap host tools diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..49212efb --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "lib/third-party/dmcomp"] + path = lib/third-party/dmcomp + url = https://github.com/choco-technologies/dmcomp.git diff --git a/CMakeLists.txt b/CMakeLists.txt index e7ca3aa4..2368ceb6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -224,6 +224,9 @@ target_link_libraries(dmod INTERFACE dmod_inc dmod_common) if (DMOD_USE_FASTLZ) target_link_libraries(dmod INTERFACE dmod_fastlz) endif() +if (DMOD_USE_DMCOMP) + target_link_libraries(dmod INTERFACE dmod_dmcomp) +endif() if (DMOD_SYSTEM) target_link_libraries(dmod INTERFACE dmod_system) elseif (DMOD_MODULE) diff --git a/Docker/build.sh b/Docker/build.sh index 79cef54f..a3285a62 100755 --- a/Docker/build.sh +++ b/Docker/build.sh @@ -103,6 +103,15 @@ function isTargetSelected() [[ ",$TARGETS," == *",$1,"* ]] } +# +# The image is built from this directory (COPY . in the Dockerfile): the submodules (lib/third-party/dmcomp) have to be +# in it +# +function updateSubmodules() +{ + git -C "$ROOT_DIR" submodule update --init --recursive +} + # # Builds the environment image # @@ -152,6 +161,7 @@ function buildOpencode() # MAIN # prepareScript "$@" +updateSubmodules isTargetSelected env && buildEnv isTargetSelected dmod && build isTargetSelected claude && buildClaude diff --git a/cmake/runtime/CMakeLists.txt b/cmake/runtime/CMakeLists.txt index 8b4672e5..38f34d0f 100644 --- a/cmake/runtime/CMakeLists.txt +++ b/cmake/runtime/CMakeLists.txt @@ -27,3 +27,7 @@ if(DMOD_USE_FASTLZ) add_subdirectory("${DMOD_DIR}/lib/third-party/FastLZ" fastlz) target_link_libraries(dmod INTERFACE dmod_fastlz) endif() +if(DMOD_USE_DMCOMP) + add_subdirectory("${DMOD_DIR}/lib/third-party/dmod_dmcomp" dmcomp) + target_link_libraries(dmod INTERFACE dmod_dmcomp) +endif() diff --git a/dmod-cfg.cmake b/dmod-cfg.cmake index f2056759..2e854c7e 100644 --- a/dmod-cfg.cmake +++ b/dmod-cfg.cmake @@ -37,6 +37,19 @@ set(DMOD_USE_TIME ON CACHE BOOL "Enable to use time.h for system time functions # Use FastLZ compression library set(DMOD_USE_FASTLZ ON CACHE BOOL "Enable to use the FastLZ compression library") +# Use the dmcomp library: the "deflate" compression method (the decoder) +set(DMOD_USE_DMCOMP ON CACHE BOOL "Enable to use the dmcomp library (the deflate compression method)") + +# Build the deflate encoder (Dmod_Compression_Pack) of the dmcomp library: for the host tools. It allocates its work memory +# (~330 kB) with malloc, so by default it is built when DMOD_USE_STDLIB is (a firmware has no malloc of the standard +# library and nothing to compress for - it gets the decoder only) +if(DMOD_USE_STDLIB) + set(DMOD_USE_DMCOMP_ENCODER_DEFAULT ON) +else() + set(DMOD_USE_DMCOMP_ENCODER_DEFAULT OFF) +endif() +set(DMOD_USE_DMCOMP_ENCODER ${DMOD_USE_DMCOMP_ENCODER_DEFAULT} CACHE BOOL "Enable to build the deflate encoder of the dmcomp library (it needs malloc and ~330 kB of the heap)") + # Maximum number of modules set(DMOD_MAX_MODULES 30 CACHE STRING "Maximum number of modules") diff --git a/dmod-cfg.mk b/dmod-cfg.mk index 4d4128b9..3925cab9 100644 --- a/dmod-cfg.mk +++ b/dmod-cfg.mk @@ -41,6 +41,10 @@ DMOD_USE_TIME=ON # Use FastLZ compression library DMOD_USE_FASTLZ=ON +# Use the dmcomp library (the deflate compression method) and its encoder (for the host tools) +DMOD_USE_DMCOMP=ON +DMOD_USE_DMCOMP_ENCODER=ON + # Maximum number of modules DMOD_MAX_MODULES=30 diff --git a/dmod-config.h.in b/dmod-config.h.in index fb443204..c4a5bdb5 100644 --- a/dmod-config.h.in +++ b/dmod-config.h.in @@ -22,6 +22,8 @@ #define DMOD_USE_TERMIOS @DMOD_USE_TERMIOS@ #define DMOD_USE_TIME @DMOD_USE_TIME@ #define DMOD_USE_FASTLZ @DMOD_USE_FASTLZ@ +#define DMOD_USE_DMCOMP @DMOD_USE_DMCOMP@ +#define DMOD_USE_DMCOMP_ENCODER @DMOD_USE_DMCOMP_ENCODER@ #define DMOD_IMPLEMENT_PRINTF @DMOD_IMPLEMENT_PRINTF@ #define DMOD_IMPLEMENT_SCANF @DMOD_IMPLEMENT_SCANF@ #define DMOD_MODE "@DMOD_MODE@" diff --git a/dmod-defaults.cmake b/dmod-defaults.cmake index 3adb3aec..f1c5ae7a 100644 --- a/dmod-defaults.cmake +++ b/dmod-defaults.cmake @@ -68,6 +68,20 @@ if (NOT DEFINED DMOD_USE_FASTLZ) set(DMOD_USE_FASTLZ ON) endif() +if (NOT DEFINED DMOD_USE_DMCOMP) + set(DMOD_USE_DMCOMP ON) +endif() + +if (NOT DEFINED DMOD_USE_DMCOMP_ENCODER) + # The encoder allocates its work memory (~330 kB) with malloc: it is for the host. Without the standard library + # (a firmware) there is no malloc, and nothing to compress for - only the decoder is built. + if (DMOD_USE_STDLIB) + set(DMOD_USE_DMCOMP_ENCODER ON) + else() + set(DMOD_USE_DMCOMP_ENCODER OFF) + endif() +endif() + if (NOT DEFINED DMOD_MAX_MODULES) set(DMOD_MAX_MODULES 30) endif() diff --git a/dmod-defaults.mk b/dmod-defaults.mk index 88a1b0d7..4633f1ce 100644 --- a/dmod-defaults.mk +++ b/dmod-defaults.mk @@ -73,6 +73,20 @@ ifndef DMOD_USE_FASTLZ DMOD_USE_FASTLZ = ON endif +ifndef DMOD_USE_DMCOMP + DMOD_USE_DMCOMP = ON +endif + +ifndef DMOD_USE_DMCOMP_ENCODER + # The encoder allocates its work memory (~330 kB) with malloc: it is for the host. Without the standard library + # (a firmware) there is no malloc, and nothing to compress for - only the decoder is built. + ifeq ($(DMOD_USE_STDLIB),ON) + DMOD_USE_DMCOMP_ENCODER = ON + else + DMOD_USE_DMCOMP_ENCODER = OFF + endif +endif + ifndef DMOD_MAX_MODULES DMOD_MAX_MODULES = 30 endif diff --git a/docs/cmake-functions.md b/docs/cmake-functions.md index 8078d7aa..2231fdb7 100644 --- a/docs/cmake-functions.md +++ b/docs/cmake-functions.md @@ -334,7 +334,7 @@ The following variables should be set before calling `dmod_add_executable` or `d - `DMOD_MANUAL_LOAD` - Manual load flag (default: OFF) - `DMOD_MAL_IMPLS` - List of MAL (Module Abstraction Layer) interfaces implemented - `DMOD_DIF_IMPLS` - List of DIF (Device Interface) implementations -- `DMOD_COMPRESSION_METHOD` - Compression method for DMFC files (default: "fastlz") +- `DMOD_COMPRESSION_METHOD` - Compression method for DMFC files: "fastlz" (default) or "deflate" (~20 % smaller; the loader needs the `deflate` decoder, see `DMOD_USE_DMCOMP`) - `DMOD_ASSETS_PATHS` - List of directories with the sources of a module's assets (views, images, fonts). When set, they are converted into `DMOD_VIEWS_DIR` (default: `/views`) by the `dmod_assets` target, which the module depends on - with tools `dmf-get` downloads at configure time, only for the kinds of files found: - `*.dmvs` views - `todmv` - `.dmv`; - images (`DMOD_IMAGE_EXTENSIONS`, default `png jpg jpeg jpe jfif bmp qoi gif tif tiff svg`) - `todmvi` with the `dmimg` decoder of each extension (`dmimg_png`, `dmimg_jpeg` for `.jpg`, ...; `DMOD_IMAGE_DECODERS` overrides the list) - `.dmvi`, next to the views, so a view finds them by name. An optional `.ini` next to an image (`logo.png.ini`) sets its conversion: `format` (`auto`, `rgb565`, `rgb565a8`, `argb8888`, `i8`, `a8`, `a4`), `size` (`WIDTHxHEIGHT`, scaled down to fit), `compression` (`fastlz`, `none`), `dither` (`on`/`off`), `blur` (pixels: a Gaussian blur of that standard deviation, after scaling), `crop` (`X,Y,W,H`: only that of it, after scaling and blurring - what a box shows), `radius` (pixels: its corners rounded after `crop`, transparent outside them - an image in a rounded box). Keys before any section apply to every output; each `[section]` is an output `
.dmvi` of its own (e.g. `[logo]` and `[logo-mask]` from one image). diff --git a/docs/development-package.md b/docs/development-package.md index a70e68c9..57ca305c 100644 --- a/docs/development-package.md +++ b/docs/development-package.md @@ -37,7 +37,7 @@ and compiled runtime objects are created in the consumer's build directory; the installed SDK remains read-only. Existing module API declaration rules still apply; see the [CMake function reference](cmake-functions.md). -Set DMOD options before `find_package`, for example `DMOD_USE_FASTLZ`, +Set DMOD options before `find_package`, for example `DMOD_USE_FASTLZ`, `DMOD_USE_DMCOMP` (the `deflate` method) and `DMOD_USE_DMCOMP_ENCODER`, `DMOD_DMF_DIR`, `DMOD_DMFC_DIR`, `DMOD_REPO_DIR` and `DMOD_ARCH`. Set per-module options such as `DMOD_AUTHOR_NAME`, `DMOD_STACK_SIZE`, `DMOD_MAL_IMPLS` and `DMOD_DMR_PATH` before calling `dmod_add_library` or `dmod_add_executable`. diff --git a/docs/tools-installation.md b/docs/tools-installation.md index 92747216..a21349de 100644 --- a/docs/tools-installation.md +++ b/docs/tools-installation.md @@ -220,7 +220,8 @@ Full documentation: [todmm-tool.md](todmm-tool.md) todmfc input.dmf output.dmfc # Specify compression method and level -todmfc input.dmf output.dmfc fastlz 3 +todmfc input.dmf output.dmfc fastlz 1 +todmfc input.dmf output.dmfc deflate 9 # ~20 % smaller than fastlz # Display available compression methods todmfc --help @@ -229,8 +230,10 @@ todmfc --help **Parameters:** - `path/to/file.dmf` - Input DMF file - `path/to/output.dmfc` - Output DMFC file -- `[compression_method]` - Optional compression method (default: fastlz) -- `[level]` - Optional compression level (default: 2) +- `[compression_method]` - Optional compression method: `fastlz` (default) or `deflate` (RFC 1951, from the + [dmcomp](../lib/third-party/dmcomp) library). A loader has to know the method: the `deflate` decoder is 1.7 kB of code + and needs a work memory of ~3 kB from the heap, so keep `fastlz` for the modules that older firmware has to load +- `[level]` - Optional compression level: `fastlz` 1 or 2 (default: 2), `deflate` 0 (stored) to 9 (default: 9) ### todmp diff --git a/examples/system/dif_test/Makefile b/examples/system/dif_test/Makefile index a821e8ac..8627a446 100644 --- a/examples/system/dif_test/Makefile +++ b/examples/system/dif_test/Makefile @@ -35,6 +35,9 @@ DEFINITIONS = DMOD_CURRENT_ALLOCATOR="dif_test" ifeq ($(DMOD_USE_FASTLZ),ON) DMOD_LIBS += dmod_fastlz endif +ifeq ($(DMOD_USE_DMCOMP),ON) + DMOD_LIBS += dmod_dmcomp +endif # ----------------------------------------------------------------------------- # List of objects diff --git a/examples/system/dif_test/dmod-cfg.mk b/examples/system/dif_test/dmod-cfg.mk index 962e9869..bb4c147a 100644 --- a/examples/system/dif_test/dmod-cfg.mk +++ b/examples/system/dif_test/dmod-cfg.mk @@ -12,6 +12,8 @@ DMOD_USE_STDIO=ON DMOD_USE_ASSERT=ON DMOD_USE_PTHREAD=ON DMOD_USE_FASTLZ=ON +DMOD_USE_DMCOMP=ON +DMOD_USE_DMCOMP_ENCODER=ON DMOD_USE_MMAN=ON DMOD_USE_ALIGNED_ALLOC=ON DMOD_USE_ALIGNED_MALLOC_MOCK=OFF diff --git a/examples/system/dmod_loader/Makefile b/examples/system/dmod_loader/Makefile index 40777536..3f79712e 100644 --- a/examples/system/dmod_loader/Makefile +++ b/examples/system/dmod_loader/Makefile @@ -33,6 +33,9 @@ DEFINITIONS = DMOD_CURRENT_ALLOCATOR="dmod_loader" ifeq ($(DMOD_USE_FASTLZ),ON) DMOD_LIBS += dmod_fastlz endif +ifeq ($(DMOD_USE_DMCOMP),ON) + DMOD_LIBS += dmod_dmcomp +endif # ----------------------------------------------------------------------------- # List of objects diff --git a/examples/system/dmod_loader/dmod-cfg.mk b/examples/system/dmod_loader/dmod-cfg.mk index 962e9869..bb4c147a 100644 --- a/examples/system/dmod_loader/dmod-cfg.mk +++ b/examples/system/dmod_loader/dmod-cfg.mk @@ -12,6 +12,8 @@ DMOD_USE_STDIO=ON DMOD_USE_ASSERT=ON DMOD_USE_PTHREAD=ON DMOD_USE_FASTLZ=ON +DMOD_USE_DMCOMP=ON +DMOD_USE_DMCOMP_ENCODER=ON DMOD_USE_MMAN=ON DMOD_USE_ALIGNED_ALLOC=ON DMOD_USE_ALIGNED_MALLOC_MOCK=OFF diff --git a/inc/dmod_sal.h b/inc/dmod_sal.h index 7097502d..1e9f1560 100644 --- a/inc/dmod_sal.h +++ b/inc/dmod_sal.h @@ -677,6 +677,72 @@ DMOD_BUILTIN_API(Dmod, 1.0, int, _Reboot, ( void ) ); //! @} +/** + * @defgroup DMOD_SAL_FIRMWARE Firmware Interface + * @ingroup DMOD_SAL + * + * What the running firmware tells the platform, and what the platform keeps for it between the runs. + * + * A platform that has more than one firmware image (banks) starts a new one on trial: it stays only if it says that it + * runs properly (Dmod_FirmwareReady()); if it does not, the platform goes back to the previous one after a reset. The + * persistent data (Dmod_WritePersistency(), Dmod_ReadPersistency()) is kept apart from the images, so that an update of + * the firmware - or a firmware that did not start - does not touch it. + * + * All of the functions are implemented by the platform (the boot loader of the board, for instance); the default + * (weak) implementations of the host and of the platforms without them return -ENOSYS and have no side effects. + * @{ + */ + +/** + * @brief Tell the platform that the running firmware has started properly and may stay. + * + * What "properly" means is up to the firmware, so it is the firmware that calls this - when the console is there, the + * file system is mounted, the network is up, or whatever its project needs to be able to be updated again. A platform + * with a firmware on trial keeps it from then on; calling it again, or on a firmware that is not on trial, is harmless. + * Until it is called a platform may restart the system (a watchdog) and fall back to the previous firmware. + * + * @return 0 on success, a negative errno value otherwise (-ENOSYS: the platform has nothing to confirm) + */ +DMOD_BUILTIN_API(Dmod, 1.0, int, _FirmwareReady, ( void ) ); + +/** + * @brief Store data that survives a reset and an update of the firmware. + * + * The item @p Id is replaced with the @p Size bytes of @p Data. @p Data may be NULL only when @p Size is 0, which + * removes the item. The platform decides the sizes, and the number of the items, that it can keep (-ENOSPC when it + * cannot). + * + * Writing may erase and program a non-volatile memory: it can take long (hundreds of milliseconds for a sector of a + * flash) and, where the code runs from the same memory, stop the whole system meanwhile. Do not do it often, nor from + * a time critical thread; the data is for the settings, not for the logs. + * + * @param Id Identifier of the item, chosen by the caller (the same one reads it back) + * @param Data The data + * @param Size Size of the data in bytes + * + * @return 0 on success, a negative errno value otherwise (-EINVAL: wrong arguments, -ENOSPC: no room, + * -EIO: the memory failed, -ENOSYS: the platform keeps nothing) + */ +DMOD_BUILTIN_API(Dmod, 1.0, int, _WritePersistency, ( uint32_t Id, const void* Data, size_t Size ) ); + +/** + * @brief Read data stored with Dmod_WritePersistency(). + * + * Copies at most @p BufferSize bytes of the item @p Id. The result is the size of the whole item, which may be more + * than @p BufferSize (then the rest was not copied): call it with a NULL @p Buffer and 0 as @p BufferSize to learn the + * size first. + * + * @param Id Identifier of the item + * @param Buffer Where to copy the data; may be NULL when @p BufferSize is 0 + * @param BufferSize Size of the buffer in bytes + * + * @return Size of the item in bytes (0 or more), or a negative errno value (-ENOENT: there is no such item, + * -EINVAL: wrong arguments, -EIO: the memory failed, -ENOSYS: the platform keeps nothing) + */ +DMOD_BUILTIN_API(Dmod, 1.0, int, _ReadPersistency, ( uint32_t Id, void* Buffer, size_t BufferSize ) ); + +//! @} + #ifdef __cplusplus } #endif diff --git a/lib/third-party/CMakeLists.txt b/lib/third-party/CMakeLists.txt index 1ef320a5..32a22cdb 100644 --- a/lib/third-party/CMakeLists.txt +++ b/lib/third-party/CMakeLists.txt @@ -1 +1,2 @@ -add_subdirectory(FastLZ) \ No newline at end of file +add_subdirectory(FastLZ) +add_subdirectory(dmod_dmcomp) \ No newline at end of file diff --git a/lib/third-party/Makefile b/lib/third-party/Makefile index 6b6c20a7..b79e1a93 100644 --- a/lib/third-party/Makefile +++ b/lib/third-party/Makefile @@ -6,7 +6,7 @@ DMOD_DIR=../.. # ----------------------------------------------------------------------------- # Subdirectories # ----------------------------------------------------------------------------- -SUBDIRS = FastLZ +SUBDIRS = FastLZ dmod_dmcomp SYSTEM_SUBDIRS = MODULE_SUBDIRS = diff --git a/lib/third-party/dmcomp b/lib/third-party/dmcomp new file mode 160000 index 00000000..0126ec1c --- /dev/null +++ b/lib/third-party/dmcomp @@ -0,0 +1 @@ +Subproject commit 0126ec1c8a3d1c4a683356335e5f663f6b6535f4 diff --git a/lib/third-party/dmod_dmcomp/CMakeLists.txt b/lib/third-party/dmod_dmcomp/CMakeLists.txt new file mode 100644 index 00000000..b8c0ef40 --- /dev/null +++ b/lib/third-party/dmod_dmcomp/CMakeLists.txt @@ -0,0 +1,36 @@ +set(MODULE_NAME dmod_dmcomp) + +# The sources are the submodule choco-technologies/dmcomp (lib/third-party/dmcomp): this only builds them the way DMOD +# builds its libraries (and writes the Makefile of the library, which the Make based builds use) +set(DMCOMP_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../dmcomp") +if(NOT EXISTS "${DMCOMP_DIR}/include/dmcomp.h") + message(FATAL_ERROR "The dmcomp library is missing: ${DMCOMP_DIR} is empty. " + "It is a git submodule - run 'git submodule update --init --recursive'.") +endif() + +# The decoder is always there; the encoder (it allocates ~330 kB with malloc and is ~10 kB of code) is for the host: +# DMOD_USE_DMCOMP_ENCODER is OFF in a configuration without the standard library, a firmware that only loads modules +add_library(${MODULE_NAME} STATIC + ../dmcomp/src/dmcomp_inflate.c + ../dmcomp/src/dmcomp_strerror.c +) +if(DMOD_USE_DMCOMP_ENCODER) + target_sources(${MODULE_NAME} PRIVATE ../dmcomp/src/dmcomp_deflate.c) +else() + target_compile_definitions(${MODULE_NAME} PUBLIC DMCOMP_NO_ENCODER) +endif() + +target_include_directories(${MODULE_NAME} + PUBLIC + ${DMCOMP_DIR}/include + PRIVATE + ${DMCOMP_DIR}/src +) + +target_compile_options(${MODULE_NAME} PRIVATE + -Os + -ffunction-sections + -fdata-sections +) + +create_library_makefile(${MODULE_NAME}) diff --git a/lib/third-party/dmod_dmcomp/Makefile b/lib/third-party/dmod_dmcomp/Makefile new file mode 100644 index 00000000..46508b4d --- /dev/null +++ b/lib/third-party/dmod_dmcomp/Makefile @@ -0,0 +1,36 @@ +# ############################################################################## +# Makefile for the dmod_dmcomp module +# +# This Makefile is used to build the dmod_dmcomp module library +# +# DONT EDIT THIS FILE - it is automatically generated. +# Edit the scripts/Makefile-lib.in file instead. +# +# ############################################################################## +ifeq ($(DMOD_DIR),) + DMOD_DIR = ../../../ +endif + +# +# Name of the module +# +DMOD_LIB_NAME=libdmod_dmcomp.a +DMOD_SOURCES=../dmcomp/src/dmcomp_inflate.c\ + ../dmcomp/src/dmcomp_strerror.c\ + ../dmcomp/src/dmcomp_deflate.c +DMOD_INC_DIRS = $(DMOD_DIR)/lib/third-party/dmod_dmcomp/../dmcomp/include\ + $(DMOD_DIR)/lib/third-party/dmod_dmcomp/../dmcomp/src +DMOD_LIBS = +DMOD_GEN_HEADERS_IN = +DMOD_DEFINITIONS = DMOD_CURRENT_ALLOCATOR="dmod_dmcomp" + +# ----------------------------------------------------------------------------- +# Initialization of paths +# ----------------------------------------------------------------------------- +include $(DMOD_DIR)/paths.mk + +# ----------------------------------------------------------------------------- +# Including the template for the library +# ----------------------------------------------------------------------------- +DMOD_LIB_OBJS_DIR = $(DMOD_OBJS_DIR)/dmod_dmcomp +include $(DMOD_SLIB_FILE_PATH) diff --git a/lib/third-party/dmod_dmcomp/README.md b/lib/third-party/dmod_dmcomp/README.md new file mode 100644 index 00000000..634ca5ae --- /dev/null +++ b/lib/third-party/dmod_dmcomp/README.md @@ -0,0 +1,8 @@ +# dmod_dmcomp + +The `dmod_dmcomp` library of DMOD: the DEFLATE coder and decoder of +[choco-technologies/dmcomp](https://github.com/choco-technologies/dmcomp), which is the git submodule +`lib/third-party/dmcomp`. There are no sources here, only the build (`CMakeLists.txt`, and the `Makefile` that it writes). + +It is the `deflate` method of `Dmod_Compression_*` (`src/system/if/dmod_if_comp.c`). To change the library, change dmcomp +(its tests, sanitizers and size checks are there) and move the submodule to the new commit. diff --git a/paths.cmake b/paths.cmake index 38ae3b17..adc7cb13 100644 --- a/paths.cmake +++ b/paths.cmake @@ -147,6 +147,9 @@ macro(dmod_setup_external_module) if (DMOD_USE_FASTLZ) target_link_libraries(dmod INTERFACE dmod_fastlz) endif() + if (DMOD_USE_DMCOMP) + target_link_libraries(dmod INTERFACE dmod_dmcomp) + endif() if (DMOD_SYSTEM) target_link_libraries(dmod INTERFACE dmod_system) elseif (DMOD_MODULE) diff --git a/scripts/install-sdk.cmake b/scripts/install-sdk.cmake index c158e124..17a6c881 100644 --- a/scripts/install-sdk.cmake +++ b/scripts/install-sdk.cmake @@ -29,6 +29,12 @@ install(DIRECTORY "${DMOD_DIR}/lib/third-party/FastLZ" DESTINATION "${DMOD_SDK_INSTALL_DIR}/lib/third-party" COMPONENT development FILES_MATCHING PATTERN "*.c" PATTERN "*.h" PATTERN "CMakeLists.txt" PATTERN "LICENSE.MIT") +install(DIRECTORY "${DMOD_DIR}/lib/third-party/dmod_dmcomp" + DESTINATION "${DMOD_SDK_INSTALL_DIR}/lib/third-party" COMPONENT development + FILES_MATCHING PATTERN "CMakeLists.txt") +install(DIRECTORY "${DMOD_DIR}/lib/third-party/dmcomp/src" "${DMOD_DIR}/lib/third-party/dmcomp/include" + DESTINATION "${DMOD_SDK_INSTALL_DIR}/lib/third-party/dmcomp" COMPONENT development + FILES_MATCHING PATTERN "*.c" PATTERN "*.h") install(DIRECTORY "${DMOD_DIR}/scripts/" DESTINATION "${DMOD_SDK_INSTALL_DIR}/scripts" COMPONENT development FILES_MATCHING PATTERN "CMakeLists.txt" PATTERN "*.in" PATTERN "*.ld" @@ -44,5 +50,7 @@ install(FILES "${DMOD_DIR}/license.md" DESTINATION share/doc/dmod-dev RENAME copyright COMPONENT development) install(FILES "${DMOD_DIR}/lib/third-party/FastLZ/LICENSE.MIT" DESTINATION share/doc/dmod-dev RENAME copyright.FastLZ COMPONENT development) +install(FILES "${DMOD_DIR}/lib/third-party/dmcomp/LICENSE" + DESTINATION share/doc/dmod-dev RENAME copyright.dmcomp COMPONENT development) install(FILES "${DMOD_DIR}/docs/development-package.md" DESTINATION share/doc/dmod-dev COMPONENT development) diff --git a/scripts/system-tool.mk b/scripts/system-tool.mk index e7786845..2261c7ee 100644 --- a/scripts/system-tool.mk +++ b/scripts/system-tool.mk @@ -28,6 +28,9 @@ OBJ_OUTPUT_DIR = $(DMOD_OBJS_DIR)/$(PROJECT_NAME) ifeq ($(DMOD_USE_FASTLZ),ON) DMOD_LIBS += dmod_fastlz endif +ifeq ($(DMOD_USE_DMCOMP),ON) + DMOD_LIBS += dmod_dmcomp +endif # ----------------------------------------------------------------------------- # List of objects diff --git a/src/system/CMakeLists.txt b/src/system/CMakeLists.txt index e48fe771..56e53ece 100644 --- a/src/system/CMakeLists.txt +++ b/src/system/CMakeLists.txt @@ -30,6 +30,7 @@ set(SYSTEM_SOURCES if/dmod_if_input.c if/dmod_if_proc.c if/dmod_if_power.c + if/dmod_if_firmware.c if/dmod_if_time.c Dmod.cpp ) @@ -62,6 +63,7 @@ target_link_libraries(${MODULE_NAME} dmod_inc dmod_common dmod_fastlz + dmod_dmcomp ) if (DMOD_USE_PTHREAD) diff --git a/src/system/Makefile b/src/system/Makefile index fe086b88..94fb5d88 100644 --- a/src/system/Makefile +++ b/src/system/Makefile @@ -42,6 +42,7 @@ DMOD_SOURCES=private/dmod_vars.c\ if/dmod_if_input.c\ if/dmod_if_proc.c\ if/dmod_if_power.c\ + if/dmod_if_firmware.c\ if/dmod_if_time.c\ Dmod.cpp\ if/dmod_prf.c @@ -49,10 +50,12 @@ DMOD_INC_DIRS = $(DMOD_DIR)/src/system\ $(DMOD_DIR)/inc\ $(DMOD_BUILD_DIR)\ $(DMOD_DIR)/src/common\ - $(DMOD_DIR)/lib/third-party/FastLZ + $(DMOD_DIR)/lib/third-party/FastLZ\ + $(DMOD_DIR)/lib/third-party/dmod_dmcomp/../dmcomp/include DMOD_LIBS = dmod_inc\ dmod_common\ dmod_fastlz\ + dmod_dmcomp\ pthread DMOD_GEN_HEADERS_IN = DMOD_DEFINITIONS = DMOD_CURRENT_ALLOCATOR="dmod_system" diff --git a/src/system/if/dmod_if_comp.c b/src/system/if/dmod_if_comp.c index d959459c..b197d289 100644 --- a/src/system/if/dmod_if_comp.c +++ b/src/system/if/dmod_if_comp.c @@ -35,13 +35,19 @@ #if DMOD_USE_FASTLZ # include "fastlz.h" #endif +#if DMOD_USE_DMCOMP +# include "dmcomp.h" +#endif //============================================================================== // GLOBAL VARIABLES //============================================================================== static const char* SupportedCompressions[] = { #if DMOD_USE_FASTLZ - "fastlz", + "fastlz", // The first one is the default of todmfc: it has to stay fastlz (the loaders that were built before deflate do not read the other) + #endif + #if DMOD_USE_DMCOMP + "deflate", // Raw DEFLATE (RFC 1951) by dmcomp: ~20 % smaller than fastlz, 1.7 kB of the decoder #endif }; static const size_t SupportedCompressionsCount = sizeof(SupportedCompressions) / sizeof(SupportedCompressions[0]); @@ -73,9 +79,15 @@ DMOD_INPUT_WEAK_API_DECLARATION(Dmod, 1.0, size_t, _Compression_GetMaxSize, ( co max_output_size = SrcSize + (SrcSize / 16) + 64 + 3; } #endif + #if DMOD_USE_DMCOMP && DMOD_USE_DMCOMP_ENCODER + else if( strcmp( Name, "deflate" ) == 0 ) + { + max_output_size = dmcomp_deflate_bound( SrcSize ); + } + #endif else { - DMOD_LOG_ERROR("Compression algorithm '%s' is not supported\n", Name); + DMOD_LOG_ERROR("Compression algorithm '%s' is not supported (or this build cannot compress with it)\n", Name); } return max_output_size; @@ -107,9 +119,21 @@ DMOD_INPUT_WEAK_API_DECLARATION(Dmod, 1.0, size_t, _Compression_Pack, ( const ch Ret = fastlz_compress_level(Level, Src, SrcSize, Dest ); } #endif + #if DMOD_USE_DMCOMP && DMOD_USE_DMCOMP_ENCODER + else if( strcmp( Name, "deflate" ) == 0 ) + { + // Level 0 (stored) .. 9 (the smallest); dmcomp takes a level out of the range as its default + int result = dmcomp_deflate( Src, SrcSize, Dest, DestSize, &Ret, Level ); + if( result != DMCOMP_OK ) + { + DMOD_LOG_ERROR("Cannot compress with deflate: %s\n", dmcomp_strerror( result )); + Ret = 0; + } + } + #endif else { - DMOD_LOG_ERROR("Compression algorithm '%s' is not supported\n", Name); + DMOD_LOG_ERROR("Compression algorithm '%s' is not supported (or this build cannot compress with it)\n", Name); } return Ret; @@ -140,7 +164,34 @@ DMOD_INPUT_WEAK_API_DECLARATION(Dmod, 1.0, size_t, _Compression_Unpack, ( const Ret = fastlz_decompress( Src, SrcSize, Dest, DestSize ); } #endif + #if DMOD_USE_DMCOMP + #if DMOD_USE_FASTLZ + else + #endif + if( strcmp( Name, "deflate" ) == 0 ) + { + // The tables of the decoder (~3 kB) come from the heap, not from the stack: a module is loaded from whatever + // thread asks for it, and some of them have 4 kB of the stack + void* workspace = Dmod_Malloc( dmcomp_inflate_workspace_size() ); + if( workspace == NULL ) + { + DMOD_LOG_ERROR("Cannot decompress with deflate: out of memory\n"); + } + else + { + int result = dmcomp_inflate_ws( Src, SrcSize, Dest, DestSize, &Ret, workspace ); + if( result != DMCOMP_OK ) + { + DMOD_LOG_ERROR("Cannot decompress with deflate: %s\n", dmcomp_strerror( result )); + Ret = 0; + } + Dmod_Free( workspace ); + } + } + #endif + #if DMOD_USE_FASTLZ || DMOD_USE_DMCOMP else + #endif { DMOD_LOG_ERROR("Compression algorithm '%s' is not supported\n", Name); } diff --git a/src/system/if/dmod_if_firmware.c b/src/system/if/dmod_if_firmware.c new file mode 100644 index 00000000..9faea7f4 --- /dev/null +++ b/src/system/if/dmod_if_firmware.c @@ -0,0 +1,29 @@ +/** + * @brief Platform-overridable firmware interface: the confirmation of a firmware on trial and the persistent data. + * + * The platform with a boot loader that keeps more than one firmware (see DMOD_SAL_FIRMWARE in dmod_sal.h) overrides + * these; everywhere else (the host included) there is nothing to confirm and nothing is kept. + */ +#include "dmod.h" +#include + +DMOD_INPUT_WEAK_API_DECLARATION(Dmod, 1.0, int, _FirmwareReady, ( void )) +{ + return -ENOSYS; +} + +DMOD_INPUT_WEAK_API_DECLARATION(Dmod, 1.0, int, _WritePersistency, ( uint32_t Id, const void* Data, size_t Size )) +{ + (void)Id; + (void)Data; + (void)Size; + return -ENOSYS; +} + +DMOD_INPUT_WEAK_API_DECLARATION(Dmod, 1.0, int, _ReadPersistency, ( uint32_t Id, void* Buffer, size_t BufferSize )) +{ + (void)Id; + (void)Buffer; + (void)BufferSize; + return -ENOSYS; +} diff --git a/templates/system/dmod-cfg.cmake b/templates/system/dmod-cfg.cmake index 646d6cf6..226e22d4 100644 --- a/templates/system/dmod-cfg.cmake +++ b/templates/system/dmod-cfg.cmake @@ -31,6 +31,19 @@ set(DMOD_USE_REALLOC ON CACHE BOOL "Enable to use the realloc function") # Use FastLZ compression library set(DMOD_USE_FASTLZ ON CACHE BOOL "Enable to use the FastLZ compression library") +# Use the dmcomp library: the "deflate" compression method (the decoder) +set(DMOD_USE_DMCOMP ON CACHE BOOL "Enable to use the dmcomp library (the deflate compression method)") + +# Build the deflate encoder (Dmod_Compression_Pack) of the dmcomp library: for the host tools. It allocates its work memory +# (~330 kB) with malloc, so by default it is built when DMOD_USE_STDLIB is (a firmware has no malloc of the standard +# library and nothing to compress for - it gets the decoder only) +if(DMOD_USE_STDLIB) + set(DMOD_USE_DMCOMP_ENCODER_DEFAULT ON) +else() + set(DMOD_USE_DMCOMP_ENCODER_DEFAULT OFF) +endif() +set(DMOD_USE_DMCOMP_ENCODER ${DMOD_USE_DMCOMP_ENCODER_DEFAULT} CACHE BOOL "Enable to build the deflate encoder of the dmcomp library (it needs malloc and ~330 kB of the heap)") + # Maximum number of modules set(DMOD_MAX_MODULES 30 CACHE STRING "Maximum number of modules") diff --git a/templates/system/dmod-cfg.mk b/templates/system/dmod-cfg.mk index a36b0bdb..c449b4b6 100644 --- a/templates/system/dmod-cfg.mk +++ b/templates/system/dmod-cfg.mk @@ -38,6 +38,10 @@ DMOD_USE_REALLOC=ON # Use FastLZ compression library DMOD_USE_FASTLZ=ON +# Use the dmcomp library (the deflate compression method) and its encoder (for the host tools) +DMOD_USE_DMCOMP=ON +DMOD_USE_DMCOMP_ENCODER=ON + # Maximum number of modules DMOD_MAX_MODULES=30 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 97de2c8e..5cc3a85b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -38,7 +38,7 @@ foreach(test_src ${TEST_SOURCES}) get_filename_component(test_name ${test_src} NAME_WE) add_executable(${test_name} ${test_src} ut_main.cpp) dmod_set_current_allocator(${test_name}) - target_link_libraries(${test_name} gtest gtest_main gmock dmod_system dmod_common dmod_inc dmod_fastlz dmod_version dmod_manifest dmod_dependencies dmod_resource) + target_link_libraries(${test_name} gtest gtest_main gmock dmod_system dmod_common dmod_inc dmod_fastlz dmod_dmcomp dmod_version dmod_manifest dmod_dependencies dmod_resource) add_test(NAME ${test_name} COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${test_name}) list(APPEND TEST_FILES ${test_name}) diff --git a/tests/system/if/CMakeLists.txt b/tests/system/if/CMakeLists.txt index 4d9427c7..e03362ff 100644 --- a/tests/system/if/CMakeLists.txt +++ b/tests/system/if/CMakeLists.txt @@ -18,5 +18,7 @@ set(TEST_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/tests_dmod_str_merge.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tests_dmod_module_exists.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tests_dmod_reboot.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests_dmod_firmware.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests_dmod_compression.cpp PARENT_SCOPE ) diff --git a/tests/system/if/tests_dmod_compression.cpp b/tests/system/if/tests_dmod_compression.cpp new file mode 100644 index 00000000..039ae6a6 --- /dev/null +++ b/tests/system/if/tests_dmod_compression.cpp @@ -0,0 +1,186 @@ +#define DMOD_PRIVATE +#include +#include +#include +#include +#include +#include "dmod_sal.h" + +// The heap of this test: counts what the compression takes and gives back, and can refuse to give anything +static int g_LiveAllocations = 0; +static bool g_FailAllocations = false; + +extern "C" +{ + void* Dmod_MallocEx(size_t Size, const char* ModuleName) + { + (void)ModuleName; + if( g_FailAllocations ) + { + return nullptr; + } + void* ptr = malloc(Size); + if( ptr != nullptr ) + { + g_LiveAllocations++; + } + return ptr; + } + + void Dmod_FreeEx(void* ptr, bool Concatenate) + { + (void)Concatenate; + if( ptr != nullptr ) + { + g_LiveAllocations--; + free(ptr); + } + } +} + +class DmodCompressionTest : public ::testing::Test +{ +protected: + void SetUp() override + { + g_LiveAllocations = 0; + g_FailAllocations = false; + } + + // Something that compresses like a module does: structured, repetitive, with some noise + static std::vector Sample(size_t size) + { + static const char* words[] = { "module", "loader", "flash", "driver", "device", "memory", "thread", "mount" }; + std::vector data; + uint32_t state = 12345; + + while( data.size() < size ) + { + state = state * 1103515245u + 12345u; + const char* word = words[(state >> 16) % 8]; + data.insert(data.end(), word, word + strlen(word)); + data.push_back((state >> 8) % 11 == 0 ? '\n' : ' '); + if( (state >> 20) % 5 == 0 ) + { + data.push_back((uint8_t)(state >> 3)); + } + } + data.resize(size); + return data; + } + + static size_t Pack(const char* name, int level, const std::vector& data, std::vector& packed) + { + size_t max = Dmod_Compression_GetMaxSize(name, level, data.size()); + packed.assign(max, 0); + size_t size = Dmod_Compression_Pack(name, level, packed.data(), packed.size(), data.data(), data.size()); + packed.resize(size); + return size; + } +}; + +TEST_F(DmodCompressionTest, SupportedMethods) +{ + EXPECT_TRUE(Dmod_Compression_IsSupported("fastlz")); + EXPECT_TRUE(Dmod_Compression_IsSupported("deflate")); + EXPECT_FALSE(Dmod_Compression_IsSupported("lzma")); + + // fastlz stays the first: it is the default of todmfc, and what the loaders without deflate can read + ASSERT_NE(Dmod_Compression_GetNextSupported(nullptr), nullptr); + EXPECT_STREQ(Dmod_Compression_GetNextSupported(nullptr), "fastlz"); + EXPECT_STREQ(Dmod_Compression_GetNextSupported("fastlz"), "deflate"); + EXPECT_EQ(Dmod_Compression_GetNextSupported("deflate"), nullptr); +} + +TEST_F(DmodCompressionTest, DeflateRoundTripOnEveryLevel) +{ + const std::vector data = Sample(20000); + + for( int level = 0; level <= 9; level++ ) + { + std::vector packed; + size_t max = Dmod_Compression_GetMaxSize("deflate", level, data.size()); + + ASSERT_GE(max, data.size()) << "level " << level; + ASSERT_GT(Pack("deflate", level, data, packed), 0u) << "level " << level; + EXPECT_LE(packed.size(), max); + if( level > 0 ) + { + EXPECT_LT(packed.size(), data.size() / 2) << "level " << level; + } + + std::vector unpacked(data.size()); + EXPECT_EQ(Dmod_Compression_Unpack("deflate", unpacked.data(), unpacked.size(), packed.data(), packed.size()), + data.size()) << "level " << level; + EXPECT_EQ(unpacked, data) << "level " << level; + } + // The tables of the decoder come from the heap and go back to it + EXPECT_EQ(g_LiveAllocations, 0); +} + +TEST_F(DmodCompressionTest, DeflateIsSmallerThanFastlz) +{ + const std::vector data = Sample(60000); + std::vector deflate, fastlz; + + ASSERT_GT(Pack("deflate", 9, data, deflate), 0u); + ASSERT_GT(Pack("fastlz", 2, data, fastlz), 0u); + EXPECT_LT(deflate.size(), fastlz.size()); +} + +TEST_F(DmodCompressionTest, FastlzStillWorks) +{ + const std::vector data = Sample(10000); + std::vector packed; + + ASSERT_GT(Pack("fastlz", 2, data, packed), 0u); + std::vector unpacked(data.size()); + EXPECT_EQ(Dmod_Compression_Unpack("fastlz", unpacked.data(), unpacked.size(), packed.data(), packed.size()), data.size()); + EXPECT_EQ(unpacked, data); +} + +TEST_F(DmodCompressionTest, DeflateRefusesWhatIsNotDeflate) +{ + const std::vector data = Sample(5000); + std::vector packed; + std::vector out(data.size()); + + ASSERT_GT(Pack("deflate", 6, data, packed), 0u); + + // Garbage + std::vector garbage(300); + for( size_t i = 0; i < garbage.size(); i++ ) garbage[i] = (uint8_t)(i * 7 + 3); + EXPECT_EQ(Dmod_Compression_Unpack("deflate", out.data(), out.size(), garbage.data(), garbage.size()), 0u); + + // Cut short + EXPECT_EQ(Dmod_Compression_Unpack("deflate", out.data(), out.size(), packed.data(), packed.size() / 2), 0u); + + // An output buffer that is too small + EXPECT_EQ(Dmod_Compression_Unpack("deflate", out.data(), out.size() - 1, packed.data(), packed.size()), 0u); + + EXPECT_EQ(g_LiveAllocations, 0); +} + +TEST_F(DmodCompressionTest, DeflateDecoderWithoutMemory) +{ + const std::vector data = Sample(5000); + std::vector packed; + std::vector out(data.size()); + + ASSERT_GT(Pack("deflate", 6, data, packed), 0u); + + g_FailAllocations = true; + EXPECT_EQ(Dmod_Compression_Unpack("deflate", out.data(), out.size(), packed.data(), packed.size()), 0u); + g_FailAllocations = false; + EXPECT_EQ(Dmod_Compression_Unpack("deflate", out.data(), out.size(), packed.data(), packed.size()), data.size()); +} + +TEST_F(DmodCompressionTest, NothingToCompress) +{ + uint8_t out[64]; + uint8_t in = 0; + + EXPECT_EQ(Dmod_Compression_Pack("deflate", 6, out, sizeof(out), &in, 0), 0u); + EXPECT_EQ(Dmod_Compression_Unpack("deflate", out, sizeof(out), &in, 0), 0u); + EXPECT_EQ(Dmod_Compression_GetMaxSize("deflate", 6, 0), 0u); +} diff --git a/tests/system/if/tests_dmod_firmware.cpp b/tests/system/if/tests_dmod_firmware.cpp new file mode 100644 index 00000000..104fc460 --- /dev/null +++ b/tests/system/if/tests_dmod_firmware.cpp @@ -0,0 +1,34 @@ +#define DMOD_PRIVATE +#include +#include +#include "dmod_sal.h" + +// The host has no boot loader and no persistent memory: the default (weak) implementations say so and do nothing. +// A platform overrides them (dmod-boot does, with the selector in the first sector of the flash). + +TEST(DmodFirmwareTest, FirmwareReadyIsUnsupportedByDefault) +{ + EXPECT_EQ(Dmod_FirmwareReady(), -ENOSYS); + // Repeated confirmations must be harmless as well. + EXPECT_EQ(Dmod_FirmwareReady(), -ENOSYS); +} + +TEST(DmodFirmwareTest, WritePersistencyIsUnsupportedByDefault) +{ + const char data[] = "settings"; + + EXPECT_EQ(Dmod_WritePersistency(1, data, sizeof(data)), -ENOSYS); + // Removing an item is a write of nothing + EXPECT_EQ(Dmod_WritePersistency(1, nullptr, 0), -ENOSYS); +} + +TEST(DmodFirmwareTest, ReadPersistencyIsUnsupportedByDefault) +{ + char buffer[16] = { 'x' }; + + EXPECT_EQ(Dmod_ReadPersistency(1, buffer, sizeof(buffer)), -ENOSYS); + // Asking for the size only + EXPECT_EQ(Dmod_ReadPersistency(1, nullptr, 0), -ENOSYS); + // Nothing was copied + EXPECT_EQ(buffer[0], 'x'); +} diff --git a/tests/system/public/CMakeLists.txt b/tests/system/public/CMakeLists.txt index 3644572b..adcae860 100644 --- a/tests/system/public/CMakeLists.txt +++ b/tests/system/public/CMakeLists.txt @@ -1,6 +1,7 @@ set(TEST_SOURCES ${TEST_SOURCES} ${CMAKE_CURRENT_SOURCE_DIR}/tests_dmod_dmfc_api.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests_dmod_dmfc_deflate.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tests_dmod_getname.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tests_dmod_arch.cpp ${CMAKE_CURRENT_SOURCE_DIR}/tests_dmod_findmatch.cpp diff --git a/tests/system/public/tests_dmod_dmfc_deflate.cpp b/tests/system/public/tests_dmod_dmfc_deflate.cpp new file mode 100644 index 00000000..93113896 --- /dev/null +++ b/tests/system/public/tests_dmod_dmfc_deflate.cpp @@ -0,0 +1,76 @@ +#define DMOD_PRIVATE +#include +#include +#include +#include "dmod.h" +#include "dmod_defs.h" +#include "dmod_types.h" + +// DMF -> DMFC -> DMF with every compression method. A DMF for the conversion only has to start with the module header +// (the signature and the name); the rest is the module's code, which this is not - but it is as compressible. +static std::vector FakeModule(size_t size) +{ + std::vector dmf(size); + uint32_t state = 777; + + for( size_t i = 0; i < size; i++ ) + { + state = state * 1664525u + 1013904223u; + // Words of a few kinds of values, like the instructions and the tables of a real module + dmf[i] = (i % 4 == 3) ? 0xE0 : (uint8_t)((state >> 24) % 24); + } + Dmod_ModuleHeader_t* header = reinterpret_cast(dmf.data()); + memset(header, 0, sizeof(*header)); + header->Signature = DMOD_HEADER_SIGNATURE; + strncpy(header->Name, "fakemodule", sizeof(header->Name) - 1); + return dmf; +} + +class DmodDmfcMethodsTest : public ::testing::TestWithParam +{ +}; + +TEST_P(DmodDmfcMethodsTest, ToDmfcAndBack) +{ + const char* method = GetParam(); + const std::vector dmf = FakeModule(12000); + void* dmfc = nullptr; + size_t dmfcSize = 0; + + ASSERT_TRUE(Dmod_ToDMFC(method, 2, dmf.data(), dmf.size(), &dmfc, &dmfcSize)); + ASSERT_NE(dmfc, nullptr); + EXPECT_LT(dmfcSize, dmf.size()); + EXPECT_TRUE(Dmod_IsDMFC(dmfc, dmfcSize)); + + // The method is named in the header: the loader decompresses with it + const Dmod_DmfcHeader_t* header = static_cast(dmfc); + EXPECT_STREQ(header->Compression, method); + EXPECT_EQ(header->OriginalSize, dmf.size()); + + void* back = nullptr; + size_t backSize = 0; + ASSERT_TRUE(Dmod_FromDMFC(dmfc, dmfcSize, &back, &backSize)); + ASSERT_EQ(backSize, dmf.size()); + EXPECT_EQ(memcmp(back, dmf.data(), dmf.size()), 0); + + Dmod_Free(back); + Dmod_Free(dmfc); +} + +// A DMFC with the compressed data cut short: deflate notices (the stream ends before its last block), so the loader +// does not get a half of a module. (fastlz gives what it has decoded, and Dmod_FromDMFC only warns about the size.) +TEST(DmodDmfcDeflateTest, DamagedDmfcIsRefused) +{ + const std::vector dmf = FakeModule(12000); + void* dmfc = nullptr; + size_t dmfcSize = 0; + + ASSERT_TRUE(Dmod_ToDMFC("deflate", 2, dmf.data(), dmf.size(), &dmfc, &dmfcSize)); + + void* back = nullptr; + size_t backSize = 0; + EXPECT_FALSE(Dmod_FromDMFC(dmfc, dmfcSize - dmfcSize / 3, &back, &backSize)); + Dmod_Free(dmfc); +} + +INSTANTIATE_TEST_SUITE_P(Methods, DmodDmfcMethodsTest, ::testing::Values("fastlz", "deflate")); diff --git a/tools/system/todmd/dmod-cfg.mk b/tools/system/todmd/dmod-cfg.mk index 6b9df1f4..cd6e22f2 100644 --- a/tools/system/todmd/dmod-cfg.mk +++ b/tools/system/todmd/dmod-cfg.mk @@ -12,6 +12,8 @@ DMOD_USE_STDIO=ON DMOD_USE_ASSERT=ON DMOD_USE_PTHREAD=ON DMOD_USE_FASTLZ=ON +DMOD_USE_DMCOMP=ON +DMOD_USE_DMCOMP_ENCODER=ON DMOD_USE_MMAN=ON DMOD_USE_ALIGNED_ALLOC=ON DMOD_USE_ALIGNED_MALLOC_MOCK=OFF diff --git a/tools/system/todmfc/README.md b/tools/system/todmfc/README.md index 518f7f6b..44c6a9b7 100644 --- a/tools/system/todmfc/README.md +++ b/tools/system/todmfc/README.md @@ -7,7 +7,7 @@ ## Features - **Compression of DMF files**: Converts standard DMF files to compressed DMFC format -- **Multiple compression methods**: Supports various compression algorithms (currently: fastlz) +- **Multiple compression methods**: Supports various compression algorithms (currently: fastlz and deflate) - **Configurable compression levels**: Allows tuning between compression ratio and speed - **Size comparison**: Displays original and compressed file sizes with compression ratio - **Simple interface**: Easy-to-use command-line tool @@ -22,6 +22,13 @@ todmfc path/to/file.dmf path/to/output.dmfc This will compress the DMF file using the default compression method (fastlz) and level (2). +The `deflate` method (RFC 1951, [dmcomp](../../../lib/third-party/dmcomp)) makes the files ~20 % smaller than `fastlz`; its decoder is 1.7 kB of code. A loader has to know it, so the modules that older firmware has to load stay `fastlz`: + +```bash +todmfc path/to/file.dmf path/to/output.dmfc deflate # level 9 +todmfc path/to/file.dmf path/to/output.dmfc deflate 6 +``` + ### Advanced Usage ```bash @@ -32,7 +39,7 @@ todmfc path/to/file.dmf path/to/output.dmfc [compression_method] [level] - `path/to/file.dmf` - Input DMF file to compress - `path/to/output.dmfc` - Output DMFC file path - `[compression_method]` - (Optional) Compression algorithm to use (default: fastlz) -- `[level]` - (Optional) Compression level (default: 2) +- `[level]` - (Optional) Compression level: fastlz 1 or 2 (default: 2), deflate 0 (stored) .. 9 (default: 9) ### Options diff --git a/tools/system/todmfc/dmod-cfg.mk b/tools/system/todmfc/dmod-cfg.mk index b5fedbc1..ceffbf63 100644 --- a/tools/system/todmfc/dmod-cfg.mk +++ b/tools/system/todmfc/dmod-cfg.mk @@ -12,6 +12,8 @@ DMOD_USE_STDIO=ON DMOD_USE_ASSERT=ON DMOD_USE_PTHREAD=ON DMOD_USE_FASTLZ=ON +DMOD_USE_DMCOMP=ON +DMOD_USE_DMCOMP_ENCODER=ON DMOD_USE_MMAN=ON DMOD_USE_ALIGNED_ALLOC=ON DMOD_USE_ALIGNED_MALLOC_MOCK=OFF diff --git a/tools/system/todmfc/main.c b/tools/system/todmfc/main.c index 62e54229..2c5dfeb6 100644 --- a/tools/system/todmfc/main.c +++ b/tools/system/todmfc/main.c @@ -43,7 +43,9 @@ void PrintHelp( const char* AppName ) printf(" -v, --version Print version information\n"); printf(" [compression_method] (default: %s) Compression method method to be used. ", Dmod_Compression_GetNextSupported(NULL)); PrintSupportedCompressionMethods(); - printf(" [level] (default: 2) Compression level - depends on the [compression_method]\n"); + printf(" [level] Compression level - depends on the [compression_method]:\n"); + printf(" fastlz: 1 (faster) or 2 (smaller, the default)\n"); + printf(" deflate: 0 (stored) .. 9 (the smallest, the default)\n"); } // ----------------------------------------- @@ -96,7 +98,8 @@ int main( int argc, char *argv[] ) } } - int level = 2; + // The default level depends on the method: fastlz has two (1, 2), deflate ten (0 - 9) + int level = strcmp( compression, "deflate" ) == 0 ? 9 : 2; if( argc >= 5 ) { level = atoi( argv[4] ); diff --git a/tools/system/todmm/dmod-cfg.mk b/tools/system/todmm/dmod-cfg.mk index 6b9df1f4..cd6e22f2 100644 --- a/tools/system/todmm/dmod-cfg.mk +++ b/tools/system/todmm/dmod-cfg.mk @@ -12,6 +12,8 @@ DMOD_USE_STDIO=ON DMOD_USE_ASSERT=ON DMOD_USE_PTHREAD=ON DMOD_USE_FASTLZ=ON +DMOD_USE_DMCOMP=ON +DMOD_USE_DMCOMP_ENCODER=ON DMOD_USE_MMAN=ON DMOD_USE_ALIGNED_ALLOC=ON DMOD_USE_ALIGNED_MALLOC_MOCK=OFF