deflate compression method (dmcomp, a submodule) and the firmware SAL: FirmwareReady, Write/ReadPersistency

.github/workflows/ci.yml

@@ -16,6 +16,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
       - name: Clean build directory
         run: rm -rf build
@@ -190,6 +192,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
       - name: Clean build directory
         run: rm -rf build
@@ -281,6 +285,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
       - name: Build dmod in MODULE mode
         run: |
@@ -331,6 +337,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
       - name: Build dmod with Make in MODULE mode
         run: make DMOD_MODE=DMOD_MODULE

.github/workflows/debian-package.yml

@@ -14,6 +14,8 @@ jobs:
         examples: ['OFF', 'ON']
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
       - name: Install build dependencies
         run: sudo apt-get update && sudo apt-get install -y build-essential cmake dpkg-dev libcurl4-openssl-dev unzip bubblewrap python3 fonts-dejavu-core
       - name: Bootstrap host tools

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "lib/third-party/dmcomp"]
+	path = lib/third-party/dmcomp
+	url = https://github.com/choco-technologies/dmcomp.git

CMakeLists.txt

@@ -224,6 +224,9 @@ target_link_libraries(dmod INTERFACE dmod_inc dmod_common)
 if (DMOD_USE_FASTLZ)
 	target_link_libraries(dmod INTERFACE dmod_fastlz)
 endif()
+if (DMOD_USE_DMCOMP)
+	target_link_libraries(dmod INTERFACE dmod_dmcomp)
+endif()
 if (DMOD_SYSTEM)
 	target_link_libraries(dmod INTERFACE dmod_system)
 elseif (DMOD_MODULE)

Docker/build.sh

@@ -103,6 +103,15 @@ function isTargetSelected()
     [[ ",$TARGETS," == *",$1,"* ]]
 }
 
+#
+#   The image is built from this directory (COPY . in the Dockerfile): the submodules (lib/third-party/dmcomp) have to be
+#   in it
+#
+function updateSubmodules()
+{
+    git -C "$ROOT_DIR" submodule update --init --recursive
+}
+
 #
 #   Builds the environment image
 #   
@@ -152,6 +161,7 @@ function buildOpencode()
 #   MAIN
 #
 prepareScript "$@"
+updateSubmodules
 isTargetSelected env && buildEnv
 isTargetSelected dmod && build
 isTargetSelected claude && buildClaude

cmake/runtime/CMakeLists.txt

@@ -27,3 +27,7 @@ if(DMOD_USE_FASTLZ)
     add_subdirectory("${DMOD_DIR}/lib/third-party/FastLZ" fastlz)
     target_link_libraries(dmod INTERFACE dmod_fastlz)
 endif()
+if(DMOD_USE_DMCOMP)
+    add_subdirectory("${DMOD_DIR}/lib/third-party/dmod_dmcomp" dmcomp)
+    target_link_libraries(dmod INTERFACE dmod_dmcomp)
+endif()

dmod-cfg.cmake

@@ -37,6 +37,19 @@ set(DMOD_USE_TIME ON CACHE BOOL "Enable to use time.h for system time functions
 # Use FastLZ compression library
 set(DMOD_USE_FASTLZ ON CACHE BOOL "Enable to use the FastLZ compression library")
 
+# Use the dmcomp library: the "deflate" compression method (the decoder)
+set(DMOD_USE_DMCOMP ON CACHE BOOL "Enable to use the dmcomp library (the deflate compression method)")
+
+# Build the deflate encoder (Dmod_Compression_Pack) of the dmcomp library: for the host tools. It allocates its work memory
+# (~330 kB) with malloc, so by default it is built when DMOD_USE_STDLIB is (a firmware has no malloc of the standard
+# library and nothing to compress for - it gets the decoder only)
+if(DMOD_USE_STDLIB)
+    set(DMOD_USE_DMCOMP_ENCODER_DEFAULT ON)
+else()
+    set(DMOD_USE_DMCOMP_ENCODER_DEFAULT OFF)
+endif()
+set(DMOD_USE_DMCOMP_ENCODER ${DMOD_USE_DMCOMP_ENCODER_DEFAULT} CACHE BOOL "Enable to build the deflate encoder of the dmcomp library (it needs malloc and ~330 kB of the heap)")
+
 # Maximum number of modules
 set(DMOD_MAX_MODULES 30 CACHE STRING "Maximum number of modules")
 

dmod-cfg.mk

@@ -41,6 +41,10 @@ DMOD_USE_TIME=ON
 # Use FastLZ compression library
 DMOD_USE_FASTLZ=ON
 
+# Use the dmcomp library (the deflate compression method) and its encoder (for the host tools)
+DMOD_USE_DMCOMP=ON
+DMOD_USE_DMCOMP_ENCODER=ON
+
 # Maximum number of modules
 DMOD_MAX_MODULES=30
 

dmod-config.h.in

@@ -22,6 +22,8 @@
 #define DMOD_USE_TERMIOS                @DMOD_USE_TERMIOS@
 #define DMOD_USE_TIME                 @DMOD_USE_TIME@
 #define DMOD_USE_FASTLZ                 @DMOD_USE_FASTLZ@
+#define DMOD_USE_DMCOMP                 @DMOD_USE_DMCOMP@
+#define DMOD_USE_DMCOMP_ENCODER         @DMOD_USE_DMCOMP_ENCODER@
 #define DMOD_IMPLEMENT_PRINTF           @DMOD_IMPLEMENT_PRINTF@
 #define DMOD_IMPLEMENT_SCANF            @DMOD_IMPLEMENT_SCANF@
 #define DMOD_MODE           "@DMOD_MODE@"

dmod-defaults.cmake

@@ -68,6 +68,20 @@ if (NOT DEFINED DMOD_USE_FASTLZ)
     set(DMOD_USE_FASTLZ ON)
 endif()
 
+if (NOT DEFINED DMOD_USE_DMCOMP)
+    set(DMOD_USE_DMCOMP ON)
+endif()
+
+if (NOT DEFINED DMOD_USE_DMCOMP_ENCODER)
+    # The encoder allocates its work memory (~330 kB) with malloc: it is for the host. Without the standard library
+    # (a firmware) there is no malloc, and nothing to compress for - only the decoder is built.
+    if (DMOD_USE_STDLIB)
+        set(DMOD_USE_DMCOMP_ENCODER ON)
+    else()
+        set(DMOD_USE_DMCOMP_ENCODER OFF)
+    endif()
+endif()
+
 if (NOT DEFINED DMOD_MAX_MODULES)
     set(DMOD_MAX_MODULES 30)
 endif()

dmod-defaults.mk

@@ -73,6 +73,20 @@ ifndef DMOD_USE_FASTLZ
     DMOD_USE_FASTLZ = ON
 endif
 
+ifndef DMOD_USE_DMCOMP
+    DMOD_USE_DMCOMP = ON
+endif
+
+ifndef DMOD_USE_DMCOMP_ENCODER
+    # The encoder allocates its work memory (~330 kB) with malloc: it is for the host. Without the standard library
+    # (a firmware) there is no malloc, and nothing to compress for - only the decoder is built.
+    ifeq ($(DMOD_USE_STDLIB),ON)
+        DMOD_USE_DMCOMP_ENCODER = ON
+    else
+        DMOD_USE_DMCOMP_ENCODER = OFF
+    endif
+endif
+
 ifndef DMOD_MAX_MODULES
     DMOD_MAX_MODULES = 30
 endif

docs/cmake-functions.md

@@ -334,7 +334,7 @@ The following variables should be set before calling `dmod_add_executable` or `d
 - `DMOD_MANUAL_LOAD` - Manual load flag (default: OFF)
 - `DMOD_MAL_IMPLS` - List of MAL (Module Abstraction Layer) interfaces implemented
 - `DMOD_DIF_IMPLS` - List of DIF (Device Interface) implementations
-- `DMOD_COMPRESSION_METHOD` - Compression method for DMFC files (default: "fastlz")
+- `DMOD_COMPRESSION_METHOD` - Compression method for DMFC files: "fastlz" (default) or "deflate" (~20 % smaller; the loader needs the `deflate` decoder, see `DMOD_USE_DMCOMP`)
 - `DMOD_ASSETS_PATHS` - List of directories with the sources of a module's assets (views, images, fonts). When set, they are converted into `DMOD_VIEWS_DIR` (default: `<build>/views`) by the `dmod_assets` target, which the module depends on - with tools `dmf-get` downloads at configure time, only for the kinds of files found:
   - `*.dmvs` views - `todmv` - `<name>.dmv`;
   - images (`DMOD_IMAGE_EXTENSIONS`, default `png jpg jpeg jpe jfif bmp qoi gif tif tiff svg`) - `todmvi` with the `dmimg` decoder of each extension (`dmimg_png`, `dmimg_jpeg` for `.jpg`, ...; `DMOD_IMAGE_DECODERS` overrides the list) - `<name>.dmvi`, next to the views, so a view finds them by name. An optional `<image>.ini` next to an image (`logo.png.ini`) sets its conversion: `format` (`auto`, `rgb565`, `rgb565a8`, `argb8888`, `i8`, `a8`, `a4`), `size` (`WIDTHxHEIGHT`, scaled down to fit), `compression` (`fastlz`, `none`), `dither` (`on`/`off`), `blur` (pixels: a Gaussian blur of that standard deviation, after scaling), `crop` (`X,Y,W,H`: only that of it, after scaling and blurring - what a box shows), `radius` (pixels: its corners rounded after `crop`, transparent outside them - an image in a rounded box). Keys before any section apply to every output; each `[section]` is an output `<section>.dmvi` of its own (e.g. `[logo]` and `[logo-mask]` from one image).

docs/development-package.md

@@ -37,7 +37,7 @@ and compiled runtime objects are created in the consumer's build directory;
 the installed SDK remains read-only. Existing module API declaration rules
 still apply; see the [CMake function reference](cmake-functions.md).
 
-Set DMOD options before `find_package`, for example `DMOD_USE_FASTLZ`,
+Set DMOD options before `find_package`, for example `DMOD_USE_FASTLZ`, `DMOD_USE_DMCOMP` (the `deflate` method) and `DMOD_USE_DMCOMP_ENCODER`,
 `DMOD_DMF_DIR`, `DMOD_DMFC_DIR`, `DMOD_REPO_DIR` and `DMOD_ARCH`. Set per-module
 options such as `DMOD_AUTHOR_NAME`, `DMOD_STACK_SIZE`, `DMOD_MAL_IMPLS` and
 `DMOD_DMR_PATH` before calling `dmod_add_library` or `dmod_add_executable`.

docs/tools-installation.md

@@ -220,7 +220,8 @@ Full documentation: [todmm-tool.md](todmm-tool.md)
 todmfc input.dmf output.dmfc
 
 # Specify compression method and level
-todmfc input.dmf output.dmfc fastlz 3
+todmfc input.dmf output.dmfc fastlz 1
+todmfc input.dmf output.dmfc deflate 9    # ~20 % smaller than fastlz
 
 # Display available compression methods
 todmfc --help
@@ -229,8 +230,10 @@ todmfc --help
 **Parameters:**
 - `path/to/file.dmf` - Input DMF file
 - `path/to/output.dmfc` - Output DMFC file
-- `[compression_method]` - Optional compression method (default: fastlz)
-- `[level]` - Optional compression level (default: 2)
+- `[compression_method]` - Optional compression method: `fastlz` (default) or `deflate` (RFC 1951, from the
+  [dmcomp](../lib/third-party/dmcomp) library). A loader has to know the method: the `deflate` decoder is 1.7 kB of code
+  and needs a work memory of ~3 kB from the heap, so keep `fastlz` for the modules that older firmware has to load
+- `[level]` - Optional compression level: `fastlz` 1 or 2 (default: 2), `deflate` 0 (stored) to 9 (default: 9)
 
 ### todmp
 

examples/system/dif_test/Makefile

@@ -35,6 +35,9 @@ DEFINITIONS 	= DMOD_CURRENT_ALLOCATOR="dif_test"
 ifeq ($(DMOD_USE_FASTLZ),ON)
 	DMOD_LIBS += dmod_fastlz
 endif
+ifeq ($(DMOD_USE_DMCOMP),ON)
+	DMOD_LIBS += dmod_dmcomp
+endif
 
 # -----------------------------------------------------------------------------
 # 	List of objects

examples/system/dif_test/dmod-cfg.mk

@@ -12,6 +12,8 @@ DMOD_USE_STDIO=ON
 DMOD_USE_ASSERT=ON
 DMOD_USE_PTHREAD=ON
 DMOD_USE_FASTLZ=ON
+DMOD_USE_DMCOMP=ON
+DMOD_USE_DMCOMP_ENCODER=ON
 DMOD_USE_MMAN=ON
 DMOD_USE_ALIGNED_ALLOC=ON
 DMOD_USE_ALIGNED_MALLOC_MOCK=OFF

examples/system/dmod_loader/Makefile

@@ -33,6 +33,9 @@ DEFINITIONS 	= DMOD_CURRENT_ALLOCATOR="dmod_loader"
 ifeq ($(DMOD_USE_FASTLZ),ON)
 	DMOD_LIBS += dmod_fastlz
 endif
+ifeq ($(DMOD_USE_DMCOMP),ON)
+	DMOD_LIBS += dmod_dmcomp
+endif
 
 # -----------------------------------------------------------------------------
 # 	List of objects

examples/system/dmod_loader/dmod-cfg.mk

@@ -12,6 +12,8 @@ DMOD_USE_STDIO=ON
 DMOD_USE_ASSERT=ON
 DMOD_USE_PTHREAD=ON
 DMOD_USE_FASTLZ=ON
+DMOD_USE_DMCOMP=ON
+DMOD_USE_DMCOMP_ENCODER=ON
 DMOD_USE_MMAN=ON
 DMOD_USE_ALIGNED_ALLOC=ON
 DMOD_USE_ALIGNED_MALLOC_MOCK=OFF

inc/dmod_sal.h

@@ -677,6 +677,72 @@ DMOD_BUILTIN_API(Dmod, 1.0, int, _Reboot, ( void ) );
 
 //! @}
 
+/**
+ * @defgroup DMOD_SAL_FIRMWARE Firmware Interface
+ * @ingroup DMOD_SAL
+ *
+ * What the running firmware tells the platform, and what the platform keeps for it between the runs.
+ *
+ * A platform that has more than one firmware image (banks) starts a new one on trial: it stays only if it says that it
+ * runs properly (Dmod_FirmwareReady()); if it does not, the platform goes back to the previous one after a reset. The
+ * persistent data (Dmod_WritePersistency(), Dmod_ReadPersistency()) is kept apart from the images, so that an update of
+ * the firmware - or a firmware that did not start - does not touch it.
+ *
+ * All of the functions are implemented by the platform (the boot loader of the board, for instance); the default
+ * (weak) implementations of the host and of the platforms without them return -ENOSYS and have no side effects.
+ * @{
+ */
+
+/**
+ * @brief Tell the platform that the running firmware has started properly and may stay.
+ *
+ * What "properly" means is up to the firmware, so it is the firmware that calls this - when the console is there, the
+ * file system is mounted, the network is up, or whatever its project needs to be able to be updated again. A platform
+ * with a firmware on trial keeps it from then on; calling it again, or on a firmware that is not on trial, is harmless.
+ * Until it is called a platform may restart the system (a watchdog) and fall back to the previous firmware.
+ *
+ * @return 0 on success, a negative errno value otherwise (-ENOSYS: the platform has nothing to confirm)
+ */
+DMOD_BUILTIN_API(Dmod, 1.0, int, _FirmwareReady, ( void ) );
+
+/**
+ * @brief Store data that survives a reset and an update of the firmware.
+ *
+ * The item @p Id is replaced with the @p Size bytes of @p Data. @p Data may be NULL only when @p Size is 0, which
+ * removes the item. The platform decides the sizes, and the number of the items, that it can keep (-ENOSPC when it
+ * cannot).
+ *
+ * Writing may erase and program a non-volatile memory: it can take long (hundreds of milliseconds for a sector of a
+ * flash) and, where the code runs from the same memory, stop the whole system meanwhile. Do not do it often, nor from
+ * a time critical thread; the data is for the settings, not for the logs.
+ *
+ * @param Id    Identifier of the item, chosen by the caller (the same one reads it back)
+ * @param Data  The data
+ * @param Size  Size of the data in bytes
+ *
+ * @return 0 on success, a negative errno value otherwise (-EINVAL: wrong arguments, -ENOSPC: no room,
+ *         -EIO: the memory failed, -ENOSYS: the platform keeps nothing)
+ */
+DMOD_BUILTIN_API(Dmod, 1.0, int, _WritePersistency, ( uint32_t Id, const void* Data, size_t Size ) );
+
+/**
+ * @brief Read data stored with Dmod_WritePersistency().
+ *
+ * Copies at most @p BufferSize bytes of the item @p Id. The result is the size of the whole item, which may be more
+ * than @p BufferSize (then the rest was not copied): call it with a NULL @p Buffer and 0 as @p BufferSize to learn the
+ * size first.
+ *
+ * @param Id            Identifier of the item
+ * @param Buffer        Where to copy the data; may be NULL when @p BufferSize is 0
+ * @param BufferSize    Size of the buffer in bytes
+ *
+ * @return Size of the item in bytes (0 or more), or a negative errno value (-ENOENT: there is no such item,
+ *         -EINVAL: wrong arguments, -EIO: the memory failed, -ENOSYS: the platform keeps nothing)
+ */
+DMOD_BUILTIN_API(Dmod, 1.0, int, _ReadPersistency, ( uint32_t Id, void* Buffer, size_t BufferSize ) );
+
+//! @}
+
 #ifdef __cplusplus
 }
 #endif

lib/third-party/CMakeLists.txt

@@ -1 +1,2 @@
-add_subdirectory(FastLZ)
+add_subdirectory(FastLZ)
+add_subdirectory(dmod_dmcomp)

lib/third-party/Makefile

@@ -6,7 +6,7 @@ DMOD_DIR=../..
 # -----------------------------------------------------------------------------
 #   Subdirectories
 # -----------------------------------------------------------------------------
-SUBDIRS = FastLZ
+SUBDIRS = FastLZ dmod_dmcomp
 SYSTEM_SUBDIRS = 
 MODULE_SUBDIRS = 
 

lib/third-party/dmod_dmcomp/CMakeLists.txt

@@ -0,0 +1,36 @@
+set(MODULE_NAME dmod_dmcomp)
+
+# The sources are the submodule choco-technologies/dmcomp (lib/third-party/dmcomp): this only builds them the way DMOD
+# builds its libraries (and writes the Makefile of the library, which the Make based builds use)
+set(DMCOMP_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../dmcomp")
+if(NOT EXISTS "${DMCOMP_DIR}/include/dmcomp.h")
+    message(FATAL_ERROR "The dmcomp library is missing: ${DMCOMP_DIR} is empty. "
+                        "It is a git submodule - run 'git submodule update --init --recursive'.")
+endif()
+
+# The decoder is always there; the encoder (it allocates ~330 kB with malloc and is ~10 kB of code) is for the host:
+# DMOD_USE_DMCOMP_ENCODER is OFF in a configuration without the standard library, a firmware that only loads modules
+add_library(${MODULE_NAME} STATIC
+    ../dmcomp/src/dmcomp_inflate.c
+    ../dmcomp/src/dmcomp_strerror.c
+)
+if(DMOD_USE_DMCOMP_ENCODER)
+    target_sources(${MODULE_NAME} PRIVATE ../dmcomp/src/dmcomp_deflate.c)
+else()
+    target_compile_definitions(${MODULE_NAME} PUBLIC DMCOMP_NO_ENCODER)
+endif()
+
+target_include_directories(${MODULE_NAME}
+    PUBLIC
+        ${DMCOMP_DIR}/include
+    PRIVATE
+        ${DMCOMP_DIR}/src
+)
+
+target_compile_options(${MODULE_NAME} PRIVATE
+    -Os
+    -ffunction-sections
+    -fdata-sections
+)
+
+create_library_makefile(${MODULE_NAME})