feat(minimax-h3): unify native audiovisual delivery

apps/benchmark/native/benchmark_worker.cpp

@@ -421,10 +421,10 @@ Json run_generate_audio(trtmc::ITask& task, const Json& request, const Timing& t
     return measure(
         timing, [&]() { return interface.generate_audio(prompt, config); },
         [](const trtmc::AudioResult& result) {
-            const double seconds =
-                result.sample_rate > 0
-                    ? static_cast<double>(result.samples.size()) / result.sample_rate
-                    : 0.0;
+            const double seconds = result.sample_rate > 0 && result.channels > 0
+                                       ? static_cast<double>(result.samples.size()) /
+                                             result.channels / result.sample_rate
+                                       : 0.0;
             return Json{{"output_samples", result.samples.size()},
                         {"num_samples", result.samples.size()},
                         {"output_audio_seconds", seconds},
@@ -458,10 +458,10 @@ Json run_speak(trtmc::ITask& task, const Json& request, const Timing& timing) {
         [](const auto& value) {
             const auto& result = value.first;
             return Json{{"input_audio_seconds", value.second},
-                        {"output_audio_seconds",
-                         result.sample_rate > 0
-                             ? static_cast<double>(result.samples.size()) / result.sample_rate
-                             : 0.0},
+                        {"output_audio_seconds", result.sample_rate > 0 && result.channels > 0
+                                                     ? static_cast<double>(result.samples.size()) /
+                                                           result.channels / result.sample_rate
+                                                     : 0.0},
                         {"output_samples", result.samples.size()},
                         {"num_samples", result.samples.size()},
                         {"sample_rate", result.sample_rate}};

apps/benchmark/tests/native/test_benchmark_worker_e2e.cpp

@@ -11,6 +11,10 @@
 #include <nlohmann/json.hpp>
 #include <string>
 
+#if defined(_WIN32)
+#include <process.h>
+#endif
+
 namespace {
 
 using Json = nlohmann::json;
@@ -39,12 +43,14 @@ void write_bundle(const std::filesystem::path& path) {
         throw std::runtime_error("failed to write fake bundle");
 }
 
+#if !defined(_WIN32)
 std::string shell_quote(const std::string& value) {
     std::string result{"'"};
     for (const char character : value)
         result += character == '\'' ? "'\\''" : std::string(1, character);
     return result + "'";
 }
+#endif
 
 } // namespace
 
@@ -81,10 +87,19 @@ int main(int argc, char** argv) {
                 throw std::runtime_error("failed to write worker request");
         }
 
+#if defined(_WIN32)
+        const std::string worker_argument = '"' + std::string(argv[1]) + '"';
+        const std::string request_argument = '"' + request_path.string() + '"';
+        const std::string output_argument = '"' + output_path.string() + '"';
+        check(_spawnl(_P_WAIT, argv[1], worker_argument.c_str(), "--request",
+                      request_argument.c_str(), "--output", output_argument.c_str(), nullptr) == 0,
+              "worker process completed");
+#else
         const std::string command = shell_quote(argv[1]) + " --request " +
                                     shell_quote(request_path.string()) + " --output " +
                                     shell_quote(output_path.string());
         check(std::system(command.c_str()) == 0, "worker process completed");
+#endif
 
         std::ifstream output_file(output_path);
         Json result;

apps/cli/cli.cpp

@@ -6,6 +6,8 @@
 #include "cli/cli.h"
 
 #include "cli/io.h"
+#include "cli/windows_media.h"
+#include "runtime/platform/dynamic_library.h"
 #include "trtmc/runtime/family_loader.h"
 
 #include <algorithm>
@@ -14,7 +16,6 @@
 #include <cstddef>
 #include <cstdint>
 #include <cstdlib>
-#include <dlfcn.h>
 #include <filesystem>
 #include <fstream>
 #include <iomanip>
@@ -123,8 +124,10 @@ const std::unordered_map<std::string, CommandSpec>& command_specs() {
            "--num-steps", "--guidance-scale", "--cfg-scale"}}},
         {"generate-video",
          {CommandKind::kGenerateVideo,
-          {"--prompt", "--image", "--output", "--negative-prompt", "--height", "--width",
-           "--num-steps", "--seed", "--guidance-scale", "--cfg-scale", "--initial-latents-raw"}}},
+          {"--prompt", "--output", "--negative-prompt", "--height", "--width", "--num-frames",
+           "--num-steps", "--seed", "--guidance-scale", "--cfg-scale", "--initial-latents-raw",
+           "--first-frame", "--last-frame", "--reference-image", "--reference-video",
+           "--reference-audio"}}},
         {"solve", {CommandKind::kSolve, {"--branch", "--trunk"}}},
         {"forecast", {CommandKind::kForecast, {"--input", "--mask", "--frequency"}}},
         {"control", {CommandKind::kControl, {"--image", "--state", "--output"}}},
@@ -142,21 +145,24 @@ bool is_byok_option(const std::string& option) {
 
 void load_byok_extension(const Command& command) {
     using LoadKernelFn = const char* (*)(const char*, const char*, const char*) noexcept;
-    const fs::path extension = fs::path(command.runtime_root) / "libtrtmc_byok_tvm_ffi.so";
-    dlerror();
-    void* handle = dlopen(extension.c_str(), RTLD_NOW | RTLD_LOCAL);
+    const fs::path extension =
+        fs::path(command.runtime_root) / internal::dynamic_library_filename("trtmc_byok_tvm_ffi");
+    std::string loader_error;
+    auto handle = internal::open_dynamic_library(
+        extension, internal::DynamicLibraryVisibility::local, &loader_error);
     if (handle == nullptr) {
-        const char* error = dlerror();
         throw std::runtime_error("unable to load BYOK extension '" + extension.string() +
-                                 "': " + (error != nullptr ? error : "unknown dlopen error"));
+                                 "': " + loader_error);
     }
-    static auto* handles = new std::vector<void*>;
-    handles->push_back(handle);
-    dlerror();
-    auto load = reinterpret_cast<LoadKernelFn>(dlsym(handle, "trtmc_load_byok_kernel"));
-    if (const char* error = dlerror(); error != nullptr || load == nullptr) {
-        throw std::runtime_error("BYOK extension is missing trtmc_load_byok_kernel");
+    auto load = reinterpret_cast<LoadKernelFn>(
+        internal::dynamic_library_symbol(handle, "trtmc_load_byok_kernel", &loader_error));
+    if (load == nullptr) {
+        (void)internal::close_dynamic_library(handle);
+        throw std::runtime_error("BYOK extension is missing trtmc_load_byok_kernel: " +
+                                 loader_error);
     }
+    static auto* handles = new std::vector<internal::DynamicLibraryHandle>;
+    handles->push_back(handle);
     if (const char* error = load(command.options.at("--byok-library").c_str(),
                                  command.options.at("--byok-function").c_str(),
                                  command.options.at("--byok-name").c_str())) {
@@ -464,6 +470,7 @@ ImageGenerationConfig image_config(const Command& command) {
         config.negative_prompt = command.options.at("--negative-prompt");
     config.height = int_option(command, "--height", 0, 1);
     config.width = int_option(command, "--width", 0, 1);
+    config.video_num_frames = int_option(command, "--num-frames", 0, 1);
     config.num_steps = int_option(command, "--num-steps", -1, 1);
     config.seed = int_option(command, "--seed", -1);
     config.guidance_scale = float_option(command, "--guidance-scale", -1.0F);
@@ -538,6 +545,102 @@ ImageResult generate_image(const Command& command, ITask& task) {
     return require_interface<IImageGeneration>(task).generate_image(prompt, config);
 }
 
+VideoImageInput load_video_image(const std::string& path) {
+    auto decoded = read_image(path);
+    VideoImageInput result;
+    result.pixels = std::move(decoded.pixels);
+    result.height = decoded.height;
+    result.width = decoded.width;
+    result.channels = 3;
+    return result;
+}
+
+VideoGenerationRequest video_request(const Command& command, IVideoGeneration& generator) {
+    VideoGenerationRequest request;
+    request.prompt = require_option(command, "--prompt");
+    request.config = image_config(command);
+
+    const bool has_first = has_option(command, "--first-frame");
+    const bool has_last = has_option(command, "--last-frame");
+    if ((has_first || has_last) && !command.video_references.empty())
+        throw std::invalid_argument("key frames cannot be combined with media references");
+    if (has_first || has_last) {
+        request.mode = VideoGenerationMode::kFirstLastFrameToVideoAudio;
+        if (has_first)
+            request.first_frame = load_video_image(command.options.at("--first-frame"));
+        if (has_last)
+            request.last_frame = load_video_image(command.options.at("--last-frame"));
+        return request;
+    }
+    if (command.video_references.empty())
+        return request;
+
+    request.mode = VideoGenerationMode::kReferenceToVideoAudio;
+    const auto policy = generator.reference_media_decode_policy();
+    request.references.reserve(command.video_references.size());
+    for (const auto& argument : command.video_references) {
+        VideoReferenceInput reference;
+        reference.kind = argument.kind;
+        switch (argument.kind) {
+        case VideoReferenceKind::kImage:
+            reference.image = load_video_image(argument.path);
+            break;
+        case VideoReferenceKind::kVideo:
+            if (!policy)
+                throw std::runtime_error(
+                    "loaded family does not provide a reference-media decode policy");
+            reference.video = read_video_file(argument.path, *policy);
+            break;
+        case VideoReferenceKind::kAudio:
+            if (!policy)
+                throw std::runtime_error(
+                    "loaded family does not provide a reference-media decode policy");
+            reference.audio = read_audio_file(argument.path, *policy);
+            break;
+        }
+        request.references.push_back(std::move(reference));
+    }
+    return request;
+}
+
+nlohmann::json write_generated_video(const VideoResult& result, const std::string& path) {
+    if (result.frames.pixels.empty() && result.frames.height == 0 && result.frames.width == 0 &&
+        result.frames.channels == 3 && result.frames.num_frames == 0 &&
+        result.audio.samples.empty() && result.fps == 0) {
+        return {{"worker", true}};
+    }
+    validate_image_result(result.frames);
+    if (result.fps <= 0)
+        throw std::runtime_error("video result has a non-positive frame rate");
+    if (!result.audio.samples.empty()) {
+        if (result.audio.sample_rate <= 0 ||
+            (result.audio.channels != 1 && result.audio.channels != 2) ||
+            result.audio.samples.size() % static_cast<std::size_t>(result.audio.channels) != 0) {
+            throw std::runtime_error("video result has invalid interleaved audio metadata");
+        }
+        require_finite(result.audio.samples, "video audio result");
+    }
+    if (is_mp4_path(path)) {
+        write_mp4(result, path);
+        return {{"output", path},
+                {"frames", result.frames.num_frames},
+                {"fps", result.fps},
+                {"height", result.frames.height},
+                {"width", result.frames.width},
+                {"audio_channels", result.audio.samples.empty() ? 0 : result.audio.channels},
+                {"audio_sample_rate", result.audio.samples.empty() ? 0 : result.audio.sample_rate}};
+    }
+
+    auto payload = write_video(result.frames, path);
+    payload["fps"] = result.fps;
+    if (!result.audio.samples.empty()) {
+        const fs::path audio_path = fs::path(path) / "audio.wav";
+        io::write_wav(result.audio, audio_path.string());
+        payload["audio"] = audio_path.string();
+    }
+    return payload;
+}
+
 const char* event_kind_name(SpeechSessionEventKind kind) {
     switch (kind) {
     case SpeechSessionEventKind::kAgentAudio:
@@ -665,17 +768,17 @@ Command parse_args(int argc, char** argv) {
     if (name == "help") {
         if (argc != 2)
             throw std::invalid_argument("help does not accept arguments");
-        return {CommandKind::kHelp, name, {}, {}, {}, {}, {}, 0, {}, false};
+        return {CommandKind::kHelp, name, {}, {}, {}, {}, {}, {}, 0, {}, false};
     }
     if (name == "version") {
         if (argc != 2)
             throw std::invalid_argument("version does not accept arguments");
-        return {CommandKind::kVersion, name, {}, {}, {}, {}, {}, 0, {}, false};
+        return {CommandKind::kVersion, name, {}, {}, {}, {}, {}, {}, 0, {}, false};
     }
     if (name == "inspect") {
         if (argc != 3 || std::string(argv[2]).empty())
             throw std::invalid_argument("inspect requires exactly one BUNDLE path");
-        return {CommandKind::kInspect, name, argv[2], {}, {}, {}, {}, 0, {}, false};
+        return {CommandKind::kInspect, name, argv[2], {}, {}, {}, {}, {}, 0, {}, false};
     }
 
     const auto spec = command_specs().find(name);
@@ -684,7 +787,7 @@ Command parse_args(int argc, char** argv) {
     if (argc < 3 || std::string(argv[2]).empty())
         throw std::invalid_argument(name + " requires a BUNDLE path");
 
-    Command command{spec->second.kind, name, argv[2], {}, {}, {}, {}, 0, {}, false};
+    Command command{spec->second.kind, name, argv[2], {}, {}, {}, {}, {}, 0, {}, false};
     for (int index = 3; index < argc; ++index) {
         const std::string option = argv[index];
         if (option == "--runtime-root") {
@@ -721,6 +824,16 @@ Command parse_args(int argc, char** argv) {
             command.inputs.push_back(take_value(argc, argv, index, option));
             continue;
         }
+        if (option == "--reference-image" || option == "--reference-video" ||
+            option == "--reference-audio") {
+            VideoReferenceKind kind = VideoReferenceKind::kImage;
+            if (option == "--reference-video")
+                kind = VideoReferenceKind::kVideo;
+            else if (option == "--reference-audio")
+                kind = VideoReferenceKind::kAudio;
+            command.video_references.push_back({kind, take_value(argc, argv, index, option)});
+            continue;
+        }
         if (command.options.count(option) != 0)
             throw std::invalid_argument(option + " may be specified only once");
         command.options.emplace(option, take_value(argc, argv, index, option));
@@ -1195,8 +1308,10 @@ int dispatch(const Command& command, ITask& task, std::ostream& output) {
         return EXIT_SUCCESS;
     }
     case CommandKind::kGenerateVideo: {
-        const std::string directory = require_option(command, "--output");
-        write_json(output, write_video(generate_image(command, task), directory));
+        const std::string path = require_option(command, "--output");
+        auto& generator = require_interface<IVideoGeneration>(task);
+        auto request = video_request(command, generator);
+        write_json(output, write_generated_video(generator.generate_video(request), path));
         return EXIT_SUCCESS;
     }
     case CommandKind::kSolve: {
@@ -1305,6 +1420,10 @@ void print_usage(std::ostream& output) {
               "  [--kv-cache-size BYTES|GB|GiB]\n\n"
               "TensorRT-RTX runtime options:\n"
               "  [--runtime-cache PATH] [--cuda-graphs]\n\n"
+              "Video generation options:\n"
+              "  --prompt TEXT --output OUTPUT.mp4 [--num-frames N] [--height N] [--width N]\n"
+              "  [--first-frame IMAGE] [--last-frame IMAGE]\n"
+              "  [--reference-image IMAGE|--reference-video VIDEO|--reference-audio AUDIO]...\n\n"
               "Execution never searches for runtimes; --runtime-root is always required.\n";
 }
 

apps/cli/cli.h

@@ -48,6 +48,11 @@ enum class CommandKind {
     kGenerateWorld,
 };
 
+struct VideoReferenceArgument {
+    VideoReferenceKind kind{VideoReferenceKind::kImage};
+    std::string path;
+};
+
 struct Command {
     CommandKind kind{CommandKind::kHelp};
     std::string name;
@@ -56,6 +61,7 @@ struct Command {
     std::unordered_map<std::string, std::string> options;
     std::vector<std::string> frames;
     std::vector<std::string> inputs;
+    std::vector<VideoReferenceArgument> video_references;
     std::uint64_t kv_cache_size_bytes{0};
     std::string runtime_cache_path;
     bool cuda_graphs{false};

apps/cli/io.cpp

@@ -15,6 +15,7 @@
 #include <cstddef>
 #include <cstdint>
 #include <fstream>
+#include <limits>
 #include <stdexcept>
 #include <string>
 #include <vector>
@@ -24,18 +25,34 @@ namespace trtmc::cli::io {
 void write_wav(const AudioResult& audio, const std::string& path) {
     if (audio.samples.empty())
         throw std::runtime_error("write_wav: empty audio");
+    if (audio.sample_rate <= 0)
+        throw std::runtime_error("write_wav: sample rate must be positive");
+    if (audio.channels != 1 && audio.channels != 2)
+        throw std::runtime_error("write_wav: channel count must be mono or stereo");
+    if (audio.samples.size() % static_cast<std::size_t>(audio.channels) != 0)
+        throw std::runtime_error("write_wav: interleaved sample count is not channel-aligned");
+    if (audio.samples.size() >
+        static_cast<std::size_t>(std::numeric_limits<std::int32_t>::max() / sizeof(float)))
+        throw std::runtime_error("write_wav: sample buffer is too large for RIFF/WAVE");
+    if (audio.num_samples != 0 &&
+        audio.num_samples != static_cast<std::int32_t>(audio.samples.size()))
+        throw std::runtime_error("write_wav: num_samples does not match the sample buffer");
 
     std::ofstream output(path, std::ios::binary);
     if (!output)
         throw std::runtime_error("write_wav: cannot open " + path);
 
-    const auto num_samples = static_cast<std::int32_t>(audio.samples.size());
     const std::int32_t sample_rate = audio.sample_rate;
-    const std::int16_t num_channels = 1;
+    const auto num_channels = static_cast<std::int16_t>(audio.channels);
     const std::int16_t bits_per_sample = 32;
-    const std::int32_t byte_rate = sample_rate * num_channels * (bits_per_sample / 8);
     const auto block_align = static_cast<std::int16_t>(num_channels * (bits_per_sample / 8));
-    const std::int32_t data_size = num_samples * block_align;
+    const auto byte_rate_64 = static_cast<std::int64_t>(sample_rate) * block_align;
+    if (byte_rate_64 > std::numeric_limits<std::int32_t>::max())
+        throw std::runtime_error("write_wav: byte rate exceeds the RIFF/WAVE range");
+    const auto byte_rate = static_cast<std::int32_t>(byte_rate_64);
+    const auto data_size = static_cast<std::int32_t>(audio.samples.size() * sizeof(float));
+    if (data_size > std::numeric_limits<std::int32_t>::max() - 36)
+        throw std::runtime_error("write_wav: output exceeds the RIFF/WAVE range");
     const std::int32_t chunk_size = 36 + data_size;
     const std::int32_t format_size = 16;
     const std::int16_t audio_format = 3;