From 014f4650142fa5b29aa91d47b5c53555afc0b0c0 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Wed, 13 May 2026 13:40:08 +0200 Subject: [PATCH 1/3] MINIFICPP-2740 Rust bindings based on stable C API --- .github/workflows/ci.yml | 62 +- .../workflows/create-release-artifacts.yml | 10 + CMakeLists.txt | 2 + cmake/MiNiFiOptions.cmake | 1 + docker/rockylinux/Dockerfile | 3 +- minifi_rust/.cargo/config.toml | 19 + minifi_rust/.dockerignore | 12 + minifi_rust/.gitignore | 7 + minifi_rust/CMakeLists.txt | 42 ++ minifi_rust/Cargo.toml | 9 + minifi_rust/LICENSE | 202 +++++++ minifi_rust/README.md | 75 +++ .../minifi_rs_playground/.cargo/config.toml | 5 + .../minifi_rs_playground/.gitignore | 7 + .../minifi_rs_playground/Cargo.toml | 20 + .../features/basic.feature | 84 +++ .../features/environment.py | 74 +++ .../features/error-handling.feature | 70 +++ .../features/lazy-logging.feature | 13 + .../features/metrics.feature | 23 + .../features/steps/steps.py | 45 ++ .../features/streaming.feature | 34 ++ .../minifi_rs_playground.md | 260 ++++++++ .../dummy_controller_service.rs | 22 + .../lorem_ipsum_controller_service.rs | 33 ++ .../properties.rs | 13 + .../src/controller_services/mod.rs | 2 + .../minifi_rs_playground/src/lib.rs | 37 ++ .../src/processors/asciify_german.rs | 67 +++ .../asciify_german/processor_definition.rs | 15 + .../asciify_german/relationships.rs | 11 + .../src/processors/asciify_german/tests.rs | 54 ++ .../src/processors/count_actual_logging.rs | 57 ++ .../src/processors/duplicate_text.rs | 54 ++ .../src/processors/generate_flow_file.rs | 168 ++++++ .../processor_definition.rs | 21 + .../generate_flow_file/properties.rs | 61 ++ .../generate_flow_file/relationships.rs | 6 + .../processors/generate_flow_file/tests.rs | 138 +++++ .../src/processors/get_file.rs | 303 ++++++++++ .../processors/get_file/output_attributes.rs | 13 + .../get_file/processor_definition.rs | 30 + .../src/processors/get_file/properties.rs | 121 ++++ .../src/processors/get_file/relationships.rs | 6 + .../src/processors/get_file/tests.rs | 172 ++++++ .../src/processors/kamikaze_processor.rs | 103 ++++ .../processor_definition.rs | 17 + .../kamikaze_processor/properties.rs | 39 ++ .../kamikaze_processor/relationships.rs | 6 + .../processors/kamikaze_processor/tests.rs | 83 +++ .../src/processors/log_attribute.rs | 160 +++++ .../log_attribute/processor_definition.rs | 24 + .../processors/log_attribute/properties.rs | 86 +++ .../processors/log_attribute/relationships.rs | 6 + .../src/processors/log_attribute/tests.rs | 114 ++++ .../src/processors/lorem_ipsum_cs_user.rs | 75 +++ .../processor_definition.rs | 16 + .../lorem_ipsum_cs_user/properties.rs | 28 + .../lorem_ipsum_cs_user/relationships.rs | 6 + .../processors/lorem_ipsum_cs_user/tests.rs | 19 + .../src/processors/mod.rs | 9 + .../src/processors/put_file.rs | 239 ++++++++ .../put_file/processor_definition.rs | 36 ++ .../src/processors/put_file/properties.rs | 51 ++ .../src/processors/put_file/relationships.rs | 11 + .../src/processors/put_file/tests.rs | 143 +++++ .../put_file/unix_only_properties.rs | 25 + .../generate_docs/generate_docs.dockerfile | 29 + minifi_rust/generate_docs/generate_docs.sh | 29 + minifi_rust/minifi_native/.gitignore | 2 + minifi_rust/minifi_native/Cargo.toml | 15 + minifi_rust/minifi_native/src/api.rs | 25 + .../minifi_native/src/api/attribute.rs | 11 + .../src/api/component_definition_traits.rs | 22 + .../src/api/controller_service.rs | 62 ++ minifi_rust/minifi_native/src/api/errors.rs | 128 ++++ .../minifi_native/src/api/flow_file.rs | 1 + minifi_rust/minifi_native/src/api/logger.rs | 95 +++ .../minifi_native/src/api/process_context.rs | 126 ++++ .../minifi_native/src/api/process_session.rs | 60 ++ .../minifi_native/src/api/processor.rs | 60 ++ .../src/api/processor_wrappers.rs | 5 + .../processor_wrappers/complex_processor.rs | 89 +++ .../processor_wrappers/flow_file_source.rs | 129 ++++ .../flow_file_stream_transform.rs | 174 ++++++ .../processor_wrappers/flow_file_transform.rs | 184 ++++++ .../src/api/processor_wrappers/utils.rs | 2 + .../utils/context_session_flowfile_bundle.rs | 70 +++ .../utils/flow_file_content.rs | 27 + minifi_rust/minifi_native/src/api/property.rs | 98 ++++ .../src/api/raw_controller_service.rs | 11 + .../minifi_native/src/api/raw_processor.rs | 70 +++ .../minifi_native/src/api/relationship.rs | 5 + minifi_rust/minifi_native/src/c_ffi.rs | 26 + .../c_ffi/c_ffi_controller_service_context.rs | 70 +++ .../c_ffi_controller_service_definition.rs | 162 +++++ .../c_ffi/c_ffi_controller_service_list.rs | 53 ++ .../src/c_ffi/c_ffi_flow_file.rs | 22 + .../minifi_native/src/c_ffi/c_ffi_logger.rs | 52 ++ .../src/c_ffi/c_ffi_output_attribute.rs | 41 ++ .../src/c_ffi/c_ffi_primitives.rs | 82 +++ .../src/c_ffi/c_ffi_process_context.rs | 146 +++++ .../src/c_ffi/c_ffi_process_session.rs | 554 ++++++++++++++++++ .../src/c_ffi/c_ffi_processor_definition.rs | 266 +++++++++ .../src/c_ffi/c_ffi_processor_list.rs | 49 ++ .../minifi_native/src/c_ffi/c_ffi_property.rs | 132 +++++ .../src/c_ffi/c_ffi_relationship.rs | 15 + .../minifi_native/src/c_ffi/c_ffi_streams.rs | 114 ++++ minifi_rust/minifi_native/src/lib.rs | 116 ++++ minifi_rust/minifi_native/src/mock.rs | 12 + .../mock/mock_controller_service_context.rs | 20 + .../minifi_native/src/mock/mock_flow_file.rs | 45 ++ .../minifi_native/src/mock/mock_logger.rs | 83 +++ .../src/mock/mock_process_context.rs | 121 ++++ .../src/mock/mock_process_session.rs | 181 ++++++ minifi_rust/minifi_native_macros/Cargo.toml | 11 + minifi_rust/minifi_native_macros/src/lib.rs | 20 + minifi_rust/minifi_native_sys/.gitignore | 2 + minifi_rust/minifi_native_sys/Cargo.toml | 18 + minifi_rust/minifi_native_sys/build.rs | 195 ++++++ minifi_rust/minifi_native_sys/src/lib.rs | 11 + minifi_rust/minifi_rs_behave/Cargo.toml | 12 + .../minifi_rs_behave/alpine.dockerfile | 27 + minifi_rust/minifi_rs_behave/build.rs | 6 + .../minifi_rs_behave/debian.dockerfile | 26 + minifi_rust/minifi_rs_behave/linux_build.sh | 94 +++ minifi_rust/minifi_rs_behave/src/main.rs | 155 +++++ minifi_rust/rustfmt.toml | 0 128 files changed, 8243 insertions(+), 4 deletions(-) create mode 100644 minifi_rust/.cargo/config.toml create mode 100644 minifi_rust/.dockerignore create mode 100644 minifi_rust/.gitignore create mode 100644 minifi_rust/CMakeLists.txt create mode 100644 minifi_rust/Cargo.toml create mode 100644 minifi_rust/LICENSE create mode 100644 minifi_rust/README.md create mode 100644 minifi_rust/extensions/minifi_rs_playground/.cargo/config.toml create mode 100644 minifi_rust/extensions/minifi_rs_playground/.gitignore create mode 100644 minifi_rust/extensions/minifi_rs_playground/Cargo.toml create mode 100644 minifi_rust/extensions/minifi_rs_playground/features/basic.feature create mode 100644 minifi_rust/extensions/minifi_rs_playground/features/environment.py create mode 100644 minifi_rust/extensions/minifi_rs_playground/features/error-handling.feature create mode 100644 minifi_rust/extensions/minifi_rs_playground/features/lazy-logging.feature create mode 100644 minifi_rust/extensions/minifi_rs_playground/features/metrics.feature create mode 100644 minifi_rust/extensions/minifi_rs_playground/features/steps/steps.py create mode 100644 minifi_rust/extensions/minifi_rs_playground/features/streaming.feature create mode 100644 minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service/properties.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/lib.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/processor_definition.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/relationships.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/tests.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/count_actual_logging.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/duplicate_text.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/processor_definition.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/properties.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/relationships.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/tests.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/output_attributes.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/processor_definition.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/properties.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/relationships.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/tests.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/processor_definition.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/properties.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/relationships.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/processor_definition.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/properties.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/relationships.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/tests.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/processor_definition.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/properties.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/relationships.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/tests.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/mod.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/processor_definition.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/properties.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/relationships.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/tests.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/unix_only_properties.rs create mode 100644 minifi_rust/generate_docs/generate_docs.dockerfile create mode 100755 minifi_rust/generate_docs/generate_docs.sh create mode 100644 minifi_rust/minifi_native/.gitignore create mode 100644 minifi_rust/minifi_native/Cargo.toml create mode 100644 minifi_rust/minifi_native/src/api.rs create mode 100644 minifi_rust/minifi_native/src/api/attribute.rs create mode 100644 minifi_rust/minifi_native/src/api/component_definition_traits.rs create mode 100644 minifi_rust/minifi_native/src/api/controller_service.rs create mode 100644 minifi_rust/minifi_native/src/api/errors.rs create mode 100644 minifi_rust/minifi_native/src/api/flow_file.rs create mode 100644 minifi_rust/minifi_native/src/api/logger.rs create mode 100644 minifi_rust/minifi_native/src/api/process_context.rs create mode 100644 minifi_rust/minifi_native/src/api/process_session.rs create mode 100644 minifi_rust/minifi_native/src/api/processor.rs create mode 100644 minifi_rust/minifi_native/src/api/processor_wrappers.rs create mode 100644 minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs create mode 100644 minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs create mode 100644 minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs create mode 100644 minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs create mode 100644 minifi_rust/minifi_native/src/api/processor_wrappers/utils.rs create mode 100644 minifi_rust/minifi_native/src/api/processor_wrappers/utils/context_session_flowfile_bundle.rs create mode 100644 minifi_rust/minifi_native/src/api/processor_wrappers/utils/flow_file_content.rs create mode 100644 minifi_rust/minifi_native/src/api/property.rs create mode 100644 minifi_rust/minifi_native/src/api/raw_controller_service.rs create mode 100644 minifi_rust/minifi_native/src/api/raw_processor.rs create mode 100644 minifi_rust/minifi_native/src/api/relationship.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_flow_file.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_logger.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_output_attribute.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_primitives.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_relationship.rs create mode 100644 minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs create mode 100644 minifi_rust/minifi_native/src/lib.rs create mode 100644 minifi_rust/minifi_native/src/mock.rs create mode 100644 minifi_rust/minifi_native/src/mock/mock_controller_service_context.rs create mode 100644 minifi_rust/minifi_native/src/mock/mock_flow_file.rs create mode 100644 minifi_rust/minifi_native/src/mock/mock_logger.rs create mode 100644 minifi_rust/minifi_native/src/mock/mock_process_context.rs create mode 100644 minifi_rust/minifi_native/src/mock/mock_process_session.rs create mode 100644 minifi_rust/minifi_native_macros/Cargo.toml create mode 100644 minifi_rust/minifi_native_macros/src/lib.rs create mode 100644 minifi_rust/minifi_native_sys/.gitignore create mode 100644 minifi_rust/minifi_native_sys/Cargo.toml create mode 100644 minifi_rust/minifi_native_sys/build.rs create mode 100644 minifi_rust/minifi_native_sys/src/lib.rs create mode 100644 minifi_rust/minifi_rs_behave/Cargo.toml create mode 100644 minifi_rust/minifi_rs_behave/alpine.dockerfile create mode 100644 minifi_rust/minifi_rs_behave/build.rs create mode 100644 minifi_rust/minifi_rs_behave/debian.dockerfile create mode 100755 minifi_rust/minifi_rs_behave/linux_build.sh create mode 100644 minifi_rust/minifi_rs_behave/src/main.rs create mode 100644 minifi_rust/rustfmt.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1d5f8896a..482076d0db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ concurrency: env: DOCKER_CMAKE_FLAGS: -DDOCKER_VERIFY_THREAD=3 -DUSE_SHARED_LIBS= -DSTRICT_GSL_CHECKS=AUDIT -DCI_BUILD=ON -DENABLE_AWS=ON -DENABLE_KAFKA=ON -DENABLE_MQTT=ON -DENABLE_AZURE=ON -DENABLE_SQL=ON \ -DENABLE_SPLUNK=ON -DENABLE_GCP=ON -DENABLE_OPC=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_TEST_PROCESSORS=ON -DENABLE_PROMETHEUS=ON \ - -DENABLE_ELASTICSEARCH=ON -DENABLE_GRAFANA_LOKI=ON -DENABLE_COUCHBASE=ON -DENABLE_LLAMACPP=ON -DDOCKER_BUILD_ONLY=ON -DMINIFI_PERFORMANCE_TESTS=ON + -DENABLE_ELASTICSEARCH=ON -DENABLE_GRAFANA_LOKI=ON -DENABLE_COUCHBASE=ON -DENABLE_LLAMACPP=ON -DDOCKER_BUILD_ONLY=ON -DMINIFI_PERFORMANCE_TESTS=ON -DMINIFI_RUST=OFF CCACHE_DIR: ${{ GITHUB.WORKSPACE }}/.ccache jobs: macos_xcode: @@ -52,6 +52,7 @@ jobs: -DENABLE_SFTP=ON -DENABLE_SPLUNK=ON -DENABLE_SQL=ON + -DMINIFI_RUST=ON -DENABLE_TEST_PROCESSORS=ON -DFORCE_COLORED_OUTPUT=OFF -DLIBC_STATIC=OFF @@ -161,6 +162,7 @@ jobs: -DENABLE_SQL=ON -DENABLE_TEST_PROCESSORS=ON -DENABLE_WEL=ON + -DMINIFI_RUST=ON -DFORCE_COLORED_OUTPUT=ON -DINSTALLER_MERGE_MODULES=OFF -DMINIFI_FAIL_ON_WARNINGS=ON @@ -261,6 +263,7 @@ jobs: -DENABLE_SPLUNK=ON -DENABLE_SQL=ON -DENABLE_SYSTEMD=ON + -DMINIFI_RUST=ON -DENABLE_TEST_PROCESSORS=ON -DFORCE_COLORED_OUTPUT=ON -DMINIFI_FAIL_ON_WARNINGS=ON @@ -552,8 +555,55 @@ jobs: with: name: behavex_output_modular path: build/behavex_output_modular + rusty_docker_tests: + name: "Rusty Docker integration tests (x86_64)" + needs: docker_build + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - id: checkout + uses: actions/checkout@v6 + - id: run_cmake + name: Run CMake + run: | + mkdir build + cd build + cmake ${DOCKER_CMAKE_FLAGS} .. + - name: Download artifact + uses: actions/download-artifact@v8 + with: + name: minifi_docker + path: build + - name: Load Docker image + run: | + docker load --input ./build/minifi_docker.tar && docker tag apacheminificpp:1.0.0 apacheminificpp:behave + - id: install_deps + name: Install dependencies for Docker Verify + run: | + sudo apt update + sudo apt install -y python3-virtualenv + - id: test + name: Docker Verify + working-directory: ./minifi_rust + run: cargo behave-alpine + - name: Test Reporter + if: always() + uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3 + with: + name: Docker integration tests + path: minifi_rust/minifi_rs_behave/output/behave/*.xml + reporter: java-junit + only-summary: 'true' + list-tests: 'failed' + list-suites: 'failed' + - name: Upload artifact + if: failure() + uses: actions/upload-artifact@v7 + with: + name: minifi_rs_behave + path: minifi_rust/minifi_rs_behave/output linters: - name: "C++ lint + Shellcheck + Flake8" + name: "C++ lint + Shellcheck + Flake8 + Cargo check" runs-on: ubuntu-22.04-arm timeout-minutes: 15 steps: @@ -578,8 +628,14 @@ jobs: continue-on-error: true run: ./run_flake8.sh . + - id: cargo_check + name: Cargo check + continue-on-error: true + run: cargo fmt --check + working-directory: minifi_rust + - name: Check Linter Statuses - if: steps.cpp_lint.outcome == 'failure' || steps.shellcheck.outcome == 'failure' || steps.flake8_check.outcome == 'failure' + if: steps.cpp_lint.outcome == 'failure' || steps.shellcheck.outcome == 'failure' || steps.flake8_check.outcome == 'failure' || steps.cargo_check.outcome == 'failure' run: | echo "One or more linters failed. Failing the workflow." exit 1 diff --git a/.github/workflows/create-release-artifacts.yml b/.github/workflows/create-release-artifacts.yml index edac6fc903..ce0811452f 100644 --- a/.github/workflows/create-release-artifacts.yml +++ b/.github/workflows/create-release-artifacts.yml @@ -43,6 +43,16 @@ jobs: with: name: ${{ matrix.platform.rpm-artifact }} path: build/nifi-minifi-cpp-*.rpm + + - id: cargo + working-directory: minifi_rust + run: | + cargo build --release + + - uses: actions/upload-artifact@v7 + with: + name: minifi_rust_extensions + path: minifi_rust/target/release/*.so windows_VS2022: name: "Windows Server 2025 x86_64" runs-on: windows-2025 diff --git a/CMakeLists.txt b/CMakeLists.txt index 5af9c071c7..47993663f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -849,3 +849,5 @@ if (MINIFI_ADVANCED_CODE_COVERAGE) endif() add_subdirectory(packaging) + +add_subdirectory(minifi_rust) diff --git a/cmake/MiNiFiOptions.cmake b/cmake/MiNiFiOptions.cmake index 5c9820d419..59993fe6d4 100644 --- a/cmake/MiNiFiOptions.cmake +++ b/cmake/MiNiFiOptions.cmake @@ -116,6 +116,7 @@ add_minifi_option(ENABLE_EXECUTE_PROCESS "Enable ExecuteProcess processor" OFF) add_minifi_option(ENABLE_CONTROLLER "Enables the build of MiNiFi controller binary." ON) add_minifi_option(ENABLE_LLAMACPP "Enables llama.cpp support." ON) add_minifi_option(ENABLE_OPC "Instructs the build system to enable the OPC extension" ON) +add_minifi_option(MINIFI_RUST "Enables the build of rust based extensions." OFF) set_minifi_cache_variable(CUSTOM_MALLOC OFF "Overwrite malloc implementation.") set_property(CACHE CUSTOM_MALLOC PROPERTY STRINGS "jemalloc" "mimalloc" "rpmalloc" OFF) diff --git a/docker/rockylinux/Dockerfile b/docker/rockylinux/Dockerfile index 7af9a3de23..019bfe4653 100644 --- a/docker/rockylinux/Dockerfile +++ b/docker/rockylinux/Dockerfile @@ -42,11 +42,12 @@ COPY . ${MINIFI_BASE_DIR} # Install the system dependencies needed for a build # ccache is in EPEL RUN dnf -y install epel-release && dnf -y install gcc-toolset-14 gcc-toolset-14-libatomic-devel sudo git which make libarchive ccache ca-certificates perl patch bison flex libtool cmake rpmdevtools && \ + if echo "$MINIFI_OPTIONS" | grep -q "MINIFI_RUST=ON"; then dnf -y install rust cargo clang; fi && \ if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_ALL=ON\|ENABLE_PYTHON_SCRIPTING=ON\|ENABLE_OPC=ON"; then dnf -y --enablerepo=devel install python3-devel; fi && \ if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_SFTP=ON" && [ "${DOCKER_SKIP_TESTS}" == "OFF" ]; then dnf -y install java-1.8.0-openjdk maven; fi RUN cd $MINIFI_BASE_DIR && \ - ln -s /usr/bin/ccache /usr/lib64/ccache/c++ + ln -sfn /usr/bin/ccache /usr/lib64/ccache/c++ # Setup minificpp user RUN groupadd -g ${GID} ${USER} && useradd -g ${GID} ${USER} && \ diff --git a/minifi_rust/.cargo/config.toml b/minifi_rust/.cargo/config.toml new file mode 100644 index 0000000000..b871040ed5 --- /dev/null +++ b/minifi_rust/.cargo/config.toml @@ -0,0 +1,19 @@ +[target.aarch64-unknown-linux-musl] +rustflags = ["-C", "target-feature=-crt-static"] + +[target.x86_64-unknown-linux-musl] +rustflags = ["-C", "target-feature=-crt-static"] + +[target.aarch64-apple-darwin] +rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"] + +[target.x86_64-apple-darwin] +rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"] + +[alias] +behave = ["run", "--release", "--bin", "minifi_rs_behave"] +behave-alpine = ["run", "--release", "--bin", "minifi_rs_behave", "--", "--alpine"] +behave-debian = ["run", "--release", "--bin", "minifi_rs_behave", "--", "--debian"] + +[env] +MINIFI_SDK_PATH="../" diff --git a/minifi_rust/.dockerignore b/minifi_rust/.dockerignore new file mode 100644 index 0000000000..aeb3a1bf07 --- /dev/null +++ b/minifi_rust/.dockerignore @@ -0,0 +1,12 @@ +.git +.vscode +.gitmodules +.idea +output +features +target/debug +target/release/build +target/release/deps +target/release/.fingerprint +target/release/examples +target/release/incremental diff --git a/minifi_rust/.gitignore b/minifi_rust/.gitignore new file mode 100644 index 0000000000..eb4d6b29b9 --- /dev/null +++ b/minifi_rust/.gitignore @@ -0,0 +1,7 @@ +.idea +output +venv +.venv +target +.DS_Store +Cargo.lock diff --git a/minifi_rust/CMakeLists.txt b/minifi_rust/CMakeLists.txt new file mode 100644 index 0000000000..c73f684d89 --- /dev/null +++ b/minifi_rust/CMakeLists.txt @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +if (NOT MINIFI_RUST) + return() +endif() + +include(FetchContent) + +# Corrosion is a tool for integrating Rust into an existing CMake project +FetchContent_Declare(Corrosion + URL https://github.com/corrosion-rs/corrosion/archive/refs/tags/v0.6.1.tar.gz + URL_HASH SHA256=e9e95b1ee2bad52681f347993fb1a5af5cce458c5ce8a2636c9e476e4babf8e3 + SYSTEM) + +FetchContent_MakeAvailable(Corrosion) + +corrosion_import_crate(MANIFEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/Cargo.toml) + +include(CTest) + +add_test( + NAME cargo_tests + COMMAND cargo test + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) diff --git a/minifi_rust/Cargo.toml b/minifi_rust/Cargo.toml new file mode 100644 index 0000000000..6dc306f11f --- /dev/null +++ b/minifi_rust/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] +resolver = "3" +members = ["minifi_native_sys", "minifi_native", "minifi_native_macros", "minifi_rs_behave", "extensions/*"] + +[profile.release] +panic = "abort" + +[profile.dev] +panic = "abort" diff --git a/minifi_rust/LICENSE b/minifi_rust/LICENSE new file mode 100644 index 0000000000..7968421e21 --- /dev/null +++ b/minifi_rust/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 The Apache Software Foundation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/minifi_rust/README.md b/minifi_rust/README.md new file mode 100644 index 0000000000..27c5b96868 --- /dev/null +++ b/minifi_rust/README.md @@ -0,0 +1,75 @@ +# MiNiFi Native Rust + +This repository provides a safe, idiomatic, and high-performance Rust framework for building native extensions (processors) for [Apache NiFi MiNiFi C++](https://github.com/apache/nifi-minifi-cpp) + +It is designed to offer a robust developer experience, allowing you to write powerful and reliable data processing components in safe Rust. + +The framework completely encapsulates the unsafe C FFI (Foreign Function Interface) boundary, providing a pure Rust API that is fully mockable for unit testing. + +## Project Philosophy + - **Safety First**: Leverage Rust's compile-time guarantees to prevent common bugs like null pointers, buffer overflows, and data races. + - **Zero-Cost Abstraction**: The safe API wrapper is designed to compile down with zero runtime overhead compared to writing raw C code. + - **Ergonomics**: Provide a clean, idiomatic Rust API that is a pleasure to use. Developers should not need to think about unsafe code or C++ interoperability. + - **Testability**: Every component of a processor's logic should be unit-testable in a pure Rust environment, without needing a C++ host. + - **Cross Platform**: The library should work on all platforms that are supported by [Apache NiFi MiNiFi C++](https://github.com/apache/nifi-minifi-cpp) + - macOS (aarch64) + - Linux (x86_64, aarch64) + - Windows (x86_64) + + +The project is structured as a Cargo workspace with a clear, layered architecture: +### [minifi-native-sys](minifi_native_sys) +Contains the raw, unsafe FFI bindings to the minifi-c.h C API. +### [minifi-native](minifi_native) +Provides the public, safe, and idiomatic Rust API. This is the crate that developers will use to build their processors. +#### API Traits +Pure Rust traits (Processor, ProcessSession, Logger, etc.) that define the abstract behavior of the MiNiFi environment. +#### Higher level API +Pure rust traits that simplify the requirements for a working processor + - FlowFileTransform + - FlowFileSource +#### FFI Wrappers +Concrete structs (CffiSession, CffiLogger, etc.) that implement the API traits by calling the unsafe functions from minifi-native-sys. +#### Thread safety +The trait system differentiates between thread-safe (&self) and single-threaded (&mut self) processors at compile time. +#### Comprehensive Mocking: +A full suite of mock objects allows for fast and reliable unit testing of all processor logic. +### [minifi_native_macros](minifi_native_macros) +Helper crate that includes the procedural macros +### [minifi_rs_behave](minifi_rs_behave) +Run the behave integration tests using Minifi's docker framework. This will test the release artifacts against the latest released [MiNiFi native docker container](https://hub.docker.com/r/apache/nifi-minifi-cpp). +There is a handy alias to initiate all behave tests. + +`cargo behave` + +## Creating an Extension +Building an extension is straightforward. The framework provides a declare_minifi_extension! macro that automatically generates the C-compatible entry points and registers your components. + +```rust +declare_minifi_extension!( + processors: [ + (FlowFileSourceProcessorType, Concurrent, MyFlowFileSource), + (FlowFileTransformProcessorType, Exclusive, MyDataTransformer), + ], + controllers: [ + MyCustomControllerService, + ] +); +``` + + +## Deployment +Build your extension as a dynamic library (cd extensions/your_extension && cargo build --release). + +Locate the output artifact in target/release/ (it will be a .so on Linux, .dll on Windows, or .dylib on macOS). + +Copy the library file into the MiNiFi C++ application's extensions/ directory. + +Restart the MiNiFi Native agent to automatically discover and load the new processors. + +## Included Extensions +### [minifi_rs_playground](extensions/minifi_rs_playground) +A concrete example and testing ground for extensions built using the minifi-native crate. +- Demonstrates how to implement the Processor traits, define processor properties, and route to relationships. +- Includes comprehensive unit tests using the pure-Rust mocking framework. +- Includes integration testing that verifies the processor works as expected in a real, containerized MiNiFi environment. diff --git a/minifi_rust/extensions/minifi_rs_playground/.cargo/config.toml b/minifi_rust/extensions/minifi_rs_playground/.cargo/config.toml new file mode 100644 index 0000000000..cb8c02ddc4 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/.cargo/config.toml @@ -0,0 +1,5 @@ +[target.aarch64-apple-darwin] +rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"] + +[target.x86_64-apple-darwin] +rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"] diff --git a/minifi_rust/extensions/minifi_rs_playground/.gitignore b/minifi_rust/extensions/minifi_rs_playground/.gitignore new file mode 100644 index 0000000000..32c9b263bd --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/.gitignore @@ -0,0 +1,7 @@ +target +output +features/.venv +features/output +integration-test/features/.venv +integration-test/features/linux_so +integration-test/.venv diff --git a/minifi_rust/extensions/minifi_rs_playground/Cargo.toml b/minifi_rust/extensions/minifi_rs_playground/Cargo.toml new file mode 100644 index 0000000000..0100847daa --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "minifi_rs_playground" +version = "0.1.0" +edition = "2024" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +minifi_native = { path = "../../minifi_native" } +strum = "0.28.0" +walkdir = "2.5.0" +rand = "0.10.0" +hex = "0.4.3" +strum_macros = "0.28.0" +lipsum = "0.9.1" + +[dev-dependencies] +tempfile = "3.22.0" +filetime = "0.2.26" diff --git a/minifi_rust/extensions/minifi_rs_playground/features/basic.feature b/minifi_rust/extensions/minifi_rs_playground/features/basic.feature new file mode 100644 index 0000000000..4c642f8530 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/features/basic.feature @@ -0,0 +1,84 @@ +@SUPPORTS_WINDOWS +Feature: Basic scenarios + + Scenario: The rust library is loaded into minifi + Given log property "logger.org::apache::nifi::minifi::core::extension::ExtensionManager" is set to "TRACE,stderr" + And log property "logger.org::apache::nifi::minifi::core::ClassLoader" is set to "TRACE,stderr" + + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "Registering class 'GenerateFlowFileRs' at '/minifi_rs_playground'" in less than 10 seconds + And the Minifi logs contain the following message: "Registering class 'GetFileRs' at '/minifi_rs_playground'" in less than 1 seconds + And the Minifi logs contain the following message: "Registering class 'KamikazeProcessorRs' at '/minifi_rs_playground'" in less than 1 seconds + And the Minifi logs contain the following message: "Registering class 'LogAttributeRs' at '/minifi_rs_playground'" in less than 1 seconds + And the Minifi logs contain the following message: "Registering class 'PutFileRs' at '/minifi_rs_playground'" in less than 1 seconds + And the Minifi logs do not contain errors + And the Minifi logs do not contain warnings + + Scenario: Simple GenerateFlowFileRs -> PutFileRs + Given a GenerateFlowFileRs processor with the "Custom Text" property set to "Ferris the crab" + And the "Data Format" property of the GenerateFlowFileRs processor is set to "Text" + And the "Unique FlowFiles" property of the GenerateFlowFileRs processor is set to "false" + And a PutFileRs processor with the "Directory" property set to "/tmp/output" + And the "success" relationship of the GenerateFlowFileRs processor is connected to the PutFileRs + And PutFileRs's success relationship is auto-terminated + + When the MiNiFi instance starts up + + Then at least one file with the content "Ferris the crab" is placed in the "/tmp/output" directory in less than 10 seconds + And the Minifi logs do not contain errors + And the Minifi logs do not contain warnings + + Scenario: Simple GetFileRs -> PutFileRs + Given a GetFileRs processor with the "Input Directory" property set to "/tmp/input" + And a PutFileRs processor with the "Directory" property set to "/tmp/output" + And the "success" relationship of the GetFileRs processor is connected to the PutFileRs + And PutFileRs's success relationship is auto-terminated + And PutFileRs's failure relationship is auto-terminated + And a directory at "/tmp/input" has a file "test_file.log" with the content "test content" + + When the MiNiFi instance starts up + + Then at least one file with the content "test content" is placed in the "/tmp/output" directory in less than 10 seconds + And the Minifi logs do not contain errors + And the Minifi logs do not contain warnings + + Scenario Outline: The LogAttributeRs can read and log FlowFile content + Given a GenerateFlowFileRs processor with the "Custom Text" property set to "" + And the "Data Format" property of the GenerateFlowFileRs processor is set to "Text" + And the "Unique FlowFiles" property of the GenerateFlowFileRs processor is set to "false" + And a LogAttributeRs processor with the "Log Level" property set to "" + And the "Log Payload" property of the LogAttributeRs processor is set to "true" + And the "success" relationship of the GenerateFlowFileRs processor is connected to the LogAttributeRs + And LogAttributeRs's success relationship is auto-terminated + And log property "logger.minifi_rs_playground::processors::log_attribute::LogAttributeRs" is set to "TRACE,stderr" + + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "" in less than 20 seconds + And the Minifi logs contain the following message: "" in less than 1 seconds + And the Minifi logs do not contain errors + And the Minifi logs do not contain warnings + Examples: + | custom_text | log_level | expected_log_1 | expected_log_2 | + | Elephant | Critical | [critical] Logging for flow file | Elephant | + | Lynx | Info | [info] Logging for flow file | Lynx | + | Ant | Trace | [trace] Logging for flow file | Ant | + + Scenario Outline: Controller services work + Given a LoremIpsumCSUser processor with the "Lorem Ipsum Controller Service" property set to "LoremIpsumControllerService" + And the "Write Method" property of the LoremIpsumCSUser processor is set to "" + And a LoremIpsumControllerService controller service is set up and the "Length" property set to "200000" + And a PutFileRs processor with the "Directory" property set to "/tmp/output" + And the "success" relationship of the LoremIpsumCSUser processor is connected to the PutFileRs + And PutFileRs's success relationship is auto-terminated + + When the MiNiFi instance starts up + + Then at least one file with minimum size of "1 MB" is placed in the "/tmp/output" directory in less than 10 seconds + And the Minifi logs do not contain errors + And the Minifi logs do not contain warnings + Examples: + | write_method | + | Buffer | + | Stream | diff --git a/minifi_rust/extensions/minifi_rs_playground/features/environment.py b/minifi_rust/extensions/minifi_rs_playground/features/environment.py new file mode 100644 index 0000000000..c5d67ba23d --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/features/environment.py @@ -0,0 +1,74 @@ +import os +from typing import List + +from minifi_behave.containers.docker_image_builder import DockerImageBuilder +from minifi_behave.core.hooks import common_after_scenario +from minifi_behave.core.hooks import common_before_scenario, get_minifi_container_image +from minifi_behave.core.minifi_test_context import MinifiTestContext + + +def add_extension_to_minifi_container( + extension_name: str, possible_paths: List[str], context: MinifiTestContext +): + new_container_name = f"apacheminificpp:{extension_name}" + is_windows = os.name == "nt" + if is_windows: + lib_filename = f"{extension_name}.dll" + container_extension_dir = ( + "C:/Program Files/ApacheNiFiMiNiFi/nifi-minifi-cpp/extensions" + ) + else: + lib_filename = f"lib{extension_name}.so" + container_extension_dir = "/opt/minifi/minifi-current/extensions/" + + host_path = None + for path in possible_paths: + if os.path.exists(os.path.join(path, lib_filename)): + host_path = os.path.join(path, lib_filename) + break + + assert host_path is not None, ( + f"Could not find {lib_filename} in {[p for p in possible_paths]}" + ) + + with open(host_path, "rb") as f: + lib_content = f.read() + + base_img = get_minifi_container_image() + + if is_windows: + dockerfile = f""" +FROM {base_img} +COPY ["{lib_filename}", "{container_extension_dir}/{lib_filename}"] +""" + else: + dockerfile = f""" +FROM {base_img} +COPY --chown=minificpp:minificpp {lib_filename} {container_extension_dir} +RUN chmod 755 {container_extension_dir}{lib_filename} +""" + + builder = DockerImageBuilder( + image_tag=new_container_name, + dockerfile_content=dockerfile, + files_on_context={lib_filename: lib_content}, + ) + + builder.build() + return new_container_name + + +def before_all(context): + dir_path = os.path.dirname(os.path.realpath(__file__)) + build_path = os.path.normpath(os.path.join(dir_path, "../../../target/release/")) + deps_build_path = os.path.normpath(os.path.join(dir_path, "../../../target/release/deps/")) + add_extension_to_minifi_container("minifi_rs_playground", [build_path, deps_build_path], context) + + +def before_scenario(context, scenario): + context.minifi_container_image = "apacheminificpp:minifi_rs_playground" + common_before_scenario(context, scenario) + + +def after_scenario(context, scenario): + common_after_scenario(context, scenario) diff --git a/minifi_rust/extensions/minifi_rs_playground/features/error-handling.feature b/minifi_rust/extensions/minifi_rs_playground/features/error-handling.feature new file mode 100644 index 0000000000..23dbf43b44 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/features/error-handling.feature @@ -0,0 +1,70 @@ +@SUPPORTS_WINDOWS +Feature: API error handling and logging + + Scenario: The Api handles empty flow-files + Given a GenerateFlowFileRs processor with the "Custom Text" property set to "${invalid_attribute}" + And the "Data Format" property of the GenerateFlowFileRs processor is set to "Text" + And the "Unique FlowFiles" property of the GenerateFlowFileRs processor is set to "false" + And a LogAttributeRs processor with the "Log Level" property set to "Critical" + And the "success" relationship of the GenerateFlowFileRs processor is connected to the LogAttributeRs + And LogAttributeRs's success relationship is auto-terminated + + When the MiNiFi instance starts up + + Then after 3 sec have passed + And the Minifi logs do not contain errors + And the Minifi logs do not contain warnings + + Scenario: Minifi handles errors from on_schedule + Given a KamikazeProcessorRs processor with the "On Schedule Behaviour" property set to "ReturnErr" + And KamikazeProcessorRs's success relationship is auto-terminated + + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "KamikazeProcessorRs] [error] Error during on_schedule: ScheduleError("it was designed to fail during schedule")" in less than 10 seconds + And the Minifi logs contain the following message: "(KamikazeProcessorRs): Process Schedule Operation: Error while scheduling processor" in less than 10 seconds + + Scenario: Minifi handles errors from on_trigger + Given a KamikazeProcessorRs processor with the "On Schedule Behaviour" property set to "ReturnOk" + And the "On Trigger Behaviour" property of the KamikazeProcessorRs processor is set to "ReturnErr" + And KamikazeProcessorRs's success relationship is auto-terminated + + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "KamikazeProcessorRs] [error] Error during on_trigger TriggerError("it was designed to fail in trigger")" in less than 10 seconds + And the Minifi logs contain the following message: "Trigger and commit failed for processor KamikazeProcessorRs" in less than 10 seconds + + Scenario: Panic in extension's on_schedule crashes the agent aswell + Given a KamikazeProcessorRs processor with the "On Schedule Behaviour" property set to "Panic" + And KamikazeProcessorRs's success relationship is auto-terminated + + When the MiNiFi instance is started without assertions + Then Minifi crashes with the following "KamikazeProcessor::on_schedule panic" in less than 10 seconds + + Scenario: Panic in extension's on_trigger crashes the agent aswell + Given a KamikazeProcessorRs processor with the "On Schedule Behaviour" property set to "ReturnOk" + And the "On Trigger Behaviour" property of the KamikazeProcessorRs processor is set to "Panic" + And KamikazeProcessorRs's success relationship is auto-terminated + + When the MiNiFi instance is started without assertions + Then Minifi crashes with the following "KamikazeProcessor::on_trigger panic" in less than 10 seconds + + Scenario: Get not supported property + Given a KamikazeProcessorRs processor with the "On Schedule Behaviour" property set to "ReturnOk" + And the "On Trigger Behaviour" property of the KamikazeProcessorRs processor is set to "GetNotRegisteredProperty" + And KamikazeProcessorRs's success relationship is auto-terminated + + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "MinifiProcessContextGetProperty("Kamikaze Processor Property"), not supported property" in less than 10 seconds + And the Minifi logs contain the following message: "Trigger and commit failed for processor KamikazeProcessorRs" in less than 10 seconds + + Scenario: Get wrong typed Controller Service + Given a LoremIpsumCSUser processor with the "Lorem Ipsum Controller Service" property set to "My Controller Service" + And a DummyControllerService controller service is set up + And LoremIpsumCSUser's success relationship is auto-terminated + + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "MinifiProcessContextGetControllerService::<"minifi_rs_playground::controller_services::lorem_ipsum_controller_service::LoremIpsumControllerService">("My Controller Service"), validation failed" in less than 10 seconds + And the Minifi logs contain the following message: "Trigger and commit failed for processor LoremIpsumCSUser" in less than 10 seconds diff --git a/minifi_rust/extensions/minifi_rs_playground/features/lazy-logging.feature b/minifi_rust/extensions/minifi_rs_playground/features/lazy-logging.feature new file mode 100644 index 0000000000..89e11197ba --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/features/lazy-logging.feature @@ -0,0 +1,13 @@ +@SUPPORTS_WINDOWS +Feature: Logs should be lazily evaluated + + Scenario: CountActualLogging only increments self when actually logging + Given a CountActualLogging processor + And log property "logger.minifi_rs_playground::processors::count_actual_logging::CountActualLogging" is set to "INFO,stderr" + And CountActualLogging is TIMER_DRIVEN with 1 min scheduling period + + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "[minifi_rs_playground::processors::count_actual_logging::CountActualLogging] [info] info 1" in less than 10 seconds + And the Minifi logs do not contain errors + And the Minifi logs do not contain warnings diff --git a/minifi_rust/extensions/minifi_rs_playground/features/metrics.feature b/minifi_rust/extensions/minifi_rs_playground/features/metrics.feature new file mode 100644 index 0000000000..7ebf844e2a --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/features/metrics.feature @@ -0,0 +1,23 @@ +@SUPPORTS_WINDOWS +Feature: Testing custom and default metrics + + Scenario: CustomMetrics(GetFileRs), DefaultMetrics from streaming(DuplicateStreamText) API and DefaultMetrics from buffer(PutFileRs) API + Given a GetFileRs processor with the "Input Directory" property set to "/tmp/input" + And a DuplicateStreamText processor + And a PutFileRs processor with the "Directory" property set to "/tmp/output" + And the "success" relationship of the GetFileRs processor is connected to the DuplicateStreamText + And the "success" relationship of the DuplicateStreamText processor is connected to the PutFileRs + And PutFileRs's success relationship is auto-terminated + And PutFileRs's failure relationship is auto-terminated + And a directory at "/tmp/input" has a file "hello.txt" with the content "hello" + And MiNiFi logs processor metrics + + When the MiNiFi instance starts up + + Then at least one file with the content "hheelllloo" is placed in the "/tmp/output" directory in less than 10 seconds + And the Minifi logs match the following regex: "GetFileRsMetrics": {\n[ ]+\"[0-9a-z-]+\": \{\n[ a-zA-Z0-9":,\n]*"BytesRead": "0",[\n ]*"BytesWritten": "5"[ a-zA-Z0-9":,\n]*"InputBytes": "5"[ a-zA-Z0-9":,\n]*}" in less than 10 seconds + And the Minifi logs match the following regex: "DuplicateStreamTextMetrics": {\n[ ]+\"[0-9a-z-]+\": \{\n[ a-zA-Z0-9":,\n]*"BytesRead": "5",[\n ]*"BytesWritten": "10"[ a-zA-Z0-9":,\n]*}" in less than 10 seconds + And the Minifi logs match the following regex: "PutFileRsMetrics": {\n[ ]+\"[0-9a-z-]+\": \{\n[ a-zA-Z0-9":,\n]*"BytesRead": "10"[ a-zA-Z0-9":,\n]*}" in less than 10 seconds + + And the Minifi logs do not contain errors + And the Minifi logs do not contain warnings diff --git a/minifi_rust/extensions/minifi_rs_playground/features/steps/steps.py b/minifi_rust/extensions/minifi_rs_playground/features/steps/steps.py new file mode 100644 index 0000000000..181105116e --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/features/steps/steps.py @@ -0,0 +1,45 @@ +from behave import then, when, given +import humanfriendly + +from minifi_behave.steps import checking_steps # noqa: F401 +from minifi_behave.steps import configuration_steps # noqa: F401 +from minifi_behave.steps import core_steps # noqa: F401 +from minifi_behave.steps import flow_building_steps # noqa: F401 +from minifi_behave.core.helpers import wait_for_condition +from minifi_behave.core.minifi_test_context import MinifiTestContext + + +@when("the MiNiFi instance is started without assertions") +def minifi_starts_wo_assertions(context: MinifiTestContext): + context.get_or_create_default_minifi_container().deploy(context) + + +@then('Minifi crashes with the following "{crash_msg}" in less than {duration}') +def minifi_crashes(context: MinifiTestContext, crash_msg: str, duration: str): + duration_seconds = humanfriendly.parse_timespan(duration) + assert wait_for_condition( + condition=lambda: ( + context.get_or_create_default_minifi_container().exited + and crash_msg in context.get_or_create_default_minifi_container().get_logs() + ), + timeout_seconds=duration_seconds, + bail_condition=lambda: False, + context=context, + ) + + +@given("MiNiFi logs processor metrics") +def minifi_logs_processor_metrics(context: MinifiTestContext): + context.get_or_create_default_minifi_container().set_property( + "nifi.metrics.publisher.LogMetricsPublisher.metrics", + "GetFileRsMetrics,DuplicateStreamTextMetrics,PutFileRsMetrics", + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.metrics.publisher.LogMetricsPublisher.logging.interval", "1s" + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.metrics.publisher.class", "LogMetricsPublisher" + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.metrics.publisher.agent.identifier", "Agent1" + ) diff --git a/minifi_rust/extensions/minifi_rs_playground/features/streaming.feature b/minifi_rust/extensions/minifi_rs_playground/features/streaming.feature new file mode 100644 index 0000000000..d6fea8933d --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/features/streaming.feature @@ -0,0 +1,34 @@ +@SUPPORTS_WINDOWS +Feature: Testing streaming reads and writes + + Scenario: Streaming Transforms work + Given a GetFileRs processor with the "Input Directory" property set to "/tmp/input" + And a AsciifyGerman processor + And a PutFileRs processor with the "Directory" property set to "/tmp/output" + And the "success" relationship of the GetFileRs processor is connected to the AsciifyGerman + And the "success" relationship of the AsciifyGerman processor is connected to the PutFileRs + And PutFileRs's success relationship is auto-terminated + And PutFileRs's failure relationship is auto-terminated + And a directory at "/tmp/input" has a file "german.txt" with the content "Üben von Xylophon und Querflöte ist ja zweckmäßig." + + When the MiNiFi instance starts up + + Then at least one file with the content "Ueben von Xylophon und Querfloete ist ja zweckmaessig." is placed in the "/tmp/output" directory in less than 10 seconds + And the Minifi logs do not contain errors + And the Minifi logs do not contain warnings + + Scenario: Streaming can be cancelled + Given a GetFileRs processor with the "Input Directory" property set to "/tmp/input" + And a AsciifyGerman processor + And a PutFileRs processor with the "Directory" property set to "/tmp/output" + And the "success" relationship of the GetFileRs processor is connected to the AsciifyGerman + And the "failure" relationship of the AsciifyGerman processor is connected to the PutFileRs + And PutFileRs's success relationship is auto-terminated + And PutFileRs's failure relationship is auto-terminated + And a directory at "/tmp/input" has a file "french.txt" with the content "Voix ambiguë d'un cœur qui, au zéphyr, préfère les jattes de kiwis." + + When the MiNiFi instance starts up + + Then at least one file with the content "Voix ambiguë d'un cœur qui, au zéphyr, préfère les jattes de kiwis." is placed in the "/tmp/output" directory in less than 10 seconds + And the Minifi logs do not contain errors + And the Minifi logs do not contain warnings diff --git a/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md b/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md new file mode 100644 index 0000000000..9b2397fcd5 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md @@ -0,0 +1,260 @@ + + +## Table of Contents + +### Processors + +- [AsciifyGerman](#AsciifyGerman) +- [CountActualLogging](#CountActualLogging) +- [GenerateFlowFileRs](#GenerateFlowFileRs) +- [GetFileRs](#GetFileRs) +- [KamikazeProcessorRs](#KamikazeProcessorRs) +- [LogAttributeRs](#LogAttributeRs) +- [LoremIpsumCSUser](#LoremIpsumCSUser) +- [PutFileRs](#PutFileRs) +### Controller Services + +- [DummyControllerService](#DummyControllerService) +- [LoremIpsumControllerService](#LoremIpsumControllerService) + + +## AsciifyGerman + +### Description + +This processor switches German characters with their ascii counterparts. (to test stream API) + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|------|---------------|------------------|-------------| + +### Relationships + +| Name | Description | +|---------|-----------------------------------------| +| success | All asciified flowfiles are routed here | +| failure | Non-german flowfiles are routed here | + + +## CountActualLogging + +### Description + +For testing lazy logging + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|------|---------------|------------------|-------------| + +### Relationships + +| Name | Description | +|------|-------------| + + +## GenerateFlowFileRs + +### Description + +This processor creates FlowFiles with random data or custom content. GenerateFlowFile is useful for load testing, configuration, and simulation. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|----------------------|---------------|------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **File Size** | 1 kB | | The size of the file that will be used
**Supports Expression Language: true** | +| **Batch Size** | 1 | | The number of FlowFiles to be transferred in each invocation | +| **Data Format** | Binary | Text
Binary | Specifies whether the data should be Text or Binary | +| **Unique FlowFiles** | true | true
false | If true, each FlowFile that is generated will be unique. If false, a random value will be generated and all FlowFiles will get the same content but this offers much higher throughput (but see the description of Custom Text for special non-random use cases) | +| Custom Text | | | If Data Format is text and if Unique FlowFiles is false, then this custom text will be used as content of the generated FlowFiles and the File Size will be ignored. Finally, if Expression Language is used, evaluation will be performed only once per batch of generated FlowFiles
**Supports Expression Language: true** | + +### Relationships + +| Name | Description | +|---------|----------------------------------------| +| success | success operational on the flow record | + + +## GetFileRs + +### Description + +Creates FlowFiles from files in a directory. MiNiFi will ignore files for which it doesn't have read permissions. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|-------------------------|---------------|------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Input Directory** | | | The input directory from which to pull files
**Supports Expression Language: true** | +| Polling Interval | | | Indicates how long to wait before performing a directory listing | +| Recurse Subdirectories | true | true
false | Indicates whether or not to pull files from subdirectories | +| Keep Source File | false | true
false | If true, the file is not deleted after it has been copied to the Content Repository | +| Minimum File Age | | | The minimum age that a file must be in order to be pulled; any file younger than this amount of time (according to last modification date) will be ignored | +| Maximum File Age | | | The maximum age that a file must be in order to be pulled; any file older than this amount of time (according to last modification date) will be ignored | +| Minimum File Size | | | The minimum size that a file can be in order to be pulled | +| Maximum File Size | | | The maximum size that a file can be in order to be pulled | +| **Ignore Hidden Files** | true | true
false | Indicates whether or not hidden files should be ignored | +| **Batch Size** | 10 | | The maximum number of files to pull in each iteration | + +### Relationships + +| Name | Description | +|---------|----------------------------------------------| +| success | FlowFiles are transferred here after logging | + +### Output Attributes + +| Attribute | Relationship | Description | +|---------------|--------------|-------------------------------------------------------------------------------------------------------------------------------------| +| absolute.path | success | The full/absolute path from where a file was picked up. The current 'path' attribute is still populated, but may be a relative path | +| filename | success | The filename is set to the name of the file on disk | + + +## KamikazeProcessorRs + +### Description + +This processor can fail or panic in on_trigger and on_schedule calls based on configuration. Only for testing purposes. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|---------------------------|---------------|-----------------------------------------------------------------------------------------------|------------------------------------------| +| **On Schedule Behaviour** | ReturnOk | ReturnErr
ReturnOk
GetNotRegisteredProperty
GetInvalidControllerService
Panic | What to do during the on_schedule method | +| **On Trigger Behaviour** | ReturnOk | ReturnErr
ReturnOk
GetNotRegisteredProperty
GetInvalidControllerService
Panic | What to do during the on_trigger method | + +### Relationships + +| Name | Description | +|---------|----------------------| +| success | success relationship | + + +## LogAttributeRs + +### Description + +Logs attributes of flow files in the MiNiFi application log. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|-----------------------|---------------|------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Log Level** | Info | Trace
Debug
Info
Warn
Error
Critical
Off | The Log Level to use when logging the Attributes | +| Attributes to Log | | | A comma-separated list of Attributes to Log. If not specified, all attributes will be logged. | +| Attributes to Ignore | | | A comma-separated list of Attributes to ignore. If not specified, no attributes will be ignored. | +| **Log Payload** | false | true
false | If true, the FlowFile's payload will be logged, in addition to its attributes. Otherwise, just the Attributes will be logged. | +| Log Prefix | | | Log prefix appended to the log lines. It helps to distinguish the output of multiple LogAttribute processors. | +| **FlowFiles To Log** | 1 | | Number of flow files to log. If set to zero all flow files will be logged. Please note that this may block other threads from running if not used judiciously. | +| **Hexencode Payload** | false | true
false | If true, the FlowFile's payload will be logged in a hexencoded format | + +### Relationships + +| Name | Description | +|---------|----------------------------------------------| +| success | FlowFiles are transferred here after logging | + + +## LoremIpsumCSUser + +### Description + +Processor to test Controller Service API + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|------------------------------------|---------------|-------------------|--------------------------------------------| +| **Lorem Ipsum Controller Service** | | | Name of the lorem ipsum controller service | +| **Write Method** | Buffer | Buffer
Stream | Which API to test | + +### Relationships + +| Name | Description | +|---------|------------------------------| +| success | All flowfile are routed here | + + +## PutFileRs + +### Description + +Writes the contents of a FlowFile to the local file system. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|----------------------------------|---------------|-----------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Directory** | . | | The output directory to which to put files
**Supports Expression Language: true** | +| **Conflict Resolution Strategy** | fail | fail
replace
ignore | Indicates what should happen when a file with the same name already exists in the output directory | +| **Create Missing Directories** | true | true
false | If true, then missing destination directories will be created. If false, flowfiles are penalized and sent to failure. | +| Maximum File Count | | | Specifies the maximum number of files that can exist in the output directory | +| Permissions | | | Sets the permissions on the output file to the value of this attribute. Must be an octal number (e.g. 644 or 0755). Not supported on Windows systems. | +| Directory Permissions | | | Sets the permissions on the directories being created if 'Create Missing Directories' property is set. Must be an octal number (e.g. 644 or 0755). Not supported on Windows systems. | + +### Relationships + +| Name | Description | +|---------|-----------------------------------------------------------------------------------| +| success | Flowfiles that are successfully written to a file are routed to this relationship | +| failure | Failed files (conflict, write failure, etc.) are transferred to failure | + + +## DummyControllerService + +### Description + +Dummy Controller Service + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|------|---------------|------------------|-------------| + + +## LoremIpsumControllerService + +### Description + +Simple Rusty Controller Service to test API + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|------------|---------------|------------------|-------------------------------------------------------------| +| **Length** | 25 | | How many words to generate
**Sensitive Property: true** | diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs new file mode 100644 index 0000000000..ccd098ab41 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs @@ -0,0 +1,22 @@ +use minifi_native::macros::ComponentIdentifier; +use minifi_native::{ + ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, + Property, +}; + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct DummyControllerService {} + +impl EnableControllerService for DummyControllerService { + fn enable(_context: &Ctx, _logger: &L) -> Result + where + Self: Sized, + { + Ok(Self {}) + } +} + +impl ControllerServiceDefinition for DummyControllerService { + const DESCRIPTION: &'static str = "Dummy Controller Service"; + const PROPERTIES: &'static [Property] = &[]; +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs new file mode 100644 index 0000000000..eeb9ab5213 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs @@ -0,0 +1,33 @@ +mod properties; + +use crate::controller_services::lorem_ipsum_controller_service::properties::LENGTH; +use lipsum::lipsum; +use minifi_native::macros::ComponentIdentifier; +use minifi_native::{ + ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, + Property, +}; + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct LoremIpsumControllerService { + pub data: String, +} + +impl EnableControllerService for LoremIpsumControllerService { + fn enable(context: &P, _logger: &L) -> Result + where + Self: Sized, + { + let length = context + .get_u64_property(&LENGTH)? + .ok_or(MinifiError::missing_required_property("Length is required"))?; + + let data = lipsum(length as usize); + Ok(Self { data }) + } +} + +impl ControllerServiceDefinition for LoremIpsumControllerService { + const DESCRIPTION: &'static str = "Simple Rusty Controller Service to test API"; + const PROPERTIES: &'static [Property] = &[LENGTH]; +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service/properties.rs new file mode 100644 index 0000000000..b895647a7c --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service/properties.rs @@ -0,0 +1,13 @@ +use minifi_native::{Property, StandardPropertyValidator}; + +pub(crate) const LENGTH: Property = Property { + name: "Length", + description: "How many words to generate", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("25"), + validator: StandardPropertyValidator::U64Validator, + allowed_values: &[], + allowed_type: "", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs new file mode 100644 index 0000000000..7935987e7f --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod dummy_controller_service; +pub(crate) mod lorem_ipsum_controller_service; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/lib.rs b/minifi_rust/extensions/minifi_rs_playground/src/lib.rs new file mode 100644 index 0000000000..ad83792f28 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/lib.rs @@ -0,0 +1,37 @@ +mod controller_services; +mod processors; + +use crate::controller_services::dummy_controller_service::DummyControllerService; +use crate::controller_services::lorem_ipsum_controller_service::LoremIpsumControllerService; +use crate::processors::asciify_german::AsciifyGerman; +use crate::processors::count_actual_logging::CountActualLogging; +use crate::processors::duplicate_text::DuplicateStreamText; +use crate::processors::generate_flow_file::GenerateFlowFileRs; +use crate::processors::get_file::GetFileRs; +use crate::processors::kamikaze_processor::KamikazeProcessorRs; +use crate::processors::log_attribute::LogAttributeRs; +use crate::processors::lorem_ipsum_cs_user::LoremIpsumCSUser; +use crate::processors::put_file::PutFileRs; + +use minifi_native::{ + ComplexProcessorType, Concurrent, Exclusive, FlowFileSourceProcessorType, + FlowFileStreamTransformProcessorType, FlowFileTransformProcessorType, +}; + +minifi_native::declare_minifi_extension!( +processors: [ + (ComplexProcessorType, Concurrent, GenerateFlowFileRs), + (ComplexProcessorType, Concurrent, LogAttributeRs), + (ComplexProcessorType, Concurrent, GetFileRs), + (ComplexProcessorType, Concurrent, KamikazeProcessorRs), + (ComplexProcessorType, Exclusive, CountActualLogging), + (FlowFileSourceProcessorType, Concurrent, LoremIpsumCSUser), + (FlowFileTransformProcessorType, Concurrent, PutFileRs), + (FlowFileStreamTransformProcessorType, Concurrent, AsciifyGerman), + (FlowFileStreamTransformProcessorType, Exclusive, DuplicateStreamText), +], +controllers: [ + LoremIpsumControllerService, + DummyControllerService, +] +); diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs new file mode 100644 index 0000000000..e85949a07f --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs @@ -0,0 +1,67 @@ +use crate::processors::asciify_german::relationships::FAILURE; +use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::{ + FlowFileStreamTransform, GetProperty, InputStream, Logger, MinifiError, OutputStream, Schedule, + TransformStreamResult, +}; +use std::collections::HashMap; + +mod relationships; + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct AsciifyGerman {} + +impl Schedule for AsciifyGerman { + fn schedule(_context: &P, _logger: &L) -> Result + where + Self: Sized, + { + Ok(Self {}) + } +} + +impl FlowFileStreamTransform for AsciifyGerman { + fn transform( + &self, + _context: &Ctx, + input_stream: &mut dyn InputStream, + output_stream: &mut dyn OutputStream, + _logger: &LoggerImpl, + ) -> Result { + let mut byte = [0u8; 1]; + + while input_stream.read(&mut byte)? > 0 { + match byte[0] { + 0..=127 => { + output_stream.write_all(&byte)?; + } + 0xC3 => { + let mut next = [0u8; 1]; + if input_stream.read(&mut next)? > 0 { + match next[0] { + 0xA4 => output_stream.write_all(b"ae")?, // ä + 0xB6 => output_stream.write_all(b"oe")?, // ö + 0xBC => output_stream.write_all(b"ue")?, // ü + 0x84 => output_stream.write_all(b"Ae")?, // Ä + 0x96 => output_stream.write_all(b"Oe")?, // Ö + 0x9C => output_stream.write_all(b"Ue")?, // Ü + 0x9F => output_stream.write_all(b"ss")?, // ß + _ => return Ok(TransformStreamResult::route_without_changes(&FAILURE)), + } + } + } + _ => return Ok(TransformStreamResult::route_without_changes(&FAILURE)), + } + } + + output_stream.flush()?; + Ok(TransformStreamResult::new( + &relationships::SUCCESS, + HashMap::new(), + )) + } +} + +mod processor_definition; +#[cfg(test)] +mod tests; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/processor_definition.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/processor_definition.rs new file mode 100644 index 0000000000..62f135741d --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/processor_definition.rs @@ -0,0 +1,15 @@ +use crate::processors::asciify_german::AsciifyGerman; +use minifi_native::{ + OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, +}; + +impl ProcessorDefinition for AsciifyGerman { + const DESCRIPTION: &'static str = "This processor switches German characters with their ascii counterparts. (to test stream API)"; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[]; + const RELATIONSHIPS: &'static [Relationship] = + &[super::relationships::SUCCESS, super::relationships::FAILURE]; + const PROPERTIES: &'static [Property] = &[]; +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/relationships.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/relationships.rs new file mode 100644 index 0000000000..ccb190d200 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/relationships.rs @@ -0,0 +1,11 @@ +use minifi_native::Relationship; + +pub(crate) const SUCCESS: Relationship = Relationship { + name: "success", + description: "All asciified flowfiles are routed here", +}; + +pub(crate) const FAILURE: Relationship = Relationship { + name: "failure", + description: "Non-german flowfiles are routed here", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/tests.rs new file mode 100644 index 0000000000..268a95ff92 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german/tests.rs @@ -0,0 +1,54 @@ +use super::*; +use crate::processors::asciify_german::relationships::SUCCESS; +use minifi_native::{IoState, MockLogger, MockProcessContext}; +use std::io::BufReader; + +#[test] +fn schedule_succeeds_with_default_values() { + assert!(AsciifyGerman::schedule(&MockProcessContext::new(), &MockLogger::new()).is_ok()); +} + +#[test] +fn simple_test() { + let process_context = MockProcessContext::new(); + let context = MockProcessContext::new(); + let logger = MockLogger::new(); + + let asciify_german = + AsciifyGerman::schedule(&process_context, &logger).expect("Should succeed"); + let input_str = "Falsches Üben von Xylophonmusik quält jeden größeren Zwerg."; + let mut input_stream = BufReader::new(input_str.as_bytes()); + let mut output_vec: Vec = Vec::new(); + { + let result = asciify_german + .transform(&context, &mut input_stream, &mut output_vec, &logger) + .expect("Should succeed"); + assert_eq!(result.write_status(), IoState::Ok); + assert_eq!(result.target_relationship_name(), SUCCESS.name); + } + assert_eq!( + output_vec, + "Falsches Ueben von Xylophonmusik quaelt jeden groesseren Zwerg.".as_bytes() + ); +} + +#[test] +fn simple_failure_test() { + let process_context = MockProcessContext::new(); + let context = MockProcessContext::new(); + let logger = MockLogger::new(); + + let asciify_german = + AsciifyGerman::schedule(&process_context, &logger).expect("Should succeed"); + let input_str = "Üldögélő műújságíró"; + let mut input_stream = BufReader::new(input_str.as_bytes()); + let mut output_vec: Vec = Vec::new(); + { + let result = asciify_german + .transform(&context, &mut input_stream, &mut output_vec, &logger) + .expect("Should succeed"); + assert_eq!(result.write_status(), IoState::Cancel); + assert_eq!(result.target_relationship_name(), FAILURE.name); + } + assert_eq!(output_vec, "Ueldoeg".as_bytes()); +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/count_actual_logging.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/count_actual_logging.rs new file mode 100644 index 0000000000..f712f37c9a --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/count_actual_logging.rs @@ -0,0 +1,57 @@ +use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::{ + GetProperty, Logger, MinifiError, MutTrigger, OnTriggerResult, OutputAttribute, ProcessContext, + ProcessSession, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, + Schedule, debug, info, trace, +}; + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct CountActualLogging { + log_count: usize, +} + +impl CountActualLogging { + fn get_incremented_log_count(&mut self) -> usize { + self.log_count += 1; + self.log_count + } +} + +impl Schedule for CountActualLogging { + fn schedule(_context: &P, _logger: &L) -> Result + where + Self: Sized, + { + Ok(Self { log_count: 0 }) + } +} + +impl MutTrigger for CountActualLogging { + fn trigger( + &mut self, + _context: &mut PC, + _session: &mut PS, + logger: &L, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession, + L: Logger, + { + trace!(logger, "trace {}", self.get_incremented_log_count()); + debug!(logger, "debug {}", self.get_incremented_log_count()); + info!(logger, "info {}", self.get_incremented_log_count()); + + Ok(OnTriggerResult::Ok) + } +} + +impl ProcessorDefinition for CountActualLogging { + const DESCRIPTION: &'static str = "For testing lazy logging"; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Forbidden; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[]; + const RELATIONSHIPS: &'static [Relationship] = &[]; + const PROPERTIES: &'static [Property] = &[]; +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/duplicate_text.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/duplicate_text.rs new file mode 100644 index 0000000000..38ff066e7b --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/duplicate_text.rs @@ -0,0 +1,54 @@ +use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::{ + GetAttribute, GetControllerService, GetProperty, InputStream, Logger, MinifiError, + MutFlowFileStreamTransform, OutputAttribute, OutputStream, ProcessorDefinition, + ProcessorInputRequirement, Property, Relationship, Schedule, TransformStreamResult, +}; +use std::collections::HashMap; + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct DuplicateStreamText {} + +pub(crate) const SUCCESS: Relationship = Relationship { + name: "success", + description: "", +}; + +impl Schedule for DuplicateStreamText { + fn schedule( + _context: &Ctx, + _logger: &L, + ) -> Result + where + Self: Sized, + { + Ok(Self {}) + } +} + +impl MutFlowFileStreamTransform for DuplicateStreamText { + fn transform( + &mut self, + _context: &Ctx, + input_stream: &mut dyn InputStream, + output_stream: &mut dyn OutputStream, + _logger: &LoggerImpl, + ) -> Result { + let mut byte = [0u8; 1]; + while input_stream.read(&mut byte)? > 0 { + output_stream.write(&byte)?; + output_stream.write(&byte)?; + } + Ok(TransformStreamResult::new(&SUCCESS, HashMap::new())) + } +} + +impl ProcessorDefinition for DuplicateStreamText { + const DESCRIPTION: &'static str = "Duplicate text"; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[]; + const RELATIONSHIPS: &'static [Relationship] = &[SUCCESS]; + const PROPERTIES: &'static [Property] = &[]; +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs new file mode 100644 index 0000000000..1e9800ce5f --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs @@ -0,0 +1,168 @@ +use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::{ + GetProperty, Logger, MinifiError, OnTriggerResult, ProcessContext, + ProcessSession, Schedule, Trigger, +}; +use rand::RngExt; +use rand::distr::Alphanumeric; +use std::cmp::PartialEq; + +mod properties; +mod relationships; + +#[derive(Debug, PartialEq)] +enum Mode { + UniqueBytes, + UniqueText, + NotUniqueBytes, + NotUniqueText, + CustomText, + Empty, +} + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct GenerateFlowFileRs { + mode: Mode, + batch_size: u64, + file_size: u64, + data_generated_during_on_schedule: Vec, +} + +impl Schedule for GenerateFlowFileRs { + fn schedule(context: &P, _logger: &L) -> Result + where + Self: Sized, + { + let is_unique = context + .get_bool_property(&properties::UNIQUE_FLOW_FILES)? + .expect("Required property"); + let is_text = context + .get_property(&properties::DATA_FORMAT)? + .expect("Required property") + .as_str() + == "Text"; + let has_custom_text = context.get_property(&properties::CUSTOM_TEXT)?.is_some(); + + let file_size = context + .get_size_property(&properties::FILE_SIZE)? + .expect("Required property"); + let batch_size = context + .get_u64_property(&properties::BATCH_SIZE)? + .expect("Required property"); + + let mode = Self::get_mode(is_unique, is_text, has_custom_text, file_size); + let data_generated_during_on_schedule = + if mode == Mode::NotUniqueText || mode == Mode::NotUniqueBytes { + let mut data = vec![0; file_size as usize]; + Self::generate_data(&mut data, is_text); + data + } else { + vec![] + }; + + Ok(Self { + mode, + batch_size, + file_size, + data_generated_during_on_schedule, + }) + } +} + +impl GenerateFlowFileRs { + fn is_unique(&self) -> bool { + match self.mode { + Mode::UniqueBytes => true, + Mode::UniqueText => true, + Mode::NotUniqueBytes => false, + Mode::NotUniqueText => false, + Mode::CustomText => false, + Mode::Empty => false, + } + } + + fn is_text(&self) -> bool { + match self.mode { + Mode::UniqueBytes => false, + Mode::UniqueText => true, + Mode::NotUniqueBytes => false, + Mode::NotUniqueText => true, + Mode::CustomText => true, + Mode::Empty => false, + } + } + + fn get_mode(is_unique: bool, is_text: bool, has_custom_text: bool, file_size: u64) -> Mode { + if is_text && !is_unique && has_custom_text { + return Mode::CustomText; + } + + if file_size == 0 { + return Mode::Empty; + } + + match (is_unique, is_text) { + (true, true) => Mode::UniqueText, + (true, false) => Mode::UniqueBytes, + (false, true) => Mode::NotUniqueText, + (false, false) => Mode::NotUniqueBytes, + } + } + + fn generate_data(data: &mut [u8], text_data: bool) { + let mut rng = rand::rng(); + + if text_data { + for byte in data.iter_mut() { + *byte = rng.sample(Alphanumeric); + } + } else { + rng.fill(data); + } + } +} + +impl Trigger for GenerateFlowFileRs { + fn trigger( + &self, + context: &mut PC, + session: &mut PS, + _logger: &L, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession, + { + let non_unique_data_buffer: &[u8]; + let custom_text_for_batch: Option; + + if self.mode == Mode::CustomText + && let Some(custom_text) = context.get_property(&properties::CUSTOM_TEXT, None)? + { + custom_text_for_batch = Some(custom_text); + non_unique_data_buffer = custom_text_for_batch.as_ref().unwrap().as_bytes(); + } else { + non_unique_data_buffer = self.data_generated_during_on_schedule.as_slice(); + } + + for _ in 0..self.batch_size { + let mut ff = session.create()?; + if self.mode != Mode::Empty { + if self.is_unique() { + let mut unique_data: Vec = vec![0; self.file_size as usize]; + Self::generate_data(&mut unique_data, self.is_text()); + session.write(&mut ff, unique_data.as_slice())?; + } else { + session.write(&mut ff, non_unique_data_buffer)?; + } + } + session.transfer(ff, relationships::SUCCESS.name)?; + } + Ok(OnTriggerResult::Ok) + } +} + +pub(crate) mod processor_definition; + +#[cfg(test)] +mod tests; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/processor_definition.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/processor_definition.rs new file mode 100644 index 0000000000..7e0c21b520 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/processor_definition.rs @@ -0,0 +1,21 @@ +use super::properties::*; +use super::{GenerateFlowFileRs, relationships}; +use minifi_native::{ + OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, +}; + +impl ProcessorDefinition for GenerateFlowFileRs { + const DESCRIPTION: &'static str = "This processor creates FlowFiles with random data or custom content. GenerateFlowFile is useful for load testing, configuration, and simulation."; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Forbidden; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[]; + const RELATIONSHIPS: &'static [Relationship] = &[relationships::SUCCESS]; + const PROPERTIES: &'static [Property] = &[ + FILE_SIZE, + BATCH_SIZE, + DATA_FORMAT, + UNIQUE_FLOW_FILES, + CUSTOM_TEXT, + ]; +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/properties.rs new file mode 100644 index 0000000000..19d81836c9 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/properties.rs @@ -0,0 +1,61 @@ +use minifi_native::{Property, StandardPropertyValidator}; + +pub(crate) const FILE_SIZE: Property = Property { + name: "File Size", + description: "The size of the file that will be used", + is_required: true, + is_sensitive: false, + supports_expr_lang: true, + default_value: Some("1 kB"), + validator: StandardPropertyValidator::DataSizeValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const BATCH_SIZE: Property = Property { + name: "Batch Size", + description: "The number of FlowFiles to be transferred in each invocation", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("1"), + validator: StandardPropertyValidator::U64Validator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const DATA_FORMAT: Property = Property { + name: "Data Format", + description: "Specifies whether the data should be Text or Binary", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("Binary"), + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &["Text", "Binary"], + allowed_type: "", +}; + +pub(crate) const UNIQUE_FLOW_FILES: Property = Property { + name: "Unique FlowFiles", + description: "If true, each FlowFile that is generated will be unique. If false, a random value will be generated and all FlowFiles will get the same content but this offers much higher throughput (but see the description of Custom Text for special non-random use cases)", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("true"), + validator: StandardPropertyValidator::BoolValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const CUSTOM_TEXT: Property = Property { + name: "Custom Text", + description: "If Data Format is text and if Unique FlowFiles is false, then this custom text will be used as content of the generated FlowFiles and the File Size will be ignored. Finally, if Expression Language is used, evaluation will be performed only once per batch of generated FlowFiles", + is_required: false, + is_sensitive: false, + supports_expr_lang: true, + default_value: None, + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &[], + allowed_type: "", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/relationships.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/relationships.rs new file mode 100644 index 0000000000..3fb72c7114 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/relationships.rs @@ -0,0 +1,6 @@ +use minifi_native::Relationship; + +pub(crate) const SUCCESS: Relationship = Relationship { + name: "success", + description: "success operational on the flow record", // TODO(wtf... but copy paste) +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/tests.rs new file mode 100644 index 0000000000..2a1f4108f8 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/tests.rs @@ -0,0 +1,138 @@ +use super::*; +use crate::processors::generate_flow_file::properties::{ + BATCH_SIZE, CUSTOM_TEXT, DATA_FORMAT, UNIQUE_FLOW_FILES, +}; +use minifi_native::{MockLogger, MockProcessContext, MockProcessSession}; + +#[test] +fn schedule_succeeds_with_default_values() { + assert!(GenerateFlowFileRs::schedule(&MockProcessContext::new(), &MockLogger::new()).is_ok()); +} + +#[test] +fn generate_flow_file_empty_test() { + let logger = MockLogger::new(); + let mut context = MockProcessContext::new(); + context + .properties + .insert(properties::FILE_SIZE.name.to_string(), "0".to_string()); + context + .properties + .insert(UNIQUE_FLOW_FILES.name.to_string(), "false".to_string()); + context + .properties + .insert(DATA_FORMAT.name.to_string(), "Text".to_string()); + + let processor = GenerateFlowFileRs::schedule(&context, &logger).unwrap(); + let mut session = MockProcessSession::new(); + assert_eq!( + processor + .trigger(&mut context, &mut session, &logger) + .unwrap(), + OnTriggerResult::Ok + ); + let result_flow_files = session.transferred_flow_files.borrow(); + assert_eq!(result_flow_files.len(), 1); + assert_eq!(result_flow_files[0].flow_file.content_len(), 0); +} + +#[test] +fn generate_custom_text() { + let mut context = MockProcessContext::new(); + context + .properties + .insert(properties::FILE_SIZE.name.to_string(), "0".to_string()); + context + .properties + .insert(UNIQUE_FLOW_FILES.name.to_string(), "false".to_string()); + context + .properties + .insert(DATA_FORMAT.name.to_string(), "Text".to_string()); + context + .properties + .insert(CUSTOM_TEXT.name.to_string(), "foo bar baz".to_string()); + + let logger = MockLogger::new(); + let processor = GenerateFlowFileRs::schedule(&context, &logger).unwrap(); + + let mut session = MockProcessSession::new(); + assert_eq!( + processor + .trigger(&mut context, &mut session, &logger) + .expect("Should trigger successfully"), + OnTriggerResult::Ok + ); + let result_flow_files = session.transferred_flow_files.borrow(); + assert_eq!(result_flow_files.len(), 1); + assert!(result_flow_files[0].flow_file.content_eq("foo bar baz"),); +} + +#[test] +fn random_bytes_unique() { + let mut context = MockProcessContext::new(); + context + .properties + .insert(properties::FILE_SIZE.name.to_string(), "40 B".to_string()); + context + .properties + .insert(UNIQUE_FLOW_FILES.name.to_string(), "true".to_string()); + context + .properties + .insert(DATA_FORMAT.name.to_string(), "Bytes".to_string()); + context + .properties + .insert(BATCH_SIZE.name.to_string(), "2".to_string()); + + let logger = MockLogger::new(); + let processor = GenerateFlowFileRs::schedule(&context, &logger).unwrap(); + let mut session = MockProcessSession::new(); + assert_eq!( + processor + .trigger(&mut context, &mut session, &logger) + .expect("Should trigger successfully"), + OnTriggerResult::Ok + ); + let result_flow_files = session.transferred_flow_files.borrow(); + assert_eq!(result_flow_files.len(), 2); + assert_eq!(result_flow_files[0].flow_file.content_len(), 40); + assert_eq!(result_flow_files[1].flow_file.content_len(), 40); + assert_ne!( + *result_flow_files[0].flow_file.content.borrow(), + *result_flow_files[1].flow_file.content.borrow() + ); +} + +#[test] +fn random_bytes_non_unique() { + let mut context = MockProcessContext::new(); + context + .properties + .insert(properties::FILE_SIZE.name.to_string(), "40 B".to_string()); + context + .properties + .insert(UNIQUE_FLOW_FILES.name.to_string(), "false".to_string()); + context + .properties + .insert(DATA_FORMAT.name.to_string(), "Bytes".to_string()); + context + .properties + .insert(BATCH_SIZE.name.to_string(), "2".to_string()); + + let logger = MockLogger::new(); + let processor = GenerateFlowFileRs::schedule(&context, &logger).unwrap(); + let mut session = MockProcessSession::new(); + assert_eq!( + processor + .trigger(&mut context, &mut session, &logger) + .expect("Should trigger successfully"), + OnTriggerResult::Ok + ); + let result_flow_files = session.transferred_flow_files.borrow(); + assert_eq!(result_flow_files.len(), 2); + assert_eq!(result_flow_files[0].flow_file.content_len(), 40); + assert_eq!(result_flow_files[1].flow_file.content_len(), 40); + assert_eq!( + *result_flow_files[0].flow_file.content.borrow(), + *result_flow_files[1].flow_file.content.borrow() + ); +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs new file mode 100644 index 0000000000..1b05b030af --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs @@ -0,0 +1,303 @@ +use crate::processors::get_file::output_attributes::{ + ABSOLUTE_PATH_OUTPUT_ATTRIBUTE, FILENAME_OUTPUT_ATTRIBUTE, +}; +use crate::processors::get_file::properties::{ + BATCH_SIZE, DIRECTORY, IGNORE_HIDDEN_FILES, KEEP_SOURCE_FILE, MAX_AGE, MAX_SIZE, MIN_AGE, + MIN_SIZE, RECURSE, +}; +use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::{ + GetProperty, IoState, Logger, MinifiError, OnTriggerResult, ProcessContext, + ProcessSession, Schedule, Trigger, debug, info, trace, warn, +}; +use std::collections::VecDeque; +use std::error; +use std::fs::File; +use std::path::PathBuf; +use std::sync::Mutex; +use std::time::{Duration, Instant, SystemTime}; +use walkdir::{DirEntry, WalkDir}; + +mod properties; +mod relationships; + +#[derive(Debug)] +struct GetFileMetrics { + accepted_files: u32, + input_bytes: u64, +} + +#[derive(Debug)] +struct DirectoryListing { + paths: VecDeque, + last_polling_time: Option, +} + +impl DirectoryListing { + fn new() -> Self { + Self { + paths: VecDeque::new(), + last_polling_time: None, + } + } +} + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct GetFileRs { + recursive: bool, + keep_source_file: bool, + input_directory: PathBuf, + poll_interval: Option, + directory_listing: Mutex, + batch_size: u64, + min_size: Option, + max_size: Option, + min_age: Option, + max_age: Option, + ignore_hidden_files: bool, + metrics: Mutex, +} + +impl GetFileRs { + fn is_listing_empty(&self) -> bool { + let directory_listing = self.directory_listing.lock().unwrap(); + directory_listing.paths.is_empty() + } + + fn poll_listing(&self, batch_size: u64) -> VecDeque { + let mut directory_listings = self.directory_listing.lock().unwrap(); + + let mut res = VecDeque::new(); + for _ in 0..batch_size { + if let Some(path) = directory_listings.paths.pop_back() { + res.push_back(path); + } else { + break; + } + } + + res + } + + fn should_poll(&self) -> bool { + if self.poll_interval.is_none() { + return true; + } + let directory_listings = self.directory_listing.lock().unwrap(); + + if directory_listings.last_polling_time.is_none() { + return true; + } + Instant::now() - directory_listings.last_polling_time.unwrap() > self.poll_interval.unwrap() + } + + fn perform_listing(&self) { + let mut directory_listings = self.directory_listing.lock().unwrap(); + let mut walker = WalkDir::new(&self.input_directory); + + if !self.recursive { + walker = walker.max_depth(1); + } + + let mut files_added = 0u32; + let mut bytes_added = 0u64; + for entry in walker.into_iter().filter_map(Result::ok) { + if self.entry_matches_criteria(&entry).unwrap_or(false) { + let file_size = entry.metadata().map(|m| m.len()).unwrap_or(0); + directory_listings.paths.push_back(entry.into_path()); + files_added += 1; + bytes_added += file_size; + } + } + let mut metrics = self.metrics.lock().unwrap(); + metrics.accepted_files += files_added; + metrics.input_bytes += bytes_added; + directory_listings.last_polling_time = Some(Instant::now()); + } + + fn entry_matches_criteria(&self, dir_entry: &DirEntry) -> Result> { + let metadata = dir_entry.metadata()?; + if !metadata.is_file() { + return Ok(false); + } + let age = SystemTime::now().duration_since(metadata.modified()?)?; + let size = metadata.len(); + + if self.min_age.is_some() && age < self.min_age.unwrap() { + return Ok(false); + } + if self.max_age.is_some() && age > self.max_age.unwrap() { + return Ok(false); + } + if self.min_size.is_some() && size < self.min_size.unwrap() { + return Ok(false); + } + if self.max_size.is_some() && size > self.max_size.unwrap() { + return Ok(false); + } + + fn is_hidden(path: PathBuf) -> bool { + path.file_name() + .and_then(|f| f.to_str()) + .is_some_and(|f| f.starts_with('.')) + } + + if self.ignore_hidden_files && is_hidden(dir_entry.path().to_path_buf()) { + return Ok(false); + } + + Ok(true) + } + + fn get_single_file( + &self, + session: &mut PS, + logger: &L, + path: PathBuf, + ) -> Result<(), MinifiError> { + info!(logger, "GetFile process {:?}", &path); + let mut ff = session + .create() + .expect("Successful FlowFile creation is expected"); + + if let Some(file_name) = path.file_name().and_then(|f| f.to_str()) { + session.set_attribute(&mut ff, FILENAME_OUTPUT_ATTRIBUTE.name, file_name)?; + } else { + warn!(logger, "Couldnt get filename of {:?}", path); + } + session.set_attribute( + &mut ff, + ABSOLUTE_PATH_OUTPUT_ATTRIBUTE.name, + path.to_string_lossy().trim(), + )?; + + session.write_stream(&ff, |output_stream| { + let mut file = File::open(&path)?; + std::io::copy(&mut file, output_stream)?; + Ok(((), IoState::Ok)) + })?; + if !self.keep_source_file { + match std::fs::remove_file(&path) { + Ok(_) => {} + Err(err) => { + warn!(logger, "Failed to remove source file {:?}", err); + } + } + } + session.transfer(ff, relationships::SUCCESS.name)?; + Ok(()) + } + + fn calculate_metrics(&self) -> Vec<(String, f64)> { + let metrics = self.metrics.lock().unwrap(); + vec![ + ("accepted_files".to_string(), metrics.accepted_files as f64), + ("input_bytes".to_string(), metrics.input_bytes as f64), + ] + } +} + +impl Schedule for GetFileRs { + fn schedule(context: &P, _logger: &L) -> Result + where + Self: Sized, + { + let input_directory: PathBuf = context + .get_property(&DIRECTORY)? + .expect("Required property") + .into(); + if !input_directory.is_dir() { + return Err(MinifiError::schedule_err(format!( + "{:?} is not a valid directory", + input_directory + ))); + } + + let recursive = context + .get_bool_property(&RECURSE)? + .expect("Required property"); + + let keep_source_file = context + .get_bool_property(&KEEP_SOURCE_FILE)? + .expect("Required property"); + + let poll_interval = context.get_duration_property(&properties::POLLING_INTERVAL)?; + let min_size = context.get_size_property(&MIN_SIZE)?; + let max_size = context.get_size_property(&MAX_SIZE)?; + let min_age = context.get_duration_property(&MIN_AGE)?; + let max_age = context.get_duration_property(&MAX_AGE)?; + let batch_size = context + .get_u64_property(&BATCH_SIZE)? + .expect("required property"); + let ignore_hidden_files = context + .get_bool_property(&IGNORE_HIDDEN_FILES)? + .expect("required property"); + + Ok(GetFileRs { + recursive, + keep_source_file, + input_directory, + poll_interval, + directory_listing: Mutex::new(DirectoryListing::new()), + batch_size, + min_size, + max_size, + min_age, + max_age, + ignore_hidden_files, + metrics: Mutex::new(GetFileMetrics { + accepted_files: 0, + input_bytes: 0, + }), + }) + } +} + +impl Trigger for GetFileRs { + fn trigger( + &self, + context: &mut PC, + session: &mut PS, + logger: &L, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession, + L: Logger, + { + trace!(logger, "on_trigger: {:?}", self); + { + let is_dir_empty_before_poll = self.is_listing_empty(); + debug!( + logger, + "Listing is {} before polling directory", is_dir_empty_before_poll + ); + if is_dir_empty_before_poll && self.should_poll() { + self.perform_listing(); + } + } + { + let is_dir_empty_after_poll = self.is_listing_empty(); + debug!( + logger, + "Listing is {} after polling directory", is_dir_empty_after_poll + ); + if is_dir_empty_after_poll { + return Ok(OnTriggerResult::Yield); + } + } + + let files = self.poll_listing(self.batch_size); + for file in files { + self.get_single_file(session, logger, file)?; + } + context.report_metrics(self.calculate_metrics())?; + Ok(OnTriggerResult::Ok) + } +} + +pub(crate) mod processor_definition; + +mod output_attributes; +#[cfg(test)] +mod tests; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/output_attributes.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/output_attributes.rs new file mode 100644 index 0000000000..62b1fbc11f --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/output_attributes.rs @@ -0,0 +1,13 @@ +use minifi_native::OutputAttribute; + +pub(crate) const FILENAME_OUTPUT_ATTRIBUTE: OutputAttribute = OutputAttribute { + name: "filename", + relationships: &["success"], + description: "The filename is set to the name of the file on disk", +}; + +pub(crate) const ABSOLUTE_PATH_OUTPUT_ATTRIBUTE: OutputAttribute = OutputAttribute { + name: "absolute.path", + relationships: &["success"], + description: "The full/absolute path from where a file was picked up. The current 'path' attribute is still populated, but may be a relative path", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/processor_definition.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/processor_definition.rs new file mode 100644 index 0000000000..b3bd89146d --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/processor_definition.rs @@ -0,0 +1,30 @@ +use crate::processors::get_file::output_attributes::{ + ABSOLUTE_PATH_OUTPUT_ATTRIBUTE, FILENAME_OUTPUT_ATTRIBUTE, +}; +use crate::processors::get_file::properties::*; +use crate::processors::get_file::{GetFileRs, relationships}; +use minifi_native::{ + OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, +}; + +impl ProcessorDefinition for GetFileRs { + const DESCRIPTION: &'static str = "Creates FlowFiles from files in a directory. MiNiFi will ignore files for which it doesn't have read permissions."; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Forbidden; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = + &[ABSOLUTE_PATH_OUTPUT_ATTRIBUTE, FILENAME_OUTPUT_ATTRIBUTE]; + const RELATIONSHIPS: &'static [Relationship] = &[relationships::SUCCESS]; + const PROPERTIES: &'static [Property] = &[ + DIRECTORY, + POLLING_INTERVAL, + RECURSE, + KEEP_SOURCE_FILE, + MIN_AGE, + MAX_AGE, + MIN_SIZE, + MAX_SIZE, + IGNORE_HIDDEN_FILES, + BATCH_SIZE, + ]; +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/properties.rs new file mode 100644 index 0000000000..44f81fa717 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/properties.rs @@ -0,0 +1,121 @@ +use minifi_native::{Property, StandardPropertyValidator}; + +pub(crate) const DIRECTORY: Property = Property { + name: "Input Directory", + description: "The input directory from which to pull files", + is_required: true, + is_sensitive: false, + supports_expr_lang: true, + default_value: None, + validator: StandardPropertyValidator::NonBlankValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const RECURSE: Property = Property { + name: "Recurse Subdirectories", + description: "Indicates whether or not to pull files from subdirectories", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("true"), + validator: StandardPropertyValidator::BoolValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const KEEP_SOURCE_FILE: Property = Property { + name: "Keep Source File", + description: "If true, the file is not deleted after it has been copied to the Content Repository", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("false"), + validator: StandardPropertyValidator::BoolValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const MIN_AGE: Property = Property { + name: "Minimum File Age", + description: "The minimum age that a file must be in order to be pulled; any file younger than this amount of time (according to last modification date) will be ignored", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::TimePeriodValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const MAX_AGE: Property = Property { + name: "Maximum File Age", + description: "The maximum age that a file must be in order to be pulled; any file older than this amount of time (according to last modification date) will be ignored", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::TimePeriodValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const MIN_SIZE: Property = Property { + name: "Minimum File Size", + description: "The minimum size that a file can be in order to be pulled", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::DataSizeValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const MAX_SIZE: Property = Property { + name: "Maximum File Size", + description: "The maximum size that a file can be in order to be pulled", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::DataSizeValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const IGNORE_HIDDEN_FILES: Property = Property { + name: "Ignore Hidden Files", + description: "Indicates whether or not hidden files should be ignored", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("true"), + validator: StandardPropertyValidator::BoolValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const POLLING_INTERVAL: Property = Property { + name: "Polling Interval", + description: "Indicates how long to wait before performing a directory listing", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::TimePeriodValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const BATCH_SIZE: Property = Property { + name: "Batch Size", + description: "The maximum number of files to pull in each iteration", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("10"), + validator: StandardPropertyValidator::U64Validator, + allowed_values: &[], + allowed_type: "", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/relationships.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/relationships.rs new file mode 100644 index 0000000000..5f3c286715 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/relationships.rs @@ -0,0 +1,6 @@ +use minifi_native::Relationship; + +pub(crate) const SUCCESS: Relationship = Relationship { + name: "success", + description: "FlowFiles are transferred here after logging", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/tests.rs new file mode 100644 index 0000000000..143fadaf7f --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/tests.rs @@ -0,0 +1,172 @@ +use super::*; +use crate::processors::get_file::relationships::SUCCESS; +use filetime::FileTime; +use minifi_native::{MockLogger, MockProcessContext, MockProcessSession}; +use tempfile::TempDir; + +#[test] +fn schedule_fails_without_input_dir() { + assert!(matches!( + GetFileRs::schedule(&MockProcessContext::new(), &MockLogger::new()) + .err() + .unwrap(), + MinifiError::MissingRequiredProperty(_) + )); +} + +#[test] +fn schedule_fails_with_invalid_input_dir() { + let mut context = MockProcessContext::new(); + context.properties.insert( + "Input Directory".to_string(), + "/invalid_directory".to_string(), + ); + assert!(matches!( + GetFileRs::schedule(&context, &MockLogger::new()), + Err(MinifiError::ScheduleError(_)) + )); +} + +#[test] +fn simple_get_file_test() { + let temp_dir = tempfile::tempdir().expect("temp dir is required for testing GetFile"); + let file_path = temp_dir.path().join("input_file"); + std::fs::write(&file_path, "test").unwrap(); + + let mut context = MockProcessContext::new(); + context.properties.insert( + "Input Directory".to_string(), + temp_dir.path().to_str().unwrap().to_string(), + ); + + let get_file = GetFileRs::schedule(&context, &MockLogger::new()).unwrap(); + + let mut session = MockProcessSession::new(); + get_file + .trigger(&mut context, &mut session, &MockLogger::new()) + .expect("Should succeed"); + assert_eq!(session.num_of_transferred_flow_files(), 1); +} + +fn make_file(temp_dir: &TempDir, file_name: &str, size: usize, age: Duration) { + let path = temp_dir.path().join(file_name); + std::fs::write(&path, "a".repeat(size)).unwrap(); + let file_time = FileTime::from_system_time(SystemTime::now() - age); + filetime::set_file_mtime(path, file_time).expect("Cannot set file time"); +} + +fn create_test_directory() -> TempDir { + let temp_dir = tempfile::tempdir().expect("temp dir is required for testing GetFile"); + make_file(&temp_dir, "small_new", 10, Duration::from_secs(10)); + make_file(&temp_dir, "small_old", 11, Duration::from_secs(3600)); + + make_file(&temp_dir, "large_new", 1000, Duration::from_secs(0)); + make_file(&temp_dir, "large_old", 2000, Duration::from_secs(3600)); + make_file(&temp_dir, ".small_hidden", 10, Duration::from_secs(0)); + temp_dir +} + +#[test] +fn complex_dir_without_filters() { + let test_directory = create_test_directory(); + + let mut context = MockProcessContext::new(); + context.properties.insert( + "Input Directory".to_string(), + test_directory.path().to_str().unwrap().to_string(), + ); + context + .properties + .insert("Batch Size".to_string(), "10".to_string()); + + let mut session = MockProcessSession::new(); + let get_file = GetFileRs::schedule(&context, &MockLogger::new()).unwrap(); + get_file + .trigger(&mut context, &mut session, &MockLogger::new()) + .expect("Should succeed"); + assert_eq!(session.num_of_transferred_flow_files(), 4); +} + +fn test_complex_dir_with_filter( + property_name: &str, + property_vale: &str, + expected_filename_part: &str, +) { + let test_directory = create_test_directory(); + + let mut context = MockProcessContext::new(); + context.properties.insert( + DIRECTORY.name.to_string(), + test_directory.path().to_str().unwrap().to_string(), + ); + context + .properties + .insert(BATCH_SIZE.name.to_string(), "10".to_string()); + + context + .properties + .insert(property_name.to_string(), property_vale.to_string()); + + let mut session = MockProcessSession::new(); + let get_file = GetFileRs::schedule(&context, &MockLogger::new()).unwrap(); + get_file + .trigger(&mut context, &mut session, &MockLogger::new()) + .expect("Should succeed"); + assert_eq!(session.num_of_transferred_flow_files(), 2); + let transferred_flow_files = session.transferred_flow_files.borrow(); + assert!(transferred_flow_files.iter().all(|transfer| { + transfer.relationship == SUCCESS.name + && transfer + .flow_file + .attributes + .get("filename") + .and_then(|filename| Some(filename.contains(expected_filename_part))) + .unwrap_or(false) + })); + let sum_file_len = transferred_flow_files + .iter() + .fold(0, |acc, transfer| acc + transfer.flow_file.content_len()); + + let metrics = get_file.calculate_metrics(); + assert_eq!(metrics.len(), 2); + assert_eq!(metrics[0].0, "accepted_files".to_string()); + assert_eq!(metrics[0].1, 2.0); + assert_eq!(metrics[1].0, "input_bytes".to_string()); + assert_eq!(metrics[1].1, sum_file_len as f64); +} + +#[test] +fn complex_dir_with_filters() { + test_complex_dir_with_filter(MIN_AGE.name, "5 min", "old"); + test_complex_dir_with_filter(MAX_AGE.name, "5 min", "new"); + test_complex_dir_with_filter(MIN_SIZE.name, "50 B", "large"); + test_complex_dir_with_filter(MAX_SIZE.name, "50 B", "small"); +} + +#[test] +fn test_hidden_files_and_batch_size() { + let temp_dir = tempfile::tempdir().expect("temp dir is required for testing GetFile"); + make_file(&temp_dir, ".one", 10, Duration::from_secs(0)); + make_file(&temp_dir, ".two", 10, Duration::from_secs(0)); + make_file(&temp_dir, ".three", 10, Duration::from_secs(0)); + + let mut context = MockProcessContext::new(); + context.properties.insert( + DIRECTORY.name.to_string(), + temp_dir.path().to_str().unwrap().to_string(), + ); + context + .properties + .insert(BATCH_SIZE.name.to_string(), "2".to_string()); + + context + .properties + .insert(IGNORE_HIDDEN_FILES.name.to_string(), "false".to_string()); + + let mut session = MockProcessSession::new(); + let get_file = GetFileRs::schedule(&context, &MockLogger::new()).unwrap(); + get_file + .trigger(&mut context, &mut session, &MockLogger::new()) + .expect("Should succeed"); + assert_eq!(session.num_of_transferred_flow_files(), 2); +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs new file mode 100644 index 0000000000..ad65040ff1 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs @@ -0,0 +1,103 @@ +mod properties; +mod relationships; + +use crate::controller_services::lorem_ipsum_controller_service::LoremIpsumControllerService; +use crate::processors::kamikaze_processor::properties::NOT_REGISTERED_PROPERTY; +use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::{ + GetProperty, Logger, MinifiError, OnTriggerResult, ProcessContext, + ProcessSession, Schedule, Trigger, +}; +use strum_macros::{Display, EnumString, IntoStaticStr, VariantNames}; + +#[derive(Debug, Clone, Copy, PartialEq, Display, EnumString, VariantNames, IntoStaticStr)] +#[strum(serialize_all = "PascalCase", const_into_str)] +enum KamikazeBehaviour { + ReturnErr, + ReturnOk, + GetNotRegisteredProperty, + GetInvalidControllerService, + Panic, +} + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct KamikazeProcessorRs { + on_trigger_behaviour: KamikazeBehaviour, +} + +impl Schedule for KamikazeProcessorRs { + fn schedule(context: &P, _logger: &L) -> Result + where + Self: Sized, + { + let on_trigger_behaviour = context + .get_property(&properties::ON_TRIGGER_BEHAVIOUR)? + .expect("required property") + .parse::()?; + + let on_schedule_behaviour = context + .get_property(&properties::ON_SCHEDULE_BEHAVIOUR)? + .expect("required property") + .parse::()?; + + match on_schedule_behaviour { + KamikazeBehaviour::ReturnErr => Err(MinifiError::schedule_err( + "it was designed to fail during schedule", + )), + KamikazeBehaviour::ReturnOk => Ok(KamikazeProcessorRs { + on_trigger_behaviour, + }), + KamikazeBehaviour::GetNotRegisteredProperty => { + let _ = context.get_property(&NOT_REGISTERED_PROPERTY)?; + Ok(KamikazeProcessorRs { + on_trigger_behaviour, + }) + } + KamikazeBehaviour::Panic => { + panic!("KamikazeProcessor::on_schedule panic") + } + KamikazeBehaviour::GetInvalidControllerService => { + unimplemented!("KamikazeProcessor::get_invalid_controller_service"); + } + } + } +} + +impl Trigger for KamikazeProcessorRs { + fn trigger( + &self, + context: &mut PC, + _session: &mut PS, + _logger: &L, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession, + L: Logger, + { + match self.on_trigger_behaviour { + KamikazeBehaviour::ReturnErr => Err(MinifiError::trigger_err( + "it was designed to fail in trigger", + )), + KamikazeBehaviour::ReturnOk => Ok(OnTriggerResult::Ok), + KamikazeBehaviour::Panic => { + panic!("KamikazeProcessor::on_trigger panic") + } + KamikazeBehaviour::GetNotRegisteredProperty => { + let _ = context.get_property(&NOT_REGISTERED_PROPERTY, None)?; + Ok(OnTriggerResult::Ok) + } + KamikazeBehaviour::GetInvalidControllerService => { + let _ = context.get_controller_service::( + &NOT_REGISTERED_PROPERTY, + )?; + Ok(OnTriggerResult::Ok) + } + } + } +} + +pub(crate) mod processor_definition; + +#[cfg(test)] +mod tests; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/processor_definition.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/processor_definition.rs new file mode 100644 index 0000000000..1cbc76634c --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/processor_definition.rs @@ -0,0 +1,17 @@ +use super::*; +use minifi_native::{ + OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, +}; + +impl ProcessorDefinition for KamikazeProcessorRs { + const DESCRIPTION: &'static str = "This processor can fail or panic in on_trigger and on_schedule calls based on configuration. Only for testing purposes."; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Allowed; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[]; + const RELATIONSHIPS: &'static [Relationship] = &[relationships::SUCCESS]; + const PROPERTIES: &'static [Property] = &[ + properties::ON_SCHEDULE_BEHAVIOUR, + properties::ON_TRIGGER_BEHAVIOUR, + ]; +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/properties.rs new file mode 100644 index 0000000000..0f2e550e7e --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/properties.rs @@ -0,0 +1,39 @@ +use crate::processors::kamikaze_processor::KamikazeBehaviour; +use minifi_native::{Property, StandardPropertyValidator}; +use strum::VariantNames; + +pub(crate) const ON_SCHEDULE_BEHAVIOUR: Property = Property { + name: "On Schedule Behaviour", + description: "What to do during the on_schedule method", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some(KamikazeBehaviour::ReturnOk.into_str()), + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &KamikazeBehaviour::VARIANTS, + allowed_type: "", +}; + +pub(crate) const ON_TRIGGER_BEHAVIOUR: Property = Property { + name: "On Trigger Behaviour", + description: "What to do during the on_trigger method", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some(KamikazeBehaviour::ReturnOk.into_str()), + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &KamikazeBehaviour::VARIANTS, + allowed_type: "", +}; + +pub(crate) const NOT_REGISTERED_PROPERTY: Property = Property { + name: "Kamikaze Processor Property", + description: "Property purposely left out of Processor description", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &[], + allowed_type: "", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/relationships.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/relationships.rs new file mode 100644 index 0000000000..1bdfca63f5 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/relationships.rs @@ -0,0 +1,6 @@ +use minifi_native::Relationship; + +pub(crate) const SUCCESS: Relationship = Relationship { + name: "success", + description: "success relationship", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs new file mode 100644 index 0000000000..76acbb3e5f --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs @@ -0,0 +1,83 @@ +use super::*; +use crate::processors::kamikaze_processor::properties::{ + ON_SCHEDULE_BEHAVIOUR, ON_TRIGGER_BEHAVIOUR, +}; +use minifi_native::MinifiError::{ScheduleError, TriggerError}; +use minifi_native::{MockLogger, MockProcessContext, MockProcessSession}; +use std::panic::AssertUnwindSafe; + +#[test] +fn on_schedule_ok() { + let context = MockProcessContext::new(); + let processor = KamikazeProcessorRs::schedule(&context, &MockLogger::new()); + assert!(processor.is_ok()); +} + +#[test] +fn on_schedule_err() { + let mut context = MockProcessContext::new(); + context.properties.insert( + ON_SCHEDULE_BEHAVIOUR.name.to_string(), + "ReturnErr".to_string(), + ); + let processor = KamikazeProcessorRs::schedule(&context, &MockLogger::new()); + assert!(matches!(processor, Err(ScheduleError(_)))); +} + +#[test] +fn on_schedule_panic() { + let mut context = MockProcessContext::new(); + context + .properties + .insert(ON_SCHEDULE_BEHAVIOUR.name.to_string(), "Panic".to_string()); + + let result = std::panic::catch_unwind(AssertUnwindSafe(|| { + KamikazeProcessorRs::schedule(&context, &MockLogger::new()) + })); + assert!(result.is_err()); +} + +#[test] +fn on_trigger_ok() { + let mut context = MockProcessContext::new(); + let processor = KamikazeProcessorRs::schedule(&context, &MockLogger::new()).unwrap(); + + let mut session = MockProcessSession::new(); + assert_eq!( + processor + .trigger(&mut context, &mut session, &MockLogger::new()) + .expect("Should trigger successfully"), + OnTriggerResult::Ok + ); +} + +#[test] +fn on_trigger_err() { + let mut context = MockProcessContext::new(); + context.properties.insert( + ON_TRIGGER_BEHAVIOUR.name.to_string(), + "ReturnErr".to_string(), + ); + let processor = KamikazeProcessorRs::schedule(&context, &MockLogger::new()).unwrap(); + + let mut session = MockProcessSession::new(); + assert!(matches!( + processor.trigger(&mut context, &mut session, &MockLogger::new()), + Err(TriggerError(_)) + )); +} + +#[test] +fn on_trigger_panic() { + let mut context = MockProcessContext::new(); + context + .properties + .insert(ON_TRIGGER_BEHAVIOUR.name.to_string(), "Panic".to_string()); + let processor = KamikazeProcessorRs::schedule(&context, &MockLogger::new()).unwrap(); + + let mut session = MockProcessSession::new(); + let result = std::panic::catch_unwind(AssertUnwindSafe(|| { + processor.trigger(&mut context, &mut session, &MockLogger::new()) + })); + assert!(result.is_err()); +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs new file mode 100644 index 0000000000..b49f7dd442 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs @@ -0,0 +1,160 @@ +use crate::processors::log_attribute::properties::{FLOW_FILES_TO_LOG, LOG_LEVEL, LOG_PAYLOAD}; +use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::{ + GetProperty, LogLevel, Logger, MinifiError, OnTriggerResult, ProcessContext, ProcessSession, + Property, Schedule, Trigger, debug, log, trace, +}; + +mod properties; +mod relationships; + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct LogAttributeRs { + log_level: LogLevel, + attributes_to_log: Option>, + attributes_to_ignore: Option>, + log_payload: bool, + flow_files_to_log: usize, + dash_line: String, + hex_encode_payload: bool, +} + +impl LogAttributeRs { + fn generate_log_message(&self, session: &mut PS, flow_file: &PS::FlowFile) -> String + where + PS: ProcessSession, + { + let mut log_msg = String::with_capacity(1024); + log_msg.push_str("Logging for flow file\n"); + log_msg.push_str(self.dash_line.as_str()); + + log_msg.push_str("\nFlowFile Attributes Map Content"); + session.on_attributes(flow_file, |key, value| { + if let Some(attributes_to_ignore) = &self.attributes_to_ignore { + if attributes_to_ignore.iter().any(|ign| ign == key) { + return; + } + } + if let Some(attributes_to_log) = &self.attributes_to_log { + if !attributes_to_log.iter().any(|ign| ign == key) { + return; + } + } + log_msg.push_str(format!("\nkey:{} value:{}", &key, &value).as_str()); + }); + if self.log_payload { + log_msg.push_str("\nPayload:\n"); + if let Some(flow_file_payload) = session.read(flow_file) { + if self.hex_encode_payload { + log_msg.push_str(&hex::encode(flow_file_payload)); + } else { + log_msg.push_str( + String::from_utf8(flow_file_payload) + .unwrap_or(String::new()) + .as_str(), + ); + } + } + } + log_msg.push_str("\n"); + log_msg.push_str(self.dash_line.as_str()); + log_msg + } +} + +impl Trigger for LogAttributeRs { + fn trigger( + &self, + _context: &mut PC, + session: &mut PS, + logger: &L, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession, + L: Logger, + { + trace!( + logger, + "enter log attribute, attempting to retrieve {} flow files", self.flow_files_to_log + ); + let max_flow_files_to_process = if self.flow_files_to_log == 0 { + usize::MAX + } else { + self.flow_files_to_log + }; + let mut flow_files_processed = 0usize; + for _ in 0..max_flow_files_to_process { + if let Some(mut flow_file) = session.get() { + let log_msg = self.generate_log_message(session, &mut flow_file); + log!(logger, self.log_level, "{}", log_msg); + session.transfer(flow_file, relationships::SUCCESS.name)?; + flow_files_processed += 1; + } else { + break; + } + } + debug!(logger, "Logged {} flow files", flow_files_processed); + + Ok(OnTriggerResult::Ok) + } +} + +impl Schedule for LogAttributeRs { + fn schedule(context: &P, _logger: &L) -> Result + where + Self: Sized, + { + let log_level = context + .get_property(&LOG_LEVEL)? + .expect("required property") + .parse::()?; + + let log_payload = context + .get_bool_property(&LOG_PAYLOAD)? + .expect("required property"); + + let flow_files_to_log = context + .get_property(&FLOW_FILES_TO_LOG)? + .expect("required property") + .parse::()?; + + fn get_csv_property( + context: &P, + property: &Property, + ) -> Result>, MinifiError> { + Ok(context + .get_property(property)? + .and_then(|s| Some(s.split(",").map(|s| s.to_string()).collect::>()))) + } + + let attributes_to_log = get_csv_property(context, &properties::ATTRIBUTES_TO_LOG)?; + let attributes_to_ignore = get_csv_property(context, &properties::ATTRIBUTES_TO_IGNORE)?; + + let dash_line = format!( + "{:-^50}", + context + .get_property(&properties::LOG_PREFIX)? + .unwrap_or(String::new()) + ); + + let hex_encode_payload = context + .get_bool_property(&properties::HEX_ENCODE_PAYLOAD)? + .expect("required property"); + + Ok(LogAttributeRs { + log_level, + attributes_to_log, + attributes_to_ignore, + log_payload, + flow_files_to_log, + dash_line, + hex_encode_payload, + }) + } +} + +pub(crate) mod processor_definition; + +#[cfg(test)] +mod tests; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/processor_definition.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/processor_definition.rs new file mode 100644 index 0000000000..0d4c08fdb1 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/processor_definition.rs @@ -0,0 +1,24 @@ +use crate::processors::log_attribute::properties::*; +use crate::processors::log_attribute::{LogAttributeRs, relationships}; +use minifi_native::{ + OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, +}; + +impl ProcessorDefinition for LogAttributeRs { + const DESCRIPTION: &'static str = + "Logs attributes of flow files in the MiNiFi application log."; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[]; + const RELATIONSHIPS: &'static [Relationship] = &[relationships::SUCCESS]; + const PROPERTIES: &'static [Property] = &[ + LOG_LEVEL, + ATTRIBUTES_TO_LOG, + ATTRIBUTES_TO_IGNORE, + LOG_PAYLOAD, + LOG_PREFIX, + FLOW_FILES_TO_LOG, + HEX_ENCODE_PAYLOAD, + ]; +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/properties.rs new file mode 100644 index 0000000000..5ec30f72ea --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/properties.rs @@ -0,0 +1,86 @@ +use minifi_native::{LogLevel, Property, StandardPropertyValidator}; +use strum::VariantNames; + +pub(crate) const LOG_LEVEL: Property = Property { + name: "Log Level", + description: "The Log Level to use when logging the Attributes", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("Info"), // todo! it would be nicer to come from enum value but wasnt able to use into_const_str from another crate + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &LogLevel::VARIANTS, + allowed_type: "", +}; + +pub(crate) const ATTRIBUTES_TO_LOG: Property = Property { + name: "Attributes to Log", + description: "A comma-separated list of Attributes to Log. If not specified, all attributes will be logged.", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const ATTRIBUTES_TO_IGNORE: Property = Property { + name: "Attributes to Ignore", + description: "A comma-separated list of Attributes to ignore. If not specified, no attributes will be ignored.", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const LOG_PAYLOAD: Property = Property { + name: "Log Payload", + description: "If true, the FlowFile's payload will be logged, in addition to its attributes. Otherwise, just the Attributes will be logged.", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("false"), + validator: StandardPropertyValidator::BoolValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const LOG_PREFIX: Property = Property { + name: "Log Prefix", + description: "Log prefix appended to the log lines. It helps to distinguish the output of multiple LogAttribute processors.", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const FLOW_FILES_TO_LOG: Property = Property { + name: "FlowFiles To Log", + description: "Number of flow files to log. If set to zero all flow files will be logged. Please note that this may block other threads from running if not used judiciously.", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("1"), + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const HEX_ENCODE_PAYLOAD: Property = Property { + name: "Hexencode Payload", + description: "If true, the FlowFile's payload will be logged in a hexencoded format", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("false"), + validator: StandardPropertyValidator::BoolValidator, + allowed_values: &[], + allowed_type: "", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/relationships.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/relationships.rs new file mode 100644 index 0000000000..5f3c286715 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/relationships.rs @@ -0,0 +1,6 @@ +use minifi_native::Relationship; + +pub(crate) const SUCCESS: Relationship = Relationship { + name: "success", + description: "FlowFiles are transferred here after logging", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/tests.rs new file mode 100644 index 0000000000..c87070b9f3 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/tests.rs @@ -0,0 +1,114 @@ +use super::*; +use minifi_native::{ + ComponentIdentifier, MockFlowFile, MockLogger, MockProcessContext, MockProcessSession, +}; + +#[test] +fn test_component_id() { + assert_eq!( + LogAttributeRs::CLASS_NAME, + "minifi_rs_playground::processors::log_attribute::LogAttributeRs" + ); + assert_eq!(LogAttributeRs::GROUP_NAME, "minifi_rs_playground"); + assert_eq!(LogAttributeRs::VERSION, "0.1.0"); +} +#[test] +fn schedule_succeeds_with_default_values() { + assert!(LogAttributeRs::schedule(&MockProcessContext::new(), &MockLogger::new()).is_ok()); +} + +fn tester( + input_flow_files: Vec, + log_level: LogLevel, + properties: Box<[(&str, &str)]>, + expected_log_msg: String, +) { + let logger = MockLogger::new(); + let mut context = MockProcessContext::new(); + for (k, v) in properties { + context.properties.insert(k.to_string(), v.to_string()); + } + + let processor = LogAttributeRs::schedule(&context, &logger).unwrap(); + + let mut session = MockProcessSession::new(); + for flow_file in input_flow_files { + session.input_flow_files.push(flow_file); + } + processor + .trigger(&mut context, &mut session, &logger) + .expect("The on_trigger should succeed"); + + let logs = logger.logs.lock().unwrap(); + assert_eq!(logs[1], (log_level, expected_log_msg)); +} + +#[test] +fn warn_single_log_payload() { + let mut flow_file = MockFlowFile::with_content("Lorem ipsum dolor sit amet, consectetur adipiscing elit. Integer facilisis diam sit amet nisl interdum, vitae interdum arcu viverra. Nam placerat mi in erat pellentesque, at ultrices orci faucibus. Cras sollicitudin iaculis posuere. Sed tempus, dolor nec lacinia suscipit, tellus odio venenatis odio, nec sollicitudin dolor augue non urna. Aliquam tincidunt viverra ipsum eget hendrerit. Suspendisse varius, augue vel fermentum varius, velit elit euismod lacus, a placerat purus est a lacus. Aenean nibh neque, consectetur hendrerit egestas vitae, commodo non quam. Nullam luctus tempor ante, sed tempus quam imperdiet in. Maecenas gravida erat orci, in consequat urna pretium nec. In sodales iaculis magna at vehicula.".as_bytes()); + flow_file + .attributes + .insert(String::from("apple"), String::from("apfel")); + let vec = vec![flow_file]; + + let expected = + "Logging for flow file +-------------------------------------------------- +FlowFile Attributes Map Content +key:apple value:apfel +Payload: +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Integer facilisis diam sit amet nisl interdum, vitae interdum arcu viverra. Nam placerat mi in erat pellentesque, at ultrices orci faucibus. Cras sollicitudin iaculis posuere. Sed tempus, dolor nec lacinia suscipit, tellus odio venenatis odio, nec sollicitudin dolor augue non urna. Aliquam tincidunt viverra ipsum eget hendrerit. Suspendisse varius, augue vel fermentum varius, velit elit euismod lacus, a placerat purus est a lacus. Aenean nibh neque, consectetur hendrerit egestas vitae, commodo non quam. Nullam luctus tempor ante, sed tempus quam imperdiet in. Maecenas gravida erat orci, in consequat urna pretium nec. In sodales iaculis magna at vehicula. +--------------------------------------------------".to_string(); + let properties_set = [ + ("Log Payload", "true"), + ("Hexencode Payload", "false"), + ("Log Level", "Warn"), + ]; + tester(vec, LogLevel::Warn, Box::new(properties_set), expected); +} + +#[test] +fn critical_single_hexencode_payload() { + let mut flow_file = MockFlowFile::with_content("Lorem ipsum dolor sit amet, consectetur adipiscing elit. Integer facilisis diam sit amet nisl interdum, vitae interdum arcu viverra. Nam placerat mi in erat pellentesque, at ultrices orci faucibus. Cras sollicitudin iaculis posuere. Sed tempus, dolor nec lacinia suscipit, tellus odio venenatis odio, nec sollicitudin dolor augue non urna. Aliquam tincidunt viverra ipsum eget hendrerit. Suspendisse varius, augue vel fermentum varius, velit elit euismod lacus, a placerat purus est a lacus. Aenean nibh neque, consectetur hendrerit egestas vitae, commodo non quam. Nullam luctus tempor ante, sed tempus quam imperdiet in. Maecenas gravida erat orci, in consequat urna pretium nec. In sodales iaculis magna at vehicula.".as_bytes()); + flow_file + .attributes + .insert(String::from("apple"), String::from("apfel")); + let vec = vec![flow_file]; + + let expected = + "Logging for flow file +-------------------------------------------------- +FlowFile Attributes Map Content +key:apple value:apfel +Payload: +4c6f72656d20697073756d20646f6c6f722073697420616d65742c20636f6e73656374657475722061646970697363696e6720656c69742e20496e746567657220666163696c69736973206469616d2073697420616d6574206e69736c20696e74657264756d2c20766974616520696e74657264756d206172637520766976657272612e204e616d20706c616365726174206d6920696e20657261742070656c6c656e7465737175652c20617420756c747269636573206f7263692066617563696275732e204372617320736f6c6c696369747564696e20696163756c697320706f73756572652e205365642074656d7075732c20646f6c6f72206e6563206c6163696e69612073757363697069742c2074656c6c7573206f64696f2076656e656e61746973206f64696f2c206e656320736f6c6c696369747564696e20646f6c6f72206175677565206e6f6e2075726e612e20416c697175616d2074696e636964756e74207669766572726120697073756d20656765742068656e6472657269742e2053757370656e6469737365207661726975732c2061756775652076656c206665726d656e74756d207661726975732c2076656c697420656c697420657569736d6f64206c616375732c206120706c616365726174207075727573206573742061206c616375732e2041656e65616e206e696268206e657175652c20636f6e73656374657475722068656e64726572697420656765737461732076697461652c20636f6d6d6f646f206e6f6e207175616d2e204e756c6c616d206c75637475732074656d706f7220616e74652c207365642074656d707573207175616d20696d7065726469657420696e2e204d616563656e617320677261766964612065726174206f7263692c20696e20636f6e7365717561742075726e61207072657469756d206e65632e20496e20736f64616c657320696163756c6973206d61676e61206174207665686963756c612e +--------------------------------------------------".to_string(); + let properties_set = [ + ("Log Payload", "true"), + ("Hexencode Payload", "true"), + ("Log Level", "Critical"), + ]; + tester(vec, LogLevel::Critical, Box::new(properties_set), expected); +} + +#[test] +fn default_level_multiple_attributes() { + let mut flow_file = MockFlowFile::with_content("Lorem ipsum dolor sit amet, consectetur adipiscing elit. Integer facilisis diam sit amet nisl interdum, vitae interdum arcu viverra. Nam placerat mi in erat pellentesque, at ultrices orci faucibus. Cras sollicitudin iaculis posuere. Sed tempus, dolor nec lacinia suscipit, tellus odio venenatis odio, nec sollicitudin dolor augue non urna. Aliquam tincidunt viverra ipsum eget hendrerit. Suspendisse varius, augue vel fermentum varius, velit elit euismod lacus, a placerat purus est a lacus. Aenean nibh neque, consectetur hendrerit egestas vitae, commodo non quam. Nullam luctus tempor ante, sed tempus quam imperdiet in. Maecenas gravida erat orci, in consequat urna pretium nec. In sodales iaculis magna at vehicula.".as_bytes()); + flow_file + .attributes + .insert(String::from("apple"), String::from("apfel")); + flow_file + .attributes + .insert(String::from("pear"), String::from("birne")); + let vec = vec![flow_file]; + + let expected = "Logging for flow file +-------------------------------------------------- +FlowFile Attributes Map Content +key:apple value:apfel +key:pear value:birne +--------------------------------------------------" + .to_string(); + let properties_set = [("Log Payload", "false")]; + tester(vec, LogLevel::Info, Box::new(properties_set), expected); +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user.rs new file mode 100644 index 0000000000..50e8d9c60e --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user.rs @@ -0,0 +1,75 @@ +mod properties; + +use crate::controller_services::lorem_ipsum_controller_service::LoremIpsumControllerService; +use crate::processors::lorem_ipsum_cs_user::properties::CONTROLLER_SERVICE; +use crate::processors::lorem_ipsum_cs_user::relationships::SUCCESS; +use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::{ + Content, FlowFileSource, GeneratedFlowFile, GetControllerService, GetProperty, Logger, + MinifiError, Schedule, trace, +}; +use std::collections::HashMap; +use strum_macros::{Display, EnumString, IntoStaticStr, VariantNames}; + +#[derive(Debug, Clone, Copy, PartialEq, Display, EnumString, VariantNames, IntoStaticStr)] +#[strum(serialize_all = "PascalCase", const_into_str)] +enum WriteMethod { + Buffer, + Stream, +} + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct LoremIpsumCSUser { + write_method: WriteMethod, +} + +impl Schedule for LoremIpsumCSUser { + fn schedule(context: &P, _logger: &L) -> Result + where + Self: Sized, + { + let write_method = context + .get_property(&properties::WRITE_METHOD)? + .expect("required property") + .parse::()?; + Ok(Self { write_method }) + } +} + +impl FlowFileSource for LoremIpsumCSUser { + fn generate<'a, Context: GetProperty + GetControllerService, LoggerImpl: Logger>( + &self, + context: &'a mut Context, + logger: &LoggerImpl, + ) -> Result>, MinifiError> { + trace!(logger, "generate call {:?}", self); + let controller_service = context + .get_controller_service::(&CONTROLLER_SERVICE)? + .ok_or(MinifiError::missing_required_property( + "A valid usable controller service is required", + ))?; + match self.write_method { + WriteMethod::Buffer => { + let generated_flow_file = GeneratedFlowFile::new( + &SUCCESS, + Some(Content::from(controller_service.data.clone())), + HashMap::new(), + ); + Ok(vec![generated_flow_file]) + } + WriteMethod::Stream => { + let reader = controller_service.data.as_bytes(); + let content = Content::Stream(Box::new(reader)); + let generated_flow_file = + GeneratedFlowFile::new(&SUCCESS, Some(content), HashMap::new()); + Ok(vec![generated_flow_file]) + } + } + } +} + +pub(crate) mod processor_definition; + +mod relationships; +#[cfg(test)] +mod tests; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/processor_definition.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/processor_definition.rs new file mode 100644 index 0000000000..2ba155a80f --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/processor_definition.rs @@ -0,0 +1,16 @@ +use super::LoremIpsumCSUser; +use super::properties::*; +use crate::processors::lorem_ipsum_cs_user::relationships::SUCCESS; +use minifi_native::{ + OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, +}; + +impl ProcessorDefinition for LoremIpsumCSUser { + const DESCRIPTION: &'static str = "Processor to test Controller Service API"; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Forbidden; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[]; + const RELATIONSHIPS: &'static [Relationship] = &[SUCCESS]; + const PROPERTIES: &'static [Property] = &[CONTROLLER_SERVICE, WRITE_METHOD]; +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/properties.rs new file mode 100644 index 0000000000..c18e1a7d01 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/properties.rs @@ -0,0 +1,28 @@ +use crate::controller_services::lorem_ipsum_controller_service::LoremIpsumControllerService; +use minifi_native::ComponentIdentifier; +use minifi_native::{Property, StandardPropertyValidator}; +use strum::VariantNames; + +pub(crate) const CONTROLLER_SERVICE: Property = Property { + name: "Lorem Ipsum Controller Service", + description: "Name of the lorem ipsum controller service", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &[], + allowed_type: LoremIpsumControllerService::CLASS_NAME, +}; + +pub(crate) const WRITE_METHOD: Property = Property { + name: "Write Method", + description: "Which API to test", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some(super::WriteMethod::Buffer.into_str()), + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: super::WriteMethod::VARIANTS, + allowed_type: "", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/relationships.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/relationships.rs new file mode 100644 index 0000000000..97d0d98286 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/relationships.rs @@ -0,0 +1,6 @@ +use minifi_native::Relationship; + +pub(crate) const SUCCESS: Relationship = Relationship { + name: "success", + description: "All flowfile are routed here", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/tests.rs new file mode 100644 index 0000000000..e1eb6b54dc --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/tests.rs @@ -0,0 +1,19 @@ +use crate::processors::lorem_ipsum_cs_user::LoremIpsumCSUser; +use minifi_native::{ComponentIdentifier, MockLogger, MockProcessContext, Schedule}; + +#[test] +fn test_ids() { + assert_eq!( + LoremIpsumCSUser::CLASS_NAME, + "minifi_rs_playground::processors::lorem_ipsum_cs_user::LoremIpsumCSUser" + ); + assert_eq!(LoremIpsumCSUser::GROUP_NAME, "minifi_rs_playground"); + assert_eq!(LoremIpsumCSUser::VERSION, "0.1.0"); +} + +#[test] +fn schedules_with_controller() { + let context = MockProcessContext::new(); + let schedule_result = LoremIpsumCSUser::schedule(&context, &MockLogger::new()); + assert!(schedule_result.is_ok()); +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/mod.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/mod.rs new file mode 100644 index 0000000000..0a6e3c7271 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod asciify_german; +pub(crate) mod count_actual_logging; +pub(crate) mod duplicate_text; +pub(crate) mod generate_flow_file; +pub(crate) mod get_file; +pub(crate) mod kamikaze_processor; +pub(crate) mod log_attribute; +pub(crate) mod lorem_ipsum_cs_user; +pub(crate) mod put_file; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs new file mode 100644 index 0000000000..973e790e32 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs @@ -0,0 +1,239 @@ +use crate::processors::put_file::relationships::{FAILURE, SUCCESS}; +use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::{ + FlowFileTransform, GetAttribute, GetControllerService, GetProperty, InputStream, Logger, + MinifiError, Schedule, TransformedFlowFile, trace, warn, +}; +use std::path::{Path, PathBuf}; +use strum_macros::{Display, EnumString, IntoStaticStr, VariantNames}; +use walkdir::WalkDir; + +mod properties; +mod relationships; +#[cfg(unix)] +mod unix_only_properties; + +#[derive(Debug, Clone, Copy, PartialEq, Display, EnumString, VariantNames, IntoStaticStr)] +#[strum(serialize_all = "camelCase", const_into_str)] +enum ConflictResolutionStrategy { + Fail, + Replace, + Ignore, +} + +#[cfg(unix)] +#[derive(Debug)] +struct PutFileUnixPermissions { + file_permissions: Option, + directory_permissions: Option, +} + +#[cfg(unix)] +impl PutFileUnixPermissions { + fn set_directory_permissions(&self, path: &Path) -> std::io::Result<()> { + if let Some(permissions) = self.directory_permissions.as_ref().map(|p| p.clone()) { + return std::fs::set_permissions(path, permissions); + } + Ok(()) + } + + fn set_file_permissions(&self, file: &Path) -> std::io::Result<()> { + if let Some(permissions) = self.file_permissions.as_ref().map(|p| p.clone()) { + return std::fs::set_permissions(file, permissions); + } + Ok(()) + } +} + +#[cfg(windows)] +#[derive(Debug)] +struct PutFileUnixPermissions {} + +#[cfg(windows)] +impl PutFileUnixPermissions { + fn set_directory_permissions(&self, _path: &Path) -> std::io::Result<()> { + Ok(()) + } + + fn set_file_permissions(&self, _file: &Path) -> std::io::Result<()> { + Ok(()) + } +} + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct PutFileRs { + conflict_resolution_strategy: ConflictResolutionStrategy, + try_make_dirs: bool, + maximum_file_count: Option, + unix_permissions: PutFileUnixPermissions, +} + +impl PutFileRs { + pub(crate) fn directory_is_full(&self, p0: &Path) -> bool { + if let Some(max_file_count) = self.maximum_file_count + && let Some(parent) = p0.parent() + { + parent.exists() + && WalkDir::new(parent) + .into_iter() + .filter_map(Result::ok) + .filter(|e| e.file_type().is_file()) + .count() + >= max_file_count as usize + } else { + false + } + } + + fn get_destination_path(context: &Ctx) -> Result + where + Ctx: GetProperty + GetAttribute, + { + let directory = context + .get_property(&properties::DIRECTORY)? + .expect("required property"); + + let file_name = context + .get_attribute("filename")? + .unwrap_or("foo.txt".to_string()); // TODO fallback to UUID + Ok(PathBuf::from(directory + "/" + file_name.as_str())) + } + + fn prepare_destination(&self, destination: &Path) -> std::io::Result<()> { + if let Some(parent) = destination.parent() { + if self.try_make_dirs { + std::fs::create_dir_all(parent)?; + self.unix_permissions.set_directory_permissions(parent)?; + } + } + Ok(()) + } + + fn put_file( + &self, + input_stream: &mut dyn InputStream, + logger: &L, + destination: &Path, + ) -> Result<(), MinifiError> + where + L: Logger, + { + match self.prepare_destination(destination) { + Ok(_) => {} + Err(err) => { + warn!(logger, "Failed to prepare destination due to {:?}", err); + } + } + let mut file = std::fs::File::create(destination)?; + std::io::copy(input_stream, &mut file)?; + match self.unix_permissions.set_file_permissions(destination) { + Ok(_) => {} + Err(err) => { + warn!(logger, "Failed to set file permissions due to {:?}", err); + } + } + Ok(()) + } + + #[cfg(unix)] + fn parse_unix_permissions( + context: &P, + ) -> Result { + use std::os::unix::fs::PermissionsExt; + let parse_permission = + |property: &minifi_native::Property| -> Result, MinifiError> { + Ok(context + .get_property(&property)? + .map(|perm_str| u32::from_str_radix(&perm_str, 8)) + .transpose()? + .map(|perm| std::fs::Permissions::from_mode(perm))) + }; + let file_permissions = parse_permission(&unix_only_properties::PERMISSIONS)?; + let directory_permissions = parse_permission(&unix_only_properties::DIRECTORY_PERMISSIONS)?; + + Ok(PutFileUnixPermissions { + file_permissions, + directory_permissions, + }) + } + + #[cfg(windows)] + fn parse_unix_permissions( + _context: &P, + ) -> Result { + Ok(PutFileUnixPermissions {}) + } +} + +impl Schedule for PutFileRs { + fn schedule(context: &P, _logger: &L) -> Result { + let conflict_resolution_strategy = context + .get_property(&properties::CONFLICT_RESOLUTION)? + .expect("required property") + .parse::()?; + + let try_make_dirs = context + .get_bool_property(&properties::CREATE_DIRS)? + .expect("required property"); + + let maximum_file_count = context.get_u64_property(&properties::MAX_FILE_COUNT)?; + + let unix_permissions = Self::parse_unix_permissions(context)?; + + Ok(PutFileRs { + conflict_resolution_strategy, + try_make_dirs, + maximum_file_count, + unix_permissions, + }) + } +} + +impl FlowFileTransform for PutFileRs { + fn transform< + 'a, + Context: GetProperty + GetControllerService + GetAttribute, + LoggerImpl: Logger, + >( + &self, + context: &Context, + input_stream: &'a mut dyn InputStream, + logger: &LoggerImpl, + ) -> Result, MinifiError> { + trace!(logger, "on_trigger: {:?}", self); + + let Ok(destination_path) = Self::get_destination_path(context) else { + warn!(logger, "Invalid destination path"); + return Ok(TransformedFlowFile::route_without_changes(&FAILURE)); + }; + + if self.directory_is_full(&destination_path) { + warn!(logger, "Directory is full"); + return Ok(TransformedFlowFile::route_without_changes(&FAILURE)); + } + + if destination_path.exists() { + match self.conflict_resolution_strategy { + ConflictResolutionStrategy::Fail => { + return Ok(TransformedFlowFile::route_without_changes(&FAILURE)); + } + ConflictResolutionStrategy::Replace => { + // continue with PutFile operation + } + ConflictResolutionStrategy::Ignore => { + return Ok(TransformedFlowFile::route_without_changes(&SUCCESS)); + } + } + } + + match self.put_file(input_stream, logger, &destination_path) { + Ok(_) => Ok(TransformedFlowFile::route_without_changes(&SUCCESS)), + Err(_e) => Ok(TransformedFlowFile::route_without_changes(&FAILURE)), + } + } +} + +pub(crate) mod processor_definition; + +#[cfg(test)] +mod tests; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/processor_definition.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/processor_definition.rs new file mode 100644 index 0000000000..245782a870 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/processor_definition.rs @@ -0,0 +1,36 @@ +use super::*; +use minifi_native::{ + OutputAttribute, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, +}; + +#[cfg(windows)] +const fn get_properties() -> &'static [Property] { + &[ + properties::DIRECTORY, + properties::CONFLICT_RESOLUTION, + properties::CREATE_DIRS, + properties::MAX_FILE_COUNT, + ] +} + +#[cfg(unix)] +const fn get_properties() -> &'static [Property] { + &[ + properties::DIRECTORY, + properties::CONFLICT_RESOLUTION, + properties::CREATE_DIRS, + properties::MAX_FILE_COUNT, + unix_only_properties::PERMISSIONS, + unix_only_properties::DIRECTORY_PERMISSIONS, + ] +} + +impl ProcessorDefinition for PutFileRs { + const DESCRIPTION: &'static str = "Writes the contents of a FlowFile to the local file system."; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Required; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[]; + const RELATIONSHIPS: &'static [Relationship] = &[SUCCESS, FAILURE]; + const PROPERTIES: &'static [Property] = get_properties(); +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/properties.rs new file mode 100644 index 0000000000..91953868c9 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/properties.rs @@ -0,0 +1,51 @@ +use minifi_native::{Property, StandardPropertyValidator}; +use strum::VariantNames; + +use super::ConflictResolutionStrategy; +pub(crate) const DIRECTORY: Property = Property { + name: "Directory", + description: "The output directory to which to put files", + is_required: true, + is_sensitive: false, + supports_expr_lang: true, + default_value: Some("."), + validator: StandardPropertyValidator::NonBlankValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const CONFLICT_RESOLUTION: Property = Property { + name: "Conflict Resolution Strategy", + description: "Indicates what should happen when a file with the same name already exists in the output directory", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some(ConflictResolutionStrategy::Fail.into_str()), + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &ConflictResolutionStrategy::VARIANTS, + allowed_type: "", +}; + +pub(crate) const CREATE_DIRS: Property = Property { + name: "Create Missing Directories", + description: "If true, then missing destination directories will be created. If false, flowfiles are penalized and sent to failure.", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("true"), + validator: StandardPropertyValidator::BoolValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const MAX_FILE_COUNT: Property = Property { + name: "Maximum File Count", + description: "Specifies the maximum number of files that can exist in the output directory", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, // Diverged from the original implementation, u64 with no default describes the behavior better + validator: StandardPropertyValidator::U64Validator, + allowed_values: &[], + allowed_type: "", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/relationships.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/relationships.rs new file mode 100644 index 0000000000..a3387b65b1 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/relationships.rs @@ -0,0 +1,11 @@ +use minifi_native::Relationship; + +pub(crate) const SUCCESS: Relationship = Relationship { + name: "success", + description: "Flowfiles that are successfully written to a file are routed to this relationship", +}; + +pub(crate) const FAILURE: Relationship = Relationship { + name: "failure", + description: "Failed files (conflict, write failure, etc.) are transferred to failure", +}; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/tests.rs new file mode 100644 index 0000000000..076d8d270d --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/tests.rs @@ -0,0 +1,143 @@ +use super::*; +use crate::processors::put_file::relationships::{FAILURE, SUCCESS}; +use minifi_native::{MockLogger, MockProcessContext}; + +#[test] +fn schedule_succeeds_with_default_values() { + assert!(PutFileRs::schedule(&MockProcessContext::new(), &MockLogger::new()).is_ok()); +} + +#[test] +fn simple_put_file_test() { + let mut context = MockProcessContext::new(); + let temp_dir = tempfile::tempdir().expect("temp dir is required for testing PutFile"); + let put_file_dir = temp_dir.path().join("subdir"); + + context.properties.insert( + "Directory".to_string(), + put_file_dir.to_str().unwrap().to_string(), + ); + let put_file = PutFileRs::schedule(&context, &MockLogger::new()).expect("Should succeed"); + + let mut input_stream = std::io::Cursor::new("test".as_bytes()); + context + .attributes + .insert("filename".to_string(), "test.txt".to_string()); + let result = put_file + .transform(&context, &mut input_stream, &MockLogger::new()) + .expect("Should succeed"); + + assert_eq!(result.target_relationship(), SUCCESS.name); + + let expected_path = temp_dir.path().join("subdir/test.txt"); + assert!(expected_path.exists()); + assert_eq!(std::fs::read_to_string(expected_path).unwrap(), "test"); +} + +#[test] +fn put_file_without_create_dirs() { + let mut context = MockProcessContext::new(); + let temp_dir = tempfile::tempdir().expect("temp dir is required for testing PutFile"); + + let put_file_dir = temp_dir.path().join("subdir"); + + context.properties.insert( + "Directory".to_string(), + put_file_dir.to_str().unwrap().to_string(), + ); + + context.properties.insert( + "Create Missing Directories".to_string(), + "false".to_string(), + ); + + let put_file = PutFileRs::schedule(&context, &MockLogger::new()).expect("Should succeed"); + + let mut input_stream = std::io::Cursor::new("test".as_bytes()); + context + .attributes + .insert("filename".to_string(), "test.txt".to_string()); + let result = put_file + .transform(&context, &mut input_stream, &MockLogger::new()) + .expect("Should succeed"); + + assert_eq!(result.target_relationship(), FAILURE.name); + + let expected_path = temp_dir.path().join("subdir/test.txt"); + assert!(!expected_path.exists()); +} + +#[test] +fn directory_is_full_counts_only_files() { + let mut context = MockProcessContext::new(); + let temp_dir = tempfile::tempdir().expect("temp dir is required for testing PutFile"); + + context.properties.insert( + "Directory".to_string(), + temp_dir.path().to_str().unwrap().to_string(), + ); + context + .properties + .insert("Maximum File Count".to_string(), "2".to_string()); + + let put_file = PutFileRs::schedule(&context, &MockLogger::new()).expect("Should succeed"); + + let destination = temp_dir.path().join("test.txt"); + + // No files yet → not full + assert!(!put_file.directory_is_full(&destination)); + + // Create a subdirectory; it must not be counted as a file + std::fs::create_dir(temp_dir.path().join("subdir")).unwrap(); + assert!(!put_file.directory_is_full(&destination)); + + // Add two files → full + std::fs::write(temp_dir.path().join("a.txt"), b"a").unwrap(); + std::fs::write(temp_dir.path().join("b.txt"), b"b").unwrap(); + assert!(put_file.directory_is_full(&destination)); + + // Remove one file → not full again + std::fs::remove_file(temp_dir.path().join("a.txt")).unwrap(); + assert!(!put_file.directory_is_full(&destination)); +} + +#[cfg(unix)] +#[test] +fn put_file_test_permissions() { + use std::os::unix::fs::PermissionsExt; + let mut context = MockProcessContext::new(); + let temp_dir = tempfile::tempdir().expect("temp dir is required for testing PutFile"); + let put_file_dir = temp_dir.path().join("subdir"); + + context.properties.insert( + "Directory".to_string(), + put_file_dir.to_str().unwrap().to_string(), + ); + + context + .properties + .insert("Directory Permissions".to_string(), "0777".to_string()); + + context + .properties + .insert("Permissions".to_string(), "0777".to_string()); + let put_file = PutFileRs::schedule(&context, &MockLogger::new()).expect("Should succeed"); + + let mut input_stream = std::io::Cursor::new("test".as_bytes()); + context + .attributes + .insert("filename".to_string(), "test.txt".to_string()); + let result = put_file + .transform(&context, &mut input_stream, &MockLogger::new()) + .expect("Should succeed"); + + assert_eq!(result.target_relationship(), SUCCESS.name); + + let expected_path = temp_dir.path().join("subdir/test.txt"); + assert!(expected_path.exists()); + assert_eq!(std::fs::read_to_string(&expected_path).unwrap(), "test"); + let parent_permissions = std::fs::metadata(put_file_dir).unwrap().permissions(); + let permissions = expected_path.metadata().unwrap().permissions(); + assert_eq!(permissions.mode(), 0o100777); + assert_eq!(parent_permissions.mode(), 0o40777); +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/unix_only_properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/unix_only_properties.rs new file mode 100644 index 0000000000..8e4ba1cc6e --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/unix_only_properties.rs @@ -0,0 +1,25 @@ +use minifi_native::{Property, StandardPropertyValidator}; + +pub(crate) const PERMISSIONS: Property = Property { + name: "Permissions", + description: "Sets the permissions on the output file to the value of this attribute. Must be an octal number (e.g. 644 or 0755). Not supported on Windows systems.", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &[], + allowed_type: "", +}; + +pub(crate) const DIRECTORY_PERMISSIONS: Property = Property { + name: "Directory Permissions", + description: "Sets the permissions on the directories being created if 'Create Missing Directories' property is set. Must be an octal number (e.g. 644 or 0755). Not supported on Windows systems.", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &[], + allowed_type: "", +}; diff --git a/minifi_rust/generate_docs/generate_docs.dockerfile b/minifi_rust/generate_docs/generate_docs.dockerfile new file mode 100644 index 0000000000..3d07516176 --- /dev/null +++ b/minifi_rust/generate_docs/generate_docs.dockerfile @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +ARG BASE_IMAGE="apacheminificpp:behave" + +FROM ${BASE_IMAGE} AS builder +LABEL maintainer="Martin Zink " + +COPY ./target/release/libminifi_rs_playground.so /opt/minifi/minifi-current/extensions + +RUN /opt/minifi/minifi-current/bin/minifi --docs DOCS_OUTPUT + +FROM scratch AS docs-export + +COPY --from=builder /opt/minifi/minifi-current/DOCS_OUTPUT / diff --git a/minifi_rust/generate_docs/generate_docs.sh b/minifi_rust/generate_docs/generate_docs.sh new file mode 100755 index 0000000000..a6fd08cc2a --- /dev/null +++ b/minifi_rust/generate_docs/generate_docs.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -e + +PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$PROJECT_ROOT" + +echo "$PROJECT_ROOT" + +DOCKERFILE="generate_docs/generate_docs.dockerfile" +TARGET_DIR="target/docs" + +mkdir -p "$TARGET_DIR" + +docker buildx build \ + -f "$DOCKERFILE" \ + --target docs-export \ + --output type=local,dest="$TARGET_DIR" \ + . + +echo "Build complete. Artifacts updated in $TARGET_DIR" + +for filepath in target/docs/modules/*.md; do + filename=$(basename "$filepath") + dirname="${filename%.md}" + if [ -d "extensions/$dirname" ]; then + echo "Copying $filename to extensions/$dirname/" + cp "$filepath" "extensions/$dirname/" + fi +done diff --git a/minifi_rust/minifi_native/.gitignore b/minifi_rust/minifi_native/.gitignore new file mode 100644 index 0000000000..a9d37c560c --- /dev/null +++ b/minifi_rust/minifi_native/.gitignore @@ -0,0 +1,2 @@ +target +Cargo.lock diff --git a/minifi_rust/minifi_native/Cargo.toml b/minifi_rust/minifi_native/Cargo.toml new file mode 100644 index 0000000000..5998c4fd32 --- /dev/null +++ b/minifi_rust/minifi_native/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "minifi_native" +version = "0.1.0" +edition = "2024" +description = "A safe, idiomatic Rust wrapper for the Apache MiNiFi Native C API" +license = "Apache-2.0" + +[dependencies] +minifi_native_sys = { path = "../minifi_native_sys" } +minifi_native_macros = { path = "../minifi_native_macros" } +strum = "0.28.0" +strum_macros = "0.28.0" +humantime = "2.3.0" +byte-unit = "5.1.6" +itertools = "0.14.0" diff --git a/minifi_rust/minifi_native/src/api.rs b/minifi_rust/minifi_native/src/api.rs new file mode 100644 index 0000000000..7413ddddeb --- /dev/null +++ b/minifi_rust/minifi_native/src/api.rs @@ -0,0 +1,25 @@ +pub(crate) mod attribute; +pub(crate) mod component_definition_traits; +pub(crate) mod controller_service; +pub(crate) mod errors; +mod flow_file; +pub(crate) mod logger; +mod process_context; +pub(crate) mod process_session; +pub(crate) mod processor; +pub(crate) mod processor_wrappers; +pub(crate) mod property; +pub(crate) mod raw_controller_service; +pub(crate) mod raw_processor; +mod relationship; + +pub use flow_file::FlowFile; +pub use logger::{LogLevel, Logger}; +pub use process_context::ProcessContext; +pub use process_session::{InputStream, OutputStream, ProcessSession}; +pub use raw_controller_service::RawControllerService; +pub use raw_processor::{OnTriggerResult, ProcessorInputRequirement, RawProcessor, ThreadingModel}; + +pub use property::StandardPropertyValidator; + +pub use relationship::Relationship; diff --git a/minifi_rust/minifi_native/src/api/attribute.rs b/minifi_rust/minifi_native/src/api/attribute.rs new file mode 100644 index 0000000000..f80b9eff42 --- /dev/null +++ b/minifi_rust/minifi_native/src/api/attribute.rs @@ -0,0 +1,11 @@ +use crate::MinifiError; + +pub struct OutputAttribute { + pub name: &'static str, + pub relationships: &'static [&'static str], + pub description: &'static str, +} + +pub trait GetAttribute { + fn get_attribute(&self, name: &str) -> Result, MinifiError>; +} diff --git a/minifi_rust/minifi_native/src/api/component_definition_traits.rs b/minifi_rust/minifi_native/src/api/component_definition_traits.rs new file mode 100644 index 0000000000..671997e5fa --- /dev/null +++ b/minifi_rust/minifi_native/src/api/component_definition_traits.rs @@ -0,0 +1,22 @@ +use crate::{OutputAttribute, ProcessorInputRequirement, Property, Relationship}; + +pub trait ComponentIdentifier { + const CLASS_NAME: &'static str; + const GROUP_NAME: &'static str; + const VERSION: &'static str; +} + +pub trait ProcessorDefinition { + const DESCRIPTION: &'static str; + const INPUT_REQUIREMENT: ProcessorInputRequirement; + const SUPPORTS_DYNAMIC_PROPERTIES: bool; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute]; + const RELATIONSHIPS: &'static [Relationship]; + const PROPERTIES: &'static [Property]; +} + +pub trait ControllerServiceDefinition { + const DESCRIPTION: &'static str; + const PROPERTIES: &'static [Property]; +} diff --git a/minifi_rust/minifi_native/src/api/controller_service.rs b/minifi_rust/minifi_native/src/api/controller_service.rs new file mode 100644 index 0000000000..875dc02766 --- /dev/null +++ b/minifi_rust/minifi_native/src/api/controller_service.rs @@ -0,0 +1,62 @@ +use crate::api::RawControllerService; +use crate::{ComponentIdentifier, GetProperty, LogLevel, Logger, MinifiError}; + +pub trait EnableControllerService { + fn enable(context: &Ctx, logger: &L) -> Result + where + Self: Sized; +} + +#[derive(Debug)] +pub struct ControllerService +where + Implementation: EnableControllerService + ComponentIdentifier, + L: Logger, +{ + logger: L, + enabled_impl: Option, +} + +impl ControllerService +where + Implementation: EnableControllerService + ComponentIdentifier, + L: Logger, +{ + pub fn get_implementation(&self) -> Option<&Implementation> { + self.enabled_impl.as_ref() + } +} + +impl RawControllerService for ControllerService +where + Implementation: EnableControllerService + ComponentIdentifier, + L: Logger, +{ + type LoggerType = L; + + fn new(logger: Self::LoggerType) -> Self { + Self { + logger, + enabled_impl: None, + } + } + + fn log(&self, log_level: LogLevel, args: std::fmt::Arguments) { + self.logger.log(log_level, args); + } + + fn enable(&mut self, context: &P) -> Result<(), MinifiError> { + self.enabled_impl = Some(Implementation::enable(context, &self.logger)?); + Ok(()) + } +} + +impl ComponentIdentifier for ControllerService +where + Implementation: EnableControllerService + ComponentIdentifier, + L: Logger, +{ + const CLASS_NAME: &'static str = Implementation::CLASS_NAME; + const GROUP_NAME: &'static str = Implementation::GROUP_NAME; + const VERSION: &'static str = Implementation::VERSION; +} diff --git a/minifi_rust/minifi_native/src/api/errors.rs b/minifi_rust/minifi_native/src/api/errors.rs new file mode 100644 index 0000000000..dbe68ea31c --- /dev/null +++ b/minifi_rust/minifi_native/src/api/errors.rs @@ -0,0 +1,128 @@ +use minifi_native_sys::minifi_status; +use std::borrow::Cow; +use std::ffi::NulError; +use std::fmt; +use std::num::{NonZeroU32, ParseIntError}; +use std::str::ParseBoolError; + +#[derive(Debug, Clone)] +pub enum ParseError { + Strum(strum::ParseError), + Bool(ParseBoolError), + Int(ParseIntError), + Duration(humantime::DurationError), + Size(byte_unit::ParseError), + Nul(NulError), + Other, +} + +#[derive(Debug)] +pub enum MinifiError { + UnknownError, + StatusError((Cow<'static, str>, NonZeroU32)), + MissingRequiredProperty(Cow<'static, str>), + ControllerServiceError(Cow<'static, str>), + ValidationError(Cow<'static, str>), + ScheduleError(Cow<'static, str>), + TriggerError(Cow<'static, str>), + Parse(ParseError), + IoError(std::io::Error), +} + +impl From for MinifiError { + fn from(error: std::io::Error) -> Self { + MinifiError::IoError(error) + } +} + +impl From for MinifiError { + fn from(err: strum::ParseError) -> Self { + MinifiError::Parse(ParseError::Strum(err)) + } +} + +impl From for MinifiError { + fn from(err: ParseBoolError) -> Self { + MinifiError::Parse(ParseError::Bool(err)) + } +} + +impl From for MinifiError { + fn from(err: ParseIntError) -> Self { + MinifiError::Parse(ParseError::Int(err)) + } +} + +impl From for MinifiError { + fn from(err: humantime::DurationError) -> Self { + MinifiError::Parse(ParseError::Duration(err)) + } +} + +impl From for MinifiError { + fn from(err: byte_unit::ParseError) -> Self { + MinifiError::Parse(ParseError::Size(err)) + } +} + +impl From for MinifiError { + fn from(err: NulError) -> Self { + MinifiError::Parse(ParseError::Nul(err)) + } +} + +impl MinifiError { + pub(crate) fn to_status(&self) -> minifi_status { + // TODO expand this + minifi_native_sys::minifi_status_MINIFI_STATUS_UNKNOWN_ERROR + } + + pub fn validation_err>>(msg: S) -> Self { + MinifiError::ValidationError(msg.into()) + } + + pub fn schedule_err>>(msg: S) -> Self { + MinifiError::ScheduleError(msg.into()) + } + + pub fn trigger_err>>(msg: S) -> Self { + MinifiError::TriggerError(msg.into()) + } + + pub fn missing_required_property>>(msg: S) -> Self { + MinifiError::MissingRequiredProperty(msg.into()) + } + + pub fn controller_service_err>>(msg: S) -> Self { + MinifiError::ControllerServiceError(msg.into()) + } +} + +impl fmt::Display for MinifiError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + MinifiError::StatusError((context, code)) => match code.get() { + minifi_native_sys::minifi_status_MINIFI_STATUS_UNKNOWN_ERROR => { + write!(f, "{}, unknown error", context) + } + minifi_native_sys::minifi_status_MINIFI_STATUS_NOT_SUPPORTED_PROPERTY => { + write!(f, "{}, not supported property", context) + } + minifi_native_sys::minifi_status_MINIFI_STATUS_DYNAMIC_PROPERTIES_NOT_SUPPORTED => { + write!(f, "{}, dynamic properties not supported", context) + } + minifi_native_sys::minifi_status_MINIFI_STATUS_PROPERTY_NOT_SET => { + write!(f, "{}, property not set", context) + } + minifi_native_sys::minifi_status_MINIFI_STATUS_VALIDATION_FAILED => { + write!(f, "{}, validation failed", context) + } + minifi_native_sys::minifi_status_MINIFI_STATUS_PROCESSOR_YIELD => { + write!(f, "{}, processor yield", context) + } + _ => write!(f, "{} (Unknown Status Code: {})", context, code), + }, + _ => write!(f, "{:?}", self), + } + } +} diff --git a/minifi_rust/minifi_native/src/api/flow_file.rs b/minifi_rust/minifi_native/src/api/flow_file.rs new file mode 100644 index 0000000000..24c441e1df --- /dev/null +++ b/minifi_rust/minifi_native/src/api/flow_file.rs @@ -0,0 +1 @@ +pub trait FlowFile {} diff --git a/minifi_rust/minifi_native/src/api/logger.rs b/minifi_rust/minifi_native/src/api/logger.rs new file mode 100644 index 0000000000..a7c3fbfacb --- /dev/null +++ b/minifi_rust/minifi_native/src/api/logger.rs @@ -0,0 +1,95 @@ +use std::fmt; +use std::fmt::Debug; + +use strum_macros::{Display, EnumString, VariantNames}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Display, EnumString, VariantNames)] +#[strum(serialize_all = "PascalCase", const_into_str)] +pub enum LogLevel { + Trace, + Debug, + Info, + Warn, + Error, + Critical, + Off, +} + +pub trait Logger: Debug { + fn log(&self, level: LogLevel, args: fmt::Arguments); + fn should_log(&self, level: LogLevel) -> bool; +} + +/// The "Master" macro that handles the core logic. +/// It takes the logger instance, the level, and the format string/args. +#[macro_export] +macro_rules! log { + ($logger:expr, $level:expr, $($arg:tt)+) => { + if $logger.should_log($level) { + $logger.log($level, format_args!($($arg)+)) + } + }; +} + +/// Log at the Trace level +#[macro_export] +macro_rules! trace { + ($logger:expr, $($arg:tt)+) => { + $crate::log!($logger, $crate::LogLevel::Trace, $($arg)+) + }; +} + +/// Log at the Debug level +#[macro_export] +macro_rules! debug { + ($logger:expr, $($arg:tt)+) => { + $crate::log!($logger, $crate::LogLevel::Debug, $($arg)+) + }; +} + +/// Log at the Info level +#[macro_export] +macro_rules! info { + ($logger:expr, $($arg:tt)+) => { + $crate::log!($logger, $crate::LogLevel::Info, $($arg)+) + }; +} + +/// Log at the Warn level +#[macro_export] +macro_rules! warn { + ($logger:expr, $($arg:tt)+) => { + $crate::log!($logger, $crate::LogLevel::Warn, $($arg)+) + }; +} + +/// Log at the Error level +#[macro_export] +macro_rules! error { + ($logger:expr, $($arg:tt)+) => { + $crate::log!($logger, $crate::LogLevel::Error, $($arg)+) + }; +} + +/// Log at the Critical level +#[macro_export] +macro_rules! critical { + ($logger:expr, $($arg:tt)+) => { + $crate::log!($logger, $crate::LogLevel::Critical, $($arg)+) + }; +} + +#[cfg(test)] +mod tests { + use crate::LogLevel; + + #[test] + fn order_test() { + assert!(LogLevel::Debug > LogLevel::Trace); + assert!(LogLevel::Info > LogLevel::Debug); + assert!(LogLevel::Warn > LogLevel::Info); + assert!(LogLevel::Error > LogLevel::Warn); + assert!(LogLevel::Critical > LogLevel::Error); + assert!(LogLevel::Off > LogLevel::Critical); + } +} diff --git a/minifi_rust/minifi_native/src/api/process_context.rs b/minifi_rust/minifi_native/src/api/process_context.rs new file mode 100644 index 0000000000..fc60ca502f --- /dev/null +++ b/minifi_rust/minifi_native/src/api/process_context.rs @@ -0,0 +1,126 @@ +use crate::StandardPropertyValidator::*; +use crate::api::RawControllerService; +use crate::api::component_definition_traits::ComponentIdentifier; +use crate::api::flow_file::FlowFile; +use crate::api::property::GetControllerService; +use crate::{EnableControllerService, GetProperty, MinifiError, Property}; +use std::str::FromStr; +use std::time::Duration; + +pub trait ProcessContext { + type FlowFile: FlowFile; + + fn get_property( + &self, + property: &Property, + flow_file: Option<&Self::FlowFile>, + ) -> Result, MinifiError>; + + fn get_bool_property( + &self, + property: &Property, + flow_file: Option<&Self::FlowFile>, + ) -> Result, MinifiError> { + if property.validator != BoolValidator { + return Err(MinifiError::validation_err(format!( + "to use get_bool_property {:?} must have BoolValidator", + property + ))); + } + + if let Some(property_val) = self.get_property(property, flow_file)? { + Ok(Some(bool::from_str(&property_val)?)) + } else { + Ok(None) + } + } + + fn get_duration_property( + &self, + property: &Property, + flow_file: Option<&Self::FlowFile>, + ) -> Result, MinifiError> { + if property.validator != TimePeriodValidator { + return Err(MinifiError::validation_err(format!( + "to use get_duration_property {:?} must have TimePeriodValidator", + property + ))); + } + + if let Some(property_val) = self.get_property(property, flow_file)? { + Ok(Some(humantime::parse_duration(property_val.as_str())?)) + } else { + Ok(None) + } + } + + fn get_size_property( + &self, + property: &Property, + flow_file: Option<&Self::FlowFile>, + ) -> Result, MinifiError> { + if property.validator != DataSizeValidator { + return Err(MinifiError::validation_err(format!( + "to use get_size_property {:?} must have DataSizeValidator", + property + ))); + } + if let Some(property_val) = self.get_property(property, flow_file)? { + Ok(Some(byte_unit::Byte::from_str(&property_val)?.as_u64())) + } else { + Ok(None) + } + } + + fn get_u64_property( + &self, + property: &Property, + flow_file: Option<&Self::FlowFile>, + ) -> Result, MinifiError> { + if property.validator != U64Validator { + return Err(MinifiError::validation_err(format!( + "to use get_u64_property {:?} must have U64Validator", + property + ))); + } + if let Some(property_val) = self.get_property(property, flow_file)? { + Ok(Some(u64::from_str(&property_val)?)) + } else { + Ok(None) + } + } + + fn get_raw_controller_service( + &self, + property: &Property, + ) -> Result, MinifiError> + where + Cs: RawControllerService + ComponentIdentifier + 'static; + + fn get_controller_service(&self, property: &Property) -> Result, MinifiError> + where + Cs: EnableControllerService + ComponentIdentifier + 'static; + + fn report_metrics(&self, metrics: Vec<(String, f64)>) -> Result<(), MinifiError>; +} + +impl GetProperty for S +where + S: ProcessContext, +{ + fn get_property(&self, property: &Property) -> Result, MinifiError> { + self.get_property(property, None) + } +} + +impl GetControllerService for S +where + S: ProcessContext, +{ + fn get_controller_service(&self, property: &Property) -> Result, MinifiError> + where + Cs: EnableControllerService + ComponentIdentifier + 'static, + { + ProcessContext::get_controller_service::(self, property) + } +} diff --git a/minifi_rust/minifi_native/src/api/process_session.rs b/minifi_rust/minifi_native/src/api/process_session.rs new file mode 100644 index 0000000000..8fb78dfd18 --- /dev/null +++ b/minifi_rust/minifi_native/src/api/process_session.rs @@ -0,0 +1,60 @@ +use crate::MinifiError; +use crate::api::flow_file::FlowFile; +pub trait InputStream: std::io::BufRead + Send + std::fmt::Debug {} +pub trait OutputStream: std::io::Write + Send + std::fmt::Debug {} +impl OutputStream for T {} +impl InputStream for T {} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum IoState { + Ok, + Cancel, +} + +pub trait ProcessSession { + type FlowFile: FlowFile; + + fn create(&mut self) -> Result; + fn get(&mut self) -> Option; + fn transfer(&self, flow_file: Self::FlowFile, relationship: &str) -> Result<(), MinifiError>; + fn remove(&mut self, flow_file: Self::FlowFile) -> Result<(), MinifiError>; + + fn set_attribute( + &self, + flow_file: &mut Self::FlowFile, + attr_key: &str, + attr_value: &str, + ) -> Result<(), MinifiError>; + fn get_attribute(&self, flow_file: &Self::FlowFile, attr_key: &str) -> Option; + fn on_attributes( + &self, + flow_file: &Self::FlowFile, + process_attr: F, + ) -> bool; + + fn write(&self, flow_file: &Self::FlowFile, data: &[u8]) -> Result<(), MinifiError>; + fn write_lazy<'a>( + &self, + flow_file: &Self::FlowFile, + stream: Box, + ) -> Result<(), MinifiError>; + + fn write_stream(&self, flow_file: &Self::FlowFile, callback: F) -> Result + where + F: FnOnce(&mut dyn OutputStream) -> Result<(R, IoState), MinifiError>; + + fn read(&self, flow_file: &Self::FlowFile) -> Option>; + fn read_stream(&self, flow_file: &Self::FlowFile, callback: F) -> Result + where + F: FnOnce(&mut dyn InputStream) -> Result; + fn read_in_batches( + &self, + flow_file: &Self::FlowFile, + batch_size: usize, + process_batch: F, + ) -> Result<(), MinifiError> + where + F: FnMut(&[u8]) -> Result<(), MinifiError>; + + fn get_flow_file_id(&self, flow_file: &Self::FlowFile) -> Result; +} diff --git a/minifi_rust/minifi_native/src/api/processor.rs b/minifi_rust/minifi_native/src/api/processor.rs new file mode 100644 index 0000000000..60b15ceacf --- /dev/null +++ b/minifi_rust/minifi_native/src/api/processor.rs @@ -0,0 +1,60 @@ +use crate::api::{RawProcessor, ThreadingModel}; +use crate::{GetProperty, LogLevel, Logger, MinifiError, ProcessContext}; +use std::marker::PhantomData; + +pub trait Schedule { + fn schedule( + context: &Ctx, + logger: &L, + ) -> Result + where + Self: Sized; + + fn unschedule(&mut self) {} +} + +pub struct Processor +where + Impl: Schedule, + T: ThreadingModel, + L: Logger, +{ + pub(crate) logger: L, + pub(crate) scheduled_impl: Option, + threading_model: PhantomData, + flow_file_type: PhantomData, +} + +impl RawProcessor for Processor +where + Impl: Schedule, + T: ThreadingModel, + L: Logger, +{ + type Threading = T; + type LoggerType = L; + + fn new(logger: Self::LoggerType) -> Self { + Self { + logger, + scheduled_impl: None, + threading_model: PhantomData, + flow_file_type: PhantomData, + } + } + + fn log(&self, log_level: LogLevel, args: std::fmt::Arguments) { + self.logger.log(log_level, args); + } + + fn on_schedule(&mut self, context: &P) -> Result<(), MinifiError> { + self.scheduled_impl = Some(Impl::schedule(context, &self.logger)?); + Ok(()) + } + + fn on_unschedule(&mut self) { + if let Some(ref mut scheduled_impl) = self.scheduled_impl { + scheduled_impl.unschedule() + } + } +} diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers.rs b/minifi_rust/minifi_native/src/api/processor_wrappers.rs new file mode 100644 index 0000000000..ee5827f785 --- /dev/null +++ b/minifi_rust/minifi_native/src/api/processor_wrappers.rs @@ -0,0 +1,5 @@ +pub(crate) mod complex_processor; +pub(crate) mod flow_file_source; +pub(crate) mod flow_file_stream_transform; +pub(crate) mod flow_file_transform; +pub(crate) mod utils; diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs new file mode 100644 index 0000000000..4abebc0fcf --- /dev/null +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs @@ -0,0 +1,89 @@ +use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; +use crate::{ + ComponentIdentifier, Concurrent, Exclusive, Logger, MinifiError, OnTriggerResult, + ProcessContext, ProcessSession, Processor, ProcessorDefinition, Schedule, +}; + +pub trait MutTrigger { + fn trigger( + &mut self, + context: &mut Ctx, + session: &mut Session, + logger: &Lggr, + ) -> Result + where + Ctx: ProcessContext, + Session: ProcessSession, + Lggr: Logger; +} + +pub trait Trigger { + fn trigger( + &self, + context: &mut Context, + session: &mut Session, + logger: &Lggr, + ) -> Result + where + Context: ProcessContext, + Session: ProcessSession, + Lggr: Logger; +} + +pub struct ComplexProcessorType {} + +impl SingleThreadedTrigger + for Processor +where + Implementation: Schedule + + MutTrigger + + ComponentIdentifier + + ProcessorDefinition, + L: Logger, +{ + fn on_trigger( + &mut self, + context: &mut PC, + session: &mut PS, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession, + { + if let Some(ref mut scheduled_impl) = self.scheduled_impl { + scheduled_impl.trigger(context, session, &self.logger) + } else { + Err(MinifiError::trigger_err( + "The processor hasn't been scheduled yet", + )) + } + } +} + +impl MultiThreadedTrigger + for Processor +where + Implementation: Schedule + + Trigger + + ComponentIdentifier + + ProcessorDefinition, + L: Logger, +{ + fn on_trigger( + &self, + context: &mut PC, + session: &mut PS, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession, + { + if let Some(ref scheduled_impl) = self.scheduled_impl { + scheduled_impl.trigger(context, session, &self.logger) + } else { + Err(MinifiError::trigger_err( + "The processor hasn't been scheduled yet", + )) + } + } +} diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs new file mode 100644 index 0000000000..5bf2f33d10 --- /dev/null +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs @@ -0,0 +1,129 @@ +use crate::api::processor_wrappers::utils::flow_file_content::Content; +use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; +use crate::{ + Concurrent, Exclusive, GetControllerService, GetProperty, Logger, + MinifiError, OnTriggerResult, ProcessContext, ProcessSession, Processor, Relationship, + Schedule, +}; +use std::collections::HashMap; + +pub struct GeneratedFlowFile<'a> { + target_relationship_name: &'static str, + new_content: Option>, + attributes_to_add: HashMap, +} + +impl<'a> GeneratedFlowFile<'a> { + pub fn new( + target_relationship: &'a Relationship, + new_content: Option>, + attributes_to_add: HashMap, + ) -> Self { + Self { + target_relationship_name: target_relationship.name, + new_content, + attributes_to_add, + } + } + + pub fn target_relationship_name(&self) -> &'static str { + self.target_relationship_name + } +} + +pub trait FlowFileSource { + fn generate<'a, Context: GetProperty + GetControllerService, LoggerImpl: Logger>( + &self, + context: &'a mut Context, + logger: &LoggerImpl, + ) -> Result>, MinifiError>; +} + +pub trait MutFlowFileSource { + fn generate<'a, Context: GetProperty + GetControllerService, LoggerImpl: Logger>( + &mut self, + context: &'a mut Context, + logger: &LoggerImpl, + ) -> Result>, MinifiError>; +} + +fn handle_generated_flow_files( + session: &mut PS, + generated_flow_files: Vec, +) -> Result +where + PC: ProcessContext, + PS: ProcessSession, +{ + if generated_flow_files.is_empty() { + return Ok(OnTriggerResult::Yield); + } + + for new_flow_file_data in generated_flow_files { + let mut ff = session.create()?; + match new_flow_file_data.new_content { + None => {} + Some(Content::Buffer(buffer)) => session.write(&mut ff, &buffer)?, + Some(Content::Stream(stream)) => session.write_lazy(&mut ff, stream)?, + } + for (k, v) in &new_flow_file_data.attributes_to_add { + session.set_attribute(&mut ff, k, v)?; + } + session.transfer(ff, new_flow_file_data.target_relationship_name)?; + } + Ok(OnTriggerResult::Ok) +} + +pub struct FlowFileSourceProcessorType {} + +impl<'a, Implementation, L> MultiThreadedTrigger + for Processor +where + Implementation: Schedule + FlowFileSource, + L: Logger, +{ + fn on_trigger( + &self, + context: &mut PC, + session: &mut PS, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession, + { + if let Some(ref scheduled_impl) = self.scheduled_impl { + let files = scheduled_impl.generate(context, &self.logger)?; + handle_generated_flow_files::(session, files) + } else { + Err(MinifiError::trigger_err( + "The processor hasn't been scheduled yet", + )) + } + } +} + +impl<'a, Implementation, L> SingleThreadedTrigger + for Processor +where + Implementation: Schedule + MutFlowFileSource, + L: Logger, +{ + fn on_trigger( + &mut self, + context: &mut PC, + session: &mut PS, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession, + { + if let Some(ref mut scheduled_impl) = self.scheduled_impl { + let files = scheduled_impl.generate(context, &self.logger)?; + handle_generated_flow_files::(session, files) + } else { + Err(MinifiError::trigger_err( + "The processor hasn't been scheduled yet", + )) + } + } +} diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs new file mode 100644 index 0000000000..71dae71a4c --- /dev/null +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs @@ -0,0 +1,174 @@ +use crate::api::process_session::IoState; +use crate::api::processor_wrappers::utils::context_session_flowfile_bundle::ContextSessionFlowFileBundle; +use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; +use crate::{ + Concurrent, Exclusive, GetAttribute, GetControllerService, GetProperty, + InputStream, LogLevel, Logger, MinifiError, OnTriggerResult, OutputStream, ProcessContext, + ProcessSession, Processor, Relationship, Schedule, +}; +use std::collections::HashMap; + +pub struct TransformStreamResult { + target_relationship_name: &'static str, + attributes_to_add: HashMap, + write_status: IoState, +} + +impl TransformStreamResult { + pub fn new( + target_relationship: &Relationship, + attributes_to_add: HashMap, + ) -> Self { + Self { + target_relationship_name: target_relationship.name, + attributes_to_add, + write_status: IoState::Ok, + } + } + + pub fn route_without_changes(target_relationship: &Relationship) -> Self { + Self { + target_relationship_name: target_relationship.name, + attributes_to_add: HashMap::new(), + write_status: IoState::Cancel, + } + } + + pub fn target_relationship_name(&self) -> &'static str { + self.target_relationship_name + } + + pub fn get_attribute(&self, name: &str) -> Option { + self.attributes_to_add.get(name).cloned() + } + + pub fn write_status(&self) -> IoState { + self.write_status + } +} + +pub trait FlowFileStreamTransform { + fn transform( + &self, + context: &Ctx, + input_stream: &mut dyn InputStream, + output_stream: &mut dyn OutputStream, + logger: &LoggerImpl, + ) -> Result; +} + +pub trait MutFlowFileStreamTransform { + fn transform( + &mut self, + context: &Ctx, + input_stream: &mut dyn InputStream, + output_stream: &mut dyn OutputStream, + logger: &LoggerImpl, + ) -> Result; +} + +pub struct FlowFileStreamTransformProcessorType {} + +fn handle_stream_transform( + context: &mut PC, + session: &mut PS, + logger: &L, + mut transform_fn: F, +) -> Result +where + PC: ProcessContext, + PS: ProcessSession, + L: Logger, + F: FnMut( + &ContextSessionFlowFileBundle, + &mut dyn InputStream, + &mut dyn OutputStream, + ) -> Result, +{ + if let Some(mut flow_file) = session.get() { + let simple_context = ContextSessionFlowFileBundle::new(context, session, Some(&flow_file)); + + let (relationship, attrs) = session.read_stream(&flow_file, |input_stream| { + session.write_stream(&flow_file, |output_stream| { + let transformed = transform_fn(&simple_context, input_stream, output_stream)?; + + Ok(( + ( + transformed.target_relationship_name, + transformed.attributes_to_add, + ), + transformed.write_status, + )) + }) + })?; + + for (k, v) in attrs { + session.set_attribute(&mut flow_file, &k, &v)?; + } + + session.transfer(flow_file, relationship)?; + + Ok(OnTriggerResult::Ok) + } else { + logger.log(LogLevel::Trace, format_args!("No flowfile to transform")); + Ok(OnTriggerResult::Yield) + } +} + +// Concurrent Implementation (Multi-Threaded) +impl<'a, Implementation, L> MultiThreadedTrigger + for Processor +where + Implementation: + Schedule + FlowFileStreamTransform, + L: Logger, +{ + fn on_trigger( + &self, + context: &mut PC, + session: &mut PS, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession, + { + if let Some(ref scheduled_impl) = self.scheduled_impl { + handle_stream_transform(context, session, &self.logger, |ctx, input, output| { + scheduled_impl.transform(ctx, input, output, &self.logger) + }) + } else { + Err(MinifiError::trigger_err( + "The processor hasn't been scheduled yet", + )) + } + } +} + +// Exclusive Implementation (Single-Threaded) +impl<'a, Implementation, L> SingleThreadedTrigger + for Processor +where + Implementation: + Schedule + MutFlowFileStreamTransform, + L: Logger, +{ + fn on_trigger( + &mut self, + context: &mut PC, + session: &mut PS, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession, + { + if let Some(ref mut scheduled_impl) = self.scheduled_impl { + handle_stream_transform(context, session, &self.logger, |ctx, input, output| { + scheduled_impl.transform(ctx, input, output, &self.logger) + }) + } else { + Err(MinifiError::trigger_err( + "The processor hasn't been scheduled yet", + )) + } + } +} diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs new file mode 100644 index 0000000000..3ddd19029e --- /dev/null +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs @@ -0,0 +1,184 @@ +use crate::api::InputStream; +use crate::api::processor::{Processor}; +use crate::api::processor_wrappers::utils::context_session_flowfile_bundle::ContextSessionFlowFileBundle; +use crate::api::processor_wrappers::utils::flow_file_content::Content; +use crate::api::property::{GetControllerService, GetProperty}; +use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; +use crate::{ + Concurrent, Exclusive, GetAttribute, LogLevel, Logger, MinifiError, + OnTriggerResult, ProcessContext, ProcessSession, Relationship, Schedule, info, +}; +use std::collections::HashMap; + +#[derive(Debug)] +pub struct TransformedFlowFile<'a> { + target_relationship_name: &'static str, + new_content: Option>, + attributes_to_add: HashMap, +} + +impl<'a> TransformedFlowFile<'a> { + pub fn route_without_changes(target_relationship: &Relationship) -> Self { + Self { + target_relationship_name: target_relationship.name, + new_content: None, + attributes_to_add: HashMap::new(), + } + } + + pub fn new( + target_relationship: &Relationship, + new_content: Option>, + attributes_to_add: HashMap, + ) -> Self { + Self { + target_relationship_name: target_relationship.name, + new_content: new_content.map(|b| Content::Buffer(b)), + attributes_to_add, + } + } + + pub fn new_content(&'_ self) -> Option<&'_ Content<'_>> { + self.new_content.as_ref() + } + + pub fn target_relationship(&self) -> &'static str { + self.target_relationship_name + } + + pub fn attributes_to_add(&self) -> &HashMap { + &self.attributes_to_add + } +} + +pub trait FlowFileTransform { + fn transform< + 'a, + Context: GetProperty + GetControllerService + GetAttribute, + LoggerImpl: Logger, + >( + &self, + context: &Context, + input_stream: &'a mut dyn InputStream, + logger: &LoggerImpl, + ) -> Result, MinifiError>; +} + +pub trait MutFlowFileTransform { + fn transform< + 'a, + Context: GetProperty + GetControllerService + GetAttribute, + LoggerImpl: Logger, + >( + &mut self, + context: &Context, + input_stream: &'a mut dyn InputStream, + logger: &LoggerImpl, + ) -> Result, MinifiError>; +} + +pub struct FlowFileTransformProcessorType {} + +fn handle_transform( + context: &mut PC, + session: &mut PS, + logger: &L, + mut transform_fn: F, +) -> Result +where + PC: ProcessContext, + PS: ProcessSession, + L: Logger, + F: for<'stream> FnMut( + &ContextSessionFlowFileBundle<'_, PC, PS>, + &'stream mut dyn InputStream, + ) -> Result, MinifiError>, +{ + if let Some(mut flow_file) = session.get() { + let simple_context = ContextSessionFlowFileBundle::new(context, session, Some(&flow_file)); + + let (attrs_to_add, relationship) = session.read_stream(&flow_file, |input_stream| { + let transformed = transform_fn(&simple_context, input_stream)?; + + info!(logger, "{:?}", transformed); + match transformed.new_content { + None => {} + Some(Content::Buffer(buffer)) => { + session.write(&flow_file, &buffer)?; + } + Some(Content::Stream(stream)) => { + session.write_lazy(&flow_file, stream)?; + } + }; + + Ok(( + transformed.attributes_to_add, + transformed.target_relationship_name, + )) + })?; + + for (k, v) in attrs_to_add { + session.set_attribute(&mut flow_file, &k, &v)?; + } + + session.transfer(flow_file, relationship)?; + Ok(OnTriggerResult::Ok) + } else { + logger.log(LogLevel::Trace, format_args!("No flowfile to transform")); + Ok(OnTriggerResult::Yield) + } +} + +impl MultiThreadedTrigger + for Processor +where + Implementation: Schedule + FlowFileTransform, + L: Logger, +{ + fn on_trigger( + &self, + context: &mut PC, + session: &mut PS, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession, + { + if let Some(ref scheduled_impl) = self.scheduled_impl { + handle_transform(context, session, &self.logger, |ctx, input| { + scheduled_impl.transform(ctx, input, &self.logger) + }) + } else { + Err(MinifiError::trigger_err( + "The processor hasn't been scheduled yet", + )) + } + } +} + +impl SingleThreadedTrigger + for Processor +where + Implementation: Schedule + MutFlowFileTransform, + L: Logger, +{ + fn on_trigger( + &mut self, + context: &mut PC, + session: &mut PS, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession, + { + if let Some(ref mut scheduled_impl) = self.scheduled_impl { + handle_transform(context, session, &self.logger, |ctx, input| { + scheduled_impl.transform(ctx, input, &self.logger) + }) + } else { + Err(MinifiError::trigger_err( + "The processor hasn't been scheduled yet", + )) + } + } +} diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/utils.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/utils.rs new file mode 100644 index 0000000000..e10592e96c --- /dev/null +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/utils.rs @@ -0,0 +1,2 @@ +pub(crate) mod context_session_flowfile_bundle; +pub(crate) mod flow_file_content; diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/utils/context_session_flowfile_bundle.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/utils/context_session_flowfile_bundle.rs new file mode 100644 index 0000000000..8af603c9ff --- /dev/null +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/utils/context_session_flowfile_bundle.rs @@ -0,0 +1,70 @@ +use crate::api::attribute::GetAttribute; +use crate::api::property::{GetControllerService, GetProperty}; +use crate::{ + ComponentIdentifier, EnableControllerService, MinifiError, ProcessContext, ProcessSession, + Property, +}; + +pub struct ContextSessionFlowFileBundle<'a, PC, PS> +where + PC: ProcessContext, + PS: ProcessSession, +{ + context: &'a PC, + session: &'a PS, + flow_file: Option<&'a PC::FlowFile>, +} + +impl<'a, PC, PS> ContextSessionFlowFileBundle<'a, PC, PS> +where + PC: ProcessContext, + PS: ProcessSession, +{ + pub(crate) fn new( + context: &'a PC, + session: &'a PS, + flow_file: Option<&'a PC::FlowFile>, + ) -> Self { + Self { + context, + session, + flow_file, + } + } +} +impl<'a, PC, PS> GetProperty for ContextSessionFlowFileBundle<'a, PC, PS> +where + PC: ProcessContext, + PS: ProcessSession, +{ + fn get_property(&self, property: &Property) -> Result, MinifiError> { + self.context.get_property(property, self.flow_file) + } +} + +impl<'a, PC, PS> GetControllerService for ContextSessionFlowFileBundle<'a, PC, PS> +where + PC: ProcessContext, + PS: ProcessSession, +{ + fn get_controller_service(&self, property: &Property) -> Result, MinifiError> + where + Cs: EnableControllerService + ComponentIdentifier + 'static, + { + self.context.get_controller_service(property) + } +} + +impl<'a, PC, PS> GetAttribute for ContextSessionFlowFileBundle<'a, PC, PS> +where + PC: ProcessContext, + PS: ProcessSession, +{ + fn get_attribute(&self, name: &str) -> Result, MinifiError> { + if let Some(ff) = &self.flow_file { + Ok(self.session.get_attribute(ff, name)) + } else { + Ok(None) + } + } +} diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/utils/flow_file_content.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/utils/flow_file_content.rs new file mode 100644 index 0000000000..6bd5492605 --- /dev/null +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/utils/flow_file_content.rs @@ -0,0 +1,27 @@ +use std::fmt::Formatter; + +pub enum Content<'a> { + Buffer(Vec), + Stream(Box), +} + +impl std::fmt::Debug for Content<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Content::Buffer(b) => f.debug_struct("Content").field("Buffer", &b).finish(), + Content::Stream(_s) => f.debug_struct("Content::Stream").finish(), + } + } +} + +impl From> for Content<'_> { + fn from(v: Vec) -> Self { + Content::Buffer(v) + } +} + +impl From for Content<'_> { + fn from(s: String) -> Self { + Content::Buffer(s.into_bytes()) + } +} diff --git a/minifi_rust/minifi_native/src/api/property.rs b/minifi_rust/minifi_native/src/api/property.rs new file mode 100644 index 0000000000..3f87b5615c --- /dev/null +++ b/minifi_rust/minifi_native/src/api/property.rs @@ -0,0 +1,98 @@ +use crate::StandardPropertyValidator::{ + BoolValidator, DataSizeValidator, TimePeriodValidator, U64Validator, +}; +use crate::{ComponentIdentifier, EnableControllerService, MinifiError}; +use std::str::FromStr; +use std::time::Duration; + +#[derive(Debug, Eq, PartialEq)] +pub enum StandardPropertyValidator { + AlwaysValidValidator, + NonBlankValidator, + TimePeriodValidator, + BoolValidator, + I64Validator, + U64Validator, + DataSizeValidator, + PortValidator, +} + +#[derive(Debug)] +pub struct Property { + pub name: &'static str, + pub description: &'static str, + pub is_required: bool, + pub is_sensitive: bool, + pub supports_expr_lang: bool, + pub default_value: Option<&'static str>, + pub validator: StandardPropertyValidator, + pub allowed_values: &'static [&'static str], + pub allowed_type: &'static str, +} + +pub trait GetProperty { + fn get_property(&self, property: &Property) -> Result, MinifiError>; + fn get_bool_property(&self, property: &Property) -> Result, MinifiError> { + if property.validator != BoolValidator { + return Err(MinifiError::validation_err(format!( + "to use get_bool_property {:?} must have BoolValidator", + property + ))); + } + + if let Some(property_val) = self.get_property(property)? { + Ok(Some(bool::from_str(&property_val)?)) + } else { + Ok(None) + } + } + + fn get_duration_property(&self, property: &Property) -> Result, MinifiError> { + if property.validator != TimePeriodValidator { + return Err(MinifiError::validation_err(format!( + "to use get_duration_property {:?} must have TimePeriodValidator", + property + ))); + } + + if let Some(property_val) = self.get_property(property)? { + Ok(Some(humantime::parse_duration(property_val.as_str())?)) + } else { + Ok(None) + } + } + + fn get_size_property(&self, property: &Property) -> Result, MinifiError> { + if property.validator != DataSizeValidator { + return Err(MinifiError::validation_err(format!( + "to use get_size_property {:?} must have DataSizeValidator", + property + ))); + } + if let Some(property_val) = self.get_property(property)? { + Ok(Some(byte_unit::Byte::from_str(&property_val)?.as_u64())) + } else { + Ok(None) + } + } + + fn get_u64_property(&self, property: &Property) -> Result, MinifiError> { + if property.validator != U64Validator { + return Err(MinifiError::validation_err(format!( + "to use get_u64_property {:?} must have U64Validator", + property + ))); + } + if let Some(property_val) = self.get_property(property)? { + Ok(Some(u64::from_str(&property_val)?)) + } else { + Ok(None) + } + } +} + +pub trait GetControllerService { + fn get_controller_service(&self, property: &Property) -> Result, MinifiError> + where + Cs: EnableControllerService + ComponentIdentifier + 'static; +} diff --git a/minifi_rust/minifi_native/src/api/raw_controller_service.rs b/minifi_rust/minifi_native/src/api/raw_controller_service.rs new file mode 100644 index 0000000000..3348a398e0 --- /dev/null +++ b/minifi_rust/minifi_native/src/api/raw_controller_service.rs @@ -0,0 +1,11 @@ +use crate::{GetProperty, LogLevel, Logger, MinifiError}; + +/// This RawControllerService will be instantiated, and called on by the agent +pub trait RawControllerService: Sized { + type LoggerType: Logger; + + fn new(logger: Self::LoggerType) -> Self; + fn log(&self, log_level: LogLevel, args: std::fmt::Arguments); + fn enable(&mut self, context: &P) -> Result<(), MinifiError>; + fn disable(&mut self) {} +} diff --git a/minifi_rust/minifi_native/src/api/raw_processor.rs b/minifi_rust/minifi_native/src/api/raw_processor.rs new file mode 100644 index 0000000000..6926db17a8 --- /dev/null +++ b/minifi_rust/minifi_native/src/api/raw_processor.rs @@ -0,0 +1,70 @@ +use crate::api::errors::MinifiError; +use crate::{LogLevel, Logger, ProcessContext, ProcessSession}; + +pub enum ProcessorInputRequirement { + Required, + Allowed, + Forbidden, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum OnTriggerResult { + Ok, + Yield, +} + +/// This RawProcessor will be instantiated, and called on by the agent +pub trait RawProcessor: Sized { + type Threading: ThreadingModel; + type LoggerType: Logger; + + fn new(logger: Self::LoggerType) -> Self; + fn log(&self, log_level: LogLevel, args: std::fmt::Arguments); + fn on_schedule(&mut self, context: &P) -> Result<(), MinifiError>; + fn on_unschedule(&mut self); +} + +/// To differentiate between single and multithreaded processors +pub trait ThreadingModel: sealed::Sealed { + const IS_EXCLUSIVE: bool; +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Concurrent; +impl ThreadingModel for Concurrent { + const IS_EXCLUSIVE: bool = false; +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Exclusive; +impl ThreadingModel for Exclusive { + const IS_EXCLUSIVE: bool = true; +} + +mod sealed { + pub trait Sealed {} + impl Sealed for super::Concurrent {} + impl Sealed for super::Exclusive {} +} + +pub trait SingleThreadedTrigger: RawProcessor { + fn on_trigger( + &mut self, + context: &mut PC, + session: &mut PS, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession; +} + +pub trait MultiThreadedTrigger: RawProcessor { + fn on_trigger( + &self, + context: &mut PC, + session: &mut PS, + ) -> Result + where + PC: ProcessContext, + PS: ProcessSession; +} diff --git a/minifi_rust/minifi_native/src/api/relationship.rs b/minifi_rust/minifi_native/src/api/relationship.rs new file mode 100644 index 0000000000..dc80477fa8 --- /dev/null +++ b/minifi_rust/minifi_native/src/api/relationship.rs @@ -0,0 +1,5 @@ +#[derive(Debug, Eq, PartialEq)] +pub struct Relationship { + pub name: &'static str, + pub description: &'static str, +} diff --git a/minifi_rust/minifi_native/src/c_ffi.rs b/minifi_rust/minifi_native/src/c_ffi.rs new file mode 100644 index 0000000000..d426cda447 --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi.rs @@ -0,0 +1,26 @@ +mod c_ffi_controller_service_context; +mod c_ffi_controller_service_definition; +mod c_ffi_controller_service_list; +mod c_ffi_flow_file; +mod c_ffi_logger; +mod c_ffi_output_attribute; +mod c_ffi_primitives; +mod c_ffi_process_context; +mod c_ffi_process_session; +mod c_ffi_processor_definition; +mod c_ffi_processor_list; +mod c_ffi_property; +mod c_ffi_relationship; +mod c_ffi_streams; + +pub use c_ffi_controller_service_definition::CffiControllerServiceDefinition; +pub use c_ffi_controller_service_definition::DynRawControllerServiceDefinition; +pub use c_ffi_controller_service_definition::RegisterableControllerService; +pub use c_ffi_controller_service_list::CffiControllerServiceList; +pub use c_ffi_logger::CffiLogger; +pub use c_ffi_primitives::StaticStrAsMinifiCStr; +pub use c_ffi_processor_definition::DispatchOnTrigger; +pub use c_ffi_processor_definition::DynRawProcessorDefinition; +pub use c_ffi_processor_definition::RawProcessorDefinition; +pub use c_ffi_processor_definition::RawRegisterableProcessor; +pub use c_ffi_processor_list::CffiProcessorList; diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs new file mode 100644 index 0000000000..52ae73cbb8 --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs @@ -0,0 +1,70 @@ +use crate::c_ffi::c_ffi_primitives::StringView; +use crate::{GetProperty, MinifiError, Property}; +use minifi_native_sys::{ + minifi_controller_service_context, + minifi_controller_service_context_get_property, + minifi_status_MINIFI_STATUS_SUCCESS, minifi_string_view, +}; +use std::borrow::Cow; +use std::ffi::c_void; + +pub struct CffiControllerServiceContext<'a> { + ptr: *mut minifi_controller_service_context, + _lifetime: std::marker::PhantomData<&'a ()>, +} + +impl<'a> CffiControllerServiceContext<'a> { + pub fn new(ptr: *mut minifi_controller_service_context) -> Self { + Self { + ptr, + _lifetime: std::marker::PhantomData, + } + } +} + +unsafe extern "C" fn property_callback( + output_option: *mut c_void, + property_c_value: minifi_string_view, +) { + unsafe { + let result_target = &mut *(output_option as *mut Option); + + if property_c_value.data.is_null() || property_c_value.length == 0 { + *result_target = None; + return; + } + + let value_slice = + std::slice::from_raw_parts(property_c_value.data as *const u8, property_c_value.length); + if let Ok(string_value) = String::from_utf8(value_slice.to_vec()) { + *result_target = Some(string_value); + } + } +} + +impl<'a> GetProperty for CffiControllerServiceContext<'a> { + fn get_property(&self, property: &Property) -> Result, MinifiError> { + let mut result: Option = None; + let property_name: StringView = StringView::new(property.name); + + let status = unsafe { + minifi_controller_service_context_get_property( + self.ptr, + property_name.as_raw(), + Some(property_callback), + &mut result as *mut _ as *mut c_void, + ) + }; + + #[allow(non_upper_case_globals)] + match status { + minifi_status_MINIFI_STATUS_SUCCESS => Ok(result), + _ => match property.is_required { + true => Err(MinifiError::MissingRequiredProperty(Cow::from( + property.name, + ))), + false => Ok(None), + }, + } + } +} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs new file mode 100644 index 0000000000..6923889a7e --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs @@ -0,0 +1,162 @@ +use crate::api::RawControllerService; +use crate::c_ffi::c_ffi_controller_service_context::CffiControllerServiceContext; +use crate::c_ffi::c_ffi_property::CProperties; +use crate::c_ffi::{CffiLogger, StaticStrAsMinifiCStr}; +use crate::{ + ComponentIdentifier, ControllerService, ControllerServiceDefinition, EnableControllerService, + LogLevel, Property, +}; +use minifi_native_sys::{ + minifi_controller_service_callbacks, minifi_controller_service_class_definition, + minifi_controller_service_context, minifi_controller_service_metadata, minifi_status, +}; +use std::ffi::c_void; + +#[derive(Debug)] +pub struct ControllerServiceClassDefinition<'a> { + inner: minifi_controller_service_class_definition, + _lifetime: std::marker::PhantomData<&'a ()>, +} + +impl<'a> ControllerServiceClassDefinition<'a> { + pub(crate) fn new(inner: minifi_controller_service_class_definition) -> Self { + Self { + inner, + _lifetime: std::marker::PhantomData, + } + } + + pub unsafe fn as_raw(&self) -> minifi_controller_service_class_definition { + self.inner + } +} + +pub struct CffiControllerServiceDefinition +where + T: RawControllerService + ComponentIdentifier, +{ + name: &'static str, + description_text: &'static str, + + c_properties: CProperties, + + _phantom: std::marker::PhantomData, +} + +impl CffiControllerServiceDefinition +where + T: RawControllerService + ComponentIdentifier, +{ + pub fn new(description_text: &'static str, properties: &'static [Property]) -> Self { + let c_properties = Property::create_c_properties(properties); + + Self { + name: T::CLASS_NAME, + description_text, + c_properties, + _phantom: std::marker::PhantomData, + } + } + + unsafe extern "C" fn create_controller_service( + metadata: minifi_controller_service_metadata, + ) -> *mut c_void { + let logger = CffiLogger::new(metadata.logger); + let controller_service = Box::new(T::new(logger)); + Box::into_raw(controller_service) as *mut c_void + } + + unsafe extern "C" fn destroy_controller_service(controller_service_ptr: *mut c_void) { + unsafe { + if !controller_service_ptr.is_null() { + let _ = Box::from_raw(controller_service_ptr as *mut T); + } + } + } + + unsafe extern "C" fn enable_controller_service( + controller_service_ptr: *mut c_void, + context_ptr: *mut minifi_controller_service_context, + ) -> minifi_status { + unsafe { + let controller_service = &mut *(controller_service_ptr as *mut T); + let context = CffiControllerServiceContext::new(context_ptr); + match controller_service.enable(&context) { + Ok(_) => 0, + Err(err) => { + controller_service.log(LogLevel::Error, format_args!("{:?}", err)); + err.to_status() + } + } + } + } + + unsafe extern "C" fn disable_controller_service(controller_service_ptr: *mut c_void) { + unsafe { + let controller_service = &mut *(controller_service_ptr as *mut T); + controller_service.disable() + } + } +} + +pub trait DynRawControllerServiceDefinition { + fn class_description(&'_ self) -> ControllerServiceClassDefinition<'_>; +} + +impl DynRawControllerServiceDefinition for CffiControllerServiceDefinition +where + T: RawControllerService + ComponentIdentifier, +{ + fn class_description(&'_ self) -> ControllerServiceClassDefinition<'_> { + unsafe { + ControllerServiceClassDefinition::new(minifi_controller_service_class_definition { + full_name: self.name.as_minifi_c_type(), + description: self.description_text.as_minifi_c_type(), + properties_count: self.c_properties.len(), + properties_ptr: self.c_properties.get_ptr(), + callbacks: minifi_controller_service_callbacks { + create: Some(Self::create_controller_service), + destroy: Some(Self::destroy_controller_service), + enable: Some(Self::enable_controller_service), + disable: Some(Self::disable_controller_service), + get_interface: None, // TODO(mzink) + }, + provided_interfaces_count: 0, + provided_interfaces_ptr: std::ptr::null_mut() // TODO(mzink) + }) + } + } +} + +pub trait RegisterableControllerService { + fn get_definition() -> Box; +} + +impl RegisterableControllerService for T +where + T: ComponentIdentifier + + ControllerServiceDefinition + + RawControllerService + + 'static, +{ + fn get_definition() -> Box { + Box::new(CffiControllerServiceDefinition::::new( + T::DESCRIPTION, + T::PROPERTIES, + )) + } +} + +impl RegisterableControllerService for ControllerService +where + Implementation: + EnableControllerService + ComponentIdentifier + ControllerServiceDefinition + 'static, +{ + fn get_definition() -> Box { + Box::new(CffiControllerServiceDefinition::< + ControllerService, + >::new( + Implementation::DESCRIPTION, Implementation::PROPERTIES + )) + } +} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs new file mode 100644 index 0000000000..d527490890 --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs @@ -0,0 +1,53 @@ +use crate::c_ffi::RegisterableControllerService; +use crate::c_ffi::c_ffi_controller_service_definition::DynRawControllerServiceDefinition; +use minifi_native_sys::minifi_controller_service_class_definition; + +pub struct CffiControllerServiceList { + controller_service_definitions: Vec>, + minifi_controller_service_class_description_list: Vec, +} + +impl CffiControllerServiceList { + pub fn new() -> Self { + Self { + controller_service_definitions: Vec::new(), + minifi_controller_service_class_description_list: Vec::new(), + } + } + + pub fn add(&mut self) { + self.add_controller_service_definition(T::get_definition()) + } + + pub fn add_controller_service_definition( + &mut self, + processor_definition: Box, + ) { + unsafe { + self.controller_service_definitions + .push(processor_definition); + self.minifi_controller_service_class_description_list.push( + self.controller_service_definitions + .last() + .unwrap() + .class_description() + .as_raw(), + ); + } + } + + pub fn get_controller_service_count(&self) -> usize { + assert_eq!( + self.controller_service_definitions.len(), + self.minifi_controller_service_class_description_list.len() + ); + self.minifi_controller_service_class_description_list.len() + } + + pub unsafe fn get_controller_service_ptr( + &self, + ) -> *const minifi_controller_service_class_definition { + self.minifi_controller_service_class_description_list + .as_ptr() + } +} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_flow_file.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_flow_file.rs new file mode 100644 index 0000000000..0fdeac4b50 --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_flow_file.rs @@ -0,0 +1,22 @@ +use crate::api::FlowFile; +use minifi_native_sys::minifi_flow_file; + +pub struct CffiFlowFile<'a> { + ptr: *mut minifi_flow_file, + _lifetime: std::marker::PhantomData<&'a ()>, +} + +impl CffiFlowFile<'_> { + pub(crate) fn new(ptr: *mut minifi_flow_file) -> Self { + Self { + ptr, + _lifetime: std::marker::PhantomData, + } + } + + pub(crate) fn get_ptr(&self) -> *mut minifi_flow_file { + self.ptr + } +} + +impl FlowFile for CffiFlowFile<'_> {} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_logger.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_logger.rs new file mode 100644 index 0000000000..2ffb893c8e --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_logger.rs @@ -0,0 +1,52 @@ +use crate::api::LogLevel; +use crate::api::Logger; +use minifi_native_sys::{minifi_log_level, minifi_logger, minifi_logger_log_string, minifi_logger_should_log, minifi_string_view}; +use std::ffi::CString; +use std::fmt; + +impl From for minifi_log_level { + fn from(level: LogLevel) -> Self { + match level { + LogLevel::Trace => minifi_native_sys::minifi_log_level_MINIFI_LOG_LEVEL_TRACE, + LogLevel::Debug => minifi_native_sys::minifi_log_level_MINIFI_LOG_LEVEL_DEBUG, + LogLevel::Info => minifi_native_sys::minifi_log_level_MINIFI_LOG_LEVEL_INFO, + LogLevel::Warn => minifi_native_sys::minifi_log_level_MINIFI_LOG_LEVEL_WARNING, + LogLevel::Error => minifi_native_sys::minifi_log_level_MINIFI_LOG_LEVEL_ERROR, + LogLevel::Critical => minifi_native_sys::minifi_log_level_MINIFI_LOG_LEVEL_CRITICAL, + LogLevel::Off => minifi_native_sys::minifi_log_level_MINIFI_LOG_LEVEL_OFF, + } + } +} + +#[derive(Clone, Copy, Debug)] +pub struct CffiLogger { + ptr: *mut minifi_logger, +} + +impl CffiLogger { + pub fn new(logger: *mut minifi_logger) -> Self { + Self { ptr: logger } + } +} + +impl Logger for CffiLogger { + fn log(&self, level: LogLevel, args: fmt::Arguments) { + unsafe { + let message = fmt::format(args); + if let Ok(c_message) = CString::new(message) { + minifi_logger_log_string( + self.ptr, + level.into(), + minifi_string_view { + data: c_message.as_ptr(), + length: c_message.as_bytes().len(), + }, + ); + } + } + } + + fn should_log(&self, level: LogLevel) -> bool { + unsafe { minifi_logger_should_log(self.ptr, level.into()) } + } +} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_output_attribute.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_output_attribute.rs new file mode 100644 index 0000000000..3485b156a2 --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_output_attribute.rs @@ -0,0 +1,41 @@ +use crate::OutputAttribute; +use crate::c_ffi::StaticStrAsMinifiCStr; +use minifi_native_sys::{minifi_string_view, minifi_output_attribute_definition}; + +#[allow(dead_code)] // the c_ vecs are holding the values referenced from the output attributes +pub struct COutputAttributes { + c_relationship_names: Vec>, + c_output_attributes: Vec, +} + +impl COutputAttributes { + pub(crate) fn new(output_attributes: &[OutputAttribute]) -> Self { + let mut c_relationship_names = Vec::new(); + let mut c_output_attributes = Vec::new(); + for output_attribute in output_attributes { + let mut output_attribute_relationships = Vec::new(); + for relationship in output_attribute.relationships { + output_attribute_relationships.push(relationship.as_minifi_c_type()); + } + c_output_attributes.push(minifi_output_attribute_definition { + name: output_attribute.name.as_minifi_c_type(), + relationships_count: output_attribute_relationships.len(), + relationships_ptr: output_attribute_relationships.as_ptr(), + description: output_attribute.description.as_minifi_c_type(), + }); + c_relationship_names.push(output_attribute_relationships); + } + COutputAttributes { + c_relationship_names, + c_output_attributes, + } + } + + pub(crate) fn len(&self) -> usize { + self.c_output_attributes.len() + } + + pub(crate) unsafe fn get_ptr(&self) -> *const minifi_output_attribute_definition { + self.c_output_attributes.as_ptr() + } +} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_primitives.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_primitives.rs new file mode 100644 index 0000000000..687fbd4382 --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_primitives.rs @@ -0,0 +1,82 @@ +use crate::ProcessorInputRequirement; +use minifi_native_sys::{minifi_string_view, minifi_input_requirement_MINIFI_INPUT_REQUIRED, minifi_input_requirement_MINIFI_INPUT_ALLOWED, minifi_input_requirement_MINIFI_INPUT_FORBIDDEN, minifi_input_requirement}; +use std::os::raw::c_char; + +#[derive(Debug)] +pub enum FfiConversionError { + NullPointer, + InvalidUtf8, +} + +#[repr(transparent)] // This allows us to pass Vec as a pointer to minifi_string_view array +#[derive(Debug)] +pub(crate) struct StringView<'a> { + inner: minifi_string_view, + _marker: std::marker::PhantomData<&'a ()>, +} + +impl<'a> StringView<'a> { + pub(crate) fn new(str: &'a str) -> Self { + Self { + inner: minifi_string_view { + data: str.as_ptr() as *const c_char, + length: str.len(), + }, + _marker: std::marker::PhantomData, + } + } + + pub unsafe fn as_raw(&self) -> minifi_string_view { + self.inner + } +} + +pub trait StaticStrAsMinifiCStr { + fn as_minifi_c_type(&self) -> minifi_string_view; +} + +impl StaticStrAsMinifiCStr for &'static str { + fn as_minifi_c_type(&self) -> minifi_string_view { + minifi_string_view { + data: self.as_ptr() as *const c_char, + length: self.len(), + } + } +} + +pub trait ConvertMinifiStringView { + unsafe fn as_string(&self) -> Result; + unsafe fn as_str(&self) -> Result<&str, FfiConversionError>; +} + +impl ConvertMinifiStringView for minifi_string_view { + unsafe fn as_string(&self) -> Result { + if self.data.is_null() { + return Err(FfiConversionError::NullPointer); + } + unsafe { + let slice = std::slice::from_raw_parts(self.data as *const u8, self.length); + String::from_utf8(slice.to_vec()).map_err(|_| FfiConversionError::InvalidUtf8) + } + } + + unsafe fn as_str(&self) -> Result<&str, FfiConversionError> { + if self.data.is_null() { + return Err(FfiConversionError::NullPointer); + } + unsafe { + let slice = std::slice::from_raw_parts(self.data as *const u8, self.length); + str::from_utf8(slice).map_err(|_| FfiConversionError::InvalidUtf8) + } + } +} + +impl ProcessorInputRequirement { + pub fn as_minifi_c_type(&self) -> minifi_input_requirement { + match self { + ProcessorInputRequirement::Required => minifi_input_requirement_MINIFI_INPUT_REQUIRED, + ProcessorInputRequirement::Allowed => minifi_input_requirement_MINIFI_INPUT_ALLOWED, + ProcessorInputRequirement::Forbidden => minifi_input_requirement_MINIFI_INPUT_FORBIDDEN, + } + } +} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs new file mode 100644 index 0000000000..e84f7fa108 --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs @@ -0,0 +1,146 @@ +use super::c_ffi_flow_file::CffiFlowFile; +use super::c_ffi_primitives::StringView; +use crate::api::ProcessContext; +use crate::api::controller_service::ControllerService; +use crate::c_ffi::{CffiLogger, StaticStrAsMinifiCStr}; +use crate::{ComponentIdentifier, EnableControllerService, MinifiError, Property}; +use minifi_native_sys::*; +use std::borrow::Cow; +use std::ffi::c_void; +use std::num::NonZeroU32; + +/// A safe wrapper around a `minifi_process_context` pointer. +pub struct CffiProcessContext<'a> { + ptr: *mut minifi_process_context, + _lifetime: std::marker::PhantomData<&'a ()>, +} + +impl<'a> CffiProcessContext<'a> { + pub fn new(ptr: *mut minifi_process_context) -> Self { + Self { + ptr, + _lifetime: std::marker::PhantomData, + } + } +} + +unsafe extern "C" fn get_property_callback( + output_option: *mut c_void, + property_c_value: minifi_string_view, +) { + unsafe { + let result_target = &mut *(output_option as *mut Option); + + if property_c_value.data.is_null() || property_c_value.length == 0 { + *result_target = None; + return; + } + + let value_slice = + std::slice::from_raw_parts(property_c_value.data as *const u8, property_c_value.length); + if let Ok(string_value) = String::from_utf8(value_slice.to_vec()) { + *result_target = Some(string_value); + } + } +} + +impl<'a> ProcessContext for CffiProcessContext<'a> { + type FlowFile = CffiFlowFile<'a>; // FlowFile shouldn't outlive the ProcessContext + fn get_property( + &self, + property: &Property, + flow_file: Option<&Self::FlowFile>, + ) -> Result, MinifiError> { + let ff_ptr = flow_file.map_or(std::ptr::null_mut(), |ff| ff.get_ptr()); + + let mut result: Option = None; + let property_name: StringView = StringView::new(property.name); + + #[allow(non_upper_case_globals)] + unsafe { + match minifi_process_context_get_property( + self.ptr, + property_name.as_raw(), + ff_ptr, + Some(get_property_callback), + &mut result as *mut _ as *mut c_void, + ) { + minifi_status_MINIFI_STATUS_SUCCESS => Ok(result), + minifi_status_MINIFI_STATUS_PROPERTY_NOT_SET => match property.is_required { + true => Err(MinifiError::MissingRequiredProperty(Cow::from( + property.name, + ))), + false => Ok(None), + }, + err_code => Err(MinifiError::StatusError(( + format!("minifi_process_context_get_property({:?})", property.name).into(), + NonZeroU32::new_unchecked(err_code), + ))), + } + } + } + + fn get_raw_controller_service( + &self, + property: &Property, + ) -> Result, MinifiError> + where + Cs: ComponentIdentifier + 'static, + { + let str_view = StringView::new(property.name); + + let mut controller_service_ptr: *mut minifi_controller_service = std::ptr::null_mut(); + + unsafe { + let get_cs_status = minifi_process_context_get_controller_service_from_property( + self.ptr, + str_view.as_raw(), + Cs::CLASS_NAME.as_minifi_c_type(), + &mut controller_service_ptr, + ); + if get_cs_status != minifi_status_MINIFI_STATUS_SUCCESS { + return Err(MinifiError::StatusError(( + format!( + "minifi_process_context_get_controller_service_from_property::<{:?}>({:?})", + Cs::CLASS_NAME, + property + ) + .into(), + NonZeroU32::new_unchecked(get_cs_status), + ))); + } + let cs_ref: &Cs = { + (controller_service_ptr as *const Cs) + .as_ref() + .expect("C returned a null pointer") + }; + Ok(Some(cs_ref)) + } + } + + fn get_controller_service(&self, property: &Property) -> Result, MinifiError> + where + Cs: EnableControllerService + ComponentIdentifier + 'static, + { + match self.get_raw_controller_service::>(property)? { + None => Ok(None), + Some(f) => Ok(f.get_implementation()), + } + } + + fn report_metrics(&self, metrics: Vec<(String, f64)>) -> Result<(), MinifiError>{ + let mut keys = Vec::new(); + let mut values = Vec::new(); + for (key, value) in &metrics { + keys.push(StringView::new(key)); + values.push(*value); + } + unsafe { + let err_code = minifi_process_context_report_metrics(self.ptr, keys.len(), keys.as_ptr() as *const minifi_string_view, values.as_ptr()); + if err_code != minifi_status_MINIFI_STATUS_SUCCESS { + return Err(MinifiError::StatusError(("report_metrics".into(), NonZeroU32::new_unchecked(err_code)))); + } + } + Ok(()) + } +} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs new file mode 100644 index 0000000000..3ec6a71237 --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs @@ -0,0 +1,554 @@ +use super::c_ffi_flow_file::CffiFlowFile; +use crate::MinifiError; +use crate::api::process_session::{IoState, OutputStream}; +use crate::api::{InputStream, ProcessSession}; +use crate::c_ffi::c_ffi_primitives::{ConvertMinifiStringView, StringView}; +use crate::c_ffi::c_ffi_streams::{CffiInputStream, CffiOutputStream}; +use minifi_native_sys::{minifi_status_MINIFI_STATUS_SUCCESS, minifi_string_view, minifi_process_session, minifi_output_stream, minifi_output_stream_write, minifi_io_status_MINIFI_IO_ERROR, minifi_process_session_write, minifi_process_session_create, minifi_process_session_get, minifi_process_session_transfer, minifi_process_session_remove, minifi_process_session_set_flow_file_attribute, minifi_process_session_get_flow_file_attribute, minifi_process_session_get_flow_file_attributes, minifi_io_status_MINIFI_IO_CANCEL, minifi_input_stream, minifi_input_stream_size, minifi_input_stream_read, minifi_process_session_read, minifi_process_session_get_flow_file_id}; +use std::ffi::{CString, c_void}; +use std::io::Read; +use std::num::NonZeroU32; +use std::os::raw::c_char; + +const _: () = { + if minifi_status_MINIFI_STATUS_SUCCESS != 0 { + panic!("minifi_status_MINIFI_STATUS_SUCCESS expected to be 0"); + } +}; + +pub struct CffiProcessSession<'a> { + ptr: *mut minifi_process_session, + // The lifetime ensures the session cannot outlive the `on_trigger` call. + _lifetime: std::marker::PhantomData<&'a ()>, +} + +impl<'a> CffiProcessSession<'a> { + pub fn new(ptr: *mut minifi_process_session) -> Self { + Self { + ptr, + _lifetime: std::marker::PhantomData, + } + } + + fn write_in_batches( + &self, + flow_file: &CffiFlowFile, + produce_batch: F, + ) -> Result<(), MinifiError> + where + F: FnMut(&mut [u8]) -> Option, + { + unsafe { + struct State<'b, F: FnMut(&mut [u8]) -> Option> { + callback: F, + buffer: &'b mut [u8], + } + + let mut buffer = [0u8; 8192]; + let mut state = State { + callback: produce_batch, + buffer: &mut buffer, + }; + + unsafe extern "C" fn cb Option>( + user_ctx: *mut c_void, + output_stream: *mut minifi_output_stream, + ) -> i64 { + unsafe { + let state = &mut *(user_ctx as *mut State); + let mut overall_writes = 0; + + // The user fills our provided buffer + while let Some(n) = (state.callback)(state.buffer) { + let written = minifi_output_stream_write( + output_stream, + state.buffer.as_ptr() as *const c_char, + n, + ); + + if written < 0 { + return minifi_io_status_MINIFI_IO_ERROR; + } + overall_writes += written; + } + overall_writes + } + } + + match minifi_process_session_write( + self.ptr, + flow_file.get_ptr(), + Some(cb::), + &mut state as *mut _ as *mut c_void, + ) { + #[allow(non_upper_case_globals)] + minifi_status_MINIFI_STATUS_SUCCESS => Ok(()), + error_code => Err(MinifiError::StatusError(( + "MinifiProcessSessionWrite".into(), + NonZeroU32::new_unchecked(error_code), + ))), + } + } + } +} + +impl<'a> ProcessSession for CffiProcessSession<'a> { + type FlowFile = CffiFlowFile<'a>; // FlowFile shouldn't outlive the Session + + fn create(&mut self) -> Result { + let ff_ptr = unsafe { minifi_process_session_create(self.ptr, std::ptr::null_mut()) }; + if ff_ptr.is_null() { + Err(MinifiError::UnknownError) + } else { + Ok(CffiFlowFile::new(ff_ptr)) + } + } + + fn get(&mut self) -> Option { + let ff_ptr = unsafe { minifi_process_session_get(self.ptr) }; + if ff_ptr.is_null() { + None + } else { + Some(CffiFlowFile::new(ff_ptr)) + } + } + + fn transfer(&self, flow_file: Self::FlowFile, relationship: &str) -> Result<(), MinifiError> { + let c_relationship = CString::new(relationship)?; + unsafe { + match minifi_process_session_transfer( + self.ptr, + flow_file.get_ptr(), + minifi_string_view { + data: c_relationship.as_ptr(), + length: c_relationship.as_bytes().len(), + }, + ) { + #[allow(non_upper_case_globals)] + minifi_status_MINIFI_STATUS_SUCCESS => Ok(()), + err_code => Err(MinifiError::StatusError(( + "MinifiProcessSessionTransfer".into(), + NonZeroU32::new_unchecked(err_code), + ))), + } + } + } + + fn remove(&mut self, flow_file: Self::FlowFile) -> Result<(), MinifiError> { + unsafe { + match minifi_process_session_remove(self.ptr, flow_file.get_ptr()) { + #[allow(non_upper_case_globals)] + minifi_status_MINIFI_STATUS_SUCCESS => Ok(()), + err_code => Err(MinifiError::StatusError(( + "MinifiProcessSessionRemove".into(), + NonZeroU32::new_unchecked(err_code), + ))), + } + } + } + + fn set_attribute( + &self, + flow_file: &mut Self::FlowFile, + attr_key: &str, + attr_value: &str, + ) -> Result<(), MinifiError> { + unsafe { + let attr_key_string_view = StringView::new(attr_key); + let attr_value_string_view = StringView::new(attr_value); + match minifi_process_session_set_flow_file_attribute( + self.ptr, + flow_file.get_ptr(), + attr_key_string_view.as_raw(), + &attr_value_string_view.as_raw(), + ) { + #[allow(non_upper_case_globals)] + minifi_status_MINIFI_STATUS_SUCCESS => Ok(()), + err_code => Err(MinifiError::StatusError(( + format!( + "MinifiProcessSessionFlowFileSetAttribute({}, {})", + attr_key, attr_value + ) + .into(), + NonZeroU32::new_unchecked(err_code), + ))), + } + } + } + + fn get_attribute(&self, flow_file: &Self::FlowFile, attr_key: &str) -> Option { + let mut attr_value: Option = None; + unsafe { + unsafe extern "C" fn cb( + rs_attr_value: *mut c_void, + minifi_attr_value: minifi_string_view, + ) { + unsafe { + let result_target = &mut *(rs_attr_value as *mut Option); + *result_target = minifi_attr_value.as_string().ok() + } + } + + let attr_key_string_view = StringView::new(attr_key); + minifi_process_session_get_flow_file_attribute( + self.ptr, + flow_file.get_ptr(), + attr_key_string_view.as_raw(), + Some(cb), + &mut attr_value as *mut _ as *mut c_void, + ); + } + attr_value + } + + fn on_attributes( + &self, + flow_file: &Self::FlowFile, + process_attr: F, + ) -> bool { + struct OnAttrHelper { + result: bool, + process_attr: F, + } + + let mut on_attr_helper = OnAttrHelper { + result: true, + process_attr, + }; + + unsafe extern "C" fn get_attributes_callback<'b, F: FnMut(&str, &str)>( + user_ctx: *mut c_void, + minifi_attr_key: minifi_string_view, + minifi_attr_val: minifi_string_view, + ) { + unsafe { + let helper = &mut *(user_ctx as *mut OnAttrHelper); + let attr_key = minifi_attr_key.as_str(); + let attr_value = minifi_attr_val.as_str(); + if attr_key.is_err() || attr_value.is_err() { + helper.result = false; + return; + } + (helper.process_attr)(attr_key.unwrap(), attr_value.unwrap()); + } + } + + unsafe { + minifi_process_session_get_flow_file_attributes( + self.ptr, + flow_file.get_ptr(), + Some(get_attributes_callback::), + &mut on_attr_helper as *mut _ as *mut c_void, + ) + } + on_attr_helper.result + } + + fn write(&self, flow_file: &Self::FlowFile, data: &[u8]) -> Result<(), MinifiError> { + let mut dt: Option<&[u8]> = Some(data); + unsafe { + unsafe extern "C" fn cb( + user_ctx: *mut c_void, + output_stream: *mut minifi_output_stream, + ) -> i64 { + unsafe { + let result_target = &mut *(user_ctx as *mut Option<&[u8]>); + if result_target.is_none() { + return minifi_io_status_MINIFI_IO_ERROR; + } + + minifi_output_stream_write( + output_stream, + result_target.unwrap().as_ptr() as *const c_char, + result_target.unwrap().len(), + ) + } + } + + match minifi_process_session_write( + self.ptr, + flow_file.get_ptr(), + Some(cb), + &mut dt as *mut _ as *mut c_void, + ) { + #[allow(non_upper_case_globals)] + minifi_status_MINIFI_STATUS_SUCCESS => Ok(()), + err_code => Err(MinifiError::StatusError(( + "MinifiProcessSessionWrite".into(), + NonZeroU32::new_unchecked(err_code), + ))), + } + } + } + + fn write_lazy<'b>( + &self, + flow_file: &Self::FlowFile, + mut stream: Box, + ) -> Result<(), MinifiError> { + self.write_in_batches(flow_file, |buffer| { + match stream.read(buffer) { + Ok(0) => None, // EOF + Ok(n) => Some(n), + Err(_e) => None, // Signal failure/EOF + } + }) + } + + fn write_stream(&self, flow_file: &Self::FlowFile, callback: F) -> Result + where + F: FnOnce(&mut dyn OutputStream) -> Result<(R, IoState), MinifiError>, + { + struct WriteCallbackCtx { + callback: Option, + result: Option>, + } + + let mut ctx = WriteCallbackCtx { + callback: Some(callback), + result: None, + }; + + unsafe extern "C" fn write_cb( + user_data: *mut c_void, + stream_ptr: *mut minifi_output_stream, + ) -> i64 + where + F: FnOnce(&mut dyn OutputStream) -> Result<(R, IoState), MinifiError>, + { + unsafe { + let ctx = &mut *(user_data as *mut WriteCallbackCtx); + + let mut writer = CffiOutputStream::new(stream_ptr); + + if let Some(cb) = ctx.callback.take() { + let (cb_result, state) = match cb(&mut writer) { + Ok((f, b)) => (Ok(f), b), + Err(e) => (Err(e), IoState::Cancel), + }; + let is_err = cb_result.is_err(); + ctx.result = Some(cb_result); + if is_err { + return minifi_io_status_MINIFI_IO_ERROR; + } + if state == IoState::Cancel { + return minifi_io_status_MINIFI_IO_CANCEL; + } + } + + writer.written_bytes() as i64 + } + } + + unsafe { + let session_status = minifi_process_session_write( + self.ptr, + flow_file.get_ptr(), + Some(write_cb::), + &mut ctx as *mut _ as *mut c_void, + ); + if session_status != minifi_status_MINIFI_STATUS_SUCCESS { + return Err(MinifiError::StatusError(( + "MinifiProcessSessionWrite".into(), + NonZeroU32::new_unchecked(session_status), + ))); + } + } + + ctx.result + .expect("Agent returned with success, so ctx.result should be set") + } + + fn read(&self, flow_file: &Self::FlowFile) -> Option> { + let mut output: Option> = None; + unsafe { + unsafe extern "C" fn cb( + output_option: *mut c_void, + input_stream: *mut minifi_input_stream, + ) -> i64 { + unsafe { + let result_target = &mut *(output_option as *mut Option>); + + let stream_size = minifi_input_stream_size(input_stream); + if stream_size == 0 { + *result_target = None; + return 0; + } + let mut buffer: Vec = Vec::with_capacity(stream_size); + + let bytes_read = minifi_input_stream_read( + input_stream, + buffer.as_mut_ptr() as *mut c_char, + stream_size, + ); + + if bytes_read < 0 { + *result_target = None; + return bytes_read; + } + + buffer.set_len(bytes_read as usize); + + *result_target = Some(buffer); + bytes_read + } + } + + minifi_process_session_read( + self.ptr, + flow_file.get_ptr(), + Some(cb), + &mut output as *mut _ as *mut c_void, + ); + } + output + } + + fn read_stream(&self, flow_file: &Self::FlowFile, callback: F) -> Result + where + F: FnOnce(&mut dyn InputStream) -> Result, + { + unsafe { + struct CallbackCtx { + callback: Option, + result: Option>, + } + let mut ctx = CallbackCtx { + callback: Some(callback), + result: None, + }; + + unsafe extern "C" fn read_cb( + user_data: *mut c_void, + stream_ptr: *mut minifi_input_stream, + ) -> i64 + where + F: FnOnce(&mut dyn InputStream) -> Result, + { + unsafe { + let ctx = &mut *(user_data as *mut CallbackCtx); + + let mut reader = CffiInputStream::new(stream_ptr); + + if let Some(cb) = ctx.callback.take() { + ctx.result = Some(cb(&mut reader)); + } + + reader.total_bytes_read() as i64 + } + } + + minifi_process_session_read( + self.ptr, + flow_file.get_ptr(), + Some(read_cb::), + &mut ctx as *mut _ as *mut c_void, + ); + + ctx.result + .expect("Agent returned with success, so ctx.result should be set") + } + } + + fn read_in_batches( + &self, + flow_file: &Self::FlowFile, + batch_size: usize, + process_batch: F, + ) -> Result<(), MinifiError> + where + F: FnMut(&[u8]) -> Result<(), MinifiError>, + { + struct BatchReadHelper + where + F: FnMut(&[u8]) -> Result<(), MinifiError>, + { + batch_size: usize, + process_batch: F, + } + + let mut batch_helper = BatchReadHelper { + batch_size, + process_batch, + }; + unsafe { + unsafe extern "C" fn cb( + output_option: *mut c_void, + input_stream: *mut minifi_input_stream, + ) -> i64 + where + F: FnMut(&[u8]) -> Result<(), MinifiError>, + { + unsafe { + let batch_helper = &mut *(output_option as *mut BatchReadHelper); + + let mut remaining_size = minifi_input_stream_size(input_stream); + let mut overall_read = 0; + while remaining_size > 0 { + let read_size = remaining_size.min(batch_helper.batch_size); + let mut buffer: Vec = Vec::with_capacity(read_size); + + let bytes_read = minifi_input_stream_read( + input_stream, + buffer.as_mut_ptr() as *mut c_char, + read_size, + ); + if bytes_read < 0 || bytes_read > read_size as i64 { + return minifi_io_status_MINIFI_IO_ERROR; + } + + buffer.set_len(bytes_read as usize); + + match (batch_helper.process_batch)(&*buffer) { + Ok(_) => {} + Err(err) => { + eprintln!("Error during read_in_batch {:?}", err); + return minifi_io_status_MINIFI_IO_ERROR; + } + } + remaining_size -= bytes_read as usize; + overall_read += bytes_read; + } + overall_read + } + } + + match minifi_process_session_read( + self.ptr, + flow_file.get_ptr(), + Some(cb::), + &mut batch_helper as *mut _ as *mut c_void, + ) { + #[allow(non_upper_case_globals)] + minifi_status_MINIFI_STATUS_SUCCESS => Ok(()), + status_code => Err(MinifiError::StatusError(( + "minifi_process_session_read".into(), + NonZeroU32::new_unchecked(status_code), + ))), + } + } + } + + fn get_flow_file_id(&self, flow_file: &Self::FlowFile) -> Result { + let mut attr_value: Option = None; + unsafe { + unsafe extern "C" fn cb( + rs_flow_file_id: *mut c_void, + minifi_flow_file_id: minifi_string_view, + ) { + unsafe { + let result_target = &mut *(rs_flow_file_id as *mut Option); + *result_target = minifi_flow_file_id.as_string().ok() + } + } + + minifi_process_session_get_flow_file_id( + self.ptr, + flow_file.get_ptr(), + Some(cb), + &mut attr_value as *mut _ as *mut c_void, + ); + } + attr_value.ok_or(MinifiError::UnknownError) + } +} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs new file mode 100644 index 0000000000..67cacd566d --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs @@ -0,0 +1,266 @@ +use std::ffi::c_void; +use std::ptr; + +use super::c_ffi_primitives::{StaticStrAsMinifiCStr}; +use super::c_ffi_process_context::CffiProcessContext; +use super::c_ffi_process_session::CffiProcessSession; +use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; +use crate::api::{ProcessorInputRequirement, RawProcessor, ThreadingModel}; +use crate::c_ffi::CffiLogger; +use crate::c_ffi::c_ffi_output_attribute::COutputAttributes; +use crate::c_ffi::c_ffi_property::CProperties; +use crate::{ + ComponentIdentifier, Concurrent, Exclusive, + LogLevel, OutputAttribute, Processor, ProcessorDefinition, Property, Schedule, +}; +use crate::{OnTriggerResult, Relationship}; +use minifi_native_sys::*; + +pub trait DispatchOnTrigger { + unsafe fn dispatch_on_trigger( + processor: *mut c_void, + context: *mut minifi_process_context, + session: *mut minifi_process_session, + ) -> minifi_status; +} + +impl DispatchOnTrigger for T +where + T: MultiThreadedTrigger, +{ + unsafe fn dispatch_on_trigger( + processor_ptr: *mut c_void, + context_ptr: *mut minifi_process_context, + session_ptr: *mut minifi_process_session, + ) -> minifi_status { + unsafe { + let processor = &*(processor_ptr as *const T); + let mut context = CffiProcessContext::new(context_ptr); + let mut session = CffiProcessSession::new(session_ptr); + match processor.on_trigger(&mut context, &mut session) { + Ok(OnTriggerResult::Ok) => minifi_status_MINIFI_STATUS_SUCCESS, + Ok(OnTriggerResult::Yield) => minifi_status_MINIFI_STATUS_PROCESSOR_YIELD, + Err(minifi_error) => { + processor.log( + LogLevel::Error, + format_args!("Error during on_trigger {}", minifi_error), + ); + minifi_error.to_status() + } + } + } + } +} + +impl DispatchOnTrigger for T +where + T: SingleThreadedTrigger, +{ + unsafe fn dispatch_on_trigger( + processor_ptr: *mut c_void, + context_ptr: *mut minifi_process_context, + session_ptr: *mut minifi_process_session, + ) -> minifi_status { + unsafe { + let processor = &mut *(processor_ptr as *mut T); + let mut context = CffiProcessContext::new(context_ptr); + let mut session = CffiProcessSession::new(session_ptr); + match processor.on_trigger(&mut context, &mut session) { + Ok(OnTriggerResult::Ok) => minifi_status_MINIFI_STATUS_SUCCESS, + Ok(OnTriggerResult::Yield) => minifi_status_MINIFI_STATUS_PROCESSOR_YIELD, + Err(error_code) => error_code.to_status(), + } + } + } +} + +pub struct RawProcessorDefinition +where + T: RawProcessor + DispatchOnTrigger, +{ + name: &'static str, + description_text: &'static str, + input_requirement: ProcessorInputRequirement, + supports_dynamic_properties: bool, + supports_dynamic_relationships: bool, + + c_output_attributes: COutputAttributes, + c_relationships: Vec, + c_properties: CProperties, + + _phantom: std::marker::PhantomData, +} + +impl RawProcessorDefinition +where + T: RawProcessor + DispatchOnTrigger, +{ + pub fn new( + name: &'static str, + description_text: &'static str, + input_requirement: ProcessorInputRequirement, + supports_dynamic_properties: bool, + supports_dynamic_relationships: bool, + output_attributes: &'static [OutputAttribute], + relationships: &'static [Relationship], + properties: &'static [Property], + ) -> Self { + let c_relationships = Relationship::create_c_vec(relationships); + let c_properties = Property::create_c_properties(properties); + let c_output_attributes = COutputAttributes::new(output_attributes); + + Self { + name, + description_text, + input_requirement, + supports_dynamic_properties, + supports_dynamic_relationships, + c_output_attributes, + c_relationships, + c_properties, + _phantom: std::marker::PhantomData, + } + } + + unsafe extern "C" fn create_processor(metadata: minifi_processor_metadata) -> *mut c_void { + let logger = CffiLogger::new(metadata.logger); + let processor = Box::new(T::new(logger)); + Box::into_raw(processor) as *mut c_void + } + + unsafe extern "C" fn destroy_processor(processor_ptr: *mut c_void) { + unsafe { + if !processor_ptr.is_null() { + let _ = Box::from_raw(processor_ptr as *mut T); + } + } + } + + unsafe extern "C" fn on_trigger_processor( + processor_ptr: *mut c_void, + context_ptr: *mut minifi_process_context, + session_ptr: *mut minifi_process_session, + ) -> minifi_status { + unsafe { + >::dispatch_on_trigger( + processor_ptr, + context_ptr, + session_ptr, + ) + } + } + + unsafe extern "C" fn on_schedule_processor( + processor_ptr: *mut c_void, + context_ptr: *mut minifi_process_context, + ) -> minifi_status { + unsafe { + let processor = &mut *(processor_ptr as *mut T); + let context = CffiProcessContext::new(context_ptr); + match processor.on_schedule(&context) { + Ok(_) => 0, + Err(error_code) => { + processor.log( + LogLevel::Error, + format_args!("Error during on_schedule: {}", error_code), + ); + error_code.to_status() + } + } + } + } + + unsafe extern "C" fn on_unschedule_processor(processor_ptr: *mut c_void) { + unsafe { + let processor = &mut *(processor_ptr as *mut T); + processor.on_unschedule(); + } + } +} + +#[derive(Debug)] +pub struct ProcessorClassDefinition<'a> { + inner: minifi_processor_class_definition, + _marker: std::marker::PhantomData<&'a ()>, +} + +impl<'a> ProcessorClassDefinition<'a> { + pub(crate) fn new(inner: minifi_processor_class_definition) -> Self { + Self { + inner, + _marker: std::marker::PhantomData, + } + } + + pub unsafe fn as_raw(&self) -> minifi_processor_class_definition { + self.inner + } +} + +pub trait DynRawProcessorDefinition { + fn class_description(&'_ self) -> ProcessorClassDefinition<'_>; +} + +impl DynRawProcessorDefinition for RawProcessorDefinition +where + T: RawProcessor + DispatchOnTrigger, +{ + fn class_description(&'_ self) -> ProcessorClassDefinition<'_> { + unsafe { + ProcessorClassDefinition::new(minifi_processor_class_definition { + full_name: self.name.as_minifi_c_type(), + description: self.description_text.as_minifi_c_type(), + properties_count: self.c_properties.len(), + properties_ptr: self.c_properties.get_ptr(), + dynamic_properties_count: 0, + dynamic_properties_ptr: ptr::null(), + relationships_count: self.c_relationships.len(), + relationships_ptr: self.c_relationships.as_ptr(), + output_attributes_count: self.c_output_attributes.len(), + output_attributes_ptr: self.c_output_attributes.get_ptr(), + supports_dynamic_properties: self.supports_dynamic_properties, + supports_dynamic_relationships: self.supports_dynamic_relationships, + input_requirement: self.input_requirement.as_minifi_c_type(), + is_single_threaded: T::Threading::IS_EXCLUSIVE, + callbacks: minifi_processor_callbacks { + create: Some(Self::create_processor), + destroy: Some(Self::destroy_processor), + trigger: Some(Self::on_trigger_processor), + schedule: Some(Self::on_schedule_processor), + unschedule: Some(Self::on_unschedule_processor), + }, + }) + } + } +} + +pub trait RawRegisterableProcessor { + fn get_definition() -> Box; +} + +impl RawRegisterableProcessor + for Processor +where + Threading: ThreadingModel + 'static, + Implementation: Schedule + + ComponentIdentifier + + ProcessorDefinition + + 'static, + Processor: + RawProcessor + DispatchOnTrigger, +{ + fn get_definition() -> Box { + Box::new(RawProcessorDefinition::< + Processor, + >::new( + Implementation::CLASS_NAME, + Implementation::DESCRIPTION, + Implementation::INPUT_REQUIREMENT, + Implementation::SUPPORTS_DYNAMIC_PROPERTIES, + Implementation::SUPPORTS_DYNAMIC_RELATIONSHIPS, + Implementation::OUTPUT_ATTRIBUTES, + Implementation::RELATIONSHIPS, + Implementation::PROPERTIES, + )) + } +} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs new file mode 100644 index 0000000000..9f5bf153bf --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs @@ -0,0 +1,49 @@ +use minifi_native_sys::minifi_processor_class_definition; +use crate::c_ffi::RawRegisterableProcessor; +use crate::c_ffi::c_ffi_processor_definition::DynRawProcessorDefinition; + +pub struct CffiProcessorList { + processor_definitions: Vec>, + minifi_processor_class_description_list: Vec, +} + +impl CffiProcessorList { + pub fn new() -> Self { + Self { + processor_definitions: Vec::new(), + minifi_processor_class_description_list: Vec::new(), + } + } + + pub fn add(&mut self) { + self.add_processor_definition(T::get_definition()) + } + + pub fn add_processor_definition( + &mut self, + processor_definition: Box, + ) { + unsafe { + self.processor_definitions.push(processor_definition); + self.minifi_processor_class_description_list.push( + self.processor_definitions + .last() + .unwrap() + .class_description() + .as_raw(), + ); + } + } + + pub fn get_processor_count(&self) -> usize { + assert_eq!( + self.processor_definitions.len(), + self.minifi_processor_class_description_list.len() + ); + self.minifi_processor_class_description_list.len() + } + + pub unsafe fn get_processor_ptr(&self) -> *const minifi_processor_class_definition { + self.minifi_processor_class_description_list.as_ptr() + } +} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs new file mode 100644 index 0000000000..11ebe4c5ca --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs @@ -0,0 +1,132 @@ +use super::c_ffi_primitives::StaticStrAsMinifiCStr; +use crate::{Property, StandardPropertyValidator}; +use minifi_native_sys::{minifi_property_definition, minifi_validator, minifi_string_view, minifi_validator_MINIFI_VALIDATOR_ALWAYS_VALID, minifi_validator_MINIFI_VALIDATOR_BOOLEAN, minifi_validator_MINIFI_VALIDATOR_DATA_SIZE, minifi_validator_MINIFI_VALIDATOR_INTEGER, minifi_validator_MINIFI_VALIDATOR_NON_BLANK, minifi_validator_MINIFI_VALIDATOR_PORT, minifi_validator_MINIFI_VALIDATOR_TIME_PERIOD, minifi_validator_MINIFI_VALIDATOR_UNSIGNED_INTEGER}; +use std::ptr; + +#[allow(dead_code)] // these c_ vecs are holding the values referenced from the properties, so they live long enough for registration +pub struct CProperties { + c_default_values: Vec, + c_allowed_values: Vec>, + c_allowed_types: Vec, + properties: Vec, +} + +impl CProperties { + pub(crate) fn new( + c_default_values: Vec, + c_allowed_values: Vec>, + c_allowed_types: Vec, + properties: Vec, + ) -> Self { + Self { + c_default_values, + c_allowed_values, + c_allowed_types, + properties, + } + } + + pub(crate) fn len(&self) -> usize { + self.properties.len() + } + + pub(crate) unsafe fn get_ptr(&self) -> *const minifi_property_definition { + self.properties.as_ptr() + } +} + +impl Property { + fn create_c_default_value_holder(properties: &[Self]) -> Vec { + properties + .iter() + .map(|p| match p.default_value { + Some(dv) => dv.as_minifi_c_type(), + None => minifi_string_view { + data: ptr::null(), + length: 0, + }, + }) + .collect() + } + + fn create_c_allowed_values_vec_vec(properties: &[Self]) -> Vec> { + properties + .iter() + .map(|p| { + p.allowed_values + .iter() + .map(|av| av.as_minifi_c_type()) + .collect() + }) + .collect() + } + + fn create_c_allowed_types_vec(properties: &[Self]) -> Vec { + properties + .iter() + .map(|p| p.allowed_type.as_minifi_c_type()) + .collect() + } + + pub(crate) fn create_c_properties(properties: &[Self]) -> CProperties { + let c_default_values = Property::create_c_default_value_holder(properties); + let c_allowed_values = Property::create_c_allowed_values_vec_vec(properties); + let c_allowed_types = Property::create_c_allowed_types_vec(properties); + assert_eq!(c_default_values.len(), properties.len()); + assert_eq!(c_allowed_values.len(), properties.len()); + assert_eq!(c_allowed_types.len(), properties.len()); + + let c_properties = properties + .iter() + .zip(c_default_values.iter()) + .zip(c_allowed_values.iter()) + .zip(c_allowed_types.iter()) + .map(|(((property, def_value), allowed_values), allowed_type)| { + minifi_property_definition { + name: property.name.as_minifi_c_type(), + display_name: property.name.as_minifi_c_type(), + description: property.description.as_minifi_c_type(), + is_required: property.is_required, + is_sensitive: property.is_sensitive, + default_value: def_value, + allowed_values_count: allowed_values.len(), + allowed_values_ptr: allowed_values.as_ptr(), + validator: property.validator.as_minifi_c_type(), + allowed_type, + supports_expression_language: property.supports_expr_lang, + } + }) + .collect(); + CProperties::new( + c_default_values, + c_allowed_values, + c_allowed_types, + c_properties, + ) + } +} + +impl StandardPropertyValidator { + pub(crate) fn as_minifi_c_type(&self) -> minifi_validator { + match self { + StandardPropertyValidator::AlwaysValidValidator => { + minifi_validator_MINIFI_VALIDATOR_ALWAYS_VALID + } + StandardPropertyValidator::NonBlankValidator => { + minifi_validator_MINIFI_VALIDATOR_NON_BLANK + } + StandardPropertyValidator::TimePeriodValidator => { + minifi_validator_MINIFI_VALIDATOR_TIME_PERIOD + } + StandardPropertyValidator::BoolValidator => minifi_validator_MINIFI_VALIDATOR_BOOLEAN, + StandardPropertyValidator::I64Validator => minifi_validator_MINIFI_VALIDATOR_INTEGER, + StandardPropertyValidator::U64Validator => { + minifi_validator_MINIFI_VALIDATOR_UNSIGNED_INTEGER + } + StandardPropertyValidator::DataSizeValidator => { + minifi_validator_MINIFI_VALIDATOR_DATA_SIZE + } + StandardPropertyValidator::PortValidator => minifi_validator_MINIFI_VALIDATOR_PORT, + } + } +} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_relationship.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_relationship.rs new file mode 100644 index 0000000000..86d3ee9083 --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_relationship.rs @@ -0,0 +1,15 @@ +use super::c_ffi_primitives::StaticStrAsMinifiCStr; +use crate::Relationship; +use minifi_native_sys::minifi_relationship_definition; + +impl Relationship { + pub(crate) fn create_c_vec(relationships: &[Self]) -> Vec { + relationships + .iter() + .map(|r| minifi_relationship_definition { + name: r.name.as_minifi_c_type(), + description: r.description.as_minifi_c_type(), + }) + .collect() + } +} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs new file mode 100644 index 0000000000..64fbaf95f1 --- /dev/null +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs @@ -0,0 +1,114 @@ +use std::io::{BufRead, Error, ErrorKind, Read}; +use minifi_native_sys::{minifi_input_stream, minifi_input_stream_read, minifi_output_stream, minifi_output_stream_write}; + +#[derive(Debug)] +pub struct CffiInputStream<'a> { + ptr: *mut minifi_input_stream, + buffer: [u8; 8192], + pos: usize, + cap: usize, + total_read: usize, + _marker: std::marker::PhantomData<&'a ()>, +} + +unsafe impl<'a> Send for CffiInputStream<'a> {} + +impl<'a> CffiInputStream<'a> { + pub fn new(ptr: *mut minifi_input_stream) -> Self { + Self { + ptr, + buffer: [0u8; 8192], + pos: 0, + cap: 0, + total_read: 0, + _marker: std::marker::PhantomData, + } + } + + pub fn total_bytes_read(&self) -> usize { + self.total_read + } +} + +impl<'a> Read for CffiInputStream<'a> { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + // Delegate to the BufRead implementation to ensure consistency + let nread = { + let mut rem = self.fill_buf()?; + rem.read(buf)? + }; + self.consume(nread); + Ok(nread) + } +} + +impl<'a> BufRead for CffiInputStream<'a> { + fn fill_buf(&mut self) -> std::io::Result<&[u8]> { + if self.pos >= self.cap { + unsafe { + let ret = minifi_input_stream_read( + self.ptr, + self.buffer.as_mut_ptr() as *mut std::ffi::c_char, + self.buffer.len(), + ); + if ret < 0 { + return Err(Error::new(ErrorKind::Other, "Minifi Read Error")); + } + self.cap = ret as usize; + self.pos = 0; + } + } + Ok(&self.buffer[self.pos..self.cap]) + } + + fn consume(&mut self, amount: usize) { + let actual_consumed = std::cmp::min(amount, self.cap - self.pos); + self.pos += actual_consumed; + self.total_read += actual_consumed; + } +} + +#[derive(Debug)] +pub struct CffiOutputStream<'a> { + ptr: *mut minifi_output_stream, + written_bytes: usize, + _marker: std::marker::PhantomData<&'a ()>, +} + +impl<'a> CffiOutputStream<'a> { + pub(crate) fn new(ptr: *mut minifi_output_stream) -> Self { + Self { + ptr, + written_bytes: 0, + _marker: std::marker::PhantomData, + } + } +} + +impl<'a> CffiOutputStream<'a> { + pub fn written_bytes(&self) -> usize { + self.written_bytes + } +} + +unsafe impl<'a> Send for CffiOutputStream<'a> {} + +impl<'a> std::io::Write for CffiOutputStream<'a> { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + unsafe { + let ret = minifi_output_stream_write( + self.ptr, + buf.as_ptr() as *const std::ffi::c_char, + buf.len(), + ); + if ret < 0 { + return Err(Error::new(ErrorKind::Other, "Minifi Write Error")); + } + self.written_bytes += ret as usize; + Ok(ret as usize) + } + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) // Handled by C++ session commit + } +} diff --git a/minifi_rust/minifi_native/src/lib.rs b/minifi_rust/minifi_native/src/lib.rs new file mode 100644 index 0000000000..97286e3337 --- /dev/null +++ b/minifi_rust/minifi_native/src/lib.rs @@ -0,0 +1,116 @@ +mod api; +pub mod c_ffi; +pub mod mock; + +pub use api::errors::MinifiError; + +pub use api::component_definition_traits::{ + ComponentIdentifier, ControllerServiceDefinition, ProcessorDefinition, +}; +pub use api::controller_service::{ControllerService, EnableControllerService}; +pub use api::processor_wrappers::complex_processor::{ComplexProcessorType, MutTrigger, Trigger}; +pub use api::processor_wrappers::flow_file_source::{ + FlowFileSource, FlowFileSourceProcessorType, GeneratedFlowFile, +}; +pub use api::processor_wrappers::flow_file_stream_transform::{ + FlowFileStreamTransform, FlowFileStreamTransformProcessorType, MutFlowFileStreamTransform, + TransformStreamResult, +}; +pub use api::processor_wrappers::flow_file_transform::{ + FlowFileTransform, FlowFileTransformProcessorType, TransformedFlowFile, +}; + +pub use api::processor_wrappers::utils::flow_file_content::Content; + +pub use api::processor::{Processor, Schedule}; + +pub use api::raw_processor::{Concurrent, Exclusive}; + +pub use api::logger::{LogLevel, Logger}; + +pub use api::property::{GetControllerService, GetProperty, Property}; + +pub use api::process_session::IoState; + +pub use api::attribute::{GetAttribute, OutputAttribute}; + +pub use api::{ + FlowFile, InputStream, OnTriggerResult, OutputStream, ProcessContext, ProcessSession, + ProcessorInputRequirement, Relationship, StandardPropertyValidator, +}; + +pub use minifi_native_macros as macros; +pub use minifi_native_sys as sys; +pub use mock::{ + MockControllerServiceContext, MockFlowFile, MockLogger, MockProcessContext, MockProcessSession, + StdLogger, +}; + +#[unsafe(no_mangle)] +#[allow(non_upper_case_globals)] +#[cfg_attr(target_os = "linux", unsafe(link_section = ".rodata"))] +#[cfg_attr(target_os = "macos", unsafe(link_section = "__DATA,__const"))] +#[cfg_attr(target_os = "windows", unsafe(link_section = ".rdata"))] +pub static MinifiApiVersion: u32 = minifi_native_sys::MINIFI_API_VERSION; + +/// Defines the required MinifiInitExtension C function to register the listed processors and controller services +#[macro_export] +macro_rules! declare_minifi_extension { + ( + // Match a tuple of three types for each processor + processors: [ $( ($kind:ty, $thread:ty, $impl:ty) ),* $(,)? ], + // Match a single type for each controller service + controllers: [ $( $ctrl:ty ),* $(,)? ] + ) => { + + #[unsafe(no_mangle)] + pub extern "C" fn minifi_init_extension( + extension_context: *mut minifi_native::sys::minifi_extension_context + ) { + + use minifi_native::c_ffi::StaticStrAsMinifiCStr; + use minifi_native::c_ffi::RawRegisterableProcessor; + use minifi_native::c_ffi::RegisterableControllerService; + unsafe { + let extension_definition = minifi_native::sys::minifi_extension_definition { + name: env!("CARGO_PKG_NAME").as_minifi_c_type(), + version: env!("CARGO_PKG_VERSION").as_minifi_c_type(), + deinit: None, + user_data: std::ptr::null_mut(), + }; + + let extension = minifi_native::sys::minifi_register_extension(extension_context, &extension_definition); + + + $( + { + let processor_def = minifi_native::Processor::< + $impl, + $kind, + $thread, + minifi_native::c_ffi::CffiLogger + >::get_definition(); + + let proc_desc = processor_def.class_description(); + + minifi_native::sys::minifi_register_processor(extension, &proc_desc.as_raw()); + } + )* + + $( + { + let controller_def = + minifi_native::ControllerService::< + $ctrl, + minifi_native::c_ffi::CffiLogger + >::get_definition(); + + let controller_desc = controller_def.class_description(); + + minifi_native::sys::minifi_register_controller_service(extension, &controller_desc.as_raw()); + } + )* + } + } + }; +} diff --git a/minifi_rust/minifi_native/src/mock.rs b/minifi_rust/minifi_native/src/mock.rs new file mode 100644 index 0000000000..e98b637468 --- /dev/null +++ b/minifi_rust/minifi_native/src/mock.rs @@ -0,0 +1,12 @@ +mod mock_controller_service_context; +mod mock_flow_file; +mod mock_logger; +mod mock_process_context; +mod mock_process_session; + +pub use mock_controller_service_context::MockControllerServiceContext; +pub use mock_flow_file::MockFlowFile; +pub use mock_logger::MockLogger; +pub use mock_logger::StdLogger; +pub use mock_process_context::MockProcessContext; +pub use mock_process_session::MockProcessSession; diff --git a/minifi_rust/minifi_native/src/mock/mock_controller_service_context.rs b/minifi_rust/minifi_native/src/mock/mock_controller_service_context.rs new file mode 100644 index 0000000000..e24c0bce6d --- /dev/null +++ b/minifi_rust/minifi_native/src/mock/mock_controller_service_context.rs @@ -0,0 +1,20 @@ +use crate::mock::mock_process_context::MockPropertyMap; +use crate::{GetProperty, MinifiError, Property}; + +pub struct MockControllerServiceContext { + pub properties: MockPropertyMap, +} + +impl GetProperty for MockControllerServiceContext { + fn get_property(&self, property: &Property) -> Result, MinifiError> { + self.properties.get_property(property, None) + } +} + +impl MockControllerServiceContext { + pub fn new() -> Self { + Self { + properties: MockPropertyMap::new(), + } + } +} diff --git a/minifi_rust/minifi_native/src/mock/mock_flow_file.rs b/minifi_rust/minifi_native/src/mock/mock_flow_file.rs new file mode 100644 index 0000000000..06b2c61966 --- /dev/null +++ b/minifi_rust/minifi_native/src/mock/mock_flow_file.rs @@ -0,0 +1,45 @@ +use crate::api::FlowFile; +use std::cell::RefCell; +use std::collections::HashMap; + +pub struct MockFlowFile { + pub content: RefCell>, + pub attributes: HashMap, + pub id: String, +} + +impl FlowFile for MockFlowFile {} + +impl MockFlowFile { + pub fn new() -> MockFlowFile { + MockFlowFile { + content: RefCell::new(Vec::new()), + attributes: HashMap::new(), + id: "67e55044-10b1-426f-9247-bb680e5fe0c8".to_string(), // TODO generate something? + } + } + + pub fn with_content(content: &[u8]) -> MockFlowFile { + Self { + content: RefCell::new(content.to_vec()), + attributes: HashMap::new(), + id: "67e55044-10b1-426f-9247-bb680e5fe0c8".to_string(), // TODO generate something? + } + } + + pub fn content_len(&self) -> usize { + self.content.borrow().len() + } + + pub fn content_eq(&self, other: S) -> bool + where + S: Into, + { + let my_content = self.content.borrow(); + *my_content == other.into().as_bytes() + } + + pub fn get_stream(&self) -> std::io::Cursor> { + std::io::Cursor::new(self.content.borrow().clone()) + } +} diff --git a/minifi_rust/minifi_native/src/mock/mock_logger.rs b/minifi_rust/minifi_native/src/mock/mock_logger.rs new file mode 100644 index 0000000000..a6451ac9b4 --- /dev/null +++ b/minifi_rust/minifi_native/src/mock/mock_logger.rs @@ -0,0 +1,83 @@ +use crate::LogLevel::Trace; +use crate::api::LogLevel; +use crate::api::Logger; +use std::fmt; +use std::sync::Mutex; + +#[derive(Debug)] +pub struct MockLogger { + pub logs: Mutex>, + pub log_level: LogLevel, +} + +impl Logger for MockLogger { + fn log(&self, level: LogLevel, args: fmt::Arguments) { + let message = fmt::format(args); + let mut logs_guard = self.logs.lock().unwrap(); + logs_guard.push((level, message.to_string())); + } + + fn should_log(&self, level: LogLevel) -> bool { + level >= self.log_level + } +} + +impl MockLogger { + pub fn new() -> Self { + MockLogger { + logs: Mutex::new(Vec::new()), + log_level: Trace, + } + } +} + +/// For easier debugging +#[derive(Debug)] +pub struct StdLogger { + pub log_level: LogLevel, +} + +impl Logger for StdLogger { + fn log(&self, level: LogLevel, args: fmt::Arguments) { + let message = fmt::format(args); + println!("[{}] {}", level, message); + } + + fn should_log(&self, level: LogLevel) -> bool { + level >= self.log_level + } +} + +#[cfg(test)] +mod tests { + use crate::api::logger::Logger; + use crate::{LogLevel, MockLogger, error, trace}; + + #[test] + fn test_macro_laziness() { + let mut mock_logger = MockLogger::new(); + mock_logger.log_level = LogLevel::Warn; + + let mut call_count = 0; + + trace!( + mock_logger, + "This is a trace message {}", + || -> u32 { + call_count += 1; + call_count + }() + ); + error!( + mock_logger, + "This is an error message {}", + || -> u32 { + call_count += 1; + call_count + }() + ); + + assert_eq!(mock_logger.logs.lock().unwrap().len(), 1); + assert_eq!(call_count, 1); + } +} diff --git a/minifi_rust/minifi_native/src/mock/mock_process_context.rs b/minifi_rust/minifi_native/src/mock/mock_process_context.rs new file mode 100644 index 0000000000..c68afd96d0 --- /dev/null +++ b/minifi_rust/minifi_native/src/mock/mock_process_context.rs @@ -0,0 +1,121 @@ +use crate::api::{ProcessContext, RawControllerService}; +use crate::{ + ComponentIdentifier, EnableControllerService, GetAttribute, MinifiError, MockFlowFile, Property, +}; +use std::any::Any; +use std::borrow::Cow; +use std::collections::HashMap; + +pub struct MockPropertyMap { + pub properties: HashMap, +} + +impl MockPropertyMap { + pub fn new() -> Self { + Self { + properties: HashMap::new(), + } + } + + pub fn insert(&mut self, key: K, value: V) + where + K: Into, + V: Into, + { + self.properties.insert(key.into(), value.into()); + } + + pub fn extend(&mut self, iter: I) + where + I: IntoIterator, + K: Into, + V: Into, + { + self.properties + .extend(iter.into_iter().map(|(k, v)| (k.into(), v.into()))) + } +} + +impl MockPropertyMap { + pub fn get_property( + &self, + property: &Property, + _flow_file: Option<&MockFlowFile>, + ) -> Result, MinifiError> { + if let Some(property) = self.properties.get(property.name) { + Ok(Some(property.clone())) + } else { + if let Some(default_val) = property.default_value { + return Ok(Some(default_val.to_string())); + } + match property.is_required { + true => Err(MinifiError::MissingRequiredProperty(Cow::from( + property.name, + ))), + false => Ok(None), + } + } + } +} + +pub struct MockProcessContext { + pub properties: MockPropertyMap, + pub controller_services: HashMap>, + pub attributes: HashMap, +} + +impl ProcessContext for MockProcessContext { + type FlowFile = MockFlowFile; + + fn get_property( + &self, + property: &Property, + _flow_file: Option<&Self::FlowFile>, + ) -> Result, MinifiError> { + self.properties.get_property(property, _flow_file) + } + + fn get_raw_controller_service( + &self, + _property: &Property, + ) -> Result, MinifiError> + where + Cs: RawControllerService + ComponentIdentifier + 'static, + { + panic!("Not implemented yet"); + } + + fn get_controller_service(&self, property: &Property) -> Result, MinifiError> + where + Cs: EnableControllerService + ComponentIdentifier + 'static, + { + if let Some(service_name) = self.get_property(property, None)? { + Ok(self + .controller_services + .get(&service_name) + .and_then(|c| c.downcast_ref::())) + } else { + Ok(None) + } + } + + fn report_metrics(&self, _metrics: Vec<(String, f64)>) -> Result<(), MinifiError> { + Ok(()) + } +} + +impl MockProcessContext { + pub fn new() -> Self { + Self { + properties: MockPropertyMap::new(), + controller_services: HashMap::new(), + attributes: HashMap::new(), + } + } +} + +impl GetAttribute for MockProcessContext { + fn get_attribute(&self, name: &str) -> Result, MinifiError> { + Ok(self.attributes.get(name).cloned()) + } +} diff --git a/minifi_rust/minifi_native/src/mock/mock_process_session.rs b/minifi_rust/minifi_native/src/mock/mock_process_session.rs new file mode 100644 index 0000000000..475eb3a8cf --- /dev/null +++ b/minifi_rust/minifi_native/src/mock/mock_process_session.rs @@ -0,0 +1,181 @@ +use crate::api::process_session::IoState; +use crate::api::{InputStream, ProcessSession}; +use crate::{MinifiError, MockFlowFile}; +use itertools::Itertools; +use std::cell::RefCell; +use std::io::Read; + +pub struct TransferredFlowFile { + pub relationship: String, + pub flow_file: MockFlowFile, +} + +pub struct MockProcessSession { + pub input_flow_files: Vec, + pub transferred_flow_files: RefCell>, +} + +impl ProcessSession for MockProcessSession { + type FlowFile = MockFlowFile; + + fn create(&mut self) -> Result { + Ok(Self::FlowFile::new()) + } + fn get(&mut self) -> Option { + self.input_flow_files.pop() + } + fn transfer(&self, flow_file: Self::FlowFile, relationship: &str) -> Result<(), MinifiError> { + self.transferred_flow_files + .borrow_mut() + .push(TransferredFlowFile { + relationship: relationship.to_string(), + flow_file, + }); + Ok(()) + } + + fn remove(&mut self, _flow_file: Self::FlowFile) -> Result<(), MinifiError> { + Ok(()) + } + + fn set_attribute( + &self, + flow_file: &mut Self::FlowFile, + attr_key: &str, + attr_value: &str, + ) -> Result<(), MinifiError> { + flow_file + .attributes + .insert(attr_key.to_string(), attr_value.to_string()); + Ok(()) + } + fn get_attribute(&self, flow_file: &Self::FlowFile, attr_key: &str) -> Option { + flow_file.attributes.get(attr_key).cloned() + } + + fn on_attributes( + &self, + flow_file: &Self::FlowFile, + mut process_attr: F, + ) -> bool { + // Sorting for deterministic tests. + for (attr_key, attr_value) in flow_file.attributes.iter().sorted_by_key(|x| x.0) { + process_attr(attr_key, attr_value); + } + true + } + + fn write(&self, flow_file: &Self::FlowFile, data: &[u8]) -> Result<(), MinifiError> { + *flow_file.content.borrow_mut() = data.to_vec(); + Ok(()) + } + + fn write_lazy<'a>( + &self, + flow_file: &Self::FlowFile, + mut stream: Box, + ) -> Result<(), MinifiError> { + stream + .read_to_end(&mut flow_file.content.borrow_mut()) + .expect("Mock data should be readable"); + Ok(()) + } + + fn write_stream(&self, flow_file: &Self::FlowFile, callback: F) -> Result + where + F: FnOnce( + &mut dyn crate::api::process_session::OutputStream, + ) -> Result<(R, IoState), MinifiError>, + { + let mut new_content: Vec = Vec::new(); + let mut cursor = std::io::Cursor::new(&mut new_content); + let (r, _state) = callback(&mut cursor)?; + *flow_file.content.borrow_mut() = new_content; + Ok(r) + } + + fn read(&self, flow_file: &Self::FlowFile) -> Option> { + Some(flow_file.content.borrow().clone()) + } + + fn read_stream(&self, _flow_file: &Self::FlowFile, _callback: F) -> Result + where + F: FnOnce(&mut dyn InputStream) -> Result, + { + unimplemented!("Not implemented yet") + } + + fn read_in_batches( + &self, + flow_file: &Self::FlowFile, + batch_size: usize, + mut process_batch: F, + ) -> Result<(), MinifiError> + where + F: FnMut(&[u8]) -> Result<(), MinifiError>, + { + for chunk in flow_file.content.borrow().chunks(batch_size) { + process_batch(chunk)?; + } + Ok(()) + } + + fn get_flow_file_id(&self, flow_file: &Self::FlowFile) -> Result { + Ok(flow_file.id.clone()) + } +} + +impl MockProcessSession { + pub fn new() -> Self { + Self { + transferred_flow_files: RefCell::new(Vec::new()), + input_flow_files: Vec::new(), + } + } + + pub fn num_of_transferred_flow_files(&self) -> usize { + self.transferred_flow_files.borrow().len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::api::process_session::IoState; + + #[test] + fn test_read_in_batches() { + let session = MockProcessSession::new(); + let mut flow_file = MockFlowFile::new(); + flow_file.content = RefCell::from("Hello, World!".to_string().as_bytes().to_vec()); + let mut vec: Vec = Vec::new(); + + let res = session.read_in_batches(&mut flow_file, 1, |batch| { + assert_eq!(batch.len(), 1); + vec.push(batch[0]); + Ok(()) + }); + + assert!(res.is_ok()); + + assert_eq!(vec.len(), 13); + assert_eq!(vec, b"Hello, World!"); + } + + #[test] + fn test_write_stream_replaces_content() { + let session = MockProcessSession::new(); + let flow_file = MockFlowFile::new(); + // Pre-populate the flow file with longer content + *flow_file.content.borrow_mut() = b"Hello, World!".to_vec(); + + let res: Result<(), MinifiError> = session.write_stream(&flow_file, |stream| { + stream.write_all(b"Hi")?; + Ok(((), IoState::Ok)) + }); + + assert!(res.is_ok()); + // Old trailing bytes must not survive + assert_eq!(*flow_file.content.borrow(), b"Hi"); + } +} diff --git a/minifi_rust/minifi_native_macros/Cargo.toml b/minifi_rust/minifi_native_macros/Cargo.toml new file mode 100644 index 0000000000..c5e6edf9a0 --- /dev/null +++ b/minifi_rust/minifi_native_macros/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "minifi_native_macros" +version = "0.1.0" +edition = "2024" + +[lib] +proc-macro = true + +[dependencies] +syn = "2.0" +quote = "1.0" diff --git a/minifi_rust/minifi_native_macros/src/lib.rs b/minifi_rust/minifi_native_macros/src/lib.rs new file mode 100644 index 0000000000..9961a58664 --- /dev/null +++ b/minifi_rust/minifi_native_macros/src/lib.rs @@ -0,0 +1,20 @@ +use proc_macro::TokenStream; +use quote::quote; +use syn::{DeriveInput, parse_macro_input}; + +#[proc_macro_derive(ComponentIdentifier)] +pub fn derive_component_identifier(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as DeriveInput); + let name = &input.ident; + let name_str = name.to_string(); + + let expanded = quote! { + impl ::minifi_native::ComponentIdentifier for #name { + const CLASS_NAME: &'static str = concat!(module_path!(), "::", #name_str); + const GROUP_NAME: &'static str = env!("CARGO_PKG_NAME"); + const VERSION: &'static str = env!("CARGO_PKG_VERSION"); + } + }; + + TokenStream::from(expanded) +} diff --git a/minifi_rust/minifi_native_sys/.gitignore b/minifi_rust/minifi_native_sys/.gitignore new file mode 100644 index 0000000000..a9d37c560c --- /dev/null +++ b/minifi_rust/minifi_native_sys/.gitignore @@ -0,0 +1,2 @@ +target +Cargo.lock diff --git a/minifi_rust/minifi_native_sys/Cargo.toml b/minifi_rust/minifi_native_sys/Cargo.toml new file mode 100644 index 0000000000..db9a249ed0 --- /dev/null +++ b/minifi_rust/minifi_native_sys/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "minifi_native_sys" +version = "0.1.0" +edition = "2024" +description = "Raw FFI bindings to the Apache MiNiFi Native C API" +license = "Apache-2.0" +links = "minifi" + +[features] +default = ["last-release"] + +last-release = [] +local-sdk = [] +local-repo = [] + +[build-dependencies] +bindgen = "0.72.0" +cc = "1.2.41" diff --git a/minifi_rust/minifi_native_sys/build.rs b/minifi_rust/minifi_native_sys/build.rs new file mode 100644 index 0000000000..9b09df452e --- /dev/null +++ b/minifi_rust/minifi_native_sys/build.rs @@ -0,0 +1,195 @@ +use std::env; +use std::path::{Path, PathBuf}; +use std::process::Command; + +struct SDK { + header_path: PathBuf, + #[cfg_attr(not(windows), allow(dead_code))] + def_path: PathBuf, + behave_path: PathBuf, +} + +impl SDK { + fn from_local_repo(repo_root: &Path) -> Option { + let canon = std::fs::canonicalize(&repo_root).ok()?; + println!( + "cargo:info=MINIFI_SDK_PATH from from_local_repo: {:?} -> {:?}", + repo_root, canon + ); + + let header_path = canon + .to_path_buf() + .join("minifi-api/include/minifi-c/minifi-api.h"); + let def_path = canon.to_path_buf().join("minifi-api/minifi-api.def"); + let behave_path = canon.to_path_buf().join("behave_framework"); + + println!( + "cargo:info=header_path {:?}\n def_path {:?}\n behave_path {:?}", + header_path, def_path, behave_path + ); + + if !header_path.exists() || !def_path.exists() { + return None; + } + + Some(Self { + header_path, + def_path, + behave_path, + }) + } + + fn from_local_sdk_path(sdk_path: &Path) -> Option { + let base_path = if sdk_path.join("minifi-api.h").exists() { + sdk_path.to_path_buf() + } else if sdk_path.join("minifi-native-sdk/minifi-api.h").exists() { + sdk_path.join("minifi-native-sdk") + } else { + return None; + }; + + let header_path = base_path.join("minifi-api.h"); + let def_path = base_path.join("minifi-api.def"); + let behave_path = std::fs::read_dir(&base_path) + .ok()? + .filter_map(|e| e.ok()) + .find(|e| e.path().extension().unwrap_or_default() == "whl") + .map(|e| e.path())?; + + if !header_path.exists() || !def_path.exists() || !behave_path.exists() { + return None; + } + + Some(Self { + header_path, + def_path, + behave_path, + }) + } + + fn new_from_env_variable() -> Option { + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + let sdk_path = env::var("MINIFI_SDK_PATH").expect( + "MINIFI_SDK_PATH must be set. Please define it in your environment or .cargo/config.toml" + ); + + println!("cargo:info=Using MINIFI_SDK_PATH: {}", sdk_path); + + if sdk_path.starts_with("https://") { + let zip_path = out_dir.join("downloaded-sdk.zip"); + let extract_dir = out_dir.join("extracted-sdk"); + if !extract_dir.exists() { + Self::download_file(&sdk_path, &zip_path); + Self::extract_zip(&zip_path, &extract_dir); + } + Self::from_local_sdk_path(&extract_dir) + } else { + let local_path = if sdk_path.starts_with(".") { + PathBuf::from(format!("../{}", sdk_path)) + } else { + PathBuf::from(&sdk_path) + }; + if local_path.is_file() && local_path.extension().unwrap_or_default() == "zip" { + let extract_dir = out_dir.join("extracted-local-sdk"); + if !extract_dir.exists() { + Self::extract_zip(&local_path, &extract_dir); + } + Self::from_local_sdk_path(&extract_dir) + } else { + Self::from_local_sdk_path(&local_path) + .or_else(|| Self::from_local_repo(&local_path)) + } + } + } + + fn download_file(url: &str, dest: &Path) { + println!("cargo:warning=Downloading SDK from {}...", url); + let status = Command::new("curl") + .args(["-fSL", "-o", dest.to_str().unwrap(), url]) + .status() + .expect("Failed to execute curl to download SDK"); + assert!(status.success(), "Failed to download SDK from {}", url); + } + + fn extract_zip(archive: &Path, dest_dir: &Path) { + println!("cargo:warning=Extracting SDK archive..."); + std::fs::create_dir_all(dest_dir).unwrap(); + let status = Command::new("tar") + .args([ + "-xf", + archive.to_str().unwrap(), + "-C", + dest_dir.to_str().unwrap(), + ]) + .status() + .expect("Failed to execute tar to extract SDK zip"); + assert!(status.success(), "Failed to extract SDK zip file"); + } +} + +#[cfg(windows)] +fn generate_minifi_c_api_lib(def_path: &Path) { + let tool = cc::Build::new() + .try_get_compiler() + .expect("Failed to find the MSVC toolchain. Is Visual Studio or the C++ Build Tools workload installed?"); + + let lib_exe_path = tool + .path() + .parent() + .expect("Compiler path is expected to be in a 'bin' directory.") + .join("lib.exe"); + + if !lib_exe_path.exists() { + panic!( + "Could not find lib.exe at the expected path: {}", + lib_exe_path.display() + ); + } + + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + let lib_out_path = out_dir.join("minifi-api.lib"); + + let status = Command::new(&lib_exe_path) + .arg(format!("/def:{}", def_path.display())) + .arg(format!("/out:{}", lib_out_path.display())) + .arg(format!( + "/machine:{}", + env::var("CARGO_CFG_TARGET_ARCH").unwrap().to_uppercase() + )) + .status() + .expect("Failed to execute lib.exe."); + + if !status.success() { + panic!("lib.exe failed to create the import library from the .def file."); + } + + println!("cargo:rustc-link-lib=dylib=minifi-api"); + println!("cargo:rustc-link-search=native={}", out_dir.display()); +} + +fn main() { + println!("cargo:rerun-if-env-changed=MINIFI_SDK_PATH"); + + let sdk = SDK::new_from_env_variable() + .expect("Couldn't find valid SDK. Ensure MINIFI_SDK_PATH is set correctly."); + + #[cfg(windows)] + generate_minifi_c_api_lib(&sdk.def_path); + + println!("cargo:rerun-if-changed={}", sdk.header_path.display()); + println!( + "cargo:behave_path={}", + sdk.behave_path.canonicalize().unwrap().display() + ); + + let bindings = bindgen::Builder::default() + .header(sdk.header_path.to_str().unwrap()) + .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) + .generate() + .expect("Unable to generate bindings"); + + let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); + bindings + .write_to_file(out_path.join("bindings.rs")) + .expect("Couldn't write bindings!"); +} diff --git a/minifi_rust/minifi_native_sys/src/lib.rs b/minifi_rust/minifi_native_sys/src/lib.rs new file mode 100644 index 0000000000..1b9a6c3e45 --- /dev/null +++ b/minifi_rust/minifi_native_sys/src/lib.rs @@ -0,0 +1,11 @@ +// minifi-sys/src/lib.rs + +#![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] + +//! This crate provides raw, unsafe FFI bindings for the Apache NiFi MiNiFi C API. +//! +//! The bindings are generated automatically by `bindgen` from the `minifi-c.h` header. + +include!(concat!(env!("OUT_DIR"), "/bindings.rs")); diff --git a/minifi_rust/minifi_rs_behave/Cargo.toml b/minifi_rust/minifi_rs_behave/Cargo.toml new file mode 100644 index 0000000000..c13e8521e5 --- /dev/null +++ b/minifi_rust/minifi_rs_behave/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "minifi_rs_behave" +version = "0.1.0" +edition = "2024" + +[dependencies] +glob = "0.3.3" +minifi_rs_playground = { path = "../extensions/minifi_rs_playground" } +minifi_native_sys = { path = "../minifi_native_sys" } + +[lints.rust] +unused_crate_dependencies = "allow" diff --git a/minifi_rust/minifi_rs_behave/alpine.dockerfile b/minifi_rust/minifi_rs_behave/alpine.dockerfile new file mode 100644 index 0000000000..b32cf63f02 --- /dev/null +++ b/minifi_rust/minifi_rs_behave/alpine.dockerfile @@ -0,0 +1,27 @@ +FROM rust:alpine3.22 AS chef +# Install build dependencies required for compiling C code & extensions on Alpine +RUN apk add --no-cache musl-dev gcc g++ clang-dev lld pkgconfig curl tar +RUN cargo install cargo-chef +WORKDIR /app + +FROM chef AS planner +COPY . . +RUN cargo chef prepare --recipe-path recipe.json + +FROM chef AS builder +ARG MINIFI_SDK_PATH +ENV MINIFI_SDK_PATH=${MINIFI_SDK_PATH} + +COPY --from=planner /app/recipe.json recipe.json + +# Conditionally copy the local SDK files from the target directory +COPY target/.docker_sd[k] /app/target/.docker_sdk/ +COPY target/.docker_sdk.zi[p] /app/target/ + +RUN cargo chef cook --release --recipe-path recipe.json +COPY . . +RUN cargo build --release + +# Export Stage +FROM scratch AS bin-export +COPY --from=builder /app/target/release/libminifi_rs_playground.so / diff --git a/minifi_rust/minifi_rs_behave/build.rs b/minifi_rust/minifi_rs_behave/build.rs new file mode 100644 index 0000000000..a675fa0e2b --- /dev/null +++ b/minifi_rust/minifi_rs_behave/build.rs @@ -0,0 +1,6 @@ +fn main() { + println!("cargo:rerun-if-env-changed=DEP_MINIFI_BEHAVE_PATH"); + let behave_path = + std::env::var("DEP_MINIFI_BEHAVE_PATH").expect("DEP_MINIFI_BEHAVE_PATH is required"); + println!("cargo:rustc-env=MINIFI_BEHAVE_PATH={}", behave_path); +} diff --git a/minifi_rust/minifi_rs_behave/debian.dockerfile b/minifi_rust/minifi_rs_behave/debian.dockerfile new file mode 100644 index 0000000000..643328716d --- /dev/null +++ b/minifi_rust/minifi_rs_behave/debian.dockerfile @@ -0,0 +1,26 @@ +FROM rust:slim-bullseye AS chef +RUN apt-get update && apt-get install -y clang lld pkg-config curl tar +RUN cargo install cargo-chef +WORKDIR /app + +FROM chef AS planner +COPY . . +RUN cargo chef prepare --recipe-path recipe.json + +FROM chef AS builder +ARG MINIFI_SDK_PATH +ENV MINIFI_SDK_PATH=${MINIFI_SDK_PATH} + +COPY --from=planner /app/recipe.json recipe.json + +# Conditionally copy the local SDK files from the target directory +COPY target/.docker_sd[k] /app/target/.docker_sdk/ +COPY target/.docker_sdk.zi[p] /app/target/ + +RUN cargo chef cook --release --recipe-path recipe.json +COPY . . +RUN cargo build --release + +# Export Stage +FROM scratch AS bin-export +COPY --from=builder /app/target/release/libminifi_rs_playground.so / diff --git a/minifi_rust/minifi_rs_behave/linux_build.sh b/minifi_rust/minifi_rs_behave/linux_build.sh new file mode 100755 index 0000000000..fc94e48961 --- /dev/null +++ b/minifi_rust/minifi_rs_behave/linux_build.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +set -e + +PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$PROJECT_ROOT" + +echo "Project root: $PROJECT_ROOT" + +# --- Argument Parsing --- +FLAVOR="debian" # Default + +while [[ "$#" -gt 0 ]]; do + case $1 in + --debian) FLAVOR="debian" ;; + --alpine) FLAVOR="alpine" ;; + *) echo "Unknown parameter passed: $1"; exit 1 ;; + esac + shift +done + +# Ensure target dir exists +mkdir -p target + +# 1. Resolve MINIFI_SDK_PATH (from env, or fallback to cargo config) +RESOLVED_SDK_PATH="${MINIFI_SDK_PATH}" +if [ -z "$RESOLVED_SDK_PATH" ] && [ -f .cargo/config.toml ]; then + RESOLVED_SDK_PATH=$(grep -m 1 'MINIFI_SDK_PATH' .cargo/config.toml | cut -d '=' -f2 | tr -d ' "') +fi + +# 2. Define target locations +DOCKER_SDK_DIR="target/.docker_sdk" +DOCKER_SDK_ZIP="target/.docker_sdk.zip" +DOCKER_SDK_ARG="$RESOLVED_SDK_PATH" + +# Clean up any leftover SDK injections +rm -rf "$DOCKER_SDK_DIR" "$DOCKER_SDK_ZIP" + +if [[ "$RESOLVED_SDK_PATH" == http* ]]; then + echo "Detected URL SDK: $RESOLVED_SDK_PATH" +elif [ -f "$RESOLVED_SDK_PATH" ] && [[ "$RESOLVED_SDK_PATH" == *.zip ]]; then + echo "Detected local ZIP SDK: $RESOLVED_SDK_PATH" + cp "$RESOLVED_SDK_PATH" "$DOCKER_SDK_ZIP" + DOCKER_SDK_ARG="/app/$DOCKER_SDK_ZIP" +elif [ -d "$RESOLVED_SDK_PATH" ]; then + echo "Detected local directory SDK: $RESOLVED_SDK_PATH" + echo "Extracting only required files to minimize Docker context..." + + mkdir -p "$DOCKER_SDK_DIR" + + # Case A: It's a flat SDK layout + if [ -f "$RESOLVED_SDK_PATH/minifi-c.h" ]; then + cp "$RESOLVED_SDK_PATH/minifi-c.h" "$DOCKER_SDK_DIR/" + cp "$RESOLVED_SDK_PATH/minifi-c-api.def" "$DOCKER_SDK_DIR/" 2>/dev/null || true + cp "$RESOLVED_SDK_PATH"/*.whl "$DOCKER_SDK_DIR/" 2>/dev/null || true + + # Case B: It's a raw repository layout + elif [ -f "$RESOLVED_SDK_PATH/minifi-api/include/minifi-c/minifi-c.h" ]; then + mkdir -p "$DOCKER_SDK_DIR/minifi-api/include/minifi-c" + + cp "$RESOLVED_SDK_PATH/minifi-api/include/minifi-c/minifi-c.h" "$DOCKER_SDK_DIR/minifi-api/include/minifi-c/" + cp "$RESOLVED_SDK_PATH/minifi-api/minifi-c-api.def" "$DOCKER_SDK_DIR/minifi-api/" 2>/dev/null || true + + if [ -d "$RESOLVED_SDK_PATH/behave_framework" ]; then + cp -R "$RESOLVED_SDK_PATH/behave_framework" "$DOCKER_SDK_DIR/" + fi + else + echo "ERROR: Could not find minifi-c.h in $RESOLVED_SDK_PATH" + exit 1 + fi + + DOCKER_SDK_ARG="/app/$DOCKER_SDK_DIR" +else + echo "WARNING: MINIFI_SDK_PATH is neither a valid URL nor a local path: $RESOLVED_SDK_PATH" +fi + +# --- Set specific build configurations based on flavor --- +echo "Building for $FLAVOR" +DOCKERFILE="minifi_rs_behave/$FLAVOR.dockerfile" +TARGET_DIR="target/release" + +mkdir -p "$TARGET_DIR" + +# 3. Build using Docker +docker buildx build \ + -f "$DOCKERFILE" \ + --target bin-export \ + --build-arg MINIFI_SDK_PATH="$DOCKER_SDK_ARG" \ + --output type=local,dest="$TARGET_DIR" \ + . + +# 4. Cleanup +rm -rf "$DOCKER_SDK_DIR" "$DOCKER_SDK_ZIP" + +echo "Build complete. Artifacts updated in $TARGET_DIR" diff --git a/minifi_rust/minifi_rs_behave/src/main.rs b/minifi_rust/minifi_rs_behave/src/main.rs new file mode 100644 index 0000000000..0dc99686ae --- /dev/null +++ b/minifi_rust/minifi_rs_behave/src/main.rs @@ -0,0 +1,155 @@ +use glob::glob; +use std::env; +use std::path::{Path, PathBuf}; +use std::process::{Command, ExitStatus}; + +#[derive(Debug, PartialEq)] +enum DockerBuildFlavor { + Debian, + Alpine, + Skip, +} + +impl DockerBuildFlavor { + fn from_args() -> Self { + let args: Vec = env::args().collect(); + if args.contains(&"--alpine".to_string()) { + DockerBuildFlavor::Alpine + } else if args.contains(&"--debian".to_string()) { + DockerBuildFlavor::Debian + } else if args.contains(&"--skip-build".to_string()) { + DockerBuildFlavor::Skip + } else { + println!("No build flag provided. Defaulting to skip build."); + DockerBuildFlavor::Skip + } + } +} + +struct BehaveRunner { + venv_path: PathBuf, + minifi_behave_path: PathBuf, +} + +impl BehaveRunner { + fn new(out_dir: &Path) -> Self { + let behave_path = std::env::var("MINIFI_BEHAVE_PATH") + .map(PathBuf::from) + .expect("MINIFI_BEHAVE_PATH is required"); + + let venv_dir = out_dir.join(".venv"); + if !venv_dir.exists() { + println!("Creating virtual environment at {:?}", venv_dir); + let status = Command::new("python3") + .arg("-m") + .arg("venv") + .arg(&venv_dir) + .status() + .expect("Failed to create venv"); + + assert!(status.success(), "Failed to initialize venv"); + } + + Self { + venv_path: venv_dir, + minifi_behave_path: behave_path, + } + } + + fn get_venv_behave(&self) -> PathBuf { + if cfg!(windows) { + self.venv_path.join("Scripts").join("behavex.exe") + } else { + self.venv_path.join("bin").join("behavex") + } + } + + fn get_venv_python(&self) -> PathBuf { + if cfg!(windows) { + self.venv_path.join("Scripts").join("python.exe") + } else { + self.venv_path.join("bin").join("python") + } + } + + fn install_minifi_behave(&self) -> ExitStatus { + Command::new(&self.get_venv_python()) + .arg("-m") + .arg("pip") + .arg("install") + .arg(self.minifi_behave_path.to_string_lossy().as_ref()) + .status() + .expect("Failed to install dependencies") + } + + fn find_features(root: &Path) -> Vec { + println!("Searching for features in {:?}", root); + let pattern = format!("{}/../**/*.feature", root.display()); + let mut feature_files = Vec::new(); + for entry in glob(&pattern).expect("Failed to read glob pattern") { + match entry { + Ok(path) => { + if !path.to_string_lossy().contains(".venv") { + println!("Found feature file: {:?}", path); + feature_files.push(path); + } + } + Err(e) => println!("{:?}", e), + } + } + feature_files + } + + fn run_tests(&self, root: &Path) -> ExitStatus { + let mut cmd = Command::new(&self.get_venv_behave()); + + cmd.args(&Self::find_features(root)); + + cmd.arg("--show-progress-bar"); + cmd.arg("--parallel-processes"); + cmd.arg("2"); + #[cfg(unix)] + cmd.env("TMPDIR", "/tmp"); + + cmd.current_dir(root) + .status() + .expect("Failed to run behave") + } +} + +fn build_linux_so(root_path: &Path, flavor: &DockerBuildFlavor) { + let script_arg = match flavor { + DockerBuildFlavor::Debian => "--debian", + DockerBuildFlavor::Alpine => "--alpine", + DockerBuildFlavor::Skip => return, + }; + + println!("Triggering linux build with flavor: {}", script_arg); + + let mut build_linux = Command::new("./linux_build.sh"); + build_linux.arg(script_arg); + + let status = build_linux + .current_dir(root_path) + .status() + .expect("Failed to execute linux_build.sh"); + + assert!(status.success(), "Linux build script failed"); +} + +fn main() { + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not set"); + let root = Path::new(&manifest_dir); + + let flavor = DockerBuildFlavor::from_args(); + build_linux_so(root, &flavor); + + let out_dir = PathBuf::from(std::env::var("OUT_DIR").expect("OUT_DIR not set")); + let behave_runner = BehaveRunner::new(&out_dir); + + let install_status = behave_runner.install_minifi_behave(); + assert!(install_status.success(), "Pip install failed"); + + let status = behave_runner.run_tests(&root); + assert!(status.success(), "Behave tests failed"); +} diff --git a/minifi_rust/rustfmt.toml b/minifi_rust/rustfmt.toml new file mode 100644 index 0000000000..e69de29bb2 From fb302e44d5540484722b3816d45fc8db1181fafd Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Mon, 22 Jun 2026 17:40:25 +0200 Subject: [PATCH 2/3] rebase changes --- .../steps/flow_building_steps.py | 7 + minifi_rust/README.md | 2 +- .../features/controller_apis.feature | 14 ++ .../features/error-handling.feature | 36 +++--- .../minifi_rs_playground.md | 121 ++++++++++++++---- .../animal_controller_apis.rs | 12 ++ .../dog_controller_service.rs | 71 ++++++++++ .../duck_controller_service.rs | 40 ++++++ .../dummy_controller_service.rs | 6 +- .../lorem_ipsum_controller_service.rs | 6 +- .../properties.rs | 2 +- .../src/controller_services/mod.rs | 3 + .../minifi_rs_playground/src/lib.rs | 6 + .../generate_flow_file/properties.rs | 10 +- .../src/processors/get_file/properties.rs | 20 +-- .../src/processors/kamikaze_processor.rs | 22 ++-- .../processor_definition.rs | 4 +- .../kamikaze_processor/properties.rs | 16 +-- .../processors/kamikaze_processor/tests.rs | 10 +- .../src/processors/log_attribute.rs | 4 +- .../processors/log_attribute/properties.rs | 14 +- .../lorem_ipsum_cs_user/properties.rs | 4 +- .../src/processors/mod.rs | 1 + .../src/processors/put_file/properties.rs | 8 +- .../put_file/unix_only_properties.rs | 4 +- .../src/processors/zoo_processor.rs | 70 ++++++++++ minifi_rust/minifi_native/src/api.rs | 3 +- .../src/api/component_definition_traits.rs | 4 +- .../minifi_native/src/api/process_context.rs | 10 +- .../minifi_native/src/api/process_session.rs | 2 +- .../minifi_native/src/api/processor.rs | 4 +- .../processor_wrappers/complex_processor.rs | 4 +- .../processor_wrappers/flow_file_source.rs | 6 +- .../flow_file_stream_transform.rs | 4 +- .../processor_wrappers/flow_file_transform.rs | 8 +- minifi_rust/minifi_native/src/api/property.rs | 6 +- .../src/api/provided_interface.rs | 36 ++++++ .../minifi_native/src/api/raw_processor.rs | 8 +- .../c_ffi_controller_service_definition.rs | 83 +++++++----- .../src/c_ffi/c_ffi_process_context.rs | 46 ++++++- .../src/c_ffi/c_ffi_process_session.rs | 16 +-- .../src/c_ffi/c_ffi_processor_definition.rs | 24 ++-- .../minifi_native/src/c_ffi/c_ffi_property.rs | 12 +- minifi_rust/minifi_native/src/lib.rs | 4 +- .../src/mock/mock_process_context.rs | 8 +- .../src/mock/mock_process_session.rs | 2 +- minifi_rust/minifi_native_macros/src/lib.rs | 19 ++- minifi_rust/minifi_native_sys/src/lib.rs | 2 +- minifi_rust/minifi_rs_behave/linux_build.sh | 14 +- 49 files changed, 631 insertions(+), 207 deletions(-) create mode 100644 minifi_rust/extensions/minifi_rs_playground/features/controller_apis.feature create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/controller_services/animal_controller_apis.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/controller_services/dog_controller_service.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/controller_services/duck_controller_service.rs create mode 100644 minifi_rust/extensions/minifi_rs_playground/src/processors/zoo_processor.rs create mode 100644 minifi_rust/minifi_native/src/api/provided_interface.rs diff --git a/behave_framework/src/minifi_behave/steps/flow_building_steps.py b/behave_framework/src/minifi_behave/steps/flow_building_steps.py index 40414a9e90..b880bbfbc0 100644 --- a/behave_framework/src/minifi_behave/steps/flow_building_steps.py +++ b/behave_framework/src/minifi_behave/steps/flow_building_steps.py @@ -301,6 +301,13 @@ def setup_controller_service(context: MinifiTestContext, service_name: str): context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service) +@given('a {class_name} controller service named "{service_name}" is set up and the "{property_name}" property set to "{property_value}"') +def setup_controller_service_with_property(context: MinifiTestContext, class_name: str, service_name: str, property_name: str, property_value: str): + controller_service = ControllerService(class_name=class_name, service_name=service_name) + controller_service.add_property(property_name, property_value) + context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service) + + @given('a {service_name} controller service is set up and the "{property_name}" property set to "{property_value}"') def setup_controller_service_with_property(context: MinifiTestContext, service_name: str, property_name: str, property_value: str): controller_service = ControllerService(class_name=service_name, service_name=service_name) diff --git a/minifi_rust/README.md b/minifi_rust/README.md index 27c5b96868..950f9153b0 100644 --- a/minifi_rust/README.md +++ b/minifi_rust/README.md @@ -19,7 +19,7 @@ The framework completely encapsulates the unsafe C FFI (Foreign Function Interfa The project is structured as a Cargo workspace with a clear, layered architecture: ### [minifi-native-sys](minifi_native_sys) -Contains the raw, unsafe FFI bindings to the minifi-c.h C API. +Contains the raw, unsafe FFI bindings to the minifi-api.h C API. ### [minifi-native](minifi_native) Provides the public, safe, and idiomatic Rust API. This is the crate that developers will use to build their processors. #### API Traits diff --git a/minifi_rust/extensions/minifi_rs_playground/features/controller_apis.feature b/minifi_rust/extensions/minifi_rs_playground/features/controller_apis.feature new file mode 100644 index 0000000000..a6bcf92252 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/features/controller_apis.feature @@ -0,0 +1,14 @@ +@SUPPORTS_WINDOWS +Feature: Testing controller service api casting + + Scenario: Zoo has a jetpack dog and a normal duck + Given a ZooProcessor processor with the "Can fly service" property set to "Wolfie the magical" + And the "Number of Legs Service" property of the ZooProcessor processor is set to "Wolfie the magical" + And a DogController controller service named "Wolfie the magical" is set up and the "Has Jetpack" property set to "true" + And the "Extra information" property of the Wolfie the magical controller service is set to "The dog (Canis familiaris or Canis lupus familiaris) is a domesticated descendant of wolves." + When the MiNiFi instance starts up + + Then the Minifi logs contain the following message: "[minifi_rs_playground::processors::zoo_processor::ZooProcessor] [critical] Can DogController { has_jetpack: true, extra_info: "The dog (Canis familiaris or Canis lupus familiaris) is a domesticated descendant of wolves." } fly? true" in less than 10 seconds + And the Minifi logs do not contain errors + And the Minifi logs do not contain warnings + diff --git a/minifi_rust/extensions/minifi_rs_playground/features/error-handling.feature b/minifi_rust/extensions/minifi_rs_playground/features/error-handling.feature index 23dbf43b44..543c4a45a9 100644 --- a/minifi_rust/extensions/minifi_rs_playground/features/error-handling.feature +++ b/minifi_rust/extensions/minifi_rs_playground/features/error-handling.feature @@ -15,48 +15,48 @@ Feature: API error handling and logging And the Minifi logs do not contain errors And the Minifi logs do not contain warnings - Scenario: Minifi handles errors from on_schedule - Given a KamikazeProcessorRs processor with the "On Schedule Behaviour" property set to "ReturnErr" + Scenario: Minifi handles errors from schedule + Given a KamikazeProcessorRs processor with the "Schedule Behaviour" property set to "ReturnErr" And KamikazeProcessorRs's success relationship is auto-terminated When the MiNiFi instance starts up - Then the Minifi logs contain the following message: "KamikazeProcessorRs] [error] Error during on_schedule: ScheduleError("it was designed to fail during schedule")" in less than 10 seconds + Then the Minifi logs contain the following message: "KamikazeProcessorRs] [error] Error during schedule: ScheduleError("it was designed to fail during schedule")" in less than 10 seconds And the Minifi logs contain the following message: "(KamikazeProcessorRs): Process Schedule Operation: Error while scheduling processor" in less than 10 seconds - Scenario: Minifi handles errors from on_trigger - Given a KamikazeProcessorRs processor with the "On Schedule Behaviour" property set to "ReturnOk" - And the "On Trigger Behaviour" property of the KamikazeProcessorRs processor is set to "ReturnErr" + Scenario: Minifi handles errors from trigger + Given a KamikazeProcessorRs processor with the "Schedule Behaviour" property set to "ReturnOk" + And the "Trigger Behaviour" property of the KamikazeProcessorRs processor is set to "ReturnErr" And KamikazeProcessorRs's success relationship is auto-terminated When the MiNiFi instance starts up - Then the Minifi logs contain the following message: "KamikazeProcessorRs] [error] Error during on_trigger TriggerError("it was designed to fail in trigger")" in less than 10 seconds + Then the Minifi logs contain the following message: "KamikazeProcessorRs] [error] Error during trigger TriggerError("it was designed to fail in trigger")" in less than 10 seconds And the Minifi logs contain the following message: "Trigger and commit failed for processor KamikazeProcessorRs" in less than 10 seconds - Scenario: Panic in extension's on_schedule crashes the agent aswell - Given a KamikazeProcessorRs processor with the "On Schedule Behaviour" property set to "Panic" + Scenario: Panic in extension's schedule crashes the agent aswell + Given a KamikazeProcessorRs processor with the "Schedule Behaviour" property set to "Panic" And KamikazeProcessorRs's success relationship is auto-terminated When the MiNiFi instance is started without assertions - Then Minifi crashes with the following "KamikazeProcessor::on_schedule panic" in less than 10 seconds + Then Minifi crashes with the following "KamikazeProcessor::schedule panic" in less than 10 seconds - Scenario: Panic in extension's on_trigger crashes the agent aswell - Given a KamikazeProcessorRs processor with the "On Schedule Behaviour" property set to "ReturnOk" - And the "On Trigger Behaviour" property of the KamikazeProcessorRs processor is set to "Panic" + Scenario: Panic in extension's trigger crashes the agent aswell + Given a KamikazeProcessorRs processor with the "Schedule Behaviour" property set to "ReturnOk" + And the "Trigger Behaviour" property of the KamikazeProcessorRs processor is set to "Panic" And KamikazeProcessorRs's success relationship is auto-terminated When the MiNiFi instance is started without assertions - Then Minifi crashes with the following "KamikazeProcessor::on_trigger panic" in less than 10 seconds + Then Minifi crashes with the following "KamikazeProcessor::trigger panic" in less than 10 seconds Scenario: Get not supported property - Given a KamikazeProcessorRs processor with the "On Schedule Behaviour" property set to "ReturnOk" - And the "On Trigger Behaviour" property of the KamikazeProcessorRs processor is set to "GetNotRegisteredProperty" + Given a KamikazeProcessorRs processor with the "Schedule Behaviour" property set to "ReturnOk" + And the "Trigger Behaviour" property of the KamikazeProcessorRs processor is set to "GetNotRegisteredProperty" And KamikazeProcessorRs's success relationship is auto-terminated When the MiNiFi instance starts up - Then the Minifi logs contain the following message: "MinifiProcessContextGetProperty("Kamikaze Processor Property"), not supported property" in less than 10 seconds + Then the Minifi logs contain the following message: "minifi_process_context_get_property("Kamikaze Processor Property"), not supported property" in less than 10 seconds And the Minifi logs contain the following message: "Trigger and commit failed for processor KamikazeProcessorRs" in less than 10 seconds Scenario: Get wrong typed Controller Service @@ -66,5 +66,5 @@ Feature: API error handling and logging When the MiNiFi instance starts up - Then the Minifi logs contain the following message: "MinifiProcessContextGetControllerService::<"minifi_rs_playground::controller_services::lorem_ipsum_controller_service::LoremIpsumControllerService">("My Controller Service"), validation failed" in less than 10 seconds + Then the Minifi logs contain the following message: "Error during trigger minifi_process_context_get_controller_service_from_property::<"minifi_rs_playground::controller_services::lorem_ipsum_controller_service::LoremIpsumControllerService">" in less than 10 seconds And the Minifi logs contain the following message: "Trigger and commit failed for processor LoremIpsumCSUser" in less than 10 seconds diff --git a/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md b/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md index 9b2397fcd5..8cc65a7fbe 100644 --- a/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md +++ b/minifi_rust/extensions/minifi_rs_playground/minifi_rs_playground.md @@ -19,14 +19,18 @@ limitations under the License. - [AsciifyGerman](#AsciifyGerman) - [CountActualLogging](#CountActualLogging) +- [DuplicateStreamText](#DuplicateStreamText) - [GenerateFlowFileRs](#GenerateFlowFileRs) - [GetFileRs](#GetFileRs) - [KamikazeProcessorRs](#KamikazeProcessorRs) - [LogAttributeRs](#LogAttributeRs) - [LoremIpsumCSUser](#LoremIpsumCSUser) - [PutFileRs](#PutFileRs) +- [ZooProcessor](#ZooProcessor) ### Controller Services +- [DogController](#DogController) +- [DuckController](#DuckController) - [DummyControllerService](#DummyControllerService) - [LoremIpsumControllerService](#LoremIpsumControllerService) @@ -48,8 +52,8 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Description | |---------|-----------------------------------------| -| success | All asciified flowfiles are routed here | | failure | Non-german flowfiles are routed here | +| success | All asciified flowfiles are routed here | ## CountActualLogging @@ -71,6 +75,26 @@ In the list below, the names of required properties appear in bold. Any other pr |------|-------------| +## DuplicateStreamText + +### Description + +Duplicate text + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|------|---------------|------------------|-------------| + +### Relationships + +| Name | Description | +|---------|-------------| +| success | | + + ## GenerateFlowFileRs ### Description @@ -83,11 +107,11 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |----------------------|---------------|------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **File Size** | 1 kB | | The size of the file that will be used
**Supports Expression Language: true** | | **Batch Size** | 1 | | The number of FlowFiles to be transferred in each invocation | +| Custom Text | | | If Data Format is text and if Unique FlowFiles is false, then this custom text will be used as content of the generated FlowFiles and the File Size will be ignored. Finally, if Expression Language is used, evaluation will be performed only once per batch of generated FlowFiles
**Supports Expression Language: true** | | **Data Format** | Binary | Text
Binary | Specifies whether the data should be Text or Binary | +| **File Size** | 1 kB | | The size of the file that will be used
**Supports Expression Language: true** | | **Unique FlowFiles** | true | true
false | If true, each FlowFile that is generated will be unique. If false, a random value will be generated and all FlowFiles will get the same content but this offers much higher throughput (but see the description of Custom Text for special non-random use cases) | -| Custom Text | | | If Data Format is text and if Unique FlowFiles is false, then this custom text will be used as content of the generated FlowFiles and the File Size will be ignored. Finally, if Expression Language is used, evaluation will be performed only once per batch of generated FlowFiles
**Supports Expression Language: true** | ### Relationships @@ -108,16 +132,16 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |-------------------------|---------------|------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Batch Size** | 10 | | The maximum number of files to pull in each iteration | +| **Ignore Hidden Files** | true | true
false | Indicates whether or not hidden files should be ignored | | **Input Directory** | | | The input directory from which to pull files
**Supports Expression Language: true** | -| Polling Interval | | | Indicates how long to wait before performing a directory listing | -| Recurse Subdirectories | true | true
false | Indicates whether or not to pull files from subdirectories | | Keep Source File | false | true
false | If true, the file is not deleted after it has been copied to the Content Repository | -| Minimum File Age | | | The minimum age that a file must be in order to be pulled; any file younger than this amount of time (according to last modification date) will be ignored | | Maximum File Age | | | The maximum age that a file must be in order to be pulled; any file older than this amount of time (according to last modification date) will be ignored | -| Minimum File Size | | | The minimum size that a file can be in order to be pulled | | Maximum File Size | | | The maximum size that a file can be in order to be pulled | -| **Ignore Hidden Files** | true | true
false | Indicates whether or not hidden files should be ignored | -| **Batch Size** | 10 | | The maximum number of files to pull in each iteration | +| Minimum File Age | | | The minimum age that a file must be in order to be pulled; any file younger than this amount of time (according to last modification date) will be ignored | +| Minimum File Size | | | The minimum size that a file can be in order to be pulled | +| Polling Interval | | | Indicates how long to wait before performing a directory listing | +| Recurse Subdirectories | true | true
false | Indicates whether or not to pull files from subdirectories | ### Relationships @@ -143,10 +167,10 @@ This processor can fail or panic in on_trigger and on_schedule calls based on co In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. -| Name | Default Value | Allowable Values | Description | -|---------------------------|---------------|-----------------------------------------------------------------------------------------------|------------------------------------------| -| **On Schedule Behaviour** | ReturnOk | ReturnErr
ReturnOk
GetNotRegisteredProperty
GetInvalidControllerService
Panic | What to do during the on_schedule method | -| **On Trigger Behaviour** | ReturnOk | ReturnErr
ReturnOk
GetNotRegisteredProperty
GetInvalidControllerService
Panic | What to do during the on_trigger method | +| Name | Default Value | Allowable Values | Description | +|------------------------|---------------|-----------------------------------------------------------------------------------------------|------------------------------------------| +| **Schedule Behaviour** | ReturnOk | ReturnErr
ReturnOk
GetNotRegisteredProperty
GetInvalidControllerService
Panic | What to do during the on_schedule method | +| **Trigger Behaviour** | ReturnOk | ReturnErr
ReturnOk
GetNotRegisteredProperty
GetInvalidControllerService
Panic | What to do during the trigger method | ### Relationships @@ -167,13 +191,13 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |-----------------------|---------------|------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **Log Level** | Info | Trace
Debug
Info
Warn
Error
Critical
Off | The Log Level to use when logging the Attributes | -| Attributes to Log | | | A comma-separated list of Attributes to Log. If not specified, all attributes will be logged. | | Attributes to Ignore | | | A comma-separated list of Attributes to ignore. If not specified, no attributes will be ignored. | -| **Log Payload** | false | true
false | If true, the FlowFile's payload will be logged, in addition to its attributes. Otherwise, just the Attributes will be logged. | -| Log Prefix | | | Log prefix appended to the log lines. It helps to distinguish the output of multiple LogAttribute processors. | +| Attributes to Log | | | A comma-separated list of Attributes to Log. If not specified, all attributes will be logged. | | **FlowFiles To Log** | 1 | | Number of flow files to log. If set to zero all flow files will be logged. Please note that this may block other threads from running if not used judiciously. | | **Hexencode Payload** | false | true
false | If true, the FlowFile's payload will be logged in a hexencoded format | +| **Log Level** | Info | Trace
Debug
Info
Warn
Error
Critical
Off | The Log Level to use when logging the Attributes | +| **Log Payload** | false | true
false | If true, the FlowFile's payload will be logged, in addition to its attributes. Otherwise, just the Attributes will be logged. | +| Log Prefix | | | Log prefix appended to the log lines. It helps to distinguish the output of multiple LogAttribute processors. | ### Relationships @@ -216,19 +240,70 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |----------------------------------|---------------|-----------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **Directory** | . | | The output directory to which to put files
**Supports Expression Language: true** | | **Conflict Resolution Strategy** | fail | fail
replace
ignore | Indicates what should happen when a file with the same name already exists in the output directory | | **Create Missing Directories** | true | true
false | If true, then missing destination directories will be created. If false, flowfiles are penalized and sent to failure. | +| **Directory** | . | | The output directory to which to put files
**Supports Expression Language: true** | +| Directory Permissions | | | Sets the permissions on the directories being created if 'Create Missing Directories' property is set. Must be an octal number (e.g. 644 or 0755). Not supported on Windows systems. | | Maximum File Count | | | Specifies the maximum number of files that can exist in the output directory | | Permissions | | | Sets the permissions on the output file to the value of this attribute. Must be an octal number (e.g. 644 or 0755). Not supported on Windows systems. | -| Directory Permissions | | | Sets the permissions on the directories being created if 'Create Missing Directories' property is set. Must be an octal number (e.g. 644 or 0755). Not supported on Windows systems. | ### Relationships | Name | Description | |---------|-----------------------------------------------------------------------------------| -| success | Flowfiles that are successfully written to a file are routed to this relationship | | failure | Failed files (conflict, write failure, etc.) are transferred to failure | +| success | Flowfiles that are successfully written to a file are routed to this relationship | + + +## ZooProcessor + +### Description + +Test ZooProcessor + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|----------------------------|---------------|------------------|--------------------------| +| **Can fly service** | | | Test CanFlyService | +| **Number of Legs Service** | | | Test NumberOfLegsService | + +### Relationships + +| Name | Description | +|------|-------------| + + +## DogController + +### Description + +Test DogController + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|-------------------|---------------|------------------|-------------------------------------------------------| +| Extra information | | | We need this to verify the casting was done correctly | +| **Has Jetpack** | false | true
false | Whether or not the dog has a jetpack | + + +## DuckController + +### Description + +Test DuckController + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|------|---------------|------------------|-------------| ## DummyControllerService @@ -255,6 +330,6 @@ Simple Rusty Controller Service to test API In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. -| Name | Default Value | Allowable Values | Description | -|------------|---------------|------------------|-------------------------------------------------------------| -| **Length** | 25 | | How many words to generate
**Sensitive Property: true** | +| Name | Default Value | Allowable Values | Description | +|------------|---------------|------------------|----------------------------| +| **Length** | 25 | | How many words to generate | diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/animal_controller_apis.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/animal_controller_apis.rs new file mode 100644 index 0000000000..358963784e --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/animal_controller_apis.rs @@ -0,0 +1,12 @@ +use std::fmt::Debug; +use minifi_native::macros::controller_service_api; + +#[controller_service_api] +pub trait NumberOfLegsControllerApi: Debug { + fn number_of_legs(&self) -> u8; +} + +#[controller_service_api] +pub trait CanFlyControllerApi: Debug { + fn can_fly(&self) -> bool; +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dog_controller_service.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dog_controller_service.rs new file mode 100644 index 0000000000..9649b9d1f4 --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dog_controller_service.rs @@ -0,0 +1,71 @@ +use minifi_native::ControllerServiceApi; +use minifi_native::{create_provided_interface, ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, Property, ProvidedInterface, StandardPropertyValidator}; +use minifi_native::macros::ComponentIdentifier; +use crate::controller_services::animal_controller_apis::{CanFlyControllerApi, NumberOfLegsControllerApi}; + +pub(crate) const HAS_JETPACK: Property = Property { + name: "Has Jetpack", + description: "Whether or not the dog has a jetpack", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: Some("false"), + validator: StandardPropertyValidator::BoolValidator, + allowed_values: &[], + allowed_type: None, +}; + +pub(crate) const EXTRA_INFO: Property = Property { + name: "Extra information", + description: "We need this to verify the casting was done correctly", + is_required: false, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &[], + allowed_type: None, +}; + +#[allow(dead_code)] // extra_info is only used by {:?} +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct DogController { + has_jetpack: bool, + extra_info: String +} + +impl NumberOfLegsControllerApi for DogController { + fn number_of_legs(&self) -> u8 { + 4 + } +} + +impl CanFlyControllerApi for DogController { + fn can_fly(&self) -> bool { + self.has_jetpack + } +} + +impl EnableControllerService for DogController { + fn enable(context: &Ctx, _logger: &L) -> Result + where + Self: Sized + { + let has_jetpack = context + .get_bool_property(&HAS_JETPACK)? + .ok_or(MinifiError::missing_required_property("Has jetpack is required"))?; + + let extra_info = context.get_property(&EXTRA_INFO)?.unwrap_or("".into()); + + Ok(Self { has_jetpack, extra_info }) + } +} + +impl ControllerServiceDefinition for DogController { + const DESCRIPTION: &'static str = "Test DogController"; + const PROPERTIES: &'static [Property] = &[HAS_JETPACK, EXTRA_INFO]; + const PROVIDED_APIS: &'static [ProvidedInterface] = &[ + create_provided_interface!(dyn CanFlyControllerApi), + create_provided_interface!(dyn NumberOfLegsControllerApi), + ]; +} \ No newline at end of file diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/duck_controller_service.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/duck_controller_service.rs new file mode 100644 index 0000000000..a565ccbffb --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/duck_controller_service.rs @@ -0,0 +1,40 @@ +use minifi_native::ControllerServiceApi; +use minifi_native::{create_provided_interface, ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, Property, ProvidedInterface}; +use minifi_native::macros::ComponentIdentifier; +use crate::controller_services::animal_controller_apis::{CanFlyControllerApi, NumberOfLegsControllerApi}; + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct DuckController { + +} + + +impl NumberOfLegsControllerApi for DuckController { + fn number_of_legs(&self) -> u8 { + 2 + } +} + +impl CanFlyControllerApi for DuckController { + fn can_fly(&self) -> bool { + true + } +} + +impl EnableControllerService for DuckController { + fn enable(_context: &Ctx, _logger: &L) -> Result + where + Self: Sized + { + Ok(Self{}) + } +} + +impl ControllerServiceDefinition for DuckController { + const DESCRIPTION: &'static str = "Test DuckController"; + const PROPERTIES: &'static [Property] = &[]; + const PROVIDED_APIS: &'static [ProvidedInterface] = &[ + create_provided_interface!(dyn CanFlyControllerApi), + create_provided_interface!(dyn NumberOfLegsControllerApi), + ]; +} \ No newline at end of file diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs index ccd098ab41..fb346e7444 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs @@ -1,8 +1,5 @@ use minifi_native::macros::ComponentIdentifier; -use minifi_native::{ - ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, - Property, -}; +use minifi_native::{ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, Property, ProvidedInterface}; #[derive(Debug, ComponentIdentifier)] pub(crate) struct DummyControllerService {} @@ -19,4 +16,5 @@ impl EnableControllerService for DummyControllerService { impl ControllerServiceDefinition for DummyControllerService { const DESCRIPTION: &'static str = "Dummy Controller Service"; const PROPERTIES: &'static [Property] = &[]; + const PROVIDED_APIS: &'static [ProvidedInterface] = &[]; } diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs index eeb9ab5213..bafc80c5c3 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs @@ -3,10 +3,7 @@ mod properties; use crate::controller_services::lorem_ipsum_controller_service::properties::LENGTH; use lipsum::lipsum; use minifi_native::macros::ComponentIdentifier; -use minifi_native::{ - ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, - Property, -}; +use minifi_native::{ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, Property, ProvidedInterface}; #[derive(Debug, ComponentIdentifier)] pub(crate) struct LoremIpsumControllerService { @@ -30,4 +27,5 @@ impl EnableControllerService for LoremIpsumControllerService { impl ControllerServiceDefinition for LoremIpsumControllerService { const DESCRIPTION: &'static str = "Simple Rusty Controller Service to test API"; const PROPERTIES: &'static [Property] = &[LENGTH]; + const PROVIDED_APIS: &'static [ProvidedInterface] = &[]; } diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service/properties.rs index b895647a7c..f15fd95f21 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service/properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service/properties.rs @@ -9,5 +9,5 @@ pub(crate) const LENGTH: Property = Property { default_value: Some("25"), validator: StandardPropertyValidator::U64Validator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs index 7935987e7f..0467a80410 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs @@ -1,2 +1,5 @@ pub(crate) mod dummy_controller_service; pub(crate) mod lorem_ipsum_controller_service; +pub(crate) mod dog_controller_service; +pub(crate) mod animal_controller_apis; +pub(crate) mod duck_controller_service; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/lib.rs b/minifi_rust/extensions/minifi_rs_playground/src/lib.rs index ad83792f28..ed55e963f8 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/lib.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/lib.rs @@ -1,6 +1,8 @@ mod controller_services; mod processors; +use crate::controller_services::dog_controller_service::DogController; +use crate::controller_services::duck_controller_service::DuckController; use crate::controller_services::dummy_controller_service::DummyControllerService; use crate::controller_services::lorem_ipsum_controller_service::LoremIpsumControllerService; use crate::processors::asciify_german::AsciifyGerman; @@ -12,6 +14,7 @@ use crate::processors::kamikaze_processor::KamikazeProcessorRs; use crate::processors::log_attribute::LogAttributeRs; use crate::processors::lorem_ipsum_cs_user::LoremIpsumCSUser; use crate::processors::put_file::PutFileRs; +use crate::processors::zoo_processor::ZooProcessor; use minifi_native::{ ComplexProcessorType, Concurrent, Exclusive, FlowFileSourceProcessorType, @@ -29,9 +32,12 @@ processors: [ (FlowFileTransformProcessorType, Concurrent, PutFileRs), (FlowFileStreamTransformProcessorType, Concurrent, AsciifyGerman), (FlowFileStreamTransformProcessorType, Exclusive, DuplicateStreamText), + (ComplexProcessorType, Concurrent, ZooProcessor), ], controllers: [ LoremIpsumControllerService, DummyControllerService, + DogController, + DuckController ] ); diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/properties.rs index 19d81836c9..58840edb99 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file/properties.rs @@ -9,7 +9,7 @@ pub(crate) const FILE_SIZE: Property = Property { default_value: Some("1 kB"), validator: StandardPropertyValidator::DataSizeValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const BATCH_SIZE: Property = Property { @@ -21,7 +21,7 @@ pub(crate) const BATCH_SIZE: Property = Property { default_value: Some("1"), validator: StandardPropertyValidator::U64Validator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const DATA_FORMAT: Property = Property { @@ -33,7 +33,7 @@ pub(crate) const DATA_FORMAT: Property = Property { default_value: Some("Binary"), validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &["Text", "Binary"], - allowed_type: "", + allowed_type: None, }; pub(crate) const UNIQUE_FLOW_FILES: Property = Property { @@ -45,7 +45,7 @@ pub(crate) const UNIQUE_FLOW_FILES: Property = Property { default_value: Some("true"), validator: StandardPropertyValidator::BoolValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const CUSTOM_TEXT: Property = Property { @@ -57,5 +57,5 @@ pub(crate) const CUSTOM_TEXT: Property = Property { default_value: None, validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/properties.rs index 44f81fa717..5945ce29d5 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file/properties.rs @@ -9,7 +9,7 @@ pub(crate) const DIRECTORY: Property = Property { default_value: None, validator: StandardPropertyValidator::NonBlankValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const RECURSE: Property = Property { @@ -21,7 +21,7 @@ pub(crate) const RECURSE: Property = Property { default_value: Some("true"), validator: StandardPropertyValidator::BoolValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const KEEP_SOURCE_FILE: Property = Property { @@ -33,7 +33,7 @@ pub(crate) const KEEP_SOURCE_FILE: Property = Property { default_value: Some("false"), validator: StandardPropertyValidator::BoolValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const MIN_AGE: Property = Property { @@ -45,7 +45,7 @@ pub(crate) const MIN_AGE: Property = Property { default_value: None, validator: StandardPropertyValidator::TimePeriodValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const MAX_AGE: Property = Property { @@ -57,7 +57,7 @@ pub(crate) const MAX_AGE: Property = Property { default_value: None, validator: StandardPropertyValidator::TimePeriodValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const MIN_SIZE: Property = Property { @@ -69,7 +69,7 @@ pub(crate) const MIN_SIZE: Property = Property { default_value: None, validator: StandardPropertyValidator::DataSizeValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const MAX_SIZE: Property = Property { @@ -81,7 +81,7 @@ pub(crate) const MAX_SIZE: Property = Property { default_value: None, validator: StandardPropertyValidator::DataSizeValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const IGNORE_HIDDEN_FILES: Property = Property { @@ -93,7 +93,7 @@ pub(crate) const IGNORE_HIDDEN_FILES: Property = Property { default_value: Some("true"), validator: StandardPropertyValidator::BoolValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const POLLING_INTERVAL: Property = Property { @@ -105,7 +105,7 @@ pub(crate) const POLLING_INTERVAL: Property = Property { default_value: None, validator: StandardPropertyValidator::TimePeriodValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const BATCH_SIZE: Property = Property { @@ -117,5 +117,5 @@ pub(crate) const BATCH_SIZE: Property = Property { default_value: Some("10"), validator: StandardPropertyValidator::U64Validator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs index ad65040ff1..bd01eab573 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs @@ -22,7 +22,7 @@ enum KamikazeBehaviour { #[derive(Debug, ComponentIdentifier)] pub(crate) struct KamikazeProcessorRs { - on_trigger_behaviour: KamikazeBehaviour, + trigger_behaviour: KamikazeBehaviour, } impl Schedule for KamikazeProcessorRs { @@ -30,31 +30,31 @@ impl Schedule for KamikazeProcessorRs { where Self: Sized, { - let on_trigger_behaviour = context - .get_property(&properties::ON_TRIGGER_BEHAVIOUR)? + let trigger_behaviour = context + .get_property(&properties::TRIGGER_BEHAVIOUR)? .expect("required property") .parse::()?; - let on_schedule_behaviour = context - .get_property(&properties::ON_SCHEDULE_BEHAVIOUR)? + let schedule_behaviour = context + .get_property(&properties::SCHEDULE_BEHAVIOUR)? .expect("required property") .parse::()?; - match on_schedule_behaviour { + match schedule_behaviour { KamikazeBehaviour::ReturnErr => Err(MinifiError::schedule_err( "it was designed to fail during schedule", )), KamikazeBehaviour::ReturnOk => Ok(KamikazeProcessorRs { - on_trigger_behaviour, + trigger_behaviour, }), KamikazeBehaviour::GetNotRegisteredProperty => { let _ = context.get_property(&NOT_REGISTERED_PROPERTY)?; Ok(KamikazeProcessorRs { - on_trigger_behaviour, + trigger_behaviour, }) } KamikazeBehaviour::Panic => { - panic!("KamikazeProcessor::on_schedule panic") + panic!("KamikazeProcessor::schedule panic") } KamikazeBehaviour::GetInvalidControllerService => { unimplemented!("KamikazeProcessor::get_invalid_controller_service"); @@ -75,13 +75,13 @@ impl Trigger for KamikazeProcessorRs { PS: ProcessSession, L: Logger, { - match self.on_trigger_behaviour { + match self.trigger_behaviour { KamikazeBehaviour::ReturnErr => Err(MinifiError::trigger_err( "it was designed to fail in trigger", )), KamikazeBehaviour::ReturnOk => Ok(OnTriggerResult::Ok), KamikazeBehaviour::Panic => { - panic!("KamikazeProcessor::on_trigger panic") + panic!("KamikazeProcessor::trigger panic") } KamikazeBehaviour::GetNotRegisteredProperty => { let _ = context.get_property(&NOT_REGISTERED_PROPERTY, None)?; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/processor_definition.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/processor_definition.rs index 1cbc76634c..5c38f209da 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/processor_definition.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/processor_definition.rs @@ -11,7 +11,7 @@ impl ProcessorDefinition for KamikazeProcessorRs { const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[]; const RELATIONSHIPS: &'static [Relationship] = &[relationships::SUCCESS]; const PROPERTIES: &'static [Property] = &[ - properties::ON_SCHEDULE_BEHAVIOUR, - properties::ON_TRIGGER_BEHAVIOUR, + properties::SCHEDULE_BEHAVIOUR, + properties::TRIGGER_BEHAVIOUR, ]; } diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/properties.rs index 0f2e550e7e..ba81c6b025 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/properties.rs @@ -2,8 +2,8 @@ use crate::processors::kamikaze_processor::KamikazeBehaviour; use minifi_native::{Property, StandardPropertyValidator}; use strum::VariantNames; -pub(crate) const ON_SCHEDULE_BEHAVIOUR: Property = Property { - name: "On Schedule Behaviour", +pub(crate) const SCHEDULE_BEHAVIOUR: Property = Property { + name: "Schedule Behaviour", description: "What to do during the on_schedule method", is_required: true, is_sensitive: false, @@ -11,19 +11,19 @@ pub(crate) const ON_SCHEDULE_BEHAVIOUR: Property = Property { default_value: Some(KamikazeBehaviour::ReturnOk.into_str()), validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &KamikazeBehaviour::VARIANTS, - allowed_type: "", + allowed_type: None, }; -pub(crate) const ON_TRIGGER_BEHAVIOUR: Property = Property { - name: "On Trigger Behaviour", - description: "What to do during the on_trigger method", +pub(crate) const TRIGGER_BEHAVIOUR: Property = Property { + name: "Trigger Behaviour", + description: "What to do during the trigger method", is_required: true, is_sensitive: false, supports_expr_lang: false, default_value: Some(KamikazeBehaviour::ReturnOk.into_str()), validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &KamikazeBehaviour::VARIANTS, - allowed_type: "", + allowed_type: None, }; pub(crate) const NOT_REGISTERED_PROPERTY: Property = Property { @@ -35,5 +35,5 @@ pub(crate) const NOT_REGISTERED_PROPERTY: Property = Property { default_value: None, validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs index 76acbb3e5f..ab959f6e91 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs @@ -1,6 +1,6 @@ use super::*; use crate::processors::kamikaze_processor::properties::{ - ON_SCHEDULE_BEHAVIOUR, ON_TRIGGER_BEHAVIOUR, + SCHEDULE_BEHAVIOUR, TRIGGER_BEHAVIOUR, }; use minifi_native::MinifiError::{ScheduleError, TriggerError}; use minifi_native::{MockLogger, MockProcessContext, MockProcessSession}; @@ -17,7 +17,7 @@ fn on_schedule_ok() { fn on_schedule_err() { let mut context = MockProcessContext::new(); context.properties.insert( - ON_SCHEDULE_BEHAVIOUR.name.to_string(), + SCHEDULE_BEHAVIOUR.name.to_string(), "ReturnErr".to_string(), ); let processor = KamikazeProcessorRs::schedule(&context, &MockLogger::new()); @@ -29,7 +29,7 @@ fn on_schedule_panic() { let mut context = MockProcessContext::new(); context .properties - .insert(ON_SCHEDULE_BEHAVIOUR.name.to_string(), "Panic".to_string()); + .insert(SCHEDULE_BEHAVIOUR.name.to_string(), "Panic".to_string()); let result = std::panic::catch_unwind(AssertUnwindSafe(|| { KamikazeProcessorRs::schedule(&context, &MockLogger::new()) @@ -55,7 +55,7 @@ fn on_trigger_ok() { fn on_trigger_err() { let mut context = MockProcessContext::new(); context.properties.insert( - ON_TRIGGER_BEHAVIOUR.name.to_string(), + TRIGGER_BEHAVIOUR.name.to_string(), "ReturnErr".to_string(), ); let processor = KamikazeProcessorRs::schedule(&context, &MockLogger::new()).unwrap(); @@ -72,7 +72,7 @@ fn on_trigger_panic() { let mut context = MockProcessContext::new(); context .properties - .insert(ON_TRIGGER_BEHAVIOUR.name.to_string(), "Panic".to_string()); + .insert(TRIGGER_BEHAVIOUR.name.to_string(), "Panic".to_string()); let processor = KamikazeProcessorRs::schedule(&context, &MockLogger::new()).unwrap(); let mut session = MockProcessSession::new(); diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs index b49f7dd442..3143db9d44 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs @@ -101,9 +101,7 @@ impl Trigger for LogAttributeRs { } impl Schedule for LogAttributeRs { - fn schedule(context: &P, _logger: &L) -> Result - where - Self: Sized, + fn schedule(context: &P, _logger: &L) -> Result { let log_level = context .get_property(&LOG_LEVEL)? diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/properties.rs index 5ec30f72ea..b104224e00 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute/properties.rs @@ -10,7 +10,7 @@ pub(crate) const LOG_LEVEL: Property = Property { default_value: Some("Info"), // todo! it would be nicer to come from enum value but wasnt able to use into_const_str from another crate validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &LogLevel::VARIANTS, - allowed_type: "", + allowed_type: None, }; pub(crate) const ATTRIBUTES_TO_LOG: Property = Property { @@ -22,7 +22,7 @@ pub(crate) const ATTRIBUTES_TO_LOG: Property = Property { default_value: None, validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const ATTRIBUTES_TO_IGNORE: Property = Property { @@ -34,7 +34,7 @@ pub(crate) const ATTRIBUTES_TO_IGNORE: Property = Property { default_value: None, validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const LOG_PAYLOAD: Property = Property { @@ -46,7 +46,7 @@ pub(crate) const LOG_PAYLOAD: Property = Property { default_value: Some("false"), validator: StandardPropertyValidator::BoolValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const LOG_PREFIX: Property = Property { @@ -58,7 +58,7 @@ pub(crate) const LOG_PREFIX: Property = Property { default_value: None, validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const FLOW_FILES_TO_LOG: Property = Property { @@ -70,7 +70,7 @@ pub(crate) const FLOW_FILES_TO_LOG: Property = Property { default_value: Some("1"), validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const HEX_ENCODE_PAYLOAD: Property = Property { @@ -82,5 +82,5 @@ pub(crate) const HEX_ENCODE_PAYLOAD: Property = Property { default_value: Some("false"), validator: StandardPropertyValidator::BoolValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/properties.rs index c18e1a7d01..687cbbec01 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user/properties.rs @@ -12,7 +12,7 @@ pub(crate) const CONTROLLER_SERVICE: Property = Property { default_value: None, validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &[], - allowed_type: LoremIpsumControllerService::CLASS_NAME, + allowed_type: Some(LoremIpsumControllerService::CLASS_NAME), }; pub(crate) const WRITE_METHOD: Property = Property { @@ -24,5 +24,5 @@ pub(crate) const WRITE_METHOD: Property = Property { default_value: Some(super::WriteMethod::Buffer.into_str()), validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: super::WriteMethod::VARIANTS, - allowed_type: "", + allowed_type: None, }; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/mod.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/mod.rs index 0a6e3c7271..f1a885c728 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/mod.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/mod.rs @@ -7,3 +7,4 @@ pub(crate) mod kamikaze_processor; pub(crate) mod log_attribute; pub(crate) mod lorem_ipsum_cs_user; pub(crate) mod put_file; +pub(crate) mod zoo_processor; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/properties.rs index 91953868c9..a6bd205c2d 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/properties.rs @@ -11,7 +11,7 @@ pub(crate) const DIRECTORY: Property = Property { default_value: Some("."), validator: StandardPropertyValidator::NonBlankValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const CONFLICT_RESOLUTION: Property = Property { @@ -23,7 +23,7 @@ pub(crate) const CONFLICT_RESOLUTION: Property = Property { default_value: Some(ConflictResolutionStrategy::Fail.into_str()), validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &ConflictResolutionStrategy::VARIANTS, - allowed_type: "", + allowed_type: None, }; pub(crate) const CREATE_DIRS: Property = Property { @@ -35,7 +35,7 @@ pub(crate) const CREATE_DIRS: Property = Property { default_value: Some("true"), validator: StandardPropertyValidator::BoolValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const MAX_FILE_COUNT: Property = Property { @@ -47,5 +47,5 @@ pub(crate) const MAX_FILE_COUNT: Property = Property { default_value: None, // Diverged from the original implementation, u64 with no default describes the behavior better validator: StandardPropertyValidator::U64Validator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/unix_only_properties.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/unix_only_properties.rs index 8e4ba1cc6e..8c98a861f8 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/unix_only_properties.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file/unix_only_properties.rs @@ -9,7 +9,7 @@ pub(crate) const PERMISSIONS: Property = Property { default_value: None, validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; pub(crate) const DIRECTORY_PERMISSIONS: Property = Property { @@ -21,5 +21,5 @@ pub(crate) const DIRECTORY_PERMISSIONS: Property = Property { default_value: None, validator: StandardPropertyValidator::AlwaysValidValidator, allowed_values: &[], - allowed_type: "", + allowed_type: None, }; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/zoo_processor.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/zoo_processor.rs new file mode 100644 index 0000000000..92ce6f89ad --- /dev/null +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/zoo_processor.rs @@ -0,0 +1,70 @@ +use crate::controller_services::animal_controller_apis::{CanFlyControllerApi, NumberOfLegsControllerApi}; +use minifi_native::{critical, info, GetProperty, Logger, MinifiError, OnTriggerResult, OutputAttribute, ProcessContext, ProcessSession, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, Schedule, StandardPropertyValidator, Trigger}; +use minifi_native::ControllerServiceApi; +use minifi_native::macros::ComponentIdentifier; + +pub(crate) const CAN_FLY_SERVICE: Property = Property { + name: "Can fly service", + description: "Test CanFlyService", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &[], + allowed_type: Some(::INTERFACE_NAME), +}; + +pub(crate) const NUMBER_OF_LEGS: Property = Property { + name: "Number of Legs Service", + description: "Test NumberOfLegsService", + is_required: true, + is_sensitive: false, + supports_expr_lang: false, + default_value: None, + validator: StandardPropertyValidator::AlwaysValidValidator, + allowed_values: &[], + allowed_type: Some(::INTERFACE_NAME), +}; + +#[derive(Debug, ComponentIdentifier)] +pub(crate) struct ZooProcessor { + +} + +impl Schedule for ZooProcessor { + fn schedule(_context: &Ctx, _logger: &L) -> Result + where + Self: Sized + { + Ok(Self{}) + } +} + +impl Trigger for ZooProcessor { + fn trigger(&self, context: &mut Context, _session: &mut Session, logger: &Lggr) -> Result + where + Context: ProcessContext, + Session: ProcessSession, + Lggr: Logger + { + info!(logger, "{:?}", self); + if let Some(maybe_flyer) = context.get_controller_service_api::(&CAN_FLY_SERVICE)? { + critical!(logger, "Can {:?} fly? {}", maybe_flyer, maybe_flyer.can_fly()); + } + if let Some(legged) = context.get_controller_service_api::(&NUMBER_OF_LEGS)? { + critical!(logger, "{:?} has {} legs", legged, legged.number_of_legs()); + } + Ok(OnTriggerResult::Ok) + } +} + +impl ProcessorDefinition for ZooProcessor { + const DESCRIPTION: &'static str = "Test ZooProcessor"; + const INPUT_REQUIREMENT: ProcessorInputRequirement = ProcessorInputRequirement::Forbidden; + const SUPPORTS_DYNAMIC_PROPERTIES: bool = false; + const SUPPORTS_DYNAMIC_RELATIONSHIPS: bool = false; + const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[]; + const RELATIONSHIPS: &'static [Relationship] = &[]; + const PROPERTIES: &'static [Property] = &[CAN_FLY_SERVICE, NUMBER_OF_LEGS]; +} \ No newline at end of file diff --git a/minifi_rust/minifi_native/src/api.rs b/minifi_rust/minifi_native/src/api.rs index 7413ddddeb..34d9e2e496 100644 --- a/minifi_rust/minifi_native/src/api.rs +++ b/minifi_rust/minifi_native/src/api.rs @@ -11,7 +11,8 @@ pub(crate) mod processor_wrappers; pub(crate) mod property; pub(crate) mod raw_controller_service; pub(crate) mod raw_processor; -mod relationship; +pub(crate) mod relationship; +pub(crate) mod provided_interface; pub use flow_file::FlowFile; pub use logger::{LogLevel, Logger}; diff --git a/minifi_rust/minifi_native/src/api/component_definition_traits.rs b/minifi_rust/minifi_native/src/api/component_definition_traits.rs index 671997e5fa..9afc25a193 100644 --- a/minifi_rust/minifi_native/src/api/component_definition_traits.rs +++ b/minifi_rust/minifi_native/src/api/component_definition_traits.rs @@ -1,4 +1,5 @@ use crate::{OutputAttribute, ProcessorInputRequirement, Property, Relationship}; +use crate::api::provided_interface::ProvidedInterface; pub trait ComponentIdentifier { const CLASS_NAME: &'static str; @@ -16,7 +17,8 @@ pub trait ProcessorDefinition { const PROPERTIES: &'static [Property]; } -pub trait ControllerServiceDefinition { +pub trait ControllerServiceDefinition: Sized + 'static { const DESCRIPTION: &'static str; const PROPERTIES: &'static [Property]; + const PROVIDED_APIS: &'static [ProvidedInterface]; } diff --git a/minifi_rust/minifi_native/src/api/process_context.rs b/minifi_rust/minifi_native/src/api/process_context.rs index fc60ca502f..cc8d184062 100644 --- a/minifi_rust/minifi_native/src/api/process_context.rs +++ b/minifi_rust/minifi_native/src/api/process_context.rs @@ -3,7 +3,7 @@ use crate::api::RawControllerService; use crate::api::component_definition_traits::ComponentIdentifier; use crate::api::flow_file::FlowFile; use crate::api::property::GetControllerService; -use crate::{EnableControllerService, GetProperty, MinifiError, Property}; +use crate::{ControllerServiceApi, ControllerServiceDefinition, EnableControllerService, GetProperty, MinifiError, Property}; use std::str::FromStr; use std::time::Duration; @@ -101,6 +101,12 @@ pub trait ProcessContext { where Cs: EnableControllerService + ComponentIdentifier + 'static; + fn get_controller_service_api( + &self, + property: &Property, + ) -> Result>, MinifiError>; + + fn report_metrics(&self, metrics: Vec<(String, f64)>) -> Result<(), MinifiError>; } @@ -119,7 +125,7 @@ where { fn get_controller_service(&self, property: &Property) -> Result, MinifiError> where - Cs: EnableControllerService + ComponentIdentifier + 'static, + Cs: EnableControllerService + ComponentIdentifier + ControllerServiceDefinition + 'static, { ProcessContext::get_controller_service::(self, property) } diff --git a/minifi_rust/minifi_native/src/api/process_session.rs b/minifi_rust/minifi_native/src/api/process_session.rs index 8fb78dfd18..f58dc9cc60 100644 --- a/minifi_rust/minifi_native/src/api/process_session.rs +++ b/minifi_rust/minifi_native/src/api/process_session.rs @@ -33,7 +33,7 @@ pub trait ProcessSession { ) -> bool; fn write(&self, flow_file: &Self::FlowFile, data: &[u8]) -> Result<(), MinifiError>; - fn write_lazy<'a>( + fn write_from_stream<'a>( &self, flow_file: &Self::FlowFile, stream: Box, diff --git a/minifi_rust/minifi_native/src/api/processor.rs b/minifi_rust/minifi_native/src/api/processor.rs index 60b15ceacf..c8900036f0 100644 --- a/minifi_rust/minifi_native/src/api/processor.rs +++ b/minifi_rust/minifi_native/src/api/processor.rs @@ -47,12 +47,12 @@ where self.logger.log(log_level, args); } - fn on_schedule(&mut self, context: &P) -> Result<(), MinifiError> { + fn schedule(&mut self, context: &P) -> Result<(), MinifiError> { self.scheduled_impl = Some(Impl::schedule(context, &self.logger)?); Ok(()) } - fn on_unschedule(&mut self) { + fn unschedule(&mut self) { if let Some(ref mut scheduled_impl) = self.scheduled_impl { scheduled_impl.unschedule() } diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs index 4abebc0fcf..d70ea846ff 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs @@ -41,7 +41,7 @@ where + ProcessorDefinition, L: Logger, { - fn on_trigger( + fn trigger( &mut self, context: &mut PC, session: &mut PS, @@ -69,7 +69,7 @@ where + ProcessorDefinition, L: Logger, { - fn on_trigger( + fn trigger( &self, context: &mut PC, session: &mut PS, diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs index 5bf2f33d10..27052a0273 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs @@ -64,7 +64,7 @@ where match new_flow_file_data.new_content { None => {} Some(Content::Buffer(buffer)) => session.write(&mut ff, &buffer)?, - Some(Content::Stream(stream)) => session.write_lazy(&mut ff, stream)?, + Some(Content::Stream(stream)) => session.write_from_stream(&mut ff, stream)?, } for (k, v) in &new_flow_file_data.attributes_to_add { session.set_attribute(&mut ff, k, v)?; @@ -82,7 +82,7 @@ where Implementation: Schedule + FlowFileSource, L: Logger, { - fn on_trigger( + fn trigger( &self, context: &mut PC, session: &mut PS, @@ -108,7 +108,7 @@ where Implementation: Schedule + MutFlowFileSource, L: Logger, { - fn on_trigger( + fn trigger( &mut self, context: &mut PC, session: &mut PS, diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs index 71dae71a4c..0a47a48da1 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs @@ -123,7 +123,7 @@ where Schedule + FlowFileStreamTransform, L: Logger, { - fn on_trigger( + fn trigger( &self, context: &mut PC, session: &mut PS, @@ -152,7 +152,7 @@ where Schedule + MutFlowFileStreamTransform, L: Logger, { - fn on_trigger( + fn trigger( &mut self, context: &mut PC, session: &mut PS, diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs index 3ddd19029e..620d57afba 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs @@ -13,7 +13,7 @@ use std::collections::HashMap; #[derive(Debug)] pub struct TransformedFlowFile<'a> { target_relationship_name: &'static str, - new_content: Option>, + new_content: Option>, // If None the content doesn't change attributes_to_add: HashMap, } @@ -107,7 +107,7 @@ where session.write(&flow_file, &buffer)?; } Some(Content::Stream(stream)) => { - session.write_lazy(&flow_file, stream)?; + session.write_from_stream(&flow_file, stream)?; } }; @@ -135,7 +135,7 @@ where Implementation: Schedule + FlowFileTransform, L: Logger, { - fn on_trigger( + fn trigger( &self, context: &mut PC, session: &mut PS, @@ -162,7 +162,7 @@ where Implementation: Schedule + MutFlowFileTransform, L: Logger, { - fn on_trigger( + fn trigger( &mut self, context: &mut PC, session: &mut PS, diff --git a/minifi_rust/minifi_native/src/api/property.rs b/minifi_rust/minifi_native/src/api/property.rs index 3f87b5615c..e5233571c3 100644 --- a/minifi_rust/minifi_native/src/api/property.rs +++ b/minifi_rust/minifi_native/src/api/property.rs @@ -1,7 +1,7 @@ use crate::StandardPropertyValidator::{ BoolValidator, DataSizeValidator, TimePeriodValidator, U64Validator, }; -use crate::{ComponentIdentifier, EnableControllerService, MinifiError}; +use crate::{ComponentIdentifier, ControllerServiceDefinition, EnableControllerService, MinifiError}; use std::str::FromStr; use std::time::Duration; @@ -27,7 +27,7 @@ pub struct Property { pub default_value: Option<&'static str>, pub validator: StandardPropertyValidator, pub allowed_values: &'static [&'static str], - pub allowed_type: &'static str, + pub allowed_type: Option<&'static str>, } pub trait GetProperty { @@ -94,5 +94,5 @@ pub trait GetProperty { pub trait GetControllerService { fn get_controller_service(&self, property: &Property) -> Result, MinifiError> where - Cs: EnableControllerService + ComponentIdentifier + 'static; + Cs: EnableControllerService + ComponentIdentifier + ControllerServiceDefinition + 'static; } diff --git a/minifi_rust/minifi_native/src/api/provided_interface.rs b/minifi_rust/minifi_native/src/api/provided_interface.rs new file mode 100644 index 0000000000..2ce3021aed --- /dev/null +++ b/minifi_rust/minifi_native/src/api/provided_interface.rs @@ -0,0 +1,36 @@ +pub trait ControllerServiceApi { + const INTERFACE_NAME: &'static str; +} + +#[macro_export] +macro_rules! impl_interface_fqn { + ($trait_name:ident) => { + impl ControllerServiceApi for dyn $trait_name { + const INTERFACE_NAME: &'static str = concat!( + module_path!(), + "::", + stringify!($trait_name) + ); + } + }; +} + +#[derive(Debug)] +pub struct ProvidedInterface { + pub name: &'static str, + pub cast: fn(&T) -> *mut std::ffi::c_void, +} + +#[macro_export] +macro_rules! create_provided_interface { + ($trait_type:ty) => { + ProvidedInterface { + name: <$trait_type as ControllerServiceApi>::INTERFACE_NAME, + cast: |instance| { + let trait_ref: &$trait_type = instance; + let boxed_ref: Box<&$trait_type> = Box::new(trait_ref); + Box::into_raw(boxed_ref) as *mut std::ffi::c_void + }, + } + }; +} \ No newline at end of file diff --git a/minifi_rust/minifi_native/src/api/raw_processor.rs b/minifi_rust/minifi_native/src/api/raw_processor.rs index 6926db17a8..ea1d3d7f67 100644 --- a/minifi_rust/minifi_native/src/api/raw_processor.rs +++ b/minifi_rust/minifi_native/src/api/raw_processor.rs @@ -20,8 +20,8 @@ pub trait RawProcessor: Sized { fn new(logger: Self::LoggerType) -> Self; fn log(&self, log_level: LogLevel, args: std::fmt::Arguments); - fn on_schedule(&mut self, context: &P) -> Result<(), MinifiError>; - fn on_unschedule(&mut self); + fn schedule(&mut self, context: &P) -> Result<(), MinifiError>; + fn unschedule(&mut self); } /// To differentiate between single and multithreaded processors @@ -48,7 +48,7 @@ mod sealed { } pub trait SingleThreadedTrigger: RawProcessor { - fn on_trigger( + fn trigger( &mut self, context: &mut PC, session: &mut PS, @@ -59,7 +59,7 @@ pub trait SingleThreadedTrigger: RawProcessor { } pub trait MultiThreadedTrigger: RawProcessor { - fn on_trigger( + fn trigger( &self, context: &mut PC, session: &mut PS, diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs index 6923889a7e..7d8a1367f0 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs @@ -2,14 +2,8 @@ use crate::api::RawControllerService; use crate::c_ffi::c_ffi_controller_service_context::CffiControllerServiceContext; use crate::c_ffi::c_ffi_property::CProperties; use crate::c_ffi::{CffiLogger, StaticStrAsMinifiCStr}; -use crate::{ - ComponentIdentifier, ControllerService, ControllerServiceDefinition, EnableControllerService, - LogLevel, Property, -}; -use minifi_native_sys::{ - minifi_controller_service_callbacks, minifi_controller_service_class_definition, - minifi_controller_service_context, minifi_controller_service_metadata, minifi_status, -}; +use crate::{ComponentIdentifier, ControllerService, ControllerServiceDefinition, EnableControllerService, LogLevel, Property, ProvidedInterface}; +use minifi_native_sys::{minifi_controller_service_callbacks, minifi_controller_service_class_definition, minifi_controller_service_context, minifi_controller_service_metadata, minifi_status, minifi_string_view}; use std::ffi::c_void; #[derive(Debug)] @@ -31,30 +25,39 @@ impl<'a> ControllerServiceClassDefinition<'a> { } } -pub struct CffiControllerServiceDefinition +pub struct CffiControllerServiceDefinition where T: RawControllerService + ComponentIdentifier, + P: ControllerServiceDefinition { name: &'static str, description_text: &'static str, c_properties: CProperties, + c_provided_interfaces: Vec, _phantom: std::marker::PhantomData, + _phantom_2: std::marker::PhantomData

, } -impl CffiControllerServiceDefinition +impl CffiControllerServiceDefinition where T: RawControllerService + ComponentIdentifier, + P: ControllerServiceDefinition { - pub fn new(description_text: &'static str, properties: &'static [Property]) -> Self { + pub fn new(description_text: &'static str, properties: &'static [Property], provided_interfaces: &'static [ProvidedInterface

]) -> Self { let c_properties = Property::create_c_properties(properties); - + let mut c_provided_apis = Vec::new(); + for provided_interface in provided_interfaces { + c_provided_apis.push(provided_interface.name.as_minifi_c_type()); + } Self { name: T::CLASS_NAME, description_text, c_properties, + c_provided_interfaces: c_provided_apis, _phantom: std::marker::PhantomData, + _phantom_2: std::marker::PhantomData } } @@ -97,15 +100,42 @@ where controller_service.disable() } } + + extern "C" fn get_interface( + self_ptr: *mut std::ffi::c_void, + interface_name: minifi_native_sys::minifi_string_view, + ) -> *mut std::ffi::c_void { + let name = unsafe { + let slice = std::slice::from_raw_parts( + interface_name.data as *const u8, + interface_name.length + ); + std::str::from_utf8_unchecked(slice) + }; + + let wrapper = unsafe { + &*(self_ptr as *const ControllerService) + }; + + if let Some(inner_instance) = wrapper.get_implementation() { + for iface in C::PROVIDED_APIS { + if iface.name == name { + return (iface.cast)(inner_instance); + } + } + } + + std::ptr::null_mut() + } } pub trait DynRawControllerServiceDefinition { fn class_description(&'_ self) -> ControllerServiceClassDefinition<'_>; } -impl DynRawControllerServiceDefinition for CffiControllerServiceDefinition +impl DynRawControllerServiceDefinition for CffiControllerServiceDefinition, Impl> where - T: RawControllerService + ComponentIdentifier, + Impl: EnableControllerService + ComponentIdentifier + ControllerServiceDefinition + 'static, { fn class_description(&'_ self) -> ControllerServiceClassDefinition<'_> { unsafe { @@ -119,10 +149,10 @@ where destroy: Some(Self::destroy_controller_service), enable: Some(Self::enable_controller_service), disable: Some(Self::disable_controller_service), - get_interface: None, // TODO(mzink) + get_interface: Some(Self::get_interface::) }, - provided_interfaces_count: 0, - provided_interfaces_ptr: std::ptr::null_mut() // TODO(mzink) + provided_interfaces_count: self.c_provided_interfaces.len(), + provided_interfaces_ptr: self.c_provided_interfaces.as_ptr() }) } } @@ -132,21 +162,6 @@ pub trait RegisterableControllerService { fn get_definition() -> Box; } -impl RegisterableControllerService for T -where - T: ComponentIdentifier - + ControllerServiceDefinition - + RawControllerService - + 'static, -{ - fn get_definition() -> Box { - Box::new(CffiControllerServiceDefinition::::new( - T::DESCRIPTION, - T::PROPERTIES, - )) - } -} - impl RegisterableControllerService for ControllerService where Implementation: @@ -154,9 +169,9 @@ where { fn get_definition() -> Box { Box::new(CffiControllerServiceDefinition::< - ControllerService, + ControllerService, Implementation >::new( - Implementation::DESCRIPTION, Implementation::PROPERTIES + Implementation::DESCRIPTION, Implementation::PROPERTIES, Implementation::PROVIDED_APIS )) } } diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs index e84f7fa108..9fc2104f1a 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs @@ -3,7 +3,7 @@ use super::c_ffi_primitives::StringView; use crate::api::ProcessContext; use crate::api::controller_service::ControllerService; use crate::c_ffi::{CffiLogger, StaticStrAsMinifiCStr}; -use crate::{ComponentIdentifier, EnableControllerService, MinifiError, Property}; +use crate::{ComponentIdentifier, ControllerServiceApi, EnableControllerService, MinifiError, Property}; use minifi_native_sys::*; use std::borrow::Cow; use std::ffi::c_void; @@ -128,6 +128,50 @@ impl<'a> ProcessContext for CffiProcessContext<'a> { } } + fn get_controller_service_api( + &self, + property: &Property, + ) -> Result>, MinifiError> { + let str_view = StringView::new(property.name); + let interface_view = StringView::new(Trait::INTERFACE_NAME); + + let mut interface_ptr: *mut minifi_controller_service = std::ptr::null_mut(); + + unsafe { + let get_cs_status = minifi_process_context_get_controller_service_from_property( + self.ptr, + str_view.as_raw(), + interface_view.as_raw(), + &mut interface_ptr, + ); + + if get_cs_status == minifi_status_MINIFI_STATUS_PROPERTY_NOT_SET { + return Ok(None); + } + + if get_cs_status != minifi_status_MINIFI_STATUS_SUCCESS { + return Err(MinifiError::StatusError(( + format!( + "Failed to get controller service interface <{}> for property {:?}", + Trait::INTERFACE_NAME, property.name + ) + .into(), + NonZeroU32::new_unchecked(get_cs_status), + ))); + } + + if interface_ptr.is_null() { + return Ok(None); + } + + // MAGIC: We receive the thin void pointer from the C API, + // cast it back to a Box of a Fat Pointer, and unbox it! + let boxed_ref = Box::from_raw(interface_ptr as *mut &Trait); + + Ok(Some(boxed_ref)) + } + } + fn report_metrics(&self, metrics: Vec<(String, f64)>) -> Result<(), MinifiError>{ let mut keys = Vec::new(); let mut values = Vec::new(); diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs index 3ec6a71237..6e4364a626 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs @@ -18,7 +18,7 @@ const _: () = { pub struct CffiProcessSession<'a> { ptr: *mut minifi_process_session, - // The lifetime ensures the session cannot outlive the `on_trigger` call. + // The lifetime ensures the session cannot outlive the `trigger` call. _lifetime: std::marker::PhantomData<&'a ()>, } @@ -84,7 +84,7 @@ impl<'a> CffiProcessSession<'a> { #[allow(non_upper_case_globals)] minifi_status_MINIFI_STATUS_SUCCESS => Ok(()), error_code => Err(MinifiError::StatusError(( - "MinifiProcessSessionWrite".into(), + "minifi_process_session_write".into(), NonZeroU32::new_unchecked(error_code), ))), } @@ -127,7 +127,7 @@ impl<'a> ProcessSession for CffiProcessSession<'a> { #[allow(non_upper_case_globals)] minifi_status_MINIFI_STATUS_SUCCESS => Ok(()), err_code => Err(MinifiError::StatusError(( - "MinifiProcessSessionTransfer".into(), + "minifi_process_session_transfer".into(), NonZeroU32::new_unchecked(err_code), ))), } @@ -140,7 +140,7 @@ impl<'a> ProcessSession for CffiProcessSession<'a> { #[allow(non_upper_case_globals)] minifi_status_MINIFI_STATUS_SUCCESS => Ok(()), err_code => Err(MinifiError::StatusError(( - "MinifiProcessSessionRemove".into(), + "minifi_process_session_remove".into(), NonZeroU32::new_unchecked(err_code), ))), } @@ -166,7 +166,7 @@ impl<'a> ProcessSession for CffiProcessSession<'a> { minifi_status_MINIFI_STATUS_SUCCESS => Ok(()), err_code => Err(MinifiError::StatusError(( format!( - "MinifiProcessSessionFlowFileSetAttribute({}, {})", + "minifi_process_session_set_flow_file_attribute({}, {})", attr_key, attr_value ) .into(), @@ -274,14 +274,14 @@ impl<'a> ProcessSession for CffiProcessSession<'a> { #[allow(non_upper_case_globals)] minifi_status_MINIFI_STATUS_SUCCESS => Ok(()), err_code => Err(MinifiError::StatusError(( - "MinifiProcessSessionWrite".into(), + "minifi_process_session_write".into(), NonZeroU32::new_unchecked(err_code), ))), } } } - fn write_lazy<'b>( + fn write_from_stream<'b>( &self, flow_file: &Self::FlowFile, mut stream: Box, @@ -349,7 +349,7 @@ impl<'a> ProcessSession for CffiProcessSession<'a> { ); if session_status != minifi_status_MINIFI_STATUS_SUCCESS { return Err(MinifiError::StatusError(( - "MinifiProcessSessionWrite".into(), + "minifi_process_session_write".into(), NonZeroU32::new_unchecked(session_status), ))); } diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs index 67cacd566d..a9db67e876 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs @@ -37,13 +37,13 @@ where let processor = &*(processor_ptr as *const T); let mut context = CffiProcessContext::new(context_ptr); let mut session = CffiProcessSession::new(session_ptr); - match processor.on_trigger(&mut context, &mut session) { + match processor.trigger(&mut context, &mut session) { Ok(OnTriggerResult::Ok) => minifi_status_MINIFI_STATUS_SUCCESS, Ok(OnTriggerResult::Yield) => minifi_status_MINIFI_STATUS_PROCESSOR_YIELD, Err(minifi_error) => { processor.log( LogLevel::Error, - format_args!("Error during on_trigger {}", minifi_error), + format_args!("Error during trigger {}", minifi_error), ); minifi_error.to_status() } @@ -65,7 +65,7 @@ where let processor = &mut *(processor_ptr as *mut T); let mut context = CffiProcessContext::new(context_ptr); let mut session = CffiProcessSession::new(session_ptr); - match processor.on_trigger(&mut context, &mut session) { + match processor.trigger(&mut context, &mut session) { Ok(OnTriggerResult::Ok) => minifi_status_MINIFI_STATUS_SUCCESS, Ok(OnTriggerResult::Yield) => minifi_status_MINIFI_STATUS_PROCESSOR_YIELD, Err(error_code) => error_code.to_status(), @@ -136,7 +136,7 @@ where } } - unsafe extern "C" fn on_trigger_processor( + unsafe extern "C" fn trigger_processor( processor_ptr: *mut c_void, context_ptr: *mut minifi_process_context, session_ptr: *mut minifi_process_session, @@ -150,19 +150,19 @@ where } } - unsafe extern "C" fn on_schedule_processor( + unsafe extern "C" fn schedule_processor( processor_ptr: *mut c_void, context_ptr: *mut minifi_process_context, ) -> minifi_status { unsafe { let processor = &mut *(processor_ptr as *mut T); let context = CffiProcessContext::new(context_ptr); - match processor.on_schedule(&context) { + match processor.schedule(&context) { Ok(_) => 0, Err(error_code) => { processor.log( LogLevel::Error, - format_args!("Error during on_schedule: {}", error_code), + format_args!("Error during schedule: {}", error_code), ); error_code.to_status() } @@ -170,10 +170,10 @@ where } } - unsafe extern "C" fn on_unschedule_processor(processor_ptr: *mut c_void) { + unsafe extern "C" fn unschedule_processor(processor_ptr: *mut c_void) { unsafe { let processor = &mut *(processor_ptr as *mut T); - processor.on_unschedule(); + processor.unschedule(); } } } @@ -225,9 +225,9 @@ where callbacks: minifi_processor_callbacks { create: Some(Self::create_processor), destroy: Some(Self::destroy_processor), - trigger: Some(Self::on_trigger_processor), - schedule: Some(Self::on_schedule_processor), - unschedule: Some(Self::on_unschedule_processor), + trigger: Some(Self::trigger_processor), + schedule: Some(Self::schedule_processor), + unschedule: Some(Self::unschedule_processor), }, }) } diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs index 11ebe4c5ca..bf3d8f5830 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs @@ -64,7 +64,13 @@ impl Property { fn create_c_allowed_types_vec(properties: &[Self]) -> Vec { properties .iter() - .map(|p| p.allowed_type.as_minifi_c_type()) + .map(|p| match p.allowed_type { + Some(dv) => dv.as_minifi_c_type(), + None => minifi_string_view { + data: ptr::null(), + length: 0, + }, + }) .collect() } @@ -88,11 +94,11 @@ impl Property { description: property.description.as_minifi_c_type(), is_required: property.is_required, is_sensitive: property.is_sensitive, - default_value: def_value, + default_value: if def_value.data == std::ptr::null() { std::ptr::null() } else { def_value }, allowed_values_count: allowed_values.len(), allowed_values_ptr: allowed_values.as_ptr(), validator: property.validator.as_minifi_c_type(), - allowed_type, + allowed_type: if allowed_type.data == std::ptr::null() { std::ptr::null() } else { allowed_type }, supports_expression_language: property.supports_expr_lang, } }) diff --git a/minifi_rust/minifi_native/src/lib.rs b/minifi_rust/minifi_native/src/lib.rs index 97286e3337..6f50d9beeb 100644 --- a/minifi_rust/minifi_native/src/lib.rs +++ b/minifi_rust/minifi_native/src/lib.rs @@ -30,6 +30,8 @@ pub use api::logger::{LogLevel, Logger}; pub use api::property::{GetControllerService, GetProperty, Property}; +pub use api::provided_interface::{ProvidedInterface, ControllerServiceApi}; + pub use api::process_session::IoState; pub use api::attribute::{GetAttribute, OutputAttribute}; @@ -51,7 +53,7 @@ pub use mock::{ #[cfg_attr(target_os = "linux", unsafe(link_section = ".rodata"))] #[cfg_attr(target_os = "macos", unsafe(link_section = "__DATA,__const"))] #[cfg_attr(target_os = "windows", unsafe(link_section = ".rdata"))] -pub static MinifiApiVersion: u32 = minifi_native_sys::MINIFI_API_VERSION; +pub static minifi_api_version: u32 = minifi_native_sys::MINIFI_API_VERSION; /// Defines the required MinifiInitExtension C function to register the listed processors and controller services #[macro_export] diff --git a/minifi_rust/minifi_native/src/mock/mock_process_context.rs b/minifi_rust/minifi_native/src/mock/mock_process_context.rs index c68afd96d0..b939319d9c 100644 --- a/minifi_rust/minifi_native/src/mock/mock_process_context.rs +++ b/minifi_rust/minifi_native/src/mock/mock_process_context.rs @@ -1,7 +1,5 @@ use crate::api::{ProcessContext, RawControllerService}; -use crate::{ - ComponentIdentifier, EnableControllerService, GetAttribute, MinifiError, MockFlowFile, Property, -}; +use crate::{ComponentIdentifier, ControllerServiceApi, EnableControllerService, GetAttribute, MinifiError, MockFlowFile, Property}; use std::any::Any; use std::borrow::Cow; use std::collections::HashMap; @@ -99,6 +97,10 @@ impl ProcessContext for MockProcessContext { } } + fn get_controller_service_api(&self, _property: &Property) -> Result>, MinifiError> { + todo!() + } + fn report_metrics(&self, _metrics: Vec<(String, f64)>) -> Result<(), MinifiError> { Ok(()) } diff --git a/minifi_rust/minifi_native/src/mock/mock_process_session.rs b/minifi_rust/minifi_native/src/mock/mock_process_session.rs index 475eb3a8cf..d9c5864df1 100644 --- a/minifi_rust/minifi_native/src/mock/mock_process_session.rs +++ b/minifi_rust/minifi_native/src/mock/mock_process_session.rs @@ -70,7 +70,7 @@ impl ProcessSession for MockProcessSession { Ok(()) } - fn write_lazy<'a>( + fn write_from_stream<'a>( &self, flow_file: &Self::FlowFile, mut stream: Box, diff --git a/minifi_rust/minifi_native_macros/src/lib.rs b/minifi_rust/minifi_native_macros/src/lib.rs index 9961a58664..f093b82a7a 100644 --- a/minifi_rust/minifi_native_macros/src/lib.rs +++ b/minifi_rust/minifi_native_macros/src/lib.rs @@ -1,6 +1,6 @@ use proc_macro::TokenStream; use quote::quote; -use syn::{DeriveInput, parse_macro_input}; +use syn::{DeriveInput, parse_macro_input, ItemTrait}; #[proc_macro_derive(ComponentIdentifier)] pub fn derive_component_identifier(input: TokenStream) -> TokenStream { @@ -18,3 +18,20 @@ pub fn derive_component_identifier(input: TokenStream) -> TokenStream { TokenStream::from(expanded) } + +#[proc_macro_attribute] +pub fn controller_service_api(_attr: TokenStream, item: TokenStream) -> TokenStream { + let input = parse_macro_input!(item as ItemTrait); + let name = &input.ident; + let name_str = name.to_string(); + + let expanded = quote! { + #input + + impl ::minifi_native::ControllerServiceApi for dyn #name { + const INTERFACE_NAME: &'static str = concat!(module_path!(), "::", #name_str); + } + }; + + TokenStream::from(expanded) +} diff --git a/minifi_rust/minifi_native_sys/src/lib.rs b/minifi_rust/minifi_native_sys/src/lib.rs index 1b9a6c3e45..fd4dddab9a 100644 --- a/minifi_rust/minifi_native_sys/src/lib.rs +++ b/minifi_rust/minifi_native_sys/src/lib.rs @@ -6,6 +6,6 @@ //! This crate provides raw, unsafe FFI bindings for the Apache NiFi MiNiFi C API. //! -//! The bindings are generated automatically by `bindgen` from the `minifi-c.h` header. +//! The bindings are generated automatically by `bindgen` from the `minifi-api.h` header. include!(concat!(env!("OUT_DIR"), "/bindings.rs")); diff --git a/minifi_rust/minifi_rs_behave/linux_build.sh b/minifi_rust/minifi_rs_behave/linux_build.sh index fc94e48961..10b9fb2e77 100755 --- a/minifi_rust/minifi_rs_behave/linux_build.sh +++ b/minifi_rust/minifi_rs_behave/linux_build.sh @@ -48,23 +48,23 @@ elif [ -d "$RESOLVED_SDK_PATH" ]; then mkdir -p "$DOCKER_SDK_DIR" # Case A: It's a flat SDK layout - if [ -f "$RESOLVED_SDK_PATH/minifi-c.h" ]; then - cp "$RESOLVED_SDK_PATH/minifi-c.h" "$DOCKER_SDK_DIR/" - cp "$RESOLVED_SDK_PATH/minifi-c-api.def" "$DOCKER_SDK_DIR/" 2>/dev/null || true + if [ -f "$RESOLVED_SDK_PATH/minifi-api.h" ]; then + cp "$RESOLVED_SDK_PATH/minifi-api.h" "$DOCKER_SDK_DIR/" + cp "$RESOLVED_SDK_PATH/minifi-api.def" "$DOCKER_SDK_DIR/" 2>/dev/null || true cp "$RESOLVED_SDK_PATH"/*.whl "$DOCKER_SDK_DIR/" 2>/dev/null || true # Case B: It's a raw repository layout - elif [ -f "$RESOLVED_SDK_PATH/minifi-api/include/minifi-c/minifi-c.h" ]; then + elif [ -f "$RESOLVED_SDK_PATH/minifi-api/include/minifi-c/minifi-api.h" ]; then mkdir -p "$DOCKER_SDK_DIR/minifi-api/include/minifi-c" - cp "$RESOLVED_SDK_PATH/minifi-api/include/minifi-c/minifi-c.h" "$DOCKER_SDK_DIR/minifi-api/include/minifi-c/" - cp "$RESOLVED_SDK_PATH/minifi-api/minifi-c-api.def" "$DOCKER_SDK_DIR/minifi-api/" 2>/dev/null || true + cp "$RESOLVED_SDK_PATH/minifi-api/include/minifi-c/minifi-api.h" "$DOCKER_SDK_DIR/minifi-api/include/minifi-c/" + cp "$RESOLVED_SDK_PATH/minifi-api/minifi-api.def" "$DOCKER_SDK_DIR/minifi-api/" 2>/dev/null || true if [ -d "$RESOLVED_SDK_PATH/behave_framework" ]; then cp -R "$RESOLVED_SDK_PATH/behave_framework" "$DOCKER_SDK_DIR/" fi else - echo "ERROR: Could not find minifi-c.h in $RESOLVED_SDK_PATH" + echo "ERROR: Could not find minifi-api.h in $RESOLVED_SDK_PATH" exit 1 fi From 3f23c26bf23fe8bfee44eba818ef10187661e650 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Thu, 25 Jun 2026 14:44:03 +0200 Subject: [PATCH 3/3] cargo fmt --- .../animal_controller_apis.rs | 2 +- .../dog_controller_service.rs | 28 ++++++---- .../duck_controller_service.rs | 20 +++---- .../dummy_controller_service.rs | 5 +- .../lorem_ipsum_controller_service.rs | 5 +- .../src/controller_services/mod.rs | 6 +-- .../src/processors/asciify_german.rs | 2 +- .../src/processors/count_actual_logging.rs | 2 +- .../src/processors/duplicate_text.rs | 2 +- .../src/processors/generate_flow_file.rs | 6 +-- .../src/processors/get_file.rs | 6 +-- .../src/processors/kamikaze_processor.rs | 14 ++--- .../processors/kamikaze_processor/tests.rs | 18 +++---- .../src/processors/log_attribute.rs | 5 +- .../src/processors/lorem_ipsum_cs_user.rs | 2 +- .../src/processors/put_file.rs | 2 +- .../src/processors/zoo_processor.rs | 51 ++++++++++++------ minifi_rust/minifi_native/src/api.rs | 2 +- .../src/api/component_definition_traits.rs | 2 +- .../minifi_native/src/api/process_context.rs | 6 ++- .../processor_wrappers/complex_processor.rs | 10 +--- .../processor_wrappers/flow_file_source.rs | 5 +- .../flow_file_stream_transform.rs | 12 ++--- .../processor_wrappers/flow_file_transform.rs | 8 +-- minifi_rust/minifi_native/src/api/property.rs | 4 +- .../src/api/provided_interface.rs | 9 ++-- .../c_ffi/c_ffi_controller_service_context.rs | 3 +- .../c_ffi_controller_service_definition.rs | 53 ++++++++++++------- .../c_ffi/c_ffi_controller_service_list.rs | 3 +- .../minifi_native/src/c_ffi/c_ffi_logger.rs | 5 +- .../src/c_ffi/c_ffi_output_attribute.rs | 2 +- .../src/c_ffi/c_ffi_primitives.rs | 8 ++- .../src/c_ffi/c_ffi_process_context.rs | 23 +++++--- .../src/c_ffi/c_ffi_process_session.rs | 11 +++- .../src/c_ffi/c_ffi_processor_definition.rs | 11 ++-- .../src/c_ffi/c_ffi_processor_list.rs | 2 +- .../minifi_native/src/c_ffi/c_ffi_property.rs | 21 ++++++-- .../minifi_native/src/c_ffi/c_ffi_streams.rs | 4 +- minifi_rust/minifi_native/src/lib.rs | 2 +- .../src/mock/mock_process_context.rs | 10 +++- minifi_rust/minifi_native_macros/src/lib.rs | 2 +- 41 files changed, 237 insertions(+), 157 deletions(-) diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/animal_controller_apis.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/animal_controller_apis.rs index 358963784e..f2f1e26760 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/animal_controller_apis.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/animal_controller_apis.rs @@ -1,5 +1,5 @@ -use std::fmt::Debug; use minifi_native::macros::controller_service_api; +use std::fmt::Debug; #[controller_service_api] pub trait NumberOfLegsControllerApi: Debug { diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dog_controller_service.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dog_controller_service.rs index 9649b9d1f4..3fc17cafc9 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dog_controller_service.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dog_controller_service.rs @@ -1,7 +1,12 @@ +use crate::controller_services::animal_controller_apis::{ + CanFlyControllerApi, NumberOfLegsControllerApi, +}; use minifi_native::ControllerServiceApi; -use minifi_native::{create_provided_interface, ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, Property, ProvidedInterface, StandardPropertyValidator}; use minifi_native::macros::ComponentIdentifier; -use crate::controller_services::animal_controller_apis::{CanFlyControllerApi, NumberOfLegsControllerApi}; +use minifi_native::{ + ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, + Property, ProvidedInterface, StandardPropertyValidator, create_provided_interface, +}; pub(crate) const HAS_JETPACK: Property = Property { name: "Has Jetpack", @@ -27,11 +32,11 @@ pub(crate) const EXTRA_INFO: Property = Property { allowed_type: None, }; -#[allow(dead_code)] // extra_info is only used by {:?} +#[allow(dead_code)] // extra_info is only used by {:?} #[derive(Debug, ComponentIdentifier)] pub(crate) struct DogController { has_jetpack: bool, - extra_info: String + extra_info: String, } impl NumberOfLegsControllerApi for DogController { @@ -49,15 +54,18 @@ impl CanFlyControllerApi for DogController { impl EnableControllerService for DogController { fn enable(context: &Ctx, _logger: &L) -> Result where - Self: Sized + Self: Sized, { - let has_jetpack = context - .get_bool_property(&HAS_JETPACK)? - .ok_or(MinifiError::missing_required_property("Has jetpack is required"))?; + let has_jetpack = context.get_bool_property(&HAS_JETPACK)?.ok_or( + MinifiError::missing_required_property("Has jetpack is required"), + )?; let extra_info = context.get_property(&EXTRA_INFO)?.unwrap_or("".into()); - Ok(Self { has_jetpack, extra_info }) + Ok(Self { + has_jetpack, + extra_info, + }) } } @@ -68,4 +76,4 @@ impl ControllerServiceDefinition for DogController { create_provided_interface!(dyn CanFlyControllerApi), create_provided_interface!(dyn NumberOfLegsControllerApi), ]; -} \ No newline at end of file +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/duck_controller_service.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/duck_controller_service.rs index a565ccbffb..41132ce69b 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/duck_controller_service.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/duck_controller_service.rs @@ -1,13 +1,15 @@ +use crate::controller_services::animal_controller_apis::{ + CanFlyControllerApi, NumberOfLegsControllerApi, +}; use minifi_native::ControllerServiceApi; -use minifi_native::{create_provided_interface, ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, Property, ProvidedInterface}; use minifi_native::macros::ComponentIdentifier; -use crate::controller_services::animal_controller_apis::{CanFlyControllerApi, NumberOfLegsControllerApi}; +use minifi_native::{ + ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, + Property, ProvidedInterface, create_provided_interface, +}; #[derive(Debug, ComponentIdentifier)] -pub(crate) struct DuckController { - -} - +pub(crate) struct DuckController {} impl NumberOfLegsControllerApi for DuckController { fn number_of_legs(&self) -> u8 { @@ -24,9 +26,9 @@ impl CanFlyControllerApi for DuckController { impl EnableControllerService for DuckController { fn enable(_context: &Ctx, _logger: &L) -> Result where - Self: Sized + Self: Sized, { - Ok(Self{}) + Ok(Self {}) } } @@ -37,4 +39,4 @@ impl ControllerServiceDefinition for DuckController { create_provided_interface!(dyn CanFlyControllerApi), create_provided_interface!(dyn NumberOfLegsControllerApi), ]; -} \ No newline at end of file +} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs index fb346e7444..d69efb8f98 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/dummy_controller_service.rs @@ -1,5 +1,8 @@ use minifi_native::macros::ComponentIdentifier; -use minifi_native::{ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, Property, ProvidedInterface}; +use minifi_native::{ + ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, + Property, ProvidedInterface, +}; #[derive(Debug, ComponentIdentifier)] pub(crate) struct DummyControllerService {} diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs index bafc80c5c3..6b3809917d 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/lorem_ipsum_controller_service.rs @@ -3,7 +3,10 @@ mod properties; use crate::controller_services::lorem_ipsum_controller_service::properties::LENGTH; use lipsum::lipsum; use minifi_native::macros::ComponentIdentifier; -use minifi_native::{ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, Property, ProvidedInterface}; +use minifi_native::{ + ControllerServiceDefinition, EnableControllerService, GetProperty, Logger, MinifiError, + Property, ProvidedInterface, +}; #[derive(Debug, ComponentIdentifier)] pub(crate) struct LoremIpsumControllerService { diff --git a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs index 0467a80410..774dd42076 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/controller_services/mod.rs @@ -1,5 +1,5 @@ -pub(crate) mod dummy_controller_service; -pub(crate) mod lorem_ipsum_controller_service; -pub(crate) mod dog_controller_service; pub(crate) mod animal_controller_apis; +pub(crate) mod dog_controller_service; pub(crate) mod duck_controller_service; +pub(crate) mod dummy_controller_service; +pub(crate) mod lorem_ipsum_controller_service; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs index e85949a07f..7491fc60dd 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/asciify_german.rs @@ -1,5 +1,5 @@ use crate::processors::asciify_german::relationships::FAILURE; -use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::macros::ComponentIdentifier; use minifi_native::{ FlowFileStreamTransform, GetProperty, InputStream, Logger, MinifiError, OutputStream, Schedule, TransformStreamResult, diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/count_actual_logging.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/count_actual_logging.rs index f712f37c9a..2aa3b4c4a2 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/count_actual_logging.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/count_actual_logging.rs @@ -1,4 +1,4 @@ -use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::macros::ComponentIdentifier; use minifi_native::{ GetProperty, Logger, MinifiError, MutTrigger, OnTriggerResult, OutputAttribute, ProcessContext, ProcessSession, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/duplicate_text.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/duplicate_text.rs index 38ff066e7b..708371aabf 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/duplicate_text.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/duplicate_text.rs @@ -1,4 +1,4 @@ -use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::macros::ComponentIdentifier; use minifi_native::{ GetAttribute, GetControllerService, GetProperty, InputStream, Logger, MinifiError, MutFlowFileStreamTransform, OutputAttribute, OutputStream, ProcessorDefinition, diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs index 1e9800ce5f..217c5d03ee 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/generate_flow_file.rs @@ -1,7 +1,7 @@ -use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::macros::ComponentIdentifier; use minifi_native::{ - GetProperty, Logger, MinifiError, OnTriggerResult, ProcessContext, - ProcessSession, Schedule, Trigger, + GetProperty, Logger, MinifiError, OnTriggerResult, ProcessContext, ProcessSession, Schedule, + Trigger, }; use rand::RngExt; use rand::distr::Alphanumeric; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs index 1b05b030af..1366b71bb6 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/get_file.rs @@ -5,10 +5,10 @@ use crate::processors::get_file::properties::{ BATCH_SIZE, DIRECTORY, IGNORE_HIDDEN_FILES, KEEP_SOURCE_FILE, MAX_AGE, MAX_SIZE, MIN_AGE, MIN_SIZE, RECURSE, }; -use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::macros::ComponentIdentifier; use minifi_native::{ - GetProperty, IoState, Logger, MinifiError, OnTriggerResult, ProcessContext, - ProcessSession, Schedule, Trigger, debug, info, trace, warn, + GetProperty, IoState, Logger, MinifiError, OnTriggerResult, ProcessContext, ProcessSession, + Schedule, Trigger, debug, info, trace, warn, }; use std::collections::VecDeque; use std::error; diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs index bd01eab573..b6ed74b499 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor.rs @@ -3,10 +3,10 @@ mod relationships; use crate::controller_services::lorem_ipsum_controller_service::LoremIpsumControllerService; use crate::processors::kamikaze_processor::properties::NOT_REGISTERED_PROPERTY; -use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::macros::ComponentIdentifier; use minifi_native::{ - GetProperty, Logger, MinifiError, OnTriggerResult, ProcessContext, - ProcessSession, Schedule, Trigger, + GetProperty, Logger, MinifiError, OnTriggerResult, ProcessContext, ProcessSession, Schedule, + Trigger, }; use strum_macros::{Display, EnumString, IntoStaticStr, VariantNames}; @@ -44,14 +44,10 @@ impl Schedule for KamikazeProcessorRs { KamikazeBehaviour::ReturnErr => Err(MinifiError::schedule_err( "it was designed to fail during schedule", )), - KamikazeBehaviour::ReturnOk => Ok(KamikazeProcessorRs { - trigger_behaviour, - }), + KamikazeBehaviour::ReturnOk => Ok(KamikazeProcessorRs { trigger_behaviour }), KamikazeBehaviour::GetNotRegisteredProperty => { let _ = context.get_property(&NOT_REGISTERED_PROPERTY)?; - Ok(KamikazeProcessorRs { - trigger_behaviour, - }) + Ok(KamikazeProcessorRs { trigger_behaviour }) } KamikazeBehaviour::Panic => { panic!("KamikazeProcessor::schedule panic") diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs index ab959f6e91..9bd44336f8 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/kamikaze_processor/tests.rs @@ -1,7 +1,5 @@ use super::*; -use crate::processors::kamikaze_processor::properties::{ - SCHEDULE_BEHAVIOUR, TRIGGER_BEHAVIOUR, -}; +use crate::processors::kamikaze_processor::properties::{SCHEDULE_BEHAVIOUR, TRIGGER_BEHAVIOUR}; use minifi_native::MinifiError::{ScheduleError, TriggerError}; use minifi_native::{MockLogger, MockProcessContext, MockProcessSession}; use std::panic::AssertUnwindSafe; @@ -16,10 +14,9 @@ fn on_schedule_ok() { #[test] fn on_schedule_err() { let mut context = MockProcessContext::new(); - context.properties.insert( - SCHEDULE_BEHAVIOUR.name.to_string(), - "ReturnErr".to_string(), - ); + context + .properties + .insert(SCHEDULE_BEHAVIOUR.name.to_string(), "ReturnErr".to_string()); let processor = KamikazeProcessorRs::schedule(&context, &MockLogger::new()); assert!(matches!(processor, Err(ScheduleError(_)))); } @@ -54,10 +51,9 @@ fn on_trigger_ok() { #[test] fn on_trigger_err() { let mut context = MockProcessContext::new(); - context.properties.insert( - TRIGGER_BEHAVIOUR.name.to_string(), - "ReturnErr".to_string(), - ); + context + .properties + .insert(TRIGGER_BEHAVIOUR.name.to_string(), "ReturnErr".to_string()); let processor = KamikazeProcessorRs::schedule(&context, &MockLogger::new()).unwrap(); let mut session = MockProcessSession::new(); diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs index 3143db9d44..318d8e8f60 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/log_attribute.rs @@ -1,5 +1,5 @@ use crate::processors::log_attribute::properties::{FLOW_FILES_TO_LOG, LOG_LEVEL, LOG_PAYLOAD}; -use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::macros::ComponentIdentifier; use minifi_native::{ GetProperty, LogLevel, Logger, MinifiError, OnTriggerResult, ProcessContext, ProcessSession, Property, Schedule, Trigger, debug, log, trace, @@ -101,8 +101,7 @@ impl Trigger for LogAttributeRs { } impl Schedule for LogAttributeRs { - fn schedule(context: &P, _logger: &L) -> Result - { + fn schedule(context: &P, _logger: &L) -> Result { let log_level = context .get_property(&LOG_LEVEL)? .expect("required property") diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user.rs index 50e8d9c60e..6361669d36 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/lorem_ipsum_cs_user.rs @@ -3,7 +3,7 @@ mod properties; use crate::controller_services::lorem_ipsum_controller_service::LoremIpsumControllerService; use crate::processors::lorem_ipsum_cs_user::properties::CONTROLLER_SERVICE; use crate::processors::lorem_ipsum_cs_user::relationships::SUCCESS; -use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::macros::ComponentIdentifier; use minifi_native::{ Content, FlowFileSource, GeneratedFlowFile, GetControllerService, GetProperty, Logger, MinifiError, Schedule, trace, diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs index 973e790e32..4f682229e4 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/put_file.rs @@ -1,5 +1,5 @@ use crate::processors::put_file::relationships::{FAILURE, SUCCESS}; -use minifi_native::macros::{ComponentIdentifier}; +use minifi_native::macros::ComponentIdentifier; use minifi_native::{ FlowFileTransform, GetAttribute, GetControllerService, GetProperty, InputStream, Logger, MinifiError, Schedule, TransformedFlowFile, trace, warn, diff --git a/minifi_rust/extensions/minifi_rs_playground/src/processors/zoo_processor.rs b/minifi_rust/extensions/minifi_rs_playground/src/processors/zoo_processor.rs index 92ce6f89ad..40058c1c5f 100644 --- a/minifi_rust/extensions/minifi_rs_playground/src/processors/zoo_processor.rs +++ b/minifi_rust/extensions/minifi_rs_playground/src/processors/zoo_processor.rs @@ -1,7 +1,13 @@ -use crate::controller_services::animal_controller_apis::{CanFlyControllerApi, NumberOfLegsControllerApi}; -use minifi_native::{critical, info, GetProperty, Logger, MinifiError, OnTriggerResult, OutputAttribute, ProcessContext, ProcessSession, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, Schedule, StandardPropertyValidator, Trigger}; +use crate::controller_services::animal_controller_apis::{ + CanFlyControllerApi, NumberOfLegsControllerApi, +}; use minifi_native::ControllerServiceApi; use minifi_native::macros::ComponentIdentifier; +use minifi_native::{ + GetProperty, Logger, MinifiError, OnTriggerResult, OutputAttribute, ProcessContext, + ProcessSession, ProcessorDefinition, ProcessorInputRequirement, Property, Relationship, + Schedule, StandardPropertyValidator, Trigger, critical, info, +}; pub(crate) const CAN_FLY_SERVICE: Property = Property { name: "Can fly service", @@ -28,31 +34,46 @@ pub(crate) const NUMBER_OF_LEGS: Property = Property { }; #[derive(Debug, ComponentIdentifier)] -pub(crate) struct ZooProcessor { - -} +pub(crate) struct ZooProcessor {} impl Schedule for ZooProcessor { - fn schedule(_context: &Ctx, _logger: &L) -> Result + fn schedule( + _context: &Ctx, + _logger: &L, + ) -> Result where - Self: Sized + Self: Sized, { - Ok(Self{}) + Ok(Self {}) } } impl Trigger for ZooProcessor { - fn trigger(&self, context: &mut Context, _session: &mut Session, logger: &Lggr) -> Result + fn trigger( + &self, + context: &mut Context, + _session: &mut Session, + logger: &Lggr, + ) -> Result where Context: ProcessContext, - Session: ProcessSession, - Lggr: Logger + Session: ProcessSession, + Lggr: Logger, { info!(logger, "{:?}", self); - if let Some(maybe_flyer) = context.get_controller_service_api::(&CAN_FLY_SERVICE)? { - critical!(logger, "Can {:?} fly? {}", maybe_flyer, maybe_flyer.can_fly()); + if let Some(maybe_flyer) = + context.get_controller_service_api::(&CAN_FLY_SERVICE)? + { + critical!( + logger, + "Can {:?} fly? {}", + maybe_flyer, + maybe_flyer.can_fly() + ); } - if let Some(legged) = context.get_controller_service_api::(&NUMBER_OF_LEGS)? { + if let Some(legged) = + context.get_controller_service_api::(&NUMBER_OF_LEGS)? + { critical!(logger, "{:?} has {} legs", legged, legged.number_of_legs()); } Ok(OnTriggerResult::Ok) @@ -67,4 +88,4 @@ impl ProcessorDefinition for ZooProcessor { const OUTPUT_ATTRIBUTES: &'static [OutputAttribute] = &[]; const RELATIONSHIPS: &'static [Relationship] = &[]; const PROPERTIES: &'static [Property] = &[CAN_FLY_SERVICE, NUMBER_OF_LEGS]; -} \ No newline at end of file +} diff --git a/minifi_rust/minifi_native/src/api.rs b/minifi_rust/minifi_native/src/api.rs index 34d9e2e496..e68e7aebd7 100644 --- a/minifi_rust/minifi_native/src/api.rs +++ b/minifi_rust/minifi_native/src/api.rs @@ -9,10 +9,10 @@ pub(crate) mod process_session; pub(crate) mod processor; pub(crate) mod processor_wrappers; pub(crate) mod property; +pub(crate) mod provided_interface; pub(crate) mod raw_controller_service; pub(crate) mod raw_processor; pub(crate) mod relationship; -pub(crate) mod provided_interface; pub use flow_file::FlowFile; pub use logger::{LogLevel, Logger}; diff --git a/minifi_rust/minifi_native/src/api/component_definition_traits.rs b/minifi_rust/minifi_native/src/api/component_definition_traits.rs index 9afc25a193..947c10518e 100644 --- a/minifi_rust/minifi_native/src/api/component_definition_traits.rs +++ b/minifi_rust/minifi_native/src/api/component_definition_traits.rs @@ -1,5 +1,5 @@ -use crate::{OutputAttribute, ProcessorInputRequirement, Property, Relationship}; use crate::api::provided_interface::ProvidedInterface; +use crate::{OutputAttribute, ProcessorInputRequirement, Property, Relationship}; pub trait ComponentIdentifier { const CLASS_NAME: &'static str; diff --git a/minifi_rust/minifi_native/src/api/process_context.rs b/minifi_rust/minifi_native/src/api/process_context.rs index cc8d184062..1a967ffea3 100644 --- a/minifi_rust/minifi_native/src/api/process_context.rs +++ b/minifi_rust/minifi_native/src/api/process_context.rs @@ -3,7 +3,10 @@ use crate::api::RawControllerService; use crate::api::component_definition_traits::ComponentIdentifier; use crate::api::flow_file::FlowFile; use crate::api::property::GetControllerService; -use crate::{ControllerServiceApi, ControllerServiceDefinition, EnableControllerService, GetProperty, MinifiError, Property}; +use crate::{ + ControllerServiceApi, ControllerServiceDefinition, EnableControllerService, GetProperty, + MinifiError, Property, +}; use std::str::FromStr; use std::time::Duration; @@ -106,7 +109,6 @@ pub trait ProcessContext { property: &Property, ) -> Result>, MinifiError>; - fn report_metrics(&self, metrics: Vec<(String, f64)>) -> Result<(), MinifiError>; } diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs index d70ea846ff..a907f77e59 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/complex_processor.rs @@ -35,10 +35,7 @@ pub struct ComplexProcessorType {} impl SingleThreadedTrigger for Processor where - Implementation: Schedule - + MutTrigger - + ComponentIdentifier - + ProcessorDefinition, + Implementation: Schedule + MutTrigger + ComponentIdentifier + ProcessorDefinition, L: Logger, { fn trigger( @@ -63,10 +60,7 @@ where impl MultiThreadedTrigger for Processor where - Implementation: Schedule - + Trigger - + ComponentIdentifier - + ProcessorDefinition, + Implementation: Schedule + Trigger + ComponentIdentifier + ProcessorDefinition, L: Logger, { fn trigger( diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs index 27052a0273..0776ee4782 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_source.rs @@ -1,9 +1,8 @@ use crate::api::processor_wrappers::utils::flow_file_content::Content; use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; use crate::{ - Concurrent, Exclusive, GetControllerService, GetProperty, Logger, - MinifiError, OnTriggerResult, ProcessContext, ProcessSession, Processor, Relationship, - Schedule, + Concurrent, Exclusive, GetControllerService, GetProperty, Logger, MinifiError, OnTriggerResult, + ProcessContext, ProcessSession, Processor, Relationship, Schedule, }; use std::collections::HashMap; diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs index 0a47a48da1..ef938c7b31 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_stream_transform.rs @@ -2,9 +2,9 @@ use crate::api::process_session::IoState; use crate::api::processor_wrappers::utils::context_session_flowfile_bundle::ContextSessionFlowFileBundle; use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; use crate::{ - Concurrent, Exclusive, GetAttribute, GetControllerService, GetProperty, - InputStream, LogLevel, Logger, MinifiError, OnTriggerResult, OutputStream, ProcessContext, - ProcessSession, Processor, Relationship, Schedule, + Concurrent, Exclusive, GetAttribute, GetControllerService, GetProperty, InputStream, LogLevel, + Logger, MinifiError, OnTriggerResult, OutputStream, ProcessContext, ProcessSession, Processor, + Relationship, Schedule, }; use std::collections::HashMap; @@ -119,8 +119,7 @@ where impl<'a, Implementation, L> MultiThreadedTrigger for Processor where - Implementation: - Schedule + FlowFileStreamTransform, + Implementation: Schedule + FlowFileStreamTransform, L: Logger, { fn trigger( @@ -148,8 +147,7 @@ where impl<'a, Implementation, L> SingleThreadedTrigger for Processor where - Implementation: - Schedule + MutFlowFileStreamTransform, + Implementation: Schedule + MutFlowFileStreamTransform, L: Logger, { fn trigger( diff --git a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs index 620d57afba..d9a2c0d5ce 100644 --- a/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs +++ b/minifi_rust/minifi_native/src/api/processor_wrappers/flow_file_transform.rs @@ -1,19 +1,19 @@ use crate::api::InputStream; -use crate::api::processor::{Processor}; +use crate::api::processor::Processor; use crate::api::processor_wrappers::utils::context_session_flowfile_bundle::ContextSessionFlowFileBundle; use crate::api::processor_wrappers::utils::flow_file_content::Content; use crate::api::property::{GetControllerService, GetProperty}; use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; use crate::{ - Concurrent, Exclusive, GetAttribute, LogLevel, Logger, MinifiError, - OnTriggerResult, ProcessContext, ProcessSession, Relationship, Schedule, info, + Concurrent, Exclusive, GetAttribute, LogLevel, Logger, MinifiError, OnTriggerResult, + ProcessContext, ProcessSession, Relationship, Schedule, info, }; use std::collections::HashMap; #[derive(Debug)] pub struct TransformedFlowFile<'a> { target_relationship_name: &'static str, - new_content: Option>, // If None the content doesn't change + new_content: Option>, // If None the content doesn't change attributes_to_add: HashMap, } diff --git a/minifi_rust/minifi_native/src/api/property.rs b/minifi_rust/minifi_native/src/api/property.rs index e5233571c3..63f8210a58 100644 --- a/minifi_rust/minifi_native/src/api/property.rs +++ b/minifi_rust/minifi_native/src/api/property.rs @@ -1,7 +1,9 @@ use crate::StandardPropertyValidator::{ BoolValidator, DataSizeValidator, TimePeriodValidator, U64Validator, }; -use crate::{ComponentIdentifier, ControllerServiceDefinition, EnableControllerService, MinifiError}; +use crate::{ + ComponentIdentifier, ControllerServiceDefinition, EnableControllerService, MinifiError, +}; use std::str::FromStr; use std::time::Duration; diff --git a/minifi_rust/minifi_native/src/api/provided_interface.rs b/minifi_rust/minifi_native/src/api/provided_interface.rs index 2ce3021aed..bee680e128 100644 --- a/minifi_rust/minifi_native/src/api/provided_interface.rs +++ b/minifi_rust/minifi_native/src/api/provided_interface.rs @@ -6,11 +6,8 @@ pub trait ControllerServiceApi { macro_rules! impl_interface_fqn { ($trait_name:ident) => { impl ControllerServiceApi for dyn $trait_name { - const INTERFACE_NAME: &'static str = concat!( - module_path!(), - "::", - stringify!($trait_name) - ); + const INTERFACE_NAME: &'static str = + concat!(module_path!(), "::", stringify!($trait_name)); } }; } @@ -33,4 +30,4 @@ macro_rules! create_provided_interface { }, } }; -} \ No newline at end of file +} diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs index 52ae73cbb8..b39bbfd3e5 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_context.rs @@ -1,8 +1,7 @@ use crate::c_ffi::c_ffi_primitives::StringView; use crate::{GetProperty, MinifiError, Property}; use minifi_native_sys::{ - minifi_controller_service_context, - minifi_controller_service_context_get_property, + minifi_controller_service_context, minifi_controller_service_context_get_property, minifi_status_MINIFI_STATUS_SUCCESS, minifi_string_view, }; use std::borrow::Cow; diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs index 7d8a1367f0..2fcd7ede5b 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_definition.rs @@ -2,8 +2,15 @@ use crate::api::RawControllerService; use crate::c_ffi::c_ffi_controller_service_context::CffiControllerServiceContext; use crate::c_ffi::c_ffi_property::CProperties; use crate::c_ffi::{CffiLogger, StaticStrAsMinifiCStr}; -use crate::{ComponentIdentifier, ControllerService, ControllerServiceDefinition, EnableControllerService, LogLevel, Property, ProvidedInterface}; -use minifi_native_sys::{minifi_controller_service_callbacks, minifi_controller_service_class_definition, minifi_controller_service_context, minifi_controller_service_metadata, minifi_status, minifi_string_view}; +use crate::{ + ComponentIdentifier, ControllerService, ControllerServiceDefinition, EnableControllerService, + LogLevel, Property, ProvidedInterface, +}; +use minifi_native_sys::{ + minifi_controller_service_callbacks, minifi_controller_service_class_definition, + minifi_controller_service_context, minifi_controller_service_metadata, minifi_status, + minifi_string_view, +}; use std::ffi::c_void; #[derive(Debug)] @@ -28,7 +35,7 @@ impl<'a> ControllerServiceClassDefinition<'a> { pub struct CffiControllerServiceDefinition where T: RawControllerService + ComponentIdentifier, - P: ControllerServiceDefinition + P: ControllerServiceDefinition, { name: &'static str, description_text: &'static str, @@ -40,12 +47,16 @@ where _phantom_2: std::marker::PhantomData

, } -impl CffiControllerServiceDefinition +impl CffiControllerServiceDefinition where T: RawControllerService + ComponentIdentifier, - P: ControllerServiceDefinition + P: ControllerServiceDefinition, { - pub fn new(description_text: &'static str, properties: &'static [Property], provided_interfaces: &'static [ProvidedInterface

]) -> Self { + pub fn new( + description_text: &'static str, + properties: &'static [Property], + provided_interfaces: &'static [ProvidedInterface

], + ) -> Self { let c_properties = Property::create_c_properties(properties); let mut c_provided_apis = Vec::new(); for provided_interface in provided_interfaces { @@ -57,7 +68,7 @@ where c_properties, c_provided_interfaces: c_provided_apis, _phantom: std::marker::PhantomData, - _phantom_2: std::marker::PhantomData + _phantom_2: std::marker::PhantomData, } } @@ -101,21 +112,19 @@ where } } - extern "C" fn get_interface( + extern "C" fn get_interface< + C: ControllerServiceDefinition + EnableControllerService + ComponentIdentifier, + >( self_ptr: *mut std::ffi::c_void, interface_name: minifi_native_sys::minifi_string_view, ) -> *mut std::ffi::c_void { let name = unsafe { - let slice = std::slice::from_raw_parts( - interface_name.data as *const u8, - interface_name.length - ); + let slice = + std::slice::from_raw_parts(interface_name.data as *const u8, interface_name.length); std::str::from_utf8_unchecked(slice) }; - let wrapper = unsafe { - &*(self_ptr as *const ControllerService) - }; + let wrapper = unsafe { &*(self_ptr as *const ControllerService) }; if let Some(inner_instance) = wrapper.get_implementation() { for iface in C::PROVIDED_APIS { @@ -133,7 +142,8 @@ pub trait DynRawControllerServiceDefinition { fn class_description(&'_ self) -> ControllerServiceClassDefinition<'_>; } -impl DynRawControllerServiceDefinition for CffiControllerServiceDefinition, Impl> +impl DynRawControllerServiceDefinition + for CffiControllerServiceDefinition, Impl> where Impl: EnableControllerService + ComponentIdentifier + ControllerServiceDefinition + 'static, { @@ -149,10 +159,10 @@ where destroy: Some(Self::destroy_controller_service), enable: Some(Self::enable_controller_service), disable: Some(Self::disable_controller_service), - get_interface: Some(Self::get_interface::) + get_interface: Some(Self::get_interface::), }, provided_interfaces_count: self.c_provided_interfaces.len(), - provided_interfaces_ptr: self.c_provided_interfaces.as_ptr() + provided_interfaces_ptr: self.c_provided_interfaces.as_ptr(), }) } } @@ -169,9 +179,12 @@ where { fn get_definition() -> Box { Box::new(CffiControllerServiceDefinition::< - ControllerService, Implementation + ControllerService, + Implementation, >::new( - Implementation::DESCRIPTION, Implementation::PROPERTIES, Implementation::PROVIDED_APIS + Implementation::DESCRIPTION, + Implementation::PROPERTIES, + Implementation::PROVIDED_APIS, )) } } diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs index d527490890..4bec07d636 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_controller_service_list.rs @@ -4,7 +4,8 @@ use minifi_native_sys::minifi_controller_service_class_definition; pub struct CffiControllerServiceList { controller_service_definitions: Vec>, - minifi_controller_service_class_description_list: Vec, + minifi_controller_service_class_description_list: + Vec, } impl CffiControllerServiceList { diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_logger.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_logger.rs index 2ffb893c8e..bb1ea0980e 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_logger.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_logger.rs @@ -1,6 +1,9 @@ use crate::api::LogLevel; use crate::api::Logger; -use minifi_native_sys::{minifi_log_level, minifi_logger, minifi_logger_log_string, minifi_logger_should_log, minifi_string_view}; +use minifi_native_sys::{ + minifi_log_level, minifi_logger, minifi_logger_log_string, minifi_logger_should_log, + minifi_string_view, +}; use std::ffi::CString; use std::fmt; diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_output_attribute.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_output_attribute.rs index 3485b156a2..40858bdd04 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_output_attribute.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_output_attribute.rs @@ -1,6 +1,6 @@ use crate::OutputAttribute; use crate::c_ffi::StaticStrAsMinifiCStr; -use minifi_native_sys::{minifi_string_view, minifi_output_attribute_definition}; +use minifi_native_sys::{minifi_output_attribute_definition, minifi_string_view}; #[allow(dead_code)] // the c_ vecs are holding the values referenced from the output attributes pub struct COutputAttributes { diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_primitives.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_primitives.rs index 687fbd4382..78b144cee0 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_primitives.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_primitives.rs @@ -1,5 +1,9 @@ use crate::ProcessorInputRequirement; -use minifi_native_sys::{minifi_string_view, minifi_input_requirement_MINIFI_INPUT_REQUIRED, minifi_input_requirement_MINIFI_INPUT_ALLOWED, minifi_input_requirement_MINIFI_INPUT_FORBIDDEN, minifi_input_requirement}; +use minifi_native_sys::{ + minifi_input_requirement, minifi_input_requirement_MINIFI_INPUT_ALLOWED, + minifi_input_requirement_MINIFI_INPUT_FORBIDDEN, + minifi_input_requirement_MINIFI_INPUT_REQUIRED, minifi_string_view, +}; use std::os::raw::c_char; #[derive(Debug)] @@ -8,7 +12,7 @@ pub enum FfiConversionError { InvalidUtf8, } -#[repr(transparent)] // This allows us to pass Vec as a pointer to minifi_string_view array +#[repr(transparent)] // This allows us to pass Vec as a pointer to minifi_string_view array #[derive(Debug)] pub(crate) struct StringView<'a> { inner: minifi_string_view, diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs index 9fc2104f1a..d95e809d23 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_context.rs @@ -3,7 +3,9 @@ use super::c_ffi_primitives::StringView; use crate::api::ProcessContext; use crate::api::controller_service::ControllerService; use crate::c_ffi::{CffiLogger, StaticStrAsMinifiCStr}; -use crate::{ComponentIdentifier, ControllerServiceApi, EnableControllerService, MinifiError, Property}; +use crate::{ + ComponentIdentifier, ControllerServiceApi, EnableControllerService, MinifiError, Property, +}; use minifi_native_sys::*; use std::borrow::Cow; use std::ffi::c_void; @@ -153,9 +155,10 @@ impl<'a> ProcessContext for CffiProcessContext<'a> { return Err(MinifiError::StatusError(( format!( "Failed to get controller service interface <{}> for property {:?}", - Trait::INTERFACE_NAME, property.name + Trait::INTERFACE_NAME, + property.name ) - .into(), + .into(), NonZeroU32::new_unchecked(get_cs_status), ))); } @@ -172,7 +175,7 @@ impl<'a> ProcessContext for CffiProcessContext<'a> { } } - fn report_metrics(&self, metrics: Vec<(String, f64)>) -> Result<(), MinifiError>{ + fn report_metrics(&self, metrics: Vec<(String, f64)>) -> Result<(), MinifiError> { let mut keys = Vec::new(); let mut values = Vec::new(); for (key, value) in &metrics { @@ -180,9 +183,17 @@ impl<'a> ProcessContext for CffiProcessContext<'a> { values.push(*value); } unsafe { - let err_code = minifi_process_context_report_metrics(self.ptr, keys.len(), keys.as_ptr() as *const minifi_string_view, values.as_ptr()); + let err_code = minifi_process_context_report_metrics( + self.ptr, + keys.len(), + keys.as_ptr() as *const minifi_string_view, + values.as_ptr(), + ); if err_code != minifi_status_MINIFI_STATUS_SUCCESS { - return Err(MinifiError::StatusError(("report_metrics".into(), NonZeroU32::new_unchecked(err_code)))); + return Err(MinifiError::StatusError(( + "report_metrics".into(), + NonZeroU32::new_unchecked(err_code), + ))); } } Ok(()) diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs index 6e4364a626..72cd14dccd 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_process_session.rs @@ -4,7 +4,16 @@ use crate::api::process_session::{IoState, OutputStream}; use crate::api::{InputStream, ProcessSession}; use crate::c_ffi::c_ffi_primitives::{ConvertMinifiStringView, StringView}; use crate::c_ffi::c_ffi_streams::{CffiInputStream, CffiOutputStream}; -use minifi_native_sys::{minifi_status_MINIFI_STATUS_SUCCESS, minifi_string_view, minifi_process_session, minifi_output_stream, minifi_output_stream_write, minifi_io_status_MINIFI_IO_ERROR, minifi_process_session_write, minifi_process_session_create, minifi_process_session_get, minifi_process_session_transfer, minifi_process_session_remove, minifi_process_session_set_flow_file_attribute, minifi_process_session_get_flow_file_attribute, minifi_process_session_get_flow_file_attributes, minifi_io_status_MINIFI_IO_CANCEL, minifi_input_stream, minifi_input_stream_size, minifi_input_stream_read, minifi_process_session_read, minifi_process_session_get_flow_file_id}; +use minifi_native_sys::{ + minifi_input_stream, minifi_input_stream_read, minifi_input_stream_size, + minifi_io_status_MINIFI_IO_CANCEL, minifi_io_status_MINIFI_IO_ERROR, minifi_output_stream, + minifi_output_stream_write, minifi_process_session, minifi_process_session_create, + minifi_process_session_get, minifi_process_session_get_flow_file_attribute, + minifi_process_session_get_flow_file_attributes, minifi_process_session_get_flow_file_id, + minifi_process_session_read, minifi_process_session_remove, + minifi_process_session_set_flow_file_attribute, minifi_process_session_transfer, + minifi_process_session_write, minifi_status_MINIFI_STATUS_SUCCESS, minifi_string_view, +}; use std::ffi::{CString, c_void}; use std::io::Read; use std::num::NonZeroU32; diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs index a9db67e876..6d90eda48c 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_definition.rs @@ -1,7 +1,7 @@ use std::ffi::c_void; use std::ptr; -use super::c_ffi_primitives::{StaticStrAsMinifiCStr}; +use super::c_ffi_primitives::StaticStrAsMinifiCStr; use super::c_ffi_process_context::CffiProcessContext; use super::c_ffi_process_session::CffiProcessSession; use crate::api::raw_processor::{MultiThreadedTrigger, SingleThreadedTrigger}; @@ -10,8 +10,8 @@ use crate::c_ffi::CffiLogger; use crate::c_ffi::c_ffi_output_attribute::COutputAttributes; use crate::c_ffi::c_ffi_property::CProperties; use crate::{ - ComponentIdentifier, Concurrent, Exclusive, - LogLevel, OutputAttribute, Processor, ProcessorDefinition, Property, Schedule, + ComponentIdentifier, Concurrent, Exclusive, LogLevel, OutputAttribute, Processor, + ProcessorDefinition, Property, Schedule, }; use crate::{OnTriggerResult, Relationship}; use minifi_native_sys::*; @@ -242,10 +242,7 @@ impl RawRegisterableProcessor for Processor where Threading: ThreadingModel + 'static, - Implementation: Schedule - + ComponentIdentifier - + ProcessorDefinition - + 'static, + Implementation: Schedule + ComponentIdentifier + ProcessorDefinition + 'static, Processor: RawProcessor + DispatchOnTrigger, { diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs index 9f5bf153bf..1ac5389163 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_processor_list.rs @@ -1,6 +1,6 @@ -use minifi_native_sys::minifi_processor_class_definition; use crate::c_ffi::RawRegisterableProcessor; use crate::c_ffi::c_ffi_processor_definition::DynRawProcessorDefinition; +use minifi_native_sys::minifi_processor_class_definition; pub struct CffiProcessorList { processor_definitions: Vec>, diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs index bf3d8f5830..85aff0426a 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_property.rs @@ -1,6 +1,13 @@ use super::c_ffi_primitives::StaticStrAsMinifiCStr; use crate::{Property, StandardPropertyValidator}; -use minifi_native_sys::{minifi_property_definition, minifi_validator, minifi_string_view, minifi_validator_MINIFI_VALIDATOR_ALWAYS_VALID, minifi_validator_MINIFI_VALIDATOR_BOOLEAN, minifi_validator_MINIFI_VALIDATOR_DATA_SIZE, minifi_validator_MINIFI_VALIDATOR_INTEGER, minifi_validator_MINIFI_VALIDATOR_NON_BLANK, minifi_validator_MINIFI_VALIDATOR_PORT, minifi_validator_MINIFI_VALIDATOR_TIME_PERIOD, minifi_validator_MINIFI_VALIDATOR_UNSIGNED_INTEGER}; +use minifi_native_sys::{ + minifi_property_definition, minifi_string_view, minifi_validator, + minifi_validator_MINIFI_VALIDATOR_ALWAYS_VALID, minifi_validator_MINIFI_VALIDATOR_BOOLEAN, + minifi_validator_MINIFI_VALIDATOR_DATA_SIZE, minifi_validator_MINIFI_VALIDATOR_INTEGER, + minifi_validator_MINIFI_VALIDATOR_NON_BLANK, minifi_validator_MINIFI_VALIDATOR_PORT, + minifi_validator_MINIFI_VALIDATOR_TIME_PERIOD, + minifi_validator_MINIFI_VALIDATOR_UNSIGNED_INTEGER, +}; use std::ptr; #[allow(dead_code)] // these c_ vecs are holding the values referenced from the properties, so they live long enough for registration @@ -94,11 +101,19 @@ impl Property { description: property.description.as_minifi_c_type(), is_required: property.is_required, is_sensitive: property.is_sensitive, - default_value: if def_value.data == std::ptr::null() { std::ptr::null() } else { def_value }, + default_value: if def_value.data == std::ptr::null() { + std::ptr::null() + } else { + def_value + }, allowed_values_count: allowed_values.len(), allowed_values_ptr: allowed_values.as_ptr(), validator: property.validator.as_minifi_c_type(), - allowed_type: if allowed_type.data == std::ptr::null() { std::ptr::null() } else { allowed_type }, + allowed_type: if allowed_type.data == std::ptr::null() { + std::ptr::null() + } else { + allowed_type + }, supports_expression_language: property.supports_expr_lang, } }) diff --git a/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs b/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs index 64fbaf95f1..5684dbc7a9 100644 --- a/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs +++ b/minifi_rust/minifi_native/src/c_ffi/c_ffi_streams.rs @@ -1,5 +1,7 @@ +use minifi_native_sys::{ + minifi_input_stream, minifi_input_stream_read, minifi_output_stream, minifi_output_stream_write, +}; use std::io::{BufRead, Error, ErrorKind, Read}; -use minifi_native_sys::{minifi_input_stream, minifi_input_stream_read, minifi_output_stream, minifi_output_stream_write}; #[derive(Debug)] pub struct CffiInputStream<'a> { diff --git a/minifi_rust/minifi_native/src/lib.rs b/minifi_rust/minifi_native/src/lib.rs index 6f50d9beeb..37efc942a4 100644 --- a/minifi_rust/minifi_native/src/lib.rs +++ b/minifi_rust/minifi_native/src/lib.rs @@ -30,7 +30,7 @@ pub use api::logger::{LogLevel, Logger}; pub use api::property::{GetControllerService, GetProperty, Property}; -pub use api::provided_interface::{ProvidedInterface, ControllerServiceApi}; +pub use api::provided_interface::{ControllerServiceApi, ProvidedInterface}; pub use api::process_session::IoState; diff --git a/minifi_rust/minifi_native/src/mock/mock_process_context.rs b/minifi_rust/minifi_native/src/mock/mock_process_context.rs index b939319d9c..5c4a3ffab6 100644 --- a/minifi_rust/minifi_native/src/mock/mock_process_context.rs +++ b/minifi_rust/minifi_native/src/mock/mock_process_context.rs @@ -1,5 +1,8 @@ use crate::api::{ProcessContext, RawControllerService}; -use crate::{ComponentIdentifier, ControllerServiceApi, EnableControllerService, GetAttribute, MinifiError, MockFlowFile, Property}; +use crate::{ + ComponentIdentifier, ControllerServiceApi, EnableControllerService, GetAttribute, MinifiError, + MockFlowFile, Property, +}; use std::any::Any; use std::borrow::Cow; use std::collections::HashMap; @@ -97,7 +100,10 @@ impl ProcessContext for MockProcessContext { } } - fn get_controller_service_api(&self, _property: &Property) -> Result>, MinifiError> { + fn get_controller_service_api( + &self, + _property: &Property, + ) -> Result>, MinifiError> { todo!() } diff --git a/minifi_rust/minifi_native_macros/src/lib.rs b/minifi_rust/minifi_native_macros/src/lib.rs index f093b82a7a..4b486e84f9 100644 --- a/minifi_rust/minifi_native_macros/src/lib.rs +++ b/minifi_rust/minifi_native_macros/src/lib.rs @@ -1,6 +1,6 @@ use proc_macro::TokenStream; use quote::quote; -use syn::{DeriveInput, parse_macro_input, ItemTrait}; +use syn::{DeriveInput, ItemTrait, parse_macro_input}; #[proc_macro_derive(ComponentIdentifier)] pub fn derive_component_identifier(input: TokenStream) -> TokenStream {