diff --git a/.github/workflows/release-version.yaml b/.github/workflows/release-version.yaml new file mode 100644 index 0000000..c409275 --- /dev/null +++ b/.github/workflows/release-version.yaml @@ -0,0 +1,115 @@ +# Stamps the version of a published GitHub release into the source tree: +# - src/Directory.Build.props (drives AssemblyVersion / FileVersion) +# - changelog.md the in-progress section gets the version and release date +# and commits the result back to the default branch. +# +# Note: the release tag points at the commit that was tagged, which is *before* this bump. +# The bump therefore lands on the default branch after the release, not on the tag itself. + +name: Update Version on Release + +on: + release: + types: [published] + # Manual re-run / dry run, e.g. after fixing up a release by hand. + workflow_dispatch: + inputs: + version: + description: "Version to stamp, e.g. 0.7.0 (or v0.7.0)" + required: true + type: string + +permissions: + contents: write + +# Never let two version bumps race each other onto the default branch. +concurrency: + group: release-version + cancel-in-progress: false + +jobs: + bump: + name: Update Version + # Pre-releases keep the numbering of the release they lead up to, so they must not + # bump the released version. Manual runs are always intentional, so they always apply. + if: github.event_name == 'workflow_dispatch' || github.event.release.prerelease == false + runs-on: ubuntu-latest + steps: + - name: Checkout default branch + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ github.event.repository.default_branch }} + + - name: Stamp version into sources + shell: pwsh + # The tag name is repository data, not trusted input: pass it through the + # environment rather than interpolating it into the script body. + env: + RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.version || github.event.release.tag_name }} + RELEASE_DATE: ${{ github.event.release.published_at }} + run: | + $tag = $env:RELEASE_TAG.Trim() + $bare = $tag -replace '^[vV]', '' + if ($bare -notmatch '^(\d+)\.(\d+)(?:\.(\d+))?$') { + throw "Tag '$tag' is not a plain version number (expected 0.7, 0.7.0, v0.7.0). Nothing was changed." + } + # AssemblyVersion needs all parts, so a two-part tag gets a .0 patch. + $patch = if ($Matches[3]) { $Matches[3] } else { '0' } + $version = "$($Matches[1]).$($Matches[2]).$patch" + + $date = if ($env:RELEASE_DATE) { + [datetime]::Parse($env:RELEASE_DATE, [cultureinfo]::InvariantCulture).ToString('yyyy-MM-dd') + } else { + (Get-Date).ToString('yyyy-MM-dd') + } + Write-Host "Stamping version $version (released $date)" + + # --- src/Directory.Build.props --------------------------------------------- + $propsPath = 'src/Directory.Build.props' + $props = Get-Content $propsPath -Raw + if ($props -notmatch '[^<]*') { + throw "No element found in $propsPath." + } + $props = $props -replace '(?<=)[^<]*(?=)', $version + Set-Content -Path $propsPath -Value $props -NoNewline + + # --- changelog.md ---------------------------------------------------------- + # Matches the in-progress heading in either form -- "## [Unreleased]" or + # "## [0.7.0] - Unreleased" -- and leaves already-dated releases alone. + $changelogPath = 'changelog.md' + $changelog = Get-Content $changelogPath -Raw + $heading = [regex]'(?m)^##\s+\[(?:Unreleased\]|\d+\.\d+(?:\.\d+)?\]\s+-\s+Unreleased)\s*$' + # Re-running the job for a release that was already stamped must not append a + # second, empty section for the same version. + $alreadyDated = [regex]::IsMatch($changelog, "(?m)^##\s+\[$([regex]::Escape($version))\]\s+-\s+\d{4}-\d{2}-\d{2}\s*$") + if ($alreadyDated) { + Write-Host "$changelogPath already has a dated section for $version -- leaving it as is." + } elseif ($heading.IsMatch($changelog)) { + $replacement = "## [Unreleased]`n`n## [$version] - $date" + # Count 1: only the topmost in-progress section becomes this release. + $changelog = $heading.Replace($changelog, $replacement, 1) + Set-Content -Path $changelogPath -Value $changelog -NoNewline + } else { + Write-Warning "No in-progress section found in $changelogPath -- only the version was updated." + } + + "Stamped version ``$version`` (released $date)." >> $env:GITHUB_STEP_SUMMARY + + - name: Commit and push + shell: pwsh + env: + RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.version || github.event.release.tag_name }} + run: | + $files = @('src/Directory.Build.props', 'changelog.md') + # --porcelain rather than "git diff --quiet": a legitimate non-zero exit code + # from git aborts the whole step under pwsh's error handling. + if (-not (git status --porcelain -- $files)) { + Write-Host "Sources already carry this version, nothing to commit." + "Sources already up to date -- no commit made." >> $env:GITHUB_STEP_SUMMARY + exit 0 + } + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add -- $files + git commit -m "chore: Set version to $($env:RELEASE_TAG) [skip ci]" + git push diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml new file mode 100644 index 0000000..ad71b95 --- /dev/null +++ b/.github/workflows/unit-tests.yaml @@ -0,0 +1,57 @@ +# Runs the xUnit test projects on every push to an open pull request. + +name: Unit Tests + +on: + pull_request: + branches: + - main + # Allows running the tests manually from the Actions tab. + workflow_dispatch: + +permissions: + contents: read + +# A new push to the same PR cancels the run still in flight for the previous one. +concurrency: + group: unit-tests-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Unit Tests + # The solution contains RTDSimulatorDesktopApp, a WinForms project targeting + # net10.0-windows, so the whole solution only restores and builds on Windows. + runs-on: windows-latest + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + + - name: Setup .NET + uses: actions/setup-dotnet@9a946fdbd5fb07b82b2f5a4466058b876ab72bb2 # v5.3.0 + with: + dotnet-version: 10.0.x + + - name: Restore + run: dotnet restore src/RealTimeDataSimulator.sln + + - name: Build + run: dotnet build src/RealTimeDataSimulator.sln --configuration Release --no-restore + + # No LogFileName: both test projects write into the same results directory + # and a fixed name would make the second run overwrite the first one's trx. + - name: Test + run: > + dotnet test src/RealTimeDataSimulator.sln + --configuration Release + --no-build + --logger trx + --results-directory TestResults + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: test-results-${{ github.run_id }} + path: TestResults/ + retention-days: 7 diff --git a/.gitignore b/.gitignore index 8a30d25..4880d58 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Test templates +*.template + ## Ignore Visual Studio temporary files, build results, and ## files generated by popular Visual Studio add-ons. ## diff --git a/README.md b/README.md index 64660fd..3085abc 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,159 @@ # Real-Time Data Simulator -Windows Desktop application to generate and stream data into EventHub. +Windows desktop app **and** command-line tool (`rtdsim`) that generate dynamic data from a JSON template and stream it to Azure messaging and analytics targets: **Azure Event Hubs**, **Azure Service Bus**, and **Azure Data Explorer (Kusto) / Microsoft Fabric Eventhouse**. # Screenshot - + # Main Features -- Configurable target EventHub -- Configurable size of workload (amount of messages to be sent) +- Multiple target [connectors](#connectors): Azure Event Hubs, Azure Service Bus, and Azure Data Explorer (Kusto) / Fabric Eventhouse +- Desktop GUI and a scriptable [CLI](#cli) (`rtdsim`) +- Configurable size of workload (amount of messages to be sent) and parallelism - Easily editable JSON payload -- Sending defined payload into the target endpoint - Load/Save payload as a file locally - Runtime evaluation of pre-defined variables - Runtime evaluation of expression in C# (slower) +- Built-in expression helpers (e.g. random strings, sub-second timestamps) - Microsoft Entra ID (Interactive Browser) Authentication # Scenarios -1. Generate dynamic data based on JSON template and send it to Microsoft Fabric EventHub (custom endpoint). -2. Generate dynamic data based on JSON template and send it to EventHub. +1. Generate dynamic data from a JSON template and send it to a **Microsoft Fabric EventStream** custom endpoint (Event Hubs-compatible). +2. Generate dynamic data and send it to an **Azure Event Hub**. +3. Generate dynamic data and send it to an **Azure Service Bus** queue or topic (e.g. to load-test consumers or messaging pipelines). +4. Generate dynamic data and ingest it into an **Azure Data Explorer (Kusto)** table or a **Microsoft Fabric Eventhouse** for KQL analytics / dashboards. -# Configuration -## Endpoint Connection String -Example of connection string for **Microsoft Fabric**: -`Endpoint=sb://esehlnx12nbglyi6y2kwx9.servicebus.windows.net/;SharedAccessKeyName=key_aaee3a28-***;SharedAccessKey=vFX2zae****=;EntityPath=es_5c952fa4-***` +# Connectors +The simulator can send generated load to several targets. Pick the connector in the GUI (**Destination → Service**) or with `--target` on the CLI; each connector declares its own parameters. -Example of connection string for **Azure Event Hub** (Interactive Browser Authentication): -`sb://ehplayerdev.servicebus.windows.net/` +| Connector | `--target` | Sends to | Authentication | +|--|--|--|--| +| [Azure Event Hubs](#azure-event-hubs) | `eventhubs` | An event hub (incl. Fabric EventStream custom endpoints) | SAS connection string **or** Microsoft Entra ID | +| [Azure Service Bus](#azure-service-bus) | `servicebus` | A queue or topic | SAS connection string **or** Microsoft Entra ID | +| [Azure Data Explorer / Kusto — Streaming](#azure-data-explorer-kusto--fabric-eventhouse) | `kusto-streaming` | A database table (incl. Fabric Eventhouse) | Microsoft Entra ID | +| [Azure Data Explorer / Kusto — Queued](#azure-data-explorer-kusto--fabric-eventhouse) | `kusto-queued` | A database table (incl. Fabric Eventhouse) | Microsoft Entra ID | + +Two authentication styles are supported by the messaging connectors: +- **SAS connection string** — paste the full `Endpoint=sb://...;SharedAccessKey=...` string into the connection field. No sign-in needed. +- **Microsoft Entra ID** — click **Azure: Sign in** in the GUI (Interactive Browser). The connection field can then be just the fully-qualified namespace (`.servicebus.windows.net`); a full connection string is also accepted, the host is extracted from it. The sign-in is **cached across runs** in an encrypted token cache (DPAPI), so you only log in once — until the refresh token expires. While signed in, the button shows **Azure: Sign out** (hover shows the signed-in user); clicking it (or **File → Azure: Sign out**) clears the cached sign-in. (The CLI uses `DefaultAzureCredential`, which reuses your `az login` session, so it also doesn't prompt per run.) + +## Azure Event Hubs +Sends each generated message as an Event Hubs event (batched). + +| Parameter | GUI field | CLI | Description | +|--|--|--|--| +| `connection` | Connection string / Namespace | `-p connection=` | SAS connection string, or the namespace when using Entra ID | +| `eventHub` | Event Hub name | `-p eventHub=` | Target event hub (entity) name | + +**Connection string (SAS):** +``` +Endpoint=sb://.servicebus.windows.net/;SharedAccessKeyName=;SharedAccessKey= +``` +Set **Event Hub name** to the hub you are sending to. + +**Microsoft Fabric (EventStream custom endpoint):** the Fabric portal's *custom endpoint → SAS key* view gives a connection string that already contains an `EntityPath`, e.g. +``` +Endpoint=sb://<...>.servicebus.windows.net/;SharedAccessKeyName=key_***;SharedAccessKey=***;EntityPath=es_5c952fa4-*** +``` +Put that same `es_***` value in the **Event Hub name** field (it must match the `EntityPath`). + +**CLI example:** +```powershell +rtdsim --target eventhubs ` + -p connection="Endpoint=sb://ns.servicebus.windows.net/;SharedAccessKeyName=send;SharedAccessKey=***" ` + -p eventHub=my-hub ` + --template .\payload.json --messages 100 +``` + +## Azure Service Bus +Sends each generated message to a queue or topic (batched). + +| Parameter | GUI field | CLI | Description | +|--|--|--|--| +| `connection` | Connection string / Namespace | `-p connection=` | SAS connection string, or the namespace when using Entra ID | +| `entity` | Queue / Topic name | `-p entity=` | Target queue or topic name | + +**Connection string (SAS):** +``` +Endpoint=sb://.servicebus.windows.net/;SharedAccessKeyName=;SharedAccessKey= +``` +Set **Queue / Topic name** to the destination entity. A namespace-level policy (e.g. `RootManageSharedAccessKey`) can send to any entity; an entity-scoped policy's connection string contains `;EntityPath=` and only works for that entity. + +**CLI example:** +```powershell +rtdsim --target servicebus ` + -p connection="Endpoint=sb://ns.servicebus.windows.net/;SharedAccessKeyName=send;SharedAccessKey=***" ` + -p entity=my-queue ` + --template .\payload.json --messages 100 +``` + +## Azure Data Explorer (Kusto) / Fabric Eventhouse +Ingests each batch into a database **table** as `multijson`. Two ingestion modes are available as separate connectors: + +| `--target` | GUI service | Ingestion | Setup required | +|--|--|--|--| +| `kusto-streaming` | Azure Data Explorer — Streaming ingestion | [Streaming](https://learn.microsoft.com/en-us/azure/data-explorer/kusto/management/streaming-ingestion-policy) — low latency, sends to the engine endpoint | **Streaming ingestion policy must be enabled** on the cluster and target table | +| `kusto-queued` | Azure Data Explorer — Queued ingestion | [Queued](https://learn.microsoft.com/en-us/azure/data-explorer/kusto/api/netfx/about-kusto-ingest#queued-ingestion) — batched server-side, sends to the `ingest-` data-management endpoint | **None** — works out of the box | + +Use **streaming** for near-real-time delivery; use **queued** when you don't want to (or can't) enable the streaming policy, e.g. for throughput measurements on a stock cluster. Both take the same parameters: + +| Parameter | GUI field | CLI | Description | +|--|--|--|--| +| `clusterUri` | Cluster URI | `-p clusterUri=` | The cluster's query/engine URI (see below) | +| `database` | Database | `-p database=` | Target database name | +| `table` | Table | `-p table=` | Target table name | + +**Cluster URI** — provide the query/engine URI for **both** connectors; the queued connector derives the `ingest-` data-management endpoint automatically (and either connector also accepts a URI already in the other form): +- **Azure Data Explorer:** `https://..kusto.windows.net` +- **Microsoft Fabric Eventhouse:** the **Query URI** shown on the Eventhouse / KQL Database page (form `https://<...>.kusto.fabric.microsoft.com`). + +**Authentication — Microsoft Entra ID only:** +- **GUI:** click **Azure: Sign in** before running. +- **CLI:** uses the ambient credential chain (`DefaultAzureCredential`) — e.g. `az login`, environment variables, or a managed identity. + +The identity needs at least **Table/Database Ingestor** rights on the target. + +**Payload format, schema & mappings:** messages are ingested as `multijson`. Without an ingestion mapping, Kusto maps JSON **by property name (case-sensitive)** to columns of the same name — any property that doesn't match a column is dropped, and any column without a matching property is set to **null**. So if your field names don't line up with the table, ingestion *succeeds* but every row is **blank/null** (this is not an ingestion failure, so `--verify-ingestion` won't flag it). Two ways to get real data in: + +1. **Match the names** — make each JSON property name equal the target column name, e.g. + ```json + { "StringColumn": "test-{{$RandomString(6)}}", "EventTime": "{{$DateTime.UtcNow.ToString("O")}}" } + ``` +2. **Use an ingestion mapping** — create a JSON mapping on the table and pass its name via the optional **Ingestion mapping** field (`-p mapping=`). This lets arbitrary field names map to the intended columns: + ```kusto + .create table MyTable ingestion json mapping "MyMapping" + '[{"column":"Fuel","Properties":{"Path":"$.FuelType"}}, {"column":"Gen","Properties":{"Path":"$.Generation"}}]' + ``` + +**Prerequisite (streaming only) — enable streaming ingestion** on both the database/cluster policy and the target table, otherwise `kusto-streaming` ingestion fails. `kusto-queued` needs no such setup. +```kusto +.alter table policy streamingingestion enable +``` + +**Error handling & verification.** Payloads are checked to be valid JSON before sending (a malformed template fails immediately with a clear error). For **streaming**, a schema/data failure is reported by the service on the send call and surfaced as an error. For **queued**, ingestion happens asynchronously server-side, so a schema mismatch is *not* visible at send time — the run can report success while rows are rejected. To catch that, pass `--verify-ingestion` (CLI) or tick **Verify ingestion** (GUI): after the run it queries `.show ingestion failures` for the target table and reports any failures. Queued failures can lag, so re-check later if in doubt: +```kusto +.show ingestion failures | where Table == "
" | order by FailedOn desc +``` + +**CLI examples:** +```powershell +# Streaming ingestion (requires the streaming policy) +rtdsim --target kusto-streaming ` + -p clusterUri=https://mycluster.westeurope.kusto.windows.net ` + -p database=Telemetry -p table=Events ` + --template .\payload.json --messages 100 --parallelism 3 + +# Queued ingestion (no setup required) +rtdsim --target kusto-queued ` + -p clusterUri=https://mycluster.westeurope.kusto.windows.net ` + -p database=Telemetry -p table=Events ` + --template .\payload.json --messages 100 --parallelism 3 +``` # Tokens The biggest advantage of the application is the ability to generate value for the tokens listed in the PAYLOAD definition. Data generation takes place on the fly for each message prepared for sending. ## Variables Format of a variable: `{{VariableName}}` -The following examples depict predefined Variables you can use as Tokens: +Variables are **defined in a TOML file** (see [Variable definitions](#variable-definitions-toml)), so you can add, remove or retune them without rebuilding. The built-in defaults are: |Variable | Generates | |--|--| @@ -42,6 +164,41 @@ The following examples depict predefined Variables you can use as Tokens: | `{{FuelType(MessageIndex)}}` | Value selected on `MessageIndex` from list: BIOMASS,CCGT,COAL,INTELEC,INTEW,INTFR,INTIFA2,INTIRL,INTNED,INTNEM,INTNSL,INTVKL,NPSHYD,NUCLEAR,OCGT,OIL,OTHER,PS,WIND | | `{{SettlementPeriod}}` | Hardcoded Int value: `48` | +### Variable definitions (TOML) +Definitions live in a `variables.toml` file shipped next to the executable. Each **table** defines one variable; the table name is the token (quote names that contain dots or parentheses, e.g. `["DateTime.Now"]`). + +```toml +[UserId] +type = "randomInt" # integer in [min, max) — max exclusive +min = 5000 +max = 5100 + +[Device] +type = "randomItem" # a random element of items +items = ["mobile", "tablet", "pc"] + +["FuelType(MessageIndex)"] +type = "indexedItem" # items[messageIndex % count] +items = ["BIOMASS", "CCGT", "COAL"] + +["DateTime.Now"] +type = "dateTimeNow" # current local time; optional `format` (default "O") + +[SettlementPeriod] +type = "literal" # the fixed string `value` +value = "48" +``` + +| type | Parameters | Produces | +|--|--|--| +| `randomInt` | `min`, `max` | Random integer in `[min, max)` (max exclusive) | +| `randomItem` | `items` | A random element of `items` | +| `indexedItem` | `items` | `items[messageIndex % items.Count]` | +| `dateTimeNow` | `format` (optional, default `"O"`) | Current local date/time | +| `literal` | `value` | The fixed string `value` | + +**How the file is resolved:** the apps load `--variables ` if given (CLI) or a file chosen via **File → Load Variable Definitions...** (GUI); otherwise `variables.toml` next to the executable; otherwise the built-in defaults. + ## C# Expressions Format of an expression: `{{$Expression}}` Expressions are **slower** in generating values, but more flexible. Use can use any C# compatible code from preloaded namespaces (for example: `System`). @@ -54,10 +211,113 @@ Expressions are being evaluated using [Microsoft.CodeAnalysis.CSharp.Scripting]( | `{{$new Random().Next(100,200)}}` | Random numeric (int) value from range 100-199 | | `{{$DateTime.Now.AddSeconds(-5).ToString("s")+"Z"}}` | Date & time 5 seconds ago in [Sortable date/time pattern](https://learn.microsoft.com/en-us/dotnet/standard/base-types/standard-date-and-time-format-strings#Sortable) with 'Z' at the end. | | `{{$DateTime.Now.ToString("dd/MM/yyyy")}}` | Current date & time formatted to `31/12/2020` | -| `{{$DateTime.Now.Hour*2+Math.Floor(DateTime.Now.Minute/30d)` | Calculation based on Current Time | +| `{{$DateTime.Now.Hour*2+Math.Floor(DateTime.Now.Minute/30d)}}` | Calculation based on Current Time | +| `{{$DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.ffffff")+"Z"}}` | UTC timestamp with **microsecond** precision (see [Sub-second timestamps](#sub-second-timestamps)) | +| `{{$RandomString(10)}}` | 10-character random alphanumeric string (see [Expression helpers](#expression-helpers)) | + +## Expression helpers +In addition to the `System` namespace, expressions can call these built-in helper functions. + +### RandomString +`RandomString(length, charset?)` generates a random string. + +| Argument | Required | Description | +|--|--|--| +| `length` | yes | Number of characters to generate. `0` (or less) yields an empty string. | +| `charset` | no | The set of characters to choose from. Defaults to `A–Z`, `a–z`, `0–9`. | + +| Expression | Generates | +|--|--| +| `{{$RandomString(8)}}` | 8 random alphanumeric characters, e.g. `wD4QUgYn` | +| `{{$RandomString(6, "ABCDEF0123456789")}}` | 6 random hex characters, e.g. `79DEF1` | +| `id-{{$RandomString(4)}}` | Composes with literals, e.g. `id-LxRU` | + +> **Note:** each *distinct* expression is evaluated once per message, and all identical occurrences are replaced with the same value. To emit two independent random values in one message, make the tokens differ slightly — e.g. `{{$RandomString(8)}}` and `{{$RandomString( 8)}}`. + +## Sub-second timestamps +The sortable format `"s"` has **no** fractional seconds. For higher resolution, use custom [date/time format specifiers](https://learn.microsoft.com/en-us/dotnet/standard/base-types/custom-date-and-time-format-strings) — `ffffff` = microseconds (6 digits), `fffffff` = 100-nanosecond ticks (7 digits, .NET's maximum): + +| Expression | Generates | +|--|--| +| `{{$DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.ffffff")+"Z"}}` | `2026-07-01T09:20:15.123456Z` (microseconds) | +| `{{$DateTime.UtcNow.ToString("O")}}` | `2026-07-01T09:20:15.1234567Z` (round-trip, 100-ns ticks) | + +> **Use `DateTime.UtcNow` (not `DateTime.Now`) when appending `Z`.** `DateTime.Now` returns *local* time; tagging it with `Z` mislabels it as UTC. Also note that the actual precision is bounded by the OS clock (~1–15 ms on Windows), so trailing microsecond digits may be padding rather than distinct values. + +# CLI + +## Usage +```text +rtdsim - Real-Time Data Simulator CLI + +Generates and sends load to a target service. + +Usage: + rtdsim --template [--target ] [--param key=value ...] [options] + +Required: + -t, --template Path to the message template file. + +Options: + --target Target service (default 'eventhubs'). + -p, --param key=value Connector parameter (repeatable). See per-target list below. + --variables Variable definitions TOML (default: variables.toml next to the exe). + --parallelism Number of concurrent senders (default 1). + --batches Batches per sender (default 1). + --messages Messages per batch (default 1). + --wait Seconds to wait between batches (default 0). + --verify-ingestion After the run, report Kusto ingestion failures (queued targets). + -h, --help Show this help. + +Targets and their parameters: + --target eventhubs (Event Hubs) + -p connection= Connection string / Namespace [required] + SAS connection string, or the fully-qualified namespace when signed in with Azure Identity. + -p eventHub= Event Hub name [required] + --target servicebus (Azure Service Bus) + -p connection= Connection string / Namespace [required] + SAS connection string, or the fully-qualified namespace when signed in with Azure Identity. + -p entity= Queue / Topic name [required] + --target kusto-streaming (Azure Data Explorer - Streaming ingestion) + -p clusterUri= Cluster URI [required] + Engine/query cluster URI. Streaming ingestion must be enabled on the cluster and target table. + -p database= Database [required] + -p table= Table [required] + --target kusto-queued (Azure Data Explorer - Queued ingestion) + -p clusterUri= Cluster URI [required] + Engine/query cluster URI (the 'ingest-' data-management endpoint is derived automatically). No streaming policy required. + -p database= Database [required] + -p table= Table [required] +``` + +## Example CLI run +```text +& rtdsim.exe --target kusto-streaming ` + -p clusterUri=https://mycluster.westeurope.kusto.windows.net ` + -p database=test ` + -p table=test ` + --template .\test.template ` + --messages 100 --parallelism 3 +Target : Azure Data Explorer - Streaming ingestion +Cluster URI : https://mycluster.westeurope.kusto.windows.net +Database : test +Table : test +Parallelism: 3 +Batches : 1 per sender (3 total) +Messages : 100 per batch (300 total) +Wait time : 0s between batches + +Batches: 2/3 | Messages: 200 | TPS: 61.69 | 36,600 bytes (0.06 Mbps) + +Completed: sent 300 messages in 3 batches. +Total time: 00:00:03.2427018 | TPS: 92.52 | 36,600 bytes (0.09 Mbps) +``` # References - [Get a custom endpoint without creating an EventHub for EventStreams](https://www.youtube.com/watch?v=ftb2nN3eukg) +- [Azure Service Bus connection strings & SAS](https://learn.microsoft.com/en-us/azure/service-bus-messaging/service-bus-sas) +- [Azure Data Explorer — streaming ingestion policy](https://learn.microsoft.com/en-us/azure/data-explorer/kusto/management/streaming-ingestion-policy) +- [Microsoft Fabric Eventhouse overview](https://learn.microsoft.com/en-us/fabric/real-time-intelligence/eventhouse) - Similar project: [Mockingbird](https://www.tinybird.co/blog-posts/mockingbird-announcement-mock-data-generator) # Release Notes diff --git a/changelog.md b/changelog.md index 3d53c9e..180708b 100644 --- a/changelog.md +++ b/changelog.md @@ -4,6 +4,20 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [0.7.0] - Unreleased +### Added +* Kusto connectors: optional **Ingestion mapping** parameter (`-p mapping=`) referencing a JSON ingestion mapping on the table. Without a mapping, Kusto maps JSON to columns by name (case-sensitive), so mismatched field names silently ingest as null/blank rows; a mapping lets arbitrary field names land in the right columns. +* Kusto ingestion error handling: payloads are validated as JSON before sending (malformed templates fail fast), and synchronously-reported ingestion failures (e.g. streaming schema mismatches) now surface as errors instead of being ignored. Added an opt-in post-run check for asynchronous (queued) failures — `--verify-ingestion` (CLI) / **Verify ingestion** (GUI) — which queries `.show ingestion failures` and reports them. +* GUI: **persistent Microsoft Entra ID sign-in**. The interactive sign-in is now stored in an encrypted, on-disk token cache with a saved authentication record, so it is silently reused across app runs (until the refresh token expires). The **Azure: Sign in** button toggles to **Azure: Sign out** while signed in (showing the signed-in user on hover), and **File → Azure: Sign out** does the same — clearing the cached sign-in. +* Azure Data Explorer (Kusto) **queued ingestion** connector (`--target kusto-queued`) — batched server-side ingestion that needs no streaming policy, so it works against any cluster/table without setup. Automatically targets the `ingest-` data-management endpoint. +* Soft-coded variable definitions: the built-in variables (`UserId`, `ProductId`, `Device`, `DateTime.Now`, `FuelType(MessageIndex)`, `SettlementPeriod`) are now defined in a `variables.toml` file instead of being hardcoded. Each variable is a TOML table named by its token, with a `type` of `randomInt`, `randomItem`, `indexedItem`, `dateTimeNow` or `literal`. +* GUI: **File → Load Variable Definitions...** to load a custom definitions file at runtime. +* CLI: `--variables ` option to use a custom definitions file. + +### Changed +* Renamed the streaming Kusto connector to `--target kusto-streaming` (was `--target kusto`). **Breaking:** `--target kusto` no longer resolves. +* Variable definitions are resolved from `--variables` / the GUI menu, then a `variables.toml` shipped next to the executable, then the built-in embedded defaults. + ## [0.5.2] - 2024-10-28 * Fixed: Reset `_TotalSizeInBytes` counter before each run to avoid overpriced Mbps after the first run diff --git a/media/real-time-data-simulator-ver-0.6-main-screen.png b/media/real-time-data-simulator-ver-0.6-main-screen.png new file mode 100644 index 0000000..6939707 Binary files /dev/null and b/media/real-time-data-simulator-ver-0.6-main-screen.png differ diff --git a/src/Directory.Build.props b/src/Directory.Build.props new file mode 100644 index 0000000..da1b0a8 --- /dev/null +++ b/src/Directory.Build.props @@ -0,0 +1,17 @@ + + + + + 0.6.0 + $(VersionPrefix) + $(VersionPrefix) + + + diff --git a/src/RTDSimulator.Cli/CliOptions.cs b/src/RTDSimulator.Cli/CliOptions.cs new file mode 100644 index 0000000..e7ccd7f --- /dev/null +++ b/src/RTDSimulator.Cli/CliOptions.cs @@ -0,0 +1,158 @@ +namespace RTDSimulator.Cli; + +/// +/// Parsed command-line options for the simulator CLI. Connection parameters are kept +/// generic (a key/value bag) so the set of options follows whichever connector the +/// --target selects, rather than being hard-coded here. +/// +internal sealed class CliOptions +{ + public required string TargetKey { get; init; } + public required Dictionary Parameters { get; init; } + public required string TemplatePath { get; init; } + public string? VariablesPath { get; init; } + public bool VerifyIngestion { get; init; } + public int Parallelism { get; init; } = 1; + public int BatchesPerSender { get; init; } = 1; + public int MessagesPerBatch { get; init; } = 1; + public int WaitTimeSeconds { get; init; } + + /// + /// Parses the supplied arguments. Returns null when parsing fails or + /// help was requested; in the failure case is set. + /// + public static CliOptions? Parse(string[] args, out string? error) + { + error = null; + + string targetKey = "eventhubs"; + string? templatePath = null; + string? variablesPath = null; + bool verifyIngestion = false; + var parameters = new Dictionary(); + int parallelism = 1; + int batches = 1; + int messages = 1; + int wait = 0; + + for (int i = 0; i < args.Length; i++) + { + string arg = args[i]; + string name = arg; + string? inlineValue = null; + + int eq = arg.IndexOf('='); + if (arg.StartsWith("--") && eq > 0) + { + name = arg.Substring(0, eq); + inlineValue = arg.Substring(eq + 1); + } + + switch (name) + { + case "-h": + case "--help": + return null; + + case "--target": + targetKey = NextValue(args, ref i, inlineValue, name, ref error) ?? targetKey; + break; + case "-t": + case "--template": + templatePath = NextValue(args, ref i, inlineValue, name, ref error); + break; + case "--variables": + variablesPath = NextValue(args, ref i, inlineValue, name, ref error); + break; + case "--verify-ingestion": + verifyIngestion = true; + break; + case "-p": + case "--param": + AddParam(NextValue(args, ref i, inlineValue, name, ref error), parameters, ref error); + break; + case "--parallelism": + parallelism = ParseInt(NextValue(args, ref i, inlineValue, name, ref error), name, ref error, parallelism); + break; + case "--batches": + batches = ParseInt(NextValue(args, ref i, inlineValue, name, ref error), name, ref error, batches); + break; + case "--messages": + messages = ParseInt(NextValue(args, ref i, inlineValue, name, ref error), name, ref error, messages); + break; + case "--wait": + wait = ParseInt(NextValue(args, ref i, inlineValue, name, ref error), name, ref error, wait); + break; + default: + error = $"unknown argument '{arg}'."; + return null; + } + + if (error is not null) + return null; + } + + if (string.IsNullOrWhiteSpace(templatePath)) + { + error = "missing required --template."; + return null; + } + + return new CliOptions + { + TargetKey = targetKey, + Parameters = parameters, + TemplatePath = templatePath, + VariablesPath = variablesPath, + VerifyIngestion = verifyIngestion, + Parallelism = parallelism, + BatchesPerSender = batches, + MessagesPerBatch = messages, + WaitTimeSeconds = wait, + }; + } + + private static void AddParam(string? value, Dictionary parameters, ref string? error) + { + if (error is not null) + return; + if (value is null) + return; + + int eq = value.IndexOf('='); + if (eq <= 0) + { + error = $"--param expects 'key=value', got '{value}'."; + return; + } + + string key = value.Substring(0, eq); + string val = value.Substring(eq + 1); + parameters[key] = val; + } + + private static string? NextValue(string[] args, ref int i, string? inlineValue, string name, ref string? error) + { + if (inlineValue is not null) + return inlineValue; + + if (i + 1 >= args.Length) + { + error = $"missing value for '{name}'."; + return null; + } + + return args[++i]; + } + + private static int ParseInt(string? value, string name, ref string? error, int fallback) + { + if (error is not null) + return fallback; + if (int.TryParse(value, out int parsed)) + return parsed; + + error = $"'{name}' expects an integer value."; + return fallback; + } +} diff --git a/src/RTDSimulator.Cli/Program.cs b/src/RTDSimulator.Cli/Program.cs new file mode 100644 index 0000000..4d5eaf6 --- /dev/null +++ b/src/RTDSimulator.Cli/Program.cs @@ -0,0 +1,267 @@ +using System.Diagnostics; +using RTDSimulator.Core; +using RTDSimulator.EventHubs; +using RTDSimulator.Kusto; +using RTDSimulator.ServiceBus; + +namespace RTDSimulator.Cli; + +internal static class Program +{ + private const double BytesToMbps = 1024 * 1024 / 8; + + /// Available connectors. Add new connectors here. + private static readonly IReadOnlyList Connectors = new ITargetConnector[] + { + new EventHubsConnector(), + new ServiceBusConnector(), + new KustoStreamingConnector(), + new KustoQueuedConnector(), + }; + + private static async Task Main(string[] args) + { + var options = CliOptions.Parse(args, out string? error); + if (options is null) + { + if (error is not null) + { + Console.Error.WriteLine($"Error: {error}"); + Console.Error.WriteLine(); + } + PrintUsage(); + return error is null ? 0 : 1; + } + + ITargetConnector? connector = Connectors.FirstOrDefault( + c => string.Equals(c.Key, options.TargetKey, StringComparison.OrdinalIgnoreCase)); + if (connector is null) + { + Console.Error.WriteLine($"Error: unknown --target '{options.TargetKey}'. Known targets: {string.Join(", ", Connectors.Select(c => c.Key))}."); + return 1; + } + + if (!ValidateParameters(connector, options.Parameters, out string? paramError)) + { + Console.Error.WriteLine($"Error: {paramError}"); + Console.Error.WriteLine(); + PrintConnectorParameters(connector); + return 1; + } + + string payload; + try + { + payload = await File.ReadAllTextAsync(options.TemplatePath); + } + catch (Exception ex) + { + Console.Error.WriteLine($"Error: could not read template '{options.TemplatePath}': {ex.Message}"); + return 1; + } + + IReadOnlyList variables; + try + { + variables = ResolveVariables(options.VariablesPath); + } + catch (Exception ex) + { + Console.Error.WriteLine($"Error: could not load variable definitions: {ex.Message}"); + return 1; + } + + long totalBatches = (long)options.Parallelism * options.BatchesPerSender; + long totalMessages = totalBatches * options.MessagesPerBatch; + Console.WriteLine($"Target : {connector.DisplayName}"); + foreach (ConnectionParameter p in connector.Parameters) + { + string value = options.Parameters.GetValueOrDefault(p.Key, ""); + Console.WriteLine($"{p.Label,-12}: {(p.Secret ? Mask(value) : value)}"); + } + Console.WriteLine($"Parallelism: {options.Parallelism}"); + Console.WriteLine($"Batches : {options.BatchesPerSender} per sender ({totalBatches} total)"); + Console.WriteLine($"Messages : {options.MessagesPerBatch} per batch ({totalMessages} total)"); + Console.WriteLine($"Wait time : {options.WaitTimeSeconds}s between batches"); + Console.WriteLine(); + + // Sender clients are thread-safe, so a single connection is shared across all + // senders. (The CLI authenticates via the ambient Azure credential chain + // for connectors that require Azure Identity, e.g. Kusto.) + await using ITargetConnection connection = connector.CreateConnection(options.Parameters); + + using var cts = new CancellationTokenSource(); + Console.CancelKeyPress += (_, e) => + { + e.Cancel = true; + Console.WriteLine(); + Console.WriteLine("Cancellation requested, stopping..."); + cts.Cancel(); + }; + + long batchesSent = 0; + long messagesSent = 0; + long bytesSent = 0; + DateTime runStartUtc = DateTime.UtcNow; + var stopwatch = Stopwatch.StartNew(); + + var generator = new LoadGenerator(new PayloadGenerator(payload, variables)) + { + BatchesNo = options.BatchesPerSender, + EventsPerBatch = options.MessagesPerBatch, + WaitTime = TimeSpan.FromSeconds(options.WaitTimeSeconds), + }; + + generator.BatchSent += (_, e) => + { + long sent = Interlocked.Increment(ref batchesSent); + Interlocked.Add(ref messagesSent, e.MessageCount); + Interlocked.Add(ref bytesSent, e.SizeInBytes); + + double seconds = stopwatch.Elapsed.TotalSeconds; + double tps = seconds > 0 ? messagesSent / seconds : 0; + double mbps = seconds > 0 ? bytesSent / BytesToMbps / seconds : 0; + Console.Write($"\rBatches: {sent}/{totalBatches} | Messages: {messagesSent} | TPS: {tps:F2} | {bytesSent:0,0} bytes ({mbps:F2} Mbps) "); + }; + + int exitCode = 0; + try + { + await generator.Send(connection, options.Parallelism, cts.Token); + } + catch (Exception ex) + { + exitCode = 1; + Console.Error.WriteLine(); + Console.Error.WriteLine($"Error during send: {ex.Message}"); + } + + stopwatch.Stop(); + Console.WriteLine(); + Console.WriteLine(); + string status = cts.IsCancellationRequested ? "Cancelled" : exitCode == 0 ? "Completed" : "Failed"; + double totalSeconds = stopwatch.Elapsed.TotalSeconds; + double finalTps = totalSeconds > 0 ? messagesSent / totalSeconds : 0; + double finalMbps = totalSeconds > 0 ? bytesSent / BytesToMbps / totalSeconds : 0; + Console.WriteLine($"{status}: sent {messagesSent} messages in {batchesSent} batches."); + Console.WriteLine($"Total time: {stopwatch.Elapsed:c} | TPS: {finalTps:F2} | {bytesSent:0,0} bytes ({finalMbps:F2} Mbps)"); + + if (options.VerifyIngestion && connection is IIngestionVerifier verifier) + { + exitCode = await VerifyIngestionAsync(verifier, runStartUtc, exitCode); + } + + return exitCode; + } + + private static async Task VerifyIngestionAsync(IIngestionVerifier verifier, DateTime sinceUtc, int exitCode) + { + Console.WriteLine(); + Console.WriteLine("Verifying ingestion (querying '.show ingestion failures')..."); + try + { + IReadOnlyList failures = await verifier.GetIngestionFailuresSinceAsync(sinceUtc); + if (failures.Count == 0) + { + Console.WriteLine("No ingestion failures reported. Note: queued failures can lag — re-run the check later if in doubt."); + return exitCode; + } + + Console.Error.WriteLine($"{failures.Count} ingestion failure(s) reported:"); + foreach (string failure in failures) + Console.Error.WriteLine($" {failure}"); + return exitCode == 0 ? 1 : exitCode; + } + catch (Exception ex) + { + Console.Error.WriteLine($"Could not query ingestion failures: {ex.Message}"); + return exitCode; + } + } + + private static bool ValidateParameters(ITargetConnector connector, IReadOnlyDictionary values, out string? error) + { + var knownKeys = connector.Parameters.Select(p => p.Key).ToHashSet(StringComparer.OrdinalIgnoreCase); + foreach (string suppliedKey in values.Keys) + { + if (!knownKeys.Contains(suppliedKey)) + { + error = $"unknown parameter '{suppliedKey}' for target '{connector.Key}'."; + return false; + } + } + + foreach (ConnectionParameter p in connector.Parameters) + { + if (p.Required && string.IsNullOrWhiteSpace(values.GetValueOrDefault(p.Key))) + { + error = $"missing required parameter '{p.Key}' ({p.Label}) for target '{connector.Key}'."; + return false; + } + } + + error = null; + return true; + } + + private static string Mask(string value) + => string.IsNullOrEmpty(value) ? value : "***"; + + // Resolution order: explicit --variables path, then a variables.toml next to the + // executable, then the built-in embedded defaults. + private static IReadOnlyList ResolveVariables(string? explicitPath) + { + if (!string.IsNullOrWhiteSpace(explicitPath)) + return VariableDefinitions.Load(explicitPath); + + string bundled = Path.Combine(AppContext.BaseDirectory, VariableDefinitions.DefaultFileName); + if (File.Exists(bundled)) + return VariableDefinitions.Load(bundled); + + return VariableDefinitions.LoadDefaults(); + } + + private static void PrintUsage() + { + Console.WriteLine("rtdsim - Real-Time Data Simulator CLI"); + Console.WriteLine(); + Console.WriteLine("Generates and sends load to a target service."); + Console.WriteLine(); + Console.WriteLine("Usage:"); + Console.WriteLine(" rtdsim --template [--target ] [--param key=value ...] [options]"); + Console.WriteLine(); + Console.WriteLine("Required:"); + Console.WriteLine(" -t, --template Path to the message template file."); + Console.WriteLine(); + Console.WriteLine("Options:"); + Console.WriteLine(" --variables Variable definitions TOML (default: variables.toml next to the exe)."); + Console.WriteLine(" --target Target service (default 'eventhubs')."); + Console.WriteLine(" -p, --param key=value Connector parameter (repeatable). See per-target list below."); + Console.WriteLine(" --parallelism Number of concurrent senders (default 1)."); + Console.WriteLine(" --batches Batches per sender (default 1)."); + Console.WriteLine(" --messages Messages per batch (default 1)."); + Console.WriteLine(" --wait Seconds to wait between batches (default 0)."); + Console.WriteLine(" --verify-ingestion After the run, report Kusto ingestion failures (queued targets)."); + Console.WriteLine(" -h, --help Show this help."); + Console.WriteLine(); + Console.WriteLine("Targets and their parameters:"); + foreach (ITargetConnector connector in Connectors) + { + PrintConnectorParameters(connector); + } + } + + private static void PrintConnectorParameters(ITargetConnector connector) + { + Console.WriteLine($" --target {connector.Key,-12} ({connector.DisplayName})"); + foreach (ConnectionParameter p in connector.Parameters) + { + string flags = p.Required ? "required" : "optional"; + Console.WriteLine($" -p {p.Key}=".PadRight(34) + $"{p.Label} [{flags}]"); + if (!string.IsNullOrEmpty(p.HelpText)) + { + Console.WriteLine($" {p.HelpText}"); + } + } + } +} diff --git a/src/RTDSimulator.Cli/RTDSimulator.Cli.csproj b/src/RTDSimulator.Cli/RTDSimulator.Cli.csproj new file mode 100644 index 0000000..e8d1045 --- /dev/null +++ b/src/RTDSimulator.Cli/RTDSimulator.Cli.csproj @@ -0,0 +1,26 @@ + + + + Exe + net10.0 + enable + enable + rtdsim + RTDSimulator.Cli + https://github.com/Azure-Player/Real-Time-Data-Simulator/ + + + + + PreserveNewest + + + + + + + + + + + diff --git a/src/RTDSimulator.Connectors.Tests/ConnectorTests.cs b/src/RTDSimulator.Connectors.Tests/ConnectorTests.cs new file mode 100644 index 0000000..b41012f --- /dev/null +++ b/src/RTDSimulator.Connectors.Tests/ConnectorTests.cs @@ -0,0 +1,175 @@ +using RTDSimulator.Core; +using RTDSimulator.EventHubs; +using RTDSimulator.Kusto; +using RTDSimulator.ServiceBus; +using Xunit; + +namespace RTDSimulator.Connectors.Tests; + +public class ConnectorTests +{ + // --- Shared descriptor invariants --- + + private static void AssertValidDescriptor(ITargetConnector connector) + { + Assert.False(string.IsNullOrWhiteSpace(connector.Key)); + Assert.False(string.IsNullOrWhiteSpace(connector.DisplayName)); + Assert.NotEmpty(connector.Parameters); + + var keys = connector.Parameters.Select(p => p.Key).ToList(); + Assert.Equal(keys.Count, keys.Distinct(StringComparer.OrdinalIgnoreCase).Count()); + Assert.All(connector.Parameters, p => + { + Assert.False(string.IsNullOrWhiteSpace(p.Key)); + Assert.False(string.IsNullOrWhiteSpace(p.Label)); + }); + } + + [Fact] + public void EventHubs_HasValidDescriptor() => AssertValidDescriptor(new EventHubsConnector()); + + [Fact] + public void ServiceBus_HasValidDescriptor() => AssertValidDescriptor(new ServiceBusConnector()); + + [Fact] + public void KustoStreaming_HasValidDescriptor() => AssertValidDescriptor(new KustoStreamingConnector()); + + [Fact] + public void KustoQueued_HasValidDescriptor() => AssertValidDescriptor(new KustoQueuedConnector()); + + // --- Event Hubs --- + + [Fact] + public void EventHubs_DeclaresExpectedParameters() + { + var connector = new EventHubsConnector(); + + Assert.Equal("eventhubs", connector.Key); + Assert.Equal("Event Hubs", connector.DisplayName); + Assert.Equal( + new[] { EventHubsConnector.ConnectionKey, EventHubsConnector.EventHubKey }, + connector.Parameters.Select(p => p.Key)); + + ConnectionParameter connection = connector.Parameters.Single(p => p.Key == EventHubsConnector.ConnectionKey); + Assert.True(connection.Secret); + Assert.True(connection.Required); + } + + [Fact] + public void EventHubs_CreateConnection_ReturnsEventHubsConnection() + { + var values = new Dictionary + { + [EventHubsConnector.ConnectionKey] = "Endpoint=sb://ns.servicebus.windows.net/;SharedAccessKey=k", + [EventHubsConnector.EventHubKey] = "hub", + }; + + ITargetConnection connection = new EventHubsConnector().CreateConnection(values); + Assert.IsType(connection); + } + + // --- Service Bus --- + + [Fact] + public void ServiceBus_UsesQueueOrTopicTerminology() + { + var connector = new ServiceBusConnector(); + + Assert.Equal("servicebus", connector.Key); + Assert.Equal("Azure Service Bus", connector.DisplayName); + Assert.Equal( + new[] { ServiceBusConnector.ConnectionKey, ServiceBusConnector.EntityKey }, + connector.Parameters.Select(p => p.Key)); + + ConnectionParameter entity = connector.Parameters.Single(p => p.Key == ServiceBusConnector.EntityKey); + Assert.Equal("Queue / Topic name", entity.Label); + } + + [Fact] + public void ServiceBus_CreateConnection_ReturnsServiceBusConnection() + { + var values = new Dictionary + { + [ServiceBusConnector.ConnectionKey] = "Endpoint=sb://ns.servicebus.windows.net/;SharedAccessKey=k", + [ServiceBusConnector.EntityKey] = "my-queue", + }; + + ITargetConnection connection = new ServiceBusConnector().CreateConnection(values); + Assert.IsType(connection); + } + + // --- Kusto --- + + [Theory] + [InlineData("kusto-streaming")] + [InlineData("kusto-queued")] + public void Kusto_DeclaresClusterDatabaseAndTable_AllRequired(string key) + { + ITargetConnector connector = key == "kusto-streaming" + ? new KustoStreamingConnector() + : new KustoQueuedConnector(); + + Assert.Equal(key, connector.Key); + Assert.Equal( + new[] { KustoParameters.ClusterUriKey, KustoParameters.DatabaseKey, KustoParameters.TableKey, KustoParameters.MappingKey }, + connector.Parameters.Select(p => p.Key)); + + // Cluster/database/table are required; the ingestion mapping is optional. + Assert.True(connector.Parameters.Single(p => p.Key == KustoParameters.ClusterUriKey).Required); + Assert.True(connector.Parameters.Single(p => p.Key == KustoParameters.DatabaseKey).Required); + Assert.True(connector.Parameters.Single(p => p.Key == KustoParameters.TableKey).Required); + Assert.False(connector.Parameters.Single(p => p.Key == KustoParameters.MappingKey).Required); + } + + [Fact] + public void KustoStreaming_CreateConnection_ReturnsStreamingConnection() + { + ITargetConnection connection = new KustoStreamingConnector().CreateConnection(KustoValues()); + Assert.IsType(connection); + } + + [Fact] + public void KustoQueued_CreateConnection_ReturnsQueuedConnection() + { + ITargetConnection connection = new KustoQueuedConnector().CreateConnection(KustoValues()); + Assert.IsType(connection); + } + + private static Dictionary KustoValues() => new() + { + [KustoParameters.ClusterUriKey] = "https://mycluster.westeurope.kusto.windows.net", + [KustoParameters.DatabaseKey] = "db", + [KustoParameters.TableKey] = "MyTable", + }; + + [Fact] + public async Task Kusto_SendBatch_RejectsMalformedJson_BeforeContactingService() + { + // Validation happens before any network call, so this fails fast without a live cluster. + var connection = new KustoStreamingConnection("https://mycluster.westeurope.kusto.windows.net", "db", "t"); + await Assert.ThrowsAsync( + () => connection.SendBatchAsync(new[] { "{ not valid json" })); + } + + [Fact] + public void Kusto_ConnectionsImplement_IIngestionVerifier() + { + Assert.IsAssignableFrom( + new KustoQueuedConnector().CreateConnection(KustoValues())); + Assert.IsAssignableFrom( + new KustoStreamingConnector().CreateConnection(KustoValues())); + } + + // --- Missing values do not throw at construction (validation lives in the UI/CLI) --- + + [Fact] + public void CreateConnection_WithMissingValues_DoesNotThrow() + { + var empty = new Dictionary(); + + Assert.NotNull(new EventHubsConnector().CreateConnection(empty)); + Assert.NotNull(new ServiceBusConnector().CreateConnection(empty)); + Assert.NotNull(new KustoStreamingConnector().CreateConnection(empty)); + Assert.NotNull(new KustoQueuedConnector().CreateConnection(empty)); + } +} diff --git a/src/RTDSimulator.Connectors.Tests/KustoUrisTests.cs b/src/RTDSimulator.Connectors.Tests/KustoUrisTests.cs new file mode 100644 index 0000000..eaf00dd --- /dev/null +++ b/src/RTDSimulator.Connectors.Tests/KustoUrisTests.cs @@ -0,0 +1,43 @@ +using RTDSimulator.Kusto; +using Xunit; + +namespace RTDSimulator.Connectors.Tests; + +public class KustoUrisTests +{ + [Fact] + public void ToIngest_AddsIngestPrefix() + => Assert.Equal("https://ingest-c.westeurope.kusto.windows.net", + KustoUris.ToIngest("https://c.westeurope.kusto.windows.net")); + + [Fact] + public void ToIngest_LeavesAlreadyIngestUnchanged() + => Assert.Equal("https://ingest-c.westeurope.kusto.windows.net", + KustoUris.ToIngest("https://ingest-c.westeurope.kusto.windows.net")); + + [Fact] + public void ToEngine_StripsIngestPrefix() + => Assert.Equal("https://c.westeurope.kusto.windows.net", + KustoUris.ToEngine("https://ingest-c.westeurope.kusto.windows.net")); + + [Fact] + public void ToEngine_LeavesEngineUnchanged() + => Assert.Equal("https://c.westeurope.kusto.windows.net", + KustoUris.ToEngine("https://c.westeurope.kusto.windows.net")); + + [Fact] + public void ToIngest_DropsTrailingSlash() + => Assert.Equal("https://ingest-c.kusto.windows.net", + KustoUris.ToIngest("https://c.kusto.windows.net/")); + + [Fact] + public void RoundTrip_EngineToIngestToEngine() + { + const string engine = "https://mycluster.westeurope.kusto.windows.net"; + Assert.Equal(engine, KustoUris.ToEngine(KustoUris.ToIngest(engine))); + } + + [Fact] + public void NonUri_ReturnedUnchanged() + => Assert.Equal("not a uri", KustoUris.ToIngest("not a uri")); +} diff --git a/src/RTDSimulator.Connectors.Tests/RTDSimulator.Connectors.Tests.csproj b/src/RTDSimulator.Connectors.Tests/RTDSimulator.Connectors.Tests.csproj new file mode 100644 index 0000000..c338a7d --- /dev/null +++ b/src/RTDSimulator.Connectors.Tests/RTDSimulator.Connectors.Tests.csproj @@ -0,0 +1,24 @@ + + + + net10.0 + enable + enable + false + true + + + + + + + + + + + + + + + + diff --git a/src/RTDSimulator.Core.Tests/AzureConnectionTests.cs b/src/RTDSimulator.Core.Tests/AzureConnectionTests.cs new file mode 100644 index 0000000..864534b --- /dev/null +++ b/src/RTDSimulator.Core.Tests/AzureConnectionTests.cs @@ -0,0 +1,48 @@ +using RTDSimulator.Core; +using Xunit; + +namespace RTDSimulator.Core.Tests; + +public class AzureConnectionTests +{ + [Fact] + public void ResolveNamespace_ExtractsHostFromConnectionString() + { + const string cs = "Endpoint=sb://my-ns.servicebus.windows.net/;SharedAccessKeyName=key;SharedAccessKey=secret"; + Assert.Equal("my-ns.servicebus.windows.net", AzureConnection.ResolveNamespace(cs)); + } + + [Fact] + public void ResolveNamespace_BareNamespace_IsReturnedUnchanged() + { + const string ns = "my-ns.servicebus.windows.net"; + Assert.Equal(ns, AzureConnection.ResolveNamespace(ns)); + } + + [Fact] + public void ResolveNamespace_SchemeWithoutTrailingSlash_ExtractsHost() + { + Assert.Equal("my-ns.servicebus.windows.net", AzureConnection.ResolveNamespace("sb://my-ns.servicebus.windows.net")); + } + + [Fact] + public void ResolveNamespace_IsCaseInsensitiveForScheme_AndPreservesHostCase() + { + const string cs = "Endpoint=SB://My-NS.servicebus.windows.net/;SharedAccessKey=secret"; + Assert.Equal("My-NS.servicebus.windows.net", AzureConnection.ResolveNamespace(cs)); + } + + [Fact] + public void ResolveNamespace_TrimsSurroundingWhitespace() + { + Assert.Equal("my-ns.servicebus.windows.net", AzureConnection.ResolveNamespace(" my-ns.servicebus.windows.net ")); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + public void ResolveNamespace_EmptyOrWhitespace_IsReturnedAsIs(string input) + { + Assert.Equal(input, AzureConnection.ResolveNamespace(input)); + } +} diff --git a/src/RTDSimulator.Core.Tests/LoadGeneratorTests.cs b/src/RTDSimulator.Core.Tests/LoadGeneratorTests.cs new file mode 100644 index 0000000..1769328 --- /dev/null +++ b/src/RTDSimulator.Core.Tests/LoadGeneratorTests.cs @@ -0,0 +1,109 @@ +using System.Collections.Concurrent; +using RTDSimulator.Core; +using Xunit; + +namespace RTDSimulator.Core.Tests; + +public class LoadGeneratorTests +{ + /// In-memory connection that records the batches it is asked to send. + private sealed class RecordingConnection : ITargetConnection + { + private int _connectCount; + public int ConnectCount => _connectCount; + public ConcurrentQueue BatchSizes { get; } = new(); + public int Batches => BatchSizes.Count; + + public Task ConnectAsync(CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref _connectCount); + return Task.CompletedTask; + } + + public Task SendBatchAsync(IReadOnlyCollection payloads, CancellationToken cancellationToken = default) + { + BatchSizes.Enqueue(payloads.Count); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + [Fact] + public async Task Send_SingleSender_DispatchesConfiguredBatches() + { + var connection = new RecordingConnection(); + var generator = new LoadGenerator("x") { BatchesNo = 3, EventsPerBatch = 2 }; + + await generator.Send(connection, CancellationToken.None); + + Assert.Equal(3, connection.Batches); + Assert.All(connection.BatchSizes, size => Assert.Equal(2, size)); + Assert.Equal(1, connection.ConnectCount); + } + + [Fact] + public async Task Send_WithParallelism_RunsEverySender() + { + var connection = new RecordingConnection(); + var generator = new LoadGenerator("x") { BatchesNo = 2, EventsPerBatch = 5, WaitTime = TimeSpan.Zero }; + + long messages = 0; + int batchEvents = 0; + generator.BatchSent += (_, e) => + { + Interlocked.Add(ref messages, e.MessageCount); + Interlocked.Increment(ref batchEvents); + }; + + await generator.Send(connection, parallelism: 3, CancellationToken.None); + + Assert.Equal(6, connection.Batches); // 3 senders * 2 batches + Assert.Equal(6, batchEvents); + Assert.Equal(30, messages); // 6 batches * 5 messages + Assert.All(connection.BatchSizes, size => Assert.Equal(5, size)); + Assert.Equal(1, connection.ConnectCount); // connected once before fan-out + } + + [Fact] + public async Task BatchSentEventArgs_ReportsPerBatchSize() + { + var connection = new RecordingConnection(); + // Payload "abcd" (4 bytes) x 3 per batch => 12 bytes per batch. + var generator = new LoadGenerator("abcd") { BatchesNo = 1, EventsPerBatch = 3 }; + + BatchSentEventArgs? captured = null; + generator.BatchSent += (_, e) => captured = e; + + await generator.Send(connection, CancellationToken.None); + + Assert.NotNull(captured); + Assert.Equal(3, captured!.MessageCount); + Assert.Equal(12, captured.SizeInBytes); + } + + [Fact] + public async Task Send_WhenCancelled_StopsEarly() + { + var connection = new RecordingConnection(); + var generator = new LoadGenerator("x") { BatchesNo = 1000, EventsPerBatch = 1 }; + + using var cts = new CancellationTokenSource(); + generator.BatchSent += (_, _) => cts.Cancel(); // cancel after the first batch + + await generator.Send(connection, cts.Token); + + Assert.InRange(connection.Batches, 1, 999); + } + + [Fact] + public async Task Send_ParallelismBelowOne_IsTreatedAsSingleSender() + { + var connection = new RecordingConnection(); + var generator = new LoadGenerator("x") { BatchesNo = 4, EventsPerBatch = 1 }; + + await generator.Send(connection, parallelism: 0, CancellationToken.None); + + Assert.Equal(4, connection.Batches); + } +} diff --git a/src/RTDSimulator.Core.Tests/PayloadGeneratorTests.cs b/src/RTDSimulator.Core.Tests/PayloadGeneratorTests.cs new file mode 100644 index 0000000..53ea318 --- /dev/null +++ b/src/RTDSimulator.Core.Tests/PayloadGeneratorTests.cs @@ -0,0 +1,164 @@ +using System.Globalization; +using RTDSimulator.Core; +using Xunit; + +namespace RTDSimulator.Core.Tests; + +public class PayloadGeneratorTests +{ + // --- Plain text / passthrough --- + + [Fact] + public async Task PlainText_IsReturnedUnchanged() + { + var gen = new PayloadGenerator("hello world"); + Assert.Equal("hello world", await gen.GetPayload(0)); + } + + [Fact] + public async Task UnknownPlaceholder_IsLeftUntouched() + { + var gen = new PayloadGenerator("{{DoesNotExist}}"); + Assert.Equal("{{DoesNotExist}}", await gen.GetPayload(0)); + } + + // --- Built-in variables --- + + [Fact] + public async Task SettlementPeriod_ReplacedWithLiteral() + { + var gen = new PayloadGenerator("p={{SettlementPeriod}}"); + Assert.Equal("p=48", await gen.GetPayload(0)); + } + + [Fact] + public async Task MultipleOccurrencesOfVariable_AreAllReplaced() + { + var gen = new PayloadGenerator("{{SettlementPeriod}}-{{SettlementPeriod}}"); + Assert.Equal("48-48", await gen.GetPayload(0)); + } + + [Fact] + public async Task UserId_IsWithinConfiguredRange() + { + var gen = new PayloadGenerator("{{UserId}}"); + int value = int.Parse(await gen.GetPayload(0)); + Assert.InRange(value, 5000, 5099); // Random(5000,5100) -> max exclusive + } + + [Fact] + public async Task ProductId_IsWithinConfiguredRange() + { + var gen = new PayloadGenerator("{{ProductId}}"); + int value = int.Parse(await gen.GetPayload(0)); + Assert.InRange(value, 700, 998); // Random(700,999) -> max exclusive + } + + [Fact] + public async Task Device_IsOneOfAllowedItems() + { + var gen = new PayloadGenerator("{{Device}}"); + string value = await gen.GetPayload(0); + Assert.Contains(value, new[] { "mobile", "tablet", "pc" }); + } + + [Theory] + [InlineData(0, "BIOMASS")] + [InlineData(1, "CCGT")] + [InlineData(2, "COAL")] + [InlineData(19, "BIOMASS")] // wraps: 19 % 19 == 0 + public async Task FuelType_IteratesByMessageIndex(int index, string expected) + { + var gen = new PayloadGenerator("{{FuelType(MessageIndex)}}"); + Assert.Equal(expected, await gen.GetPayload(index)); + } + + [Fact] + public async Task DateTimeNowVariable_ProducesParseableTimestamp() + { + var gen = new PayloadGenerator("{{DateTime.Now}}"); + string value = await gen.GetPayload(0); + Assert.True(DateTime.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out _)); + } + + // --- Expressions ({{$ ... }}) --- + + [Fact] + public async Task Expression_ArithmeticIsEvaluated() + { + var gen = new PayloadGenerator("{{$1+2}}"); + Assert.Equal("3", await gen.GetPayload(0)); + } + + [Fact] + public async Task Expression_HasSystemNamespaceImported() + { + var gen = new PayloadGenerator("{{$Math.Max(3, 7)}}"); + Assert.Equal("7", await gen.GetPayload(0)); + } + + [Fact] + public async Task DistinctExpressions_AreEvaluatedIndependently() + { + var gen = new PayloadGenerator("{{$1+1}}/{{$2+2}}"); + Assert.Equal("2/4", await gen.GetPayload(0)); + } + + // --- RandomString helper --- + + [Fact] + public async Task RandomString_HasRequestedLength() + { + var gen = new PayloadGenerator("{{$RandomString(12)}}"); + Assert.Equal(12, (await gen.GetPayload(0)).Length); + } + + [Fact] + public async Task RandomString_DefaultCharset_IsAlphanumeric() + { + var gen = new PayloadGenerator("{{$RandomString(200)}}"); + Assert.Matches("^[A-Za-z0-9]+$", await gen.GetPayload(0)); + } + + [Fact] + public async Task RandomString_RespectsCustomCharset() + { + var gen = new PayloadGenerator("{{$RandomString(50, \"AB\")}}"); + string value = await gen.GetPayload(0); + Assert.Equal(50, value.Length); + Assert.Matches("^[AB]+$", value); + } + + [Fact] + public async Task RandomString_ZeroLength_IsEmpty() + { + var gen = new PayloadGenerator("x{{$RandomString(0)}}y"); + Assert.Equal("xy", await gen.GetPayload(0)); + } + + [Fact] + public async Task IdenticalExpressionTokens_ProduceTheSameValueWithinAMessage() + { + // The engine evaluates each distinct token once and replaces all occurrences, + // so two identical tokens resolve to the same value. + var gen = new PayloadGenerator("{{$RandomString(16)}}|{{$RandomString(16)}}"); + string[] parts = (await gen.GetPayload(0)).Split('|'); + Assert.Equal(parts[0], parts[1]); + } + + // --- Combined / realistic template --- + + [Fact] + public async Task RealisticTemplate_AllPlaceholdersResolved() + { + const string template = """ + { "id": "{{$RandomString(8)}}", "device": "{{Device}}", "period": {{SettlementPeriod}} } + """; + var gen = new PayloadGenerator(template); + string result = await gen.GetPayload(0); + + Assert.DoesNotContain("{{", result); + Assert.DoesNotContain("}}", result); + Assert.Contains("\"period\": 48", result); + } +} diff --git a/src/RTDSimulator.Core.Tests/RTDSimulator.Core.Tests.csproj b/src/RTDSimulator.Core.Tests/RTDSimulator.Core.Tests.csproj new file mode 100644 index 0000000..dc698b8 --- /dev/null +++ b/src/RTDSimulator.Core.Tests/RTDSimulator.Core.Tests.csproj @@ -0,0 +1,21 @@ + + + + net10.0 + enable + enable + false + true + + + + + + + + + + + + + diff --git a/src/RTDSimulator.Core.Tests/VariableDefinitionsTests.cs b/src/RTDSimulator.Core.Tests/VariableDefinitionsTests.cs new file mode 100644 index 0000000..9e06e7c --- /dev/null +++ b/src/RTDSimulator.Core.Tests/VariableDefinitionsTests.cs @@ -0,0 +1,126 @@ +using RTDSimulator.Core; +using Xunit; + +namespace RTDSimulator.Core.Tests; + +public class VariableDefinitionsTests +{ + private const string Sample = """ + [UserId] + type = "randomInt" + min = 5000 + max = 5100 + + [Env] + type = "randomItem" + items = ["dev", "prod"] + + ["FuelType(MessageIndex)"] + type = "indexedItem" + items = ["A", "B", "C"] + + ["DateTime.Now"] + type = "dateTimeNow" + + [Fixed] + type = "literal" + value = "42" + """; + + [Fact] + public void Parse_ReadsAllDefinitions_WithNameAsTableHeader() + { + var defs = VariableDefinitions.Parse(Sample); + Assert.Equal(5, defs.Count); + Assert.Equal( + new[] { "UserId", "Env", "FuelType(MessageIndex)", "DateTime.Now", "Fixed" }, + defs.Select(d => d.Name)); + } + + [Fact] + public void Parse_MapsTypesAndParameters() + { + var byName = VariableDefinitions.Parse(Sample).ToDictionary(d => d.Name); + + Assert.Equal(VariableType.RandomInt, byName["UserId"].Type); + Assert.Equal(5000, byName["UserId"].Min); + Assert.Equal(5100, byName["UserId"].Max); + + Assert.Equal(VariableType.RandomItem, byName["Env"].Type); + Assert.Equal(new[] { "dev", "prod" }, byName["Env"].Items); + + Assert.Equal(VariableType.IndexedItem, byName["FuelType(MessageIndex)"].Type); + Assert.Equal(VariableType.DateTimeNow, byName["DateTime.Now"].Type); + + Assert.Equal(VariableType.Literal, byName["Fixed"].Type); + Assert.Equal("42", byName["Fixed"].Value); + } + + [Theory] + [InlineData(0, "A")] + [InlineData(1, "B")] + [InlineData(3, "A")] // wraps + public void Evaluate_IndexedItem_SelectsByMessageIndex(int index, string expected) + { + var def = VariableDefinitions.Parse(Sample).Single(d => d.Name == "FuelType(MessageIndex)"); + Assert.Equal(expected, def.Evaluate(index, new Random())); + } + + [Fact] + public void Evaluate_RandomInt_IsWithinRange() + { + var def = VariableDefinitions.Parse(Sample).Single(d => d.Name == "UserId"); + int value = int.Parse(def.Evaluate(0, new Random())); + Assert.InRange(value, 5000, 5099); // max exclusive + } + + [Fact] + public void LoadDefaults_ReturnsTheBuiltInVariables() + { + var byName = VariableDefinitions.LoadDefaults().ToDictionary(d => d.Name); + + Assert.Equal(6, byName.Count); + Assert.Equal(VariableType.RandomInt, byName["UserId"].Type); + Assert.Equal(VariableType.RandomItem, byName["Device"].Type); + Assert.Equal(VariableType.IndexedItem, byName["FuelType(MessageIndex)"].Type); + Assert.Equal(VariableType.DateTimeNow, byName["DateTime.Now"].Type); + Assert.Equal("48", byName["SettlementPeriod"].Value); + } + + [Fact] + public async Task PayloadGenerator_UsesSuppliedDefinitions() + { + var defs = VariableDefinitions.Parse(""" + [Greeting] + type = "literal" + value = "hi" + """); + var gen = new PayloadGenerator("{{Greeting}}, world", defs); + Assert.Equal("hi, world", await gen.GetPayload(0)); + } + + [Fact] + public void Parse_UnknownType_Throws() + { + Assert.Throws(() => VariableDefinitions.Parse(""" + [X] + type = "bogus" + """)); + } + + [Fact] + public void Parse_MissingRequiredParameter_Throws() + { + Assert.Throws(() => VariableDefinitions.Parse(""" + [X] + type = "randomInt" + min = 1 + """)); // missing max + } + + [Fact] + public void Parse_NonTableEntry_Throws() + { + Assert.Throws(() => VariableDefinitions.Parse("Loose = 123")); + } +} diff --git a/src/RTDSimulator.Core/AzureConnection.cs b/src/RTDSimulator.Core/AzureConnection.cs new file mode 100644 index 0000000..30ca05c --- /dev/null +++ b/src/RTDSimulator.Core/AzureConnection.cs @@ -0,0 +1,37 @@ +namespace RTDSimulator.Core; + +/// +/// Helpers for working with Azure messaging connection inputs that may be supplied +/// either as a SAS connection string or as a fully-qualified namespace. +/// +public static class AzureConnection +{ + /// + /// Returns the fully-qualified namespace (e.g. my-ns.servicebus.windows.net) + /// for use with Azure Identity (token-credential) authentication. Accepts either a + /// bare namespace or a full Endpoint=sb://.../;SharedAccessKey=... connection + /// string and extracts the host in the latter case. + /// + public static string ResolveNamespace(string connectionStringOrNamespace) + { + if (string.IsNullOrWhiteSpace(connectionStringOrNamespace)) + return connectionStringOrNamespace; + + string value = connectionStringOrNamespace.Trim(); + + // Full connection string: pull out the Endpoint=sb://host/ portion. + int schemeIndex = value.IndexOf("sb://", StringComparison.OrdinalIgnoreCase); + if (schemeIndex >= 0) + { + int hostStart = schemeIndex + "sb://".Length; + int hostEnd = value.IndexOfAny(new[] { '/', ';' }, hostStart); + string host = hostEnd < 0 + ? value.Substring(hostStart) + : value.Substring(hostStart, hostEnd - hostStart); + return host.Trim(); + } + + // Already a bare namespace. + return value; + } +} diff --git a/src/RTDSimulator.Core/BatchSentEventArgs.cs b/src/RTDSimulator.Core/BatchSentEventArgs.cs new file mode 100644 index 0000000..7832411 --- /dev/null +++ b/src/RTDSimulator.Core/BatchSentEventArgs.cs @@ -0,0 +1,21 @@ +namespace RTDSimulator.Core; + +/// +/// Reports a single dispatched batch. Carrying the per-batch figures on the event +/// (rather than on shared fields) keeps progress +/// reporting correct when several senders run concurrently. +/// +public sealed class BatchSentEventArgs : EventArgs +{ + public BatchSentEventArgs(int messageCount, long sizeInBytes) + { + MessageCount = messageCount; + SizeInBytes = sizeInBytes; + } + + /// Number of messages in the batch. + public int MessageCount { get; } + + /// Total payload size of the batch, in bytes. + public long SizeInBytes { get; } +} diff --git a/src/RTDSimulator.Core/ConnectionParameter.cs b/src/RTDSimulator.Core/ConnectionParameter.cs new file mode 100644 index 0000000..ac040bc --- /dev/null +++ b/src/RTDSimulator.Core/ConnectionParameter.cs @@ -0,0 +1,44 @@ +namespace RTDSimulator.Core; + +/// +/// Describes a single connection parameter that a connector needs in order to +/// reach its target service. Connectors expose a list of these so that UIs (GUI, +/// CLI) can render the correct fields and labels without hard-coding per-service +/// knowledge. +/// +public sealed class ConnectionParameter +{ + /// Stable machine key, used as the CLI --param name and dictionary key. + public string Key { get; } + + /// Human-readable label shown in the UI (the "proper term" for the field). + public string Label { get; } + + /// When true, the value is sensitive and should be masked in UIs. + public bool Secret { get; } + + /// When true, a non-empty value must be supplied. + public bool Required { get; } + + /// Optional pre-filled value. + public string? DefaultValue { get; } + + /// Optional help / hint text. + public string? HelpText { get; } + + public ConnectionParameter( + string key, + string label, + bool secret = false, + bool required = true, + string? defaultValue = null, + string? helpText = null) + { + Key = key; + Label = label; + Secret = secret; + Required = required; + DefaultValue = defaultValue; + HelpText = helpText; + } +} diff --git a/src/RTDSimulator.Core/IIngestionVerifier.cs b/src/RTDSimulator.Core/IIngestionVerifier.cs new file mode 100644 index 0000000..7a3a6d3 --- /dev/null +++ b/src/RTDSimulator.Core/IIngestionVerifier.cs @@ -0,0 +1,16 @@ +namespace RTDSimulator.Core; + +/// +/// Optional capability for connections whose ingestion completes asynchronously +/// (e.g. Kusto queued ingestion), where per-batch failures cannot be observed at send +/// time. Callers can query for failures after a run to verify the data actually landed. +/// +public interface IIngestionVerifier +{ + /// + /// Returns human-readable descriptions of ingestion failures recorded since + /// . An empty list means no failures were reported + /// (note that asynchronous failures may lag behind the send). + /// + Task> GetIngestionFailuresSinceAsync(DateTime sinceUtc, CancellationToken cancellationToken = default); +} diff --git a/src/RTDSimulator.Core/ITargetConnection.cs b/src/RTDSimulator.Core/ITargetConnection.cs new file mode 100644 index 0000000..3198696 --- /dev/null +++ b/src/RTDSimulator.Core/ITargetConnection.cs @@ -0,0 +1,20 @@ +namespace RTDSimulator.Core; + +/// +/// Abstraction over a target service that generated load can be sent to. +/// Implement this interface to add support for additional service types +/// (Azure Service Bus / Event Hubs, Kafka, HTTP endpoints, ...). +/// +public interface ITargetConnection : IAsyncDisposable +{ + /// + /// Establishes (or validates) the connection to the target service. + /// Implementations should be safe to call more than once. + /// + Task ConnectAsync(CancellationToken cancellationToken = default); + + /// + /// Sends a single batch of payloads to the target service. + /// + Task SendBatchAsync(IReadOnlyCollection payloads, CancellationToken cancellationToken = default); +} diff --git a/src/RTDSimulator.Core/ITargetConnector.cs b/src/RTDSimulator.Core/ITargetConnector.cs new file mode 100644 index 0000000..961866c --- /dev/null +++ b/src/RTDSimulator.Core/ITargetConnector.cs @@ -0,0 +1,31 @@ +using Azure.Core; + +namespace RTDSimulator.Core; + +/// +/// Describes a connectable target service and acts as a factory for +/// instances. Each connector declares the +/// parameters it needs () so that callers can collect +/// them generically and stay decoupled from service-specific details. +/// +public interface ITargetConnector +{ + /// Stable machine key (e.g. "servicebus"), used by the CLI --target option. + string Key { get; } + + /// Human-readable name (e.g. "Azure Service Bus"), shown in the GUI selector. + string DisplayName { get; } + + /// The ordered set of parameters this connector requires. + IReadOnlyList Parameters { get; } + + /// + /// Builds a connection from collected parameter values. + /// + /// Values keyed by . + /// + /// Optional Azure Identity credential. When supplied, connectors that support it + /// should authenticate with the credential instead of a connection string / key. + /// + ITargetConnection CreateConnection(IReadOnlyDictionary values, TokenCredential? credential = null); +} diff --git a/src/RTDSimulator.Core/LoadGenerator.cs b/src/RTDSimulator.Core/LoadGenerator.cs new file mode 100644 index 0000000..a47ac14 --- /dev/null +++ b/src/RTDSimulator.Core/LoadGenerator.cs @@ -0,0 +1,110 @@ +namespace RTDSimulator.Core; + +/// +/// Drives load generation: builds batches of generated payloads and dispatches +/// them through an . The connection is responsible +/// for the service-specific transport, keeping this orchestration generic. +/// +/// Concurrency is owned here: +/// runs the requested number of senders as concurrent async tasks. The batch loop keeps +/// no shared mutable state, so a single instance is safe to run in parallel. +/// +public class LoadGenerator +{ + /// Number of batches each sender dispatches. + public int BatchesNo { get; set; } = 1; + + /// Number of messages per batch. + public int EventsPerBatch { get; set; } = 1; + + /// Delay between consecutive batches within a single sender. + public TimeSpan WaitTime { get; set; } = TimeSpan.Zero; + + /// + /// The payload generator used to produce each message. Exposed so callers + /// (e.g. a preview screen) can inspect . + /// + public PayloadGenerator Generator { get; } + + /// + /// Raised after each batch is sent. May be invoked concurrently when + /// parallelism > 1; handlers must be thread-safe (or marshal to a UI thread). + /// + public event EventHandler? BatchSent; + + public LoadGenerator(string payload) + : this(new PayloadGenerator(payload)) + { + } + + public LoadGenerator(PayloadGenerator generator) + { + Generator = generator; + } + + /// Runs a single sender. + public Task Send(ITargetConnection connection, CancellationToken cancellationToken) + => Send(connection, 1, cancellationToken); + + /// + /// Runs senders concurrently, each dispatching + /// batches. The (thread-safe) connection is shared. + /// + public async Task Send(ITargetConnection connection, int parallelism, CancellationToken cancellationToken) + { + if (parallelism < 1) + parallelism = 1; + + await connection.ConnectAsync(cancellationToken); + + var senders = Enumerable.Range(0, parallelism) + .Select(_ => RunSenderAsync(connection, cancellationToken)); + await Task.WhenAll(senders); + } + + private async Task RunSenderAsync(ITargetConnection connection, CancellationToken cancellationToken) + { + DateTime nextRun = DateTime.Now; + for (int batch = 0; batch < BatchesNo; batch++) + { + if (cancellationToken.IsCancellationRequested) + return; + + long batchSizeInBytes = 0; + var payloads = new List(EventsPerBatch); + for (int m = 0; m < EventsPerBatch; m++) + { + if (cancellationToken.IsCancellationRequested) + return; + + string payload = await Generator.GetPayload(m); + payloads.Add(payload); + batchSizeInBytes += payload.Length; + } + + await SleepUntil(nextRun, cancellationToken); + + if (cancellationToken.IsCancellationRequested) + return; + + await connection.SendBatchAsync(payloads, cancellationToken); + BatchSent?.Invoke(this, new BatchSentEventArgs(payloads.Count, batchSizeInBytes)); + + nextRun = DateTime.Now.Add(WaitTime); + } + } + + private async Task SleepUntil(DateTime t, CancellationToken cancellationToken) + { + TimeSpan waitTime = t.Subtract(DateTime.Now); + if (waitTime <= TimeSpan.Zero) return; + try + { + await Task.Delay(waitTime, cancellationToken); + } + catch (Exception) + { + //don't throw if cancelled + } + } +} diff --git a/src/RTDSimulator.Core/PayloadGenerator.cs b/src/RTDSimulator.Core/PayloadGenerator.cs new file mode 100644 index 0000000..186a3e7 --- /dev/null +++ b/src/RTDSimulator.Core/PayloadGenerator.cs @@ -0,0 +1,81 @@ +using System.Collections.Concurrent; +using System.Text.RegularExpressions; +using Microsoft.CodeAnalysis.CSharp.Scripting; +using Microsoft.CodeAnalysis.Scripting; + +namespace RTDSimulator.Core; + +/// +/// Generates message payloads from a template. The template can contain: +/// +/// Variables, e.g. {{UserId}}, resolved from . +/// C# expressions, e.g. {{$DateTime.Now}}, evaluated via Roslyn scripting. +/// +/// This type is service-agnostic and lives in Core so it can be reused by any connector. +/// +public class PayloadGenerator +{ + /// The (soft-coded) variable definitions applied to the template. + public IReadOnlyList Variables { get; } + + private readonly string _payload; + private readonly Dictionary Expressions = new Dictionary(); + private ConcurrentDictionary> preCompiledScripts { get; set; } = new ConcurrentDictionary>(); + private readonly ScriptFunctions _functions = new ScriptFunctions(); + + /// Creates a generator using the built-in default variable definitions. + public PayloadGenerator(string payload) + : this(payload, VariableDefinitions.LoadDefaults()) + { + } + + /// Creates a generator using the supplied variable definitions. + public PayloadGenerator(string payload, IEnumerable variables) + { + _payload = payload; + Variables = variables.ToList(); + + // Finding expressions... + // Example of an expression definition: {{$DateTime.Now}} {{$new Random().Next(100,200)}} + var pattern = @"{{(\$.*?)}}"; + var matches = Regex.Matches(_payload, pattern); + foreach (Match m in matches) + { + if (!Expressions.ContainsKey(m.Value)) + { + Expressions.Add(m.Value, m.Value.Substring(3, m.Value.Length - 5)); + } + } + } + + public async Task GetPayload(int msgIndex) + { + Random rnd = new Random(); + string payload = _payload; + + // Evaluating Variables + foreach (VariableDefinition v in Variables) + { + payload = payload.Replace("{{" + v.Name + "}}", v.Evaluate(msgIndex, rnd)); + } + + // Evaluating Expressions + // https://github.com/dotnet/roslyn/blob/main/docs/wiki/Scripting-API-Samples.md + foreach (string key in Expressions.Keys) + { + String exp = Expressions[key]; + + if (!preCompiledScripts.TryGetValue(exp, out Script script)) + { + script = CSharpScript.Create(exp, ScriptOptions.Default.WithImports("System"), typeof(ScriptFunctions)); + preCompiledScripts.TryAdd(exp, script); + } + + var result = await script.RunAsync(_functions); + + payload = payload.Replace(key, result.ReturnValue.ToString()); + } + + return payload; + } +} diff --git a/src/RTDSimulator.Core/RTDSimulator.Core.csproj b/src/RTDSimulator.Core/RTDSimulator.Core.csproj new file mode 100644 index 0000000..f72c265 --- /dev/null +++ b/src/RTDSimulator.Core/RTDSimulator.Core.csproj @@ -0,0 +1,20 @@ + + + + net10.0 + enable + enable + https://github.com/Azure-Player/Real-Time-Data-Simulator/ + + + + + + + + + + + + + diff --git a/src/RTDSimulator.Core/ScriptFunctions.cs b/src/RTDSimulator.Core/ScriptFunctions.cs new file mode 100644 index 0000000..faa3eca --- /dev/null +++ b/src/RTDSimulator.Core/ScriptFunctions.cs @@ -0,0 +1,28 @@ +namespace RTDSimulator.Core; + +/// +/// Helper methods exposed to {{$ ... }} template expressions as globals, +/// so they can be called directly, e.g. {{$RandomString(10)}}. +/// +public sealed class ScriptFunctions +{ + private const string DefaultCharset = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + + /// + /// Returns a random string of the given , drawing + /// characters from (alphanumeric by default). + /// + /// Number of characters to generate. Values <= 0 yield an empty string. + /// Optional set of characters to choose from. Empty/null uses the default alphanumeric set. + public string RandomString(int length, string? charset = null) + { + if (length <= 0) return string.Empty; + charset = string.IsNullOrEmpty(charset) ? DefaultCharset : charset; + + var chars = new char[length]; + for (int i = 0; i < length; i++) + chars[i] = charset[Random.Shared.Next(charset.Length)]; + return new string(chars); + } +} diff --git a/src/RTDSimulator.Core/VariableDefinition.cs b/src/RTDSimulator.Core/VariableDefinition.cs new file mode 100644 index 0000000..91cf5f8 --- /dev/null +++ b/src/RTDSimulator.Core/VariableDefinition.cs @@ -0,0 +1,54 @@ +namespace RTDSimulator.Core; + +/// The kind of value a produces. +public enum VariableType +{ + /// A fixed string (). + Literal, + + /// Random integer in [Min, Max) (max exclusive). + RandomInt, + + /// A random element of . + RandomItem, + + /// An element of chosen by message index (index % count). + IndexedItem, + + /// Current local date/time, formatted with (default "O"). + DateTimeNow, +} + +/// +/// A soft-coded variable definition, typically loaded from a TOML file. The +/// is the token replaced in a template (e.g. {{UserId}}). +/// +public sealed class VariableDefinition +{ + public required string Name { get; init; } + public required VariableType Type { get; init; } + + // randomInt + public int Min { get; init; } + public int Max { get; init; } + + // randomItem / indexedItem + public IReadOnlyList Items { get; init; } = Array.Empty(); + + // literal + public string? Value { get; init; } + + // dateTimeNow + public string Format { get; init; } = "O"; + + /// Produces this variable's value for the given message index. + public string Evaluate(int messageIndex, Random random) => Type switch + { + VariableType.Literal => Value ?? string.Empty, + VariableType.RandomInt => random.Next(Min, Max).ToString(), + VariableType.RandomItem => Items.Count == 0 ? string.Empty : Items[random.Next(Items.Count)], + VariableType.IndexedItem => Items.Count == 0 ? string.Empty : Items[messageIndex % Items.Count], + VariableType.DateTimeNow => DateTime.Now.ToString(Format), + _ => string.Empty, + }; +} diff --git a/src/RTDSimulator.Core/VariableDefinitions.cs b/src/RTDSimulator.Core/VariableDefinitions.cs new file mode 100644 index 0000000..d591740 --- /dev/null +++ b/src/RTDSimulator.Core/VariableDefinitions.cs @@ -0,0 +1,140 @@ +using Tomlyn; +using Tomlyn.Model; + +namespace RTDSimulator.Core; + +/// +/// Loads s from TOML. Each top-level table defines one +/// variable, keyed by its token name, e.g.: +/// +/// [UserId] +/// type = "randomInt" +/// min = 5000 +/// max = 5100 +/// +/// +public static class VariableDefinitions +{ + /// Conventional file name the apps look for next to the executable. + public const string DefaultFileName = "variables.toml"; + + /// Parses variable definitions from a TOML string. + public static IReadOnlyList Parse(string toml) + { + TomlTable model; + try + { + model = Toml.ToModel(toml); + } + catch (Exception ex) + { + throw new FormatException($"Invalid variable definitions TOML: {ex.Message}", ex); + } + + var definitions = new List(); + foreach (var entry in model) + { + if (entry.Value is not TomlTable table) + { + throw new FormatException( + $"Variable '{entry.Key}' must be a table, e.g. [{entry.Key}] followed by 'type = ...'."); + } + + definitions.Add(Build(entry.Key, table)); + } + + return definitions; + } + + /// Loads variable definitions from a TOML file on disk. + public static IReadOnlyList Load(string path) + => Parse(File.ReadAllText(path)); + + /// Loads the built-in default definitions embedded in this assembly. + public static IReadOnlyList LoadDefaults() + { + var assembly = typeof(VariableDefinitions).Assembly; + string resourceName = assembly.GetManifestResourceNames() + .Single(n => n.EndsWith("default-variables.toml", StringComparison.OrdinalIgnoreCase)); + + using Stream stream = assembly.GetManifestResourceStream(resourceName)!; + using var reader = new StreamReader(stream); + return Parse(reader.ReadToEnd()); + } + + private static VariableDefinition Build(string name, TomlTable table) + { + string typeText = GetString(table, "type", name) + ?? throw new FormatException($"Variable '{name}' is missing required 'type'."); + + VariableType type = typeText.ToLowerInvariant() switch + { + "literal" => VariableType.Literal, + "randomint" => VariableType.RandomInt, + "randomitem" => VariableType.RandomItem, + "indexeditem" => VariableType.IndexedItem, + "datetimenow" => VariableType.DateTimeNow, + _ => throw new FormatException( + $"Variable '{name}' has unknown type '{typeText}'. " + + "Valid types: literal, randomInt, randomItem, indexedItem, dateTimeNow."), + }; + + return type switch + { + VariableType.RandomInt => new VariableDefinition + { + Name = name, + Type = type, + Min = GetInt(table, "min", name), + Max = GetInt(table, "max", name), + }, + VariableType.RandomItem or VariableType.IndexedItem => new VariableDefinition + { + Name = name, + Type = type, + Items = GetItems(table, name), + }, + VariableType.Literal => new VariableDefinition + { + Name = name, + Type = type, + Value = GetString(table, "value", name) ?? string.Empty, + }, + _ => new VariableDefinition // DateTimeNow + { + Name = name, + Type = type, + Format = GetString(table, "format", name) ?? "O", + }, + }; + } + + private static string? GetString(TomlTable table, string key, string variableName) + { + if (!table.TryGetValue(key, out object? value)) + return null; + if (value is string s) + return s; + throw new FormatException($"Variable '{variableName}': '{key}' must be a string."); + } + + private static int GetInt(TomlTable table, string key, string variableName) + { + if (!table.TryGetValue(key, out object? value)) + throw new FormatException($"Variable '{variableName}' is missing required '{key}'."); + return value switch + { + long l => (int)l, + int i => i, + _ => throw new FormatException($"Variable '{variableName}': '{key}' must be an integer."), + }; + } + + private static IReadOnlyList GetItems(TomlTable table, string variableName) + { + if (!table.TryGetValue("items", out object? value) || value is not TomlArray array || array.Count == 0) + throw new FormatException($"Variable '{variableName}' requires a non-empty 'items' array."); + + return array.Select(item => item?.ToString() ?? string.Empty).ToList(); + } +} diff --git a/src/RTDSimulator.Core/default-variables.toml b/src/RTDSimulator.Core/default-variables.toml new file mode 100644 index 0000000..1e84c2d --- /dev/null +++ b/src/RTDSimulator.Core/default-variables.toml @@ -0,0 +1,41 @@ +# Default variable definitions for the Real-Time Data Simulator. +# +# Each [table] defines one variable; the table name is the token used in the +# payload template, e.g. [UserId] is emitted for {{UserId}}. Names that contain +# dots or parentheses must be quoted, e.g. ["DateTime.Now"]. +# +# Supported types: +# randomInt - integer in [min, max) (max is exclusive) +# randomItem - a random element of `items` +# indexedItem - element of `items` chosen by message index (index % count) +# dateTimeNow - current local date/time; optional `format` (default "O") +# literal - the fixed string `value` + +[UserId] +type = "randomInt" +min = 5000 +max = 5100 + +[ProductId] +type = "randomInt" +min = 700 +max = 999 + +[Device] +type = "randomItem" +items = ["mobile", "tablet", "pc"] + +["DateTime.Now"] +type = "dateTimeNow" + +["FuelType(MessageIndex)"] +type = "indexedItem" +items = [ + "BIOMASS", "CCGT", "COAL", "INTELEC", "INTEW", "INTFR", "INTIFA2", "INTIRL", + "INTNED", "INTNEM", "INTNSL", "INTVKL", "NPSHYD", "NUCLEAR", "OCGT", "OIL", + "OTHER", "PS", "WIND", +] + +[SettlementPeriod] +type = "literal" +value = "48" diff --git a/src/RTDSimulator.EventHubs/EventHubsConnection.cs b/src/RTDSimulator.EventHubs/EventHubsConnection.cs new file mode 100644 index 0000000..f5a1882 --- /dev/null +++ b/src/RTDSimulator.EventHubs/EventHubsConnection.cs @@ -0,0 +1,64 @@ +using System.Text; +using Azure.Core; +using Azure.Messaging.EventHubs; +using Azure.Messaging.EventHubs.Producer; +using RTDSimulator.Core; + +namespace RTDSimulator.EventHubs; + +/// +/// implementation that sends generated load to an +/// Azure Event Hubs endpoint via the Event Hubs producer client. +/// +public sealed class EventHubsConnection : ITargetConnection +{ + private readonly string _connectionString; + private readonly string _eventHubName; + private readonly TokenCredential? _credential; + + private EventHubProducerClient? _producerClient; + + /// + /// Event Hubs connection string, or the fully-qualified namespace + /// (e.g. my-namespace.servicebus.windows.net) when a credential is supplied. + /// + /// Name of the target event hub. + /// Optional token credential for Entra ID authentication. + public EventHubsConnection(string connectionString, string eventHubName, TokenCredential? credential = null) + { + _connectionString = connectionString; + _eventHubName = eventHubName; + _credential = credential; + } + + public Task ConnectAsync(CancellationToken cancellationToken = default) + { + _producerClient ??= _credential is null + ? new EventHubProducerClient(_connectionString, _eventHubName) + : new EventHubProducerClient(AzureConnection.ResolveNamespace(_connectionString), _eventHubName, _credential); + + return Task.CompletedTask; + } + + public async Task SendBatchAsync(IReadOnlyCollection payloads, CancellationToken cancellationToken = default) + { + await ConnectAsync(cancellationToken); + + using EventDataBatch eventBatch = await _producerClient!.CreateBatchAsync(cancellationToken); + foreach (string payload in payloads) + { + eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes(payload))); + } + + await _producerClient.SendAsync(eventBatch, cancellationToken); + } + + public async ValueTask DisposeAsync() + { + if (_producerClient is not null) + { + await _producerClient.DisposeAsync(); + _producerClient = null; + } + } +} diff --git a/src/RTDSimulator.EventHubs/EventHubsConnector.cs b/src/RTDSimulator.EventHubs/EventHubsConnector.cs new file mode 100644 index 0000000..83c4e88 --- /dev/null +++ b/src/RTDSimulator.EventHubs/EventHubsConnector.cs @@ -0,0 +1,38 @@ +using Azure.Core; +using RTDSimulator.Core; + +namespace RTDSimulator.EventHubs; + +/// +/// describing an Azure Event Hubs target. +/// +public sealed class EventHubsConnector : ITargetConnector +{ + public const string ConnectionKey = "connection"; + public const string EventHubKey = "eventHub"; + + public string Key => "eventhubs"; + + public string DisplayName => "Event Hubs"; + + public IReadOnlyList Parameters { get; } = new[] + { + new ConnectionParameter( + ConnectionKey, + "Connection string / Namespace", + secret: true, + defaultValue: "Endpoint=sb://****fkawjq507mc.servicebus.windows.net/;SharedAccessKeyName=key_0000;SharedAccessKey=****", + helpText: "SAS connection string, or the fully-qualified namespace when signed in with Azure Identity."), + new ConnectionParameter( + EventHubKey, + "Event Hub name", + defaultValue: "es_08960000000000000000"), + }; + + public ITargetConnection CreateConnection(IReadOnlyDictionary values, TokenCredential? credential = null) + { + string connection = values.GetValueOrDefault(ConnectionKey, string.Empty); + string eventHub = values.GetValueOrDefault(EventHubKey, string.Empty); + return new EventHubsConnection(connection, eventHub, credential); + } +} diff --git a/src/RTDSimulator.EventHubs/RTDSimulator.EventHubs.csproj b/src/RTDSimulator.EventHubs/RTDSimulator.EventHubs.csproj new file mode 100644 index 0000000..f29e877 --- /dev/null +++ b/src/RTDSimulator.EventHubs/RTDSimulator.EventHubs.csproj @@ -0,0 +1,19 @@ + + + + net10.0 + enable + enable + https://github.com/Azure-Player/Real-Time-Data-Simulator/ + + + + + + + + + + + + diff --git a/src/RTDSimulator.Kusto/KustoConnectionBase.cs b/src/RTDSimulator.Kusto/KustoConnectionBase.cs new file mode 100644 index 0000000..98be6dc --- /dev/null +++ b/src/RTDSimulator.Kusto/KustoConnectionBase.cs @@ -0,0 +1,163 @@ +using System.Text; +using System.Text.Json; +using Azure.Core; +using Azure.Identity; +using Kusto.Data; +using Kusto.Data.Common; +using Kusto.Data.Ingestion; +using Kusto.Data.Net.Client; +using Kusto.Ingest; +using RTDSimulator.Core; + +namespace RTDSimulator.Kusto; + +/// +/// Shared base for the Kusto connections. Each batch of payloads is ingested into a +/// database table as newline-separated JSON objects (multijson). Subclasses choose the +/// endpoint form and the ingest-client kind (streaming vs. queued). +/// +public abstract class KustoConnectionBase : ITargetConnection, IIngestionVerifier +{ + private readonly string _clusterUri; + private readonly string _database; + private readonly string _table; + private readonly string? _ingestionMappingName; + private readonly TokenCredential _credential; + + private IKustoIngestClient? _ingestClient; + private KustoIngestionProperties? _ingestionProperties; + + protected KustoConnectionBase(string clusterUri, string database, string table, string? ingestionMappingName, TokenCredential? credential) + { + _clusterUri = clusterUri; + _database = database; + _table = table; + _ingestionMappingName = ingestionMappingName; + _credential = credential ?? new DefaultAzureCredential(); + } + + /// Maps the supplied cluster URI to the endpoint this ingestion mode needs. + protected abstract string ResolveEndpoint(string clusterUri); + + /// Creates the streaming or queued ingest client. + protected abstract IKustoIngestClient CreateIngestClient(KustoConnectionStringBuilder connectionStringBuilder); + + public Task ConnectAsync(CancellationToken cancellationToken = default) + { + if (_ingestClient is null) + { + var kcsb = new KustoConnectionStringBuilder(ResolveEndpoint(_clusterUri)) + .WithAadAzureTokenCredentialsAuthentication(_credential); + + _ingestClient = CreateIngestClient(kcsb); + _ingestionProperties = new KustoIngestionProperties(_database, _table) + { + Format = DataSourceFormat.multijson, + }; + + // Without a mapping, Kusto maps JSON properties to columns by name + // (case-sensitive); mismatched names produce all-null rows. A named + // mapping lets arbitrary field names map to the intended columns. + if (!string.IsNullOrWhiteSpace(_ingestionMappingName)) + { + _ingestionProperties.IngestionMapping = new IngestionMapping + { + IngestionMappingReference = _ingestionMappingName, + IngestionMappingKind = IngestionMappingKind.Json, + }; + } + } + + return Task.CompletedTask; + } + + public async Task SendBatchAsync(IReadOnlyCollection payloads, CancellationToken cancellationToken = default) + { + // #2: fail fast on malformed JSON before touching the service — multijson + // ingestion requires each message to be a JSON object. + ValidatePayloadsAreJson(payloads); + + await ConnectAsync(cancellationToken); + + var stream = new MemoryStream(); + await using (var writer = new StreamWriter(stream, new UTF8Encoding(false), leaveOpen: true)) + { + foreach (string payload in payloads) + { + await writer.WriteAsync(payload); + await writer.WriteAsync('\n'); + } + } + stream.Position = 0; + + IKustoIngestionResult result = await _ingestClient!.IngestFromStreamAsync( + stream, + _ingestionProperties!, + new StreamSourceOptions { LeaveOpen = false }); + + // #1: surface any synchronously-reported failure (streaming reports the real + // outcome here; queued reports 'Queued' and is verified after the run instead). + ThrowIfIngestionFailed(result); + } + + private static void ValidatePayloadsAreJson(IReadOnlyCollection payloads) + { + int index = 0; + foreach (string payload in payloads) + { + try + { + using var _ = JsonDocument.Parse(payload); + } + catch (JsonException ex) + { + throw new FormatException( + $"Payload #{index} is not valid JSON (Kusto multijson ingestion requires JSON objects): {ex.Message}", ex); + } + index++; + } + } + + private static void ThrowIfIngestionFailed(IKustoIngestionResult result) + { + foreach (IngestionStatus status in result.GetIngestionStatusCollection()) + { + if (status.Status == Status.Failed) + { + throw new InvalidOperationException( + $"Kusto ingestion failed ({status.ErrorCode}): {status.Details}"); + } + } + } + + public async Task> GetIngestionFailuresSinceAsync(DateTime sinceUtc, CancellationToken cancellationToken = default) + { + // Ingestion failures are recorded cluster-side; query the engine endpoint. + var kcsb = new KustoConnectionStringBuilder(KustoUris.ToEngine(_clusterUri)) + .WithAadAzureTokenCredentialsAuthentication(_credential); + + string since = sinceUtc.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ"); + string command = + $".show ingestion failures | where Table == '{_table}' and FailedOn > datetime({since}) " + + "| project FailedOn, FailureKind, Details"; + + var failures = new List(); + using ICslAdminProvider admin = KustoClientFactory.CreateCslAdminProvider(kcsb); + using System.Data.IDataReader reader = await Task.Run( + () => admin.ExecuteControlCommand(_database, command), cancellationToken); + + while (reader.Read()) + { + failures.Add($"{reader["FailedOn"]:o} [{reader["FailureKind"]}] {reader["Details"]}"); + } + + return failures; + } + + public ValueTask DisposeAsync() + { + _ingestClient?.Dispose(); + _ingestClient = null; + return ValueTask.CompletedTask; + } +} diff --git a/src/RTDSimulator.Kusto/KustoConnectors.cs b/src/RTDSimulator.Kusto/KustoConnectors.cs new file mode 100644 index 0000000..0e57703 --- /dev/null +++ b/src/RTDSimulator.Kusto/KustoConnectors.cs @@ -0,0 +1,81 @@ +using Azure.Core; +using RTDSimulator.Core; + +namespace RTDSimulator.Kusto; + +/// Shared parameter keys for the Kusto connectors. +public static class KustoParameters +{ + public const string ClusterUriKey = "clusterUri"; + public const string DatabaseKey = "database"; + public const string TableKey = "table"; + public const string MappingKey = "mapping"; + + internal const string DefaultClusterUri = "https://mycluster.westeurope.kusto.windows.net"; + + internal const string MappingHelp = + "Optional. Name of a JSON ingestion mapping on the table. Without a mapping, " + + "JSON property names must exactly match column names (case-sensitive) or rows ingest as nulls."; +} + +/// +/// for Azure Data Explorer (Kusto) using streaming +/// ingestion. Authentication uses Azure Identity (GUI sign-in or the CLI credential chain). +/// +public sealed class KustoStreamingConnector : ITargetConnector +{ + public string Key => "kusto-streaming"; + + public string DisplayName => "Azure Data Explorer — Streaming ingestion"; + + public IReadOnlyList Parameters { get; } = new[] + { + new ConnectionParameter( + KustoParameters.ClusterUriKey, + "Cluster URI", + defaultValue: KustoParameters.DefaultClusterUri, + helpText: "Engine/query cluster URI. Streaming ingestion must be enabled on the cluster and target table."), + new ConnectionParameter(KustoParameters.DatabaseKey, "Database"), + new ConnectionParameter(KustoParameters.TableKey, "Table"), + new ConnectionParameter(KustoParameters.MappingKey, "Ingestion mapping", required: false, helpText: KustoParameters.MappingHelp), + }; + + public ITargetConnection CreateConnection(IReadOnlyDictionary values, TokenCredential? credential = null) + => new KustoStreamingConnection( + values.GetValueOrDefault(KustoParameters.ClusterUriKey, string.Empty), + values.GetValueOrDefault(KustoParameters.DatabaseKey, string.Empty), + values.GetValueOrDefault(KustoParameters.TableKey, string.Empty), + values.GetValueOrDefault(KustoParameters.MappingKey, string.Empty), + credential); +} + +/// +/// for Azure Data Explorer (Kusto) using queued ingestion. +/// Requires no streaming policy, so it works against any cluster/table without setup. +/// +public sealed class KustoQueuedConnector : ITargetConnector +{ + public string Key => "kusto-queued"; + + public string DisplayName => "Azure Data Explorer — Queued ingestion"; + + public IReadOnlyList Parameters { get; } = new[] + { + new ConnectionParameter( + KustoParameters.ClusterUriKey, + "Cluster URI", + defaultValue: KustoParameters.DefaultClusterUri, + helpText: "Engine/query cluster URI (the 'ingest-' data-management endpoint is derived automatically). No streaming policy required."), + new ConnectionParameter(KustoParameters.DatabaseKey, "Database"), + new ConnectionParameter(KustoParameters.TableKey, "Table"), + new ConnectionParameter(KustoParameters.MappingKey, "Ingestion mapping", required: false, helpText: KustoParameters.MappingHelp), + }; + + public ITargetConnection CreateConnection(IReadOnlyDictionary values, TokenCredential? credential = null) + => new KustoQueuedConnection( + values.GetValueOrDefault(KustoParameters.ClusterUriKey, string.Empty), + values.GetValueOrDefault(KustoParameters.DatabaseKey, string.Empty), + values.GetValueOrDefault(KustoParameters.TableKey, string.Empty), + values.GetValueOrDefault(KustoParameters.MappingKey, string.Empty), + credential); +} diff --git a/src/RTDSimulator.Kusto/KustoQueuedConnection.cs b/src/RTDSimulator.Kusto/KustoQueuedConnection.cs new file mode 100644 index 0000000..5da9f88 --- /dev/null +++ b/src/RTDSimulator.Kusto/KustoQueuedConnection.cs @@ -0,0 +1,23 @@ +using Azure.Core; +using Kusto.Data; +using Kusto.Ingest; + +namespace RTDSimulator.Kusto; + +/// +/// Ingests into a Kusto table using queued ingestion (batched server-side). Uses +/// the data-management (ingest-) endpoint and needs no streaming policy, so it +/// works against any cluster/table out of the box. +/// +public sealed class KustoQueuedConnection : KustoConnectionBase +{ + public KustoQueuedConnection(string clusterUri, string database, string table, string? ingestionMappingName = null, TokenCredential? credential = null) + : base(clusterUri, database, table, ingestionMappingName, credential) + { + } + + protected override string ResolveEndpoint(string clusterUri) => KustoUris.ToIngest(clusterUri); + + protected override IKustoIngestClient CreateIngestClient(KustoConnectionStringBuilder connectionStringBuilder) + => KustoIngestFactory.CreateQueuedIngestClient(connectionStringBuilder); +} diff --git a/src/RTDSimulator.Kusto/KustoStreamingConnection.cs b/src/RTDSimulator.Kusto/KustoStreamingConnection.cs new file mode 100644 index 0000000..f7899fd --- /dev/null +++ b/src/RTDSimulator.Kusto/KustoStreamingConnection.cs @@ -0,0 +1,23 @@ +using Azure.Core; +using Kusto.Data; +using Kusto.Ingest; + +namespace RTDSimulator.Kusto; + +/// +/// Ingests into a Kusto table using streaming ingestion (low latency). Uses the +/// engine/query endpoint and requires the streaming ingestion policy to be enabled on +/// the cluster and target table. +/// +public sealed class KustoStreamingConnection : KustoConnectionBase +{ + public KustoStreamingConnection(string clusterUri, string database, string table, string? ingestionMappingName = null, TokenCredential? credential = null) + : base(clusterUri, database, table, ingestionMappingName, credential) + { + } + + protected override string ResolveEndpoint(string clusterUri) => KustoUris.ToEngine(clusterUri); + + protected override IKustoIngestClient CreateIngestClient(KustoConnectionStringBuilder connectionStringBuilder) + => KustoIngestFactory.CreateStreamingIngestClient(connectionStringBuilder); +} diff --git a/src/RTDSimulator.Kusto/KustoUris.cs b/src/RTDSimulator.Kusto/KustoUris.cs new file mode 100644 index 0000000..f80e13d --- /dev/null +++ b/src/RTDSimulator.Kusto/KustoUris.cs @@ -0,0 +1,37 @@ +namespace RTDSimulator.Kusto; + +/// +/// Helpers for switching a Kusto cluster URI between its engine/query endpoint +/// (used for streaming ingestion) and its data-management endpoint, which carries an +/// ingest- host prefix (used for queued ingestion). Both connectors therefore +/// accept the same "Cluster URI" and derive the endpoint they need. +/// +internal static class KustoUris +{ + private const string IngestPrefix = "ingest-"; + + /// Returns the data-management (ingest-) endpoint for queued ingestion. + public static string ToIngest(string clusterUri) => WithIngestPrefix(clusterUri, add: true); + + /// Returns the engine/query endpoint (no ingest- prefix) for streaming ingestion. + public static string ToEngine(string clusterUri) => WithIngestPrefix(clusterUri, add: false); + + private static string WithIngestPrefix(string clusterUri, bool add) + { + if (!Uri.TryCreate(clusterUri, UriKind.Absolute, out Uri? uri)) + return clusterUri; + + bool hasPrefix = uri.Host.StartsWith(IngestPrefix, StringComparison.OrdinalIgnoreCase); + string host; + if (add && !hasPrefix) + host = IngestPrefix + uri.Host; + else if (!add && hasPrefix) + host = uri.Host.Substring(IngestPrefix.Length); + else + return clusterUri; // already in the desired form + + string portPart = uri.IsDefaultPort ? string.Empty : ":" + uri.Port; + string path = uri.AbsolutePath == "/" ? string.Empty : uri.AbsolutePath; + return $"{uri.Scheme}://{host}{portPart}{path}"; + } +} diff --git a/src/RTDSimulator.Kusto/RTDSimulator.Kusto.csproj b/src/RTDSimulator.Kusto/RTDSimulator.Kusto.csproj new file mode 100644 index 0000000..02d2a8c --- /dev/null +++ b/src/RTDSimulator.Kusto/RTDSimulator.Kusto.csproj @@ -0,0 +1,23 @@ + + + + net10.0 + enable + enable + https://github.com/Azure-Player/Real-Time-Data-Simulator/ + + + + + + + + + + + + + + + + diff --git a/src/RTDSimulator.ServiceBus/RTDSimulator.ServiceBus.csproj b/src/RTDSimulator.ServiceBus/RTDSimulator.ServiceBus.csproj new file mode 100644 index 0000000..31c04d6 --- /dev/null +++ b/src/RTDSimulator.ServiceBus/RTDSimulator.ServiceBus.csproj @@ -0,0 +1,19 @@ + + + + net10.0 + enable + enable + https://github.com/Azure-Player/Real-Time-Data-Simulator/ + + + + + + + + + + + + diff --git a/src/RTDSimulator.ServiceBus/ServiceBusConnection.cs b/src/RTDSimulator.ServiceBus/ServiceBusConnection.cs new file mode 100644 index 0000000..bdf2c59 --- /dev/null +++ b/src/RTDSimulator.ServiceBus/ServiceBusConnection.cs @@ -0,0 +1,73 @@ +using Azure.Core; +using Azure.Messaging.ServiceBus; +using RTDSimulator.Core; + +namespace RTDSimulator.ServiceBus; + +/// +/// implementation that sends generated load to an +/// Azure Service Bus queue or topic via the Service Bus sender client. +/// +public sealed class ServiceBusConnection : ITargetConnection +{ + private readonly string _connectionString; + private readonly string _entityName; + private readonly TokenCredential? _credential; + + private ServiceBusClient? _client; + private ServiceBusSender? _sender; + + /// + /// Service Bus connection string, or the fully-qualified namespace + /// (e.g. my-namespace.servicebus.windows.net) when a credential is supplied. + /// + /// Name of the target queue or topic. + /// Optional token credential for Entra ID authentication. + public ServiceBusConnection(string connectionString, string entityName, TokenCredential? credential = null) + { + _connectionString = connectionString; + _entityName = entityName; + _credential = credential; + } + + public Task ConnectAsync(CancellationToken cancellationToken = default) + { + if (_client is null) + { + _client = _credential is null + ? new ServiceBusClient(_connectionString) + : new ServiceBusClient(AzureConnection.ResolveNamespace(_connectionString), _credential); + _sender = _client.CreateSender(_entityName); + } + + return Task.CompletedTask; + } + + public async Task SendBatchAsync(IReadOnlyCollection payloads, CancellationToken cancellationToken = default) + { + await ConnectAsync(cancellationToken); + + using ServiceBusMessageBatch messageBatch = await _sender!.CreateMessageBatchAsync(cancellationToken); + foreach (string payload in payloads) + { + messageBatch.TryAddMessage(new ServiceBusMessage(payload)); + } + + await _sender.SendMessagesAsync(messageBatch, cancellationToken); + } + + public async ValueTask DisposeAsync() + { + if (_sender is not null) + { + await _sender.DisposeAsync(); + _sender = null; + } + + if (_client is not null) + { + await _client.DisposeAsync(); + _client = null; + } + } +} diff --git a/src/RTDSimulator.ServiceBus/ServiceBusConnector.cs b/src/RTDSimulator.ServiceBus/ServiceBusConnector.cs new file mode 100644 index 0000000..c8c08f5 --- /dev/null +++ b/src/RTDSimulator.ServiceBus/ServiceBusConnector.cs @@ -0,0 +1,36 @@ +using Azure.Core; +using RTDSimulator.Core; + +namespace RTDSimulator.ServiceBus; + +/// +/// describing an Azure Service Bus target (queue or topic). +/// +public sealed class ServiceBusConnector : ITargetConnector +{ + public const string ConnectionKey = "connection"; + public const string EntityKey = "entity"; + + public string Key => "servicebus"; + + public string DisplayName => "Azure Service Bus"; + + public IReadOnlyList Parameters { get; } = new[] + { + new ConnectionParameter( + ConnectionKey, + "Connection string / Namespace", + secret: true, + helpText: "SAS connection string, or the fully-qualified namespace when signed in with Azure Identity."), + new ConnectionParameter( + EntityKey, + "Queue / Topic name"), + }; + + public ITargetConnection CreateConnection(IReadOnlyDictionary values, TokenCredential? credential = null) + { + string connection = values.GetValueOrDefault(ConnectionKey, string.Empty); + string entity = values.GetValueOrDefault(EntityKey, string.Empty); + return new ServiceBusConnection(connection, entity, credential); + } +} diff --git a/src/RTDSimulatorDesktopApp/EventSender.cs b/src/RTDSimulatorDesktopApp/EventSender.cs deleted file mode 100644 index 6dd1634..0000000 --- a/src/RTDSimulatorDesktopApp/EventSender.cs +++ /dev/null @@ -1,170 +0,0 @@ -using System.Collections.Concurrent; -using System.Text; -using System.Text.RegularExpressions; -using Azure.Messaging.EventHubs; -using Azure.Messaging.EventHubs.Producer; -using Microsoft.CodeAnalysis.CSharp.Scripting; -using Microsoft.CodeAnalysis.CSharp.Syntax; -using Microsoft.CodeAnalysis.Scripting; -using Azure.Identity; - -namespace RTDSimulatorDesktopApp -{ - public class EventSender - { - public Int64 BatchSizeInBytes = 0; - public int messagesCount = 0; - public int BatchesNo = 1; - public int EventsPerBatch = 1; - public Dictionary Variables = new Dictionary(); - public TimeSpan WaitTime = TimeSpan.FromSeconds(0); - - string _payload = ""; - - //private Lazy _producerClient; - private EventHubProducerClient _producerClient; - - private Dictionary Expressions = new Dictionary(); - private ConcurrentDictionary> preCompiledScripts { get; set; } = new ConcurrentDictionary>(); - private TargetConnection _conn; - - public EventSender(string payload) - { - _payload = payload; - - Variables.Add("UserId", "Random(5000,5100)"); - Variables.Add("ProductId", "Random(700,999)"); - Variables.Add("Device", "RandomItem(mobile|tablet|pc)"); - Variables.Add("DateTime.Now", "$DateTime.Now"); - Variables.Add("FuelType(MessageIndex)", "BIOMASS|CCGT|COAL|INTELEC|INTEW|INTFR|INTIFA2|INTIRL|INTNED|INTNEM|INTNSL|INTVKL|NPSHYD|NUCLEAR|OCGT|OIL|OTHER|PS|WIND"); - Variables.Add("SettlementPeriod", "48"); - - // Finding expressions... - // Example of an expression definition: {{$DateTime.Now}} {{$new Random().Next(100,200)}} - var pattern = @"{{(\$.*?)}}"; - var matches = Regex.Matches(_payload, pattern); - foreach (Match m in matches) - { - if (!Expressions.ContainsKey(m.Value)) - { - Expressions.Add(m.Value, m.Value.Substring(3, m.Value.Length - 5)); - } - } - } - - - - ~EventSender() - { - //if( _producerClient.IsValueCreated) - //{ - // _producerClient.DisposeAsync().GetAwaiter().GetResult(); - //} - } - - public async Task Send(TargetConnection conn, CancellationToken cancellationToken) - { - _conn = conn; - _producerClient = conn.Connect(); - - DateTime nextRun = DateTime.Now; - for (int batch = 0; batch < BatchesNo; batch++) - { - BatchSizeInBytes = 0; - using EventDataBatch eventBatch = await _producerClient.CreateBatchAsync(); - for (int m = 0; m < EventsPerBatch; m++) - { - if (cancellationToken.IsCancellationRequested) - return; - - string payload = await GetPayload(m); - eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes(payload))); - messagesCount++; - BatchSizeInBytes += payload.Length; - } - - await SleepUntil(nextRun, cancellationToken); - - if (cancellationToken.IsCancellationRequested) - return; - - await _producerClient.SendAsync(eventBatch); - OnBatchSent(this, new EventArgs()); - - nextRun = DateTime.Now.Add(this.WaitTime); //Thread.Sleep(WaitTime); - } - } - - private async Task SleepUntil(DateTime t, CancellationToken cancellationToken) - { - TimeSpan waitTime = t.Subtract(DateTime.Now); - if (waitTime <= TimeSpan.Zero) return; - try - { - await Task.Delay(waitTime, cancellationToken); - } - catch (Exception ex) - { - //don't throw if cancelled - } - } - - public async Task GetPayload(int msgIndex) - { - Random rnd = new Random(); - string payload = _payload; - - // Evaluating Variables - foreach (var v in Variables) - { - string val = v.Value; - string key = v.Key; - if (val == "$DateTime.Now") { val = DateTime.Now.ToString("O"); }; - if (val.StartsWith("Random(") && val.EndsWith(")")) - { - val = val.Substring(7, val.Length - 8); - int min = int.Parse(val.Split(',')[0]); - int max = int.Parse(val.Split(',')[1]); - val = rnd.Next(min, max).ToString(); - } - if (val.StartsWith("RandomItem(") && val.EndsWith(")")) - { - val = val.Substring(11, val.Length - 12); - String[] arr = val.Split("|"); - int i = rnd.Next(0, arr.Length); - val = arr[i]; - } - if (key == "FuelType(MessageIndex)") - { - String[] arr = val.Split("|"); - int i = msgIndex % arr.Length; // iterate through fuel types - val = arr[i]; - } - payload = payload.Replace("{{" + v.Key + "}}", val); - } - - // Evaluating Expressions - // https://github.com/dotnet/roslyn/blob/main/docs/wiki/Scripting-API-Samples.md - foreach (string key in Expressions.Keys) { - String exp = Expressions[key]; - - if(!preCompiledScripts.TryGetValue(exp, out Script script)) - { - script = CSharpScript.Create(exp, ScriptOptions.Default.WithImports("System")); - preCompiledScripts.TryAdd(exp, script); - } - - var result = await script.RunAsync(); - - payload = payload.Replace(key, result.ReturnValue.ToString()); - } - - return payload; - } - - public event EventHandler OnBatchSent; - - - - } -} diff --git a/src/RTDSimulatorDesktopApp/ProductVisitMessage.cs b/src/RTDSimulatorDesktopApp/ProductVisitMessage.cs deleted file mode 100644 index 76a1a1a..0000000 --- a/src/RTDSimulatorDesktopApp/ProductVisitMessage.cs +++ /dev/null @@ -1,68 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace RTDSimulatorDesktopApp -{ - //private static ProductVisitMessage GenerateProductVisitMessage(DateTime dateValue) - //{ - // //DateTime eventdatetime2 = DateTime.Now; - // Random rnduserid = new Random(); - // int userid = rnduserid.Next(5000, 5100); - - // Random rndproductnumber = new Random(); - // int productnumber = rndproductnumber.Next(700, 999); - - // Random rndpageviewlength = new Random(); - // int pageviewlength = rndpageviewlength.Next(1, 420); - - // Random rnd = new Random(); - - // String device; - - // int devicetype = rnd.Next(1, 4); - - // if (devicetype == 1) - // { - // device = "mobile"; - // } - // else if (devicetype == 2) - // { - // device = "tablet"; - // } - // else - // { - // device = "pc"; - // } - - // Random rndeventtype = new Random(); - - // String eventtype; - - // int eventtypenum = rnd.Next(1, 10); - - // if (eventtypenum <= 8) - // { - // eventtype = "browseproduct"; - // } - // else - // { - // eventtype = "putinbasket"; - // } - - // ProductVisitMessage productbrowsemessage = new ProductVisitMessage - // { - // UserID = userid, - // EventType = eventtype, - // EventDateTime = dateValue, - // ProductID = Int32.Parse(productnumber.ToString()), - // URL = "/product/" + productnumber.ToString(), - // Device = device, - // SessionViewSeconds = pageviewlength - // }; - - // return productbrowsemessage; - //} -} diff --git a/src/RTDSimulatorDesktopApp/Program.cs b/src/RTDSimulatorDesktopApp/Program.cs index 4aca79e..d192824 100644 --- a/src/RTDSimulatorDesktopApp/Program.cs +++ b/src/RTDSimulatorDesktopApp/Program.cs @@ -1,23 +1,22 @@ -namespace RTDSimulatorDesktopApp +namespace RTDSimulatorDesktopApp; + +internal static class Program { - internal static class Program + /// + /// The main entry point for the application. + /// + [STAThread] + static void Main() { - /// - /// The main entry point for the application. - /// - [STAThread] - static void Main() - { - // To customize application configuration such as set high DPI settings or default font, - // see https://aka.ms/applicationconfiguration. - ApplicationConfiguration.Initialize(); - Application.Run(new frmGenerator()); - //Program.RunForm(new Form1(), ""); - } + // To customize application configuration such as set high DPI settings or default font, + // see https://aka.ms/applicationconfiguration. + ApplicationConfiguration.Initialize(); + Application.Run(new frmGenerator()); + //Program.RunForm(new Form1(), ""); + } - static void RunForm(frmGenerator f, string template) - { - f.Show(); - } + static void RunForm(frmGenerator f, string template) + { + f.Show(); } } \ No newline at end of file diff --git a/src/RTDSimulatorDesktopApp/RTDSimulatorDesktopApp.csproj b/src/RTDSimulatorDesktopApp/RTDSimulatorDesktopApp.csproj index da8aea1..22a5426 100644 --- a/src/RTDSimulatorDesktopApp/RTDSimulatorDesktopApp.csproj +++ b/src/RTDSimulatorDesktopApp/RTDSimulatorDesktopApp.csproj @@ -8,8 +8,6 @@ enable Kamil Nowinski AzurePlayer - 0.6.0 - 0.6.0 2024 sc-real-time-computing-stream-computing-512.png https://github.com/Azure-Player/Real-Time-Data-Simulator/ @@ -17,11 +15,19 @@ - - - - - + + + + + PreserveNewest + + + + + + + + diff --git a/src/RTDSimulatorDesktopApp/TargetConnection.cs b/src/RTDSimulatorDesktopApp/TargetConnection.cs deleted file mode 100644 index 7542403..0000000 --- a/src/RTDSimulatorDesktopApp/TargetConnection.cs +++ /dev/null @@ -1,32 +0,0 @@ -using Azure.Identity; -using Azure.Messaging.EventHubs.Producer; - -namespace RTDSimulatorDesktopApp -{ - public class TargetConnection - { - string _connectionString = "sb://"; - string _eventHubName = "es_"; - InteractiveBrowserCredential _credential; - - public TargetConnection(string connectionString, string eventHubName, InteractiveBrowserCredential credential) - { - _connectionString = connectionString; - _eventHubName = eventHubName; - _credential = credential; - } - - public EventHubProducerClient Connect() - { - //return new Lazy(() => new EventHubProducerClient(_connectionString, _eventHubName, cred)); - if (_credential == null) - { - return new EventHubProducerClient(_connectionString, _eventHubName); - } - else - { - return new EventHubProducerClient(_connectionString, _eventHubName, _credential); - } - } - } -} diff --git a/src/RTDSimulatorDesktopApp/frmGenerator.Designer.cs b/src/RTDSimulatorDesktopApp/frmGenerator.Designer.cs index d0e0aaf..b65e586 100644 --- a/src/RTDSimulatorDesktopApp/frmGenerator.Designer.cs +++ b/src/RTDSimulatorDesktopApp/frmGenerator.Designer.cs @@ -32,10 +32,6 @@ private void InitializeComponent() { components = new System.ComponentModel.Container(); System.ComponentModel.ComponentResourceManager resources = new System.ComponentModel.ComponentResourceManager(typeof(frmGenerator)); - label1 = new Label(); - label2 = new Label(); - txtCnnStr = new TextBox(); - txtEventHubName = new TextBox(); label3 = new Label(); txtPayload = new TextBox(); btnRun = new Button(); @@ -48,8 +44,8 @@ private void InitializeComponent() label8 = new Label(); SettingsTotalMsgCount = new NumericUpDown(); SettingsMsgPerBatchNumber = new NumericUpDown(); - SettingsBatchesPerThreadNumber = new NumericUpDown(); - SettingsThreadsNumber = new NumericUpDown(); + SettingsBatchesPerSenderNumber = new NumericUpDown(); + SettingsParallelismNumber = new NumericUpDown(); statusStrip1 = new StatusStrip(); statusBatches = new ToolStripStatusLabel(); status = new ToolStripStatusLabel(); @@ -59,6 +55,8 @@ private void InitializeComponent() loadMessageTemplateToolStripMenuItem = new ToolStripMenuItem(); saveMessageTemplateToolStripMenuItem = new ToolStripMenuItem(); saveAsToolStripMenuItem = new ToolStripMenuItem(); + loadVariableDefinitionsToolStripMenuItem = new ToolStripMenuItem(); + azureSignOutToolStripMenuItem = new ToolStripMenuItem(); toolStripSeparator1 = new ToolStripSeparator(); exitToolStripMenuItem = new ToolStripMenuItem(); helpToolStripMenuItem = new ToolStripMenuItem(); @@ -70,53 +68,23 @@ private void InitializeComponent() lastErrorTextBox = new TextBox(); groupBox3 = new GroupBox(); btnAzureAuth = new Button(); + cboTargetType = new ComboBox(); + lblTargetType = new Label(); + pnlParams = new TableLayoutPanel(); + toolTip1 = new ToolTip(components); + chkVerifyIngestion = new CheckBox(); groupBox1.SuspendLayout(); ((System.ComponentModel.ISupportInitialize)SettingsWaitTimeSec).BeginInit(); ((System.ComponentModel.ISupportInitialize)SettingsTotalMsgCount).BeginInit(); ((System.ComponentModel.ISupportInitialize)SettingsMsgPerBatchNumber).BeginInit(); - ((System.ComponentModel.ISupportInitialize)SettingsBatchesPerThreadNumber).BeginInit(); - ((System.ComponentModel.ISupportInitialize)SettingsThreadsNumber).BeginInit(); + ((System.ComponentModel.ISupportInitialize)SettingsBatchesPerSenderNumber).BeginInit(); + ((System.ComponentModel.ISupportInitialize)SettingsParallelismNumber).BeginInit(); statusStrip1.SuspendLayout(); menuStrip1.SuspendLayout(); groupBox2.SuspendLayout(); groupBox3.SuspendLayout(); SuspendLayout(); - // - // label1 - // - label1.AutoSize = true; - label1.Location = new Point(15, 35); - label1.Name = "label1"; - label1.Size = new Size(157, 15); - label1.TabIndex = 0; - label1.Text = "Endpoint &Connection String:"; - // - // label2 - // - label2.AutoSize = true; - label2.Location = new Point(15, 97); - label2.Name = "label2"; - label2.Size = new Size(139, 15); - label2.TabIndex = 1; - label2.Text = "&Topic / Event Hub Name:"; - // - // txtCnnStr - // - txtCnnStr.Location = new Point(15, 53); - txtCnnStr.Name = "txtCnnStr"; - txtCnnStr.Size = new Size(424, 23); - txtCnnStr.TabIndex = 2; - txtCnnStr.Text = "Endpoint=sb://****fkawjq507mc.servicebus.windows.net/;SharedAccessKeyName=key_0000;SharedAccessKey=****"; - // - // txtEventHubName - // - txtEventHubName.Font = new Font("Courier New", 9F, FontStyle.Regular, GraphicsUnit.Point, 0); - txtEventHubName.Location = new Point(15, 115); - txtEventHubName.Name = "txtEventHubName"; - txtEventHubName.Size = new Size(424, 21); - txtEventHubName.TabIndex = 3; - txtEventHubName.Text = "es_08960000000000000000"; - // + // // label3 // label3.AutoSize = true; @@ -148,7 +116,7 @@ private void InitializeComponent() btnRun.ImageAlign = ContentAlignment.MiddleLeft; btnRun.ImageKey = "Visualpharm-Must-Have-Play.ico"; btnRun.ImageList = imageList1; - btnRun.Location = new Point(900, 447); + btnRun.Location = new Point(900, 497); btnRun.Name = "btnRun"; btnRun.Size = new Size(201, 54); btnRun.TabIndex = 6; @@ -184,7 +152,7 @@ private void InitializeComponent() label4.Name = "label4"; label4.Size = new Size(398, 21); label4.TabIndex = 8; - label4.Text = "Threads x Batches x Messages = TOTAL Messages"; + label4.Text = "Parallelism x Batches x Messages = TOTAL Messages"; // // groupBox1 // @@ -194,8 +162,8 @@ private void InitializeComponent() groupBox1.Controls.Add(label8); groupBox1.Controls.Add(SettingsTotalMsgCount); groupBox1.Controls.Add(SettingsMsgPerBatchNumber); - groupBox1.Controls.Add(SettingsBatchesPerThreadNumber); - groupBox1.Controls.Add(SettingsThreadsNumber); + groupBox1.Controls.Add(SettingsBatchesPerSenderNumber); + groupBox1.Controls.Add(SettingsParallelismNumber); groupBox1.Controls.Add(label4); groupBox1.Font = new Font("Segoe UI", 12F, FontStyle.Regular, GraphicsUnit.Point, 0); groupBox1.Location = new Point(643, 279); @@ -260,29 +228,29 @@ private void InitializeComponent() SettingsMsgPerBatchNumber.Value = new decimal(new int[] { 1, 0, 0, 0 }); SettingsMsgPerBatchNumber.ValueChanged += SettingsMsgPerBatchNumber_ValueChanged; // - // SettingsBatchesPerThreadNumber - // - SettingsBatchesPerThreadNumber.Font = new Font("Segoe UI Semibold", 12F, FontStyle.Bold); - SettingsBatchesPerThreadNumber.Location = new Point(106, 76); - SettingsBatchesPerThreadNumber.Maximum = new decimal(new int[] { 1000, 0, 0, 0 }); - SettingsBatchesPerThreadNumber.Name = "SettingsBatchesPerThreadNumber"; - SettingsBatchesPerThreadNumber.Size = new Size(76, 29); - SettingsBatchesPerThreadNumber.TabIndex = 13; - SettingsBatchesPerThreadNumber.TextAlign = HorizontalAlignment.Right; - SettingsBatchesPerThreadNumber.Value = new decimal(new int[] { 1, 0, 0, 0 }); - SettingsBatchesPerThreadNumber.ValueChanged += SettingsBatchesPerThreadNumber_ValueChanged; - // - // SettingsThreadsNumber - // - SettingsThreadsNumber.Font = new Font("Segoe UI Semibold", 12F, FontStyle.Bold); - SettingsThreadsNumber.Location = new Point(15, 76); - SettingsThreadsNumber.Maximum = new decimal(new int[] { 50, 0, 0, 0 }); - SettingsThreadsNumber.Name = "SettingsThreadsNumber"; - SettingsThreadsNumber.Size = new Size(85, 29); - SettingsThreadsNumber.TabIndex = 11; - SettingsThreadsNumber.TextAlign = HorizontalAlignment.Right; - SettingsThreadsNumber.Value = new decimal(new int[] { 1, 0, 0, 0 }); - SettingsThreadsNumber.ValueChanged += SettingsThreadsNumber_ValueChanged; + // SettingsBatchesPerSenderNumber + // + SettingsBatchesPerSenderNumber.Font = new Font("Segoe UI Semibold", 12F, FontStyle.Bold); + SettingsBatchesPerSenderNumber.Location = new Point(106, 76); + SettingsBatchesPerSenderNumber.Maximum = new decimal(new int[] { 1000, 0, 0, 0 }); + SettingsBatchesPerSenderNumber.Name = "SettingsBatchesPerSenderNumber"; + SettingsBatchesPerSenderNumber.Size = new Size(76, 29); + SettingsBatchesPerSenderNumber.TabIndex = 13; + SettingsBatchesPerSenderNumber.TextAlign = HorizontalAlignment.Right; + SettingsBatchesPerSenderNumber.Value = new decimal(new int[] { 1, 0, 0, 0 }); + SettingsBatchesPerSenderNumber.ValueChanged += SettingsBatchesPerSenderNumber_ValueChanged; + // + // SettingsParallelismNumber + // + SettingsParallelismNumber.Font = new Font("Segoe UI Semibold", 12F, FontStyle.Bold); + SettingsParallelismNumber.Location = new Point(15, 76); + SettingsParallelismNumber.Maximum = new decimal(new int[] { 50, 0, 0, 0 }); + SettingsParallelismNumber.Name = "SettingsParallelismNumber"; + SettingsParallelismNumber.Size = new Size(85, 29); + SettingsParallelismNumber.TabIndex = 11; + SettingsParallelismNumber.TextAlign = HorizontalAlignment.Right; + SettingsParallelismNumber.Value = new decimal(new int[] { 1, 0, 0, 0 }); + SettingsParallelismNumber.ValueChanged += SettingsParallelismNumber_ValueChanged; // // statusStrip1 // @@ -324,7 +292,7 @@ private void InitializeComponent() // // FileMenu // - FileMenu.DropDownItems.AddRange(new ToolStripItem[] { loadMessageTemplateToolStripMenuItem, saveMessageTemplateToolStripMenuItem, saveAsToolStripMenuItem, toolStripSeparator1, exitToolStripMenuItem }); + FileMenu.DropDownItems.AddRange(new ToolStripItem[] { loadMessageTemplateToolStripMenuItem, saveMessageTemplateToolStripMenuItem, saveAsToolStripMenuItem, loadVariableDefinitionsToolStripMenuItem, azureSignOutToolStripMenuItem, toolStripSeparator1, exitToolStripMenuItem }); FileMenu.Name = "FileMenu"; FileMenu.Size = new Size(37, 20); FileMenu.Text = "&File"; @@ -352,6 +320,20 @@ private void InitializeComponent() saveAsToolStripMenuItem.Size = new Size(186, 22); saveAsToolStripMenuItem.Text = "Save As..."; saveAsToolStripMenuItem.Click += saveAsToolStripMenuItem_Click; + // + // loadVariableDefinitionsToolStripMenuItem + // + loadVariableDefinitionsToolStripMenuItem.Name = "loadVariableDefinitionsToolStripMenuItem"; + loadVariableDefinitionsToolStripMenuItem.Size = new Size(186, 22); + loadVariableDefinitionsToolStripMenuItem.Text = "Load Variable Definitions..."; + loadVariableDefinitionsToolStripMenuItem.Click += loadVariableDefinitionsToolStripMenuItem_Click; + // + // azureSignOutToolStripMenuItem + // + azureSignOutToolStripMenuItem.Name = "azureSignOutToolStripMenuItem"; + azureSignOutToolStripMenuItem.Size = new Size(186, 22); + azureSignOutToolStripMenuItem.Text = "Azure: Sign out"; + azureSignOutToolStripMenuItem.Click += azureSignOutToolStripMenuItem_Click; // // toolStripSeparator1 // @@ -383,19 +365,60 @@ private void InitializeComponent() // groupBox2 // groupBox2.Anchor = AnchorStyles.Top | AnchorStyles.Right; + groupBox2.Controls.Add(chkVerifyIngestion); + groupBox2.Controls.Add(pnlParams); + groupBox2.Controls.Add(lblTargetType); + groupBox2.Controls.Add(cboTargetType); groupBox2.Controls.Add(btnAzureAuth); groupBox2.Controls.Add(btnTestCnn); - groupBox2.Controls.Add(label1); - groupBox2.Controls.Add(txtCnnStr); - groupBox2.Controls.Add(label2); - groupBox2.Controls.Add(txtEventHubName); groupBox2.Location = new Point(643, 88); groupBox2.Name = "groupBox2"; groupBox2.Size = new Size(458, 185); groupBox2.TabIndex = 12; groupBox2.TabStop = false; - groupBox2.Text = "Destination: EventHub"; - // + groupBox2.Text = "Destination"; + // + // lblTargetType + // + lblTargetType.AutoSize = true; + lblTargetType.Location = new Point(15, 28); + lblTargetType.Name = "lblTargetType"; + lblTargetType.Size = new Size(48, 15); + lblTargetType.TabIndex = 0; + lblTargetType.Text = "&Service:"; + // + // cboTargetType + // + cboTargetType.DropDownStyle = ComboBoxStyle.DropDownList; + cboTargetType.FormattingEnabled = true; + cboTargetType.Location = new Point(75, 25); + cboTargetType.Name = "cboTargetType"; + cboTargetType.Size = new Size(220, 23); + cboTargetType.TabIndex = 1; + cboTargetType.SelectedIndexChanged += cboTargetType_SelectedIndexChanged; + // + // chkVerifyIngestion + // + chkVerifyIngestion.AutoSize = true; + chkVerifyIngestion.Location = new Point(301, 27); + chkVerifyIngestion.Name = "chkVerifyIngestion"; + chkVerifyIngestion.Size = new Size(110, 19); + chkVerifyIngestion.TabIndex = 8; + chkVerifyIngestion.Text = "Verify ingestion"; + chkVerifyIngestion.UseVisualStyleBackColor = true; + // + // pnlParams + // + pnlParams.Anchor = AnchorStyles.Top | AnchorStyles.Left | AnchorStyles.Right; + pnlParams.AutoScroll = true; + pnlParams.ColumnCount = 2; + pnlParams.ColumnStyles.Add(new ColumnStyle(SizeType.Absolute, 175F)); + pnlParams.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 100F)); + pnlParams.Location = new Point(15, 54); + pnlParams.Name = "pnlParams"; + pnlParams.Size = new Size(424, 84); + pnlParams.TabIndex = 2; + // // btnTestCnn // btnTestCnn.Location = new Point(257, 141); @@ -429,7 +452,7 @@ private void InitializeComponent() btnCancel.ImageAlign = ContentAlignment.MiddleLeft; btnCancel.ImageKey = "Visualpharm-Must-Have-Stop.ico"; btnCancel.ImageList = imageList1; - btnCancel.Location = new Point(693, 447); + btnCancel.Location = new Point(693, 497); btnCancel.Name = "btnCancel"; btnCancel.Size = new Size(201, 54); btnCancel.TabIndex = 14; @@ -454,7 +477,7 @@ private void InitializeComponent() groupBox3.AutoSize = true; groupBox3.Controls.Add(lastErrorTextBox); groupBox3.Controls.Add(progressBar1); - groupBox3.Location = new Point(10, 507); + groupBox3.Location = new Point(10, 557); groupBox3.Name = "groupBox3"; groupBox3.Size = new Size(1100, 187); groupBox3.TabIndex = 13; @@ -475,7 +498,7 @@ private void InitializeComponent() // AutoScaleDimensions = new SizeF(7F, 15F); AutoScaleMode = AutoScaleMode.Font; - ClientSize = new Size(1124, 716); + ClientSize = new Size(1124, 766); Controls.Add(groupBox3); Controls.Add(btnCancel); Controls.Add(btnPreview); @@ -489,7 +512,7 @@ private void InitializeComponent() Icon = (Icon)resources.GetObject("$this.Icon"); KeyPreview = true; MainMenuStrip = menuStrip1; - MinimumSize = new Size(1140, 753); + MinimumSize = new Size(1140, 803); Name = "frmGenerator"; StartPosition = FormStartPosition.CenterScreen; Tag = "Real-Time Data Simulator (for Windows)"; @@ -501,8 +524,8 @@ private void InitializeComponent() ((System.ComponentModel.ISupportInitialize)SettingsWaitTimeSec).EndInit(); ((System.ComponentModel.ISupportInitialize)SettingsTotalMsgCount).EndInit(); ((System.ComponentModel.ISupportInitialize)SettingsMsgPerBatchNumber).EndInit(); - ((System.ComponentModel.ISupportInitialize)SettingsBatchesPerThreadNumber).EndInit(); - ((System.ComponentModel.ISupportInitialize)SettingsThreadsNumber).EndInit(); + ((System.ComponentModel.ISupportInitialize)SettingsBatchesPerSenderNumber).EndInit(); + ((System.ComponentModel.ISupportInitialize)SettingsParallelismNumber).EndInit(); statusStrip1.ResumeLayout(false); statusStrip1.PerformLayout(); menuStrip1.ResumeLayout(false); @@ -517,26 +540,24 @@ private void InitializeComponent() #endregion - private Label label1; - private Label label2; - private TextBox txtCnnStr; - private TextBox txtEventHubName; private Label label3; private TextBox txtPayload; private Button btnRun; private Label label4; private GroupBox groupBox1; - private NumericUpDown SettingsThreadsNumber; + private NumericUpDown SettingsParallelismNumber; private StatusStrip statusStrip1; private ToolStripStatusLabel statusBatches; private ToolStripStatusLabel status; - private NumericUpDown SettingsBatchesPerThreadNumber; + private NumericUpDown SettingsBatchesPerSenderNumber; private NumericUpDown SettingsTotalMsgCount; private NumericUpDown SettingsMsgPerBatchNumber; private MenuStrip menuStrip1; private ToolStripMenuItem FileMenu; private ToolStripMenuItem loadMessageTemplateToolStripMenuItem; private ToolStripMenuItem saveMessageTemplateToolStripMenuItem; + private ToolStripMenuItem loadVariableDefinitionsToolStripMenuItem; + private ToolStripMenuItem azureSignOutToolStripMenuItem; private ToolStripSeparator toolStripSeparator1; private ToolStripMenuItem exitToolStripMenuItem; private GroupBox groupBox2; @@ -555,5 +576,10 @@ private void InitializeComponent() private ImageList imageList1; private Button btnTestCnn; private Button btnAzureAuth; + private ComboBox cboTargetType; + private Label lblTargetType; + private TableLayoutPanel pnlParams; + private ToolTip toolTip1; + private CheckBox chkVerifyIngestion; } } diff --git a/src/RTDSimulatorDesktopApp/frmGenerator.cs b/src/RTDSimulatorDesktopApp/frmGenerator.cs index 9eaeff9..2924fef 100644 --- a/src/RTDSimulatorDesktopApp/frmGenerator.cs +++ b/src/RTDSimulatorDesktopApp/frmGenerator.cs @@ -6,315 +6,575 @@ using System.Data; using Azure.Core; using Azure.Identity; -using Microsoft.Graph; -using Microsoft.Identity.Client.Platforms.Features.DesktopOs.Kerberos; using System.Net; +using RTDSimulator.Core; +using RTDSimulator.EventHubs; +using RTDSimulator.ServiceBus; +using RTDSimulator.Kusto; -namespace RTDSimulatorDesktopApp +namespace RTDSimulatorDesktopApp; + +public partial class frmGenerator : Form { - public partial class frmGenerator : Form + public frmGenerator() + { + InitializeComponent(); + } + + private const string FILES_FILTER = "Text files (*.txt)|*.txt|JSON files (*.json)|*.json|All files (*.*)|*.*"; + private const Double BytesToMbps = 1024 * 1024 / 8; + + private string _FileName = "GenerationByFuelType"; // default save target + private bool _hasNamedFile = false; // true once a template is opened/saved + private bool _IsPayloadChanged = false; + private decimal _TotalMsgCount = 0; + private decimal _TotalBatchCount = 0; + private int _BatchSent = 0; + private int _MsgSent = 0; + private Int64 _TotalSizeInBytes = 0; + private DateTime _StartTime = DateTime.MinValue; + private DateTime _EndTime = DateTime.MinValue; + private CancellationTokenSource _CancellationTokenSource; + private ITargetConnection _conn; + private InteractiveBrowserCredential _credential; + private string? _signedInUser; + + // Persisted, encrypted token cache + authentication record so the interactive + // sign-in survives across app runs (silent re-auth until the refresh token expires). + private const string TokenCacheName = "RealTimeDataSimulator"; + private static readonly string AuthCacheDir = System.IO.Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "RealTimeDataSimulator"); + private static readonly string AuthRecordPath = System.IO.Path.Combine(AuthCacheDir, "auth.record"); + + private static InteractiveBrowserCredential CreatePersistentCredential(AuthenticationRecord? record) { - public frmGenerator() + var options = new InteractiveBrowserCredentialOptions { - InitializeComponent(); - } + TokenCachePersistenceOptions = new TokenCachePersistenceOptions { Name = TokenCacheName }, + AuthenticationRecord = record, + }; + return new InteractiveBrowserCredential(options); + } - private const string FILES_FILTER = "Text files (*.txt)|*.txt|JSON files (*.json)|*.json|All files (*.*)|*.*"; - private const Double BytesToMbps = 1024 * 1024 / 8; - - private string _FileName = "GenerationByFuelType"; - private bool _IsPayloadChanged = false; - private decimal _TotalMsgCount = 0; - private decimal _TotalBatchCount = 0; - private int _BatchSent = 0; - private int _MsgSent = 0; - private Int64 _TotalSizeInBytes = 0; - private DateTime _StartTime = DateTime.MinValue; - private DateTime _EndTime = DateTime.MinValue; - private CancellationTokenSource _CancellationTokenSource; - private TargetConnection _conn; - private InteractiveBrowserCredential _credential; - - [DesignerSerializationVisibility(DesignerSerializationVisibility.Hidden)] - public bool IsPayloadChanged + // Restores a previously cached sign-in (if any) so the user doesn't have to log in again. + private void TryRestoreCachedSignIn() + { + if (!System.IO.File.Exists(AuthRecordPath)) + return; + try { - get { return _IsPayloadChanged; } - set - { - _IsPayloadChanged = value; - RefreshAppTitle(); - } + using var stream = System.IO.File.OpenRead(AuthRecordPath); + AuthenticationRecord record = AuthenticationRecord.Deserialize(stream); + _credential = CreatePersistentCredential(record); + _signedInUser = record.Username; } - - private void ResetCounters() + catch (Exception ex) { - _BatchSent = 0; - _MsgSent = 0; - _StartTime = DateTime.Now; - _TotalSizeInBytes = 0; + lastErrorTextBox.Text = $"Could not restore cached sign-in: {ex.Message}"; } + } - private async void btnRun_Click(object sender, EventArgs e) - { - RecalcTotal(); - lastErrorTextBox.Text = ""; - progressBar1.ForeColor = Color.LimeGreen; + // Reflects sign-in state in the auth button (and its tooltip), which persists + // regardless of other status messages. + private void UpdateAuthUi() + { + bool signedIn = _credential != null; + btnAzureAuth.Text = signedIn ? "Azure: Sign out" : "Azure: Sign in"; + toolTip1.SetToolTip(btnAzureAuth, signedIn + ? $"Signed in as {_signedInUser} — click to sign out" + : "Sign in with Microsoft Entra ID"); + } - btnRun.Enabled = false; - progressBar1.Maximum = (int)_TotalBatchCount; - progressBar1.Value = 0; - ResetCounters(); + /// Soft-coded variable definitions used to expand the payload template. + private IReadOnlyList _variables = Array.Empty(); - statusBatches.Text = $"Batches sent: {_BatchSent} / {_TotalBatchCount}"; - status.Text = "Status: In Progress..."; - statusTime.Text = ""; + /// Connectors available for selection. Add new connectors here. + private readonly IReadOnlyList _connectors = new ITargetConnector[] + { + new EventHubsConnector(), + new ServiceBusConnector(), + new KustoStreamingConnector(), + new KustoQueuedConnector(), + }; - List<(int, Task)> sendTasks = new List<(int, Task)>(); + /// Live parameter text boxes for the selected connector, keyed by parameter key. + private readonly Dictionary _paramInputs = new(); - _conn = new TargetConnection(txtCnnStr.Text, txtEventHubName.Text, _credential); + private ITargetConnector SelectedConnector => _connectors[Math.Max(0, cboTargetType.SelectedIndex)]; - EventSender s = new EventSender(txtPayload.Text); - s.OnBatchSent += EventSender_OnBatchSent; - _CancellationTokenSource = new CancellationTokenSource(); + /// Rebuilds the parameter input rows to match the selected connector. + private void RebuildParameterInputs() + { + pnlParams.SuspendLayout(); + pnlParams.Controls.Clear(); + pnlParams.RowStyles.Clear(); + pnlParams.RowCount = 0; + _paramInputs.Clear(); - for (int i = 0; i < this.SettingsThreadsNumber.Value; i++) + foreach (ConnectionParameter p in SelectedConnector.Parameters) + { + var label = new Label { - s.BatchesNo = (int)SettingsBatchesPerThreadNumber.Value; - s.EventsPerBatch = (int)SettingsMsgPerBatchNumber.Value; - s.WaitTime = new TimeSpan(0, 0, (int)SettingsWaitTimeSec.Value); - sendTasks.Add((i, s.Send(_conn, _CancellationTokenSource.Token))); - } + Text = p.Label + ":", + AutoSize = true, + Anchor = AnchorStyles.Left, + Margin = new Padding(3, 5, 3, 0), + }; - try + var input = new System.Windows.Forms.TextBox { - btnCancel.Enabled = true; - await Task.WhenAll(sendTasks.Select(x => x.Item2)); - if (_CancellationTokenSource.IsCancellationRequested) - { - btnCancel.Enabled = false; - OnCancelled(); - } - else - { - btnCancel.Enabled = false; - OnCompleted(); - } - } - catch (Exception ex) + Text = p.DefaultValue ?? string.Empty, + Dock = DockStyle.Fill, + UseSystemPasswordChar = p.Secret, + Margin = new Padding(3, 2, 3, 2), + }; + if (!string.IsNullOrEmpty(p.HelpText)) { - progressBar1.ForeColor = Color.Red; - var allExceptions = sendTasks.Where(x => x.Item2.Exception != null).Select(x => (x.Item1, x.Item2.Exception)); - - StringBuilder sb = new StringBuilder(); - allExceptions.ToList().ForEach(x => sb.AppendLine($"Exception on thread {x.Item1}: {x.Exception?.Message ?? "Unknown exception occured during sending"}")); - OnFailed(sb.ToString()); + toolTip1.SetToolTip(input, p.HelpText); } - } + _paramInputs[p.Key] = input; - private void UpdateStatus(string Status) - { - _EndTime = DateTime.Now; - TimeSpan ts = _EndTime - _StartTime; - status.Text = $"Status: {Status}."; - statusBatches.Text = $"Sent {_MsgSent} messages in {_TotalBatchCount} batches. Total time: {ts:c} (TPS: {(_MsgSent / ts.TotalSeconds):F2}) | Bytes sent: {_TotalSizeInBytes:0,0} ({(_TotalSizeInBytes / BytesToMbps / ts.TotalSeconds):F2} Mbps)"; + int row = pnlParams.RowCount++; + pnlParams.RowStyles.Add(new RowStyle(SizeType.AutoSize)); + pnlParams.Controls.Add(label, 0, row); + pnlParams.Controls.Add(input, 1, row); } - private void OnCompleted() + pnlParams.ResumeLayout(true); + LayoutDestinationPanels(); + } + + // Grows the parameter panel to show every field (no clipping) and stacks the + // connection buttons and the Publisher settings group below it, so switching to a + // connector with more parameters (e.g. Kusto) reflows instead of hiding fields. + private void LayoutDestinationPanels() + { + int contentHeight = pnlParams.GetPreferredSize(new Size(pnlParams.Width, 0)).Height; + pnlParams.Height = Math.Max(contentHeight, 23); + + int buttonsTop = pnlParams.Bottom + 8; + btnAzureAuth.Top = buttonsTop; + btnTestCnn.Top = buttonsTop; + + groupBox2.Height = btnTestCnn.Bottom + 12; + groupBox1.Top = groupBox2.Bottom + 8; + } + + private void cboTargetType_SelectedIndexChanged(object sender, EventArgs e) + { + RebuildParameterInputs(); + } + + /// + /// Builds the target connection for the connector and parameters selected in the UI. + /// + private ITargetConnection CreateConnection() + { + var values = _paramInputs.ToDictionary(kv => kv.Key, kv => kv.Value.Text); + return SelectedConnector.CreateConnection(values, _credential); + } + + [DesignerSerializationVisibility(DesignerSerializationVisibility.Hidden)] + public bool IsPayloadChanged + { + get { return _IsPayloadChanged; } + set { - progressBar1.Maximum = progressBar1.Value; - UpdateStatus("Completed"); - btnRun.Enabled = true; + _IsPayloadChanged = value; + RefreshAppTitle(); } + } + + private DateTime _runStartUtc = DateTime.MinValue; + + private void ResetCounters() + { + _BatchSent = 0; + _MsgSent = 0; + _StartTime = DateTime.Now; + _runStartUtc = DateTime.UtcNow; + _TotalSizeInBytes = 0; + } - private void OnCancelled() + /// + /// Verifies all required parameters for the selected connector have a value. + /// Shows a message and focuses the offending field if not. + /// + private bool ValidateRequiredParameters() + { + foreach (ConnectionParameter p in SelectedConnector.Parameters) { - if (progressBar1.Value == 0) + if (p.Required && _paramInputs.TryGetValue(p.Key, out var input) && string.IsNullOrWhiteSpace(input.Text)) { - progressBar1.Value = 1; + lastErrorTextBox.Text = $"Please provide a value for '{p.Label}'."; + input.Focus(); + return false; } - progressBar1.Maximum = progressBar1.Value; - UpdateStatus("Cancelled"); - btnRun.Enabled = true; } + return true; + } + + private async void btnRun_Click(object sender, EventArgs e) + { + if (!ValidateRequiredParameters()) + return; + + RecalcTotal(); + lastErrorTextBox.Text = ""; + progressBar1.ForeColor = Color.LimeGreen; + + btnRun.Enabled = false; + progressBar1.Maximum = (int)_TotalBatchCount; + progressBar1.Value = 0; + ResetCounters(); + + statusBatches.Text = $"Batches sent: {_BatchSent} / {_TotalBatchCount}"; + status.Text = "Status: In Progress..."; + statusTime.Text = ""; + + _conn = CreateConnection(); + + LoadGenerator generator = new LoadGenerator(new PayloadGenerator(txtPayload.Text, _variables)) + { + BatchesNo = (int)SettingsBatchesPerSenderNumber.Value, + EventsPerBatch = (int)SettingsMsgPerBatchNumber.Value, + WaitTime = new TimeSpan(0, 0, (int)SettingsWaitTimeSec.Value), + }; + generator.BatchSent += LoadGenerator_BatchSent; + _CancellationTokenSource = new CancellationTokenSource(); + + int concurrency = (int)SettingsParallelismNumber.Value; - private void OnFailed(string errorMessage) + try { - if (progressBar1.Value == 0) + btnCancel.Enabled = true; + await generator.Send(_conn, concurrency, _CancellationTokenSource.Token); + btnCancel.Enabled = false; + if (_CancellationTokenSource.IsCancellationRequested) { - progressBar1.Value = 1; + OnCancelled(); + } + else + { + OnCompleted(); + await VerifyIngestionIfRequested(); } - progressBar1.Maximum = progressBar1.Value; - UpdateStatus("Failed"); - lastErrorTextBox.Text = errorMessage; - btnRun.Enabled = true; - btnCancel.Enabled = false; } - - private void EventSender_OnBatchSent(object? sender, EventArgs e) + catch (Exception ex) { - progressBar1.Increment(1); - _BatchSent++; - _MsgSent += (int)SettingsMsgPerBatchNumber.Value; - TimeSpan ts = DateTime.Now - _StartTime; - _TotalSizeInBytes += ((EventSender)sender).BatchSizeInBytes; - statusBatches.Text = $"Batches sent: {_BatchSent} / {_TotalBatchCount} (TPS: {(_MsgSent / ts.TotalSeconds):F2}) | Bytes sent: {_TotalSizeInBytes:0,0} ({(_TotalSizeInBytes / BytesToMbps / ts.TotalSeconds):F2} Mbps)"; - //https://learn.microsoft.com/en-us/dotnet/csharp/language-reference/tokens/interpolated + progressBar1.ForeColor = Color.Red; + OnFailed(ex.Message); } - private void RecalcTotal() + } + + // #3: opt-in post-run check for asynchronous (queued) ingestion failures. + private async Task VerifyIngestionIfRequested() + { + if (!chkVerifyIngestion.Checked || _conn is not IIngestionVerifier verifier) + return; + try { - _TotalBatchCount = SettingsThreadsNumber.Value * SettingsBatchesPerThreadNumber.Value; - _TotalMsgCount = _TotalBatchCount * SettingsMsgPerBatchNumber.Value; - SettingsTotalMsgCount.Value = _TotalMsgCount; + IReadOnlyList failures = await verifier.GetIngestionFailuresSinceAsync(_runStartUtc); + lastErrorTextBox.Text = failures.Count == 0 + ? "Ingestion verification: no failures reported (queued failures may lag)." + : $"Ingestion verification: {failures.Count} failure(s):\r\n" + string.Join("\r\n", failures); } - private void RefreshAppTitle() + catch (Exception ex) { - this.Text = (_IsPayloadChanged ? "*" : "") + _FileName + (_FileName.Length > 0 ? " - " : "") + this.Tag; + lastErrorTextBox.Text = $"Ingestion verification could not run: {ex.Message}"; } + } - private void SettingsThreadsNumber_ValueChanged(object sender, EventArgs e) - { - RecalcTotal(); - } + private void UpdateStatus(string Status) + { + _EndTime = DateTime.Now; + TimeSpan ts = _EndTime - _StartTime; + status.Text = $"Status: {Status}."; + statusBatches.Text = $"Sent {_MsgSent} messages in {_TotalBatchCount} batches. Total time: {ts:c} (TPS: {(_MsgSent / ts.TotalSeconds):F2}) | Bytes sent: {_TotalSizeInBytes:0,0} ({(_TotalSizeInBytes / BytesToMbps / ts.TotalSeconds):F2} Mbps)"; + } - private void SettingsBatchesPerThreadNumber_ValueChanged(object sender, EventArgs e) - { - RecalcTotal(); - } + private void OnCompleted() + { + progressBar1.Maximum = progressBar1.Value; + UpdateStatus("Completed"); + btnRun.Enabled = true; + } - private void SettingsMsgPerBatchNumber_ValueChanged(object sender, EventArgs e) + private void OnCancelled() + { + if (progressBar1.Value == 0) { - RecalcTotal(); + progressBar1.Value = 1; } + progressBar1.Maximum = progressBar1.Value; + UpdateStatus("Cancelled"); + btnRun.Enabled = true; + } - private void exitToolStripMenuItem_Click(object sender, EventArgs e) + private void OnFailed(string errorMessage) + { + if (progressBar1.Value == 0) { - this.Close(); + progressBar1.Value = 1; } + progressBar1.Maximum = progressBar1.Value; + UpdateStatus("Failed"); + lastErrorTextBox.Text = errorMessage; + btnRun.Enabled = true; + btnCancel.Enabled = false; + } - private void saveAsToolStripMenuItem_Click(object sender, EventArgs e) - { - SaveFileDialog saveFileDialog = new SaveFileDialog(); - saveFileDialog.Filter = FILES_FILTER; - saveFileDialog.FileName = _FileName; - DialogResult dr = saveFileDialog.ShowDialog(); - if (dr == DialogResult.OK && saveFileDialog.FileName.Length > 0) - { - SaveFile(saveFileDialog.FileName); - } - } + // Invoked once per sent batch. With multiple concurrent senders the LoadGenerator + // raises BatchSent on continuations that resume on this (UI) thread, so these + // control/counter updates stay single-threaded and safe. + private void LoadGenerator_BatchSent(object? sender, BatchSentEventArgs e) + { + progressBar1.Increment(1); + _BatchSent++; + _MsgSent += e.MessageCount; + TimeSpan ts = DateTime.Now - _StartTime; + _TotalSizeInBytes += e.SizeInBytes; + statusBatches.Text = $"Batches sent: {_BatchSent} / {_TotalBatchCount} (TPS: {(_MsgSent / ts.TotalSeconds):F2}) | Bytes sent: {_TotalSizeInBytes:0,0} ({(_TotalSizeInBytes / BytesToMbps / ts.TotalSeconds):F2} Mbps)"; + //https://learn.microsoft.com/en-us/dotnet/csharp/language-reference/tokens/interpolated + } - private void saveMessageTemplateToolStripMenuItem_Click(object sender, EventArgs e) - { - SaveFile(_FileName); - } + private void RecalcTotal() + { + _TotalBatchCount = SettingsParallelismNumber.Value * SettingsBatchesPerSenderNumber.Value; + _TotalMsgCount = _TotalBatchCount * SettingsMsgPerBatchNumber.Value; + SettingsTotalMsgCount.Value = _TotalMsgCount; + } + private void RefreshAppTitle() + { + this.Text = (_IsPayloadChanged ? "*" : "") + (_hasNamedFile ? _FileName + " - " : "") + this.Tag; + } - private void SaveFile(string FileName) - { - System.IO.File.WriteAllText(FileName, txtPayload.Text); - _FileName = new System.IO.FileInfo(FileName).Name; - IsPayloadChanged = false; - } + private void SettingsParallelismNumber_ValueChanged(object sender, EventArgs e) + { + RecalcTotal(); + } + + private void SettingsBatchesPerSenderNumber_ValueChanged(object sender, EventArgs e) + { + RecalcTotal(); + } + + private void SettingsMsgPerBatchNumber_ValueChanged(object sender, EventArgs e) + { + RecalcTotal(); + } - private void loadMessageTemplateToolStripMenuItem_Click(object sender, EventArgs e) + private void exitToolStripMenuItem_Click(object sender, EventArgs e) + { + this.Close(); + } + + private void saveAsToolStripMenuItem_Click(object sender, EventArgs e) + { + SaveFileDialog saveFileDialog = new SaveFileDialog(); + saveFileDialog.Filter = FILES_FILTER; + saveFileDialog.FileName = _FileName; + DialogResult dr = saveFileDialog.ShowDialog(); + if (dr == DialogResult.OK && saveFileDialog.FileName.Length > 0) { - OpenFileDialog openFileDialog = new OpenFileDialog(); - openFileDialog.Filter = FILES_FILTER; - openFileDialog.ShowDialog(); - if (openFileDialog.FileName.Length > 0) - { - txtPayload.Text = System.IO.File.ReadAllText(openFileDialog.FileName); - _FileName = openFileDialog.SafeFileName; - } - IsPayloadChanged = false; + SaveFile(saveFileDialog.FileName); } + } - private void txtPayload_TextChanged(object sender, EventArgs e) + private void saveMessageTemplateToolStripMenuItem_Click(object sender, EventArgs e) + { + SaveFile(_FileName); + } + + private void SaveFile(string FileName) + { + System.IO.File.WriteAllText(FileName, txtPayload.Text); + _FileName = new System.IO.FileInfo(FileName).Name; + _hasNamedFile = true; + IsPayloadChanged = false; + } + + private void loadMessageTemplateToolStripMenuItem_Click(object sender, EventArgs e) + { + OpenFileDialog openFileDialog = new OpenFileDialog(); + openFileDialog.Filter = FILES_FILTER; + openFileDialog.ShowDialog(); + if (openFileDialog.FileName.Length > 0) { - IsPayloadChanged = true; + txtPayload.Text = System.IO.File.ReadAllText(openFileDialog.FileName); + _FileName = openFileDialog.SafeFileName; + _hasNamedFile = true; } + IsPayloadChanged = false; + } - private void Form1_Load(object sender, EventArgs e) + private void txtPayload_TextChanged(object sender, EventArgs e) + { + IsPayloadChanged = true; + } + + private void Form1_Load(object sender, EventArgs e) + { + cboTargetType.Items.Clear(); + foreach (ITargetConnector connector in _connectors) { - RefreshAppTitle(); + cboTargetType.Items.Add(connector.DisplayName); } + cboTargetType.SelectedIndex = 0; // triggers RebuildParameterInputs + + LoadVariableDefinitions(); + TryRestoreCachedSignIn(); + UpdateAuthUi(); + // Loading the default payload during InitializeComponent flags the payload as + // changed; reset it so the caption starts clean (no leading '*'). + IsPayloadChanged = false; // setter refreshes the title + } - private void btnPreview_Click(object sender, EventArgs e) + // Resolution order: a variables.toml next to the executable, then the built-in + // embedded defaults. A file chosen via the menu overrides this at runtime. + private void LoadVariableDefinitions() + { + string bundled = System.IO.Path.Combine(AppContext.BaseDirectory, VariableDefinitions.DefaultFileName); + try { - EventSender s = new EventSender(txtPayload.Text); - frmPreview f = new frmPreview(); - f.gen = s; - f.ShowDialog(); + _variables = System.IO.File.Exists(bundled) + ? VariableDefinitions.Load(bundled) + : VariableDefinitions.LoadDefaults(); } - - private async void btnCancel_Click(object sender, EventArgs e) + catch (Exception ex) { - progressBar1.ForeColor = Color.Yellow; - btnCancel.Enabled = false; - _CancellationTokenSource.Cancel(); + _variables = VariableDefinitions.LoadDefaults(); + lastErrorTextBox.Text = $"Could not load '{bundled}': {ex.Message}. Using built-in defaults."; } + } - private void frmGenerator_KeyUp(object sender, KeyEventArgs e) + private void loadVariableDefinitionsToolStripMenuItem_Click(object sender, EventArgs e) + { + OpenFileDialog openFileDialog = new OpenFileDialog(); + openFileDialog.Filter = "TOML files (*.toml)|*.toml|All files (*.*)|*.*"; + if (openFileDialog.ShowDialog() == DialogResult.OK && openFileDialog.FileName.Length > 0) { - if (e.KeyData == Keys.F5) + try { - btnRun_Click(sender, e); + _variables = VariableDefinitions.Load(openFileDialog.FileName); + lastErrorTextBox.Text = $"Loaded {_variables.Count} variable definition(s) from {openFileDialog.SafeFileName}."; } - if (e.KeyData == Keys.F3) + catch (Exception ex) { - btnPreview_Click(sender, e); + lastErrorTextBox.Text = $"Failed to load variable definitions: {ex.Message}"; } } + } + + private void btnPreview_Click(object sender, EventArgs e) + { + PayloadGenerator s = new PayloadGenerator(txtPayload.Text, _variables); + frmPreview f = new frmPreview(); + f.gen = s; + f.ShowDialog(); + } - private void mnuAbout_Click(object sender, EventArgs e) + private async void btnCancel_Click(object sender, EventArgs e) + { + progressBar1.ForeColor = Color.Yellow; + btnCancel.Enabled = false; + _CancellationTokenSource.Cancel(); + } + + private void frmGenerator_KeyUp(object sender, KeyEventArgs e) + { + if (e.KeyData == Keys.F5) { - frmAboutApp f = new frmAboutApp(); - f.ShowDialog(); + btnRun_Click(sender, e); } - - private async void btnTestCnn_Click(object sender, EventArgs e) + if (e.KeyData == Keys.F3) { - try - { - _conn = new TargetConnection(txtCnnStr.Text, txtEventHubName.Text, _credential); - var r = _conn.Connect(); - lastErrorTextBox.Text = "Connection successful."; - } - catch (Exception ex) - { - lastErrorTextBox.Text = ex.ToString(); - } + btnPreview_Click(sender, e); } + } + + private void mnuAbout_Click(object sender, EventArgs e) + { + frmAboutApp f = new frmAboutApp(); + f.ShowDialog(); + } + + private async void btnTestCnn_Click(object sender, EventArgs e) + { + if (!ValidateRequiredParameters()) + return; - private void btnAzureAuth_Click(object sender, EventArgs e) + try { - AzureInteractiveAuth().Wait(); + _conn = CreateConnection(); + await _conn.ConnectAsync(); + lastErrorTextBox.Text = "Connection successful."; } + catch (Exception ex) + { + lastErrorTextBox.Text = ex.ToString(); + } + } - private async Task AzureInteractiveAuth() + // The button toggles: sign in when signed out, sign out when signed in. + private async void btnAzureAuth_Click(object sender, EventArgs e) + { + if (_credential != null) + SignOut(); + else + await AzureInteractiveAuth(); + } + + private async Task AzureInteractiveAuth() + { + try { - if (_credential == null) - { - _credential = new InteractiveBrowserCredential(); - } + var credential = CreatePersistentCredential(null); - // Define the resource scope you want to access - var scope = new[] { "https://graph.microsoft.com/.default" }; + // Interactive sign-in, then persist the authentication record so future + // runs can silently reuse the token cache (no browser prompt). + AuthenticationRecord record = await credential.AuthenticateAsync(); - try + System.IO.Directory.CreateDirectory(AuthCacheDir); + using (var stream = System.IO.File.Create(AuthRecordPath)) { - // Use the credential to get an access token - AccessToken token = await _credential.GetTokenAsync(new TokenRequestContext(scope)); - lastErrorTextBox.Text = $"Access Token: {token.Token.Substring(0,8)}***"; - } - catch (Exception ex) - { - // Handle any exceptions - lastErrorTextBox.Text = $"Error: {ex.Message}"; + await record.SerializeAsync(stream); } + + _credential = credential; + _signedInUser = record.Username; + UpdateAuthUi(); + lastErrorTextBox.Text = $"Signed in as {record.Username}. Sign-in cached for future runs."; + } + catch (Exception ex) + { + lastErrorTextBox.Text = $"Error: {ex.Message}"; } + } + private void azureSignOutToolStripMenuItem_Click(object sender, EventArgs e) + { + SignOut(); + } + private void SignOut() + { + _credential = null; + _signedInUser = null; + UpdateAuthUi(); + try + { + if (System.IO.File.Exists(AuthRecordPath)) + System.IO.File.Delete(AuthRecordPath); + lastErrorTextBox.Text = "Signed out. Cached sign-in cleared; connection-string auth will be used until you sign in again."; + } + catch (Exception ex) + { + lastErrorTextBox.Text = $"Signed out, but could not delete the cached record: {ex.Message}"; + } } + + } diff --git a/src/RTDSimulatorDesktopApp/frmPreview.cs b/src/RTDSimulatorDesktopApp/frmPreview.cs index d2b3a23..d79f50c 100644 --- a/src/RTDSimulatorDesktopApp/frmPreview.cs +++ b/src/RTDSimulatorDesktopApp/frmPreview.cs @@ -7,12 +7,13 @@ using System.Text; using System.Threading.Tasks; using System.Windows.Forms; +using RTDSimulator.Core; namespace RTDSimulatorDesktopApp { public partial class frmPreview : Form { - public EventSender gen; + public PayloadGenerator gen; public frmPreview() { diff --git a/src/RealTimeDataSimulator.sln b/src/RealTimeDataSimulator.sln index 4227d4b..6f671b5 100644 --- a/src/RealTimeDataSimulator.sln +++ b/src/RealTimeDataSimulator.sln @@ -1,20 +1,136 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.5.002.0 +# Visual Studio Version 18 +VisualStudioVersion = 18.6.11828.311 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RTDSimulatorDesktopApp", "RTDSimulatorDesktopApp\RTDSimulatorDesktopApp.csproj", "{B9935827-F71D-4A48-B5C3-68674718E37E}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RTDSimulator.Core", "RTDSimulator.Core\RTDSimulator.Core.csproj", "{9ABC2C32-917C-4780-8380-EBD6F500E3C1}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RTDSimulator.ServiceBus", "RTDSimulator.ServiceBus\RTDSimulator.ServiceBus.csproj", "{B338686E-D301-4888-8E76-E3DDA32390D0}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RTDSimulator.Cli", "RTDSimulator.Cli\RTDSimulator.Cli.csproj", "{FF4DC82C-DA9F-4B51-B0E8-337DFFAAB1D0}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RTDSimulator.EventHubs", "RTDSimulator.EventHubs\RTDSimulator.EventHubs.csproj", "{058471EB-C7B6-4F08-980F-0E8AC7804238}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RTDSimulator.Kusto", "RTDSimulator.Kusto\RTDSimulator.Kusto.csproj", "{C9BF5970-3E5D-44E5-8D04-54121D26EF66}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RTDSimulator.Core.Tests", "RTDSimulator.Core.Tests\RTDSimulator.Core.Tests.csproj", "{FD4D38AB-56E5-4298-A3E8-2CAA3483499A}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{57B03023-A43D-4478-9356-8A9DC28C9EAF}" + ProjectSection(SolutionItems) = preProject + RTDSimulator.Core\default-variables.toml = RTDSimulator.Core\default-variables.toml + ..\README.md = ..\README.md + EndProjectSection +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RTDSimulator.Connectors.Tests", "RTDSimulator.Connectors.Tests\RTDSimulator.Connectors.Tests.csproj", "{2BFD8778-F7C8-40DA-8F68-0893ECD16049}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {B9935827-F71D-4A48-B5C3-68674718E37E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {B9935827-F71D-4A48-B5C3-68674718E37E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B9935827-F71D-4A48-B5C3-68674718E37E}.Debug|x64.ActiveCfg = Debug|Any CPU + {B9935827-F71D-4A48-B5C3-68674718E37E}.Debug|x64.Build.0 = Debug|Any CPU + {B9935827-F71D-4A48-B5C3-68674718E37E}.Debug|x86.ActiveCfg = Debug|Any CPU + {B9935827-F71D-4A48-B5C3-68674718E37E}.Debug|x86.Build.0 = Debug|Any CPU {B9935827-F71D-4A48-B5C3-68674718E37E}.Release|Any CPU.ActiveCfg = Release|Any CPU {B9935827-F71D-4A48-B5C3-68674718E37E}.Release|Any CPU.Build.0 = Release|Any CPU + {B9935827-F71D-4A48-B5C3-68674718E37E}.Release|x64.ActiveCfg = Release|Any CPU + {B9935827-F71D-4A48-B5C3-68674718E37E}.Release|x64.Build.0 = Release|Any CPU + {B9935827-F71D-4A48-B5C3-68674718E37E}.Release|x86.ActiveCfg = Release|Any CPU + {B9935827-F71D-4A48-B5C3-68674718E37E}.Release|x86.Build.0 = Release|Any CPU + {9ABC2C32-917C-4780-8380-EBD6F500E3C1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9ABC2C32-917C-4780-8380-EBD6F500E3C1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9ABC2C32-917C-4780-8380-EBD6F500E3C1}.Debug|x64.ActiveCfg = Debug|Any CPU + {9ABC2C32-917C-4780-8380-EBD6F500E3C1}.Debug|x64.Build.0 = Debug|Any CPU + {9ABC2C32-917C-4780-8380-EBD6F500E3C1}.Debug|x86.ActiveCfg = Debug|Any CPU + {9ABC2C32-917C-4780-8380-EBD6F500E3C1}.Debug|x86.Build.0 = Debug|Any CPU + {9ABC2C32-917C-4780-8380-EBD6F500E3C1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9ABC2C32-917C-4780-8380-EBD6F500E3C1}.Release|Any CPU.Build.0 = Release|Any CPU + {9ABC2C32-917C-4780-8380-EBD6F500E3C1}.Release|x64.ActiveCfg = Release|Any CPU + {9ABC2C32-917C-4780-8380-EBD6F500E3C1}.Release|x64.Build.0 = Release|Any CPU + {9ABC2C32-917C-4780-8380-EBD6F500E3C1}.Release|x86.ActiveCfg = Release|Any CPU + {9ABC2C32-917C-4780-8380-EBD6F500E3C1}.Release|x86.Build.0 = Release|Any CPU + {B338686E-D301-4888-8E76-E3DDA32390D0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B338686E-D301-4888-8E76-E3DDA32390D0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B338686E-D301-4888-8E76-E3DDA32390D0}.Debug|x64.ActiveCfg = Debug|Any CPU + {B338686E-D301-4888-8E76-E3DDA32390D0}.Debug|x64.Build.0 = Debug|Any CPU + {B338686E-D301-4888-8E76-E3DDA32390D0}.Debug|x86.ActiveCfg = Debug|Any CPU + {B338686E-D301-4888-8E76-E3DDA32390D0}.Debug|x86.Build.0 = Debug|Any CPU + {B338686E-D301-4888-8E76-E3DDA32390D0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B338686E-D301-4888-8E76-E3DDA32390D0}.Release|Any CPU.Build.0 = Release|Any CPU + {B338686E-D301-4888-8E76-E3DDA32390D0}.Release|x64.ActiveCfg = Release|Any CPU + {B338686E-D301-4888-8E76-E3DDA32390D0}.Release|x64.Build.0 = Release|Any CPU + {B338686E-D301-4888-8E76-E3DDA32390D0}.Release|x86.ActiveCfg = Release|Any CPU + {B338686E-D301-4888-8E76-E3DDA32390D0}.Release|x86.Build.0 = Release|Any CPU + {FF4DC82C-DA9F-4B51-B0E8-337DFFAAB1D0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FF4DC82C-DA9F-4B51-B0E8-337DFFAAB1D0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FF4DC82C-DA9F-4B51-B0E8-337DFFAAB1D0}.Debug|x64.ActiveCfg = Debug|Any CPU + {FF4DC82C-DA9F-4B51-B0E8-337DFFAAB1D0}.Debug|x64.Build.0 = Debug|Any CPU + {FF4DC82C-DA9F-4B51-B0E8-337DFFAAB1D0}.Debug|x86.ActiveCfg = Debug|Any CPU + {FF4DC82C-DA9F-4B51-B0E8-337DFFAAB1D0}.Debug|x86.Build.0 = Debug|Any CPU + {FF4DC82C-DA9F-4B51-B0E8-337DFFAAB1D0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FF4DC82C-DA9F-4B51-B0E8-337DFFAAB1D0}.Release|Any CPU.Build.0 = Release|Any CPU + {FF4DC82C-DA9F-4B51-B0E8-337DFFAAB1D0}.Release|x64.ActiveCfg = Release|Any CPU + {FF4DC82C-DA9F-4B51-B0E8-337DFFAAB1D0}.Release|x64.Build.0 = Release|Any CPU + {FF4DC82C-DA9F-4B51-B0E8-337DFFAAB1D0}.Release|x86.ActiveCfg = Release|Any CPU + {FF4DC82C-DA9F-4B51-B0E8-337DFFAAB1D0}.Release|x86.Build.0 = Release|Any CPU + {058471EB-C7B6-4F08-980F-0E8AC7804238}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {058471EB-C7B6-4F08-980F-0E8AC7804238}.Debug|Any CPU.Build.0 = Debug|Any CPU + {058471EB-C7B6-4F08-980F-0E8AC7804238}.Debug|x64.ActiveCfg = Debug|Any CPU + {058471EB-C7B6-4F08-980F-0E8AC7804238}.Debug|x64.Build.0 = Debug|Any CPU + {058471EB-C7B6-4F08-980F-0E8AC7804238}.Debug|x86.ActiveCfg = Debug|Any CPU + {058471EB-C7B6-4F08-980F-0E8AC7804238}.Debug|x86.Build.0 = Debug|Any CPU + {058471EB-C7B6-4F08-980F-0E8AC7804238}.Release|Any CPU.ActiveCfg = Release|Any CPU + {058471EB-C7B6-4F08-980F-0E8AC7804238}.Release|Any CPU.Build.0 = Release|Any CPU + {058471EB-C7B6-4F08-980F-0E8AC7804238}.Release|x64.ActiveCfg = Release|Any CPU + {058471EB-C7B6-4F08-980F-0E8AC7804238}.Release|x64.Build.0 = Release|Any CPU + {058471EB-C7B6-4F08-980F-0E8AC7804238}.Release|x86.ActiveCfg = Release|Any CPU + {058471EB-C7B6-4F08-980F-0E8AC7804238}.Release|x86.Build.0 = Release|Any CPU + {C9BF5970-3E5D-44E5-8D04-54121D26EF66}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C9BF5970-3E5D-44E5-8D04-54121D26EF66}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C9BF5970-3E5D-44E5-8D04-54121D26EF66}.Debug|x64.ActiveCfg = Debug|Any CPU + {C9BF5970-3E5D-44E5-8D04-54121D26EF66}.Debug|x64.Build.0 = Debug|Any CPU + {C9BF5970-3E5D-44E5-8D04-54121D26EF66}.Debug|x86.ActiveCfg = Debug|Any CPU + {C9BF5970-3E5D-44E5-8D04-54121D26EF66}.Debug|x86.Build.0 = Debug|Any CPU + {C9BF5970-3E5D-44E5-8D04-54121D26EF66}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C9BF5970-3E5D-44E5-8D04-54121D26EF66}.Release|Any CPU.Build.0 = Release|Any CPU + {C9BF5970-3E5D-44E5-8D04-54121D26EF66}.Release|x64.ActiveCfg = Release|Any CPU + {C9BF5970-3E5D-44E5-8D04-54121D26EF66}.Release|x64.Build.0 = Release|Any CPU + {C9BF5970-3E5D-44E5-8D04-54121D26EF66}.Release|x86.ActiveCfg = Release|Any CPU + {C9BF5970-3E5D-44E5-8D04-54121D26EF66}.Release|x86.Build.0 = Release|Any CPU + {FD4D38AB-56E5-4298-A3E8-2CAA3483499A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FD4D38AB-56E5-4298-A3E8-2CAA3483499A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FD4D38AB-56E5-4298-A3E8-2CAA3483499A}.Debug|x64.ActiveCfg = Debug|Any CPU + {FD4D38AB-56E5-4298-A3E8-2CAA3483499A}.Debug|x64.Build.0 = Debug|Any CPU + {FD4D38AB-56E5-4298-A3E8-2CAA3483499A}.Debug|x86.ActiveCfg = Debug|Any CPU + {FD4D38AB-56E5-4298-A3E8-2CAA3483499A}.Debug|x86.Build.0 = Debug|Any CPU + {FD4D38AB-56E5-4298-A3E8-2CAA3483499A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FD4D38AB-56E5-4298-A3E8-2CAA3483499A}.Release|Any CPU.Build.0 = Release|Any CPU + {FD4D38AB-56E5-4298-A3E8-2CAA3483499A}.Release|x64.ActiveCfg = Release|Any CPU + {FD4D38AB-56E5-4298-A3E8-2CAA3483499A}.Release|x64.Build.0 = Release|Any CPU + {FD4D38AB-56E5-4298-A3E8-2CAA3483499A}.Release|x86.ActiveCfg = Release|Any CPU + {FD4D38AB-56E5-4298-A3E8-2CAA3483499A}.Release|x86.Build.0 = Release|Any CPU + {2BFD8778-F7C8-40DA-8F68-0893ECD16049}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2BFD8778-F7C8-40DA-8F68-0893ECD16049}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2BFD8778-F7C8-40DA-8F68-0893ECD16049}.Debug|x64.ActiveCfg = Debug|Any CPU + {2BFD8778-F7C8-40DA-8F68-0893ECD16049}.Debug|x64.Build.0 = Debug|Any CPU + {2BFD8778-F7C8-40DA-8F68-0893ECD16049}.Debug|x86.ActiveCfg = Debug|Any CPU + {2BFD8778-F7C8-40DA-8F68-0893ECD16049}.Debug|x86.Build.0 = Debug|Any CPU + {2BFD8778-F7C8-40DA-8F68-0893ECD16049}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2BFD8778-F7C8-40DA-8F68-0893ECD16049}.Release|Any CPU.Build.0 = Release|Any CPU + {2BFD8778-F7C8-40DA-8F68-0893ECD16049}.Release|x64.ActiveCfg = Release|Any CPU + {2BFD8778-F7C8-40DA-8F68-0893ECD16049}.Release|x64.Build.0 = Release|Any CPU + {2BFD8778-F7C8-40DA-8F68-0893ECD16049}.Release|x86.ActiveCfg = Release|Any CPU + {2BFD8778-F7C8-40DA-8F68-0893ECD16049}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE