.Net: fix Tavily pagination and image handling; add search options

dotnet/src/Plugins/Plugins.Web/Tavily/TavilyImageResult.cs

@@ -9,6 +9,7 @@ namespace Microsoft.SemanticKernel.Plugins.Web.Tavily;
 /// Represents an image result.
 /// </summary>
 #pragma warning disable CA1812 // Instantiated by reflection
+[JsonConverter(typeof(TavilyImageResultConverter))]
 internal sealed class TavilyImageResult
 {
     /// <summary>

dotnet/src/Plugins/Plugins.Web/Tavily/TavilyImageResultConverter.cs

@@ -0,0 +1,66 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Microsoft.SemanticKernel.Plugins.Web.Tavily;
+
+/// <summary>
+/// Reads an image result from either a plain image URL string (returned when image descriptions are not
+/// requested) or an object with <c>url</c> and <c>description</c> (returned when they are).
+/// </summary>
+internal sealed class TavilyImageResultConverter : JsonConverter<TavilyImageResult>
+{
+    /// <inheritdoc />
+    public override TavilyImageResult? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+    {
+        if (reader.TokenType == JsonTokenType.Null)
+        {
+            return null;
+        }
+
+        if (reader.TokenType == JsonTokenType.String)
+        {
+            return new TavilyImageResult(reader.GetString()!, string.Empty);
+        }
+
+        if (reader.TokenType != JsonTokenType.StartObject)
+        {
+            throw new JsonException("An image result must be a URL string or an object.");
+        }
+
+        string? url = null;
+        string? description = null;
+        while (reader.Read() && reader.TokenType != JsonTokenType.EndObject)
+        {
+            string? name = reader.GetString();
+            reader.Read();
+            if (name == "url")
+            {
+                url = reader.TokenType == JsonTokenType.Null ? null : reader.GetString();
+            }
+            else if (name == "description")
+            {
+                description = reader.TokenType == JsonTokenType.Null ? null : reader.GetString();
+            }
+            else
+            {
+                reader.Skip();
+            }
+        }
+
+        return new TavilyImageResult(
+            url ?? throw new JsonException("Required property 'url' was not found in the image result."),
+            description ?? throw new JsonException("Required property 'description' was not found in the image result."));
+    }
+
+    /// <inheritdoc />
+    public override void Write(Utf8JsonWriter writer, TavilyImageResult value, JsonSerializerOptions options)
+    {
+        writer.WriteStartObject();
+        writer.WriteString("url", value.Url);
+        writer.WriteString("description", value.Description);
+        writer.WriteEndObject();
+    }
+}

dotnet/src/Plugins/Plugins.Web/Tavily/TavilyIncludeDomainsMode.cs

@@ -0,0 +1,19 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace Microsoft.SemanticKernel.Plugins.Web.Tavily;
+
+/// <summary>
+/// Controls how the include domain filter is applied to a Tavily search.
+/// </summary>
+public enum TavilyIncludeDomainsMode
+{
+    /// <summary>
+    /// Limit results to only the included domains (Tavily default).
+    /// </summary>
+    Restrict,
+
+    /// <summary>
+    /// Also search the rest of the web, so results outside the included domains can still surface.
+    /// </summary>
+    Prefer
+}

dotnet/src/Plugins/Plugins.Web/Tavily/TavilySearchDepth.cs

@@ -16,5 +16,15 @@ public enum TavilySearchDepth
     /// <summary>
     /// Advanced search costs 2 API Credits.
     /// </summary>
-    Advanced
+    Advanced,
+
+    /// <summary>
+    /// Fast search prioritizes lower latency while maintaining good relevance. Costs 1 API Credit.
+    /// </summary>
+    Fast,
+
+    /// <summary>
+    /// Ultra-fast search minimizes latency above all else. Costs 1 API Credit.
+    /// </summary>
+    UltraFast
 }

dotnet/src/Plugins/Plugins.Web/Tavily/TavilySearchRequest.cs

@@ -88,6 +88,60 @@ internal sealed class TavilySearchRequest
     [JsonPropertyName("include_image_descriptions")]
     public bool? IncludeImageDescriptions { get; set; }
 
+    /// <summary>
+    /// Boost search results from a specific country. Available only when topic is general.
+    /// </summary>
+    [JsonPropertyName("country")]
+    public string? Country { get; set; }
+
+    /// <summary>
+    /// Return only results after this date. Format: YYYY-MM-DD.
+    /// </summary>
+    [JsonPropertyName("start_date")]
+    public string? StartDate { get; set; }
+
+    /// <summary>
+    /// Return only results before this date. Format: YYYY-MM-DD.
+    /// </summary>
+    [JsonPropertyName("end_date")]
+    public string? EndDate { get; set; }
+
+    /// <summary>
+    /// Whether to filter out adult or unsafe content from results.
+    /// </summary>
+    [JsonPropertyName("safe_search")]
+    public bool? SafeSearch { get; set; }
+
+    /// <summary>
+    /// How include_domains is applied. Available options: restrict, prefer.
+    /// </summary>
+    [JsonPropertyName("include_domains_mode")]
+    public string? IncludeDomainsMode { get; set; }
+
+    /// <summary>
+    /// Boost search results in a specific language (ISO 639-1 code or English language name).
+    /// </summary>
+    [JsonPropertyName("language")]
+    public string? Language { get; set; }
+
+    /// <summary>
+    /// Strictly filter out results that do not match the language instead of only boosting them.
+    /// </summary>
+    [JsonPropertyName("filter_by_language")]
+    public bool? FilterByLanguage { get; set; }
+
+    /// <summary>
+    /// Only return results containing the exact quoted phrase(s) in the query.
+    /// </summary>
+    [JsonPropertyName("exact_match")]
+    public bool? ExactMatch { get; set; }
+
+    /// <summary>
+    /// Let Tavily configure search parameters automatically from the query.
+    /// </summary>
+    [JsonPropertyName("auto_parameters")]
+    public bool? AutoParameters { get; set; }
+
     /// <summary>
     /// A list of domains to specifically include in the search results.
     /// </summary>
@@ -122,7 +176,16 @@ public TavilySearchRequest(
         bool? includeRawContent,
         int? maxResults,
         IList<string>? includeDomains,
-        IList<string>? excludeDomains)
+        IList<string>? excludeDomains,
+        string? country = null,
+        string? startDate = null,
+        string? endDate = null,
+        bool? safeSearch = null,
+        string? includeDomainsMode = null,
+        string? language = null,
+        bool? filterByLanguage = null,
+        bool? exactMatch = null,
+        bool? autoParameters = null)
     {
         this.Query = query ?? throw new ArgumentNullException(nameof(query));
         this.Topic = topic;
@@ -137,5 +200,14 @@ public TavilySearchRequest(
         this.MaxResults = maxResults;
         this.IncludeDomains = includeDomains;
         this.ExcludeDomains = excludeDomains;
+        this.Country = country;
+        this.StartDate = startDate;
+        this.EndDate = endDate;
+        this.SafeSearch = safeSearch;
+        this.IncludeDomainsMode = includeDomainsMode;
+        this.Language = language;
+        this.FilterByLanguage = filterByLanguage;
+        this.ExactMatch = exactMatch;
+        this.AutoParameters = autoParameters;
     }
 }

dotnet/src/Plugins/Plugins.Web/Tavily/TavilyTextSearch.cs

@@ -2,6 +2,8 @@
 
 using System;
 using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
 using System.Linq.Expressions;
 using System.Net.Http;
 using System.Runtime.CompilerServices;
@@ -383,6 +385,8 @@ private static void ProcessMethodCallClause(MethodCallExpression methodCall, Lis
     #region Private Methods
 
     private readonly ILogger _logger;
+    private const string ClientNameHeader = "X-Client-Name";
+    private const string ClientName = "semantic-kernel";
     private readonly HttpClient _httpClient;
     private readonly string? _apiKey;
     private readonly Uri? _uri = null;
@@ -395,6 +399,12 @@ private static void ProcessMethodCallClause(MethodCallExpression methodCall, Lis
 
     private const string DefaultUri = "https://api.tavily.com/search";
 
+    /// <summary>
+    /// The maximum value of the Tavily <c>max_results</c> request parameter. The accepted
+    /// <c>Top</c> range is unchanged; only the outgoing request window is capped.
+    /// </summary>
+    private const int MaxResultsLimit = 20;
+
     private const string Topic = "topic";
     private const string TimeRange = "time_range";
     private const string Days = "days";
@@ -422,7 +432,13 @@ private static void ProcessMethodCallClause(MethodCallExpression methodCall, Lis
         // Sensitive data, logging as trace, disabled by default
         this._logger.LogTrace("Response content received: {Data}", json);
 
-        return JsonSerializer.Deserialize<TavilySearchResponse>(json);
+        TavilySearchResponse? searchResponse = JsonSerializer.Deserialize<TavilySearchResponse>(json);
+        if (skip > 0 && searchResponse?.Results is not null)
+        {
+            searchResponse.Results = [.. searchResponse.Results.Skip(skip).Take(top)];
+        }
+
+        return searchResponse;
     }
 
     /// <summary>
@@ -453,6 +469,12 @@ private async Task<HttpResponseMessage> SendGetRequestAsync(string query, int to
             httpRequestMessage.Headers.Add("Authorization", $"Bearer {this._apiKey}");
         }
 
+        // Identify this connector to Tavily unless the caller already set the header on their HttpClient.
+        if (!this._httpClient.DefaultRequestHeaders.Contains(ClientNameHeader))
+        {
+            httpRequestMessage.Headers.Add(ClientNameHeader, ClientName);
+        }
+
         return await this._httpClient.SendWithSuccessCheckAsync(httpRequestMessage, cancellationToken).ConfigureAwait(false);
     }
 
@@ -474,7 +496,7 @@ private async IAsyncEnumerable<object> GetSearchResultsAsync(TavilySearchRespons
             await Task.Yield();
         }
 
-        if (this._searchOptions?.IncludeImages ?? false && searchResponse.Images is not null)
+        if ((this._searchOptions?.IncludeImages ?? false) && searchResponse.Images is not null)
         {
             foreach (var image in searchResponse.Images!)
             {
@@ -502,7 +524,7 @@ private async IAsyncEnumerable<TavilyWebPage> GetResultsAsWebPageAsync(TavilySea
             await Task.Yield();
         }
 
-        if (this._searchOptions?.IncludeImages ?? false && searchResponse.Images is not null)
+        if ((this._searchOptions?.IncludeImages ?? false) && searchResponse.Images is not null)
         {
             foreach (var image in searchResponse.Images!)
             {
@@ -537,7 +559,7 @@ private async IAsyncEnumerable<TextSearchResult> GetResultsAsTextSearchResultAsy
             await Task.Yield();
         }
 
-        if (this._searchOptions?.IncludeImages ?? false && searchResponse.Images is not null)
+        if ((this._searchOptions?.IncludeImages ?? false) && searchResponse.Images is not null)
         {
             foreach (var image in searchResponse.Images!)
             {
@@ -571,7 +593,7 @@ private async IAsyncEnumerable<string> GetResultsAsStringAsync(TavilySearchRespo
             await Task.Yield();
         }
 
-        if (this._searchOptions?.IncludeImages ?? false && searchResponse.Images is not null)
+        if ((this._searchOptions?.IncludeImages ?? false) && searchResponse.Images is not null)
         {
             foreach (var image in searchResponse.Images!)
             {
@@ -676,7 +698,9 @@ private TavilySearchRequest BuildRequestContent(string query, int top, int skip,
         string? topic = null;
         string? timeRange = null;
         int? days = null;
-        int? maxResults = top - skip;
+        // Tavily has no offset parameter, so skipped results are requested and then dropped client-side.
+        // The window is limited by max_results, so top + skip beyond MaxResultsLimit returns fewer than top results.
+        int? maxResults = (int)Math.Min((long)top + Math.Max(skip, 0), MaxResultsLimit);
         IList<string>? includeDomains = null;
         IList<string>? excludeDomains = null;
 
@@ -718,7 +742,7 @@ private TavilySearchRequest BuildRequestContent(string query, int top, int skip,
             timeRange,
             days,
 #pragma warning disable CA1308 // Lower is preferred over uppercase
-            this._searchOptions?.SearchDepth?.ToString()?.ToLowerInvariant(),
+            this._searchOptions?.SearchDepth is { } searchDepth ? ToApiSearchDepth(searchDepth) : null,
 #pragma warning restore CA1308
             this._searchOptions?.ChunksPerSource,
             this._searchOptions?.IncludeImages,
@@ -727,9 +751,35 @@ private TavilySearchRequest BuildRequestContent(string query, int top, int skip,
             this._searchOptions?.IncludeRawContent,
             maxResults,
             includeDomains,
-            excludeDomains);
+            excludeDomains,
+            this._searchOptions?.Country,
+            FormatDate(this._searchOptions?.StartDate),
+            FormatDate(this._searchOptions?.EndDate),
+            this._searchOptions?.SafeSearch,
+            includeDomains is { Count: > 0 } ? ToApiIncludeDomainsMode(this._searchOptions?.IncludeDomainsMode) : null,
+            this._searchOptions?.Language,
+            this._searchOptions?.FilterByLanguage,
+            this._searchOptions?.ExactMatch,
+            this._searchOptions?.AutoParameters);
     }
 
+    private static string? FormatDate(DateTime? date) => date?.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture);
+
+    private static string? ToApiIncludeDomainsMode(TavilyIncludeDomainsMode? mode) => mode switch
+    {
+        TavilyIncludeDomainsMode.Restrict => "restrict",
+        TavilyIncludeDomainsMode.Prefer => "prefer",
+        _ => null,
+    };
+
+    private static string ToApiSearchDepth(TavilySearchDepth searchDepth) => searchDepth switch
+    {
+        TavilySearchDepth.UltraFast => "ultra-fast",
+#pragma warning disable CA1308 // Lower is preferred over uppercase
+        _ => searchDepth.ToString().ToLowerInvariant(),
+#pragma warning restore CA1308
+    };
+
     private static readonly JsonSerializerOptions s_jsonOptionsCache = new()
     {
         PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,