diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 427b8ec..21f6056 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "2.9.0" + ".": "2.10.0" } \ No newline at end of file diff --git a/.stats.yml b/.stats.yml index 97db86d..f9b95cb 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ configured_endpoints: 40 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-a0dda03bbb600917cfb9add468cc4c8c84351a8dbbf61644dbc353263ca1748f.yml -openapi_spec_hash: c24264f32a46d9317aac5af9d6a396f7 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-7f75f39699c1298d16691b5a58b1312a229b78288fba6e2de4040ba1422e530f.yml +openapi_spec_hash: b3ad781596e4c52fde918cfaae54a5d4 config_hash: 920678668dd2da6f8966fbf1b8fde4e2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ac7ed8..eecf584 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 2.10.0 (2026-08-10) + +Full Changelog: [v2.9.0...v2.10.0](https://github.com/context-dot-dev/context-go-sdk/compare/v2.9.0...v2.10.0) + +### Features + +* **api:** api update ([678ec8a](https://github.com/context-dot-dev/context-go-sdk/commit/678ec8a06083f5d8a920ea7c6e8debb9172844eb)) +* **api:** api update ([e65edd1](https://github.com/context-dot-dev/context-go-sdk/commit/e65edd1676b33ef36bdfd6f9bd73007b57819d00)) + ## 2.9.0 (2026-08-07) Full Changelog: [v2.8.0...v2.9.0](https://github.com/context-dot-dev/context-go-sdk/compare/v2.8.0...v2.9.0) diff --git a/README.md b/README.md index ae38305..0618225 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Or to pin the version: ```sh -go get -u 'github.com/context-dot-dev/context-go-sdk@v2.9.0' +go get -u 'github.com/context-dot-dev/context-go-sdk@v2.10.0' ``` diff --git a/batch.go b/batch.go index c58dac2..289b610 100644 --- a/batch.go +++ b/batch.go @@ -1149,7 +1149,8 @@ type BatchGetResultsResponseDataOk struct { Status constant.Ok `json:"status" default:"ok"` // URL as submitted, or as discovered by the crawl. URL string `json:"url" api:"required"` - // Raw page HTML. Present on html batches. + // Page HTML. Present on html batches, and on markdown batches submitted with + // `options.includeHTML`. HTML string `json:"html"` // Caller-supplied identifier echoed from submission. ItemID string `json:"itemId"` @@ -1202,6 +1203,9 @@ type BatchGetResultsResponseDataOkMetadata struct { Description string `json:"description"` // Resolved favicon URL, when present. Favicon string `json:"favicon"` + // Page headings (h1–h6) in document order, extracted from the unfiltered document. + // Capped at the first 500 headings. Omitted when the page has none. + Headings []BatchGetResultsResponseDataOkMetadataHeading `json:"headings"` // Primary resolved preview image from Open Graph, Twitter, or image metadata. Image string `json:"image"` // JSON-LD structured data blocks parsed from the page. @@ -1234,6 +1238,7 @@ type BatchGetResultsResponseDataOkMetadata struct { CanonicalURL respjson.Field Description respjson.Field Favicon respjson.Field + Headings respjson.Field Image respjson.Field JsonLd respjson.Field Keywords respjson.Field @@ -1318,6 +1323,26 @@ func (r *BatchGetResultsResponseDataOkMetadataAlternate) UnmarshalJSON(data []by return apijson.UnmarshalRoot(data, r) } +type BatchGetResultsResponseDataOkMetadataHeading struct { + // Heading level, 1–6 (from h1–h6). + Level int64 `json:"level" api:"required"` + // Heading text with whitespace collapsed, truncated to 1000 characters. + Text string `json:"text" api:"required"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + Level respjson.Field + Text respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r BatchGetResultsResponseDataOkMetadataHeading) RawJSON() string { return r.JSON.raw } +func (r *BatchGetResultsResponseDataOkMetadataHeading) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + // BatchGetResultsResponseDataOkMetadataOpenGraphUnion contains all possible // properties and values from [string], [[]string]. // @@ -1795,6 +1820,9 @@ type BatchSubmitParamsInputScrapeDataMarkdownOptions struct { // younger than this many milliseconds. Defaults to 1 day (86400000 ms) when // omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh. MaxAgeMs param.Opt[int64] `json:"maxAgeMs,omitzero"` + // Also include each page's HTML in its result record, as an `html` field alongside + // the Markdown. + IncludeHTML param.Opt[bool] `json:"includeHTML,omitzero"` // Include image references in the Markdown. IncludeImages param.Opt[bool] `json:"includeImages,omitzero"` // Include links in the Markdown. @@ -2298,6 +2326,9 @@ type BatchSubmitParamsInputCrawlDataMarkdownOptions struct { // younger than this many milliseconds. Defaults to 1 day (86400000 ms) when // omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh. MaxAgeMs param.Opt[int64] `json:"maxAgeMs,omitzero"` + // Also include each page's HTML in its result record, as an `html` field alongside + // the Markdown. + IncludeHTML param.Opt[bool] `json:"includeHTML,omitzero"` // Include image references in the Markdown. IncludeImages param.Opt[bool] `json:"includeImages,omitzero"` // Include links in the Markdown. diff --git a/batch_test.go b/batch_test.go index 2011fab..3575da8 100644 --- a/batch_test.go +++ b/batch_test.go @@ -176,6 +176,7 @@ func TestBatchSubmitWithOptionalParams(t *testing.T) { Options: contextdev.BatchSubmitParamsInputScrapeDataMarkdownOptions{ Country: "de", ExcludeSelectors: []string{"x"}, + IncludeHTML: contextdev.Bool(true), IncludeImages: contextdev.Bool(true), IncludeLinks: contextdev.Bool(true), IncludeSelectors: []string{"x"}, diff --git a/internal/version.go b/internal/version.go index 24ea833..3b2e89c 100644 --- a/internal/version.go +++ b/internal/version.go @@ -2,4 +2,4 @@ package internal -const PackageVersion = "2.9.0" // x-release-please-version +const PackageVersion = "2.10.0" // x-release-please-version diff --git a/utility.go b/utility.go index 1945e43..4b5754f 100644 --- a/utility.go +++ b/utility.go @@ -33,10 +33,11 @@ func NewUtilityService(opts ...option.RequestOption) (r UtilityService) { return } -// Signal that you may fetch brand data soon to improve latency. The type field -// selects what to prefetch (currently only 'brand') and identifier carries exactly -// one lookup key: a domain, or an email whose domain is extracted and validated -// (free email providers and disposable email addresses are not allowed). +// Signal that you may fetch data soon to improve latency. The type field selects +// what to prefetch ('brand' queues a brand data fetch, 'styleguide' queues a +// styleguide extraction) and identifier carries exactly one lookup key: a domain, +// or an email whose domain is extracted and validated (free email providers and +// disposable email addresses are not allowed). func (r *UtilityService) Prefetch(ctx context.Context, body UtilityPrefetchParams, opts ...option.RequestOption) (res *UtilityPrefetchResponse, err error) { opts = slices.Concat(r.options, opts) path := "utility/prefetch" @@ -54,10 +55,9 @@ type UtilityPrefetchResponse struct { Message string `json:"message"` // Status of the response, e.g., 'ok' Status string `json:"status"` - // The type of prefetch that was queued, echoed from the request (currently always - // 'brand') + // The type of prefetch that was queued, echoed from the request // - // Any of "brand". + // Any of "brand", "styleguide". Type UtilityPrefetchResponseType `json:"type"` // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { @@ -99,20 +99,21 @@ func (r *UtilityPrefetchResponseKeyMetadata) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } -// The type of prefetch that was queued, echoed from the request (currently always -// 'brand') +// The type of prefetch that was queued, echoed from the request type UtilityPrefetchResponseType string const ( - UtilityPrefetchResponseTypeBrand UtilityPrefetchResponseType = "brand" + UtilityPrefetchResponseTypeBrand UtilityPrefetchResponseType = "brand" + UtilityPrefetchResponseTypeStyleguide UtilityPrefetchResponseType = "styleguide" ) type UtilityPrefetchParams struct { - // Identifier of the brand to prefetch. Provide exactly one of domain or email. + // Identifier of the target to prefetch. Provide exactly one of domain or email. Identifier UtilityPrefetchParamsIdentifierUnion `json:"identifier,omitzero" api:"required"` - // What to prefetch. Currently only 'brand' is supported. + // What to prefetch: 'brand' warms the brand data cache, 'styleguide' warms the + // styleguide cache. // - // Any of "brand". + // Any of "brand", "styleguide". Type UtilityPrefetchParamsType `json:"type,omitzero" api:"required"` // Optional timeout in milliseconds for the request. If the request takes longer // than this value, it will be aborted with a 408 status code. Maximum allowed @@ -147,11 +148,11 @@ func (u *UtilityPrefetchParamsIdentifierUnion) UnmarshalJSON(data []byte) error return apijson.UnmarshalRoot(data, u) } -// Prefetch brand data by domain. +// Prefetch by domain. // // The property Domain is required. type UtilityPrefetchParamsIdentifierByDomain struct { - // Domain name to prefetch brand data for + // Domain name to prefetch data for Domain string `json:"domain" api:"required"` paramObj } @@ -164,13 +165,13 @@ func (r *UtilityPrefetchParamsIdentifierByDomain) UnmarshalJSON(data []byte) err return apijson.UnmarshalRoot(data, r) } -// Prefetch brand data by email. The domain will be extracted and validated. +// Prefetch by email. The domain will be extracted and validated. // // The property Email is required. type UtilityPrefetchParamsIdentifierByEmail struct { - // Email address to prefetch brand data for. The domain will be extracted from the - // email. Free email providers (gmail.com, yahoo.com, etc.) and disposable email - // addresses are not allowed. + // Email address to prefetch data for. The domain will be extracted from the email. + // Free email providers (gmail.com, yahoo.com, etc.) and disposable email addresses + // are not allowed. Email string `json:"email" api:"required" format:"email"` paramObj } @@ -183,9 +184,11 @@ func (r *UtilityPrefetchParamsIdentifierByEmail) UnmarshalJSON(data []byte) erro return apijson.UnmarshalRoot(data, r) } -// What to prefetch. Currently only 'brand' is supported. +// What to prefetch: 'brand' warms the brand data cache, 'styleguide' warms the +// styleguide cache. type UtilityPrefetchParamsType string const ( - UtilityPrefetchParamsTypeBrand UtilityPrefetchParamsType = "brand" + UtilityPrefetchParamsTypeBrand UtilityPrefetchParamsType = "brand" + UtilityPrefetchParamsTypeStyleguide UtilityPrefetchParamsType = "styleguide" ) diff --git a/web.go b/web.go index 1d26a24..34eb653 100644 --- a/web.go +++ b/web.go @@ -137,17 +137,17 @@ func (r *WebService) WebScrapeImages(ctx context.Context, query WebWebScrapeImag // // ### Billing & errors // -// | HTTP status | Billed? | Meaning | -// | ----------- | ----------------------------------------- | ---------------------------------------------------------------------------------------- | -// | 200 | Yes — 1 credit, or 2 credits with actions | Successful scrape, including a zero-length result when includeSelectors matched nothing | -// | 400 | No | Invalid input, skipped PDF, or the page could not be scraped | -// | 401 / 403 | No | Invalid/disabled key, insufficient permissions, or credits exhausted; inspect error_code | -// | 404 | No | Target page returned or fingerprinted as not found | -// | 408 | No | Request timed out | -// | 413 | No | Target content exceeds the maximum supported size (20 MB) | -// | 415 | No | Unsupported content type | -// | 429 | No | Per-minute rate limit exceeded; honor Retry-After | -// | 500 | No | Internal error | +// | HTTP status | Billed? | Meaning | +// | ----------- | ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +// | 200 | Yes — 1 credit, or 2 credits with actions | Successful scrape, including a zero-length result when includeSelectors matched nothing | +// | 400 | No | Invalid input, skipped PDF, or the page could not be scraped. error_code WEBSITE_BLOCKED specifically means the site answered with an anti-bot challenge, CAPTCHA wall, or login shell instead of the page (even when the site returned HTTP 200) — retrying later or from another country sometimes succeeds | +// | 401 / 403 | No | Invalid/disabled key, insufficient permissions, or credits exhausted; inspect error_code | +// | 404 | No | Target page returned or fingerprinted as not found | +// | 408 | No | Request timed out | +// | 413 | No | Target content exceeds the maximum supported size (20 MB) | +// | 415 | No | Unsupported content type | +// | 429 | No | Per-minute rate limit exceeded; honor Retry-After | +// | 500 | No | Internal error | func (r *WebService) WebScrapeMd(ctx context.Context, query WebWebScrapeMdParams, opts ...option.RequestOption) (res *WebWebScrapeMdResponse, err error) { opts = slices.Concat(r.options, opts) path := "web/scrape/markdown" @@ -1407,6 +1407,9 @@ type WebWebCrawlMdResponseResultMetadata struct { Description string `json:"description"` // Resolved favicon URL, when present. Favicon string `json:"favicon"` + // Page headings (h1–h6) in document order, extracted from the unfiltered document. + // Capped at the first 500 headings. Omitted when the page has none. + Headings []WebWebCrawlMdResponseResultMetadataHeading `json:"headings"` // Primary resolved preview image from Open Graph, Twitter, or image metadata. Image string `json:"image"` // JSON-LD structured data blocks parsed from the page. @@ -1442,6 +1445,7 @@ type WebWebCrawlMdResponseResultMetadata struct { CanonicalURL respjson.Field Description respjson.Field Favicon respjson.Field + Headings respjson.Field Image respjson.Field JsonLd respjson.Field Keywords respjson.Field @@ -1525,6 +1529,26 @@ func (r *WebWebCrawlMdResponseResultMetadataAlternate) UnmarshalJSON(data []byte return apijson.UnmarshalRoot(data, r) } +type WebWebCrawlMdResponseResultMetadataHeading struct { + // Heading level, 1–6 (from h1–h6). + Level int64 `json:"level" api:"required"` + // Heading text with whitespace collapsed, truncated to 1000 characters. + Text string `json:"text" api:"required"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + Level respjson.Field + Text respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r WebWebCrawlMdResponseResultMetadataHeading) RawJSON() string { return r.JSON.raw } +func (r *WebWebCrawlMdResponseResultMetadataHeading) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + // WebWebCrawlMdResponseResultMetadataOpenGraphUnion contains all possible // properties and values from [string], [[]string]. // @@ -1688,6 +1712,9 @@ type WebWebScrapeHTMLResponseMetadata struct { Description string `json:"description"` // Resolved favicon URL, when present. Favicon string `json:"favicon"` + // Page headings (h1–h6) in document order, extracted from the unfiltered document. + // Capped at the first 500 headings. Omitted when the page has none. + Headings []WebWebScrapeHTMLResponseMetadataHeading `json:"headings"` // Primary resolved preview image from Open Graph, Twitter, or image metadata. Image string `json:"image"` // JSON-LD structured data blocks parsed from the page. @@ -1720,6 +1747,7 @@ type WebWebScrapeHTMLResponseMetadata struct { CanonicalURL respjson.Field Description respjson.Field Favicon respjson.Field + Headings respjson.Field Image respjson.Field JsonLd respjson.Field Keywords respjson.Field @@ -1804,6 +1832,26 @@ func (r *WebWebScrapeHTMLResponseMetadataAlternate) UnmarshalJSON(data []byte) e return apijson.UnmarshalRoot(data, r) } +type WebWebScrapeHTMLResponseMetadataHeading struct { + // Heading level, 1–6 (from h1–h6). + Level int64 `json:"level" api:"required"` + // Heading text with whitespace collapsed, truncated to 1000 characters. + Text string `json:"text" api:"required"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + Level respjson.Field + Text respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r WebWebScrapeHTMLResponseMetadataHeading) RawJSON() string { return r.JSON.raw } +func (r *WebWebScrapeHTMLResponseMetadataHeading) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + // WebWebScrapeHTMLResponseMetadataOpenGraphUnion contains all possible properties // and values from [string], [[]string]. // @@ -2092,6 +2140,10 @@ type WebWebScrapeMdResponse struct { // True when an action was applied but the returned content could not be refreshed // afterward. ActionsHTMLStale bool `json:"actionsHtmlStale"` + // Only present when includeHTML=true: the page HTML the Markdown was converted + // from — the same body the Scrape HTML endpoint returns for the equivalent + // request. + HTML string `json:"html"` // Metadata about the API key used for the request. Included in every response // whenever a valid API key is provided, even when the response status is not 200. KeyMetadata WebWebScrapeMdResponseKeyMetadata `json:"key_metadata"` @@ -2104,6 +2156,7 @@ type WebWebScrapeMdResponse struct { URL respjson.Field ActionsApplied respjson.Field ActionsHTMLStale respjson.Field + HTML respjson.Field KeyMetadata respjson.Field ExtraFields map[string]respjson.Field raw string @@ -2135,6 +2188,9 @@ type WebWebScrapeMdResponseMetadata struct { Description string `json:"description"` // Resolved favicon URL, when present. Favicon string `json:"favicon"` + // Page headings (h1–h6) in document order, extracted from the unfiltered document. + // Capped at the first 500 headings. Omitted when the page has none. + Headings []WebWebScrapeMdResponseMetadataHeading `json:"headings"` // Primary resolved preview image from Open Graph, Twitter, or image metadata. Image string `json:"image"` // JSON-LD structured data blocks parsed from the page. @@ -2167,6 +2223,7 @@ type WebWebScrapeMdResponseMetadata struct { CanonicalURL respjson.Field Description respjson.Field Favicon respjson.Field + Headings respjson.Field Image respjson.Field JsonLd respjson.Field Keywords respjson.Field @@ -2251,6 +2308,26 @@ func (r *WebWebScrapeMdResponseMetadataAlternate) UnmarshalJSON(data []byte) err return apijson.UnmarshalRoot(data, r) } +type WebWebScrapeMdResponseMetadataHeading struct { + // Heading level, 1–6 (from h1–h6). + Level int64 `json:"level" api:"required"` + // Heading text with whitespace collapsed, truncated to 1000 characters. + Text string `json:"text" api:"required"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + Level respjson.Field + Text respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r WebWebScrapeMdResponseMetadataHeading) RawJSON() string { return r.JSON.raw } +func (r *WebWebScrapeMdResponseMetadataHeading) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + // WebWebScrapeMdResponseMetadataOpenGraphUnion contains all possible properties // and values from [string], [[]string]. // @@ -4483,6 +4560,10 @@ type WebWebScrapeMdParams struct { Headers map[string]string `query:"headers,omitzero" json:"-"` // When true, the contents of iframes are rendered to Markdown. IncludeFrames WebWebScrapeMdParamsIncludeFramesUnion `query:"includeFrames,omitzero" json:"-"` + // When true, the response also includes an `html` field with the page HTML the + // Markdown was converted from — the same body the Scrape HTML endpoint returns for + // the equivalent request. + IncludeHTML WebWebScrapeMdParamsIncludeHTMLUnion `query:"includeHTML,omitzero" json:"-"` // Include image references in Markdown output IncludeImages WebWebScrapeMdParamsIncludeImagesUnion `query:"includeImages,omitzero" json:"-"` // Preserve hyperlinks in Markdown output @@ -4805,6 +4886,24 @@ const ( WebWebScrapeMdParamsIncludeFramesStringFalse WebWebScrapeMdParamsIncludeFramesString = "false" ) +// Only one field can be non-zero. +// +// Use [param.IsOmitted] to confirm if a field is set. +type WebWebScrapeMdParamsIncludeHTMLUnion struct { + OfBool param.Opt[bool] `query:",omitzero,inline"` + // Check if union is this variant with + // !param.IsOmitted(union.OfWebWebScrapeMdsIncludeHTMLString) + OfWebWebScrapeMdsIncludeHTMLString param.Opt[string] `query:",omitzero,inline"` + paramUnion +} + +type WebWebScrapeMdParamsIncludeHTMLString string + +const ( + WebWebScrapeMdParamsIncludeHTMLStringTrue WebWebScrapeMdParamsIncludeHTMLString = "true" + WebWebScrapeMdParamsIncludeHTMLStringFalse WebWebScrapeMdParamsIncludeHTMLString = "false" +) + // Only one field can be non-zero. // // Use [param.IsOmitted] to confirm if a field is set. diff --git a/web_test.go b/web_test.go index c5ba889..2953278 100644 --- a/web_test.go +++ b/web_test.go @@ -430,6 +430,9 @@ func TestWebWebScrapeMdWithOptionalParams(t *testing.T) { IncludeFrames: contextdev.WebWebScrapeMdParamsIncludeFramesUnion{ OfWebWebScrapeMdsIncludeFramesString: contextdev.String("true"), }, + IncludeHTML: contextdev.WebWebScrapeMdParamsIncludeHTMLUnion{ + OfWebWebScrapeMdsIncludeHTMLString: contextdev.String("true"), + }, IncludeImages: contextdev.WebWebScrapeMdParamsIncludeImagesUnion{ OfWebWebScrapeMdsIncludeImagesString: contextdev.String("true"), },