diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 427b8ec..21f6056 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "2.9.0" + ".": "2.10.0" } \ No newline at end of file diff --git a/.stats.yml b/.stats.yml index 97db86d..0829be8 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ configured_endpoints: 40 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-a0dda03bbb600917cfb9add468cc4c8c84351a8dbbf61644dbc353263ca1748f.yml -openapi_spec_hash: c24264f32a46d9317aac5af9d6a396f7 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-5a88256ce88b7bc7854211e67446bc71bc52975a2d62263e1194bec7cf6c2f16.yml +openapi_spec_hash: f5011cea07c6c4a7d2984d6be85ffe49 config_hash: 920678668dd2da6f8966fbf1b8fde4e2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f203b9..84f18dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 2.10.0 (2026-08-11) + +Full Changelog: [v2.9.0...v2.10.0](https://github.com/context-dot-dev/context-php-sdk/compare/v2.9.0...v2.10.0) + +### Features + +* **api:** api update ([0e1448d](https://github.com/context-dot-dev/context-php-sdk/commit/0e1448d609afec7df8d16031c39922d6b8cc5cba)) +* **api:** api update ([2f3a923](https://github.com/context-dot-dev/context-php-sdk/commit/2f3a923880a205a86d61f10d2aad236496f56144)) + ## 2.9.0 (2026-08-07) Full Changelog: [v2.8.0...v2.9.0](https://github.com/context-dot-dev/context-php-sdk/compare/v2.8.0...v2.9.0) diff --git a/README.md b/README.md index 8272f7d..462850f 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ The REST API documentation can be found on [docs.context.dev](https://docs.conte ``` -composer require "context-dev/context-dev-php 2.9.0" +composer require "context-dev/context-dev-php 2.10.0" ``` diff --git a/src/Batch/BatchGetResultsResponse/Data/ScrapedPage.php b/src/Batch/BatchGetResultsResponse/Data/ScrapedPage.php index d5c7edd..c8c21cf 100644 --- a/src/Batch/BatchGetResultsResponse/Data/ScrapedPage.php +++ b/src/Batch/BatchGetResultsResponse/Data/ScrapedPage.php @@ -66,7 +66,7 @@ final class ScrapedPage implements BaseModel public string $url; /** - * Raw page HTML. Present on html batches. + * Page HTML. Present on html batches, and on markdown batches submitted with `options.includeHTML`. */ #[Optional] public ?string $html; @@ -215,7 +215,7 @@ public function withURL(string $url): self } /** - * Raw page HTML. Present on html batches. + * Page HTML. Present on html batches, and on markdown batches submitted with `options.includeHTML`. */ public function withHTML(string $html): self { diff --git a/src/Batch/BatchGetResultsResponse/Data/ScrapedPage/Metadata.php b/src/Batch/BatchGetResultsResponse/Data/ScrapedPage/Metadata.php index d3619e8..db6b217 100644 --- a/src/Batch/BatchGetResultsResponse/Data/ScrapedPage/Metadata.php +++ b/src/Batch/BatchGetResultsResponse/Data/ScrapedPage/Metadata.php @@ -6,6 +6,7 @@ use ContextDev\Batch\BatchGetResultsResponse\Data\ScrapedPage\Metadata\AdditionalMeta; use ContextDev\Batch\BatchGetResultsResponse\Data\ScrapedPage\Metadata\Alternate; +use ContextDev\Batch\BatchGetResultsResponse\Data\ScrapedPage\Metadata\Heading; use ContextDev\Batch\BatchGetResultsResponse\Data\ScrapedPage\Metadata\OpenGraph; use ContextDev\Batch\BatchGetResultsResponse\Data\ScrapedPage\Metadata\Twitter; use ContextDev\Core\Attributes\Optional; @@ -22,6 +23,7 @@ * @phpstan-import-type TwitterVariants from \ContextDev\Batch\BatchGetResultsResponse\Data\ScrapedPage\Metadata\Twitter * @phpstan-import-type AdditionalMetaShape from \ContextDev\Batch\BatchGetResultsResponse\Data\ScrapedPage\Metadata\AdditionalMeta * @phpstan-import-type AlternateShape from \ContextDev\Batch\BatchGetResultsResponse\Data\ScrapedPage\Metadata\Alternate + * @phpstan-import-type HeadingShape from \ContextDev\Batch\BatchGetResultsResponse\Data\ScrapedPage\Metadata\Heading * @phpstan-import-type OpenGraphShape from \ContextDev\Batch\BatchGetResultsResponse\Data\ScrapedPage\Metadata\OpenGraph * @phpstan-import-type TwitterShape from \ContextDev\Batch\BatchGetResultsResponse\Data\ScrapedPage\Metadata\Twitter * @@ -34,6 +36,7 @@ * canonicalURL?: string|null, * description?: string|null, * favicon?: string|null, + * headings?: list|null, * image?: string|null, * jsonLd?: list>|null, * keywords?: list|null, @@ -104,6 +107,14 @@ final class Metadata implements BaseModel #[Optional] public ?string $favicon; + /** + * Page headings (h1–h6) in document order, extracted from the unfiltered document. Capped at the first 500 headings. Omitted when the page has none. + * + * @var list|null $headings + */ + #[Optional(list: Heading::class)] + public ?array $headings; + /** * Primary resolved preview image from Open Graph, Twitter, or image metadata. */ @@ -204,6 +215,7 @@ public function __construct() * * @param array|null $additionalMeta * @param list|null $alternates + * @param list|null $headings * @param list>|null $jsonLd * @param list|null $keywords * @param array|null $openGraph @@ -218,6 +230,7 @@ public static function with( ?string $canonicalURL = null, ?string $description = null, ?string $favicon = null, + ?array $headings = null, ?string $image = null, ?array $jsonLd = null, ?array $keywords = null, @@ -241,6 +254,7 @@ public static function with( null !== $canonicalURL && $self['canonicalURL'] = $canonicalURL; null !== $description && $self['description'] = $description; null !== $favicon && $self['favicon'] = $favicon; + null !== $headings && $self['headings'] = $headings; null !== $image && $self['image'] = $image; null !== $jsonLd && $self['jsonLd'] = $jsonLd; null !== $keywords && $self['keywords'] = $keywords; @@ -348,6 +362,19 @@ public function withFavicon(string $favicon): self return $self; } + /** + * Page headings (h1–h6) in document order, extracted from the unfiltered document. Capped at the first 500 headings. Omitted when the page has none. + * + * @param list $headings + */ + public function withHeadings(array $headings): self + { + $self = clone $this; + $self['headings'] = $headings; + + return $self; + } + /** * Primary resolved preview image from Open Graph, Twitter, or image metadata. */ diff --git a/src/Batch/BatchGetResultsResponse/Data/ScrapedPage/Metadata/Heading.php b/src/Batch/BatchGetResultsResponse/Data/ScrapedPage/Metadata/Heading.php new file mode 100644 index 0000000..f6e9aa2 --- /dev/null +++ b/src/Batch/BatchGetResultsResponse/Data/ScrapedPage/Metadata/Heading.php @@ -0,0 +1,86 @@ + */ + use SdkModel; + + /** + * Heading level, 1–6 (from h1–h6). + */ + #[Required] + public int $level; + + /** + * Heading text with whitespace collapsed, truncated to 1000 characters. + */ + #[Required] + public string $text; + + /** + * `new Heading()` is missing required properties by the API. + * + * To enforce required parameters use + * ``` + * Heading::with(level: ..., text: ...) + * ``` + * + * Otherwise ensure the following setters are called + * + * ``` + * (new Heading)->withLevel(...)->withText(...) + * ``` + */ + public function __construct() + { + $this->initialize(); + } + + /** + * Construct an instance from the required parameters. + * + * You must use named parameters to construct any parameters with a default value. + */ + public static function with(int $level, string $text): self + { + $self = new self; + + $self['level'] = $level; + $self['text'] = $text; + + return $self; + } + + /** + * Heading level, 1–6 (from h1–h6). + */ + public function withLevel(int $level): self + { + $self = clone $this; + $self['level'] = $level; + + return $self; + } + + /** + * Heading text with whitespace collapsed, truncated to 1000 characters. + */ + public function withText(string $text): self + { + $self = clone $this; + $self['text'] = $text; + + return $self; + } +} diff --git a/src/Batch/BatchSubmitParams/Input/Crawl/Data/Markdown/Options.php b/src/Batch/BatchSubmitParams/Input/Crawl/Data/Markdown/Options.php index 20e33f6..a62dc97 100644 --- a/src/Batch/BatchSubmitParams/Input/Crawl/Data/Markdown/Options.php +++ b/src/Batch/BatchSubmitParams/Input/Crawl/Data/Markdown/Options.php @@ -18,6 +18,7 @@ * @phpstan-type OptionsShape = array{ * country?: null|Country|value-of, * excludeSelectors?: list|null, + * includeHTML?: bool|null, * includeImages?: bool|null, * includeLinks?: bool|null, * includeSelectors?: list|null, @@ -50,6 +51,12 @@ final class Options implements BaseModel #[Optional(list: 'string', nullable: true)] public ?array $excludeSelectors; + /** + * Also include each page's HTML in its result record, as an `html` field alongside the Markdown. + */ + #[Optional] + public ?bool $includeHTML; + /** * Include image references in the Markdown. */ @@ -124,6 +131,7 @@ public function __construct() public static function with( Country|string|null $country = null, ?array $excludeSelectors = null, + ?bool $includeHTML = null, ?bool $includeImages = null, ?bool $includeLinks = null, ?array $includeSelectors = null, @@ -138,6 +146,7 @@ public static function with( null !== $country && $self['country'] = $country; null !== $excludeSelectors && $self['excludeSelectors'] = $excludeSelectors; + null !== $includeHTML && $self['includeHTML'] = $includeHTML; null !== $includeImages && $self['includeImages'] = $includeImages; null !== $includeLinks && $self['includeLinks'] = $includeLinks; null !== $includeSelectors && $self['includeSelectors'] = $includeSelectors; @@ -177,6 +186,17 @@ public function withExcludeSelectors(?array $excludeSelectors): self return $self; } + /** + * Also include each page's HTML in its result record, as an `html` field alongside the Markdown. + */ + public function withIncludeHTML(bool $includeHTML): self + { + $self = clone $this; + $self['includeHTML'] = $includeHTML; + + return $self; + } + /** * Include image references in the Markdown. */ diff --git a/src/Batch/BatchSubmitParams/Input/Scrape/Data/Markdown/Options.php b/src/Batch/BatchSubmitParams/Input/Scrape/Data/Markdown/Options.php index b734ecf..d2cb2cc 100644 --- a/src/Batch/BatchSubmitParams/Input/Scrape/Data/Markdown/Options.php +++ b/src/Batch/BatchSubmitParams/Input/Scrape/Data/Markdown/Options.php @@ -18,6 +18,7 @@ * @phpstan-type OptionsShape = array{ * country?: null|Country|value-of, * excludeSelectors?: list|null, + * includeHTML?: bool|null, * includeImages?: bool|null, * includeLinks?: bool|null, * includeSelectors?: list|null, @@ -50,6 +51,12 @@ final class Options implements BaseModel #[Optional(list: 'string', nullable: true)] public ?array $excludeSelectors; + /** + * Also include each page's HTML in its result record, as an `html` field alongside the Markdown. + */ + #[Optional] + public ?bool $includeHTML; + /** * Include image references in the Markdown. */ @@ -124,6 +131,7 @@ public function __construct() public static function with( Country|string|null $country = null, ?array $excludeSelectors = null, + ?bool $includeHTML = null, ?bool $includeImages = null, ?bool $includeLinks = null, ?array $includeSelectors = null, @@ -138,6 +146,7 @@ public static function with( null !== $country && $self['country'] = $country; null !== $excludeSelectors && $self['excludeSelectors'] = $excludeSelectors; + null !== $includeHTML && $self['includeHTML'] = $includeHTML; null !== $includeImages && $self['includeImages'] = $includeImages; null !== $includeLinks && $self['includeLinks'] = $includeLinks; null !== $includeSelectors && $self['includeSelectors'] = $includeSelectors; @@ -177,6 +186,17 @@ public function withExcludeSelectors(?array $excludeSelectors): self return $self; } + /** + * Also include each page's HTML in its result record, as an `html` field alongside the Markdown. + */ + public function withIncludeHTML(bool $includeHTML): self + { + $self = clone $this; + $self['includeHTML'] = $includeHTML; + + return $self; + } + /** * Include image references in the Markdown. */ diff --git a/src/ServiceContracts/UtilityContract.php b/src/ServiceContracts/UtilityContract.php index eebf6f1..ad22f2d 100644 --- a/src/ServiceContracts/UtilityContract.php +++ b/src/ServiceContracts/UtilityContract.php @@ -20,8 +20,8 @@ interface UtilityContract /** * @api * - * @param IdentifierShape $identifier Identifier of the brand to prefetch. Provide exactly one of domain or email. - * @param Type|value-of $type What to prefetch. Currently only 'brand' is supported. + * @param IdentifierShape $identifier Identifier of the target to prefetch. Provide exactly one of domain or email. + * @param Type|value-of $type what to prefetch: 'brand' warms the brand data cache, 'styleguide' warms the styleguide cache * @param list $tags Optional tags for tracking usage. Up to 20 tags, each 1 to 50 characters. * @param int $timeoutMs Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes). * @param RequestOpts|null $requestOptions diff --git a/src/ServiceContracts/WebContract.php b/src/ServiceContracts/WebContract.php index beefa33..ca76efa 100644 --- a/src/ServiceContracts/WebContract.php +++ b/src/ServiceContracts/WebContract.php @@ -45,6 +45,7 @@ * @phpstan-import-type EnrichmentShape from \ContextDev\Web\WebWebScrapeImagesParams\Enrichment * @phpstan-import-type ActionShape from \ContextDev\Web\WebWebScrapeMdParams\Action as ActionShape2 * @phpstan-import-type IncludeFramesShape from \ContextDev\Web\WebWebScrapeMdParams\IncludeFrames as IncludeFramesShape1 + * @phpstan-import-type IncludeHTMLShape from \ContextDev\Web\WebWebScrapeMdParams\IncludeHTML * @phpstan-import-type IncludeImagesShape from \ContextDev\Web\WebWebScrapeMdParams\IncludeImages * @phpstan-import-type IncludeLinksShape from \ContextDev\Web\WebWebScrapeMdParams\IncludeLinks * @phpstan-import-type PdfShape from \ContextDev\Web\WebWebScrapeMdParams\Pdf as PdfShape3 @@ -365,6 +366,7 @@ public function webScrapeImages( * @param list|null $excludeSelectors CSS selectors to remove before conversion to Markdown. Applied after includeSelectors. Exclusion takes precedence: an element matching both is removed. Examples: "nav", "footer", ".ad-banner", "[aria-hidden=true]". * @param array $headers Optional outbound HTTP headers forwarded only to the target URL, sent as deep-object query params such as headers[X-Custom]=value. When provided, caching is bypassed: the result is neither read from nor written to cache. * @param IncludeFramesShape1 $includeFrames when true, the contents of iframes are rendered to Markdown + * @param IncludeHTMLShape $includeHTML when true, the response also includes an `html` field with the page HTML the Markdown was converted from — the same body the Scrape HTML endpoint returns for the equivalent request * @param IncludeImagesShape $includeImages Include image references in Markdown output * @param IncludeLinksShape $includeLinks Preserve hyperlinks in Markdown output * @param list|null $includeSelectors CSS selectors. When provided, only matching HTML subtrees (and their descendants) are kept before conversion to Markdown. When omitted, the entire document is kept. Examples: "article.main", "#content", "[role=main]". @@ -388,6 +390,7 @@ public function webScrapeMd( ?array $excludeSelectors = null, ?array $headers = null, bool|\ContextDev\Web\WebWebScrapeMdParams\IncludeFrames\UnionMember1|string $includeFrames = false, + bool|\ContextDev\Web\WebWebScrapeMdParams\IncludeHTML\UnionMember1|string $includeHTML = false, bool|\ContextDev\Web\WebWebScrapeMdParams\IncludeImages\UnionMember1|string $includeImages = false, bool|\ContextDev\Web\WebWebScrapeMdParams\IncludeLinks\UnionMember1|string $includeLinks = true, ?array $includeSelectors = null, diff --git a/src/Services/UtilityRawService.php b/src/Services/UtilityRawService.php index 1b10d93..84b0c87 100644 --- a/src/Services/UtilityRawService.php +++ b/src/Services/UtilityRawService.php @@ -28,7 +28,7 @@ public function __construct(private Client $client) {} /** * @api * - * Signal that you may fetch brand data soon to improve latency. The type field selects what to prefetch (currently only 'brand') and identifier carries exactly one lookup key: a domain, or an email whose domain is extracted and validated (free email providers and disposable email addresses are not allowed). + * Signal that you may fetch data soon to improve latency. The type field selects what to prefetch ('brand' queues a brand data fetch, 'styleguide' queues a styleguide extraction) and identifier carries exactly one lookup key: a domain, or an email whose domain is extracted and validated (free email providers and disposable email addresses are not allowed). * * @param array{ * identifier: IdentifierShape, diff --git a/src/Services/UtilityService.php b/src/Services/UtilityService.php index 3ba999f..6afb213 100644 --- a/src/Services/UtilityService.php +++ b/src/Services/UtilityService.php @@ -36,10 +36,10 @@ public function __construct(private Client $client) /** * @api * - * Signal that you may fetch brand data soon to improve latency. The type field selects what to prefetch (currently only 'brand') and identifier carries exactly one lookup key: a domain, or an email whose domain is extracted and validated (free email providers and disposable email addresses are not allowed). + * Signal that you may fetch data soon to improve latency. The type field selects what to prefetch ('brand' queues a brand data fetch, 'styleguide' queues a styleguide extraction) and identifier carries exactly one lookup key: a domain, or an email whose domain is extracted and validated (free email providers and disposable email addresses are not allowed). * - * @param IdentifierShape $identifier Identifier of the brand to prefetch. Provide exactly one of domain or email. - * @param Type|value-of $type What to prefetch. Currently only 'brand' is supported. + * @param IdentifierShape $identifier Identifier of the target to prefetch. Provide exactly one of domain or email. + * @param Type|value-of $type what to prefetch: 'brand' warms the brand data cache, 'styleguide' warms the styleguide cache * @param list $tags Optional tags for tracking usage. Up to 20 tags, each 1 to 50 characters. * @param int $timeoutMs Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes). * @param RequestOpts|null $requestOptions diff --git a/src/Services/WebRawService.php b/src/Services/WebRawService.php index 953b50d..ebf4721 100644 --- a/src/Services/WebRawService.php +++ b/src/Services/WebRawService.php @@ -59,6 +59,7 @@ * @phpstan-import-type EnrichmentShape from \ContextDev\Web\WebWebScrapeImagesParams\Enrichment * @phpstan-import-type ActionShape from \ContextDev\Web\WebWebScrapeMdParams\Action as ActionShape2 * @phpstan-import-type IncludeFramesShape from \ContextDev\Web\WebWebScrapeMdParams\IncludeFrames as IncludeFramesShape1 + * @phpstan-import-type IncludeHTMLShape from \ContextDev\Web\WebWebScrapeMdParams\IncludeHTML * @phpstan-import-type IncludeImagesShape from \ContextDev\Web\WebWebScrapeMdParams\IncludeImages * @phpstan-import-type IncludeLinksShape from \ContextDev\Web\WebWebScrapeMdParams\IncludeLinks * @phpstan-import-type PdfShape from \ContextDev\Web\WebWebScrapeMdParams\Pdf as PdfShape3 @@ -482,7 +483,7 @@ public function webScrapeImages( * | HTTP status | Billed? | Meaning | * | --- | --- | --- | * | 200 | Yes — 1 credit, or 2 credits with actions | Successful scrape, including a zero-length result when includeSelectors matched nothing | - * | 400 | No | Invalid input, skipped PDF, or the page could not be scraped | + * | 400 | No | Invalid input, skipped PDF, or the page could not be scraped. error_code WEBSITE_BLOCKED specifically means the site answered with an anti-bot challenge, CAPTCHA wall, or login shell instead of the page (even when the site returned HTTP 200) — retrying later or from another country sometimes succeeds | * | 401 / 403 | No | Invalid/disabled key, insufficient permissions, or credits exhausted; inspect error_code | * | 404 | No | Target page returned or fingerprinted as not found | * | 408 | No | Request timed out | @@ -498,6 +499,7 @@ public function webScrapeImages( * excludeSelectors?: list|null, * headers?: array, * includeFrames?: IncludeFramesShape1, + * includeHTML?: IncludeHTMLShape, * includeImages?: IncludeImagesShape, * includeLinks?: IncludeLinksShape, * includeSelectors?: list|null, diff --git a/src/Services/WebService.php b/src/Services/WebService.php index 27e6e7e..61798e6 100644 --- a/src/Services/WebService.php +++ b/src/Services/WebService.php @@ -48,6 +48,7 @@ * @phpstan-import-type EnrichmentShape from \ContextDev\Web\WebWebScrapeImagesParams\Enrichment * @phpstan-import-type ActionShape from \ContextDev\Web\WebWebScrapeMdParams\Action as ActionShape2 * @phpstan-import-type IncludeFramesShape from \ContextDev\Web\WebWebScrapeMdParams\IncludeFrames as IncludeFramesShape1 + * @phpstan-import-type IncludeHTMLShape from \ContextDev\Web\WebWebScrapeMdParams\IncludeHTML * @phpstan-import-type IncludeImagesShape from \ContextDev\Web\WebWebScrapeMdParams\IncludeImages * @phpstan-import-type IncludeLinksShape from \ContextDev\Web\WebWebScrapeMdParams\IncludeLinks * @phpstan-import-type PdfShape from \ContextDev\Web\WebWebScrapeMdParams\Pdf as PdfShape3 @@ -593,7 +594,7 @@ public function webScrapeImages( * | HTTP status | Billed? | Meaning | * | --- | --- | --- | * | 200 | Yes — 1 credit, or 2 credits with actions | Successful scrape, including a zero-length result when includeSelectors matched nothing | - * | 400 | No | Invalid input, skipped PDF, or the page could not be scraped | + * | 400 | No | Invalid input, skipped PDF, or the page could not be scraped. error_code WEBSITE_BLOCKED specifically means the site answered with an anti-bot challenge, CAPTCHA wall, or login shell instead of the page (even when the site returned HTTP 200) — retrying later or from another country sometimes succeeds | * | 401 / 403 | No | Invalid/disabled key, insufficient permissions, or credits exhausted; inspect error_code | * | 404 | No | Target page returned or fingerprinted as not found | * | 408 | No | Request timed out | @@ -608,6 +609,7 @@ public function webScrapeImages( * @param list|null $excludeSelectors CSS selectors to remove before conversion to Markdown. Applied after includeSelectors. Exclusion takes precedence: an element matching both is removed. Examples: "nav", "footer", ".ad-banner", "[aria-hidden=true]". * @param array $headers Optional outbound HTTP headers forwarded only to the target URL, sent as deep-object query params such as headers[X-Custom]=value. When provided, caching is bypassed: the result is neither read from nor written to cache. * @param IncludeFramesShape1 $includeFrames when true, the contents of iframes are rendered to Markdown + * @param IncludeHTMLShape $includeHTML when true, the response also includes an `html` field with the page HTML the Markdown was converted from — the same body the Scrape HTML endpoint returns for the equivalent request * @param IncludeImagesShape $includeImages Include image references in Markdown output * @param IncludeLinksShape $includeLinks Preserve hyperlinks in Markdown output * @param list|null $includeSelectors CSS selectors. When provided, only matching HTML subtrees (and their descendants) are kept before conversion to Markdown. When omitted, the entire document is kept. Examples: "article.main", "#content", "[role=main]". @@ -631,6 +633,7 @@ public function webScrapeMd( ?array $excludeSelectors = null, ?array $headers = null, bool|\ContextDev\Web\WebWebScrapeMdParams\IncludeFrames\UnionMember1|string $includeFrames = false, + bool|\ContextDev\Web\WebWebScrapeMdParams\IncludeHTML\UnionMember1|string $includeHTML = false, bool|\ContextDev\Web\WebWebScrapeMdParams\IncludeImages\UnionMember1|string $includeImages = false, bool|\ContextDev\Web\WebWebScrapeMdParams\IncludeLinks\UnionMember1|string $includeLinks = true, ?array $includeSelectors = null, @@ -655,6 +658,7 @@ public function webScrapeMd( 'excludeSelectors' => $excludeSelectors, 'headers' => $headers, 'includeFrames' => $includeFrames, + 'includeHTML' => $includeHTML, 'includeImages' => $includeImages, 'includeLinks' => $includeLinks, 'includeSelectors' => $includeSelectors, diff --git a/src/Utility/UtilityPrefetchParams.php b/src/Utility/UtilityPrefetchParams.php index 2025df6..6fa9680 100644 --- a/src/Utility/UtilityPrefetchParams.php +++ b/src/Utility/UtilityPrefetchParams.php @@ -14,7 +14,7 @@ use ContextDev\Utility\UtilityPrefetchParams\Type; /** - * Signal that you may fetch brand data soon to improve latency. The type field selects what to prefetch (currently only 'brand') and identifier carries exactly one lookup key: a domain, or an email whose domain is extracted and validated (free email providers and disposable email addresses are not allowed). + * Signal that you may fetch data soon to improve latency. The type field selects what to prefetch ('brand' queues a brand data fetch, 'styleguide' queues a styleguide extraction) and identifier carries exactly one lookup key: a domain, or an email whose domain is extracted and validated (free email providers and disposable email addresses are not allowed). * * @see ContextDev\Services\UtilityService::prefetch() * @@ -35,7 +35,7 @@ final class UtilityPrefetchParams implements BaseModel use SdkParams; /** - * Identifier of the brand to prefetch. Provide exactly one of domain or email. + * Identifier of the target to prefetch. Provide exactly one of domain or email. * * @var IdentifierVariants $identifier */ @@ -43,7 +43,7 @@ final class UtilityPrefetchParams implements BaseModel public UtilityPrefetchDomainIdentifier|UtilityPrefetchEmailIdentifier $identifier; /** - * What to prefetch. Currently only 'brand' is supported. + * What to prefetch: 'brand' warms the brand data cache, 'styleguide' warms the styleguide cache. * * @var value-of $type */ @@ -110,7 +110,7 @@ public static function with( } /** - * Identifier of the brand to prefetch. Provide exactly one of domain or email. + * Identifier of the target to prefetch. Provide exactly one of domain or email. * * @param IdentifierShape $identifier */ @@ -124,7 +124,7 @@ public function withIdentifier( } /** - * What to prefetch. Currently only 'brand' is supported. + * What to prefetch: 'brand' warms the brand data cache, 'styleguide' warms the styleguide cache. * * @param Type|value-of $type */ diff --git a/src/Utility/UtilityPrefetchParams/Identifier.php b/src/Utility/UtilityPrefetchParams/Identifier.php index 1fdb5ab..8e90897 100644 --- a/src/Utility/UtilityPrefetchParams/Identifier.php +++ b/src/Utility/UtilityPrefetchParams/Identifier.php @@ -11,7 +11,7 @@ use ContextDev\Utility\UtilityPrefetchParams\Identifier\UtilityPrefetchEmailIdentifier; /** - * Identifier of the brand to prefetch. Provide exactly one of domain or email. + * Identifier of the target to prefetch. Provide exactly one of domain or email. * * @phpstan-import-type UtilityPrefetchDomainIdentifierShape from \ContextDev\Utility\UtilityPrefetchParams\Identifier\UtilityPrefetchDomainIdentifier * @phpstan-import-type UtilityPrefetchEmailIdentifierShape from \ContextDev\Utility\UtilityPrefetchParams\Identifier\UtilityPrefetchEmailIdentifier diff --git a/src/Utility/UtilityPrefetchParams/Identifier/UtilityPrefetchDomainIdentifier.php b/src/Utility/UtilityPrefetchParams/Identifier/UtilityPrefetchDomainIdentifier.php index 615f897..7bb39d4 100644 --- a/src/Utility/UtilityPrefetchParams/Identifier/UtilityPrefetchDomainIdentifier.php +++ b/src/Utility/UtilityPrefetchParams/Identifier/UtilityPrefetchDomainIdentifier.php @@ -9,7 +9,7 @@ use ContextDev\Core\Contracts\BaseModel; /** - * Prefetch brand data by domain. + * Prefetch by domain. * * @phpstan-type UtilityPrefetchDomainIdentifierShape = array{domain: string} */ @@ -19,7 +19,7 @@ final class UtilityPrefetchDomainIdentifier implements BaseModel use SdkModel; /** - * Domain name to prefetch brand data for. + * Domain name to prefetch data for. */ #[Required] public string $domain; @@ -58,7 +58,7 @@ public static function with(string $domain): self } /** - * Domain name to prefetch brand data for. + * Domain name to prefetch data for. */ public function withDomain(string $domain): self { diff --git a/src/Utility/UtilityPrefetchParams/Identifier/UtilityPrefetchEmailIdentifier.php b/src/Utility/UtilityPrefetchParams/Identifier/UtilityPrefetchEmailIdentifier.php index 44142c3..8ea8534 100644 --- a/src/Utility/UtilityPrefetchParams/Identifier/UtilityPrefetchEmailIdentifier.php +++ b/src/Utility/UtilityPrefetchParams/Identifier/UtilityPrefetchEmailIdentifier.php @@ -9,7 +9,7 @@ use ContextDev\Core\Contracts\BaseModel; /** - * Prefetch brand data by email. The domain will be extracted and validated. + * Prefetch by email. The domain will be extracted and validated. * * @phpstan-type UtilityPrefetchEmailIdentifierShape = array{email: string} */ @@ -19,7 +19,7 @@ final class UtilityPrefetchEmailIdentifier implements BaseModel use SdkModel; /** - * Email address to prefetch brand data for. The domain will be extracted from the email. Free email providers (gmail.com, yahoo.com, etc.) and disposable email addresses are not allowed. + * Email address to prefetch data for. The domain will be extracted from the email. Free email providers (gmail.com, yahoo.com, etc.) and disposable email addresses are not allowed. */ #[Required] public string $email; @@ -58,7 +58,7 @@ public static function with(string $email): self } /** - * Email address to prefetch brand data for. The domain will be extracted from the email. Free email providers (gmail.com, yahoo.com, etc.) and disposable email addresses are not allowed. + * Email address to prefetch data for. The domain will be extracted from the email. Free email providers (gmail.com, yahoo.com, etc.) and disposable email addresses are not allowed. */ public function withEmail(string $email): self { diff --git a/src/Utility/UtilityPrefetchParams/Type.php b/src/Utility/UtilityPrefetchParams/Type.php index 79bdffa..ed700b9 100644 --- a/src/Utility/UtilityPrefetchParams/Type.php +++ b/src/Utility/UtilityPrefetchParams/Type.php @@ -5,9 +5,11 @@ namespace ContextDev\Utility\UtilityPrefetchParams; /** - * What to prefetch. Currently only 'brand' is supported. + * What to prefetch: 'brand' warms the brand data cache, 'styleguide' warms the styleguide cache. */ enum Type: string { case BRAND = 'brand'; + + case STYLEGUIDE = 'styleguide'; } diff --git a/src/Utility/UtilityPrefetchResponse.php b/src/Utility/UtilityPrefetchResponse.php index f095c3d..a0f01e2 100644 --- a/src/Utility/UtilityPrefetchResponse.php +++ b/src/Utility/UtilityPrefetchResponse.php @@ -51,7 +51,7 @@ final class UtilityPrefetchResponse implements BaseModel public ?string $status; /** - * The type of prefetch that was queued, echoed from the request (currently always 'brand'). + * The type of prefetch that was queued, echoed from the request. * * @var value-of|null $type */ @@ -136,7 +136,7 @@ public function withStatus(string $status): self } /** - * The type of prefetch that was queued, echoed from the request (currently always 'brand'). + * The type of prefetch that was queued, echoed from the request. * * @param Type|value-of $type */ diff --git a/src/Utility/UtilityPrefetchResponse/Type.php b/src/Utility/UtilityPrefetchResponse/Type.php index b127055..024523c 100644 --- a/src/Utility/UtilityPrefetchResponse/Type.php +++ b/src/Utility/UtilityPrefetchResponse/Type.php @@ -5,9 +5,11 @@ namespace ContextDev\Utility\UtilityPrefetchResponse; /** - * The type of prefetch that was queued, echoed from the request (currently always 'brand'). + * The type of prefetch that was queued, echoed from the request. */ enum Type: string { case BRAND = 'brand'; + + case STYLEGUIDE = 'styleguide'; } diff --git a/src/Version.php b/src/Version.php index fc11686..220673a 100644 --- a/src/Version.php +++ b/src/Version.php @@ -5,5 +5,5 @@ namespace ContextDev; // x-release-please-start-version -const VERSION = '2.9.0'; +const VERSION = '2.10.0'; // x-release-please-end diff --git a/src/Web/WebWebCrawlMdResponse/Result/Metadata.php b/src/Web/WebWebCrawlMdResponse/Result/Metadata.php index 6da696c..26e7d76 100644 --- a/src/Web/WebWebCrawlMdResponse/Result/Metadata.php +++ b/src/Web/WebWebCrawlMdResponse/Result/Metadata.php @@ -11,6 +11,7 @@ use ContextDev\Core\Conversion\MapOf; use ContextDev\Web\WebWebCrawlMdResponse\Result\Metadata\AdditionalMeta; use ContextDev\Web\WebWebCrawlMdResponse\Result\Metadata\Alternate; +use ContextDev\Web\WebWebCrawlMdResponse\Result\Metadata\Heading; use ContextDev\Web\WebWebCrawlMdResponse\Result\Metadata\OpenGraph; use ContextDev\Web\WebWebCrawlMdResponse\Result\Metadata\Twitter; @@ -20,6 +21,7 @@ * @phpstan-import-type TwitterVariants from \ContextDev\Web\WebWebCrawlMdResponse\Result\Metadata\Twitter * @phpstan-import-type AdditionalMetaShape from \ContextDev\Web\WebWebCrawlMdResponse\Result\Metadata\AdditionalMeta * @phpstan-import-type AlternateShape from \ContextDev\Web\WebWebCrawlMdResponse\Result\Metadata\Alternate + * @phpstan-import-type HeadingShape from \ContextDev\Web\WebWebCrawlMdResponse\Result\Metadata\Heading * @phpstan-import-type OpenGraphShape from \ContextDev\Web\WebWebCrawlMdResponse\Result\Metadata\OpenGraph * @phpstan-import-type TwitterShape from \ContextDev\Web\WebWebCrawlMdResponse\Result\Metadata\Twitter * @@ -37,6 +39,7 @@ * canonicalURL?: string|null, * description?: string|null, * favicon?: string|null, + * headings?: list|null, * image?: string|null, * jsonLd?: list>|null, * keywords?: list|null, @@ -136,6 +139,14 @@ final class Metadata implements BaseModel #[Optional] public ?string $favicon; + /** + * Page headings (h1–h6) in document order, extracted from the unfiltered document. Capped at the first 500 headings. Omitted when the page has none. + * + * @var list|null $headings + */ + #[Optional(list: Heading::class)] + public ?array $headings; + /** * Primary resolved preview image from Open Graph, Twitter, or image metadata. */ @@ -245,6 +256,7 @@ public function __construct() * * @param array|null $additionalMeta * @param list|null $alternates + * @param list|null $headings * @param list>|null $jsonLd * @param list|null $keywords * @param array|null $openGraph @@ -264,6 +276,7 @@ public static function with( ?string $canonicalURL = null, ?string $description = null, ?string $favicon = null, + ?array $headings = null, ?string $image = null, ?array $jsonLd = null, ?array $keywords = null, @@ -291,6 +304,7 @@ public static function with( null !== $canonicalURL && $self['canonicalURL'] = $canonicalURL; null !== $description && $self['description'] = $description; null !== $favicon && $self['favicon'] = $favicon; + null !== $headings && $self['headings'] = $headings; null !== $image && $self['image'] = $image; null !== $jsonLd && $self['jsonLd'] = $jsonLd; null !== $keywords && $self['keywords'] = $keywords; @@ -452,6 +466,19 @@ public function withFavicon(string $favicon): self return $self; } + /** + * Page headings (h1–h6) in document order, extracted from the unfiltered document. Capped at the first 500 headings. Omitted when the page has none. + * + * @param list $headings + */ + public function withHeadings(array $headings): self + { + $self = clone $this; + $self['headings'] = $headings; + + return $self; + } + /** * Primary resolved preview image from Open Graph, Twitter, or image metadata. */ diff --git a/src/Web/WebWebCrawlMdResponse/Result/Metadata/Heading.php b/src/Web/WebWebCrawlMdResponse/Result/Metadata/Heading.php new file mode 100644 index 0000000..560f6f5 --- /dev/null +++ b/src/Web/WebWebCrawlMdResponse/Result/Metadata/Heading.php @@ -0,0 +1,86 @@ + */ + use SdkModel; + + /** + * Heading level, 1–6 (from h1–h6). + */ + #[Required] + public int $level; + + /** + * Heading text with whitespace collapsed, truncated to 1000 characters. + */ + #[Required] + public string $text; + + /** + * `new Heading()` is missing required properties by the API. + * + * To enforce required parameters use + * ``` + * Heading::with(level: ..., text: ...) + * ``` + * + * Otherwise ensure the following setters are called + * + * ``` + * (new Heading)->withLevel(...)->withText(...) + * ``` + */ + public function __construct() + { + $this->initialize(); + } + + /** + * Construct an instance from the required parameters. + * + * You must use named parameters to construct any parameters with a default value. + */ + public static function with(int $level, string $text): self + { + $self = new self; + + $self['level'] = $level; + $self['text'] = $text; + + return $self; + } + + /** + * Heading level, 1–6 (from h1–h6). + */ + public function withLevel(int $level): self + { + $self = clone $this; + $self['level'] = $level; + + return $self; + } + + /** + * Heading text with whitespace collapsed, truncated to 1000 characters. + */ + public function withText(string $text): self + { + $self = clone $this; + $self['text'] = $text; + + return $self; + } +} diff --git a/src/Web/WebWebScrapeHTMLResponse/Metadata.php b/src/Web/WebWebScrapeHTMLResponse/Metadata.php index 23b827a..13f3662 100644 --- a/src/Web/WebWebScrapeHTMLResponse/Metadata.php +++ b/src/Web/WebWebScrapeHTMLResponse/Metadata.php @@ -11,6 +11,7 @@ use ContextDev\Core\Conversion\MapOf; use ContextDev\Web\WebWebScrapeHTMLResponse\Metadata\AdditionalMeta; use ContextDev\Web\WebWebScrapeHTMLResponse\Metadata\Alternate; +use ContextDev\Web\WebWebScrapeHTMLResponse\Metadata\Heading; use ContextDev\Web\WebWebScrapeHTMLResponse\Metadata\OpenGraph; use ContextDev\Web\WebWebScrapeHTMLResponse\Metadata\Twitter; @@ -22,6 +23,7 @@ * @phpstan-import-type TwitterVariants from \ContextDev\Web\WebWebScrapeHTMLResponse\Metadata\Twitter * @phpstan-import-type AdditionalMetaShape from \ContextDev\Web\WebWebScrapeHTMLResponse\Metadata\AdditionalMeta * @phpstan-import-type AlternateShape from \ContextDev\Web\WebWebScrapeHTMLResponse\Metadata\Alternate + * @phpstan-import-type HeadingShape from \ContextDev\Web\WebWebScrapeHTMLResponse\Metadata\Heading * @phpstan-import-type OpenGraphShape from \ContextDev\Web\WebWebScrapeHTMLResponse\Metadata\OpenGraph * @phpstan-import-type TwitterShape from \ContextDev\Web\WebWebScrapeHTMLResponse\Metadata\Twitter * @@ -34,6 +36,7 @@ * canonicalURL?: string|null, * description?: string|null, * favicon?: string|null, + * headings?: list|null, * image?: string|null, * jsonLd?: list>|null, * keywords?: list|null, @@ -104,6 +107,14 @@ final class Metadata implements BaseModel #[Optional] public ?string $favicon; + /** + * Page headings (h1–h6) in document order, extracted from the unfiltered document. Capped at the first 500 headings. Omitted when the page has none. + * + * @var list|null $headings + */ + #[Optional(list: Heading::class)] + public ?array $headings; + /** * Primary resolved preview image from Open Graph, Twitter, or image metadata. */ @@ -204,6 +215,7 @@ public function __construct() * * @param array|null $additionalMeta * @param list|null $alternates + * @param list|null $headings * @param list>|null $jsonLd * @param list|null $keywords * @param array|null $openGraph @@ -218,6 +230,7 @@ public static function with( ?string $canonicalURL = null, ?string $description = null, ?string $favicon = null, + ?array $headings = null, ?string $image = null, ?array $jsonLd = null, ?array $keywords = null, @@ -241,6 +254,7 @@ public static function with( null !== $canonicalURL && $self['canonicalURL'] = $canonicalURL; null !== $description && $self['description'] = $description; null !== $favicon && $self['favicon'] = $favicon; + null !== $headings && $self['headings'] = $headings; null !== $image && $self['image'] = $image; null !== $jsonLd && $self['jsonLd'] = $jsonLd; null !== $keywords && $self['keywords'] = $keywords; @@ -348,6 +362,19 @@ public function withFavicon(string $favicon): self return $self; } + /** + * Page headings (h1–h6) in document order, extracted from the unfiltered document. Capped at the first 500 headings. Omitted when the page has none. + * + * @param list $headings + */ + public function withHeadings(array $headings): self + { + $self = clone $this; + $self['headings'] = $headings; + + return $self; + } + /** * Primary resolved preview image from Open Graph, Twitter, or image metadata. */ diff --git a/src/Web/WebWebScrapeHTMLResponse/Metadata/Heading.php b/src/Web/WebWebScrapeHTMLResponse/Metadata/Heading.php new file mode 100644 index 0000000..08aa555 --- /dev/null +++ b/src/Web/WebWebScrapeHTMLResponse/Metadata/Heading.php @@ -0,0 +1,86 @@ + */ + use SdkModel; + + /** + * Heading level, 1–6 (from h1–h6). + */ + #[Required] + public int $level; + + /** + * Heading text with whitespace collapsed, truncated to 1000 characters. + */ + #[Required] + public string $text; + + /** + * `new Heading()` is missing required properties by the API. + * + * To enforce required parameters use + * ``` + * Heading::with(level: ..., text: ...) + * ``` + * + * Otherwise ensure the following setters are called + * + * ``` + * (new Heading)->withLevel(...)->withText(...) + * ``` + */ + public function __construct() + { + $this->initialize(); + } + + /** + * Construct an instance from the required parameters. + * + * You must use named parameters to construct any parameters with a default value. + */ + public static function with(int $level, string $text): self + { + $self = new self; + + $self['level'] = $level; + $self['text'] = $text; + + return $self; + } + + /** + * Heading level, 1–6 (from h1–h6). + */ + public function withLevel(int $level): self + { + $self = clone $this; + $self['level'] = $level; + + return $self; + } + + /** + * Heading text with whitespace collapsed, truncated to 1000 characters. + */ + public function withText(string $text): self + { + $self = clone $this; + $self['text'] = $text; + + return $self; + } +} diff --git a/src/Web/WebWebScrapeMdParams.php b/src/Web/WebWebScrapeMdParams.php index 99db851..954637b 100644 --- a/src/Web/WebWebScrapeMdParams.php +++ b/src/Web/WebWebScrapeMdParams.php @@ -13,6 +13,7 @@ use ContextDev\Web\WebWebScrapeMdParams\Country; use ContextDev\Web\WebWebScrapeMdParams\IncludeFrames; use ContextDev\Web\WebWebScrapeMdParams\IncludeFrames\UnionMember1; +use ContextDev\Web\WebWebScrapeMdParams\IncludeHTML; use ContextDev\Web\WebWebScrapeMdParams\IncludeImages; use ContextDev\Web\WebWebScrapeMdParams\IncludeLinks; use ContextDev\Web\WebWebScrapeMdParams\Pdf; @@ -33,7 +34,7 @@ * | HTTP status | Billed? | Meaning | * | --- | --- | --- | * | 200 | Yes — 1 credit, or 2 credits with actions | Successful scrape, including a zero-length result when includeSelectors matched nothing | - * | 400 | No | Invalid input, skipped PDF, or the page could not be scraped | + * | 400 | No | Invalid input, skipped PDF, or the page could not be scraped. error_code WEBSITE_BLOCKED specifically means the site answered with an anti-bot challenge, CAPTCHA wall, or login shell instead of the page (even when the site returned HTTP 200) — retrying later or from another country sometimes succeeds | * | 401 / 403 | No | Invalid/disabled key, insufficient permissions, or credits exhausted; inspect error_code | * | 404 | No | Target page returned or fingerprinted as not found | * | 408 | No | Request timed out | @@ -46,6 +47,7 @@ * * @phpstan-import-type ActionVariants from \ContextDev\Web\WebWebScrapeMdParams\Action * @phpstan-import-type IncludeFramesVariants from \ContextDev\Web\WebWebScrapeMdParams\IncludeFrames + * @phpstan-import-type IncludeHTMLVariants from \ContextDev\Web\WebWebScrapeMdParams\IncludeHTML * @phpstan-import-type IncludeImagesVariants from \ContextDev\Web\WebWebScrapeMdParams\IncludeImages * @phpstan-import-type IncludeLinksVariants from \ContextDev\Web\WebWebScrapeMdParams\IncludeLinks * @phpstan-import-type SettleAnimationsVariants from \ContextDev\Web\WebWebScrapeMdParams\SettleAnimations @@ -53,6 +55,7 @@ * @phpstan-import-type UseMainContentOnlyVariants from \ContextDev\Web\WebWebScrapeMdParams\UseMainContentOnly * @phpstan-import-type ActionShape from \ContextDev\Web\WebWebScrapeMdParams\Action * @phpstan-import-type IncludeFramesShape from \ContextDev\Web\WebWebScrapeMdParams\IncludeFrames + * @phpstan-import-type IncludeHTMLShape from \ContextDev\Web\WebWebScrapeMdParams\IncludeHTML * @phpstan-import-type IncludeImagesShape from \ContextDev\Web\WebWebScrapeMdParams\IncludeImages * @phpstan-import-type IncludeLinksShape from \ContextDev\Web\WebWebScrapeMdParams\IncludeLinks * @phpstan-import-type PdfShape from \ContextDev\Web\WebWebScrapeMdParams\Pdf @@ -67,6 +70,7 @@ * excludeSelectors?: list|null, * headers?: array|null, * includeFrames?: IncludeFramesShape|null, + * includeHTML?: IncludeHTMLShape|null, * includeImages?: IncludeImagesShape|null, * includeLinks?: IncludeLinksShape|null, * includeSelectors?: list|null, @@ -133,6 +137,14 @@ final class WebWebScrapeMdParams implements BaseModel #[Optional(union: IncludeFrames::class)] public bool|string|null $includeFrames; + /** + * When true, the response also includes an `html` field with the page HTML the Markdown was converted from — the same body the Scrape HTML endpoint returns for the equivalent request. + * + * @var IncludeHTMLVariants|null $includeHTML + */ + #[Optional(union: IncludeHTML::class)] + public bool|string|null $includeHTML; + /** * Include image references in Markdown output. * @@ -250,6 +262,7 @@ public function __construct() * @param list|null $excludeSelectors * @param array|null $headers * @param IncludeFramesShape|null $includeFrames + * @param IncludeHTMLShape|null $includeHTML * @param IncludeImagesShape|null $includeImages * @param IncludeLinksShape|null $includeLinks * @param list|null $includeSelectors @@ -267,6 +280,7 @@ public static function with( ?array $excludeSelectors = null, ?array $headers = null, bool|UnionMember1|string|null $includeFrames = null, + bool|IncludeHTML\UnionMember1|string|null $includeHTML = null, bool|IncludeImages\UnionMember1|string|null $includeImages = null, bool|IncludeLinks\UnionMember1|string|null $includeLinks = null, ?array $includeSelectors = null, @@ -289,6 +303,7 @@ public static function with( null !== $excludeSelectors && $self['excludeSelectors'] = $excludeSelectors; null !== $headers && $self['headers'] = $headers; null !== $includeFrames && $self['includeFrames'] = $includeFrames; + null !== $includeHTML && $self['includeHTML'] = $includeHTML; null !== $includeImages && $self['includeImages'] = $includeImages; null !== $includeLinks && $self['includeLinks'] = $includeLinks; null !== $includeSelectors && $self['includeSelectors'] = $includeSelectors; @@ -382,6 +397,20 @@ public function withIncludeFrames( return $self; } + /** + * When true, the response also includes an `html` field with the page HTML the Markdown was converted from — the same body the Scrape HTML endpoint returns for the equivalent request. + * + * @param IncludeHTMLShape $includeHTML + */ + public function withIncludeHTML( + bool|IncludeHTML\UnionMember1|string $includeHTML, + ): self { + $self = clone $this; + $self['includeHTML'] = $includeHTML; + + return $self; + } + /** * Include image references in Markdown output. * diff --git a/src/Web/WebWebScrapeMdParams/IncludeHTML.php b/src/Web/WebWebScrapeMdParams/IncludeHTML.php new file mode 100644 index 0000000..cf7d394 --- /dev/null +++ b/src/Web/WebWebScrapeMdParams/IncludeHTML.php @@ -0,0 +1,29 @@ + + * @phpstan-type IncludeHTMLShape = IncludeHTMLVariants + */ +final class IncludeHTML implements ConverterSource +{ + use SdkUnion; + + /** + * @return list|array + */ + public static function variants(): array + { + return ['bool', UnionMember1::class]; + } +} diff --git a/src/Web/WebWebScrapeMdParams/IncludeHTML/UnionMember1.php b/src/Web/WebWebScrapeMdParams/IncludeHTML/UnionMember1.php new file mode 100644 index 0000000..764cb0d --- /dev/null +++ b/src/Web/WebWebScrapeMdParams/IncludeHTML/UnionMember1.php @@ -0,0 +1,12 @@ +|null, * actionsHTMLStale?: bool|null, + * html?: string|null, * keyMetadata?: null|KeyMetadata|KeyMetadataShape, * } */ @@ -77,6 +78,12 @@ final class WebWebScrapeMdResponse implements BaseModel #[Optional('actionsHtmlStale')] public ?bool $actionsHTMLStale; + /** + * Only present when includeHTML=true: the page HTML the Markdown was converted from — the same body the Scrape HTML endpoint returns for the equivalent request. + */ + #[Optional] + public ?string $html; + /** * Metadata about the API key used for the request. Included in every response whenever a valid API key is provided, even when the response status is not 200. */ @@ -126,6 +133,7 @@ public static function with( string $url, ?array $actionsApplied = null, ?bool $actionsHTMLStale = null, + ?string $html = null, KeyMetadata|array|null $keyMetadata = null, ): self { $self = new self; @@ -138,6 +146,7 @@ public static function with( null !== $actionsApplied && $self['actionsApplied'] = $actionsApplied; null !== $actionsHTMLStale && $self['actionsHTMLStale'] = $actionsHTMLStale; + null !== $html && $self['html'] = $html; null !== $keyMetadata && $self['keyMetadata'] = $keyMetadata; return $self; @@ -224,6 +233,17 @@ public function withActionsHTMLStale(bool $actionsHTMLStale): self return $self; } + /** + * Only present when includeHTML=true: the page HTML the Markdown was converted from — the same body the Scrape HTML endpoint returns for the equivalent request. + */ + public function withHTML(string $html): self + { + $self = clone $this; + $self['html'] = $html; + + return $self; + } + /** * Metadata about the API key used for the request. Included in every response whenever a valid API key is provided, even when the response status is not 200. * diff --git a/src/Web/WebWebScrapeMdResponse/Metadata.php b/src/Web/WebWebScrapeMdResponse/Metadata.php index b7e67b7..2d33288 100644 --- a/src/Web/WebWebScrapeMdResponse/Metadata.php +++ b/src/Web/WebWebScrapeMdResponse/Metadata.php @@ -11,6 +11,7 @@ use ContextDev\Core\Conversion\MapOf; use ContextDev\Web\WebWebScrapeMdResponse\Metadata\AdditionalMeta; use ContextDev\Web\WebWebScrapeMdResponse\Metadata\Alternate; +use ContextDev\Web\WebWebScrapeMdResponse\Metadata\Heading; use ContextDev\Web\WebWebScrapeMdResponse\Metadata\OpenGraph; use ContextDev\Web\WebWebScrapeMdResponse\Metadata\Twitter; @@ -22,6 +23,7 @@ * @phpstan-import-type TwitterVariants from \ContextDev\Web\WebWebScrapeMdResponse\Metadata\Twitter * @phpstan-import-type AdditionalMetaShape from \ContextDev\Web\WebWebScrapeMdResponse\Metadata\AdditionalMeta * @phpstan-import-type AlternateShape from \ContextDev\Web\WebWebScrapeMdResponse\Metadata\Alternate + * @phpstan-import-type HeadingShape from \ContextDev\Web\WebWebScrapeMdResponse\Metadata\Heading * @phpstan-import-type OpenGraphShape from \ContextDev\Web\WebWebScrapeMdResponse\Metadata\OpenGraph * @phpstan-import-type TwitterShape from \ContextDev\Web\WebWebScrapeMdResponse\Metadata\Twitter * @@ -34,6 +36,7 @@ * canonicalURL?: string|null, * description?: string|null, * favicon?: string|null, + * headings?: list|null, * image?: string|null, * jsonLd?: list>|null, * keywords?: list|null, @@ -104,6 +107,14 @@ final class Metadata implements BaseModel #[Optional] public ?string $favicon; + /** + * Page headings (h1–h6) in document order, extracted from the unfiltered document. Capped at the first 500 headings. Omitted when the page has none. + * + * @var list|null $headings + */ + #[Optional(list: Heading::class)] + public ?array $headings; + /** * Primary resolved preview image from Open Graph, Twitter, or image metadata. */ @@ -204,6 +215,7 @@ public function __construct() * * @param array|null $additionalMeta * @param list|null $alternates + * @param list|null $headings * @param list>|null $jsonLd * @param list|null $keywords * @param array|null $openGraph @@ -218,6 +230,7 @@ public static function with( ?string $canonicalURL = null, ?string $description = null, ?string $favicon = null, + ?array $headings = null, ?string $image = null, ?array $jsonLd = null, ?array $keywords = null, @@ -241,6 +254,7 @@ public static function with( null !== $canonicalURL && $self['canonicalURL'] = $canonicalURL; null !== $description && $self['description'] = $description; null !== $favicon && $self['favicon'] = $favicon; + null !== $headings && $self['headings'] = $headings; null !== $image && $self['image'] = $image; null !== $jsonLd && $self['jsonLd'] = $jsonLd; null !== $keywords && $self['keywords'] = $keywords; @@ -348,6 +362,19 @@ public function withFavicon(string $favicon): self return $self; } + /** + * Page headings (h1–h6) in document order, extracted from the unfiltered document. Capped at the first 500 headings. Omitted when the page has none. + * + * @param list $headings + */ + public function withHeadings(array $headings): self + { + $self = clone $this; + $self['headings'] = $headings; + + return $self; + } + /** * Primary resolved preview image from Open Graph, Twitter, or image metadata. */ diff --git a/src/Web/WebWebScrapeMdResponse/Metadata/Heading.php b/src/Web/WebWebScrapeMdResponse/Metadata/Heading.php new file mode 100644 index 0000000..0d6516c --- /dev/null +++ b/src/Web/WebWebScrapeMdResponse/Metadata/Heading.php @@ -0,0 +1,86 @@ + */ + use SdkModel; + + /** + * Heading level, 1–6 (from h1–h6). + */ + #[Required] + public int $level; + + /** + * Heading text with whitespace collapsed, truncated to 1000 characters. + */ + #[Required] + public string $text; + + /** + * `new Heading()` is missing required properties by the API. + * + * To enforce required parameters use + * ``` + * Heading::with(level: ..., text: ...) + * ``` + * + * Otherwise ensure the following setters are called + * + * ``` + * (new Heading)->withLevel(...)->withText(...) + * ``` + */ + public function __construct() + { + $this->initialize(); + } + + /** + * Construct an instance from the required parameters. + * + * You must use named parameters to construct any parameters with a default value. + */ + public static function with(int $level, string $text): self + { + $self = new self; + + $self['level'] = $level; + $self['text'] = $text; + + return $self; + } + + /** + * Heading level, 1–6 (from h1–h6). + */ + public function withLevel(int $level): self + { + $self = clone $this; + $self['level'] = $level; + + return $self; + } + + /** + * Heading text with whitespace collapsed, truncated to 1000 characters. + */ + public function withText(string $text): self + { + $self = clone $this; + $self['text'] = $text; + + return $self; + } +} diff --git a/tests/Services/BatchTest.php b/tests/Services/BatchTest.php index e215546..d9fa5d1 100644 --- a/tests/Services/BatchTest.php +++ b/tests/Services/BatchTest.php @@ -148,6 +148,7 @@ public function testSubmitWithOptionalParams(): void 'options' => [ 'country' => 'de', 'excludeSelectors' => ['x'], + 'includeHTML' => true, 'includeImages' => true, 'includeLinks' => true, 'includeSelectors' => ['x'], diff --git a/tests/Services/WebTest.php b/tests/Services/WebTest.php index eb41d56..2b048c1 100644 --- a/tests/Services/WebTest.php +++ b/tests/Services/WebTest.php @@ -369,6 +369,7 @@ public function testWebScrapeMdWithOptionalParams(): void excludeSelectors: ['x'], headers: ['foo' => 'J!'], includeFrames: 'true', + includeHTML: 'true', includeImages: 'true', includeLinks: 'true', includeSelectors: ['x'],