From d816c33a50931ff5902c33657ba6e887f4316bb4 Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Fri, 14 Aug 2026 18:57:39 -0700 Subject: [PATCH 01/10] feat(ocr): adopt DoclingDocument as the layout contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the two disconnected OCR paths with one parser-agnostic seam that persists layout with real lineage: every item carries a ProvenanceItem (page_no + bbox + charspan), so an extraction record can cite its source region. Parsers are swappable behind `contexts/ocr/parsers`: - pdf_inspector (MIT, zero deps, required) — the always-available default. Normalizes its three quirks at the boundary: inconsistent page indexing across its own API, bottom-left coordinates, and no page dimensions (read from MediaBox with pikepdf). Structure-tree roles joined on (page, mcid) yield real heading levels and table regions on tagged PDFs. - pymupdf (AGPL, optional `pymupdf` extra) — higher fidelity; the only one that yields per-cell table geometry. Kept optional so the Apache-2.0 distribution never requires an AGPL dependency. The three tolerant scrapers become single-item appenders driven by one ordered pass, which fixes the duplication (text inside tables was emitted twice) and the lost reading order. Tables and pictures are added first so containment dedup can run, then body order is restored geometrically. Layout persists as canonical DoclingDocument JSON in S3 plus a columnar Parquet projection (one row per (DocItem, ProvenanceItem), so multi-page items expand naturally). Not documents.metadata_ — that column is returned in full by every GET /documents. Adds GET /documents/{id}/layout returning a flat projection rather than the raw DoclingDocument, whose 73 recursive $defs would make OpenAPI and hand-written frontend types unusable, plus the matching frontend entities and repository method. Deletes, rather than migrates: the marker extra (marker-pdf, torch, torchvision, transformers), async_marker_layout_job, the deprecated Unstructured-shaped chunks.py, and the unreferenced GPU_QUEUE. Test fixtures are generated by tests/fixtures/pdf/generate.py using only pikepdf, including a tagged variant that exercises the mcid path; the blanket **/*.pdf ignore is narrowed so they can be committed. --- .gitignore | 2 + .../entities/document/DocumentLayout.ts | 136 ++ .../repositories/DocumentRepository.test.ts | 208 +++ .../repositories/DocumentRepository.ts | 97 ++ extralit-server/pyproject.toml | 59 +- .../api/handlers/v1/documents.py | 65 + .../api/handlers/v1/workflows.py | 1 + .../api/schemas/v1/document/chunks.py | 108 -- .../api/schemas/v1/document/layout.py | 63 + .../api/schemas/v1/document/metadata.py | 20 + .../api/schemas/v1/workflows.py | 3 + .../src/extralit_server/contexts/ocr/arrow.py | 129 ++ .../contexts/ocr/docling_builder.py | 182 +++ .../extralit_server/contexts/ocr/figures.py | 208 +-- .../contexts/ocr/parsers/__init__.py | 64 + .../contexts/ocr/parsers/pdf_inspector.py | 270 ++++ .../contexts/ocr/parsers/pymupdf.py | 170 ++ .../contexts/ocr/projection.py | 87 + .../extralit_server/contexts/ocr/storage.py | 102 ++ .../extralit_server/contexts/ocr/tables.py | 227 +-- .../src/extralit_server/contexts/ocr/text.py | 192 +-- .../src/extralit_server/jobs/ocr_jobs.py | 320 ++-- .../src/extralit_server/jobs/preload.py | 2 +- .../src/extralit_server/jobs/queues.py | 1 - .../extralit_server/workflows/documents.py | 29 +- .../tests/fixtures/pdf/generate.py | 170 ++ extralit-server/tests/fixtures/pdf/sample.pdf | Bin 0 -> 1177 bytes .../tests/fixtures/pdf/sample_tagged.pdf | Bin 0 -> 2055 bytes .../api/handlers/v1/test_document_layout.py | 200 +++ .../tests/unit/contexts/ocr/__init__.py | 0 .../tests/unit/contexts/ocr/test_arrow.py | 212 +++ .../unit/contexts/ocr/test_docling_builder.py | 307 ++++ .../contexts/ocr/test_pdf_inspector_parser.py | 199 +++ .../unit/contexts/ocr/test_pymupdf_parser.py | 157 ++ extralit-server/uv.lock | 1418 +++-------------- 35 files changed, 3283 insertions(+), 2125 deletions(-) create mode 100644 extralit-frontend/v1/domain/entities/document/DocumentLayout.ts create mode 100644 extralit-frontend/v1/infrastructure/repositories/DocumentRepository.test.ts delete mode 100644 extralit-server/src/extralit_server/api/schemas/v1/document/chunks.py create mode 100644 extralit-server/src/extralit_server/api/schemas/v1/document/layout.py create mode 100644 extralit-server/src/extralit_server/contexts/ocr/arrow.py create mode 100644 extralit-server/src/extralit_server/contexts/ocr/docling_builder.py create mode 100644 extralit-server/src/extralit_server/contexts/ocr/parsers/__init__.py create mode 100644 extralit-server/src/extralit_server/contexts/ocr/parsers/pdf_inspector.py create mode 100644 extralit-server/src/extralit_server/contexts/ocr/parsers/pymupdf.py create mode 100644 extralit-server/src/extralit_server/contexts/ocr/projection.py create mode 100644 extralit-server/src/extralit_server/contexts/ocr/storage.py create mode 100644 extralit-server/tests/fixtures/pdf/generate.py create mode 100644 extralit-server/tests/fixtures/pdf/sample.pdf create mode 100644 extralit-server/tests/fixtures/pdf/sample_tagged.pdf create mode 100644 extralit-server/tests/unit/api/handlers/v1/test_document_layout.py create mode 100644 extralit-server/tests/unit/contexts/ocr/__init__.py create mode 100644 extralit-server/tests/unit/contexts/ocr/test_arrow.py create mode 100644 extralit-server/tests/unit/contexts/ocr/test_docling_builder.py create mode 100644 extralit-server/tests/unit/contexts/ocr/test_pdf_inspector_parser.py create mode 100644 extralit-server/tests/unit/contexts/ocr/test_pymupdf_parser.py diff --git a/.gitignore b/.gitignore index c3ff35bc8..d502620ad 100644 --- a/.gitignore +++ b/.gitignore @@ -165,6 +165,8 @@ extralit/site # Development files **/*.db **/*.pdf +# Layout parser test fixtures are committed; regenerate with tests/fixtures/pdf/generate.py +!extralit-server/tests/fixtures/pdf/*.pdf output/ .output/ .nuxtrc diff --git a/extralit-frontend/v1/domain/entities/document/DocumentLayout.ts b/extralit-frontend/v1/domain/entities/document/DocumentLayout.ts new file mode 100644 index 000000000..075c430e3 --- /dev/null +++ b/extralit-frontend/v1/domain/entities/document/DocumentLayout.ts @@ -0,0 +1,136 @@ +/** + * Extracted document layout, mirroring the server's `DoclingDocument` projection. + * + * Every bounding box is in page points with a top-left origin, relative to the `LayoutPage` + * of the same `pageNo` — a viewer must scale by its own rendered page size, never assume 72dpi. + */ + +export interface Rect { + left: number; + top: number; + width: number; + height: number; +} + +export class BoundingBox { + constructor( + public readonly l: number, + public readonly t: number, + public readonly r: number, + public readonly b: number, + public readonly coordOrigin: string = "TOPLEFT" + ) {} + + get width(): number { + return this.r - this.l; + } + + get height(): number { + return this.b - this.t; + } + + /** Scale into the coordinate space of a rendered page of the given size. */ + toRect(pageWidth: number, pageHeight: number, renderedWidth?: number, renderedHeight?: number): Rect { + const scaleX = (renderedWidth ?? pageWidth) / pageWidth; + const scaleY = (renderedHeight ?? pageHeight) / pageHeight; + + return { + left: this.l * scaleX, + top: this.t * scaleY, + width: this.width * scaleX, + height: this.height * scaleY, + }; + } + + /** Fractions of the page, for overlays that position with percentages. */ + toRelativeRect(pageWidth: number, pageHeight: number): Rect { + return { + left: this.l / pageWidth, + top: this.t / pageHeight, + width: this.width / pageWidth, + height: this.height / pageHeight, + }; + } +} + +export class Provenance { + constructor( + public readonly pageNo: number, + public readonly bbox: BoundingBox, + /** Item-local character span — an offset into this item's own text, not the document. */ + public readonly charspan: [number, number] + ) {} +} + +export class LayoutItem { + constructor( + /** Citation anchor, e.g. `#/texts/12`. Stable for the lifetime of the stored layout. */ + public readonly selfRef: string, + public readonly label: string, + public readonly readingOrder: number, + public readonly prov: Provenance[] = [], + public readonly parentRef: string | null = null, + public readonly contentLayer: string | null = null, + public readonly level: number | null = null, + public readonly text: string | null = null, + public readonly html: string | null = null + ) {} + + /** Every page this item touches; more than one when it spans a page break. */ + get pageNumbers(): number[] { + return [...new Set(this.prov.map((p) => p.pageNo))].sort((a, b) => a - b); + } + + get isTable(): boolean { + return this.label === "table"; + } + + get isPicture(): boolean { + return this.label === "picture"; + } + + get isHeading(): boolean { + return this.label === "section_header" || this.label === "title"; + } + + provenanceOnPage(pageNo: number): Provenance[] { + return this.prov.filter((p) => p.pageNo === pageNo); + } +} + +export class LayoutPage { + constructor( + public readonly pageNo: number, + public readonly width: number, + public readonly height: number + ) {} +} + +export class DocumentLayout { + constructor( + public readonly documentId: string, + public readonly doclingVersion: string, + public readonly pages: LayoutPage[] = [], + public readonly items: LayoutItem[] = [] + ) {} + + get numPages(): number { + return this.pages.length; + } + + get numItems(): number { + return this.items.length; + } + + page(pageNo: number): LayoutPage | undefined { + return this.pages.find((p) => p.pageNo === pageNo); + } + + itemsOnPage(pageNo: number): LayoutItem[] { + return this.items.filter((item) => item.pageNumbers.includes(pageNo)); + } + + itemByRef(selfRef: string): LayoutItem | undefined { + return this.items.find((item) => item.selfRef === selfRef); + } +} diff --git a/extralit-frontend/v1/infrastructure/repositories/DocumentRepository.test.ts b/extralit-frontend/v1/infrastructure/repositories/DocumentRepository.test.ts new file mode 100644 index 000000000..bf4dfcbb9 --- /dev/null +++ b/extralit-frontend/v1/infrastructure/repositories/DocumentRepository.test.ts @@ -0,0 +1,208 @@ +import { describe, expect, it, vi } from "vitest"; +import type { AxiosInstance } from "axios"; +import { DocumentRepository } from "./DocumentRepository"; + +const axiosMock = (getImpl: (url: string) => unknown) => + ({ get: vi.fn(async (url: string) => ({ data: getImpl(url) })) }) as unknown as AxiosInstance; + +const rejectingAxiosMock = () => + ({ get: vi.fn(async () => Promise.reject(new Error("boom"))) }) as unknown as AxiosInstance; + +const BACKEND_LAYOUT = { + document_id: "d-1", + docling_version: "1.10.0", + num_items: 3, + num_pages: 2, + pages: [ + { page_no: 1, width: 612, height: 792 }, + { page_no: 2, width: 612, height: 792 }, + ], + items: [ + { + self_ref: "#/texts/0", + parent_ref: "#/body", + label: "section_header", + content_layer: "body", + level: 2, + reading_order: 0, + text: "Methods", + html: null, + prov: [ + { + page_no: 1, + bbox: { l: 72, t: 54, r: 300, b: 72, coord_origin: "TOPLEFT" }, + charspan: [0, 7], + }, + ], + }, + { + self_ref: "#/tables/0", + parent_ref: "#/body", + label: "table", + content_layer: "body", + level: null, + reading_order: 1, + text: null, + html: "
a
", + // A table that runs across a page break carries one entry per page. + prov: [ + { page_no: 1, bbox: { l: 72, t: 600, r: 372, b: 792, coord_origin: "TOPLEFT" }, charspan: [0, 0] }, + { page_no: 2, bbox: { l: 72, t: 0, r: 372, b: 200, coord_origin: "TOPLEFT" }, charspan: [0, 0] }, + ], + }, + { + self_ref: "#/pictures/0", + parent_ref: "#/body", + label: "picture", + content_layer: "body", + level: null, + reading_order: 2, + text: null, + html: null, + prov: [{ page_no: 2, bbox: { l: 72, t: 300, r: 192, b: 390, coord_origin: "TOPLEFT" }, charspan: [0, 0] }], + }, + ], +}; + +describe("DocumentRepository", () => { + describe("getDocumentLayout", () => { + it("fetches the layout and maps it to domain entities", async () => { + const axios = axiosMock(() => BACKEND_LAYOUT); + const repository = new DocumentRepository(axios); + + const layout = await repository.getDocumentLayout("d-1"); + + expect(axios.get).toHaveBeenCalledWith("/v1/documents/d-1/layout", { params: {} }); + expect(layout.documentId).toBe("d-1"); + expect(layout.doclingVersion).toBe("1.10.0"); + expect(layout.numPages).toBe(2); + expect(layout.numItems).toBe(3); + }); + + it("maps snake_case provenance onto the camelCase domain shape", async () => { + const repository = new DocumentRepository(axiosMock(() => BACKEND_LAYOUT)); + + const layout = await repository.getDocumentLayout("d-1"); + const heading = layout.itemByRef("#/texts/0"); + + expect(heading.selfRef).toBe("#/texts/0"); + expect(heading.parentRef).toBe("#/body"); + expect(heading.readingOrder).toBe(0); + expect(heading.contentLayer).toBe("body"); + expect(heading.level).toBe(2); + expect(heading.prov[0].pageNo).toBe(1); + expect(heading.prov[0].charspan).toEqual([0, 7]); + }); + + it("passes page and label filters as query params", async () => { + const axios = axiosMock(() => BACKEND_LAYOUT); + const repository = new DocumentRepository(axios); + + await repository.getDocumentLayout("d-1", { pages: [2], labels: ["table"] }); + + expect(axios.get).toHaveBeenCalledWith("/v1/documents/d-1/layout", { + params: { pages: [2], labels: ["table"] }, + }); + }); + + it("omits empty filters rather than sending empty arrays", async () => { + const axios = axiosMock(() => BACKEND_LAYOUT); + const repository = new DocumentRepository(axios); + + await repository.getDocumentLayout("d-1", { pages: [], labels: [] }); + + expect(axios.get).toHaveBeenCalledWith("/v1/documents/d-1/layout", { params: {} }); + }); + + it("throws a typed error when the request fails", async () => { + const repository = new DocumentRepository(rejectingAxiosMock()); + + await expect(repository.getDocumentLayout("d-1")).rejects.toEqual({ + response: "ERROR_FETCHING_LAYOUT", + }); + }); + }); + + describe("DocumentLayout entity", () => { + const layoutOf = async () => new DocumentRepository(axiosMock(() => BACKEND_LAYOUT)).getDocumentLayout("d-1"); + + it("reports every page an item touches", async () => { + const layout = await layoutOf(); + + expect(layout.itemByRef("#/tables/0").pageNumbers).toEqual([1, 2]); + expect(layout.itemByRef("#/pictures/0").pageNumbers).toEqual([2]); + }); + + it("returns the items that appear on a page", async () => { + const layout = await layoutOf(); + + expect(layout.itemsOnPage(1).map((i) => i.selfRef)).toEqual(["#/texts/0", "#/tables/0"]); + expect(layout.itemsOnPage(2).map((i) => i.selfRef)).toEqual(["#/tables/0", "#/pictures/0"]); + }); + + it("returns only the provenance on the requested page", async () => { + const layout = await layoutOf(); + + const onPageTwo = layout.itemByRef("#/tables/0").provenanceOnPage(2); + + expect(onPageTwo).toHaveLength(1); + expect(onPageTwo[0].bbox.t).toBe(0); + }); + + it("classifies items by label", async () => { + const layout = await layoutOf(); + + expect(layout.itemByRef("#/tables/0").isTable).toBe(true); + expect(layout.itemByRef("#/pictures/0").isPicture).toBe(true); + expect(layout.itemByRef("#/texts/0").isHeading).toBe(true); + }); + + it("exposes page geometry by page number", async () => { + const layout = await layoutOf(); + + expect(layout.page(1).height).toBe(792); + expect(layout.page(99)).toBeUndefined(); + }); + }); + + describe("BoundingBox geometry", () => { + const bboxOf = async () => + (await new DocumentRepository(axiosMock(() => BACKEND_LAYOUT)).getDocumentLayout("d-1")).itemByRef("#/texts/0") + .prov[0].bbox; + + it("computes width and height from the edges", async () => { + const bbox = await bboxOf(); + + expect(bbox.width).toBe(228); + expect(bbox.height).toBe(18); + }); + + it("returns page-point coordinates when no rendered size is given", async () => { + const bbox = await bboxOf(); + + expect(bbox.toRect(612, 792)).toEqual({ left: 72, top: 54, width: 228, height: 18 }); + }); + + it("scales into the rendered page size for an overlay", async () => { + const bbox = await bboxOf(); + + // A page rendered at double scale doubles every coordinate. + expect(bbox.toRect(612, 792, 1224, 1584)).toEqual({ left: 144, top: 108, width: 456, height: 36 }); + }); + + it("expresses the box as fractions of the page", async () => { + const bbox = await bboxOf(); + const rect = bbox.toRelativeRect(612, 792); + + expect(rect.left).toBeCloseTo(72 / 612); + expect(rect.top).toBeCloseTo(54 / 792); + expect(rect.width).toBeCloseTo(228 / 612); + }); + + it("defaults the coordinate origin to top-left", async () => { + const bbox = await bboxOf(); + + expect(bbox.coordOrigin).toBe("TOPLEFT"); + }); + }); +}); diff --git a/extralit-frontend/v1/infrastructure/repositories/DocumentRepository.ts b/extralit-frontend/v1/infrastructure/repositories/DocumentRepository.ts index 276d606f3..4dce1bc52 100644 --- a/extralit-frontend/v1/infrastructure/repositories/DocumentRepository.ts +++ b/extralit-frontend/v1/infrastructure/repositories/DocumentRepository.ts @@ -1,12 +1,90 @@ import type { AxiosInstance } from "axios"; import { Document, Segment, type Segments } from "@/v1/domain/entities/document/Document"; +import { + BoundingBox, + DocumentLayout, + LayoutItem, + LayoutPage, + Provenance, +} from "@/v1/domain/entities/document/DocumentLayout"; const DOCUMENT_API_ERRORS = { ERROR_FETCHING_DOCUMENT: "ERROR_FETCHING_DOCUMENT", ERROR_LISTING_DOCUMENTS: "ERROR_LISTING_DOCUMENTS", ERROR_FETCHING_SEGMENTS: "ERROR_FETCHING_SEGMENTS", + ERROR_FETCHING_LAYOUT: "ERROR_FETCHING_LAYOUT", }; +interface BackendBoundingBox { + l: number; + t: number; + r: number; + b: number; + coord_origin: string; +} + +interface BackendProvenance { + page_no: number; + bbox: BackendBoundingBox; + charspan: [number, number]; +} + +interface BackendLayoutItem { + self_ref: string; + parent_ref: string | null; + label: string; + content_layer: string | null; + level: number | null; + reading_order: number; + text: string | null; + html: string | null; + prov: BackendProvenance[]; +} + +interface BackendLayoutPage { + page_no: number; + width: number; + height: number; +} + +interface BackendDocumentLayout { + document_id: string; + docling_version: string; + num_items: number; + num_pages: number; + pages: BackendLayoutPage[]; + items: BackendLayoutItem[]; +} + +const toBoundingBox = (bbox: BackendBoundingBox): BoundingBox => + new BoundingBox(bbox.l, bbox.t, bbox.r, bbox.b, bbox.coord_origin ?? "TOPLEFT"); + +const toProvenance = (prov: BackendProvenance): Provenance => + new Provenance(prov.page_no, toBoundingBox(prov.bbox), prov.charspan); + +const toLayoutItem = (item: BackendLayoutItem): LayoutItem => + new LayoutItem( + item.self_ref, + item.label, + item.reading_order, + (item.prov ?? []).map(toProvenance), + item.parent_ref ?? null, + item.content_layer ?? null, + item.level ?? null, + item.text ?? null, + item.html ?? null + ); + +const toLayoutPage = (page: BackendLayoutPage): LayoutPage => new LayoutPage(page.page_no, page.width, page.height); + +const toDocumentLayout = (layout: BackendDocumentLayout): DocumentLayout => + new DocumentLayout( + layout.document_id, + layout.docling_version, + (layout.pages ?? []).map(toLayoutPage), + (layout.items ?? []).map(toLayoutItem) + ); + export class DocumentRepository { constructor(private readonly axios: AxiosInstance) {} @@ -59,4 +137,23 @@ export class DocumentRepository { }; } } + + async getDocumentLayout( + documentId: string, + filters?: { pages?: number[]; labels?: string[] } + ): Promise { + try { + const params: Record = {}; + if (filters?.pages?.length) params.pages = filters.pages; + if (filters?.labels?.length) params.labels = filters.labels; + + const { data } = await this.axios.get(`/v1/documents/${documentId}/layout`, { params }); + + return toDocumentLayout(data); + } catch (error) { + throw { + response: DOCUMENT_API_ERRORS.ERROR_FETCHING_LAYOUT, + }; + } + } } diff --git a/extralit-server/pyproject.toml b/extralit-server/pyproject.toml index e7018c311..928aebdd9 100644 --- a/extralit-server/pyproject.toml +++ b/extralit-server/pyproject.toml @@ -17,42 +17,42 @@ authors = [{ name = "Extralit Labs", email = "extralit.contact@gmail.com" }] maintainers = [{ name = "Extralit Labs", email = "extralit.contact@gmail.com" }] dependencies = [ # Basic dependencies - "fastapi ~= 0.115.0", - "pydantic ~= 2.9.0", - "pydantic-settings ~= 2.6.0", - "uvicorn[standard] ~= 0.32.0", - "opensearch-py ~= 2.0.0", - "elasticsearch8[async] ~= 8.7.0", - "brotli-asgi ~= 1.4.0", + "fastapi >= 0.115.0", + "pydantic >= 2.9.0", + "pydantic-settings >= 2.6.0", + "uvicorn[standard] >= 0.32.0", + "opensearch-py >= 2.0.0", + "elasticsearch8[async] >= 8.7.0", + "brotli-asgi >= 1.4.0", "tenacity>=9.1.2", # Database dependencies - "alembic ~= 1.13.0", - "SQLAlchemy ~= 2.0.0", - "greenlet ~= 3.1.0", + "alembic >= 1.13.0", + "SQLAlchemy >= 2.0.0", + "greenlet >= 3.1.0", # Async SQLite "aiosqlite == 0.20.0", # Statics server - "aiofiles ~= 24.1.0", + "aiofiles >= 24.1.0", "PyYAML >= 5.4.1,< 6.1.0", # security dependencies - "python-jose[cryptography] ~= 3.3.0", - "bcrypt ~= 4.2.0", + "python-jose[cryptography] >= 3.3.0", + "bcrypt >= 4.2.0", # required by fastapi - "python-multipart ~= 0.0.16", + "python-multipart >= 0.0.16", # OAuth2 integration - "httpx ~= 0.27.0", - "oauthlib ~= 3.2.0", - "social-auth-core ~= 4.5.0", + "httpx >= 0.27.0", + "oauthlib >= 3.2.0", + "social-auth-core >= 4.5.0", # LiteLLM for chat integration "litellm >= 1.80.0,<=1.82.6", # GitHub Copilot OAuth device flow + token file locking "authlib>=1.6.9", "filelock>=3.25.2", # Background processing - "rq~=2.4.1", + "rq>=2.4.1", "lazy-loader>=0.4", # Info status - "psutil ~= 5.8, <5.10", + "psutil >= 5.8, <5.10", # For logging, tracebacks, printing, progressbars "rich != 13.1.0", # For CLI @@ -67,7 +67,7 @@ dependencies = [ # NumPy 2.x compatibility "numpy>=2.0.0,<3.0.0", # For Telemetry - "huggingface-hub~=0.34.0", + "huggingface-hub>=0.34.0", "Jinja2>=3.1.4", # Used by huggingface-hub to render dataset card templates # For file storage "aioboto3>=13.1.1", @@ -79,20 +79,19 @@ dependencies = [ "pandera[io]>=0.20", "lancedb>=0.34.0", "duckdb>=1.5.4", + "docling-core>=2.91.0,<3.0.0", + "pyarrow>=23.0.1", + "pdf-inspector>=1.14.2", ] [project.optional-dependencies] postgresql = [ - "psycopg2 ~= 2.9.0", + "psycopg2 >= 2.9.0", # Async PostgreSQL - "asyncpg ~= 0.30.0", -] -marker = [ - "marker-pdf>=1.9.3", - "torch>=2.5.0", - "torchvision>=0.20.0", - "transformers>=4.51.0", + "asyncpg >= 0.30.0", ] +# AGPL — must stay optional so the Apache-2.0 distribution never requires it +pymupdf = ["pymupdf4llm~=0.0.27"] [project.urls] homepage = "https://extralit.ai" @@ -119,9 +118,9 @@ dev = [ "pytest>=7.4.4", "pytest-cov>=4.1.0", "pytest-mock>=3.12.0", - "pytest-asyncio~=1.1.0", + "pytest-asyncio>=1.1.0", "pytest-env>=1.1.3", - "factory-boy~=3.2.1", + "factory-boy>=3.2.1", "httpx>=0.26.0", "pytest-randomly>=3.16.0", # For mocking httpx requests and responses diff --git a/extralit-server/src/extralit_server/api/handlers/v1/documents.py b/extralit-server/src/extralit_server/api/handlers/v1/documents.py index 05e6b2afe..32ff39395 100644 --- a/extralit-server/src/extralit_server/api/handlers/v1/documents.py +++ b/extralit-server/src/extralit_server/api/handlers/v1/documents.py @@ -3,14 +3,19 @@ from typing import TYPE_CHECKING, Annotated from uuid import UUID, uuid4 +from docling_core.types.doc.document import CURRENT_VERSION from fastapi import APIRouter, Body, Depends, File, Form, HTTPException, Path, Query, Security, UploadFile, status +from pydantic import ValidationError from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from extralit_server.api.policies.v1 import DocumentPolicy, authorize +from extralit_server.api.schemas.v1.document.layout import DocumentLayoutOut from extralit_server.api.schemas.v1.documents import DocumentCreate, DocumentDelete, DocumentListItem, DocumentUpdate from extralit_server.api.schemas.v1.imports import DocumentsBulkCreate, DocumentsBulkResponse from extralit_server.contexts import files, imports +from extralit_server.contexts.ocr import storage +from extralit_server.contexts.ocr.projection import project_layout from extralit_server.database import get_async_db from extralit_server.models import User, Workspace from extralit_server.models.database import Document @@ -246,6 +251,66 @@ async def list_documents( return documents +@router.get( + "/documents/{document_id}/layout", + status_code=status.HTTP_200_OK, + description="Get the extracted layout of a document, with per-item page regions.", +) +async def get_document_layout( + *, + db: Annotated[AsyncSession, Depends(get_async_db)], + document_id: Annotated[UUID, Path(title="The UUID of the document whose layout will be retrieved")], + pages: Annotated[list[int] | None, Query(description="1-indexed pages to include")] = None, + labels: Annotated[list[str] | None, Query(description="DocItemLabels to include, e.g. `table`")] = None, + s3_client=Depends(files.get_s3_client), + current_user: User = Security(auth.get_current_user), +) -> DocumentLayoutOut: + await authorize(current_user, DocumentPolicy.get()) + + document = await db.get(Document, document_id) + if document is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Document with id `{document_id}` not found", + ) + + workspace = await Workspace.get(db, document.workspace_id) + if workspace is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Workspace with id `{document.workspace_id}` not found", + ) + + layout_metadata = (document.metadata_ or {}).get("layout_metadata") + if not layout_metadata: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"No layout has been extracted for document `{document_id}`", + ) + + try: + doc = await storage.load_layout( + s3_client, + workspace.name, + document_id, + object_path=layout_metadata.get("layout_url"), + ) + except ValidationError as e: + # A layout written by a newer docling-core cannot be read back by this server. + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"Stored layout is not readable by docling-core {CURRENT_VERSION}: {e}", + ) from e + except Exception as e: + _LOGGER.error(f"Error loading layout for document {document_id}: {e}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Layout for document `{document_id}` could not be loaded", + ) from e + + return project_layout(doc, document_id, pages=pages, labels=labels) + + @router.post("/documents/bulk", status_code=status.HTTP_201_CREATED) async def create_documents_bulk( *, diff --git a/extralit-server/src/extralit_server/api/handlers/v1/workflows.py b/extralit-server/src/extralit_server/api/handlers/v1/workflows.py index 5dd3e4581..20140f453 100644 --- a/extralit-server/src/extralit_server/api/handlers/v1/workflows.py +++ b/extralit-server/src/extralit_server/api/handlers/v1/workflows.py @@ -76,6 +76,7 @@ async def start_workflow( reference=document.reference, workspace_name=request.workspace_name, workspace_id=workspace.id, + layout_parser=request.layout_parser, ) # Get the created workflow diff --git a/extralit-server/src/extralit_server/api/schemas/v1/document/chunks.py b/extralit-server/src/extralit_server/api/schemas/v1/document/chunks.py deleted file mode 100644 index efad8303f..000000000 --- a/extralit-server/src/extralit_server/api/schemas/v1/document/chunks.py +++ /dev/null @@ -1,108 +0,0 @@ -import uuid -from typing import Any, Optional, Union - -from pydantic import BaseModel, Field, validator - -""" -This is deprecated code that is outdated and should be used for reference only. -We may want to switch to using LlamaIndexDocument or other document models in the future. -""" - - -class Segments(BaseModel): - items: list[Union["TextSegment", "TableSegment", "FigureSegment"]] = Field( - default_factory=list, - description="List of segments in the reading order of the document", - ) - - def get(self, id: str, header: str | None = None, default=None): - for item in self.items: - if item.id == id or (header and item.header == header): - return item - - return default - - def __repr_str__(self, join_str: str) -> str: - return "\n " + f"{join_str}\n ".join(f"{type(item).__name__}({item})" for item in self.items) - - @validator("items", pre=True, each_item=True) - def parse_segments(cls, v): - if not isinstance(v, dict): - v = v.dict() - - segment_type = v.get("type", "").lower() - if segment_type in {"figure", "image"}: - return FigureSegment(**v) - elif segment_type == "table" or "html" in v: - return TableSegment(**v) - else: - return TextSegment(**v) - - def __getitem__(self, index): - return self.items[index] - - def __len__(self): - return len(self.items) - - -class Coordinates(BaseModel): - points: list[list[float]] = Field( - ..., description="List of 4 points, e.g. [[x1, y1], [x2, y1], [x1, y2], [x2, y2]]" - ) - layout_width: int | None = Field(None, description="Width of the layout") - layout_height: int | None = Field(None, description="Height of the layout") - system: str | None = Field(description="System of coordinates") - - def __repr_str__(self, join_str: str) -> str: - return "" - - -class TextSegment(BaseModel): - id: str = Field( - default_factory=lambda: str(uuid.uuid4()), description="Unique identifier of the segment", repr=False - ) - - header: str | None = Field( - None, - description="Header of the element", - ) - text: str = Field(..., description="Content as plain text", repr=False) - summary: str | None = Field(None, description="Summary of the content") - page_number: int | None = Field(None, description="Page number of the segment") - coordinates: Optional["Coordinates"] = Field( - None, description="Coordinates of the element in the document", repr=False - ) - level: int | None = Field(None, description="Level of the header") - source: str | None = Field(None, description="Source of the element", repr=False) - type: str | None = Field("text", description="Type of the element", repr=False) - original: Any | None = Field( - None, exclude=True, description="Original object from which the segment was extracted", repr=False - ) - - def text_cleaned(self): - return self.text.replace(" | ", " ").replace("---", "").strip() - - def __repr_str__(self, join_str: str) -> str: - return join_str.join( - repr(v) - if a is None - else ( - f'{a}="{v[:100]}...{v[-100:]}"'.replace("\n", "") - if isinstance(v, str) and len(v) > 200 - else f"{a}={v!r}" - ) - for a, v in self.__repr_args__() - if v and a not in {"INCLUDE_METADATA_KEYS"} - ) - - -class TableSegment(TextSegment): - footer: str | None = Field(None, description="Footer of the table or figure, to explain variable acronyms.") - html: str | None = Field(None, description="Content as HTML structured", repr=False) - image: str | None = Field(None, description="URL/filepath of the element's image", repr=False) - probability: float | None = Field(None, description="Probability or confidence of the segment's extraction") - type: str | None = Field("table", description="Type of the element", repr=False) - - -class FigureSegment(TableSegment): - type: str | None = Field("figure", description="Type of the element", repr=False) diff --git a/extralit-server/src/extralit_server/api/schemas/v1/document/layout.py b/extralit-server/src/extralit_server/api/schemas/v1/document/layout.py new file mode 100644 index 000000000..fd3edda58 --- /dev/null +++ b/extralit-server/src/extralit_server/api/schemas/v1/document/layout.py @@ -0,0 +1,63 @@ +"""Wire schema for extracted document layout. + +A flat projection of `DoclingDocument`, structurally identical to its `ProvenanceItem` / +`BoundingBox` and to `ITEM_SCHEMA` in `contexts.ocr.arrow`. Deliberately not the raw +`DoclingDocument` — its 73 recursive `$defs` would make the OpenAPI schema and any +hand-written frontend types unusable. +""" + +from typing import Optional + +from pydantic import BaseModel, Field + + +class BoundingBoxOut(BaseModel): + """A rectangle in page points.""" + + # Field names mirror docling's BoundingBox exactly — renaming would break the contract. + l: float = Field(..., description="Left edge") # noqa: E741 + t: float = Field(..., description="Top edge") + r: float = Field(..., description="Right edge") + b: float = Field(..., description="Bottom edge") + coord_origin: str = Field(default="TOPLEFT", description="Origin the coordinates are measured from") + + +class ProvenanceOut(BaseModel): + """Where an item came from: the page, the region, and the span of its own text.""" + + page_no: int = Field(..., description="1-indexed page number") + bbox: BoundingBoxOut = Field(..., description="Region on the page") + charspan: tuple[int, int] = Field(..., description="Item-local character span, not a document offset") + + +class LayoutItemOut(BaseModel): + """One layout element, anchored by `self_ref`.""" + + self_ref: str = Field(..., description="Citation anchor, e.g. `#/texts/12`") + parent_ref: Optional[str] = Field(None, description="Reference of the parent node") + label: str = Field(..., description="DocItemLabel, e.g. `text`, `table`, `picture`") + content_layer: Optional[str] = Field(None, description="Content layer, e.g. `body` or `furniture`") + level: Optional[int] = Field(None, description="Heading level, when the item is a section header") + reading_order: int = Field(..., description="Position in document reading order") + text: Optional[str] = Field(None, description="Item text, when it has any") + html: Optional[str] = Field(None, description="Rendered HTML, for tables") + prov: list[ProvenanceOut] = Field(default_factory=list, description="One entry per page region") + + +class LayoutPageOut(BaseModel): + """Page geometry every bbox on that page is relative to.""" + + page_no: int = Field(..., description="1-indexed page number") + width: float = Field(..., description="Page width in points") + height: float = Field(..., description="Page height in points") + + +class DocumentLayoutOut(BaseModel): + """Extracted layout for one document.""" + + document_id: str = Field(..., description="Document ID") + docling_version: str = Field(..., description="docling-core schema version of the stored document") + num_items: int = Field(..., description="Number of items in this response") + num_pages: int = Field(..., description="Number of pages in this response") + pages: list[LayoutPageOut] = Field(default_factory=list, description="Page geometry") + items: list[LayoutItemOut] = Field(default_factory=list, description="Layout items in reading order") diff --git a/extralit-server/src/extralit_server/api/schemas/v1/document/metadata.py b/extralit-server/src/extralit_server/api/schemas/v1/document/metadata.py index 053cf6d92..54fa4aaa6 100644 --- a/extralit-server/src/extralit_server/api/schemas/v1/document/metadata.py +++ b/extralit-server/src/extralit_server/api/schemas/v1/document/metadata.py @@ -51,6 +51,25 @@ class TextExtractionMetadata(BaseModel): extraction_method: str = Field(..., description="Method used for extraction") +class LayoutMetadata(BaseModel): + """Layout extraction job results. + + Only pointers and counts — the layout itself lives in object storage, because + `documents.metadata_` is returned in full by every `GET /documents` listing. + """ + + layout_url: str = Field(..., description="S3 object path of the canonical DoclingDocument JSON") + items_parquet_url: Optional[str] = Field(None, description="S3 object path of the per-item Parquet sidecar") + pages_parquet_url: Optional[str] = Field(None, description="S3 object path of the per-page Parquet sidecar") + parser: str = Field(..., description="Name of the layout parser that produced the document") + docling_version: str = Field(..., description="docling-core schema version the JSON was written with") + num_items: int = Field(default=0, description="Number of layout items extracted") + num_pages: int = Field(default=0, description="Number of pages with registered geometry") + pages_needing_ocr: list[int] = Field( + default_factory=list, description="1-indexed pages with no reliable text layer" + ) + + class DocumentProcessingMetadata(BaseModel): """Complete document processing metadata stored in documents.metadata_.""" @@ -58,6 +77,7 @@ class DocumentProcessingMetadata(BaseModel): analysis_metadata: Optional[AnalysisMetadata] = Field(None, description="Analysis results") preprocessing_metadata: Optional[PreprocessingMetadata] = Field(None, description="Preprocessing results") text_extraction_metadata: Optional[TextExtractionMetadata] = Field(None, description="Text extraction results") + layout_metadata: Optional[LayoutMetadata] = Field(None, description="Layout extraction results") workflow_status: str = Field(default="running", description="Overall workflow status") def update_analysis_results(self, analysis_result: dict) -> None: diff --git a/extralit-server/src/extralit_server/api/schemas/v1/workflows.py b/extralit-server/src/extralit_server/api/schemas/v1/workflows.py index a4b9a1077..6e8943fd2 100644 --- a/extralit-server/src/extralit_server/api/schemas/v1/workflows.py +++ b/extralit-server/src/extralit_server/api/schemas/v1/workflows.py @@ -14,6 +14,9 @@ class StartWorkflowRequest(BaseModel): force: bool = Field(False, description="Force restart if workflow already exists") wait: bool = Field(True, description="Wait for job to finish before returning") timeout: Optional[int] = Field(60, description="Max seconds to wait if wait=True") + layout_parser: Optional[str] = Field( + None, description="Layout parser to run (e.g. `pdf_inspector`, `pymupdf`); omit to skip layout extraction" + ) class StartWorkflowResponse(BaseModel): diff --git a/extralit-server/src/extralit_server/contexts/ocr/arrow.py b/extralit-server/src/extralit_server/contexts/ocr/arrow.py new file mode 100644 index 000000000..60136d74a --- /dev/null +++ b/extralit-server/src/extralit_server/contexts/ocr/arrow.py @@ -0,0 +1,129 @@ +"""Columnar projection of a `DoclingDocument`. + +One row per `(DocItem, ProvenanceItem)` so an item spanning a page break expands naturally +instead of hiding a second page inside a nested column. Lance-native, so `index/lancedb_engine` +can ingest these tables unchanged for cross-document provenance search. +""" + +from __future__ import annotations + +from typing import Any, Optional + +import pyarrow as pa +from docling_core.types.doc import DoclingDocument +from docling_core.types.doc.document import DocItem, TableItem + +ITEM_SCHEMA = pa.schema( + [ + ("document_id", pa.string()), + ("self_ref", pa.string()), + ("parent_ref", pa.string()), + ("label", pa.dictionary(pa.int8(), pa.string())), + ("content_layer", pa.dictionary(pa.int8(), pa.string())), + ("level", pa.int8()), + ("reading_order", pa.int32()), + ("prov_index", pa.int16()), + ("page_no", pa.int32()), + ("bbox", pa.list_(pa.float32(), 4)), + ("coord_origin", pa.dictionary(pa.int8(), pa.string())), + ("charspan_start", pa.int32()), + ("charspan_end", pa.int32()), + ("text", pa.string()), + ("html", pa.string()), + ] +) + +PAGE_SCHEMA = pa.schema( + [ + ("document_id", pa.string()), + ("page_no", pa.int32()), + ("width", pa.float32()), + ("height", pa.float32()), + ] +) + + +def _enum_value(value: Any) -> Optional[str]: + if value is None: + return None + return getattr(value, "value", value) + + +def _table_html(doc: DoclingDocument, item: DocItem) -> Optional[str]: + if not isinstance(item, TableItem): + return None + try: + return item.export_to_html(doc=doc) or None + except Exception: # a malformed table must not sink the whole projection + return None + + +def item_rows(doc: DoclingDocument, document_id: str) -> list[dict[str, Any]]: + """Flatten every item's provenance into rows matching `ITEM_SCHEMA`.""" + rows: list[dict[str, Any]] = [] + + for reading_order, (item, _level) in enumerate(doc.iterate_items(with_groups=False)): + base = { + "document_id": document_id, + "self_ref": item.self_ref, + "parent_ref": item.parent.cref if item.parent else None, + "label": _enum_value(getattr(item, "label", None)), + "content_layer": _enum_value(getattr(item, "content_layer", None)), + "level": getattr(item, "level", None), + "reading_order": reading_order, + "text": getattr(item, "text", None) or None, + "html": _table_html(doc, item), + } + + provs = list(getattr(item, "prov", []) or []) + if not provs: + rows.append( + { + **base, + "prov_index": 0, + "page_no": None, + "bbox": None, + "coord_origin": None, + "charspan_start": None, + "charspan_end": None, + } + ) + continue + + for prov_index, prov in enumerate(provs): + rows.append( + { + **base, + "prov_index": prov_index, + "page_no": prov.page_no, + "bbox": [prov.bbox.l, prov.bbox.t, prov.bbox.r, prov.bbox.b], + "coord_origin": _enum_value(prov.bbox.coord_origin), + "charspan_start": prov.charspan[0] if prov.charspan else None, + "charspan_end": prov.charspan[1] if prov.charspan else None, + } + ) + + return rows + + +def items_table(doc: DoclingDocument, document_id: str) -> pa.Table: + """Project every `(DocItem, ProvenanceItem)` pair into an Arrow table.""" + return pa.Table.from_pylist(item_rows(doc, document_id), schema=ITEM_SCHEMA) + + +def page_rows(doc: DoclingDocument, document_id: str) -> list[dict[str, Any]]: + """One row per registered page, carrying the size every bbox is relative to.""" + return [ + { + "document_id": document_id, + "page_no": page_no, + "width": page.size.width if page.size else None, + "height": page.size.height if page.size else None, + } + for page_no, page in sorted(doc.pages.items()) + ] + + +def pages_table(doc: DoclingDocument, document_id: str) -> pa.Table: + """Project page geometry into an Arrow table.""" + return pa.Table.from_pylist(page_rows(doc, document_id), schema=PAGE_SCHEMA) diff --git a/extralit-server/src/extralit_server/contexts/ocr/docling_builder.py b/extralit-server/src/extralit_server/contexts/ocr/docling_builder.py new file mode 100644 index 000000000..a13abc133 --- /dev/null +++ b/extralit-server/src/extralit_server/contexts/ocr/docling_builder.py @@ -0,0 +1,182 @@ +"""Shared seam between layout parsers and `DoclingDocument`. + +Parsers normalize whatever their backend produces into `LayoutBlock`s; everything after that +— reading order, provenance, containment dedup — happens here, once, for every parser. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + +from docling_core.types.doc import ( + BoundingBox, + CoordOrigin, + DocItemLabel, + DoclingDocument, + ImageRef, + ProvenanceItem, + Size, + TableCell, +) +from docling_core.types.doc.document import DocumentOrigin, NodeItem + +PDF_MIMETYPE = "application/pdf" + +#: Fraction of a text block that must fall inside a table/picture for it to count as duplicated. +CONTAINMENT_THRESHOLD = 0.6 + +#: Labels routed to `add_table` / `add_picture` rather than `add_text`. +TABLE_LABELS = frozenset({DocItemLabel.TABLE, DocItemLabel.DOCUMENT_INDEX}) +PICTURE_LABELS = frozenset({DocItemLabel.PICTURE, DocItemLabel.CHART}) + +#: Captions and footnotes legitimately overlap their figure, so containment must not drop them. +CONTAINMENT_EXEMPT_LABELS = frozenset({DocItemLabel.CAPTION, DocItemLabel.FOOTNOTE}) + + +@dataclass(frozen=True) +class LayoutBlock: + """One parser-agnostic layout element, already in top-left page points.""" + + label: DocItemLabel + bbox: BoundingBox + text: str = "" + level: Optional[int] = None + cells: Optional[list[TableCell]] = None + image: Optional[ImageRef] = None + + +@dataclass(frozen=True) +class PageContext: + """The page a batch of blocks belongs to, and the geometry needed to place them.""" + + page_no: int + size: Size + + @property + def height(self) -> float: + return self.size.height + + @property + def width(self) -> float: + return self.size.width + + +def new_document( + name: str, + *, + filename: Optional[str] = None, + binary_hash: Optional[int] = None, + mimetype: str = PDF_MIMETYPE, +) -> DoclingDocument: + """Create an empty `DoclingDocument`, recording an origin when the source is identified.""" + origin = None + if filename is not None: + origin = DocumentOrigin(mimetype=mimetype, filename=filename, binary_hash=binary_hash or 0) + return DoclingDocument(name=name, origin=origin) + + +def register_page(doc: DoclingDocument, ctx: PageContext) -> None: + """Register the page size. Must happen before any prov is added, or bboxes get clamped to nothing.""" + if ctx.page_no not in doc.pages: + doc.add_page(page_no=ctx.page_no, size=ctx.size) + + +def flip_to_top_left(bbox: BoundingBox, page_height: float) -> BoundingBox: + """Convert a PDF-native bottom-left bbox to docling's top-left convention.""" + if bbox.coord_origin == CoordOrigin.TOPLEFT: + return bbox + return bbox.to_top_left_origin(page_height=page_height) + + +def clamp_to_page(bbox: BoundingBox, ctx: PageContext) -> BoundingBox: + """Clip a bbox into the page rect. docling only warns on out-of-page boxes, it does not fix them.""" + return BoundingBox( + l=max(0.0, min(bbox.l, ctx.width)), + r=max(0.0, min(bbox.r, ctx.width)), + t=max(0.0, min(bbox.t, ctx.height)), + b=max(0.0, min(bbox.b, ctx.height)), + coord_origin=bbox.coord_origin, + ) + + +def make_prov(ctx: PageContext, bbox: BoundingBox, text: Optional[str]) -> ProvenanceItem: + """Build the lineage triple. Charspans are item-local, matching docling-eval's adapters.""" + charspan = (0, len(text)) if text else (0, 0) + return ProvenanceItem( + page_no=ctx.page_no, + bbox=clamp_to_page(flip_to_top_left(bbox, ctx.height), ctx), + charspan=charspan, + ) + + +def is_contained( + doc: DoclingDocument, + bbox: BoundingBox, + page_no: int, + threshold: float = CONTAINMENT_THRESHOLD, +) -> bool: + """Whether `bbox` is mostly swallowed by a table or picture already on this page.""" + for item in [*doc.tables, *doc.pictures]: + for prov in item.prov: + if prov.page_no != page_no: + continue + if bbox.intersection_over_self(prov.bbox) >= threshold: + return True + return False + + +def _sort_key(doc: DoclingDocument, node: NodeItem) -> tuple[float, float, float]: + """Position a body child by the earliest provenance anywhere in its subtree.""" + positions = [] + stack = [node] + while stack: + current = stack.pop() + for prov in getattr(current, "prov", []) or []: + positions.append((float(prov.page_no), prov.bbox.t, prov.bbox.l)) + stack.extend(child.resolve(doc) for child in current.children) + return min(positions) if positions else (float("inf"),) * 3 + + +def sort_body_by_position(doc: DoclingDocument) -> None: + """Restore geometric reading order across text, tables and pictures alike. + + The dedup pass has to add tables and pictures first, which leaves them ahead of every + paragraph in the body; this puts the page back in the order a reader would see it. + """ + doc.body.children.sort(key=lambda ref: _sort_key(doc, ref.resolve(doc))) + + +def append_blocks( + doc: DoclingDocument, + ctx: PageContext, + blocks: list[LayoutBlock], +) -> list[NodeItem]: + """Add one page's blocks, then restore reading order. + + Tables and pictures go in first so the text pass can drop anything they already contain — + without that ordering the same words land in the document twice, once as a table cell and + once as a stray paragraph. + """ + from extralit_server.contexts.ocr.figures import add_picture_block + from extralit_server.contexts.ocr.tables import add_table_block + from extralit_server.contexts.ocr.text import add_text_block + + register_page(doc, ctx) + + added: list[NodeItem] = [] + for block in blocks: + if block.label in TABLE_LABELS: + added.append(add_table_block(doc, block, ctx)) + elif block.label in PICTURE_LABELS: + added.append(add_picture_block(doc, block, ctx)) + + for block in blocks: + if block.label in TABLE_LABELS or block.label in PICTURE_LABELS: + continue + item = add_text_block(doc, block, ctx) + if item is not None: + added.append(item) + + sort_body_by_position(doc) + return added diff --git a/extralit-server/src/extralit_server/contexts/ocr/figures.py b/extralit-server/src/extralit_server/contexts/ocr/figures.py index 72bf00922..e0aabacc7 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/figures.py +++ b/extralit-server/src/extralit_server/contexts/ocr/figures.py @@ -1,199 +1,21 @@ -"""Figure detection and bounding box utilities for OCR processing.""" +"""Appends a single picture `LayoutBlock` to a `DoclingDocument`.""" -from typing import Any +from __future__ import annotations +from typing import Optional -def extract_figure_bboxes(marker_layout: dict[str, Any]) -> list[dict[str, Any]]: - """ - Extract figure bounding boxes from Marker layout detection results. +from docling_core.types.doc import DoclingDocument +from docling_core.types.doc.document import NodeItem, PictureItem - Args: - marker_layout: Dictionary containing Marker's layout detection results +from extralit_server.contexts.ocr.docling_builder import LayoutBlock, PageContext, make_prov - Returns: - List of dictionaries containing figure bounding box information: - - page: Page number (0-indexed) - - bbox: Bounding box coordinates [x0, y0, x1, y1] - - score: Confidence score (if available) - - type: Element type ('figure') - """ - figures = [] - # Input validation - if not marker_layout or not isinstance(marker_layout, dict): - return figures - - # Handle different possible Marker output formats - pages_data = marker_layout.get("pages", []) - if not pages_data and "blocks" in marker_layout: - # Single page format - pages_data = [marker_layout] - - # Validate pages data - if not isinstance(pages_data, list): - return figures - - for page_idx, page_data in enumerate(pages_data): - # Validate page data structure - if not isinstance(page_data, dict): - continue - - page_number = page_data.get("page", page_idx) - blocks = page_data.get("blocks", []) - - # Validate blocks structure - if not isinstance(blocks, list): - continue - - for block in blocks: - # Validate block structure - if not isinstance(block, dict): - continue - - # Try different naming conventions for block type - block_type = ( - block.get("type") or block.get("block_type") or block.get("category") or block.get("label") or "" - ).lower() - - # Multiple patterns for figure detection (including Marker-specific types) - if any( - keyword in block_type - for keyword in [ - "figure", - "image", - "graphic", - "chart", - "diagram", - "plot", - "picture", - "picturegroup", - "figuregroup", - "figureblock", - "figure_block", - "imageblock", - "image_block", # - ] - ): - # Try different naming conventions for bounding box - bbox = ( - block.get("bbox") - or block.get("coordinates") - or block.get("bounding_box") - or block.get("rect") - or block.get("box") - ) - - # Additional fallback: try nested structure - if not bbox and "geometry" in block: - bbox = block["geometry"].get("bbox") or block["geometry"].get("coordinates") - - # Validate bbox format with multiple possible formats - valid_bbox = None - if bbox: - if isinstance(bbox, list) and len(bbox) == 4: - # Standard [x1, y1, x2, y2] format - try: - valid_bbox = [float(x) for x in bbox] - except (ValueError, TypeError): - pass - elif isinstance(bbox, dict): - # Object format like {x1, y1, x2, y2} or {left, top, right, bottom} - try: - if all(k in bbox for k in ["x1", "y1", "x2", "y2"]): - valid_bbox = [ - float(bbox["x1"]), - float(bbox["y1"]), - float(bbox["x2"]), - float(bbox["y2"]), - ] - elif all(k in bbox for k in ["left", "top", "right", "bottom"]): - valid_bbox = [ - float(bbox["left"]), - float(bbox["top"]), - float(bbox["right"]), - float(bbox["bottom"]), - ] - except (ValueError, TypeError, KeyError): - pass - - if valid_bbox: - # Try different naming conventions for caption/description - caption = ( - block.get("caption") - or block.get("text") - or block.get("content") - or block.get("description") - or block.get("alt_text") - or "" - ) - - # Try different naming conventions for confidence score - score = ( - block.get("score") - or block.get("confidence") - or block.get("probability") - or block.get("certainty") - ) - figures.append( - { - "page": page_number, - "bbox": valid_bbox, - "score": score, - "type": "figure", - "caption": caption, - "metadata": { - "source": "marker", - "block_id": block.get("id") or block.get("block_id"), - "polygon": block.get("polygon") or block.get("shape"), - "original_type": block.get("type") or block.get("block_type"), - }, - } - ) - - return figures - - -def normalize_figure_bbox(bbox: list[float], page_width: float, page_height: float) -> list[float]: - """ - Normalize bounding box coordinates to relative values (0-1 range). - - Args: - bbox: Bounding box coordinates [x0, y0, x1, y1] - page_width: Page width in points - page_height: Page height in points - - Returns: - Normalized bounding box coordinates [x0, y0, x1, y1] - """ - if not bbox or len(bbox) != 4: - return [0.0, 0.0, 0.0, 0.0] - - x0, y0, x1, y1 = bbox - return [ - max(0.0, min(1.0, x0 / page_width)), - max(0.0, min(1.0, y0 / page_height)), - max(0.0, min(1.0, x1 / page_width)), - max(0.0, min(1.0, y1 / page_height)), - ] - - -def filter_figures_by_size(figures: list[dict[str, Any]], min_area: float = 0.001) -> list[dict[str, Any]]: - """ - Filter figures by minimum area to remove noise/small artifacts. - - Args: - figures: List of figure dictionaries with bbox information - min_area: Minimum relative area threshold (0-1 range) - - Returns: - Filtered list of figures - """ - filtered = [] - for figure in figures: - bbox = figure.get("bbox", []) - if len(bbox) == 4: - x0, y0, x1, y1 = bbox - area = abs((x1 - x0) * (y1 - y0)) - if area >= min_area: - filtered.append(figure) - return filtered +def add_picture_block( + doc: DoclingDocument, + block: LayoutBlock, + ctx: PageContext, + parent: Optional[NodeItem] = None, +) -> PictureItem: + """Add one picture, anchored by its page bbox.""" + prov = make_prov(ctx, block.bbox, text=None) + return doc.add_picture(prov=prov, image=block.image, parent=parent) diff --git a/extralit-server/src/extralit_server/contexts/ocr/parsers/__init__.py b/extralit-server/src/extralit_server/contexts/ocr/parsers/__init__.py new file mode 100644 index 000000000..f6000942f --- /dev/null +++ b/extralit-server/src/extralit_server/contexts/ocr/parsers/__init__.py @@ -0,0 +1,64 @@ +"""Swappable PDF→`DoclingDocument` parsers. + +Each parser normalizes its backend into `LayoutBlock`s and hands them to the shared builder, +so the document that comes out is the same shape regardless of which one ran. +""" + +from __future__ import annotations + +import logging +from collections.abc import Sequence +from typing import Optional, Protocol + +from docling_core.types.doc import DoclingDocument + +_LOGGER = logging.getLogger(__name__) + + +class LayoutParser(Protocol): + """Parse PDF bytes into a `DoclingDocument`.""" + + def __call__( + self, + pdf_bytes: bytes, + *, + name: str, + pages: Optional[Sequence[int]] = None, + ) -> DoclingDocument: ... + + +_PARSERS: dict[str, LayoutParser] = {} + +from extralit_server.contexts.ocr.parsers.pdf_inspector import parse as _parse_pdf_inspector + +_PARSERS["pdf_inspector"] = _parse_pdf_inspector + +_PYMUPDF_AVAILABLE = False +try: + from extralit_server.contexts.ocr.parsers.pymupdf import parse as _parse_pymupdf + + _PARSERS["pymupdf"] = _parse_pymupdf + _PYMUPDF_AVAILABLE = True +except ImportError as e: # AGPL extra, deliberately optional + _LOGGER.debug(f"pymupdf layout parser unavailable: {e}") + + +def list_parsers() -> list[str]: + """Names of every parser installed in this environment.""" + return sorted(_PARSERS) + + +def get_parser(name: str) -> LayoutParser: + """Look up a parser by name.""" + try: + return _PARSERS[name] + except KeyError: + raise ValueError(f"unknown layout parser {name!r}; available: {list_parsers()}") from None + + +def default_parser_name() -> str: + """Prefer pymupdf's higher-fidelity geometry when the extra is installed.""" + return "pymupdf" if _PYMUPDF_AVAILABLE else "pdf_inspector" + + +__all__ = ["LayoutParser", "default_parser_name", "get_parser", "list_parsers"] diff --git a/extralit-server/src/extralit_server/contexts/ocr/parsers/pdf_inspector.py b/extralit-server/src/extralit_server/contexts/ocr/parsers/pdf_inspector.py new file mode 100644 index 000000000..200fbb181 --- /dev/null +++ b/extralit-server/src/extralit_server/contexts/ocr/parsers/pdf_inspector.py @@ -0,0 +1,270 @@ +"""pdf-inspector layout parser (MIT, zero deps, the always-available default). + +Three quirks are normalized here and nowhere else: + 1. Page indexing differs across pdf-inspector's own API — `TextItem.page` and + `StructureElement.page` are 1-indexed, `PdfClassification.pages_needing_ocr` is 0-indexed. + 2. `TextItem.y` is PDF-native bottom-left, so every bbox is flipped to docling's top-left. + 3. It returns no page dimensions at all, so MediaBox is read with pikepdf. +""" + +from __future__ import annotations + +import io +import logging +from collections import Counter, defaultdict +from collections.abc import Sequence +from typing import Any, Optional + +import pdf_inspector +import pikepdf +from docling_core.types.doc import BoundingBox, CoordOrigin, DocItemLabel, DoclingDocument, Size, TableCell + +from extralit_server.contexts.ocr.docling_builder import LayoutBlock, PageContext, append_blocks, new_document +from extralit_server.contexts.ocr.tables import make_cell + +_LOGGER = logging.getLogger(__name__) + +#: Structure-tree roles resolved through /RoleMap, mapped to docling labels. +_ROLE_LABELS: dict[str, tuple[DocItemLabel, Optional[int]]] = { + "Title": (DocItemLabel.TITLE, None), + "P": (DocItemLabel.TEXT, None), + "Table": (DocItemLabel.TABLE, None), + "Figure": (DocItemLabel.PICTURE, None), + "Caption": (DocItemLabel.CAPTION, None), + "LI": (DocItemLabel.LIST_ITEM, None), + "LBody": (DocItemLabel.LIST_ITEM, None), + "Formula": (DocItemLabel.FORMULA, None), + "Code": (DocItemLabel.CODE, None), +} + +#: Two spans are on the same line when their baselines agree within this fraction of font size. +_LINE_TOLERANCE = 0.5 +#: Consecutive lines join into one paragraph when their vertical gap is under this multiple of size. +_PARAGRAPH_GAP = 1.8 +#: A font must exceed the modal body size by this factor before it reads as a heading. +_HEADING_SIZE_RATIO = 1.15 + + +def role_to_label(role: str) -> tuple[DocItemLabel, Optional[int]]: + """Map a structure-tree role onto a docling label, recovering heading level from H1..H6.""" + if len(role) == 2 and role[0] == "H" and role[1].isdigit(): + return DocItemLabel.SECTION_HEADER, int(role[1]) + return _ROLE_LABELS.get(role, (DocItemLabel.TEXT, None)) + + +def page_sizes(pdf_bytes: bytes) -> dict[int, Size]: + """Read each page's MediaBox, keyed by docling's 1-indexed page_no.""" + sizes: dict[int, Size] = {} + with pikepdf.open(io.BytesIO(pdf_bytes)) as pdf: + for index, page in enumerate(pdf.pages, start=1): + box = [float(v) for v in page.MediaBox] + width, height = abs(box[2] - box[0]), abs(box[3] - box[1]) + if int(page.get("/Rotate", 0) or 0) % 180 == 90: + width, height = height, width + sizes[index] = Size(width=width, height=height) + return sizes + + +def classify(pdf_bytes: bytes) -> dict[str, Any]: + """Cheap classification used to route parsers and to surface scanned pages.""" + result = pdf_inspector.classify_pdf_bytes(pdf_bytes) + return { + "page_count": result.page_count, + "pdf_type": str(result.pdf_type), + "confidence": result.confidence, + # classify_pdf reports 0-indexed pages; docling page_no is 1-indexed. + "pages_needing_ocr": sorted(p + 1 for p in (result.pages_needing_ocr or [])), + } + + +def _structure_roles(pdf_bytes: bytes) -> dict[tuple[int, int], str]: + """Map (page_no, mcid) -> role for tagged PDFs. Untagged files yield an empty map.""" + try: + elements = pdf_inspector.extract_structure_elements_bytes(pdf_bytes) + except Exception as e: + _LOGGER.debug(f"no structure tree available: {e}") + return {} + return {(el.page, el.mcid): el.role for el in elements if el.mcid is not None} + + +def _to_bbox(item: Any) -> BoundingBox: + """A pdf-inspector item's rect, still in PDF-native bottom-left coordinates.""" + return BoundingBox( + l=item.x, + r=item.x + item.width, + b=item.y, + t=item.y + item.height, + coord_origin=CoordOrigin.BOTTOMLEFT, + ) + + +def _union(bboxes: Sequence[BoundingBox]) -> BoundingBox: + return BoundingBox( + l=min(b.l for b in bboxes), + r=max(b.r for b in bboxes), + b=min(b.b for b in bboxes), + t=max(b.t for b in bboxes), + coord_origin=CoordOrigin.BOTTOMLEFT, + ) + + +def _body_size(items: Sequence[Any]) -> float: + """The modal font size, treated as body text when no structure tree says otherwise.""" + sizes = Counter(round(i.font_size, 1) for i in items if i.item_type == "text" and i.font_size) + return sizes.most_common(1)[0][0] if sizes else 0.0 + + +def _heading_levels(items: Sequence[Any], body_size: float) -> dict[float, int]: + """Rank the font sizes above body size, largest first, into heading levels 1..N.""" + larger = sorted( + { + round(i.font_size, 1) + for i in items + if i.item_type == "text" and i.font_size and i.font_size > body_size * _HEADING_SIZE_RATIO + }, + reverse=True, + ) + return {size: level for level, size in enumerate(larger, start=1)} + + +def _group_lines(items: Sequence[Any]) -> list[list[Any]]: + """Cluster spans sharing a baseline into lines, top of page first.""" + lines: list[list[Any]] = [] + for item in sorted(items, key=lambda i: (-i.y, i.x)): + tolerance = max(item.font_size, 1.0) * _LINE_TOLERANCE + if lines and abs(lines[-1][0].y - item.y) <= tolerance: + lines[-1].append(item) + else: + lines.append([item]) + return [sorted(line, key=lambda i: i.x) for line in lines] + + +def _merge_paragraph(lines: list[list[Any]]) -> list[list[Any]]: + """Join vertically adjacent lines of the same size into one block.""" + blocks: list[list[Any]] = [] + for line in lines: + size = max((i.font_size for i in line), default=0.0) + if blocks: + previous = blocks[-1] + prev_size = max((i.font_size for i in previous), default=0.0) + gap = min(i.y for i in previous) - max(i.y + i.height for i in line) + if abs(prev_size - size) < 0.5 and 0 <= gap <= size * _PARAGRAPH_GAP: + previous.extend(line) + continue + blocks.append(list(line)) + return blocks + + +def _line_text(items: Sequence[Any]) -> str: + return " ".join(i.text.strip() for i in items if i.text and i.text.strip()) + + +def _table_cells(items: Sequence[Any]) -> list[TableCell]: + """Recover a grid from the spans inside a table region, by row then column position.""" + rows = _group_lines(items) + if not rows: + return [] + + # Column boundaries come from the distinct left edges seen anywhere in the table. + starts = sorted({round(i.x) for row in rows for i in row}) + columns: list[float] = [] + for start in starts: + if not columns or start - columns[-1] > 5: + columns.append(start) + + cells: list[TableCell] = [] + for row_index, row in enumerate(rows): + for item in row: + text = (item.text or "").strip() + if not text: + continue + col_index = max(i for i, c in enumerate(columns) if round(item.x) >= c - 5) + cells.append( + make_cell( + text, + row=row_index, + col=col_index, + column_header=row_index == 0, + bbox=_to_bbox(item), + ) + ) + return cells + + +def _blocks_for_page( + items: Sequence[Any], + roles: dict[tuple[int, int], str], + page_no: int, +) -> list[LayoutBlock]: + """Turn one page's positioned items into ordered layout blocks.""" + blocks: list[LayoutBlock] = [] + + images = [i for i in items if i.item_type != "text"] + text_items = [i for i in items if i.item_type == "text"] + + tagged = [i for i in text_items if i.mcid is not None and (page_no, i.mcid) in roles] + untagged = [i for i in text_items if i not in tagged] + + for image in images: + role = roles.get((page_no, image.mcid)) if image.mcid is not None else None + label = role_to_label(role)[0] if role else DocItemLabel.PICTURE + blocks.append(LayoutBlock(label=label, bbox=_to_bbox(image))) + + grouped: dict[int, list[Any]] = defaultdict(list) + for item in tagged: + grouped[item.mcid].append(item) + + for mcid, group in grouped.items(): + label, level = role_to_label(roles[(page_no, mcid)]) + bbox = _union([_to_bbox(i) for i in group]) + if label == DocItemLabel.TABLE: + blocks.append(LayoutBlock(label=label, bbox=bbox, cells=_table_cells(group))) + else: + blocks.append(LayoutBlock(label=label, bbox=bbox, text=_line_text(group), level=level)) + + if untagged: + body_size = _body_size(untagged) + levels = _heading_levels(untagged, body_size) + for group in _merge_paragraph(_group_lines(untagged)): + size = round(max((i.font_size for i in group), default=0.0), 1) + level = levels.get(size) + label = DocItemLabel.SECTION_HEADER if level else DocItemLabel.TEXT + blocks.append( + LayoutBlock( + label=label, + bbox=_union([_to_bbox(i) for i in group]), + text=_line_text(group), + level=level, + ) + ) + + # Reading order: top of the page down, ties broken left to right (still bottom-left coords). + blocks.sort(key=lambda b: (-b.bbox.t, b.bbox.l)) + return blocks + + +def parse( + pdf_bytes: bytes, + *, + name: str, + pages: Optional[Sequence[int]] = None, + filename: Optional[str] = None, +) -> DoclingDocument: + """Parse PDF bytes into a `DoclingDocument`. `pages` is a 1-indexed allowlist.""" + doc = new_document(name, filename=filename, binary_hash=hash(pdf_bytes) & 0xFFFFFFFF) + + sizes = page_sizes(pdf_bytes) + roles = _structure_roles(pdf_bytes) + wanted = set(pages) if pages else None + + by_page: dict[int, list[Any]] = defaultdict(list) + for item in pdf_inspector.extract_text_with_positions_bytes(pdf_bytes): + by_page[item.page].append(item) + + for page_no in sorted(sizes): + if wanted is not None and page_no not in wanted: + continue + ctx = PageContext(page_no=page_no, size=sizes[page_no]) + append_blocks(doc, ctx, _blocks_for_page(by_page.get(page_no, []), roles, page_no)) + + return doc diff --git a/extralit-server/src/extralit_server/contexts/ocr/parsers/pymupdf.py b/extralit-server/src/extralit_server/contexts/ocr/parsers/pymupdf.py new file mode 100644 index 000000000..38f7391dd --- /dev/null +++ b/extralit-server/src/extralit_server/contexts/ocr/parsers/pymupdf.py @@ -0,0 +1,170 @@ +"""PyMuPDF layout parser (AGPL, optional `pymupdf` extra). + +The higher-fidelity of the two: `find_tables()` yields per-cell geometry, which pdf-inspector +cannot provide. Coordinates are natively top-left in points, so nothing is flipped here. +""" + +from __future__ import annotations + +import logging +from collections.abc import Sequence +from typing import Any, Optional + +import pymupdf +import pymupdf4llm +from docling_core.types.doc import BoundingBox, CoordOrigin, DocItemLabel, DoclingDocument, Size, TableCell + +from extralit_server.contexts.ocr.docling_builder import LayoutBlock, PageContext, append_blocks, new_document +from extralit_server.contexts.ocr.tables import make_cell + +_LOGGER = logging.getLogger(__name__) + +_TEXT_BLOCK = 0 +_IMAGE_BLOCK = 1 + + +def _bbox(rect: Sequence[float]) -> BoundingBox: + """pymupdf rects are already top-left in page points.""" + left, top, right, bottom = (float(v) for v in rect) + return BoundingBox(l=left, t=top, r=right, b=bottom, coord_origin=CoordOrigin.TOPLEFT) + + +def _header_levels(doc: pymupdf.Document, pages: Optional[Sequence[int]]) -> Optional[Any]: + """Prefer the PDF outline for heading levels; fall back to font-size ranking.""" + try: + if doc.get_toc(): + return pymupdf4llm.TocHeaders(doc) + except Exception as e: + _LOGGER.debug(f"outline unusable, ranking by font size instead: {e}") + + # IdentifyHeaders rejects indices past the end, so only pass pages that exist. + indices = None + if pages: + indices = [p - 1 for p in pages if 0 < p <= doc.page_count] + if not indices: + return None + return pymupdf4llm.IdentifyHeaders(doc, pages=indices) + + +def _heading_level(headers: Optional[Any], span: dict, page: pymupdf.Page) -> Optional[int]: + """`get_header_id` returns a markdown prefix like '## '; its hash count is the level.""" + if headers is None: + return None + try: + marker = headers.get_header_id(span, page=page) + except Exception: + return None + level = marker.count("#") + return level or None + + +def _block_text(block: dict) -> str: + lines = [] + for line in block.get("lines", []): + text = "".join(span.get("text", "") for span in line.get("spans", [])) + if text.strip(): + lines.append(text.strip()) + return " ".join(lines) + + +def _dominant_span(block: dict) -> Optional[dict]: + """The largest span in a block decides whether the block reads as a heading.""" + spans = [span for line in block.get("lines", []) for span in line.get("spans", [])] + return max(spans, key=lambda s: s.get("size", 0)) if spans else None + + +def _table_cells(table: Any) -> list[TableCell]: + """Build cells from `find_tables()`, keeping the per-cell rects it hands back.""" + rows = table.extract() + has_header = bool(table.header) and not table.header.external + cells: list[TableCell] = [] + + for row_index, (row, row_obj) in enumerate(zip(rows, table.rows, strict=False)): + for col_index, value in enumerate(row): + text = (value or "").strip() + if not text: + continue + rect = row_obj.cells[col_index] if col_index < len(row_obj.cells) else None + cells.append( + make_cell( + text, + row=row_index, + col=col_index, + column_header=has_header and row_index == 0, + bbox=_bbox(rect) if rect else None, + ) + ) + return cells + + +def _blocks_for_page(page: pymupdf.Page, headers: Any) -> list[LayoutBlock]: + """Turn one page into ordered layout blocks.""" + blocks: list[LayoutBlock] = [] + + try: + tables = page.find_tables() + except Exception as e: + _LOGGER.warning(f"table detection failed on page {page.number + 1}: {e}") + tables = None + + table_rects = [] + for table in getattr(tables, "tables", []) or []: + bbox = _bbox(table.bbox) + table_rects.append(pymupdf.Rect(*table.bbox)) + blocks.append(LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox, cells=_table_cells(table))) + + for block in page.get_text("dict").get("blocks", []): + rect = pymupdf.Rect(block["bbox"]) + + if block.get("type") == _IMAGE_BLOCK: + blocks.append(LayoutBlock(label=DocItemLabel.PICTURE, bbox=_bbox(block["bbox"]))) + continue + + if block.get("type") != _TEXT_BLOCK: + continue + + # Text inside a detected table is already carried by its cells. + if any(rect.intersects(t) and (rect & t).get_area() >= rect.get_area() * 0.6 for t in table_rects): + continue + + text = _block_text(block) + if not text: + continue + + span = _dominant_span(block) + level = _heading_level(headers, span, page) if span else None + blocks.append( + LayoutBlock( + label=DocItemLabel.SECTION_HEADER if level else DocItemLabel.TEXT, + bbox=_bbox(block["bbox"]), + text=text, + level=level, + ) + ) + + blocks.sort(key=lambda b: (b.bbox.t, b.bbox.l)) + return blocks + + +def parse( + pdf_bytes: bytes, + *, + name: str, + pages: Optional[Sequence[int]] = None, + filename: Optional[str] = None, +) -> DoclingDocument: + """Parse PDF bytes into a `DoclingDocument`. `pages` is a 1-indexed allowlist.""" + doc = new_document(name, filename=filename, binary_hash=hash(pdf_bytes) & 0xFFFFFFFF) + wanted = set(pages) if pages else None + + with pymupdf.open(stream=pdf_bytes, filetype="pdf") as pdf: + headers = _header_levels(pdf, pages) + for index in range(pdf.page_count): + page_no = index + 1 + if wanted is not None and page_no not in wanted: + continue + page = pdf[index] + ctx = PageContext(page_no=page_no, size=Size(width=page.rect.width, height=page.rect.height)) + append_blocks(doc, ctx, _blocks_for_page(page, headers)) + + return doc diff --git a/extralit-server/src/extralit_server/contexts/ocr/projection.py b/extralit-server/src/extralit_server/contexts/ocr/projection.py new file mode 100644 index 000000000..55b7d8044 --- /dev/null +++ b/extralit-server/src/extralit_server/contexts/ocr/projection.py @@ -0,0 +1,87 @@ +"""Project a `DoclingDocument` onto the flat wire schema served by the layout route.""" + +from __future__ import annotations + +from collections.abc import Collection +from typing import Optional +from uuid import UUID + +from docling_core.types.doc import DoclingDocument + +from extralit_server.api.schemas.v1.document.layout import ( + BoundingBoxOut, + DocumentLayoutOut, + LayoutItemOut, + LayoutPageOut, + ProvenanceOut, +) +from extralit_server.contexts.ocr.arrow import item_rows + + +def project_layout( + doc: DoclingDocument, + document_id: UUID | str, + pages: Optional[Collection[int]] = None, + labels: Optional[Collection[str]] = None, +) -> DocumentLayoutOut: + """Flatten a document, optionally narrowed to some pages or labels. + + Rows come from the same helper the Parquet sidecar uses, so the API and the columnar + projection can never drift apart. + """ + document_id = str(document_id) + wanted_pages = set(pages) if pages else None + wanted_labels = {label.lower() for label in labels} if labels else None + + grouped: dict[str, LayoutItemOut] = {} + for row in item_rows(doc, document_id): + if wanted_labels is not None and (row["label"] or "").lower() not in wanted_labels: + continue + + page_no = row["page_no"] + if wanted_pages is not None and page_no not in wanted_pages: + continue + + item = grouped.get(row["self_ref"]) + if item is None: + item = LayoutItemOut( + self_ref=row["self_ref"], + parent_ref=row["parent_ref"], + label=row["label"] or "text", + content_layer=row["content_layer"], + level=row["level"], + reading_order=row["reading_order"], + text=row["text"], + html=row["html"], + prov=[], + ) + grouped[row["self_ref"]] = item + + if page_no is not None and row["bbox"] is not None: + left, top, right, bottom = row["bbox"] + item.prov.append( + ProvenanceOut( + page_no=page_no, + bbox=BoundingBoxOut( + l=left, t=top, r=right, b=bottom, coord_origin=row["coord_origin"] or "TOPLEFT" + ), + charspan=(row["charspan_start"] or 0, row["charspan_end"] or 0), + ) + ) + + items = sorted(grouped.values(), key=lambda i: i.reading_order) + + page_items = [ + LayoutPageOut(page_no=page_no, width=page.size.width, height=page.size.height) + for page_no, page in sorted(doc.pages.items()) + if page.size is not None and (wanted_pages is None or page_no in wanted_pages) + ] + + return DocumentLayoutOut( + document_id=document_id, + docling_version=doc.version, + num_items=len(items), + num_pages=len(page_items), + pages=page_items, + items=items, + ) diff --git a/extralit-server/src/extralit_server/contexts/ocr/storage.py b/extralit-server/src/extralit_server/contexts/ocr/storage.py new file mode 100644 index 000000000..bc549b1de --- /dev/null +++ b/extralit-server/src/extralit_server/contexts/ocr/storage.py @@ -0,0 +1,102 @@ +"""Object-storage layout for extracted document layout. + +Canonical JSON is the source of truth; the Parquet sidecars are a columnar projection for +analytics and cross-document provenance search. Both live in S3 rather than `documents.metadata_`, +which is returned in full by every document listing. +""" + +from __future__ import annotations + +import io +import json +from typing import TYPE_CHECKING, Optional +from uuid import UUID + +import pyarrow as pa +import pyarrow.parquet as pq +from docling_core.types.doc import DoclingDocument + +from extralit_server.contexts import files +from extralit_server.contexts.ocr.arrow import items_table, pages_table + +if TYPE_CHECKING: + from types_aiobotocore_s3 import S3Client + +LAYOUT_PREFIX = "layout" + + +def layout_object_path(document_id: UUID | str) -> str: + return f"{LAYOUT_PREFIX}/{document_id}.docling.json" + + +def items_object_path(document_id: UUID | str) -> str: + return f"{LAYOUT_PREFIX}/{document_id}.items.parquet" + + +def pages_object_path(document_id: UUID | str) -> str: + return f"{LAYOUT_PREFIX}/{document_id}.pages.parquet" + + +def _to_parquet(table: pa.Table) -> bytes: + buffer = io.BytesIO() + pq.write_table(table, buffer, compression="zstd") + return buffer.getvalue() + + +async def store_layout( + s3_client: S3Client, + workspace_name: str, + document_id: UUID | str, + doc: DoclingDocument, +) -> dict[str, str]: + """Write the canonical JSON and both Parquet sidecars. Returns their object paths.""" + document_id = str(document_id) + paths = { + "layout_url": layout_object_path(document_id), + "items_parquet_url": items_object_path(document_id), + "pages_parquet_url": pages_object_path(document_id), + } + + await files.put_object( + s3_client, + workspace_name, + paths["layout_url"], + json.dumps(doc.export_to_dict(), ensure_ascii=False), + content_type="application/json", + metadata={"docling_version": doc.version, "document_id": document_id}, + ) + await files.put_object( + s3_client, + workspace_name, + paths["items_parquet_url"], + _to_parquet(items_table(doc, document_id)), + content_type="application/vnd.apache.parquet", + metadata={"document_id": document_id}, + ) + await files.put_object( + s3_client, + workspace_name, + paths["pages_parquet_url"], + _to_parquet(pages_table(doc, document_id)), + content_type="application/vnd.apache.parquet", + metadata={"document_id": document_id}, + ) + + return paths + + +async def load_layout( + s3_client: S3Client, + workspace_name: str, + document_id: UUID | str, + object_path: Optional[str] = None, +) -> DoclingDocument: + """Read the canonical JSON back into a `DoclingDocument`. + + A document written by a newer docling-core fails validation outright — the version validator + demands an equal major and a minor no higher than the SDK's. + """ + key = object_path or layout_object_path(document_id) + response = await s3_client.get_object(Bucket=workspace_name, Key=key) + raw = await response["Body"].read() + return DoclingDocument.model_validate(json.loads(raw)) diff --git a/extralit-server/src/extralit_server/contexts/ocr/tables.py b/extralit-server/src/extralit_server/contexts/ocr/tables.py index f7bac0f7d..01fcd2b5d 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/tables.py +++ b/extralit-server/src/extralit_server/contexts/ocr/tables.py @@ -1,165 +1,62 @@ -"""Table detection and bounding box utilities for OCR processing.""" - -from typing import Any - - -def extract_table_bboxes(marker_layout: dict[str, Any]) -> list[dict[str, Any]]: - """ - Extract table bounding boxes from Marker layout detection results. - - Args: - marker_layout: Dictionary containing Marker's layout detection results - - Returns: - List of dictionaries containing table bounding box information: - - page: Page number (0-indexed) - - bbox: Bounding box coordinates [x0, y0, x1, y1] - - score: Confidence score (if available) - - type: Element type ('table') - """ - tables = [] - - # Input validation - if not marker_layout or not isinstance(marker_layout, dict): - return tables - - # Handle different possible Marker output formats - pages_data = marker_layout.get("pages", []) - if not pages_data and "blocks" in marker_layout: - # Single page format - pages_data = [marker_layout] - - # Validate pages data - if not isinstance(pages_data, list): - return tables - - for page_idx, page_data in enumerate(pages_data): - # Validate page data structure - if not isinstance(page_data, dict): - continue - - page_number = page_data.get("page", page_idx) - blocks = page_data.get("blocks", []) - - # Validate blocks structure - if not isinstance(blocks, list): - continue - - for block in blocks: - # Validate block structure - if not isinstance(block, dict): - continue - - # Try different naming conventions for block type - block_type = ( - block.get("type") or block.get("block_type") or block.get("category") or block.get("label") or "" - ).lower() - - # Multiple patterns for table detection (including Marker-specific types) - if any( - keyword in block_type - for keyword in [ - "table", - "grid", - "matrix", - "tableblock", - "table_block", - ] - ): - # Try different naming conventions for bounding box - bbox = ( - block.get("bbox") - or block.get("coordinates") - or block.get("bounding_box") - or block.get("rect") - or block.get("box") - ) - - # Additional fallback: try nested structure - if not bbox and "geometry" in block: - bbox = block["geometry"].get("bbox") or block["geometry"].get("coordinates") - - # Validate bbox format with multiple possible formats - valid_bbox = None - if bbox: - if isinstance(bbox, list) and len(bbox) == 4: - # Standard [x1, y1, x2, y2] format - try: - valid_bbox = [float(x) for x in bbox] - except (ValueError, TypeError): - pass - elif isinstance(bbox, dict): - # Object format like {x1, y1, x2, y2} or {left, top, right, bottom} - try: - if all(k in bbox for k in ["x1", "y1", "x2", "y2"]): - valid_bbox = [ - float(bbox["x1"]), - float(bbox["y1"]), - float(bbox["x2"]), - float(bbox["y2"]), - ] - elif all(k in bbox for k in ["left", "top", "right", "bottom"]): - valid_bbox = [ - float(bbox["left"]), - float(bbox["top"]), - float(bbox["right"]), - float(bbox["bottom"]), - ] - except (ValueError, TypeError, KeyError): - pass - - if valid_bbox: - # Try different naming conventions for text content - content = ( - block.get("text") or block.get("content") or block.get("html") or block.get("markdown") or "" - ) - - # Try different naming conventions for confidence score - score = ( - block.get("score") - or block.get("confidence") - or block.get("probability") - or block.get("certainty") - ) - - tables.append( - { - "page": page_number, - "bbox": valid_bbox, - "score": score, - "type": "table", - "content": content, - "metadata": { - "source": "marker", - "block_id": block.get("id") or block.get("block_id"), - "polygon": block.get("polygon") or block.get("shape"), - "original_type": block.get("type") or block.get("block_type"), - }, - } - ) - - return tables - - -def normalize_table_bbox(bbox: list[float], page_width: float, page_height: float) -> list[float]: - """ - Normalize bounding box coordinates to relative values (0-1 range). - - Args: - bbox: Bounding box coordinates [x0, y0, x1, y1] - page_width: Page width in points - page_height: Page height in points - - Returns: - Normalized bounding box coordinates [x0, y0, x1, y1] - """ - if not bbox or len(bbox) != 4: - return [0.0, 0.0, 0.0, 0.0] - - x0, y0, x1, y1 = bbox - return [ - max(0.0, min(1.0, x0 / page_width)), - max(0.0, min(1.0, y0 / page_height)), - max(0.0, min(1.0, x1 / page_width)), - max(0.0, min(1.0, y1 / page_height)), - ] +"""Appends a single table `LayoutBlock` to a `DoclingDocument`.""" + +from __future__ import annotations + +from typing import Optional + +from docling_core.types.doc import DoclingDocument, TableCell, TableData +from docling_core.types.doc.document import NodeItem, TableItem + +from extralit_server.contexts.ocr.docling_builder import LayoutBlock, PageContext, make_prov + + +def build_table_data(cells: Optional[list[TableCell]]) -> TableData: + """Size a `TableData` from its cells. `grid` is a computed field and must never be set.""" + if not cells: + return TableData(num_rows=0, num_cols=0, table_cells=[]) + + num_rows = max(cell.end_row_offset_idx for cell in cells) + num_cols = max(cell.end_col_offset_idx for cell in cells) + return TableData(num_rows=num_rows, num_cols=num_cols, table_cells=list(cells)) + + +def make_cell( + text: str, + row: int, + col: int, + *, + row_span: int = 1, + col_span: int = 1, + column_header: bool = False, + row_header: bool = False, + bbox=None, +) -> TableCell: + """Build a cell with docling's exclusive end offsets (`end = start + span`).""" + return TableCell( + text=text, + start_row_offset_idx=row, + end_row_offset_idx=row + row_span, + start_col_offset_idx=col, + end_col_offset_idx=col + col_span, + row_span=row_span, + col_span=col_span, + column_header=column_header, + row_header=row_header, + bbox=bbox, + ) + + +def add_table_block( + doc: DoclingDocument, + block: LayoutBlock, + ctx: PageContext, + parent: Optional[NodeItem] = None, +) -> TableItem: + """Add one table, anchored by its page bbox.""" + prov = make_prov(ctx, block.bbox, text=None) + return doc.add_table( + data=build_table_data(block.cells), + prov=prov, + parent=parent, + label=block.label, + ) diff --git a/extralit-server/src/extralit_server/contexts/ocr/text.py b/extralit-server/src/extralit_server/contexts/ocr/text.py index c33d83664..96c239226 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/text.py +++ b/extralit-server/src/extralit_server/contexts/ocr/text.py @@ -1,173 +1,39 @@ -"""Text block detection and bounding box utilities for OCR processing.""" +"""Appends a single text `LayoutBlock` to a `DoclingDocument`.""" -from typing import Any +from __future__ import annotations +from typing import Optional -def extract_text_bboxes(marker_layout: dict[str, Any]) -> list[dict[str, Any]]: - """ - Extract text block bounding boxes from Marker layout detection results. +from docling_core.types.doc import DocItemLabel, DoclingDocument +from docling_core.types.doc.document import NodeItem - Args: - marker_layout: Dictionary containing Marker's layout detection results +from extralit_server.contexts.ocr.docling_builder import ( + CONTAINMENT_EXEMPT_LABELS, + LayoutBlock, + PageContext, + is_contained, + make_prov, +) - Returns: - List of dictionaries containing text bounding box information: - - page: Page number (0-indexed) - - bbox: Bounding box coordinates [x0, y0, x1, y1] - - score: Confidence score (if available) - - type: Element type ('text') - - content: Text content (if available) - """ - text_blocks = [] - # Input validation - if not marker_layout or not isinstance(marker_layout, dict): - return text_blocks +def add_text_block( + doc: DoclingDocument, + block: LayoutBlock, + ctx: PageContext, + parent: Optional[NodeItem] = None, +) -> Optional[NodeItem]: + """Add one text block, skipping anything empty or already covered by a table/picture.""" + text = (block.text or "").strip() + if not text: + return None - # Handle different possible Marker output formats - pages_data = marker_layout.get("pages", []) - if not pages_data and "blocks" in marker_layout: - # Single page format - pages_data = [marker_layout] + prov = make_prov(ctx, block.bbox, text) - # Validate pages data - if not isinstance(pages_data, list): - return text_blocks + if block.label not in CONTAINMENT_EXEMPT_LABELS and is_contained(doc, prov.bbox, ctx.page_no): + return None - for page_idx, page_data in enumerate(pages_data): - # Validate page data structure - if not isinstance(page_data, dict): - continue + # add_text dispatches SECTION_HEADER to add_heading but drops the level, so route it here. + if block.label == DocItemLabel.SECTION_HEADER: + return doc.add_heading(text=text, level=block.level or 1, prov=prov, parent=parent) - page_number = page_data.get("page", page_idx) - blocks = page_data.get("blocks", []) - - # Validate blocks structure - if not isinstance(blocks, list): - continue - - for block in blocks: - # Validate block structure - if not isinstance(block, dict): - continue - - # Try different naming conventions for block type - block_type = ( - block.get("type") or block.get("block_type") or block.get("category") or block.get("label") or "" - ).lower() - - # Multiple patterns for text detection (including Marker-specific types) - if any( - keyword in block_type - for keyword in [ - "text", - "paragraph", - "heading", - "title", - "sectionheader", - "textinlinemath", - "listitem", - "line", - "span", - "textblock", - "text_block", - "paragraphblock", - "paragraph_block", - ] - ): - # Try different naming conventions for bounding box - bbox = ( - block.get("bbox") - or block.get("coordinates") - or block.get("bounding_box") - or block.get("rect") - or block.get("box") - ) - - # Additional fallback: try nested structure - if not bbox and "geometry" in block: - bbox = block["geometry"].get("bbox") or block["geometry"].get("coordinates") - - # Validate bbox format with multiple possible formats - valid_bbox = None - if bbox: - if isinstance(bbox, list) and len(bbox) == 4: - # Standard [x1, y1, x2, y2] format - try: - valid_bbox = [float(x) for x in bbox] - except (ValueError, TypeError): - pass - elif isinstance(bbox, dict): - # Object format like {x1, y1, x2, y2} or {left, top, right, bottom} - try: - if all(k in bbox for k in ["x1", "y1", "x2", "y2"]): - valid_bbox = [ - float(bbox["x1"]), - float(bbox["y1"]), - float(bbox["x2"]), - float(bbox["y2"]), - ] - elif all(k in bbox for k in ["left", "top", "right", "bottom"]): - valid_bbox = [ - float(bbox["left"]), - float(bbox["top"]), - float(bbox["right"]), - float(bbox["bottom"]), - ] - except (ValueError, TypeError, KeyError): - pass - - if valid_bbox: - # Try different naming conventions for text content - content = block.get("text") or block.get("content") or block.get("value") or block.get("data") or "" - - # Try different naming conventions for confidence score - score = ( - block.get("score") - or block.get("confidence") - or block.get("probability") - or block.get("certainty") - ) - - text_blocks.append( - { - "page": page_number, - "bbox": valid_bbox, - "score": score, - "type": "text", - "subtype": block.get("type") or block.get("block_type"), - "content": content, - "metadata": { - "source": "marker", - "block_id": block.get("id") or block.get("block_id"), - "polygon": block.get("polygon") or block.get("shape"), - "original_type": block.get("type") or block.get("block_type"), - }, - } - ) - - return text_blocks - - -def normalize_text_bbox(bbox: list[float], page_width: float, page_height: float) -> list[float]: - """ - Normalize bounding box coordinates to relative values (0-1 range). - - Args: - bbox: Bounding box coordinates [x0, y0, x1, y1] - page_width: Page width in points - page_height: Page height in points - - Returns: - Normalized bounding box coordinates [x0, y0, x1, y1] - """ - if not bbox or len(bbox) != 4: - return [0.0, 0.0, 0.0, 0.0] - - x0, y0, x1, y1 = bbox - return [ - max(0.0, min(1.0, x0 / page_width)), - max(0.0, min(1.0, y0 / page_height)), - max(0.0, min(1.0, x1 / page_width)), - max(0.0, min(1.0, y1 / page_height)), - ] + return doc.add_text(label=block.label, text=text, prov=prov, parent=parent) diff --git a/extralit-server/src/extralit_server/jobs/ocr_jobs.py b/extralit-server/src/extralit_server/jobs/ocr_jobs.py index 2635c5067..9e02b0b42 100644 --- a/extralit-server/src/extralit_server/jobs/ocr_jobs.py +++ b/extralit-server/src/extralit_server/jobs/ocr_jobs.py @@ -1,269 +1,127 @@ -"""OCR-related job functions for document processing.""" +"""Document layout extraction jobs.""" import logging -from pathlib import Path -from pprint import pprint -from typing import TYPE_CHECKING, Any, Optional, Union +from collections.abc import Sequence +from typing import Any, Optional from uuid import UUID from rq import Retry, get_current_job from rq.decorators import job -from extralit_server.contexts.ocr.figures import extract_figure_bboxes -from extralit_server.contexts.ocr.tables import extract_table_bboxes -from extralit_server.contexts.ocr.text import extract_text_bboxes -from extralit_server.jobs.queues import DEFAULT_QUEUE, REDIS_CONNECTION - -if TYPE_CHECKING: - from marker.renderers.json import JSONOutput +from extralit_server.api.schemas.v1.document.metadata import DocumentProcessingMetadata, LayoutMetadata +from extralit_server.contexts import files +from extralit_server.contexts.ocr import storage +from extralit_server.contexts.ocr.parsers import default_parser_name, get_parser +from extralit_server.contexts.ocr.parsers.pdf_inspector import classify +from extralit_server.database import AsyncSessionLocal +from extralit_server.jobs.queues import OCR_QUEUE, REDIS_CONNECTION +from extralit_server.models.database import Document _LOGGER = logging.getLogger(__name__) -_MARKER_AVAILABLE = False -try: - from marker.config.parser import ConfigParser - from marker.converters.pdf import PdfConverter - from marker.models import create_model_dict - - _MARKER_AVAILABLE = True -except ImportError as e: - _LOGGER.warning(f"Marker dependencies not available: {e}. OCR layout jobs will be disabled.") - ConfigParser = None # type: ignore[assignment,misc] - PdfConverter = None # type: ignore[assignment,misc] - create_model_dict = None # type: ignore[assignment] +def route_parser(pdf_bytes: bytes) -> tuple[str, dict[str, Any]]: + """Pick a parser and record what classification saw. -@job(queue=DEFAULT_QUEUE, connection=REDIS_CONNECTION, timeout=1800, retry=Retry(max=2, interval=[30, 60])) -async def async_marker_layout_job( - pdf_path: Union[str, Path], - pages: Optional[str] = None, - extract_text: bool = False, - document_id: Optional[UUID] = None, -) -> dict[str, Any]: + `pages_needing_ocr` is surfaced rather than acted on, so scanned pages show up as an + explicit gap instead of silently producing an empty layout. """ - Use Marker to extract layout (tables, figures, text blocks) without running OCR. - - This job uses Marker's layout detection capabilities to identify and extract - bounding boxes for different document elements without performing OCR. + try: + classification = classify(pdf_bytes) + except Exception as e: + _LOGGER.warning(f"PDF classification failed, falling back to the default parser: {e}") + classification = {"pages_needing_ocr": [], "page_count": 0, "pdf_type": "unknown"} + return default_parser_name(), classification + + +@job( + queue=OCR_QUEUE, + connection=REDIS_CONNECTION, + timeout=1800, + result_ttl=3600, + retry=Retry(max=2, interval=[30, 60]), +) +async def async_document_layout_job( + document_id: UUID, + s3_url: str, + workspace_name: str, + parser: Optional[str] = None, + pages: Optional[Sequence[int]] = None, +) -> dict[str, Any]: + """Extract document layout into a `DoclingDocument` and persist it. Args: - pdf_path: Path to the PDF file to process - pages: Optional comma-separated page numbers to process (0-indexed). If None, processes all pages - extract_text: Whether to extract text blocks in addition to tables/figures - document_id: Optional document ID for job tracking + document_id: UUID of the document to process + s3_url: proxy URL of the PDF, as stored on the document + workspace_name: workspace bucket the artifacts are written to + parser: layout parser name; None routes automatically + pages: 1-indexed page allowlist; None processes every page Returns: - Dictionary containing structured layout information: - - tables: List of table bounding boxes - - figures: List of figure bounding boxes - - text_blocks: List of text block bounding boxes (if extract_text=True) - - metadata: Job execution metadata + Object paths and counts. Never the document itself — it does not belong in a job result. """ - if not _MARKER_AVAILABLE: - raise ImportError("Marker not installed. Install with: pip install marker-pdf") - current_job = get_current_job() if current_job is not None: current_job.meta.update( { - "pdf_path": str(pdf_path), - "document_id": str(document_id) if document_id else None, - "pages": pages, - "extract_text": extract_text, - "workflow_step": "marker_layout_extraction", + "document_id": str(document_id), + "workspace_name": workspace_name, + "workflow_step": "document_layout", } ) current_job.save_meta() try: - pdf_path = Path(pdf_path) - if not pdf_path.exists(): - raise FileNotFoundError(f"PDF file not found: {pdf_path}") + # Shared client — do not enter it as a context manager, that would close it for everyone. + s3_client = await files.get_s3_client() + pdf_bytes = await files.download_file_content(s3_client, s3_url) - _LOGGER.info(f"Starting Marker layout extraction for: {pdf_path}") + routed, classification = route_parser(pdf_bytes) + parser_name = parser or routed + _LOGGER.info(f"Extracting layout for document {document_id} with parser {parser_name}") - if pdf_path.suffix.lower() != ".pdf": - raise ValueError(f"File is not a PDF: {pdf_path}") - - try: - # Step 1: Create configuration - config_dict, model_dict = create_marker_config(pages) - - # Step 2: Run Marker - result = run_marker(str(pdf_path), config_dict, model_dict) - - # Step 3: Parse output - layout_result = parse_marker_output(result) + doc = get_parser(parser_name)( + pdf_bytes, + name=str(document_id), + pages=pages, + filename=s3_url.split("/")[-1], + ) - except Exception as e: - _LOGGER.error(f"Error calling Marker API: {e}", exc_info=True) - raise e + paths = await storage.store_layout(s3_client, workspace_name, document_id, doc) - # Extract bounding boxes using our utility functions - tables = extract_table_bboxes(layout_result) - figures = extract_figure_bboxes(layout_result) - text_blocks = extract_text_bboxes(layout_result) + layout = LayoutMetadata( + **paths, + parser=parser_name, + docling_version=doc.version, + num_items=sum(1 for _ in doc.iterate_items(with_groups=False)), + num_pages=len(doc.pages), + pages_needing_ocr=classification.get("pages_needing_ocr", []), + ) - print(f"Extracted {len(tables)} tables, {len(figures)} figures, {len(text_blocks)} text blocks") - output = { - "tables": tables, - "figures": figures, - "text_blocks": text_blocks, - "metadata": { - "source": "marker", - "pdf_path": str(pdf_path), - "pages_processed": pages or "all", - "total_elements": len(tables) + len(figures) + len(text_blocks), - "processing_time": None, - }, + async with AsyncSessionLocal() as db: + document = await db.get(Document, document_id) + if document is not None: + metadata = DocumentProcessingMetadata(**(document.metadata_ or {})) + metadata.layout_metadata = layout + document.metadata_ = metadata.model_dump() + await db.commit() + + result = { + "document_id": str(document_id), + "parser": parser_name, + **layout.model_dump(), } - pprint(output) - - # Update job metadata with outputs - # current_job.meta.update( - # { - # "layout_extraction_complete": True, - # "tables_found": len(tables), - # "figures_found": len(figures), - # "text_blocks_found": len(text_blocks), - # } - # ) - # current_job.save_meta() + if current_job is not None: + current_job.meta["layout_complete"] = True + current_job.save_meta() - _LOGGER.info(f"Marker layout extraction completed. Found {len(tables)} tables, {len(figures)} figures") - return output + _LOGGER.info(f"Layout extraction complete for {document_id}: {layout.num_items} items") + return result except Exception as e: - _LOGGER.error(f"Error in marker layout extraction job: {e}", exc_info=True) - # current_job.meta["error"] = str(e) - # current_job.save_meta() + _LOGGER.error(f"Error in layout extraction for document {document_id}: {e}", exc_info=True) + if current_job is not None: + current_job.meta["error"] = str(e) + current_job.save_meta() raise - - -def create_marker_config(pages: Optional[str] = None) -> tuple[dict[str, Any], dict[str, Any]]: - """ - Create optimized Marker configuration for layout detection only (no OCR). - - Args: - pages: Optional comma-separated page numbers to process - - Returns: - Tuple of (config_dict, model_dict) for Marker - """ - # Configure for JSON output and layout detection only - config_dict = { - "output_format": "json", - "force_ocr": False, - "paginate_output": False, - "extract_images": False, # Skip image extraction for speed - } - - if pages is not None: - config_dict["page_range"] = pages - - # Create model dict - keep all models to avoid dependency resolution issues - # Models will be loaded but won't be used for actual OCR due to configuration - model_dict = create_model_dict() - - return config_dict, model_dict - - -def run_marker(pdf_path: str, config_dict: dict[str, Any], model_dict: dict[str, Any]) -> "JSONOutput": - """ - Run Marker layout detection on a PDF. - - Args: - pdf_path: Path to the PDF file - config_dict: Marker configuration dictionary - model_dict: Marker model dictionary - - Returns: - JSONOutput object containing layout detection results - """ - # Use ConfigParser to properly set up the renderer - config_parser = ConfigParser(config_dict) - final_config = config_parser.generate_config_dict() - - converter = PdfConverter( - config=final_config, - artifact_dict=model_dict, - processor_list=config_parser.get_processors(), - renderer=config_parser.get_renderer(), - ) - - # This should return JSONOutput because of our config - result = converter(pdf_path) - - # Verify we got JSONOutput as expected - if not hasattr(result, "model_dump"): - raise ValueError(f"Expected a Pydantic model with model_dump (like JSONOutput), but got {type(result)}") - - return result - - -def parse_marker_output(result: "JSONOutput") -> dict[str, Any]: - """ - Parse Marker JSONOutput into our application's expected layout format. - - Args: - result: JSONOutput object from Marker - - Returns: - A dictionary with a structured list of pages and their blocks. - """ - layout_data = {"pages": []} - - if result.children: - for page_idx, page in enumerate(result.children): - page_data = {"page": page_idx, "blocks": []} - - if page.children: - for block in page.children: - block_data = { - "type": block.block_type or "unknown", - "bbox": block.bbox or [], - "content": (block.html or "").strip(), - "id": block.id or "", - "score": None, # Marker doesn't provide confidence scores - } - page_data["blocks"].append(block_data) - - layout_data["pages"].append(page_data) - - return layout_data - - -if __name__ == "__main__": - import argparse - import asyncio - import json - from uuid import UUID - - parser = argparse.ArgumentParser(description="Test async_marker_layout_job from CLI.") - parser.add_argument("pdf_path", type=str, help="Path to the PDF file to process.") - parser.add_argument( - "--pages", - type=str, - default=None, - help="Comma-separated list of page numbers to process (0-indexed). If omitted, all pages are processed.", - ) - parser.add_argument( - "--extract-text", action="store_true", help="Extract text blocks in addition to tables/figures." - ) - args = parser.parse_args() - - pdf_path: str = args.pdf_path - pages: str = args.pages - extract_text: bool = args.extract_text - - async def _main(): - # Call the underlying logic directly, not as an RQ job - result = await async_marker_layout_job( - pdf_path=pdf_path, - pages=pages, - extract_text=extract_text, - ) - print(json.dumps(result, indent=2, ensure_ascii=False)) - - asyncio.run(_main()) diff --git a/extralit-server/src/extralit_server/jobs/preload.py b/extralit-server/src/extralit_server/jobs/preload.py index b1e24eb09..60bb95ea7 100644 --- a/extralit-server/src/extralit_server/jobs/preload.py +++ b/extralit-server/src/extralit_server/jobs/preload.py @@ -17,7 +17,7 @@ document_jobs, hub_jobs, import_jobs, - # ocr_jobs, + ocr_jobs, webhook_jobs, ) from extralit_server.models.database import Dataset, Document, Record, User, Workspace # noqa: F401 diff --git a/extralit-server/src/extralit_server/jobs/queues.py b/extralit-server/src/extralit_server/jobs/queues.py index d07e35486..9f4b46b84 100644 --- a/extralit-server/src/extralit_server/jobs/queues.py +++ b/extralit-server/src/extralit_server/jobs/queues.py @@ -12,6 +12,5 @@ DEFAULT_QUEUE = Queue("default", connection=REDIS_CONNECTION) HIGH_QUEUE = Queue("high", connection=REDIS_CONNECTION) OCR_QUEUE = Queue("ocr", connection=REDIS_CONNECTION) -GPU_QUEUE = Queue("gpu", connection=REDIS_CONNECTION) JOB_TIMEOUT_DISABLED = -1 diff --git a/extralit-server/src/extralit_server/workflows/documents.py b/extralit-server/src/extralit_server/workflows/documents.py index ac6ccd909..90f8ddf38 100644 --- a/extralit-server/src/extralit_server/workflows/documents.py +++ b/extralit-server/src/extralit_server/workflows/documents.py @@ -5,6 +5,7 @@ from extralit_server.database import AsyncSessionLocal from extralit_server.jobs.document_jobs import analysis_and_preprocess_job +from extralit_server.jobs.ocr_jobs import async_document_layout_job from extralit_server.jobs.queues import DEFAULT_QUEUE, OCR_QUEUE, REDIS_CONNECTION from extralit_server.models.database import DocumentWorkflow @@ -12,7 +13,12 @@ async def create_document_workflow( - document_id: UUID, s3_url: str, reference: str, workspace_name: str, workspace_id: UUID + document_id: UUID, + s3_url: str, + reference: str, + workspace_name: str, + workspace_id: UUID, + layout_parser: str | None = None, ) -> Group: """ Start PDF processing workflow using RQ Groups for job tracking. @@ -26,6 +32,7 @@ async def create_document_workflow( reference: Reference key for tracking workspace_name: Workspace name for job context workspace_id: UUID of the workspace + layout_parser: Name of a layout parser to run; None skips layout extraction Returns: Dictionary containing workflow_id and group_id for tracking @@ -75,8 +82,26 @@ async def create_document_workflow( }, ) + ocr_job_datas = [text_extraction_job_data] + + if layout_parser: + ocr_job_datas.append( + OCR_QUEUE.prepare_data( + async_document_layout_job, + (document_id, s3_url, workspace_name, layout_parser), + timeout=1800, + job_id=f"document_layout_{document_id}", + meta={ + "document_id": str(document_id), + "reference": reference, + "workflow_step": "document_layout", + "workflow_id": str(workflow.id), + }, + ) + ) + group.enqueue_many(queue=DEFAULT_QUEUE, job_datas=[analysis_job_data]) - group.enqueue_many(queue=OCR_QUEUE, job_datas=[text_extraction_job_data]) + group.enqueue_many(queue=OCR_QUEUE, job_datas=ocr_job_datas) # Step 6: Future table extraction job (conditional based on analysis results) # This will be added when table extraction is implemented diff --git a/extralit-server/tests/fixtures/pdf/generate.py b/extralit-server/tests/fixtures/pdf/generate.py new file mode 100644 index 000000000..1a9a82ca3 --- /dev/null +++ b/extralit-server/tests/fixtures/pdf/generate.py @@ -0,0 +1,170 @@ +"""Regenerate the layout fixture PDFs. + +Committed output: `sample.pdf` (untagged) and `sample_tagged.pdf` (structure tree + MCIDs). +Run with `uv run python tests/fixtures/pdf/generate.py`. Uses only pikepdf, already a dependency, +so the fixtures stay reproducible without pulling in a PDF writer. +""" + +import zlib +from pathlib import Path + +import pikepdf +from pikepdf import Array, Dictionary, Name, Stream, String + +HERE = Path(__file__).parent + +PAGE_WIDTH = 612.0 +PAGE_HEIGHT = 792.0 + +# All coordinates below are PDF-native (bottom-left origin), which is what the parsers must flip. +TITLE = ("A Study of Layout Extraction", 72, 720, 18) +HEADING = ("Methods", 72, 680, 14) +BODY = ("We evaluated two parsers on a shared corpus of documents.", 72, 655, 11) + +TABLE_LEFT, TABLE_RIGHT = 72, 372 +TABLE_TOP, TABLE_BOTTOM = 600, 540 +TABLE_MID_Y = 570 +TABLE_MID_X = 222 +TABLE_CELLS = [ + ("Group", 80, 580), + ("N", 230, 580), + ("control", 80, 550), + ("42", 230, 550), +] + +IMAGE_X, IMAGE_Y, IMAGE_W, IMAGE_H = 72, 380, 120, 90 +CAPTION = ("Figure 1. A red square.", 72, 360, 10) + + +def _text_op(text: str, x: float, y: float, size: float, font: str = "F1") -> str: + escaped = text.replace("\\", r"\\").replace("(", r"\(").replace(")", r"\)") + return f"BT /{font} {size} Tf 1 0 0 1 {x} {y} Tm ({escaped}) Tj ET\n" + + +def _table_ops() -> str: + return ( + "0.5 w 0 0 0 RG\n" + f"{TABLE_LEFT} {TABLE_BOTTOM} {TABLE_RIGHT - TABLE_LEFT} {TABLE_TOP - TABLE_BOTTOM} re S\n" + f"{TABLE_LEFT} {TABLE_MID_Y} m {TABLE_RIGHT} {TABLE_MID_Y} l S\n" + f"{TABLE_MID_X} {TABLE_BOTTOM} m {TABLE_MID_X} {TABLE_TOP} l S\n" + ) + + +def _image_ops() -> str: + return f"q {IMAGE_W} 0 0 {IMAGE_H} {IMAGE_X} {IMAGE_Y} cm /Im1 Do Q\n" + + +def _make_image(pdf: pikepdf.Pdf) -> Stream: + width, height = 8, 6 + raw = bytes([255, 0, 0] * width * height) + image = Stream(pdf, zlib.compress(raw)) + image.Type = Name.XObject + image.Subtype = Name.Image + image.Width = width + image.Height = height + image.ColorSpace = Name.DeviceRGB + image.BitsPerComponent = 8 + image.Filter = Name.FlateDecode + return image + + +def _base_pdf() -> tuple[pikepdf.Pdf, Dictionary]: + pdf = pikepdf.Pdf.new() + font = pdf.make_indirect( + Dictionary( + Type=Name.Font, + Subtype=Name.Type1, + BaseFont=Name.Helvetica, + Encoding=Name.WinAnsiEncoding, + ) + ) + resources = Dictionary(Font=Dictionary(F1=font), XObject=Dictionary(Im1=_make_image(pdf))) + return pdf, resources + + +def _add_page(pdf: pikepdf.Pdf, resources: Dictionary, content: str) -> Dictionary: + stream = Stream(pdf, content.encode("latin-1")) + page = pdf.make_indirect( + Dictionary( + Type=Name.Page, + MediaBox=Array([0, 0, PAGE_WIDTH, PAGE_HEIGHT]), + Resources=resources, + Contents=stream, + ) + ) + pdf.pages.append(pikepdf.Page(page)) + return page + + +def build_untagged(path: Path) -> None: + pdf, resources = _base_pdf() + content = ( + _text_op(*TITLE) + + _text_op(*HEADING) + + _text_op(*BODY) + + _table_ops() + + "".join(_text_op(t, x, y, 10) for t, x, y in TABLE_CELLS) + + _image_ops() + + _text_op(*CAPTION) + ) + _add_page(pdf, resources, content) + pdf.save(path) + + +def build_tagged(path: Path) -> None: + """Same content, wrapped in marked-content sequences and a structure tree.""" + pdf, resources = _base_pdf() + + def marked(tag: str, mcid: int, ops: str) -> str: + return f"/{tag} <> BDC\n{ops}EMC\n" + + content = ( + marked("H1", 0, _text_op(*TITLE)) + + marked("H2", 1, _text_op(*HEADING)) + + marked("P", 2, _text_op(*BODY)) + + marked( + "Table", + 3, + _table_ops() + "".join(_text_op(t, x, y, 10) for t, x, y in TABLE_CELLS), + ) + + marked("Figure", 4, _image_ops()) + + marked("Caption", 5, _text_op(*CAPTION)) + ) + page = _add_page(pdf, resources, content) + + struct_root = pdf.make_indirect(Dictionary(Type=Name.StructTreeRoot)) + kids = Array() + # Role names here are non-standard on purpose; /RoleMap is what resolves them. + for tag, mcid in [("H1", 0), ("H2", 1), ("P", 2), ("Table", 3), ("Figure", 4), ("Caption", 5)]: + kids.append( + pdf.make_indirect( + Dictionary( + Type=Name.StructElem, + S=Name("/" + tag), + P=struct_root, + Pg=page, + K=mcid, + ) + ) + ) + struct_root.K = kids + struct_root.RoleMap = Dictionary( + **{ + "H1": Name.H1, + "H2": Name.H2, + "P": Name.P, + "Table": Name.Table, + "Figure": Name.Figure, + "Caption": Name.Caption, + } + ) + pdf.Root.StructTreeRoot = struct_root + pdf.Root.MarkInfo = Dictionary(Marked=True) + pdf.Root.Lang = String("en-US") + pdf.save(path) + + +if __name__ == "__main__": + build_untagged(HERE / "sample.pdf") + build_tagged(HERE / "sample_tagged.pdf") + print(f"wrote {HERE / 'sample.pdf'} and {HERE / 'sample_tagged.pdf'}") diff --git a/extralit-server/tests/fixtures/pdf/sample.pdf b/extralit-server/tests/fixtures/pdf/sample.pdf new file mode 100644 index 0000000000000000000000000000000000000000..44267e28ce6b2d0063108f275775b78859e163f5 GIT binary patch literal 1177 zcmY!laBP0Gbt$SOioEb5>Alaeir0GSHnM7rW&HWxxzW z1yis=c6JK-5&lV8smTxt&s;+VGnfoW65=nA(~vv`asB^e4v=B7aNGjmE( zi-2@aVo9n?YI1%GSXXgLQEFl?SH+yx38(!I8;H1k|JxOLNNww63*FOUTSSEv9F;e2 z6Phk|wD1-C()#|n>=!s1-?W-qS?w#IX1Oez%gTKpGwa9pB(dM#6TD+|H*8&J;jySct`2)XpDNV$x-r@5 z@K(WF4p$!-RIjUz)3|FPd`v6!;gS_=JKOj@jud3=WjQ~)Gko(D1w|IgMzI^)F7`Mm zy^NpA|9A(p>LG3ASSugzI=|QA$$yKa*Txvfy=Cf~od4^izfndcyV$C2QKA34tL=K( zzi>#E@4i1FAp6nU)+1m1g6?S9{C&rjaBbza_VDM?iv-0uLT4}jJhgu7Z_Y0@?bAT1 z1R6Tf^a6?&SkO8p7N>%vOy46lr!2K3GdWQ~-!%^yteJV~3i{!hd5(F-nFxX4(xeh_ zG6Y2$FquK(4r~GAaX@O3bAE0?ejX^*SOAl7eolT-a6w`+Fgd%VmSrZV z2Dv+7k6NI!GSf3kfC)w)ZnbAFD5*fLhNUc^Q&WIx1ZX-mwHTU!QWHE~{62r+L%o3_ z!|rbVi5e42^)f3Qf$& z+#Ey9%m`Djxe+k;!}OLEC1&OT(>f?O1?A_1vNgCI2+piZRWMi3_jFN+wn;WkF-}c0 zNi|7IHa9X&G%`*#Of^qQwlp(Ou}DryOS2=aA{G)*iA5#gV6d>X^a}0L3hZ ADF6Tf literal 0 HcmV?d00001 diff --git a/extralit-server/tests/fixtures/pdf/sample_tagged.pdf b/extralit-server/tests/fixtures/pdf/sample_tagged.pdf new file mode 100644 index 0000000000000000000000000000000000000000..a879835ec366cfeb5109f7595102958b19dda96e GIT binary patch literal 2055 zcmah~e{2&~99J}oHv@@Vs>VbfgqjT)*X#9XSO#0Sxv|yNc4G)E;d*^7hqt?Wcb$WQ zKQl!)0&E$}BE)Qlk*Qe*$>tvz!JvpSY|bENCXOYt1!DrnnBfmZzwfTs&X&zf@80$M z-uLr9@B8_Fzh@1URTMJoS;~6$R{tG}K{h0ZcT+`0h_2zJ7|O>|VST6o(SA;OzDkP9 zh)j_`ID#}Kfl+ZWq61tEtH=({K}3hZ%xiTD#z9%u5NlZL_QWxweVoRLatxd(EJesp z#L3*zCnqEgF^H}fA}ZRB9J*^Gq77$bm$WNEHeIV>xl>>6(w}bq>CvBF{mIz$0nmc7 zi2Yn#KZ%cvYl18hX33WdA(l{psyS8yAN{t&2zNmL)p21F78wHxfk83j>}js#C?;2* zENNKMRK$>QL&W?zB5@G2sII*0$_|627w1b zB1E>`6e3(2MUVI0xPS_afDql&AnZa^OcC|eG^EQo6G@#kkYgbrGglU&q^DmR z%DE6!Di0UK`cIi+DW>g0EDMkof~a8yMv>ES8Rq4PF0-mBm}{b1jA6z9e+A8BdqQ`t?=6^#Ae`qj>w0~24j&tCdT{azhDns>f$WUe7} zcw%YS;MI5TAD+$oZP}-mywCRK-50f#9G;GL99-X@d(E=^_UfF2lRaCCF1Pkht@{1k zy6cy&yfnV;^c(&S;>q)KH`dH;yMDamSIg{R-|(`w=7OQtfsd&JwNE~}qkZY0`A1&p z&gu3IOkaF)NAI5e`YqRH)~XSHO8(=4+l7{cA2qgiebWBpWal)VV26f$J9C#D z+Vqe(_TIfEOU_Jv9=*8LS`)ZY6!g=}S8iMp?vMZMv23HdQDGjaFl!E3)`zBoAU=A8UL1hH#T$Vn`6xz zj(0wDaW zM3+mTi2!|-ZV;prNfpwDP$I19iAaQp>M``!5z=i$FXq$^f~E$r;**==vP9a12ilk{ z%1S8C@z9*gaI?VU;M1jdPYmEC#CB=O1ufg$yQ+zFhUDI6CjqVz$S^>!F>;M@kg`u@ z)nDW1etFrp@}4v8^h9=0S`-|mY>07Cw$z1OPABAR6s2qo^dNE}DP?2qP<@#;50rL> zjdelO%&<8b=p&gnPqxkF%(l7h(8)4<-QH|29yc_mjCEc|);fmG-h**C=bcwmI6;K6 zB~4xbWF|igAz?3uxFNc#3~evsoe>sC9oP}(-F7EuXEB4_5#H-^M?8Ea8ZG|+6^({$ SoTBNXFfP_XS*_)@71X~1b#?>* literal 0 HcmV?d00001 diff --git a/extralit-server/tests/unit/api/handlers/v1/test_document_layout.py b/extralit-server/tests/unit/api/handlers/v1/test_document_layout.py new file mode 100644 index 000000000..a2191cff9 --- /dev/null +++ b/extralit-server/tests/unit/api/handlers/v1/test_document_layout.py @@ -0,0 +1,200 @@ +"""Tests for `GET /documents/{document_id}/layout`.""" + +from unittest.mock import AsyncMock, patch +from uuid import uuid4 + +import pytest +from docling_core.types.doc import BoundingBox, CoordOrigin, DocItemLabel, Size +from docling_core.types.doc.document import CURRENT_VERSION +from httpx import AsyncClient +from sqlalchemy.ext.asyncio import AsyncSession + +from extralit_server.contexts.ocr.docling_builder import ( + LayoutBlock, + PageContext, + append_blocks, + new_document, +) +from extralit_server.models.database import Document +from tests.factories import DocumentFactory, WorkspaceFactory + +PAGE_WIDTH, PAGE_HEIGHT = 612.0, 792.0 + + +def bbox(t: float, b: float, left: float = 10.0, right: float = 100.0) -> BoundingBox: + return BoundingBox(l=left, t=t, r=right, b=b, coord_origin=CoordOrigin.TOPLEFT) + + +def build_layout(): + doc = new_document("sample") + append_blocks( + doc, + PageContext(page_no=1, size=Size(width=PAGE_WIDTH, height=PAGE_HEIGHT)), + [ + LayoutBlock(label=DocItemLabel.SECTION_HEADER, bbox=bbox(t=10, b=40), text="Methods", level=2), + LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=50, b=90), text="Body text."), + LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=200, b=400)), + ], + ) + append_blocks( + doc, + PageContext(page_no=2, size=Size(width=PAGE_WIDTH, height=PAGE_HEIGHT)), + [LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=10, b=30), text="Page two.")], + ) + return doc + + +async def make_document(db: AsyncSession, layout_metadata: dict | None) -> Document: + workspace = await WorkspaceFactory.create(name=f"ws-{uuid4().hex[:8]}") + metadata = {"layout_metadata": layout_metadata} if layout_metadata else {} + return await DocumentFactory.create(workspace_id=workspace.id, metadata_=metadata) + + +LAYOUT_METADATA = { + "layout_url": "layout/doc.docling.json", + "parser": "pdf_inspector", + "docling_version": CURRENT_VERSION, + "num_items": 4, + "num_pages": 2, + "pages_needing_ocr": [], +} + + +@pytest.fixture +def load_layout(): + with patch( + "extralit_server.api.handlers.v1.documents.storage.load_layout", + new=AsyncMock(return_value=build_layout()), + ) as mock: + yield mock + + +@pytest.mark.asyncio +class TestGetDocumentLayout: + async def test_returns_the_projected_layout(self, async_client: AsyncClient, db, owner_auth_header, load_layout): + document = await make_document(db, LAYOUT_METADATA) + + response = await async_client.get(f"/api/v1/documents/{document.id}/layout", headers=owner_auth_header) + + assert response.status_code == 200 + body = response.json() + assert body["document_id"] == str(document.id) + assert body["docling_version"] == CURRENT_VERSION + assert body["num_items"] == 4 + assert body["num_pages"] == 2 + + async def test_items_carry_the_full_provenance_triple( + self, async_client: AsyncClient, db, owner_auth_header, load_layout + ): + document = await make_document(db, LAYOUT_METADATA) + + body = (await async_client.get(f"/api/v1/documents/{document.id}/layout", headers=owner_auth_header)).json() + + for item in body["items"]: + for prov in item["prov"]: + assert prov["page_no"] >= 1 + assert set(prov["bbox"]) >= {"l", "t", "r", "b", "coord_origin"} + assert len(prov["charspan"]) == 2 + + async def test_self_ref_is_returned_as_the_citation_anchor( + self, async_client: AsyncClient, db, owner_auth_header, load_layout + ): + document = await make_document(db, LAYOUT_METADATA) + + body = (await async_client.get(f"/api/v1/documents/{document.id}/layout", headers=owner_auth_header)).json() + + assert all(item["self_ref"].startswith("#/") for item in body["items"]) + + async def test_items_are_in_reading_order(self, async_client: AsyncClient, db, owner_auth_header, load_layout): + document = await make_document(db, LAYOUT_METADATA) + + body = (await async_client.get(f"/api/v1/documents/{document.id}/layout", headers=owner_auth_header)).json() + + orders = [item["reading_order"] for item in body["items"]] + assert orders == sorted(orders) + + async def test_pages_carry_their_geometry(self, async_client: AsyncClient, db, owner_auth_header, load_layout): + document = await make_document(db, LAYOUT_METADATA) + + body = (await async_client.get(f"/api/v1/documents/{document.id}/layout", headers=owner_auth_header)).json() + + assert [p["page_no"] for p in body["pages"]] == [1, 2] + assert body["pages"][0]["height"] == PAGE_HEIGHT + + async def test_heading_level_is_exposed(self, async_client: AsyncClient, db, owner_auth_header, load_layout): + document = await make_document(db, LAYOUT_METADATA) + + body = (await async_client.get(f"/api/v1/documents/{document.id}/layout", headers=owner_auth_header)).json() + + heading = next(i for i in body["items"] if i["label"] == "section_header") + assert heading["level"] == 2 + + async def test_pages_filter_narrows_the_response( + self, async_client: AsyncClient, db, owner_auth_header, load_layout + ): + document = await make_document(db, LAYOUT_METADATA) + + response = await async_client.get( + f"/api/v1/documents/{document.id}/layout", params={"pages": [2]}, headers=owner_auth_header + ) + + body = response.json() + assert [p["page_no"] for p in body["pages"]] == [2] + assert {prov["page_no"] for i in body["items"] for prov in i["prov"]} == {2} + + async def test_labels_filter_narrows_the_response( + self, async_client: AsyncClient, db, owner_auth_header, load_layout + ): + document = await make_document(db, LAYOUT_METADATA) + + response = await async_client.get( + f"/api/v1/documents/{document.id}/layout", params={"labels": ["table"]}, headers=owner_auth_header + ) + + body = response.json() + assert [i["label"] for i in body["items"]] == ["table"] + + async def test_missing_document_is_404(self, async_client: AsyncClient, owner_auth_header): + response = await async_client.get(f"/api/v1/documents/{uuid4()}/layout", headers=owner_auth_header) + + assert response.status_code == 404 + + async def test_document_without_extracted_layout_is_404(self, async_client: AsyncClient, db, owner_auth_header): + document = await make_document(db, layout_metadata=None) + + response = await async_client.get(f"/api/v1/documents/{document.id}/layout", headers=owner_auth_header) + + assert response.status_code == 404 + assert "No layout" in response.json()["detail"] + + async def test_unreadable_stored_layout_is_404(self, async_client: AsyncClient, db, owner_auth_header): + document = await make_document(db, LAYOUT_METADATA) + + with patch( + "extralit_server.api.handlers.v1.documents.storage.load_layout", + new=AsyncMock(side_effect=RuntimeError("object missing")), + ): + response = await async_client.get(f"/api/v1/documents/{document.id}/layout", headers=owner_auth_header) + + assert response.status_code == 404 + + async def test_layout_from_a_newer_docling_is_409(self, async_client: AsyncClient, db, owner_auth_header): + from pydantic import ValidationError + + document = await make_document(db, LAYOUT_METADATA) + error = ValidationError.from_exception_data("DoclingDocument", []) + + with patch( + "extralit_server.api.handlers.v1.documents.storage.load_layout", + new=AsyncMock(side_effect=error), + ): + response = await async_client.get(f"/api/v1/documents/{document.id}/layout", headers=owner_auth_header) + + assert response.status_code == 409 + + async def test_requires_authentication(self, async_client: AsyncClient, db, load_layout): + document = await make_document(db, LAYOUT_METADATA) + + response = await async_client.get(f"/api/v1/documents/{document.id}/layout") + + assert response.status_code == 401 diff --git a/extralit-server/tests/unit/contexts/ocr/__init__.py b/extralit-server/tests/unit/contexts/ocr/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/extralit-server/tests/unit/contexts/ocr/test_arrow.py b/extralit-server/tests/unit/contexts/ocr/test_arrow.py new file mode 100644 index 000000000..547ece47d --- /dev/null +++ b/extralit-server/tests/unit/contexts/ocr/test_arrow.py @@ -0,0 +1,212 @@ +"""Tests for the columnar projection of a `DoclingDocument`.""" + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest +from docling_core.types.doc import BoundingBox, CoordOrigin, DocItemLabel, ProvenanceItem, Size + +from extralit_server.contexts.ocr.arrow import ( + ITEM_SCHEMA, + PAGE_SCHEMA, + items_table, + pages_table, +) +from extralit_server.contexts.ocr.docling_builder import ( + LayoutBlock, + PageContext, + append_blocks, + new_document, +) + +DOCUMENT_ID = "11111111-2222-3333-4444-555555555555" + + +def bbox(t: float, b: float, left: float = 10.0, right: float = 100.0) -> BoundingBox: + return BoundingBox(l=left, t=t, r=right, b=b, coord_origin=CoordOrigin.TOPLEFT) + + +def _cells(): + from extralit_server.contexts.ocr.tables import make_cell + + return [ + make_cell("Group", row=0, col=0, column_header=True), + make_cell("N", row=0, col=1, column_header=True), + make_cell("control", row=1, col=0), + make_cell("42", row=1, col=1), + ] + + +@pytest.fixture +def doc(): + document = new_document("sample") + ctx = PageContext(page_no=1, size=Size(width=612, height=792)) + append_blocks( + document, + ctx, + [ + LayoutBlock(label=DocItemLabel.TITLE, bbox=bbox(t=10, b=40), text="A Paper"), + LayoutBlock(label=DocItemLabel.SECTION_HEADER, bbox=bbox(t=50, b=70), text="Methods", level=2), + LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=80, b=120), text="Body text here."), + LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=200, b=400), cells=_cells()), + LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=450, b=600)), + ], + ) + ctx2 = PageContext(page_no=2, size=Size(width=612, height=1008)) + append_blocks(document, ctx2, [LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=10, b=30), text="Page two.")]) + return document + + +class TestItemsTable: + def test_schema_matches_exactly(self, doc): + assert items_table(doc, DOCUMENT_ID).schema.equals(ITEM_SCHEMA) + + def test_row_count_equals_total_provenance_entries(self, doc): + expected = sum(len(item.prov) for item, _ in doc.iterate_items(with_groups=False)) + + assert items_table(doc, DOCUMENT_ID).num_rows == expected + + def test_multi_prov_item_expands_to_multiple_rows(self, doc): + # A table spanning a page break carries two provenance entries. + doc.tables[0].prov.append( + ProvenanceItem(page_no=2, bbox=bbox(t=10, b=90), charspan=(0, 0)), + ) + + table = items_table(doc, DOCUMENT_ID) + rows = table.filter(pa.compute.equal(table["self_ref"], "#/tables/0")).to_pylist() + + assert [r["prov_index"] for r in rows] == [0, 1] + assert [r["page_no"] for r in rows] == [1, 2] + + def test_an_item_without_provenance_still_yields_one_row(self): + document = new_document("sample") + document.add_text(label=DocItemLabel.TEXT, text="no prov") + + rows = items_table(document, DOCUMENT_ID).to_pylist() + + assert len(rows) == 1 + assert rows[0]["page_no"] is None + assert rows[0]["bbox"] is None + assert rows[0]["prov_index"] == 0 + + def test_self_ref_is_the_citation_anchor(self, doc): + refs = items_table(doc, DOCUMENT_ID).column("self_ref").to_pylist() + + assert "#/tables/0" in refs + assert "#/pictures/0" in refs + assert all(r.startswith("#/") for r in refs) + + def test_reading_order_is_dense_and_monotonic(self, doc): + orders = items_table(doc, DOCUMENT_ID).column("reading_order").to_pylist() + + assert orders == sorted(orders) + assert set(orders) == set(range(len(set(orders)))) + + def test_bbox_is_a_fixed_four_float_list(self, doc): + table = items_table(doc, DOCUMENT_ID) + + assert table.schema.field("bbox").type == pa.list_(pa.float32(), 4) + first = table.column("bbox")[0].as_py() + assert len(first) == 4 + + def test_bbox_preserves_ltrb_order(self, doc): + table = items_table(doc, DOCUMENT_ID) + row = next(r for r in table.to_pylist() if r["self_ref"] == "#/tables/0") + + assert row["bbox"] == pytest.approx([10.0, 200.0, 100.0, 400.0]) + + def test_charspan_is_item_local(self, doc): + table = items_table(doc, DOCUMENT_ID) + rows = {r["self_ref"]: r for r in table.to_pylist()} + + assert rows["#/texts/0"]["charspan_start"] == 0 + assert rows["#/texts/0"]["charspan_end"] == len("A Paper") + assert rows["#/tables/0"]["charspan_end"] == 0 + + def test_heading_level_is_carried(self, doc): + rows = {r["self_ref"]: r for r in items_table(doc, DOCUMENT_ID).to_pylist()} + + assert rows["#/texts/1"]["label"] == DocItemLabel.SECTION_HEADER.value + assert rows["#/texts/1"]["level"] == 2 + + def test_non_heading_items_have_no_level(self, doc): + rows = {r["self_ref"]: r for r in items_table(doc, DOCUMENT_ID).to_pylist()} + + assert rows["#/texts/2"]["level"] is None + + def test_table_html_is_populated(self, doc): + rows = {r["self_ref"]: r for r in items_table(doc, DOCUMENT_ID).to_pylist()} + + assert rows["#/tables/0"]["html"] is not None + assert rows["#/tables/0"]["html"].startswith(" BoundingBox: + return BoundingBox(l=left, t=t, r=right, b=b, coord_origin=CoordOrigin.TOPLEFT) + + +class TestNewDocument: + def test_sets_name_and_current_version(self, doc): + from docling_core.types.doc.document import CURRENT_VERSION + + assert doc.name == "sample" + assert doc.version == CURRENT_VERSION + + def test_origin_is_recorded_when_given(self): + doc = new_document("sample", filename="paper.pdf", binary_hash=1234) + + assert doc.origin is not None + assert doc.origin.filename == "paper.pdf" + assert doc.origin.mimetype == "application/pdf" + + +class TestRegisterPage: + def test_registers_size_under_one_indexed_page_no(self, doc, ctx): + register_page(doc, ctx) + + assert doc.pages[1].size.width == PAGE_WIDTH + assert doc.pages[1].size.height == PAGE_HEIGHT + + def test_is_idempotent(self, doc, ctx): + register_page(doc, ctx) + register_page(doc, ctx) + + assert len(doc.pages) == 1 + + +class TestFlipToTopLeft: + def test_bottom_left_origin_is_flipped(self): + # A box 100pt tall sitting 100pt above the page bottom. + bl = BoundingBox(l=10, b=100, r=50, t=200, coord_origin=CoordOrigin.BOTTOMLEFT) + + flipped = flip_to_top_left(bl, PAGE_HEIGHT) + + assert flipped.coord_origin == CoordOrigin.TOPLEFT + assert flipped.t == pytest.approx(PAGE_HEIGHT - 200) + assert flipped.b == pytest.approx(PAGE_HEIGHT - 100) + assert (flipped.l, flipped.r) == (10, 50) + + def test_top_left_origin_is_left_alone(self): + tl = bbox(t=10, b=30) + + assert flip_to_top_left(tl, PAGE_HEIGHT) == tl + + def test_flipped_box_keeps_its_height(self): + bl = BoundingBox(l=0, b=100, r=10, t=200, coord_origin=CoordOrigin.BOTTOMLEFT) + + flipped = flip_to_top_left(bl, PAGE_HEIGHT) + + assert flipped.height == pytest.approx(bl.height) + + +class TestMakeProv: + def test_charspan_covers_the_whole_text(self, ctx): + prov = make_prov(ctx, bbox(t=10, b=30), text="hello world") + + assert prov.page_no == 1 + assert prov.charspan == (0, len("hello world")) + + def test_charspan_is_zero_for_non_text_items(self, ctx): + prov = make_prov(ctx, bbox(t=10, b=30), text=None) + + assert prov.charspan == (0, 0) + + def test_bbox_is_clamped_into_the_page(self, ctx): + prov = make_prov(ctx, bbox(t=10, b=30, left=-50, right=PAGE_WIDTH + 50), text="x") + + assert prov.bbox.l == 0.0 + assert prov.bbox.r == PAGE_WIDTH + + +class TestIsContained: + def test_text_inside_an_existing_table_is_contained(self, doc, ctx): + register_page(doc, ctx) + append_blocks(doc, ctx, [LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=100, b=300, left=0, right=500))]) + + assert is_contained(doc, bbox(t=150, b=170, left=10, right=200), page_no=1) + + def test_text_outside_every_table_is_not_contained(self, doc, ctx): + register_page(doc, ctx) + append_blocks(doc, ctx, [LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=100, b=300, left=0, right=500))]) + + assert not is_contained(doc, bbox(t=400, b=420, left=10, right=200), page_no=1) + + def test_overlap_below_threshold_is_not_contained(self, doc, ctx): + register_page(doc, ctx) + append_blocks(doc, ctx, [LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=100, b=200, left=0, right=100))]) + + # Half of this box lies inside the table -> IoSelf 0.5, under the 0.6 default. + assert not is_contained(doc, bbox(t=150, b=250, left=0, right=100), page_no=1) + + def test_overlap_above_threshold_is_contained(self, doc, ctx): + register_page(doc, ctx) + append_blocks(doc, ctx, [LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=100, b=200, left=0, right=100))]) + + # 70% of this box lies inside the table. + assert is_contained(doc, bbox(t=130, b=230, left=0, right=100), page_no=1) + + def test_containment_is_scoped_to_the_page(self, doc, ctx): + register_page(doc, ctx) + append_blocks(doc, ctx, [LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=100, b=300, left=0, right=500))]) + + assert not is_contained(doc, bbox(t=150, b=170, left=10, right=200), page_no=2) + + def test_pictures_also_absorb_text(self, doc, ctx): + register_page(doc, ctx) + append_blocks(doc, ctx, [LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=100, b=300, left=0, right=500))]) + + assert is_contained(doc, bbox(t=150, b=170, left=10, right=200), page_no=1) + + +class TestAppendBlocks: + def test_tables_and_pictures_are_added_before_text(self, doc, ctx): + blocks = [ + LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=10, b=30), text="intro"), + LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=400, b=500)), + LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=600, b=700)), + ] + + append_blocks(doc, ctx, blocks) + + assert len(doc.tables) == 1 + assert len(doc.pictures) == 1 + assert [t.text for t in doc.texts] == ["intro"] + + def test_reading_order_interleaves_text_tables_and_pictures(self, doc, ctx): + blocks = [ + LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=10, b=30), text="intro"), + LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=100, b=200)), + LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=250, b=270), text="middle"), + LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=300, b=400)), + LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=450, b=470), text="outro"), + ] + + append_blocks(doc, ctx, blocks) + + refs = [item.self_ref for item, _ in doc.iterate_items(with_groups=False)] + assert refs == ["#/texts/0", "#/tables/0", "#/texts/1", "#/pictures/0", "#/texts/2"] + + def test_reading_order_is_by_page_then_position(self, doc, ctx): + append_blocks(doc, ctx, [LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=700, b=750))]) + ctx2 = PageContext(page_no=2, size=Size(width=PAGE_WIDTH, height=PAGE_HEIGHT)) + append_blocks(doc, ctx2, [LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=10, b=30), text="page two top")]) + + order = [(i.prov[0].page_no, i.prov[0].bbox.t) for i, _ in doc.iterate_items(with_groups=False)] + assert order == sorted(order) + + def test_reading_order_of_text_is_preserved(self, doc, ctx): + blocks = [ + LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=10, b=30), text="first"), + LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=40, b=60), text="second"), + LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=70, b=90), text="third"), + ] + + append_blocks(doc, ctx, blocks) + + assert [t.text for t in doc.texts] == ["first", "second", "third"] + + def test_text_inside_a_table_is_dropped(self, doc, ctx): + blocks = [ + LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=100, b=300, left=0, right=500)), + LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=150, b=170, left=10, right=200), text="cell text"), + LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=400, b=420), text="body text"), + ] + + append_blocks(doc, ctx, blocks) + + assert [t.text for t in doc.texts] == ["body text"] + + def test_captions_are_kept_even_when_they_touch_a_figure(self, doc, ctx): + blocks = [ + LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=100, b=300, left=0, right=500)), + LayoutBlock(label=DocItemLabel.CAPTION, bbox=bbox(t=150, b=170, left=10, right=200), text="Figure 1."), + ] + + append_blocks(doc, ctx, blocks) + + assert [t.text for t in doc.texts] == ["Figure 1."] + + def test_every_item_carries_a_full_provenance_triple(self, doc, ctx): + blocks = [ + LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=10, b=30), text="hello"), + LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=400, b=500)), + LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=600, b=700)), + ] + + append_blocks(doc, ctx, blocks) + + for item, _ in doc.iterate_items(with_groups=False): + assert len(item.prov) == 1 + prov = item.prov[0] + assert prov.page_no == 1 + assert prov.bbox.coord_origin == CoordOrigin.TOPLEFT + assert prov.charspan is not None + + def test_headings_keep_their_level(self, doc, ctx): + blocks = [LayoutBlock(label=DocItemLabel.SECTION_HEADER, bbox=bbox(t=10, b=30), text="Methods", level=3)] + + append_blocks(doc, ctx, blocks) + + assert doc.texts[0].label == DocItemLabel.SECTION_HEADER + assert doc.texts[0].level == 3 + + def test_titles_become_title_items(self, doc, ctx): + blocks = [LayoutBlock(label=DocItemLabel.TITLE, bbox=bbox(t=10, b=30), text="A Paper")] + + append_blocks(doc, ctx, blocks) + + assert doc.texts[0].label == DocItemLabel.TITLE + + def test_empty_text_blocks_are_skipped(self, doc, ctx): + blocks = [ + LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=10, b=30), text=" "), + LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=40, b=60), text="real"), + ] + + append_blocks(doc, ctx, blocks) + + assert [t.text for t in doc.texts] == ["real"] + + def test_table_cells_are_carried_through(self, doc, ctx): + cells = [ + TableCell( + text="h1", + start_row_offset_idx=0, + end_row_offset_idx=1, + start_col_offset_idx=0, + end_col_offset_idx=1, + column_header=True, + ), + TableCell( + text="v1", + start_row_offset_idx=1, + end_row_offset_idx=2, + start_col_offset_idx=0, + end_col_offset_idx=1, + ), + ] + blocks = [LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=100, b=300), cells=cells)] + + append_blocks(doc, ctx, blocks) + + table = doc.tables[0] + assert table.data.num_rows == 2 + assert table.data.num_cols == 1 + assert table.data.table_cells[0].column_header is True + + def test_appending_a_second_page_extends_the_same_document(self, doc, ctx): + append_blocks(doc, ctx, [LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=10, b=30), text="page one")]) + ctx2 = PageContext(page_no=2, size=Size(width=PAGE_WIDTH, height=PAGE_HEIGHT)) + append_blocks(doc, ctx2, [LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=10, b=30), text="page two")]) + + assert [t.prov[0].page_no for t in doc.texts] == [1, 2] + assert set(doc.pages) == {1, 2} + + def test_document_round_trips_through_json(self, doc, ctx): + from docling_core.types.doc import DoclingDocument + + append_blocks( + doc, + ctx, + [ + LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=10, b=30), text="hello"), + LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=400, b=500)), + ], + ) + + restored = DoclingDocument.model_validate(doc.model_dump(mode="json")) + + assert restored.texts[0].text == "hello" + assert restored.tables[0].prov[0].page_no == 1 diff --git a/extralit-server/tests/unit/contexts/ocr/test_pdf_inspector_parser.py b/extralit-server/tests/unit/contexts/ocr/test_pdf_inspector_parser.py new file mode 100644 index 000000000..2f902add9 --- /dev/null +++ b/extralit-server/tests/unit/contexts/ocr/test_pdf_inspector_parser.py @@ -0,0 +1,199 @@ +"""Tests for the pdf-inspector layout parser. Always runs — pdf-inspector is a required dep.""" + +from pathlib import Path + +import pytest +from docling_core.types.doc import CoordOrigin, DocItemLabel + +from extralit_server.contexts.ocr.parsers import get_parser, list_parsers +from extralit_server.contexts.ocr.parsers.pdf_inspector import ( + classify, + page_sizes, + parse, + role_to_label, +) + +FIXTURES = Path(__file__).parents[3] / "fixtures" / "pdf" +PAGE_WIDTH, PAGE_HEIGHT = 612.0, 792.0 + + +@pytest.fixture(scope="module") +def untagged_bytes() -> bytes: + return (FIXTURES / "sample.pdf").read_bytes() + + +@pytest.fixture(scope="module") +def tagged_bytes() -> bytes: + return (FIXTURES / "sample_tagged.pdf").read_bytes() + + +@pytest.fixture(scope="module") +def untagged(untagged_bytes): + return parse(untagged_bytes, name="sample") + + +@pytest.fixture(scope="module") +def tagged(tagged_bytes): + return parse(tagged_bytes, name="sample_tagged") + + +class TestRegistry: + def test_pdf_inspector_is_always_registered(self): + assert "pdf_inspector" in list_parsers() + + def test_get_parser_returns_a_callable(self, untagged_bytes): + doc = get_parser("pdf_inspector")(untagged_bytes, name="sample") + + assert doc.name == "sample" + + def test_unknown_parser_raises(self): + with pytest.raises(ValueError, match="unknown"): + get_parser("nope") + + +class TestPageSizes: + def test_reads_mediabox_keyed_by_one_indexed_page_no(self, untagged_bytes): + sizes = page_sizes(untagged_bytes) + + assert set(sizes) == {1} + assert sizes[1].width == pytest.approx(PAGE_WIDTH) + assert sizes[1].height == pytest.approx(PAGE_HEIGHT) + + def test_page_item_size_lands_on_the_document(self, untagged): + assert untagged.pages[1].size.height == pytest.approx(PAGE_HEIGHT) + + +class TestCoordinateFlip: + def test_all_bboxes_are_top_left_origin(self, untagged): + for item, _ in untagged.iterate_items(with_groups=False): + for prov in item.prov: + assert prov.bbox.coord_origin == CoordOrigin.TOPLEFT + + def test_title_sits_near_the_top_of_the_page(self, untagged): + title = next(t for t in untagged.texts if "Study of Layout" in t.text) + + # Drawn at PDF y=720 on a 792pt page -> ~54..72pt from the top edge. + assert title.prov[0].bbox.t == pytest.approx(PAGE_HEIGHT - 738, abs=1.0) + assert title.prov[0].bbox.b == pytest.approx(PAGE_HEIGHT - 720, abs=1.0) + + def test_reading_order_runs_down_the_page(self, untagged): + tops = [t.prov[0].bbox.t for t in untagged.texts] + + assert tops == sorted(tops) + + def test_every_bbox_stays_inside_the_page(self, untagged): + for item, _ in untagged.iterate_items(with_groups=False): + for prov in item.prov: + assert 0 <= prov.bbox.l <= prov.bbox.r <= PAGE_WIDTH + assert 0 <= prov.bbox.t <= prov.bbox.b <= PAGE_HEIGHT + + +class TestTextExtraction: + def test_body_text_is_captured(self, untagged): + assert any("two parsers" in t.text for t in untagged.texts) + + def test_page_number_is_one_indexed(self, untagged): + assert {p.page_no for t in untagged.texts for p in t.prov} == {1} + + def test_images_become_pictures_not_text(self, untagged): + assert len(untagged.pictures) == 1 + assert not any("[Image:" in t.text for t in untagged.texts) + + def test_provenance_charspan_matches_the_item_text(self, untagged): + for text_item in untagged.texts: + assert text_item.prov[0].charspan == (0, len(text_item.text)) + + +class TestRoleMapping: + @pytest.mark.parametrize( + ("role", "label", "level"), + [ + ("H1", DocItemLabel.SECTION_HEADER, 1), + ("H3", DocItemLabel.SECTION_HEADER, 3), + ("H6", DocItemLabel.SECTION_HEADER, 6), + ("P", DocItemLabel.TEXT, None), + ("Table", DocItemLabel.TABLE, None), + ("Figure", DocItemLabel.PICTURE, None), + ("Caption", DocItemLabel.CAPTION, None), + ("LI", DocItemLabel.LIST_ITEM, None), + ("Title", DocItemLabel.TITLE, None), + ], + ) + def test_known_roles_map_to_labels(self, role, label, level): + assert role_to_label(role) == (label, level) + + def test_unknown_roles_fall_back_to_plain_text(self): + assert role_to_label("Sect") == (DocItemLabel.TEXT, None) + + +class TestTaggedPdf: + def test_structure_roles_produce_real_heading_levels(self, tagged): + headings = {t.text: t.level for t in tagged.texts if t.label == DocItemLabel.SECTION_HEADER} + + assert headings["A Study of Layout Extraction"] == 1 + assert headings["Methods"] == 2 + + def test_paragraph_role_stays_plain_text(self, tagged): + body = next(t for t in tagged.texts if "two parsers" in t.text) + + assert body.label == DocItemLabel.TEXT + + def test_table_role_produces_a_table_item(self, tagged): + assert len(tagged.tables) == 1 + + def test_table_cells_are_recovered_geometrically(self, tagged): + cells = tagged.tables[0].data + + assert cells.num_rows == 2 + assert cells.num_cols == 2 + assert {c.text for c in cells.table_cells} == {"Group", "N", "control", "42"} + + def test_first_table_row_is_marked_as_header(self, tagged): + header = [c for c in tagged.tables[0].data.table_cells if c.start_row_offset_idx == 0] + + assert all(c.column_header for c in header) + assert {c.text for c in header} == {"Group", "N"} + + def test_table_text_is_not_duplicated_as_body_text(self, tagged): + assert not any(t.text in {"Group", "N", "control", "42"} for t in tagged.texts) + + def test_caption_role_survives_containment_dedup(self, tagged): + assert any(t.label == DocItemLabel.CAPTION and "Figure 1" in t.text for t in tagged.texts) + + def test_figure_role_produces_a_picture(self, tagged): + assert len(tagged.pictures) == 1 + + +class TestUntaggedHeuristics: + def test_larger_font_sizes_are_promoted_to_headings(self, untagged): + title = next(t for t in untagged.texts if "Study of Layout" in t.text) + body = next(t for t in untagged.texts if "two parsers" in t.text) + + assert title.label == DocItemLabel.SECTION_HEADER + assert body.label == DocItemLabel.TEXT + + def test_heading_levels_rank_by_descending_font_size(self, untagged): + title = next(t for t in untagged.texts if "Study of Layout" in t.text) + methods = next(t for t in untagged.texts if t.text == "Methods") + + assert title.level < methods.level + + +class TestClassify: + def test_reports_page_count_and_normalized_ocr_pages(self, untagged_bytes): + result = classify(untagged_bytes) + + assert result["page_count"] == 1 + # classify_pdf reports 0-indexed pages; we normalize to docling's 1-indexed page_no. + assert result["pages_needing_ocr"] == [1] + + def test_reports_a_pdf_type(self, untagged_bytes): + assert isinstance(classify(untagged_bytes)["pdf_type"], str) + + +class TestPageSelection: + def test_pages_filter_restricts_output(self, untagged_bytes): + doc = parse(untagged_bytes, name="sample", pages=[2]) + + assert doc.texts == [] + assert 1 not in doc.pages diff --git a/extralit-server/tests/unit/contexts/ocr/test_pymupdf_parser.py b/extralit-server/tests/unit/contexts/ocr/test_pymupdf_parser.py new file mode 100644 index 000000000..60f71599d --- /dev/null +++ b/extralit-server/tests/unit/contexts/ocr/test_pymupdf_parser.py @@ -0,0 +1,157 @@ +"""Tests for the pymupdf layout parser. Opt-in — pymupdf4llm is an AGPL extra.""" + +from pathlib import Path + +import pytest +from docling_core.types.doc import CoordOrigin, DocItemLabel + +pytest.importorskip("pymupdf4llm") + +from extralit_server.contexts.ocr.parsers import get_parser, list_parsers +from extralit_server.contexts.ocr.parsers.pymupdf import parse + +FIXTURES = Path(__file__).parents[3] / "fixtures" / "pdf" +PAGE_WIDTH, PAGE_HEIGHT = 612.0, 792.0 + + +@pytest.fixture(scope="module") +def pdf_bytes() -> bytes: + return (FIXTURES / "sample.pdf").read_bytes() + + +@pytest.fixture(scope="module") +def doc(pdf_bytes): + return parse(pdf_bytes, name="sample") + + +class TestRegistry: + def test_pymupdf_is_registered_when_the_extra_is_installed(self): + assert "pymupdf" in list_parsers() + + def test_get_parser_resolves_it(self, pdf_bytes): + assert get_parser("pymupdf")(pdf_bytes, name="sample").name == "sample" + + +class TestGeometry: + def test_page_size_comes_from_the_page_rect(self, doc): + assert doc.pages[1].size.width == pytest.approx(PAGE_WIDTH) + assert doc.pages[1].size.height == pytest.approx(PAGE_HEIGHT) + + def test_coordinates_are_top_left_with_no_flip(self, doc): + title = next(t for t in doc.texts if "Study of Layout" in t.text) + + assert title.prov[0].bbox.coord_origin == CoordOrigin.TOPLEFT + # pymupdf is natively top-left, so the title keeps its ~52.7pt top directly. + assert title.prov[0].bbox.t == pytest.approx(52.7, abs=1.0) + + def test_page_numbers_are_one_indexed(self, doc): + assert {p.page_no for t in doc.texts for p in t.prov} == {1} + + def test_every_bbox_stays_inside_the_page(self, doc): + for item, _ in doc.iterate_items(with_groups=False): + for prov in item.prov: + assert 0 <= prov.bbox.l <= prov.bbox.r <= PAGE_WIDTH + assert 0 <= prov.bbox.t <= prov.bbox.b <= PAGE_HEIGHT + + def test_reading_order_runs_down_the_page(self, doc): + tops = [i.prov[0].bbox.t for i, _ in doc.iterate_items(with_groups=False)] + + assert tops == sorted(tops) + + +class TestHeadings: + def test_font_size_ranking_yields_heading_levels(self, doc): + title = next(t for t in doc.texts if "Study of Layout" in t.text) + methods = next(t for t in doc.texts if t.text == "Methods") + + assert title.label == DocItemLabel.SECTION_HEADER + assert (title.level, methods.level) == (1, 2) + + def test_body_text_is_not_promoted(self, doc): + body = next(t for t in doc.texts if "two parsers" in t.text) + + assert body.label == DocItemLabel.TEXT + # Plain text items carry no level at all — only headings do. + assert not hasattr(body, "level") + + +class TestTables: + def test_a_table_is_detected(self, doc): + assert len(doc.tables) == 1 + + def test_cell_offsets_are_exclusive_and_span_the_grid(self, doc): + data = doc.tables[0].data + + assert (data.num_rows, data.num_cols) == (2, 2) + for cell in data.table_cells: + assert cell.end_row_offset_idx == cell.start_row_offset_idx + cell.row_span + assert cell.end_col_offset_idx == cell.start_col_offset_idx + cell.col_span + + def test_cell_text_is_recovered(self, doc): + assert {c.text for c in doc.tables[0].data.table_cells} == {"Group", "N", "control", "42"} + + def test_header_row_is_flagged(self, doc): + header = [c for c in doc.tables[0].data.table_cells if c.start_row_offset_idx == 0] + + assert all(c.column_header for c in header) + + def test_every_cell_carries_its_own_bbox(self, doc): + # Per-cell geometry is the reason pymupdf is the higher-fidelity parser. + for cell in doc.tables[0].data.table_cells: + assert cell.bbox is not None + assert cell.bbox.coord_origin == CoordOrigin.TOPLEFT + assert cell.bbox.r > cell.bbox.l + + def test_cell_bboxes_sit_inside_the_table_bbox(self, doc): + table_bbox = doc.tables[0].prov[0].bbox + + for cell in doc.tables[0].data.table_cells: + assert cell.bbox.intersection_over_self(table_bbox) == pytest.approx(1.0, abs=0.01) + + def test_table_html_round_trips(self, doc): + html = doc.tables[0].export_to_html(doc=doc) + + assert "Group" in html + assert "42" in html + + def test_table_text_is_not_duplicated_as_body_text(self, doc): + assert not any(t.text in {"Group", "N", "control", "42", "Group N", "control 42"} for t in doc.texts) + + +class TestPictures: + def test_image_blocks_become_pictures(self, doc): + assert len(doc.pictures) == 1 + + def test_picture_bbox_matches_the_drawn_rect(self, doc): + bbox = doc.pictures[0].prov[0].bbox + + assert (bbox.l, bbox.t, bbox.r, bbox.b) == pytest.approx((72.0, 322.0, 192.0, 412.0), abs=1.0) + + +class TestPageSelection: + def test_pages_filter_restricts_output(self, pdf_bytes): + doc = parse(pdf_bytes, name="sample", pages=[2]) + + assert doc.texts == [] + assert 1 not in doc.pages + + +class TestParserAgreement: + def test_both_parsers_agree_on_page_geometry(self, pdf_bytes): + from extralit_server.contexts.ocr.parsers.pdf_inspector import parse as parse_pi + + other = parse_pi(pdf_bytes, name="sample") + mine = parse(pdf_bytes, name="sample") + + assert set(mine.pages) == set(other.pages) + for page_no, page in mine.pages.items(): + assert page.size.width == pytest.approx(other.pages[page_no].size.width) + assert page.size.height == pytest.approx(other.pages[page_no].size.height) + + def test_both_parsers_place_the_title_in_the_same_region(self, pdf_bytes): + from extralit_server.contexts.ocr.parsers.pdf_inspector import parse as parse_pi + + mine = next(t for t in parse(pdf_bytes, name="s").texts if "Study of Layout" in t.text) + other = next(t for t in parse_pi(pdf_bytes, name="s").texts if "Study of Layout" in t.text) + + assert mine.prov[0].bbox.intersection_over_self(other.prov[0].bbox) > 0.6 diff --git a/extralit-server/uv.lock b/extralit-server/uv.lock index 9db523964..0bf2e22bb 100644 --- a/extralit-server/uv.lock +++ b/extralit-server/uv.lock @@ -267,24 +267,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, ] -[[package]] -name = "anthropic" -version = "0.46.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "httpx" }, - { name = "jiter" }, - { name = "pydantic" }, - { name = "sniffio" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d4/68/3b4c045edf6dc6933895e8f279cc77c7684874c8aba46a4e6241c8b147cf/anthropic-0.46.0.tar.gz", hash = "sha256:eac3d43271d02321a57c3ca68aca84c3d58873e8e72d1433288adee2d46b745b", size = 202191, upload-time = "2025-02-18T20:35:33.314Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/50/6f/346beae0375df5f6907230bc63d557ef5d7659be49250ac5931a758322ae/anthropic-0.46.0-py3-none-any.whl", hash = "sha256:1445ec9be78d2de7ea51b4d5acd3574e414aea97ef903d0ecbb57bec806aaa49", size = 223228, upload-time = "2025-02-18T20:35:28.659Z" }, -] - [[package]] name = "anyio" version = "4.13.0" @@ -413,19 +395,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3b/05/2546085c6dc07a45627460a39e6291b82382b434fff2bd0167ff3bc31eb1/bcrypt-4.2.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e158009a54c4c8bc91d5e0da80920d048f918c61a581f0a63e4e93bb556d362f", size = 274652, upload-time = "2024-11-19T20:08:05.484Z" }, ] -[[package]] -name = "beautifulsoup4" -version = "4.14.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "soupsieve" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c3/b0/1c6a16426d389813b48d95e26898aff79abbde42ad353958ad95cc8c9b21/beautifulsoup4-4.14.3.tar.gz", hash = "sha256:6292b1c5186d356bba669ef9f7f051757099565ad9ada5dd630bd9de5fa7fb86", size = 627737, upload-time = "2025-11-30T15:08:26.084Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1a/39/47f9197bdd44df24d67ac8893641e16f386c984a0619ef2ee4c51fbbc019/beautifulsoup4-4.14.3-py3-none-any.whl", hash = "sha256:0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb", size = 107721, upload-time = "2025-11-30T15:08:24.087Z" }, -] - [[package]] name = "boto3" version = "1.40.61" @@ -616,15 +585,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195, upload-time = "2025-09-08T23:23:43.004Z" }, ] -[[package]] -name = "cfgv" -version = "3.5.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4e/b5/721b8799b04bf9afe054a3899c6cf4e880fcf8563cc71c15610242490a0c/cfgv-3.5.0.tar.gz", hash = "sha256:d5b1034354820651caa73ede66a6294d6e95c1b00acc5e9b098e917404669132", size = 7334, upload-time = "2025-11-19T20:55:51.612Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl", hash = "sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0", size = 7445, upload-time = "2025-11-19T20:55:50.744Z" }, -] - [[package]] name = "charset-normalizer" version = "3.4.6" @@ -929,79 +889,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/58/6b3d24e6b9bc474a2dcdee65dfd1f008867015408a271562e4b690561a4d/cryptography-46.0.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:8456928655f856c6e1533ff59d5be76578a7157224dbd9ce6872f25055ab9ab7", size = 3407605, upload-time = "2026-02-10T19:18:29.233Z" }, ] -[[package]] -name = "cuda-bindings" -version = "13.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-pathfinder", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/1a/fe/7351d7e586a8b4c9f89731bfe4cf0148223e8f9903ff09571f78b3fb0682/cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08b395f79cb89ce0cd8effff07c4a1e20101b873c256a1aeb286e8fd7bd0f556", size = 5744254, upload-time = "2026-03-11T00:12:29.798Z" }, - { url = "https://files.pythonhosted.org/packages/aa/ef/184aa775e970fc089942cd9ec6302e6e44679d4c14549c6a7ea45bf7f798/cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6f3682ec3c4769326aafc67c2ba669d97d688d0b7e63e659d36d2f8b72f32d6", size = 6329075, upload-time = "2026-03-11T00:12:32.319Z" }, - { url = "https://files.pythonhosted.org/packages/e0/a9/3a8241c6e19483ac1f1dcf5c10238205dcb8a6e9d0d4d4709240dff28ff4/cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:721104c603f059780d287969be3d194a18d0cc3b713ed9049065a1107706759d", size = 5730273, upload-time = "2026-03-11T00:12:37.18Z" }, - { url = "https://files.pythonhosted.org/packages/e9/94/2748597f47bb1600cd466b20cab4159f1530a3a33fe7f70fee199b3abb9e/cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1eba9504ac70667dd48313395fe05157518fd6371b532790e96fbb31bbb5a5e1", size = 6313924, upload-time = "2026-03-11T00:12:39.462Z" }, - { url = "https://files.pythonhosted.org/packages/52/c8/b2589d68acf7e3d63e2be330b84bc25712e97ed799affbca7edd7eae25d6/cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e865447abfb83d6a98ad5130ed3c70b1fc295ae3eeee39fd07b4ddb0671b6788", size = 5722404, upload-time = "2026-03-11T00:12:44.041Z" }, - { url = "https://files.pythonhosted.org/packages/1f/92/f899f7bbb5617bb65ec52a6eac1e9a1447a86b916c4194f8a5001b8cde0c/cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46d8776a55d6d5da9dd6e9858fba2efcda2abe6743871dee47dd06eb8cb6d955", size = 6320619, upload-time = "2026-03-11T00:12:45.939Z" }, - { url = "https://files.pythonhosted.org/packages/df/93/eef988860a3ca985f82c4f3174fc0cdd94e07331ba9a92e8e064c260337f/cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6629ca2df6f795b784752409bcaedbd22a7a651b74b56a165ebc0c9dcbd504d0", size = 5614610, upload-time = "2026-03-11T00:12:50.337Z" }, - { url = "https://files.pythonhosted.org/packages/18/23/6db3aba46864aee357ab2415135b3fe3da7e9f1fa0221fa2a86a5968099c/cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7dca0da053d3b4cc4869eff49c61c03f3c5dbaa0bcd712317a358d5b8f3f385d", size = 6149914, upload-time = "2026-03-11T00:12:52.374Z" }, - { url = "https://files.pythonhosted.org/packages/c0/87/87a014f045b77c6de5c8527b0757fe644417b184e5367db977236a141602/cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6464b30f46692d6c7f65d4a0e0450d81dd29de3afc1bb515653973d01c2cd6e", size = 5685673, upload-time = "2026-03-11T00:12:56.371Z" }, - { url = "https://files.pythonhosted.org/packages/ee/5e/c0fe77a73aaefd3fff25ffaccaac69c5a63eafdf8b9a4c476626ef0ac703/cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4af9f3e1be603fa12d5ad6cfca7844c9d230befa9792b5abdf7dd79979c3626", size = 6191386, upload-time = "2026-03-11T00:12:58.965Z" }, - { url = "https://files.pythonhosted.org/packages/5f/58/ed2c3b39c8dd5f96aa7a4abef0d47a73932c7a988e30f5fa428f00ed0da1/cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df850a1ff8ce1b3385257b08e47b70e959932f5f432d0a4e46a355962b4e4771", size = 5507469, upload-time = "2026-03-11T00:13:04.063Z" }, - { url = "https://files.pythonhosted.org/packages/1f/01/0c941b112ceeb21439b05895eace78ca1aa2eaaf695c8521a068fd9b4c00/cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8a16384c6494e5485f39314b0b4afb04bee48d49edb16d5d8593fd35bbd231b", size = 6059693, upload-time = "2026-03-11T00:13:06.003Z" }, -] - -[[package]] -name = "cuda-pathfinder" -version = "1.5.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/93/66/0c02bd330e7d976f83fa68583d6198d76f23581bcbb5c0e98a6148f326e5/cuda_pathfinder-1.5.0-py3-none-any.whl", hash = "sha256:498f90a9e9de36044a7924742aecce11c50c49f735f1bc53e05aa46de9ea4110", size = 49739, upload-time = "2026-03-24T21:14:30.869Z" }, -] - -[[package]] -name = "cuda-toolkit" -version = "13.0.2" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/57/b2/453099f5f3b698d7d0eab38916aac44c7f76229f451709e2eb9db6615dcd/cuda_toolkit-13.0.2-py2.py3-none-any.whl", hash = "sha256:b198824cf2f54003f50d64ada3a0f184b42ca0846c1c94192fa269ecd97a66eb", size = 2364, upload-time = "2025-12-19T23:24:07.328Z" }, -] - -[package.optional-dependencies] -cublas = [ - { name = "nvidia-cublas", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, -] -cudart = [ - { name = "nvidia-cuda-runtime", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, -] -cufft = [ - { name = "nvidia-cufft", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, -] -cufile = [ - { name = "nvidia-cufile", marker = "sys_platform == 'linux'" }, -] -cupti = [ - { name = "nvidia-cuda-cupti", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, -] -curand = [ - { name = "nvidia-curand", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, -] -cusolver = [ - { name = "nvidia-cusolver", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, -] -cusparse = [ - { name = "nvidia-cusparse", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, -] -nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, -] -nvrtc = [ - { name = "nvidia-cuda-nvrtc", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, -] -nvtx = [ - { name = "nvidia-nvtx", marker = "(python_full_version < '3.11' and sys_platform == 'win32') or sys_platform == 'linux'" }, -] - [[package]] name = "datasets" version = "3.6.0" @@ -1071,21 +958,50 @@ wheels = [ ] [[package]] -name = "distlib" -version = "0.4.0" +name = "distro" +version = "1.9.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/96/8e/709914eb2b5749865801041647dc7f4e6d00b549cfe88b65ca192995f07c/distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d", size = 614605, upload-time = "2025-07-17T16:52:00.465Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" }, + { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, ] [[package]] -name = "distro" -version = "1.9.0" +name = "doclang" +version = "0.7.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } +dependencies = [ + { name = "lxml" }, + { name = "typer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f5/3a/005e4856ad8e9b9879414a4df4dbc56dc3663b96f9d8c920ef210e8931cf/doclang-0.7.3.tar.gz", hash = "sha256:ca50615357e46ebf9597bb9065b9112367103ec24bd539f8ae12649224cf50b0", size = 31569, upload-time = "2026-07-15T08:11:02.917Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, + { url = "https://files.pythonhosted.org/packages/a5/81/334ccc0f0cd7c3d75996b6b596e7f4c62c4c46a0ca042003315c28170159/doclang-0.7.3-py3-none-any.whl", hash = "sha256:9440c4ca9f7e061a7b8d33bdf15b1029be69a4c13cd8952dd6ce541884e4c685", size = 32267, upload-time = "2026-07-15T08:11:01.977Z" }, +] + +[[package]] +name = "docling-core" +version = "2.91.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "defusedxml" }, + { name = "doclang" }, + { name = "jsonref" }, + { name = "jsonschema" }, + { name = "latex2mathml" }, + { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "pandas", version = "3.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "pillow" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "pyyaml" }, + { name = "tabulate" }, + { name = "typer" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e5/f5/5aaf6f1221242a4dd598b515786ffca796352a8d960a9c6a0deb614301f4/docling_core-2.91.0.tar.gz", hash = "sha256:dc40fe76524a2700f869265015a9ef86027888e73b5652f324b3b5c52a2df240", size = 344852, upload-time = "2026-08-06T14:23:07.919Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/68/de7c0ba404d035afe78ae4e602290bb6c330b34cc53fd5f92c9950be4ac0/docling_core-2.91.0-py3-none-any.whl", hash = "sha256:4949a5dd77ae1daf4153c095897d3bdde1c870f2bbe401bf94d8834bef867998", size = 286832, upload-time = "2026-08-06T14:23:05.908Z" }, ] [[package]] @@ -1142,15 +1058,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/a3/460c57f094a4a165c84a1341c373b0a4f5ec6ac244b998d5021aade89b77/ecdsa-0.19.1-py2.py3-none-any.whl", hash = "sha256:30638e27cf77b7e15c4c4cc1973720149e1033827cfd00661ca5c8cc0cdb24c3", size = 150607, upload-time = "2025-03-13T11:52:41.757Z" }, ] -[[package]] -name = "einops" -version = "0.8.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2c/77/850bef8d72ffb9219f0b1aac23fbc1bf7d038ee6ea666f331fa273031aa2/einops-0.8.2.tar.gz", hash = "sha256:609da665570e5e265e27283aab09e7f279ade90c4f01bcfca111f3d3e13f2827", size = 56261, upload-time = "2026-01-26T04:13:17.638Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl", hash = "sha256:54058201ac7087911181bfec4af6091bb59380360f069276601256a76af08193", size = 65638, upload-time = "2026-01-26T04:13:18.546Z" }, -] - [[package]] name = "elastic-transport" version = "8.17.1" @@ -1186,7 +1093,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -1206,6 +1113,7 @@ dependencies = [ { name = "brotli-asgi" }, { name = "click" }, { name = "datasets" }, + { name = "docling-core" }, { name = "duckdb" }, { name = "elasticsearch8", extra = ["async"] }, { name = "fastapi" }, @@ -1225,9 +1133,11 @@ dependencies = [ { name = "opensearch-py" }, { name = "packaging" }, { name = "pandera", extra = ["io"] }, + { name = "pdf-inspector" }, { name = "pdf2image" }, { name = "pillow" }, { name = "psutil" }, + { name = "pyarrow" }, { name = "pydantic" }, { name = "pydantic-settings" }, { name = "python-jose", extra = ["cryptography"] }, @@ -1245,16 +1155,13 @@ dependencies = [ ] [package.optional-dependencies] -marker = [ - { name = "marker-pdf" }, - { name = "torch" }, - { name = "torchvision" }, - { name = "transformers" }, -] postgresql = [ { name = "asyncpg" }, { name = "psycopg2" }, ] +pymupdf = [ + { name = "pymupdf4llm" }, +] [package.dev-dependencies] dev = [ @@ -1274,64 +1181,64 @@ dev = [ [package.metadata] requires-dist = [ { name = "aioboto3", specifier = ">=13.1.1" }, - { name = "aiofiles", specifier = "~=24.1.0" }, + { name = "aiofiles", specifier = ">=24.1.0" }, { name = "aiosqlite", specifier = "==0.20.0" }, - { name = "alembic", specifier = "~=1.13.0" }, - { name = "asyncpg", marker = "extra == 'postgresql'", specifier = "~=0.30.0" }, + { name = "alembic", specifier = ">=1.13.0" }, + { name = "asyncpg", marker = "extra == 'postgresql'", specifier = ">=0.30.0" }, { name = "authlib", specifier = ">=1.6.9" }, - { name = "bcrypt", specifier = "~=4.2.0" }, - { name = "brotli-asgi", specifier = "~=1.4.0" }, + { name = "bcrypt", specifier = ">=4.2.0" }, + { name = "brotli-asgi", specifier = ">=1.4.0" }, { name = "click", specifier = ">=8.2.0" }, { name = "datasets", specifier = ">=3.6.0" }, + { name = "docling-core", specifier = ">=2.91.0,<3.0.0" }, { name = "duckdb", specifier = ">=1.5.4" }, - { name = "elasticsearch8", extras = ["async"], specifier = "~=8.7.0" }, - { name = "fastapi", specifier = "~=0.115.0" }, + { name = "elasticsearch8", extras = ["async"], specifier = ">=8.7.0" }, + { name = "fastapi", specifier = ">=0.115.0" }, { name = "filelock", specifier = ">=3.25.2" }, - { name = "greenlet", specifier = "~=3.1.0" }, - { name = "httpx", specifier = "~=0.27.0" }, - { name = "huggingface-hub", specifier = "~=0.34.0" }, + { name = "greenlet", specifier = ">=3.1.0" }, + { name = "httpx", specifier = ">=0.27.0" }, + { name = "huggingface-hub", specifier = ">=0.34.0" }, { name = "jinja2", specifier = ">=3.1.4" }, { name = "lancedb", specifier = ">=0.34.0" }, { name = "lazy-loader", specifier = ">=0.4" }, { name = "litellm", specifier = ">=1.80.0,<=1.82.6" }, - { name = "marker-pdf", marker = "extra == 'marker'", specifier = ">=1.9.3" }, { name = "numpy", specifier = ">=2.0.0,<3.0.0" }, - { name = "oauthlib", specifier = "~=3.2.0" }, + { name = "oauthlib", specifier = ">=3.2.0" }, { name = "ocrmypdf", specifier = ">=16.11.0" }, { name = "opencv-python-headless", specifier = ">=4.11.0.86" }, - { name = "opensearch-py", specifier = "~=2.0.0" }, + { name = "opensearch-py", specifier = ">=2.0.0" }, { name = "packaging", specifier = ">=23.2" }, { name = "pandera", extras = ["io"], specifier = ">=0.20" }, + { name = "pdf-inspector", specifier = ">=1.14.2" }, { name = "pdf2image", specifier = ">=1.17.0" }, { name = "pillow", specifier = ">=10.1.0" }, - { name = "psutil", specifier = "~=5.8,<5.10" }, - { name = "psycopg2", marker = "extra == 'postgresql'", specifier = "~=2.9.0" }, - { name = "pydantic", specifier = "~=2.9.0" }, - { name = "pydantic-settings", specifier = "~=2.6.0" }, - { name = "python-jose", extras = ["cryptography"], specifier = "~=3.3.0" }, - { name = "python-multipart", specifier = "~=0.0.16" }, + { name = "psutil", specifier = ">=5.8,<5.10" }, + { name = "psycopg2", marker = "extra == 'postgresql'", specifier = ">=2.9.0" }, + { name = "pyarrow", specifier = ">=23.0.1" }, + { name = "pydantic", specifier = ">=2.9.0" }, + { name = "pydantic-settings", specifier = ">=2.6.0" }, + { name = "pymupdf4llm", marker = "extra == 'pymupdf'", specifier = "~=0.0.27" }, + { name = "python-jose", extras = ["cryptography"], specifier = ">=3.3.0" }, + { name = "python-multipart", specifier = ">=0.0.16" }, { name = "pyyaml", specifier = ">=5.4.1,<6.1.0" }, { name = "rich", specifier = "!=13.1.0" }, - { name = "rq", specifier = "~=2.4.1" }, - { name = "social-auth-core", specifier = "~=4.5.0" }, - { name = "sqlalchemy", specifier = "~=2.0.0" }, + { name = "rq", specifier = ">=2.4.1" }, + { name = "social-auth-core", specifier = ">=4.5.0" }, + { name = "sqlalchemy", specifier = ">=2.0.0" }, { name = "standardwebhooks", specifier = ">=1.0.0" }, { name = "tenacity", specifier = ">=9.1.2" }, - { name = "torch", marker = "extra == 'marker'", specifier = ">=2.5.0" }, - { name = "torchvision", marker = "extra == 'marker'", specifier = ">=0.20.0" }, - { name = "transformers", marker = "extra == 'marker'", specifier = ">=4.51.0" }, { name = "typer", specifier = ">=0.19.1" }, { name = "types-aiobotocore-s3", specifier = "==2.24.2" }, - { name = "uvicorn", extras = ["standard"], specifier = "~=0.32.0" }, + { name = "uvicorn", extras = ["standard"], specifier = ">=0.32.0" }, ] -provides-extras = ["marker", "postgresql"] +provides-extras = ["postgresql", "pymupdf"] [package.metadata.requires-dev] dev = [ - { name = "factory-boy", specifier = "~=3.2.1" }, + { name = "factory-boy", specifier = ">=3.2.1" }, { name = "httpx", specifier = ">=0.26.0" }, { name = "pytest", specifier = ">=7.4.4" }, - { name = "pytest-asyncio", specifier = "~=1.1.0" }, + { name = "pytest-asyncio", specifier = ">=1.1.0" }, { name = "pytest-cov", specifier = ">=4.1.0" }, { name = "pytest-env", specifier = ">=1.1.3" }, { name = "pytest-mock", specifier = ">=3.12.0" }, @@ -1451,15 +1358,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a4/a5/842ae8f0c08b61d6484b52f99a03510a3a72d23141942d216ebe81fefbce/filelock-3.25.2-py3-none-any.whl", hash = "sha256:ca8afb0da15f229774c9ad1b455ed96e85a81373065fb10446672f64444ddf70", size = 26759, upload-time = "2026-03-11T20:45:37.437Z" }, ] -[[package]] -name = "filetype" -version = "1.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bb/29/745f7d30d47fe0f251d3ad3dc2978a23141917661998763bebb6da007eb1/filetype-1.2.0.tar.gz", hash = "sha256:66b56cd6474bf41d8c54660347d37afcc3f7d1970648de365c102ef77548aadb", size = 998020, upload-time = "2022-11-02T17:34:04.141Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/18/79/1b8fa1bb3568781e84c9200f951c735f3f157429f44be0495da55894d620/filetype-1.2.0-py2.py3-none-any.whl", hash = "sha256:7ce71b6880181241cf7ac8697a2f1eb6a8bd9b429f7ad6d27b8db9ba5f1c2d25", size = 19970, upload-time = "2022-11-02T17:34:01.425Z" }, -] - [[package]] name = "frozenlist" version = "1.8.0" @@ -1595,46 +1493,6 @@ http = [ { name = "aiohttp" }, ] -[[package]] -name = "ftfy" -version = "6.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wcwidth" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a5/d3/8650919bc3c7c6e90ee3fa7fd618bf373cbbe55dff043bd67353dbb20cd8/ftfy-6.3.1.tar.gz", hash = "sha256:9b3c3d90f84fb267fe64d375a07b7f8912d817cf86009ae134aa03e1819506ec", size = 308927, upload-time = "2024-10-26T00:50:35.149Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ab/6e/81d47999aebc1b155f81eca4477a616a70f238a2549848c38983f3c22a82/ftfy-6.3.1-py3-none-any.whl", hash = "sha256:7c70eb532015cd2f9adb53f101fb6c7945988d023a085d127d1573dc49dd0083", size = 44821, upload-time = "2024-10-26T00:50:33.425Z" }, -] - -[[package]] -name = "google-auth" -version = "2.49.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cryptography" }, - { name = "pyasn1-modules" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c6/fc/e925290a1ad95c975c459e2df070fac2b90954e13a0370ac505dff78cb99/google_auth-2.49.2.tar.gz", hash = "sha256:c1ae38500e73065dcae57355adb6278cf8b5c8e391994ae9cbadbcb9631ab409", size = 333958, upload-time = "2026-04-10T00:41:21.888Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/73/76/d241a5c927433420507215df6cac1b1fa4ac0ba7a794df42a84326c68da8/google_auth-2.49.2-py3-none-any.whl", hash = "sha256:c2720924dfc82dedb962c9f52cabb2ab16714fd0a6a707e40561d217574ed6d5", size = 240638, upload-time = "2026-04-10T00:41:14.501Z" }, -] - -[[package]] -name = "google-genai" -version = "1.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-auth" }, - { name = "pydantic" }, - { name = "requests" }, - { name = "typing-extensions" }, - { name = "websockets" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/0d/ed/985f2d2e2b5fbd912ab0fdb11d6dc48c22553a6c4edffabb8146d53b974a/google_genai-1.2.0-py3-none-any.whl", hash = "sha256:609d61bee73f1a6ae5b47e9c7dd4b469d50318f050c5ceacf835b0f80f79d2d9", size = 130744, upload-time = "2025-02-12T16:40:03.601Z" }, -] - [[package]] name = "greenlet" version = "3.1.1" @@ -1818,15 +1676,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/92/1e/4157be4835fd0c064ca4c1a2cea577b3b33defa4b677ed7119372244357a/huggingface_hub-0.34.6-py3-none-any.whl", hash = "sha256:3387ec9045f9dc5b5715e4e7392c25b0d23fd539eb925111a1b301e60f2b4883", size = 562617, upload-time = "2025-09-16T08:10:49.372Z" }, ] -[[package]] -name = "identify" -version = "2.6.18" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/46/c4/7fb4db12296cdb11893d61c92048fe617ee853f8523b9b296ac03b43757e/identify-2.6.18.tar.gz", hash = "sha256:873ac56a5e3fd63e7438a7ecbc4d91aca692eb3fefa4534db2b7913f3fc352fd", size = 99580, upload-time = "2026-03-15T18:39:50.319Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/46/33/92ef41c6fad0233e41d3d84ba8e8ad18d1780f1e5d99b3c683e6d7f98b63/identify-2.6.18-py2.py3-none-any.whl", hash = "sha256:8db9d3c8ea9079db92cafb0ebf97abdc09d52e97f4dcf773a2e694048b7cd737", size = 99394, upload-time = "2026-03-15T18:39:48.915Z" }, -] - [[package]] name = "idna" version = "3.11" @@ -1995,12 +1844,12 @@ wheels = [ ] [[package]] -name = "joblib" -version = "1.5.3" +name = "jsonref" +version = "1.1.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/41/f2/d34e8b3a08a9cc79a50b2208a93dce981fe615b64d5a4d4abee421d898df/joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3", size = 331603, upload-time = "2025-12-15T08:41:46.427Z" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/0d/c1f3277e90ccdb50d33ed5ba1ec5b3f0a242ed8c1b1a85d3afeb68464dca/jsonref-1.1.0.tar.gz", hash = "sha256:32fe8e1d85af0fdefbebce950af85590b22b60f9e95443176adbde4e1ecea552", size = 8814, upload-time = "2023-01-16T16:10:04.455Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071, upload-time = "2025-12-15T08:41:44.973Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ec/e1db9922bceb168197a558a2b8c03a7963f1afe93517ddd3cf99f202f996/jsonref-1.1.0-py3-none-any.whl", hash = "sha256:590dc7773df6c21cbf948b5dac07a72a251db28b0238ceecce0a2abfa8ec30a9", size = 9425, upload-time = "2023-01-16T16:10:02.255Z" }, ] [[package]] @@ -2079,6 +1928,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d9/5d/8ca165f1386caf6c4d1c515afd52f345b66432264eecfdfb7fd33eefd9af/lancedb-0.34.0-cp39-abi3-win_amd64.whl", hash = "sha256:51cbc11808f9e3332819b9367c975b3a888541447a8e7bea09c57c852a279153", size = 63530726, upload-time = "2026-07-02T17:13:41.612Z" }, ] +[[package]] +name = "latex2mathml" +version = "3.81.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3b/62/35bb816c5c19d4d0cde5bdfb82ebb996306243d5f94e03f201658c629960/latex2mathml-3.81.0.tar.gz", hash = "sha256:4b959cdc3cac8686bc0e3e5aece8127dfb1b81ca1241bed8e00ef31b82bb4022", size = 77584, upload-time = "2026-04-15T00:55:27.977Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/b1/c488b530994c4f68e46efa99a4d6ca6741aaf158e35779fe6c4d8a9a427d/latex2mathml-3.81.0-py3-none-any.whl", hash = "sha256:d317710393fe20579aea39cfe8928fa2ad9b8780896e585326c75e89c1d1d1a4", size = 79185, upload-time = "2026-04-15T00:55:29.301Z" }, +] + [[package]] name = "lazy-loader" version = "0.5" @@ -2262,61 +2120,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, ] -[[package]] -name = "markdown2" -version = "2.5.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e4/ae/07d4a5fcaa5509221287d289323d75ac8eda5a5a4ac9de2accf7bbcc2b88/markdown2-2.5.5.tar.gz", hash = "sha256:001547e68f6e7fcf0f1cb83f7e82f48aa7d48b2c6a321f0cd20a853a8a2d1664", size = 157249, upload-time = "2026-03-02T20:46:53.411Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/43/af/4b3891eb0a49d6cfd5cbf3e9bf514c943afc2b0f13e2c57cc57cd88ecc21/markdown2-2.5.5-py3-none-any.whl", hash = "sha256:be798587e09d1f52d2e4d96a649c4b82a778c75f9929aad52a2c95747fa26941", size = 56250, upload-time = "2026-03-02T20:46:52.032Z" }, -] - -[[package]] -name = "markdownify" -version = "1.2.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "beautifulsoup4" }, - { name = "six" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3f/bc/c8c8eea5335341306b0fa7e1cb33c5e1c8d24ef70ddd684da65f41c49c92/markdownify-1.2.2.tar.gz", hash = "sha256:b274f1b5943180b031b699b199cbaeb1e2ac938b75851849a31fd0c3d6603d09", size = 18816, upload-time = "2025-11-16T19:21:18.565Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/43/ce/f1e3e9d959db134cedf06825fae8d5b294bd368aacdd0831a3975b7c4d55/markdownify-1.2.2-py3-none-any.whl", hash = "sha256:3f02d3cc52714084d6e589f70397b6fc9f2f3a8531481bf35e8cc39f975e186a", size = 15724, upload-time = "2025-11-16T19:21:17.622Z" }, -] - -[[package]] -name = "marker-pdf" -version = "1.10.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anthropic" }, - { name = "click" }, - { name = "filetype" }, - { name = "ftfy" }, - { name = "google-genai" }, - { name = "markdown2" }, - { name = "markdownify" }, - { name = "openai" }, - { name = "pdftext" }, - { name = "pillow" }, - { name = "pre-commit" }, - { name = "pydantic" }, - { name = "pydantic-settings" }, - { name = "python-dotenv" }, - { name = "rapidfuzz" }, - { name = "regex" }, - { name = "scikit-learn", version = "1.7.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scikit-learn", version = "1.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "surya-ocr" }, - { name = "torch" }, - { name = "tqdm" }, - { name = "transformers" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/04/bc/a9e72fd015f562b1f86c9dbc201c80365e7baf43d5aa616ba1c89453537f/marker_pdf-1.10.2.tar.gz", hash = "sha256:ce0fc839e11ad7519a576d254ca9d51a0f9454b9d7da02211f722b141317f9f1", size = 140056, upload-time = "2026-01-31T00:04:55.179Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d3/44/51f6a5673d4815fa6d8d355f4a7119cbd6d8ae2e3cb697c066a8012beb18/marker_pdf-1.10.2-py3-none-any.whl", hash = "sha256:f631737dd46d3927142b4b14c7b488962c5af278dc2c3ae7dc5b03a47f5909fb", size = 195706, upload-time = "2026-01-31T00:04:56.384Z" }, -] - [[package]] name = "markupsafe" version = "3.0.3" @@ -2411,15 +2214,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] -[[package]] -name = "mpmath" -version = "1.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, -] - [[package]] name = "multidict" version = "6.7.1" @@ -2585,60 +2379,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] -[[package]] -name = "networkx" -version = "3.4.2" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11' and sys_platform == 'darwin'", - "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux')", -] -sdist = { url = "https://files.pythonhosted.org/packages/fd/1d/06475e1cd5264c0b870ea2cc6fdb3e37177c1e565c43f56ff17a10e3937f/networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1", size = 2151368, upload-time = "2024-10-21T12:39:38.695Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b9/54/dd730b32ea14ea797530a4479b2ed46a6fb250f682a9cfb997e968bf0261/networkx-3.4.2-py3-none-any.whl", hash = "sha256:df5d4365b724cf81b8c6a7312509d0c22386097011ad1abe274afd5e9d3bbc5f", size = 1723263, upload-time = "2024-10-21T12:39:36.247Z" }, -] - -[[package]] -name = "networkx" -version = "3.6.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.14' and sys_platform == 'win32'", - "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version >= '3.14' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", - "python_full_version == '3.13.*' and sys_platform == 'win32'", - "python_full_version == '3.13.*' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version == '3.13.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", - "python_full_version == '3.12.*' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "python_full_version == '3.12.*' and sys_platform == 'win32'", - "python_full_version == '3.12.*' and sys_platform == 'emscripten'", - "(python_full_version == '3.12.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", - "python_full_version == '3.11.*' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "python_full_version == '3.11.*' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and sys_platform == 'emscripten'", - "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", -] -sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, -] - -[[package]] -name = "nodeenv" -version = "1.10.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/24/bf/d1bda4f6168e0b2e9e5958945e01910052158313224ada5ce1fb2e1113b8/nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb", size = 55611, upload-time = "2025-12-20T14:08:54.006Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, -] - [[package]] name = "numpy" version = "2.2.6" @@ -2808,209 +2548,60 @@ wheels = [ ] [[package]] -name = "nvidia-cublas" -version = "13.1.0.3" +name = "oauthlib" +version = "3.2.2" source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/fa/fbf4001037904031639e6bfbfc02badfc7e12f137a8afa254df6c4c8a670/oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918", size = 177352, upload-time = "2022-10-17T20:04:27.471Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/a5/fce49e2ae977e0ccc084e5adafceb4f0ac0c8333cb6863501618a7277f67/nvidia_cublas-13.1.0.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:c86fc7f7ae36d7528288c5d88098edcb7b02c633d262e7ddbb86b0ad91be5df2", size = 542851226, upload-time = "2025-10-09T08:59:04.818Z" }, - { url = "https://files.pythonhosted.org/packages/e7/44/423ac00af4dd95a5aeb27207e2c0d9b7118702149bf4704c3ddb55bb7429/nvidia_cublas-13.1.0.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:ee8722c1f0145ab246bccb9e452153b5e0515fd094c3678df50b2a0888b8b171", size = 423133236, upload-time = "2025-10-09T08:59:32.536Z" }, + { url = "https://files.pythonhosted.org/packages/7e/80/cab10959dc1faead58dc8384a781dfbf93cb4d33d50988f7a69f1b7c9bbe/oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca", size = 151688, upload-time = "2022-10-17T20:04:24.037Z" }, ] [[package]] -name = "nvidia-cuda-cupti" -version = "13.0.85" +name = "ocrmypdf" +version = "16.13.0" source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/2a/80353b103fc20ce05ef51e928daed4b6015db4aaa9162ed0997090fe2250/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_aarch64.whl", hash = "sha256:796bd679890ee55fb14a94629b698b6db54bcfd833d391d5e94017dd9d7d3151", size = 10310827, upload-time = "2025-09-04T08:26:42.012Z" }, - { url = "https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8", size = 10715597, upload-time = "2025-09-04T08:26:51.312Z" }, +dependencies = [ + { name = "deprecation" }, + { name = "img2pdf" }, + { name = "packaging" }, + { name = "pdfminer-six" }, + { name = "pi-heif" }, + { name = "pikepdf" }, + { name = "pillow" }, + { name = "pluggy" }, + { name = "rich" }, ] - -[[package]] -name = "nvidia-cuda-nvrtc" -version = "13.0.88" -source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/52/be1aaece0703a736757d8957c0d4f19c37561054169b501eb0e7132f15e5/ocrmypdf-16.13.0.tar.gz", hash = "sha256:29d37e915234ce717374863a9cc5dd32d29e063dfe60c51380dda71254c88248", size = 7042247, upload-time = "2025-12-24T07:58:35.86Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575", size = 90215200, upload-time = "2025-09-04T08:28:44.204Z" }, - { url = "https://files.pythonhosted.org/packages/b7/dc/6bb80850e0b7edd6588d560758f17e0550893a1feaf436807d64d2da040f/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b", size = 43015449, upload-time = "2025-09-04T08:28:20.239Z" }, + { url = "https://files.pythonhosted.org/packages/41/b1/e2e7ad98de0d3ee05b44dbc3f78ccb158a620f3add82d00c85490120e7f2/ocrmypdf-16.13.0-py3-none-any.whl", hash = "sha256:fad8a6f7cc52cdc6225095c401a1766c778c47efe9f1e854ae4dc64a550a3d37", size = 165377, upload-time = "2025-12-24T07:58:33.925Z" }, ] [[package]] -name = "nvidia-cuda-runtime" -version = "13.0.96" +name = "openai" +version = "1.109.1" source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c6/a1/a303104dc55fc546a3f6914c842d3da471c64eec92043aef8f652eb6c524/openai-1.109.1.tar.gz", hash = "sha256:d173ed8dbca665892a6db099b4a2dfac624f94d20a93f46eb0b56aae940ed869", size = 564133, upload-time = "2025-09-24T13:00:53.075Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/87/4f/17d7b9b8e285199c58ce28e31b5c5bbaa4d8271af06a89b6405258245de2/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55", size = 2261060, upload-time = "2025-10-09T08:55:15.78Z" }, - { url = "https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548", size = 2243632, upload-time = "2025-10-09T08:55:36.117Z" }, + { url = "https://files.pythonhosted.org/packages/1d/2a/7dd3d207ec669cacc1f186fd856a0f61dbc255d24f6fdc1a6715d6051b0f/openai-1.109.1-py3-none-any.whl", hash = "sha256:6bcaf57086cf59159b8e27447e4e7dd019db5d29a438072fbd49c290c7e65315", size = 948627, upload-time = "2025-09-24T13:00:50.754Z" }, ] [[package]] -name = "nvidia-cudnn-cu13" -version = "9.19.0.56" +name = "opencv-python-headless" +version = "4.11.0.86" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/f1/84/26025437c1e6b61a707442184fa0c03d083b661adf3a3eecfd6d21677740/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:6ed29ffaee1176c612daf442e4dd6cfeb6a0caa43ddcbeb59da94953030b1be4", size = 433781201, upload-time = "2026-02-03T20:40:53.805Z" }, - { url = "https://files.pythonhosted.org/packages/a3/22/0b4b932655d17a6da1b92fa92ab12844b053bb2ac2475e179ba6f043da1e/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:d20e1734305e9d68889a96e3f35094d733ff1f83932ebe462753973e53a572bf", size = 366066321, upload-time = "2026-02-03T20:44:52.837Z" }, -] - -[[package]] -name = "nvidia-cufft" -version = "12.0.0.61" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, - { url = "https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3", size = 214085489, upload-time = "2025-09-04T08:31:56.044Z" }, -] - -[[package]] -name = "nvidia-cufile" -version = "1.15.1.6" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/70/4f193de89a48b71714e74602ee14d04e4019ad36a5a9f20c425776e72cd6/nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44", size = 1223672, upload-time = "2025-09-04T08:32:22.779Z" }, - { url = "https://files.pythonhosted.org/packages/ab/73/cc4a14c9813a8a0d509417cf5f4bdaba76e924d58beb9864f5a7baceefbf/nvidia_cufile-1.15.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:bdc0deedc61f548bddf7733bdc216456c2fdb101d020e1ab4b88d232d5e2f6d1", size = 1136992, upload-time = "2025-09-04T08:32:14.119Z" }, -] - -[[package]] -name = "nvidia-curand" -version = "10.4.0.35" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/72/7c2ae24fb6b63a32e6ae5d241cc65263ea18d08802aaae087d9f013335a2/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:133df5a7509c3e292aaa2b477afd0194f06ce4ea24d714d616ff36439cee349a", size = 61962106, upload-time = "2025-08-04T10:21:41.128Z" }, - { url = "https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc", size = 59544258, upload-time = "2025-08-04T10:22:03.992Z" }, -] - -[[package]] -name = "nvidia-cusolver" -version = "12.0.4.66" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cublas", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, - { name = "nvidia-cusparse", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, - { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, - { url = "https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112", size = 200941980, upload-time = "2025-09-04T08:33:22.767Z" }, -] - -[[package]] -name = "nvidia-cusparse" -version = "12.6.3.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, - { url = "https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b", size = 145942937, upload-time = "2025-09-04T08:33:58.029Z" }, -] - -[[package]] -name = "nvidia-cusparselt-cu13" -version = "0.8.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/46/10/8dcd1175260706a2fc92a16a52e306b71d4c1ea0b0cc4a9484183399818a/nvidia_cusparselt_cu13-0.8.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:400c6ed1cf6780fc6efedd64ec9f1345871767e6a1a0a552a1ea0578117ea77c", size = 220791277, upload-time = "2025-08-13T19:22:40.982Z" }, - { url = "https://files.pythonhosted.org/packages/fd/53/43b0d71f4e702fa9733f8b4571fdca50a8813f1e450b656c239beff12315/nvidia_cusparselt_cu13-0.8.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:25e30a8a7323935d4ad0340b95a0b69926eee755767e8e0b1cf8dd85b197d3fd", size = 169884119, upload-time = "2025-08-13T19:23:41.967Z" }, -] - -[[package]] -name = "nvidia-nccl-cu13" -version = "2.28.9" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/39/55/1920646a2e43ffd4fc958536b276197ed740e9e0c54105b4bb3521591fc7/nvidia_nccl_cu13-2.28.9-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:01c873ba1626b54caa12272ed228dc5b2781545e0ae8ba3f432a8ef1c6d78643", size = 196561677, upload-time = "2025-11-18T05:49:03.45Z" }, - { url = "https://files.pythonhosted.org/packages/b0/b4/878fefaad5b2bcc6fcf8d474a25e3e3774bc5133e4b58adff4d0bca238bc/nvidia_nccl_cu13-2.28.9-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:e4553a30f34195f3fa1da02a6da3d6337d28f2003943aa0a3d247bbc25fefc42", size = 196493177, upload-time = "2025-11-18T05:49:17.677Z" }, -] - -[[package]] -name = "nvidia-nvjitlink" -version = "13.0.88" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/56/7a/123e033aaff487c77107195fa5a2b8686795ca537935a24efae476c41f05/nvidia_nvjitlink-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:13a74f429e23b921c1109976abefacc69835f2f433ebd323d3946e11d804e47b", size = 40713933, upload-time = "2025-09-04T08:35:43.553Z" }, - { url = "https://files.pythonhosted.org/packages/ab/2c/93c5250e64df4f894f1cbb397c6fd71f79813f9fd79d7cd61de3f97b3c2d/nvidia_nvjitlink-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e931536ccc7d467a98ba1d8b89ff7fa7f1fa3b13f2b0069118cd7f47bff07d0c", size = 38768748, upload-time = "2025-09-04T08:35:20.008Z" }, -] - -[[package]] -name = "nvidia-nvshmem-cu13" -version = "3.4.5" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/0f/05cc9c720236dcd2db9c1ab97fff629e96821be2e63103569da0c9b72f19/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dc2a197f38e5d0376ad52cd1a2a3617d3cdc150fd5966f4aee9bcebb1d68fe9", size = 60215947, upload-time = "2025-09-06T00:32:20.022Z" }, - { url = "https://files.pythonhosted.org/packages/3c/35/a9bf80a609e74e3b000fef598933235c908fcefcef9026042b8e6dfde2a9/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80", size = 60412546, upload-time = "2025-09-06T00:32:41.564Z" }, -] - -[[package]] -name = "nvidia-nvtx" -version = "13.0.85" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4", size = 148047, upload-time = "2025-09-04T08:29:01.761Z" }, - { url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878, upload-time = "2025-09-04T08:28:53.627Z" }, -] - -[[package]] -name = "oauthlib" -version = "3.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6d/fa/fbf4001037904031639e6bfbfc02badfc7e12f137a8afa254df6c4c8a670/oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918", size = 177352, upload-time = "2022-10-17T20:04:27.471Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/80/cab10959dc1faead58dc8384a781dfbf93cb4d33d50988f7a69f1b7c9bbe/oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca", size = 151688, upload-time = "2022-10-17T20:04:24.037Z" }, -] - -[[package]] -name = "ocrmypdf" -version = "16.13.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "deprecation" }, - { name = "img2pdf" }, - { name = "packaging" }, - { name = "pdfminer-six" }, - { name = "pi-heif" }, - { name = "pikepdf" }, - { name = "pillow" }, - { name = "pluggy" }, - { name = "rich" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8c/52/be1aaece0703a736757d8957c0d4f19c37561054169b501eb0e7132f15e5/ocrmypdf-16.13.0.tar.gz", hash = "sha256:29d37e915234ce717374863a9cc5dd32d29e063dfe60c51380dda71254c88248", size = 7042247, upload-time = "2025-12-24T07:58:35.86Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/41/b1/e2e7ad98de0d3ee05b44dbc3f78ccb158a620f3add82d00c85490120e7f2/ocrmypdf-16.13.0-py3-none-any.whl", hash = "sha256:fad8a6f7cc52cdc6225095c401a1766c778c47efe9f1e854ae4dc64a550a3d37", size = 165377, upload-time = "2025-12-24T07:58:33.925Z" }, -] - -[[package]] -name = "openai" -version = "1.109.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "httpx" }, - { name = "jiter" }, - { name = "pydantic" }, - { name = "sniffio" }, - { name = "tqdm" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c6/a1/a303104dc55fc546a3f6914c842d3da471c64eec92043aef8f652eb6c524/openai-1.109.1.tar.gz", hash = "sha256:d173ed8dbca665892a6db099b4a2dfac624f94d20a93f46eb0b56aae940ed869", size = 564133, upload-time = "2025-09-24T13:00:53.075Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/2a/7dd3d207ec669cacc1f186fd856a0f61dbc255d24f6fdc1a6715d6051b0f/openai-1.109.1-py3-none-any.whl", hash = "sha256:6bcaf57086cf59159b8e27447e4e7dd019db5d29a438072fbd49c290c7e65315", size = 948627, upload-time = "2025-09-24T13:00:50.754Z" }, -] - -[[package]] -name = "opencv-python-headless" -version = "4.11.0.86" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.4.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.4.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/36/2f/5b2b3ba52c864848885ba988f24b7f105052f68da9ab0e693cc7c25b0b30/opencv-python-headless-4.11.0.86.tar.gz", hash = "sha256:996eb282ca4b43ec6a3972414de0e2331f5d9cda2b41091a49739c19fb843798", size = 95177929, upload-time = "2025-01-16T13:53:40.22Z" } wheels = [ @@ -3064,10 +2655,10 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux')", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "python-dateutil", marker = "python_full_version < '3.11'" }, - { name = "pytz", marker = "python_full_version < '3.11'" }, - { name = "tzdata", marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } wheels = [ @@ -3147,9 +2738,9 @@ resolution-markers = [ "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", ] dependencies = [ - { name = "numpy", version = "2.4.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "python-dateutil", marker = "python_full_version >= '3.11'" }, - { name = "tzdata", marker = "(python_full_version >= '3.11' and sys_platform == 'emscripten') or (python_full_version >= '3.11' and sys_platform == 'win32')" }, + { name = "numpy", version = "2.4.3", source = { registry = "https://pypi.org/simple" } }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2e/0c/b28ed414f080ee0ad153f848586d61d1878f91689950f037f976ce15f6c8/pandas-3.0.1.tar.gz", hash = "sha256:4186a699674af418f655dbd420ed87f50d56b4cd6603784279d9eef6627823c8", size = 4641901, upload-time = "2026-02-17T22:20:16.434Z" } wheels = [ @@ -3223,6 +2814,19 @@ io = [ { name = "pyyaml" }, ] +[[package]] +name = "pdf-inspector" +version = "1.14.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/ba/6ef1dddc4151ed262bffd452d2289d59ddcb0525b64b62b36f34350c22a4/pdf_inspector-1.14.2.tar.gz", hash = "sha256:a0956ce5491112dbcbfeb8cefd1eaf0ef3409c0e008c434d74de1397cec447c7", size = 1576409, upload-time = "2026-08-13T21:19:08.486Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/66/992ab5293bc5713eec671aca1c69bcd9dbbda3e16721d5f024ab21a91cbe/pdf_inspector-1.14.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:9689427e4d8643355052acf2b837c6466f4f019a60ad59bdb5b96332c2b0e8c2", size = 2870693, upload-time = "2026-08-13T21:19:00.604Z" }, + { url = "https://files.pythonhosted.org/packages/94/02/15ae2baf2b19de2e53d874b5177d5ce7982ef8f6dd60c643bc51f113be48/pdf_inspector-1.14.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:54fa3ae38c340c51b9e507bde91c94b77ac2338327c710f50147c22303d3cf38", size = 2785049, upload-time = "2026-08-13T21:19:02.582Z" }, + { url = "https://files.pythonhosted.org/packages/36/b3/965bfec69b0bad2430d4603d14a5bc17386849fdca37c439c3c03c03fa9f/pdf_inspector-1.14.2-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99a1edd232855bdb69b7f35add87987c8e3d40c8240638b2cd159987d3e29513", size = 2958709, upload-time = "2026-08-13T21:19:04.099Z" }, + { url = "https://files.pythonhosted.org/packages/ae/51/49b486a78dfd8ec27be56004443b4bf9be27c0aa23430074d9a6e29b8190/pdf_inspector-1.14.2-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8bf3536ddbc7756d6c60b2e8e023bc21366370ad6efb1a910f959206bcadf023", size = 3059332, upload-time = "2026-08-13T21:19:05.435Z" }, + { url = "https://files.pythonhosted.org/packages/3d/23/a2896789cc1e3c078f46dd3f25eec90b854099ae797ff1eca9df0a4d401e/pdf_inspector-1.14.2-cp38-abi3-win_amd64.whl", hash = "sha256:1f3638b6f2e54afbc1940c0bfd83740713d4fde6c44bcb75f1328254801f0ff4", size = 2702638, upload-time = "2026-08-13T21:19:07.008Z" }, +] + [[package]] name = "pdf2image" version = "1.17.0" @@ -3248,21 +2852,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/8b/28c4eaec9d6b036a52cb44720408f26b1a143ca9bce76cc19e8f5de00ab4/pdfminer_six-20260107-py3-none-any.whl", hash = "sha256:366585ba97e80dffa8f00cebe303d2f381884d8637af4ce422f1df3ef38111a9", size = 6592252, upload-time = "2026-01-07T13:29:10.742Z" }, ] -[[package]] -name = "pdftext" -version = "0.6.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "pydantic" }, - { name = "pydantic-settings" }, - { name = "pypdfium2" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a9/7b/fe3205d44d6058932bbc785f0b9da2ed35b62e17479a8a7d2baca9df1cc6/pdftext-0.6.3.tar.gz", hash = "sha256:ab5c5dfe0f1fb78de1db837ccadac1ea41b07ce1890fead973c9a84cdaf54dec", size = 21968, upload-time = "2025-06-11T14:42:09.492Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bc/b9/4437bb89f04e57f48c96492a50d6168da5e201940de6620730d390449991/pdftext-0.6.3-py3-none-any.whl", hash = "sha256:528431ed8bdce39d74372cd3d27e8544af812f1f1adc81db229cf9fb48dacacb", size = 23693, upload-time = "2025-06-11T14:42:08.157Z" }, -] - [[package]] name = "pi-heif" version = "0.22.0" @@ -3415,15 +3004,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/52/3b/ce7a01026a7cf46e5452afa86f97a5e88ca97f562cafa76570178ab56d8d/pillow-10.4.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:0755ffd4a0c6f267cccbae2e9903d95477ca2f77c4fcf3a3a09570001856c8a5", size = 2554661, upload-time = "2024-07-01T09:48:20.293Z" }, ] -[[package]] -name = "platformdirs" -version = "4.9.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/19/56/8d4c30c8a1d07013911a8fdbd8f89440ef9f08d07a1b50ab8ca8be5a20f9/platformdirs-4.9.4.tar.gz", hash = "sha256:1ec356301b7dc906d83f371c8f487070e99d3ccf9e501686456394622a01a934", size = 28737, upload-time = "2026-03-05T18:34:13.271Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/63/d7/97f7e3a6abb67d8080dd406fd4df842c2be0efaf712d1c899c32a075027c/platformdirs-4.9.4-py3-none-any.whl", hash = "sha256:68a9a4619a666ea6439f2ff250c12a853cd1cbd5158d258bd824a7df6be2f868", size = 21216, upload-time = "2026-03-05T18:34:12.172Z" }, -] - [[package]] name = "pluggy" version = "1.6.0" @@ -3433,22 +3013,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] -[[package]] -name = "pre-commit" -version = "4.5.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cfgv" }, - { name = "identify" }, - { name = "nodeenv" }, - { name = "pyyaml" }, - { name = "virtualenv" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/40/f1/6d86a29246dfd2e9b6237f0b5823717f60cad94d47ddc26afa916d21f525/pre_commit-4.5.1.tar.gz", hash = "sha256:eb545fcff725875197837263e977ea257a402056661f09dae08e4b149b030a61", size = 198232, upload-time = "2025-12-16T21:14:33.552Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/19/fd3ef348460c80af7bb4669ea7926651d1f95c23ff2df18b9d24bab4f3fa/pre_commit-4.5.1-py2.py3-none-any.whl", hash = "sha256:3b3afd891e97337708c1674210f8eba659b52a38ea5f822ff142d10786221f77", size = 226437, upload-time = "2025-12-16T21:14:32.409Z" }, -] - [[package]] name = "propcache" version = "0.4.1" @@ -3656,18 +3220,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", size = 83997, upload-time = "2026-03-17T01:06:52.036Z" }, ] -[[package]] -name = "pyasn1-modules" -version = "0.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyasn1" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, -] - [[package]] name = "pycparser" version = "3.0" @@ -3760,15 +3312,16 @@ wheels = [ [[package]] name = "pydantic-settings" -version = "2.6.1" +version = "2.15.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, { name = "python-dotenv" }, + { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b5/d4/9dfbe238f45ad8b168f5c96ee49a3df0598ce18a0795a983b419949ce65b/pydantic_settings-2.6.1.tar.gz", hash = "sha256:e0f92546d8a9923cb8941689abf85d6601a8c19a23e97a34b2964a2e3f813ca0", size = 75646, upload-time = "2024-11-01T11:00:05.17Z" } +sdist = { url = "https://files.pythonhosted.org/packages/68/ca/31c57507b13119d7d3cfa1576dad2911a4861e3be07b579395f4e9d393f9/pydantic_settings-2.15.0.tar.gz", hash = "sha256:694b793e84f766ba76a90ebdefc01d0a9a045dab0382bee70393da93712ad117", size = 261253, upload-time = "2026-08-07T09:24:57.419Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/f9/ff95fd7d760af42f647ea87f9b8a383d891cdb5e5dbd4613edaeb094252a/pydantic_settings-2.6.1-py3-none-any.whl", hash = "sha256:7fb0637c786a558d3103436278a7c4f1cfd29ba8973238a50c5bb9a55387da87", size = 28595, upload-time = "2024-11-01T11:00:02.64Z" }, + { url = "https://files.pythonhosted.org/packages/30/a4/2bffa9f8e804325a09867f0e9d30795c80ea9f8d62560bd1b6ad6220eb2f/pydantic_settings-2.15.0-py3-none-any.whl", hash = "sha256:0ba092c291c94baceb5eff768aa0d56400a457585bc0175925a5a5510303da42", size = 69413, upload-time = "2026-08-07T09:24:55.839Z" }, ] [[package]] @@ -3793,23 +3346,33 @@ wheels = [ ] [[package]] -name = "pypdfium2" -version = "4.30.0" +name = "pymupdf" +version = "1.28.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a1/14/838b3ba247a0ba92e4df5d23f2bea9478edcfd72b78a39d6ca36ccd84ad2/pypdfium2-4.30.0.tar.gz", hash = "sha256:48b5b7e5566665bc1015b9d69c1ebabe21f6aee468b509531c3c8318eeee2e16", size = 140239, upload-time = "2024-05-09T18:33:17.552Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/fb/b6761fa2d5266f2cdb24c3b91f4023070ab7848381417678e7a289a1d52a/pymupdf-1.28.2.tar.gz", hash = "sha256:5e0be7908a715aa20333caddd73f1d6f01e4cd0c26e869fa2dd0b7f344da2249", size = 87903557, upload-time = "2026-08-06T21:43:23.321Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/9a/c8ff5cc352c1b60b0b97642ae734f51edbab6e28b45b4fcdfe5306ee3c83/pypdfium2-4.30.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:b33ceded0b6ff5b2b93bc1fe0ad4b71aa6b7e7bd5875f1ca0cdfb6ba6ac01aab", size = 2837254, upload-time = "2024-05-09T18:32:48.653Z" }, - { url = "https://files.pythonhosted.org/packages/21/8b/27d4d5409f3c76b985f4ee4afe147b606594411e15ac4dc1c3363c9a9810/pypdfium2-4.30.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:4e55689f4b06e2d2406203e771f78789bd4f190731b5d57383d05cf611d829de", size = 2707624, upload-time = "2024-05-09T18:32:51.458Z" }, - { url = "https://files.pythonhosted.org/packages/11/63/28a73ca17c24b41a205d658e177d68e198d7dde65a8c99c821d231b6ee3d/pypdfium2-4.30.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e6e50f5ce7f65a40a33d7c9edc39f23140c57e37144c2d6d9e9262a2a854854", size = 2793126, upload-time = "2024-05-09T18:32:53.581Z" }, - { url = "https://files.pythonhosted.org/packages/d1/96/53b3ebf0955edbd02ac6da16a818ecc65c939e98fdeb4e0958362bd385c8/pypdfium2-4.30.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3d0dd3ecaffd0b6dbda3da663220e705cb563918249bda26058c6036752ba3a2", size = 2591077, upload-time = "2024-05-09T18:32:55.99Z" }, - { url = "https://files.pythonhosted.org/packages/ec/ee/0394e56e7cab8b5b21f744d988400948ef71a9a892cbeb0b200d324ab2c7/pypdfium2-4.30.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc3bf29b0db8c76cdfaac1ec1cde8edf211a7de7390fbf8934ad2aa9b4d6dfad", size = 2864431, upload-time = "2024-05-09T18:32:57.911Z" }, - { url = "https://files.pythonhosted.org/packages/65/cd/3f1edf20a0ef4a212a5e20a5900e64942c5a374473671ac0780eaa08ea80/pypdfium2-4.30.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1f78d2189e0ddf9ac2b7a9b9bd4f0c66f54d1389ff6c17e9fd9dc034d06eb3f", size = 2812008, upload-time = "2024-05-09T18:32:59.886Z" }, - { url = "https://files.pythonhosted.org/packages/c8/91/2d517db61845698f41a2a974de90762e50faeb529201c6b3574935969045/pypdfium2-4.30.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:5eda3641a2da7a7a0b2f4dbd71d706401a656fea521b6b6faa0675b15d31a163", size = 6181543, upload-time = "2024-05-09T18:33:02.597Z" }, - { url = "https://files.pythonhosted.org/packages/ba/c4/ed1315143a7a84b2c7616569dfb472473968d628f17c231c39e29ae9d780/pypdfium2-4.30.0-py3-none-musllinux_1_1_i686.whl", hash = "sha256:0dfa61421b5eb68e1188b0b2231e7ba35735aef2d867d86e48ee6cab6975195e", size = 6175911, upload-time = "2024-05-09T18:33:05.376Z" }, - { url = "https://files.pythonhosted.org/packages/7a/c4/9e62d03f414e0e3051c56d5943c3bf42aa9608ede4e19dc96438364e9e03/pypdfium2-4.30.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:f33bd79e7a09d5f7acca3b0b69ff6c8a488869a7fab48fdf400fec6e20b9c8be", size = 6267430, upload-time = "2024-05-09T18:33:08.067Z" }, - { url = "https://files.pythonhosted.org/packages/90/47/eda4904f715fb98561e34012826e883816945934a851745570521ec89520/pypdfium2-4.30.0-py3-none-win32.whl", hash = "sha256:ee2410f15d576d976c2ab2558c93d392a25fb9f6635e8dd0a8a3a5241b275e0e", size = 2775951, upload-time = "2024-05-09T18:33:10.567Z" }, - { url = "https://files.pythonhosted.org/packages/25/bd/56d9ec6b9f0fc4e0d95288759f3179f0fcd34b1a1526b75673d2f6d5196f/pypdfium2-4.30.0-py3-none-win_amd64.whl", hash = "sha256:90dbb2ac07be53219f56be09961eb95cf2473f834d01a42d901d13ccfad64b4c", size = 2892098, upload-time = "2024-05-09T18:33:13.107Z" }, - { url = "https://files.pythonhosted.org/packages/be/7a/097801205b991bc3115e8af1edb850d30aeaf0118520b016354cf5ccd3f6/pypdfium2-4.30.0-py3-none-win_arm64.whl", hash = "sha256:119b2969a6d6b1e8d55e99caaf05290294f2d0fe49c12a3f17102d01c441bd29", size = 2752118, upload-time = "2024-05-09T18:33:15.489Z" }, + { url = "https://files.pythonhosted.org/packages/b4/51/550c9a75c4ff3245cb4ecb7bb95cbe2ab7374230b8e2b7a1f7259444150b/pymupdf-1.28.2-cp310-abi3-macosx_10_15_x86_64.whl", hash = "sha256:5fc315b425ff1f7afdd1ea2f348205cb19b806767daae7ce4d64115799c2bae1", size = 24645079, upload-time = "2026-08-06T21:37:25.001Z" }, + { url = "https://files.pythonhosted.org/packages/fa/01/3591f781b417b382a8487a2356e927acfe858b1043bab0ec47f6805bb109/pymupdf-1.28.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:7113846b35dbf0a033f088e4f4fb543dabeb4b0b12c112966a1ca1ee2d5eacae", size = 23875605, upload-time = "2026-08-06T21:37:40.369Z" }, + { url = "https://files.pythonhosted.org/packages/d2/86/4a68f080b71b46802178346af46486e1697508e760855ff5f3b218a6dff7/pymupdf-1.28.2-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3050a233dde1211efe89ada74e2add6238436434159f46097a1423aad2842545", size = 25095554, upload-time = "2026-08-06T21:37:58.485Z" }, + { url = "https://files.pythonhosted.org/packages/c7/06/dace3e27af26690cb20bead80dbac42941b0841eb689b8aabbd67dde16f0/pymupdf-1.28.2-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:397d6715c1f0df7548a92d0afd8ce370fc48fa47aeefac16be2bc04a16a8227f", size = 25762500, upload-time = "2026-08-06T21:38:17.438Z" }, + { url = "https://files.pythonhosted.org/packages/e5/61/4146dfa1d8172a1ce8d59f0eed94896ddefb8deb2274534d0522fbb8abf5/pymupdf-1.28.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f89fb2d86d07d643a269f17a093105057e20c79c1d06c103b53600067b6d2b01", size = 25986309, upload-time = "2026-08-06T21:38:35.472Z" }, + { url = "https://files.pythonhosted.org/packages/52/60/1fb6e64676f7500ebe89054b9e5bbbe14d3101c92d5f1a40ac9a35227673/pymupdf-1.28.2-cp310-abi3-win32.whl", hash = "sha256:530ef543a3885b3b81cb72a854e7c5a625a9233201221132bb6c31698c6a2bdb", size = 18525353, upload-time = "2026-08-06T21:38:47.697Z" }, + { url = "https://files.pythonhosted.org/packages/4a/61/d563bbccba262f9dd6d2d35ccb72593648184d886188efb12d9ce8f34dd6/pymupdf-1.28.2-cp310-abi3-win_amd64.whl", hash = "sha256:ebd244918798502d7b4504c90410d1711a4d7675a32584ca30f1bab419ecbffe", size = 19826532, upload-time = "2026-08-06T21:39:00.213Z" }, + { url = "https://files.pythonhosted.org/packages/e2/93/08f404a1f0155fe24137cf2d3aabd3e2b4b08c62053ed89c60f2611be3e9/pymupdf-1.28.2-cp310-abi3-win_arm64.whl", hash = "sha256:ffe91a24edc75c80da2a4b62f50fc0f54632d34fc8fe4cbc48e5c7ff07cf8fb4", size = 19759252, upload-time = "2026-08-06T21:39:12.937Z" }, + { url = "https://files.pythonhosted.org/packages/58/8c/d897dcd32a25b58186c968b15ce4324ca029e9d96460de12325314e390be/pymupdf-1.28.2-cp313-abi3-pyemscripten_2025_0_wasm32.whl", hash = "sha256:2e1b574c0fd2cb238021033fd3c0f9c4388816638df064e4bfb56d9d81736dc8", size = 18399403, upload-time = "2026-08-06T21:39:25.008Z" }, + { url = "https://files.pythonhosted.org/packages/f6/f1/de34a1c53fe2bf8c6e71db84b0ced782d408970c9810d2b456a2ae96814c/pymupdf-1.28.2-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:fd481ed48bef56305c41fb7e05a055c03345c899c7b101dad086258b438f8168", size = 25802333, upload-time = "2026-08-06T21:39:41.426Z" }, +] + +[[package]] +name = "pymupdf4llm" +version = "0.0.27" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pymupdf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/03/99/2634b56ff2b68558e742c6e10cd798ed3b5b5af6cc659454b9f37e7b4ecf/pymupdf4llm-0.0.27.tar.gz", hash = "sha256:35cc8bd6e0968bc1300b1fe4500f52e47da6b570af19af6c193e5fb6ee367008", size = 29368, upload-time = "2025-07-19T11:52:01.059Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/c8/7eed2e902b61574b15b295017ddb5738c4970aa9ab76903fbaace28a522e/pymupdf4llm-0.0.27-py3-none-any.whl", hash = "sha256:2eaaf9419c35520efda38f3806a276f2ec6cd29564fbb60a5c9c53a49fedb13c", size = 30055, upload-time = "2025-07-19T11:52:04.089Z" }, ] [[package]] @@ -3906,19 +3469,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] -[[package]] -name = "python-discovery" -version = "1.2.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "filelock" }, - { name = "platformdirs" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/de/ef/3bae0e537cfe91e8431efcba4434463d2c5a65f5a89edd47c6cf2f03c55f/python_discovery-1.2.2.tar.gz", hash = "sha256:876e9c57139eb757cb5878cbdd9ae5379e5d96266c99ef731119e04fffe533bb", size = 58872, upload-time = "2026-04-07T17:28:49.249Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d8/db/795879cc3ddfe338599bddea6388cc5100b088db0a4caf6e6c1af1c27e04/python_discovery-1.2.2-py3-none-any.whl", hash = "sha256:e1ae95d9af875e78f15e19aed0c6137ab1bb49c200f21f5061786490c9585c7a", size = 31894, upload-time = "2026-04-07T17:28:48.09Z" }, -] - [[package]] name = "python-dotenv" version = "1.2.2" @@ -4041,96 +3591,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] -[[package]] -name = "rapidfuzz" -version = "3.14.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2c/21/ef6157213316e85790041254259907eb722e00b03480256c0545d98acd33/rapidfuzz-3.14.5.tar.gz", hash = "sha256:ba10ac57884ce82112f7ed910b67e7fb6072d8ef2c06e30dc63c0f604a112e0e", size = 57901753, upload-time = "2026-04-07T11:16:31.931Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4f/b1/d6d6e7737fe3d0eb2ac2ac337686420d538f83f28495acc3cc32201c0dbf/rapidfuzz-3.14.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:071d96b957a33b9296b9284b6350a0fb6d030b154a04efd7c15e56b98b79a517", size = 1953508, upload-time = "2026-04-07T11:13:37.733Z" }, - { url = "https://files.pythonhosted.org/packages/2b/7b/94c1c953ac818bdd88b43213a9d38e4a41e953b786af3c3b2444d4a8f96d/rapidfuzz-3.14.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:667f40fe9c81ad129b198d236881b00dd9e8314d9cc72d03c3e16bdfe5879051", size = 1160895, upload-time = "2026-04-07T11:13:39.278Z" }, - { url = "https://files.pythonhosted.org/packages/7f/60/a67a7ca7c2532c6c1a4b5cd797917780eed43798b82c98b6df734a086c95/rapidfuzz-3.14.5-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f9fff308486bbd2c8c24f25e8e152c7594d3fe8db265a2d6a1ce24d58671127f", size = 1382245, upload-time = "2026-04-07T11:13:41.054Z" }, - { url = "https://files.pythonhosted.org/packages/95/ff/a42c9ce9f9e90ceb5b51136e0b8e8e6e5113ba0b45d986effbd671e7dddf/rapidfuzz-3.14.5-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dfa552338f51aec280f17b02d28bace1e162d1a84ccd80e3339a57f98aedb56b", size = 3163974, upload-time = "2026-04-07T11:13:42.662Z" }, - { url = "https://files.pythonhosted.org/packages/e3/3c/11e2d41075e6e48b7dad373631b379b7e40491f71d5412c5a98d3c58f60f/rapidfuzz-3.14.5-cp310-cp310-manylinux_2_39_riscv64.whl", hash = "sha256:068b3e965ca9d9ee4debe40001ae7c3938ba646308afd33cf0c66618147db65c", size = 1475540, upload-time = "2026-04-07T11:13:44.687Z" }, - { url = "https://files.pythonhosted.org/packages/29/fa/09be143dcc22c79f09cf90168a574725dbda49f02cbbd55d0447da8bec86/rapidfuzz-3.14.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:88b7d31ff1cc5e9bc0e4406e6b1fa00b6d37163d50bb58091e9b976ff1129faa", size = 2404128, upload-time = "2026-04-07T11:13:46.641Z" }, - { url = "https://files.pythonhosted.org/packages/32/f9/1aeb504cdcfde42881825e9c86f48238d4e01ba8a1530491e82eb17e5689/rapidfuzz-3.14.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:eacb434410b8d9ca99a8d42352ef085cf423e3c76c1f0b86be2fcba3bff2952c", size = 2508455, upload-time = "2026-04-07T11:13:48.726Z" }, - { url = "https://files.pythonhosted.org/packages/10/8e/b1b5eed8d887a29b0e18fd3222c46ca60fddfb528e7e1c41267ce42d5522/rapidfuzz-3.14.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:649712823f3abcdc48427147a5384fac15623ba435d0013959b52e6462521397", size = 4274060, upload-time = "2026-04-07T11:13:50.805Z" }, - { url = "https://files.pythonhosted.org/packages/e3/c4/7e5b0353693d4f47b8b0f96e941efc377cfb2034b67ef92d082ac4441a0f/rapidfuzz-3.14.5-cp310-cp310-win32.whl", hash = "sha256:13cb79c23ef5516e4c4e3830877be8b19aa75203636be1163d690d37803f6504", size = 1727457, upload-time = "2026-04-07T11:13:52.45Z" }, - { url = "https://files.pythonhosted.org/packages/d9/6e/f530a39b946fa71c009bc9c81fdb6b48a77bbc57ee8572ac0302b3bf6308/rapidfuzz-3.14.5-cp310-cp310-win_amd64.whl", hash = "sha256:f2073495a7f9b75e57e600747ac09510d67683fd64d3228e009740b7ef88f9fe", size = 1544657, upload-time = "2026-04-07T11:13:54.952Z" }, - { url = "https://files.pythonhosted.org/packages/bc/01/02fa075f9f59ff766d374fecbd042b3ac9782dcd5abc52d909a54f587eeb/rapidfuzz-3.14.5-cp310-cp310-win_arm64.whl", hash = "sha256:8166efddea49fdbc61185559f47593239e4794fd7c9044dd5a789d1a90af852d", size = 816587, upload-time = "2026-04-07T11:13:56.418Z" }, - { url = "https://files.pythonhosted.org/packages/e1/f9/3c41a7be8855803f4f6c713b472226a98d31d41869d98f64f4ca790510d6/rapidfuzz-3.14.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e251126d48615e1f02b4a178f2cd0cd4f0332b8a019c01a2e10480f7552554b4", size = 1952372, upload-time = "2026-04-07T11:13:58.32Z" }, - { url = "https://files.pythonhosted.org/packages/9e/89/c2557e37531d03465193bff0ab9de70b468420a807d71a26a65100635459/rapidfuzz-3.14.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5ab449c9abd0d4e1f8145dce0798a4c822a1a1933d613c764a641bea88b8bdab", size = 1159782, upload-time = "2026-04-07T11:14:00.127Z" }, - { url = "https://files.pythonhosted.org/packages/1a/b2/ffeeb7eca1a897d51b998f4c0ef0281696c3b06abcca4f88f9def708ffe1/rapidfuzz-3.14.5-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb2829fedd672dd7107267189dabe2bbe07972801d636014417c6861eb89e358", size = 1383677, upload-time = "2026-04-07T11:14:01.696Z" }, - { url = "https://files.pythonhosted.org/packages/6b/d0/4539e42a2d596e068f7738f279638a4a74edd1fbb6f8594e2458058979c6/rapidfuzz-3.14.5-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d50e5861872935fece391351cbb5ba21d1bced277cf5e1143d207a0a35f1925", size = 3168906, upload-time = "2026-04-07T11:14:03.29Z" }, - { url = "https://files.pythonhosted.org/packages/5e/1c/3ec897eb9d8b05308aa8ef6ae4ed64b088ad521a3f9d8ff469e7e97bc2b0/rapidfuzz-3.14.5-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:7092a216728f80c960bd6b3807275d1ee318b168986bd5dc523349581d4890b8", size = 1478176, upload-time = "2026-04-07T11:14:04.94Z" }, - { url = "https://files.pythonhosted.org/packages/ab/ba/970c03a12ce20a5399e22afe9f8932fd4cd1265b8a8461d0e63b00eb4eae/rapidfuzz-3.14.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9669753caef7fdc6529f6adcc5883ed98d65976445d9322e7dbdb6b697feee13", size = 2402441, upload-time = "2026-04-07T11:14:07.228Z" }, - { url = "https://files.pythonhosted.org/packages/81/93/61d351cae60c1d0e21ba5ff1a1015ad045539ed215da9d6e302204ed887a/rapidfuzz-3.14.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:823b1b9d9230809d8edcc18872770764bfe8ef4357995e16744047c8ccf0e489", size = 2511628, upload-time = "2026-04-07T11:14:09.234Z" }, - { url = "https://files.pythonhosted.org/packages/87/52/374d2d4f60fd98155142a869323aa221e30868cfa1f15171a0f64070c247/rapidfuzz-3.14.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f0b2af76b7e7060c09e1a0dfa9410eb19369cbe6164509bff2ef94094b54d2b6", size = 4275480, upload-time = "2026-04-07T11:14:11.332Z" }, - { url = "https://files.pythonhosted.org/packages/d8/04/82e7989bc9ec20a15b720a335c5cb6b0724bf6582013898f90a3280cfccd/rapidfuzz-3.14.5-cp311-cp311-win32.whl", hash = "sha256:c5801a89604c65ab4cc9e91b23bc4076d0ca80efd8c976fb63843d7879a85d7f", size = 1725627, upload-time = "2026-04-07T11:14:13.217Z" }, - { url = "https://files.pythonhosted.org/packages/b9/b5/eca8ac5609bc9bcb02bb6ff87fa5983cc92b8772d66a431556ab8a8c178f/rapidfuzz-3.14.5-cp311-cp311-win_amd64.whl", hash = "sha256:d7ca16637c0ede8243f84074044bd0b2335a0341421f8227c85756de2d18c819", size = 1545977, upload-time = "2026-04-07T11:14:14.766Z" }, - { url = "https://files.pythonhosted.org/packages/ca/e1/dbf318de28f65fa2cdd0a9dfbdee380f8199eb83b19259bc4f8592551b4e/rapidfuzz-3.14.5-cp311-cp311-win_arm64.whl", hash = "sha256:8c90cdf8516d9057e502aa6003cea71cf5ec27cc44699ca52412b502a04761bb", size = 816827, upload-time = "2026-04-07T11:14:16.788Z" }, - { url = "https://files.pythonhosted.org/packages/d3/e3/574435c6aafb80254c191ef40d7aca2cb2bb97a095ec9395e9fa59ac307a/rapidfuzz-3.14.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0d3378f471ef440473a396ce2f8e97ee12f89a78b495540e0a5617bbfe895638", size = 1944601, upload-time = "2026-04-07T11:14:18.771Z" }, - { url = "https://files.pythonhosted.org/packages/d0/1f/fbad3102a255ecc112ce9a7e779bacab7fd14398217be8868dc9082ba363/rapidfuzz-3.14.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1e910eebca9fd0eba245c0555e764597e8a0cccb673a92da2dc2397050725f48", size = 1164293, upload-time = "2026-04-07T11:14:20.534Z" }, - { url = "https://files.pythonhosted.org/packages/88/37/a3eb7ff6121ed3a5f199a8c38cc86c8e481816f879cb0e0b738b078c9a7e/rapidfuzz-3.14.5-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:01550fe5f60fd176aa66b7611289d46dc4aa4b1b904874c7b6d1d54e581c5ec1", size = 1371999, upload-time = "2026-04-07T11:14:22.63Z" }, - { url = "https://files.pythonhosted.org/packages/79/72/97a9728c711c7c1b06e107d3f0623880fb4ef90e147ed13c551a1730e7cc/rapidfuzz-3.14.5-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:48bee0b91bebfaec41e1081e351000659ab7570cc4598d617aa04d5bf827f9e6", size = 3145715, upload-time = "2026-04-07T11:14:24.508Z" }, - { url = "https://files.pythonhosted.org/packages/ed/54/d5caabbea233ac90c286c87c260e49d7641467e87438a18d858e41c82e91/rapidfuzz-3.14.5-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:7e580cb04ad849ae9b786fa21383c6b994b6e6c1444ad1cb9f22392759d72741", size = 1456304, upload-time = "2026-04-07T11:14:26.515Z" }, - { url = "https://files.pythonhosted.org/packages/fc/a7/2d1a81250ac8c01a0100c026018e76f0e7a097ff63e4c553e02a6938c6fb/rapidfuzz-3.14.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:09d6c9ba091854f07817055d795d604179c12a8f308ba4c7d56f3719dfea1646", size = 2389089, upload-time = "2026-04-07T11:14:28.635Z" }, - { url = "https://files.pythonhosted.org/packages/65/0d/c47c3872203ae88e6506997c0b576ad731f5261daa25d559be09c9756658/rapidfuzz-3.14.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:1e989f86113be66574113b9c7bdf4793f3f863d248e47d911b355e05ca6b6b10", size = 2493404, upload-time = "2026-04-07T11:14:30.577Z" }, - { url = "https://files.pythonhosted.org/packages/8f/2f/71e0a5a3130792146c8a200a2dd1e52aa16f7c1074012e17f2601eea9a90/rapidfuzz-3.14.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0ebd1a18e2e47bc0b292a07e6ed9c3642f8aaa672d12253885f599b50807a4f9", size = 4251709, upload-time = "2026-04-07T11:14:32.451Z" }, - { url = "https://files.pythonhosted.org/packages/86/45/d39874901abacef325adb5b34ae416817c8486dfb4fb87c7a9b74ec5b072/rapidfuzz-3.14.5-cp312-cp312-win32.whl", hash = "sha256:9981d38a703b86f0e315a3cd229fd1906fe1d91c989ed121fb975b3c849f89f5", size = 1710069, upload-time = "2026-04-07T11:14:34.37Z" }, - { url = "https://files.pythonhosted.org/packages/85/0b/f65572c53de8a1c704bda707f63a447b67bdbe95d7cdc70d18885e191df5/rapidfuzz-3.14.5-cp312-cp312-win_amd64.whl", hash = "sha256:d8375e3da319593389727c3187ccaf3e0e84199accc530866b8e0f2b79af05e9", size = 1540630, upload-time = "2026-04-07T11:14:36.287Z" }, - { url = "https://files.pythonhosted.org/packages/5e/c3/143be3a578f989758cae516f3270d5cbb49783a7bfdf57cc27a670e00456/rapidfuzz-3.14.5-cp312-cp312-win_arm64.whl", hash = "sha256:478b59bb018a6780d73f33e38d0b3ec5e968a6c1ed42876b993dd456b7aa20e8", size = 813137, upload-time = "2026-04-07T11:14:38.289Z" }, - { url = "https://files.pythonhosted.org/packages/11/66/252803f2010ba699618cdc048b6e1f7cc1f433c08b4a9a17579b92ab0142/rapidfuzz-3.14.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ebd8fd343bf8492a1e60bcb6dc99f90f74f65d98d8241a6b3e1fed225b76ecd6", size = 1940205, upload-time = "2026-04-07T11:14:40.319Z" }, - { url = "https://files.pythonhosted.org/packages/ea/59/b2afd98e41af9cd54554a4c1c423d84cdd60e6b1c0a09496f033b55f60ec/rapidfuzz-3.14.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6737b35d5af7479c5bf9710f7b17edd9d2c43128d974d25fb4ea653e42c64609", size = 1159639, upload-time = "2026-04-07T11:14:42.52Z" }, - { url = "https://files.pythonhosted.org/packages/a3/31/7aa7e62c4c516a7af322ed0c4f0774208b72d457d0cfec808bad0df12f4a/rapidfuzz-3.14.5-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b002c7994cc9f2bc9d9856f0fbaee6e8072c983873846c92f25cefba5b2a925f", size = 1367194, upload-time = "2026-04-07T11:14:44.25Z" }, - { url = "https://files.pythonhosted.org/packages/90/79/2fc252a63bc91d3c3b234d0a3a6ad4ebc460037a23cdcdaf9285f986e6c9/rapidfuzz-3.14.5-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:17a34330cd2a538c1ce5d400b61ba358c5b72c654b928ff87b362e88f8b864c7", size = 3151805, upload-time = "2026-04-07T11:14:46.21Z" }, - { url = "https://files.pythonhosted.org/packages/17/54/0c83508f2683ea70e2d05f8527eb07328acf7bb1e9d97a3bece5702378e7/rapidfuzz-3.14.5-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:95d937e74c1a7a1287dfb03b62a827be08ede10a155cf1af73bbf47f2b73ee6e", size = 1455667, upload-time = "2026-04-07T11:14:47.991Z" }, - { url = "https://files.pythonhosted.org/packages/71/1b/070175e873177814d58850a01ebe80e20ae11e93eb4da894d563988660fa/rapidfuzz-3.14.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:46b92a9970dcc34f0096901c792644094cab49554ac3547f35e3aebbdf0a3610", size = 2388246, upload-time = "2026-04-07T11:14:50.098Z" }, - { url = "https://files.pythonhosted.org/packages/c9/dd/77caf7aaf9c2be050ad1f128d7c24ff0f59079aa62c5f62f9df41c0af45e/rapidfuzz-3.14.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e012177c8e8a8a0754ae0d6027d63042aa5ff036d9f40f07cb3466a6082e21b8", size = 2494333, upload-time = "2026-04-07T11:14:52.303Z" }, - { url = "https://files.pythonhosted.org/packages/2c/e2/dd7e1f2aa31a8fbbfc16b0610af1d770ffaf1287490f3c8c5b1c52da264f/rapidfuzz-3.14.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a2ae6f53f99c9a0eca7a0afc5b4e45fc73bc1dd4ac74c00509031d76df80ed98", size = 4258579, upload-time = "2026-04-07T11:14:54.538Z" }, - { url = "https://files.pythonhosted.org/packages/9c/0a/ac99e1ba347ba0e85e0bb60b74231d55fb93c0eff43f2920ccb413d0be08/rapidfuzz-3.14.5-cp313-cp313-win32.whl", hash = "sha256:4a60f0057231188e3bd30216f7b4e0f279b11fa4ec818bb6c1d9f014d1562fbc", size = 1709231, upload-time = "2026-04-07T11:14:56.524Z" }, - { url = "https://files.pythonhosted.org/packages/cf/cb/0e251d731b3166378644238e8f0cf9e89858c024e19f75ca9f7e3ae83fd5/rapidfuzz-3.14.5-cp313-cp313-win_amd64.whl", hash = "sha256:11bfc2ed8fbe4ab86bd516fadefab126f90e6dcadffa761739fcb304707dfd35", size = 1538519, upload-time = "2026-04-07T11:14:58.635Z" }, - { url = "https://files.pythonhosted.org/packages/30/6f/4548132acc947db6d5346a248e44a8b3a22d608ef30e770fb578caaf2d00/rapidfuzz-3.14.5-cp313-cp313-win_arm64.whl", hash = "sha256:b486b5218808f6f4dc471b114b1054e63553db69705c97da0271f47bd706aedd", size = 812628, upload-time = "2026-04-07T11:15:00.552Z" }, - { url = "https://files.pythonhosted.org/packages/00/60/69b177577290c5eab892c6f75fe89c3aff3f9ae80298a78d9372b1cecb9a/rapidfuzz-3.14.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:39ef8658aaf67d51667e7bdaf7096f432333377d8302ac43c70b5df8a4cf89b8", size = 1970231, upload-time = "2026-04-07T11:15:02.603Z" }, - { url = "https://files.pythonhosted.org/packages/48/38/2fd790052659cc4e2907b63c25433f0987864b445c1aeec1a302ef5ad948/rapidfuzz-3.14.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:9ad37a0be705b544af6296da8edddc260d10a8ae5462530fc9991f66498bb1f9", size = 1194394, upload-time = "2026-04-07T11:15:04.572Z" }, - { url = "https://files.pythonhosted.org/packages/80/f4/28430ad8472fc3536e8ebd51a864a226e979cfe924c6e3f83d111373aa74/rapidfuzz-3.14.5-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d45e06f60729e07d9b20c205f7e5cff90b6ef2584e852eecf46e045aea69627d", size = 1377051, upload-time = "2026-04-07T11:15:06.728Z" }, - { url = "https://files.pythonhosted.org/packages/77/7e/9aeacabcfd1e77397968362e5b98fe14248b8307011136b17daf99752a8e/rapidfuzz-3.14.5-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e52da10236aa6212de71b9e170bace65b64b129c0dea7fc243d6c9ce976f5074", size = 3160565, upload-time = "2026-04-07T11:15:08.667Z" }, - { url = "https://files.pythonhosted.org/packages/56/f4/db4dd7be0cd2f2022117ac5407d905f435d60e48baaea313a567ad27e865/rapidfuzz-3.14.5-cp313-cp313t-manylinux_2_39_riscv64.whl", hash = "sha256:440d30faaf682ca496170a7f0cc5453ec942e3e079f0fd802c9a7f938dfb50a3", size = 1442113, upload-time = "2026-04-07T11:15:11.138Z" }, - { url = "https://files.pythonhosted.org/packages/a4/99/0e9f6aa57f3e32a767216f797e56dc96b720fcecfb9d8ee907ecc82f8d66/rapidfuzz-3.14.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:56227a61fd3d17b0cd9793132431f3a3d07c8654be96794ba9f89fe0fc8b2d09", size = 2396618, upload-time = "2026-04-07T11:15:13.154Z" }, - { url = "https://files.pythonhosted.org/packages/60/94/44a78e39ffce17cbdd3e2b53b696acc751d5d153be0f499d052b07a4d904/rapidfuzz-3.14.5-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2e83cd2e25bb4edd97b689d9979d9c3acccdaaf26ceac08212ceece202febcfa", size = 2478220, upload-time = "2026-04-07T11:15:15.193Z" }, - { url = "https://files.pythonhosted.org/packages/dd/df/454311469a09a507e9d784a35796742bec22e4cebe75551e2da4e0e290fd/rapidfuzz-3.14.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:af3b859726cd3374287e405e14b9634563c078c5531a4f62375508addebddad1", size = 4265027, upload-time = "2026-04-07T11:15:17.28Z" }, - { url = "https://files.pythonhosted.org/packages/fc/01/175465a9ab3e3b70ba669058372f009d1d49c1746e2dcd56b69df188d3a5/rapidfuzz-3.14.5-cp313-cp313t-win32.whl", hash = "sha256:8ce1d850b3c0178440efde9e884d98421b5e87ff925f364d6d79e23910d7593f", size = 1766814, upload-time = "2026-04-07T11:15:19.687Z" }, - { url = "https://files.pythonhosted.org/packages/1b/a0/a9b84a47af06ebed94a1439eb2f02adebfb8628bcd30af1fe3e02f5ef56c/rapidfuzz-3.14.5-cp313-cp313t-win_amd64.whl", hash = "sha256:c84af70bcf34e99aee894e46a0f1ac77f17d0ef828179c387407642e2466d28a", size = 1582448, upload-time = "2026-04-07T11:15:21.98Z" }, - { url = "https://files.pythonhosted.org/packages/1e/f1/5937800238b3f8248e70860d79f69ba8f73e764fff47e36bc9e2f26dbcc6/rapidfuzz-3.14.5-cp313-cp313t-win_arm64.whl", hash = "sha256:aac0ad28c686a5e72b81668b906c030ee28050b244544b8af68e12fb32543895", size = 832932, upload-time = "2026-04-07T11:15:24.358Z" }, - { url = "https://files.pythonhosted.org/packages/81/41/aa3ffb3355e62e1bf91f6599b3092e866bc88487a07c524004943c7676df/rapidfuzz-3.14.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:1a31cc6d7d03e7318a0974c038959c59e19c752b81115f2e9138b3331cd64d45", size = 1943327, upload-time = "2026-04-07T11:15:26.266Z" }, - { url = "https://files.pythonhosted.org/packages/2d/e1/c2141f1840a41e07ad2db6f724945f8f8ff3065463899a22939152dd6e09/rapidfuzz-3.14.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0298d357e2bc59d572da4db0bc631009b6f8f6c9bc8c11e99a12b833f16b6575", size = 1161755, upload-time = "2026-04-07T11:15:28.659Z" }, - { url = "https://files.pythonhosted.org/packages/ca/07/66e753eeaa353161d1d331b7dd517bb349b0bacfebe8496d7b26be26f81f/rapidfuzz-3.14.5-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:59b3dba758661a318995655435c6ab20a04ade79fa51e75bc8dc107cac8df280", size = 1376571, upload-time = "2026-04-07T11:15:31.225Z" }, - { url = "https://files.pythonhosted.org/packages/c8/85/9535df0b78ba51f478c9ce7eb6d1f85535cc31fe356773b48fd9d3e563ca/rapidfuzz-3.14.5-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4900143d82071bdda533b00300c40b14b963ff826b3642cc463b6dd0f036585e", size = 3156468, upload-time = "2026-04-07T11:15:33.428Z" }, - { url = "https://files.pythonhosted.org/packages/81/ee/b667eb93bba6dc4e0de658edd778e1619dc4d6aab68fa5e5c7f075152735/rapidfuzz-3.14.5-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:feedf219672eef83ea6be6f3bb093bba396a8560fc75be85ba225f082903df0a", size = 1458311, upload-time = "2026-04-07T11:15:35.557Z" }, - { url = "https://files.pythonhosted.org/packages/7d/ce/479074f5624364a48df3403c538797ef22d3ac49c19dc76c3f79fcdcc70c/rapidfuzz-3.14.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:419e4397a36e2665ec992d8d64c20ba4b2a42500c76ecadeca78a4f19cb9cc32", size = 2398228, upload-time = "2026-04-07T11:15:37.669Z" }, - { url = "https://files.pythonhosted.org/packages/0b/15/a8982f649150fffbdcd6f17565974501f6ab33b2795267bffbd4a7ba905b/rapidfuzz-3.14.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:97131ab2be39043054ee28d99e09efe316e6d53449b7e962dfcf3c2de8b2b246", size = 2497226, upload-time = "2026-04-07T11:15:39.857Z" }, - { url = "https://files.pythonhosted.org/packages/19/52/5267c03ef6759831b7d4625a0c9c06e87baa2fae084b61ac9c388858317b/rapidfuzz-3.14.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:593c00dac4e30231c35bf3b4f1da8ec0998762e9e94425586a5d636fcd57f9d0", size = 4262283, upload-time = "2026-04-07T11:15:42.279Z" }, - { url = "https://files.pythonhosted.org/packages/71/c0/2579f343a97f5254c43bb5853baccc01488357dcb64a27bcb869b7888a4a/rapidfuzz-3.14.5-cp314-cp314-win32.whl", hash = "sha256:0084b687b02b4e569b46d8d6d4ad25659528e6081cd6d067ca453a69035f07e4", size = 1744614, upload-time = "2026-04-07T11:15:44.498Z" }, - { url = "https://files.pythonhosted.org/packages/17/eb/8edfed1e80119dc9c35b11df4bc701eea85622ad681fff0263b6961d3224/rapidfuzz-3.14.5-cp314-cp314-win_amd64.whl", hash = "sha256:5dfa89d78f22cd773054caff44827b846161a29f2dcf7e78b8f90d086621e502", size = 1588971, upload-time = "2026-04-07T11:15:46.86Z" }, - { url = "https://files.pythonhosted.org/packages/f6/04/5676df93c85cfa57a3045d8047318df9f3cd58c7b8a99340dd95f874795e/rapidfuzz-3.14.5-cp314-cp314-win_arm64.whl", hash = "sha256:67f3f9d2b444268ab53e47d31bab89954888d23c04c6789f2c727e51fe4b1d13", size = 834985, upload-time = "2026-04-07T11:15:49.411Z" }, - { url = "https://files.pythonhosted.org/packages/f7/0d/4a8988cea658fe335048ddef8c876addff1b6daa3c9ca8ad65a5a2196e69/rapidfuzz-3.14.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:77eac0526899b3c3ad1454bb2b03cdb491d67358ec8ef0c9c48bd61b632b431d", size = 1972517, upload-time = "2026-04-07T11:15:51.819Z" }, - { url = "https://files.pythonhosted.org/packages/1c/a3/f5cfd9965a9d9a9e32249159797c47b5d6299ea6d1629f9126b25f1c10a3/rapidfuzz-3.14.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b9c6bd754d11f6e78ac54e3d86b4b11dc1ba2f13e5fc958899574532897f5a99", size = 1196056, upload-time = "2026-04-07T11:15:54.292Z" }, - { url = "https://files.pythonhosted.org/packages/64/07/561c2e40cfd10e6630a7b0ac5a2a813aef50d944bcd1f3d260319d659d5b/rapidfuzz-3.14.5-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:738c96944d076deeaff70e92b65696ab4f7ecb8081d7791c5403a3257dfaf8ff", size = 1374732, upload-time = "2026-04-07T11:15:56.584Z" }, - { url = "https://files.pythonhosted.org/packages/c2/39/123bb94fee40e2fb3b7c49b80827c7ef42d838e18def3fc2fef5a3cf817a/rapidfuzz-3.14.5-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4c1bca487a17fe4226b4ffb2d30e799d2b274d692cffa76bd0746f56235fca3", size = 3166902, upload-time = "2026-04-07T11:15:58.768Z" }, - { url = "https://files.pythonhosted.org/packages/75/0a/45716fafc9fd2e028cf20b5ac5bc704887081cd312f84edb0e325599414b/rapidfuzz-3.14.5-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:af6a90a4ed2a48fa1a2d17e9d824e6c7c950bea5bad0b707c77fd55751e6bfef", size = 1452130, upload-time = "2026-04-07T11:16:01.453Z" }, - { url = "https://files.pythonhosted.org/packages/ca/49/4e96c413114398481c0a5b0086af32c364a18613c9a2ea578d17c4bea4ee/rapidfuzz-3.14.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bf5018938208d4597b2e679a4f8cff9fd252f1df53583130ae56281a21801b64", size = 2396308, upload-time = "2026-04-07T11:16:03.588Z" }, - { url = "https://files.pythonhosted.org/packages/89/b7/49fea9fc6878d59bd259d01dd1972d9b86117992b1c66d9b16f0a65273c3/rapidfuzz-3.14.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:c0919d1f89ddf91129906705723118ea09754171e4116f5a5dbc667c7bc9b261", size = 2488210, upload-time = "2026-04-07T11:16:05.871Z" }, - { url = "https://files.pythonhosted.org/packages/0c/44/a1f732b93ffacbdad077b7c801149549b2938e1bece6addb5ad85ed74df8/rapidfuzz-3.14.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:93d8da883a35116d6813432177f35e570db5b0a5e30ecb0cbd7cb39c815735df", size = 4270621, upload-time = "2026-04-07T11:16:08.483Z" }, - { url = "https://files.pythonhosted.org/packages/bb/ce/ff942d19fce5385054650bb71a58495ddda299d94661ccc4e6e7fa44868b/rapidfuzz-3.14.5-cp314-cp314t-win32.whl", hash = "sha256:0f23e37019ec07712d58976b1ab2b889f8649a7f7c2f626a2f34ea9139e79279", size = 1803950, upload-time = "2026-04-07T11:16:10.873Z" }, - { url = "https://files.pythonhosted.org/packages/5c/0f/9aafc63f9661222b819b391c187eed29fc90ad5935f9690e5ecc2d2047a4/rapidfuzz-3.14.5-cp314-cp314t-win_amd64.whl", hash = "sha256:7d5ca9c7832e6879a707296d1463685f7c243a27846227044504741640caec66", size = 1632357, upload-time = "2026-04-07T11:16:13.1Z" }, - { url = "https://files.pythonhosted.org/packages/70/a6/51fc1b0e61e3326e1c68a61cfd0c6b3c34c843681c4b1eefbf0596f59162/rapidfuzz-3.14.5-cp314-cp314t-win_arm64.whl", hash = "sha256:3e91dcd2549b8f8d843f98ba03a17e01f3d8b72ce942adbbb6761bc58ffce813", size = 855409, upload-time = "2026-04-07T11:16:15.787Z" }, - { url = "https://files.pythonhosted.org/packages/d9/ee/e71853bf82846c5c2174b924b71d8e8099fb05ff87c958a720380b434ba3/rapidfuzz-3.14.5-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:578e6051f6d5e6200c259b47a103cf06bb875ab5814d17333fc0b5c290b22f4c", size = 1888603, upload-time = "2026-04-07T11:16:18.223Z" }, - { url = "https://files.pythonhosted.org/packages/36/82/40f67b730f32be2ebad9f62add1571c754f52249254b2e88af094b907eee/rapidfuzz-3.14.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:fbf1b8bb2695415b347f3727da1addca2acb82c9b97ac86bebf8b1bead1eb12d", size = 1120599, upload-time = "2026-04-07T11:16:20.682Z" }, - { url = "https://files.pythonhosted.org/packages/ef/9f/a3635cc4ec8fc6e14b46e7db1f7f8763d8c4bef33dcc124eea2e6cb2c8f3/rapidfuzz-3.14.5-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8f4a8f5cc84c7ad6bffa0e9947b33eb343ad66e6b53e94fe54378a5508c5ed53", size = 1348524, upload-time = "2026-04-07T11:16:23.451Z" }, - { url = "https://files.pythonhosted.org/packages/cc/1b/2b229520f0b48464cfcd7aa758f74551d12c9bc4ab544022a60210aab064/rapidfuzz-3.14.5-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:97c6d85283629646fa87acc22c66b30ea9d4de7f6fdf887daa2e30fa041829b5", size = 3099302, upload-time = "2026-04-07T11:16:25.858Z" }, - { url = "https://files.pythonhosted.org/packages/aa/b5/363906b1064fc6fe611783a61764927bbd91919aaaabe8cba82151ca93ef/rapidfuzz-3.14.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:dfef96543ced67d9513a422755db422ae1dc34dade0a1485e0b43e7342ed3ebf", size = 1509889, upload-time = "2026-04-07T11:16:28.487Z" }, -] - [[package]] name = "redis" version = "7.4.0" @@ -4463,316 +3923,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/f0/ae7ca09223a81a1d890b2557186ea015f6e0502e9b8cb8e1813f1d8cfa4e/s3transfer-0.14.0-py3-none-any.whl", hash = "sha256:ea3b790c7077558ed1f02a3072fb3cb992bbbd253392f4b6e9e8976941c7d456", size = 85712, upload-time = "2025-09-09T19:23:30.041Z" }, ] -[[package]] -name = "safetensors" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/29/9c/6e74567782559a63bd040a236edca26fd71bc7ba88de2ef35d75df3bca5e/safetensors-0.7.0.tar.gz", hash = "sha256:07663963b67e8bd9f0b8ad15bb9163606cd27cc5a1b96235a50d8369803b96b0", size = 200878, upload-time = "2025-11-19T15:18:43.199Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/47/aef6c06649039accf914afef490268e1067ed82be62bcfa5b7e886ad15e8/safetensors-0.7.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c82f4d474cf725255d9e6acf17252991c3c8aac038d6ef363a4bf8be2f6db517", size = 467781, upload-time = "2025-11-19T15:18:35.84Z" }, - { url = "https://files.pythonhosted.org/packages/e8/00/374c0c068e30cd31f1e1b46b4b5738168ec79e7689ca82ee93ddfea05109/safetensors-0.7.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:94fd4858284736bb67a897a41608b5b0c2496c9bdb3bf2af1fa3409127f20d57", size = 447058, upload-time = "2025-11-19T15:18:34.416Z" }, - { url = "https://files.pythonhosted.org/packages/f1/06/578ffed52c2296f93d7fd2d844cabfa92be51a587c38c8afbb8ae449ca89/safetensors-0.7.0-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e07d91d0c92a31200f25351f4acb2bc6aff7f48094e13ebb1d0fb995b54b6542", size = 491748, upload-time = "2025-11-19T15:18:09.79Z" }, - { url = "https://files.pythonhosted.org/packages/ae/33/1debbbb70e4791dde185edb9413d1fe01619255abb64b300157d7f15dddd/safetensors-0.7.0-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8469155f4cb518bafb4acf4865e8bb9d6804110d2d9bdcaa78564b9fd841e104", size = 503881, upload-time = "2025-11-19T15:18:16.145Z" }, - { url = "https://files.pythonhosted.org/packages/8e/1c/40c2ca924d60792c3be509833df711b553c60effbd91da6f5284a83f7122/safetensors-0.7.0-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:54bef08bf00a2bff599982f6b08e8770e09cc012d7bba00783fc7ea38f1fb37d", size = 623463, upload-time = "2025-11-19T15:18:21.11Z" }, - { url = "https://files.pythonhosted.org/packages/9b/3a/13784a9364bd43b0d61eef4bea2845039bc2030458b16594a1bd787ae26e/safetensors-0.7.0-cp38-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:42cb091236206bb2016d245c377ed383aa7f78691748f3bb6ee1bfa51ae2ce6a", size = 532855, upload-time = "2025-11-19T15:18:25.719Z" }, - { url = "https://files.pythonhosted.org/packages/a0/60/429e9b1cb3fc651937727befe258ea24122d9663e4d5709a48c9cbfceecb/safetensors-0.7.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dac7252938f0696ddea46f5e855dd3138444e82236e3be475f54929f0c510d48", size = 507152, upload-time = "2025-11-19T15:18:33.023Z" }, - { url = "https://files.pythonhosted.org/packages/3c/a8/4b45e4e059270d17af60359713ffd83f97900d45a6afa73aaa0d737d48b6/safetensors-0.7.0-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1d060c70284127fa805085d8f10fbd0962792aed71879d00864acda69dbab981", size = 541856, upload-time = "2025-11-19T15:18:31.075Z" }, - { url = "https://files.pythonhosted.org/packages/06/87/d26d8407c44175d8ae164a95b5a62707fcc445f3c0c56108e37d98070a3d/safetensors-0.7.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:cdab83a366799fa730f90a4ebb563e494f28e9e92c4819e556152ad55e43591b", size = 674060, upload-time = "2025-11-19T15:18:37.211Z" }, - { url = "https://files.pythonhosted.org/packages/11/f5/57644a2ff08dc6325816ba7217e5095f17269dada2554b658442c66aed51/safetensors-0.7.0-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:672132907fcad9f2aedcb705b2d7b3b93354a2aec1b2f706c4db852abe338f85", size = 771715, upload-time = "2025-11-19T15:18:38.689Z" }, - { url = "https://files.pythonhosted.org/packages/86/31/17883e13a814bd278ae6e266b13282a01049b0c81341da7fd0e3e71a80a3/safetensors-0.7.0-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:5d72abdb8a4d56d4020713724ba81dac065fedb7f3667151c4a637f1d3fb26c0", size = 714377, upload-time = "2025-11-19T15:18:40.162Z" }, - { url = "https://files.pythonhosted.org/packages/4a/d8/0c8a7dc9b41dcac53c4cbf9df2b9c83e0e0097203de8b37a712b345c0be5/safetensors-0.7.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b0f6d66c1c538d5a94a73aa9ddca8ccc4227e6c9ff555322ea40bdd142391dd4", size = 677368, upload-time = "2025-11-19T15:18:41.627Z" }, - { url = "https://files.pythonhosted.org/packages/05/e5/cb4b713c8a93469e3c5be7c3f8d77d307e65fe89673e731f5c2bfd0a9237/safetensors-0.7.0-cp38-abi3-win32.whl", hash = "sha256:c74af94bf3ac15ac4d0f2a7c7b4663a15f8c2ab15ed0fc7531ca61d0835eccba", size = 326423, upload-time = "2025-11-19T15:18:45.74Z" }, - { url = "https://files.pythonhosted.org/packages/5d/e6/ec8471c8072382cb91233ba7267fd931219753bb43814cbc71757bfd4dab/safetensors-0.7.0-cp38-abi3-win_amd64.whl", hash = "sha256:d1239932053f56f3456f32eb9625590cc7582e905021f94636202a864d470755", size = 341380, upload-time = "2025-11-19T15:18:44.427Z" }, - { url = "https://files.pythonhosted.org/packages/a7/6a/4d08d89a6fcbe905c5ae68b8b34f0791850882fc19782d0d02c65abbdf3b/safetensors-0.7.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4729811a6640d019a4b7ba8638ee2fd21fa5ca8c7e7bdf0fed62068fcaac737", size = 492430, upload-time = "2025-11-19T15:18:11.884Z" }, - { url = "https://files.pythonhosted.org/packages/dd/29/59ed8152b30f72c42d00d241e58eaca558ae9dbfa5695206e2e0f54c7063/safetensors-0.7.0-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:12f49080303fa6bb424b362149a12949dfbbf1e06811a88f2307276b0c131afd", size = 503977, upload-time = "2025-11-19T15:18:17.523Z" }, - { url = "https://files.pythonhosted.org/packages/d3/0b/4811bfec67fa260e791369b16dab105e4bae82686120554cc484064e22b4/safetensors-0.7.0-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0071bffba4150c2f46cae1432d31995d77acfd9f8db598b5d1a2ce67e8440ad2", size = 623890, upload-time = "2025-11-19T15:18:22.666Z" }, - { url = "https://files.pythonhosted.org/packages/58/5b/632a58724221ef03d78ab65062e82a1010e1bef8e8e0b9d7c6d7b8044841/safetensors-0.7.0-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:473b32699f4200e69801bf5abf93f1a4ecd432a70984df164fc22ccf39c4a6f3", size = 531885, upload-time = "2025-11-19T15:18:27.146Z" }, -] - -[[package]] -name = "scikit-learn" -version = "1.7.2" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11' and sys_platform == 'darwin'", - "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux')", -] -dependencies = [ - { name = "joblib", marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "threadpoolctl", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/98/c2/a7855e41c9d285dfe86dc50b250978105dce513d6e459ea66a6aeb0e1e0c/scikit_learn-1.7.2.tar.gz", hash = "sha256:20e9e49ecd130598f1ca38a1d85090e1a600147b9c02fa6f15d69cb53d968fda", size = 7193136, upload-time = "2025-09-09T08:21:29.075Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ba/3e/daed796fd69cce768b8788401cc464ea90b306fb196ae1ffed0b98182859/scikit_learn-1.7.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b33579c10a3081d076ab403df4a4190da4f4432d443521674637677dc91e61f", size = 9336221, upload-time = "2025-09-09T08:20:19.328Z" }, - { url = "https://files.pythonhosted.org/packages/1c/ce/af9d99533b24c55ff4e18d9b7b4d9919bbc6cd8f22fe7a7be01519a347d5/scikit_learn-1.7.2-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:36749fb62b3d961b1ce4fedf08fa57a1986cd409eff2d783bca5d4b9b5fce51c", size = 8653834, upload-time = "2025-09-09T08:20:22.073Z" }, - { url = "https://files.pythonhosted.org/packages/58/0e/8c2a03d518fb6bd0b6b0d4b114c63d5f1db01ff0f9925d8eb10960d01c01/scikit_learn-1.7.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7a58814265dfc52b3295b1900cfb5701589d30a8bb026c7540f1e9d3499d5ec8", size = 9660938, upload-time = "2025-09-09T08:20:24.327Z" }, - { url = "https://files.pythonhosted.org/packages/2b/75/4311605069b5d220e7cf5adabb38535bd96f0079313cdbb04b291479b22a/scikit_learn-1.7.2-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a847fea807e278f821a0406ca01e387f97653e284ecbd9750e3ee7c90347f18", size = 9477818, upload-time = "2025-09-09T08:20:26.845Z" }, - { url = "https://files.pythonhosted.org/packages/7f/9b/87961813c34adbca21a6b3f6b2bea344c43b30217a6d24cc437c6147f3e8/scikit_learn-1.7.2-cp310-cp310-win_amd64.whl", hash = "sha256:ca250e6836d10e6f402436d6463d6c0e4d8e0234cfb6a9a47835bd392b852ce5", size = 8886969, upload-time = "2025-09-09T08:20:29.329Z" }, - { url = "https://files.pythonhosted.org/packages/43/83/564e141eef908a5863a54da8ca342a137f45a0bfb71d1d79704c9894c9d1/scikit_learn-1.7.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c7509693451651cd7361d30ce4e86a1347493554f172b1c72a39300fa2aea79e", size = 9331967, upload-time = "2025-09-09T08:20:32.421Z" }, - { url = "https://files.pythonhosted.org/packages/18/d6/ba863a4171ac9d7314c4d3fc251f015704a2caeee41ced89f321c049ed83/scikit_learn-1.7.2-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:0486c8f827c2e7b64837c731c8feff72c0bd2b998067a8a9cbc10643c31f0fe1", size = 8648645, upload-time = "2025-09-09T08:20:34.436Z" }, - { url = "https://files.pythonhosted.org/packages/ef/0e/97dbca66347b8cf0ea8b529e6bb9367e337ba2e8be0ef5c1a545232abfde/scikit_learn-1.7.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:89877e19a80c7b11a2891a27c21c4894fb18e2c2e077815bcade10d34287b20d", size = 9715424, upload-time = "2025-09-09T08:20:36.776Z" }, - { url = "https://files.pythonhosted.org/packages/f7/32/1f3b22e3207e1d2c883a7e09abb956362e7d1bd2f14458c7de258a26ac15/scikit_learn-1.7.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8da8bf89d4d79aaec192d2bda62f9b56ae4e5b4ef93b6a56b5de4977e375c1f1", size = 9509234, upload-time = "2025-09-09T08:20:38.957Z" }, - { url = "https://files.pythonhosted.org/packages/9f/71/34ddbd21f1da67c7a768146968b4d0220ee6831e4bcbad3e03dd3eae88b6/scikit_learn-1.7.2-cp311-cp311-win_amd64.whl", hash = "sha256:9b7ed8d58725030568523e937c43e56bc01cadb478fc43c042a9aca1dacb3ba1", size = 8894244, upload-time = "2025-09-09T08:20:41.166Z" }, - { url = "https://files.pythonhosted.org/packages/a7/aa/3996e2196075689afb9fce0410ebdb4a09099d7964d061d7213700204409/scikit_learn-1.7.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8d91a97fa2b706943822398ab943cde71858a50245e31bc71dba62aab1d60a96", size = 9259818, upload-time = "2025-09-09T08:20:43.19Z" }, - { url = "https://files.pythonhosted.org/packages/43/5d/779320063e88af9c4a7c2cf463ff11c21ac9c8bd730c4a294b0000b666c9/scikit_learn-1.7.2-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:acbc0f5fd2edd3432a22c69bed78e837c70cf896cd7993d71d51ba6708507476", size = 8636997, upload-time = "2025-09-09T08:20:45.468Z" }, - { url = "https://files.pythonhosted.org/packages/5c/d0/0c577d9325b05594fdd33aa970bf53fb673f051a45496842caee13cfd7fe/scikit_learn-1.7.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e5bf3d930aee75a65478df91ac1225ff89cd28e9ac7bd1196853a9229b6adb0b", size = 9478381, upload-time = "2025-09-09T08:20:47.982Z" }, - { url = "https://files.pythonhosted.org/packages/82/70/8bf44b933837ba8494ca0fc9a9ab60f1c13b062ad0197f60a56e2fc4c43e/scikit_learn-1.7.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b4d6e9deed1a47aca9fe2f267ab8e8fe82ee20b4526b2c0cd9e135cea10feb44", size = 9300296, upload-time = "2025-09-09T08:20:50.366Z" }, - { url = "https://files.pythonhosted.org/packages/c6/99/ed35197a158f1fdc2fe7c3680e9c70d0128f662e1fee4ed495f4b5e13db0/scikit_learn-1.7.2-cp312-cp312-win_amd64.whl", hash = "sha256:6088aa475f0785e01bcf8529f55280a3d7d298679f50c0bb70a2364a82d0b290", size = 8731256, upload-time = "2025-09-09T08:20:52.627Z" }, - { url = "https://files.pythonhosted.org/packages/ae/93/a3038cb0293037fd335f77f31fe053b89c72f17b1c8908c576c29d953e84/scikit_learn-1.7.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0b7dacaa05e5d76759fb071558a8b5130f4845166d88654a0f9bdf3eb57851b7", size = 9212382, upload-time = "2025-09-09T08:20:54.731Z" }, - { url = "https://files.pythonhosted.org/packages/40/dd/9a88879b0c1104259136146e4742026b52df8540c39fec21a6383f8292c7/scikit_learn-1.7.2-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:abebbd61ad9e1deed54cca45caea8ad5f79e1b93173dece40bb8e0c658dbe6fe", size = 8592042, upload-time = "2025-09-09T08:20:57.313Z" }, - { url = "https://files.pythonhosted.org/packages/46/af/c5e286471b7d10871b811b72ae794ac5fe2989c0a2df07f0ec723030f5f5/scikit_learn-1.7.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:502c18e39849c0ea1a5d681af1dbcf15f6cce601aebb657aabbfe84133c1907f", size = 9434180, upload-time = "2025-09-09T08:20:59.671Z" }, - { url = "https://files.pythonhosted.org/packages/f1/fd/df59faa53312d585023b2da27e866524ffb8faf87a68516c23896c718320/scikit_learn-1.7.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a4c328a71785382fe3fe676a9ecf2c86189249beff90bf85e22bdb7efaf9ae0", size = 9283660, upload-time = "2025-09-09T08:21:01.71Z" }, - { url = "https://files.pythonhosted.org/packages/a7/c7/03000262759d7b6f38c836ff9d512f438a70d8a8ddae68ee80de72dcfb63/scikit_learn-1.7.2-cp313-cp313-win_amd64.whl", hash = "sha256:63a9afd6f7b229aad94618c01c252ce9e6fa97918c5ca19c9a17a087d819440c", size = 8702057, upload-time = "2025-09-09T08:21:04.234Z" }, - { url = "https://files.pythonhosted.org/packages/55/87/ef5eb1f267084532c8e4aef98a28b6ffe7425acbfd64b5e2f2e066bc29b3/scikit_learn-1.7.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:9acb6c5e867447b4e1390930e3944a005e2cb115922e693c08a323421a6966e8", size = 9558731, upload-time = "2025-09-09T08:21:06.381Z" }, - { url = "https://files.pythonhosted.org/packages/93/f8/6c1e3fc14b10118068d7938878a9f3f4e6d7b74a8ddb1e5bed65159ccda8/scikit_learn-1.7.2-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:2a41e2a0ef45063e654152ec9d8bcfc39f7afce35b08902bfe290c2498a67a6a", size = 9038852, upload-time = "2025-09-09T08:21:08.628Z" }, - { url = "https://files.pythonhosted.org/packages/83/87/066cafc896ee540c34becf95d30375fe5cbe93c3b75a0ee9aa852cd60021/scikit_learn-1.7.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:98335fb98509b73385b3ab2bd0639b1f610541d3988ee675c670371d6a87aa7c", size = 9527094, upload-time = "2025-09-09T08:21:11.486Z" }, - { url = "https://files.pythonhosted.org/packages/9c/2b/4903e1ccafa1f6453b1ab78413938c8800633988c838aa0be386cbb33072/scikit_learn-1.7.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:191e5550980d45449126e23ed1d5e9e24b2c68329ee1f691a3987476e115e09c", size = 9367436, upload-time = "2025-09-09T08:21:13.602Z" }, - { url = "https://files.pythonhosted.org/packages/b5/aa/8444be3cfb10451617ff9d177b3c190288f4563e6c50ff02728be67ad094/scikit_learn-1.7.2-cp313-cp313t-win_amd64.whl", hash = "sha256:57dc4deb1d3762c75d685507fbd0bc17160144b2f2ba4ccea5dc285ab0d0e973", size = 9275749, upload-time = "2025-09-09T08:21:15.96Z" }, - { url = "https://files.pythonhosted.org/packages/d9/82/dee5acf66837852e8e68df6d8d3a6cb22d3df997b733b032f513d95205b7/scikit_learn-1.7.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fa8f63940e29c82d1e67a45d5297bdebbcb585f5a5a50c4914cc2e852ab77f33", size = 9208906, upload-time = "2025-09-09T08:21:18.557Z" }, - { url = "https://files.pythonhosted.org/packages/3c/30/9029e54e17b87cb7d50d51a5926429c683d5b4c1732f0507a6c3bed9bf65/scikit_learn-1.7.2-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:f95dc55b7902b91331fa4e5845dd5bde0580c9cd9612b1b2791b7e80c3d32615", size = 8627836, upload-time = "2025-09-09T08:21:20.695Z" }, - { url = "https://files.pythonhosted.org/packages/60/18/4a52c635c71b536879f4b971c2cedf32c35ee78f48367885ed8025d1f7ee/scikit_learn-1.7.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9656e4a53e54578ad10a434dc1f993330568cfee176dff07112b8785fb413106", size = 9426236, upload-time = "2025-09-09T08:21:22.645Z" }, - { url = "https://files.pythonhosted.org/packages/99/7e/290362f6ab582128c53445458a5befd471ed1ea37953d5bcf80604619250/scikit_learn-1.7.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96dc05a854add0e50d3f47a1ef21a10a595016da5b007c7d9cd9d0bffd1fcc61", size = 9312593, upload-time = "2025-09-09T08:21:24.65Z" }, - { url = "https://files.pythonhosted.org/packages/8e/87/24f541b6d62b1794939ae6422f8023703bbf6900378b2b34e0b4384dfefd/scikit_learn-1.7.2-cp314-cp314-win_amd64.whl", hash = "sha256:bb24510ed3f9f61476181e4db51ce801e2ba37541def12dc9333b946fc7a9cf8", size = 8820007, upload-time = "2025-09-09T08:21:26.713Z" }, -] - -[[package]] -name = "scikit-learn" -version = "1.8.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.14' and sys_platform == 'win32'", - "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version >= '3.14' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", - "python_full_version == '3.13.*' and sys_platform == 'win32'", - "python_full_version == '3.13.*' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version == '3.13.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", - "python_full_version == '3.12.*' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "python_full_version == '3.12.*' and sys_platform == 'win32'", - "python_full_version == '3.12.*' and sys_platform == 'emscripten'", - "(python_full_version == '3.12.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", - "python_full_version == '3.11.*' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "python_full_version == '3.11.*' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and sys_platform == 'emscripten'", - "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", -] -dependencies = [ - { name = "joblib", marker = "python_full_version >= '3.11'" }, - { name = "numpy", version = "2.4.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "threadpoolctl", marker = "python_full_version >= '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0e/d4/40988bf3b8e34feec1d0e6a051446b1f66225f8529b9309becaeef62b6c4/scikit_learn-1.8.0.tar.gz", hash = "sha256:9bccbb3b40e3de10351f8f5068e105d0f4083b1a65fa07b6634fbc401a6287fd", size = 7335585, upload-time = "2025-12-10T07:08:53.618Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c9/92/53ea2181da8ac6bf27170191028aee7251f8f841f8d3edbfdcaf2008fde9/scikit_learn-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:146b4d36f800c013d267b29168813f7a03a43ecd2895d04861f1240b564421da", size = 8595835, upload-time = "2025-12-10T07:07:39.385Z" }, - { url = "https://files.pythonhosted.org/packages/01/18/d154dc1638803adf987910cdd07097d9c526663a55666a97c124d09fb96a/scikit_learn-1.8.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:f984ca4b14914e6b4094c5d52a32ea16b49832c03bd17a110f004db3c223e8e1", size = 8080381, upload-time = "2025-12-10T07:07:41.93Z" }, - { url = "https://files.pythonhosted.org/packages/8a/44/226142fcb7b7101e64fdee5f49dbe6288d4c7af8abf593237b70fca080a4/scikit_learn-1.8.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5e30adb87f0cc81c7690a84f7932dd66be5bac57cfe16b91cb9151683a4a2d3b", size = 8799632, upload-time = "2025-12-10T07:07:43.899Z" }, - { url = "https://files.pythonhosted.org/packages/36/4d/4a67f30778a45d542bbea5db2dbfa1e9e100bf9ba64aefe34215ba9f11f6/scikit_learn-1.8.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ada8121bcb4dac28d930febc791a69f7cb1673c8495e5eee274190b73a4559c1", size = 9103788, upload-time = "2025-12-10T07:07:45.982Z" }, - { url = "https://files.pythonhosted.org/packages/89/3c/45c352094cfa60050bcbb967b1faf246b22e93cb459f2f907b600f2ceda5/scikit_learn-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:c57b1b610bd1f40ba43970e11ce62821c2e6569e4d74023db19c6b26f246cb3b", size = 8081706, upload-time = "2025-12-10T07:07:48.111Z" }, - { url = "https://files.pythonhosted.org/packages/3d/46/5416595bb395757f754feb20c3d776553a386b661658fb21b7c814e89efe/scikit_learn-1.8.0-cp311-cp311-win_arm64.whl", hash = "sha256:2838551e011a64e3053ad7618dda9310175f7515f1742fa2d756f7c874c05961", size = 7688451, upload-time = "2025-12-10T07:07:49.873Z" }, - { url = "https://files.pythonhosted.org/packages/90/74/e6a7cc4b820e95cc38cf36cd74d5aa2b42e8ffc2d21fe5a9a9c45c1c7630/scikit_learn-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5fb63362b5a7ddab88e52b6dbb47dac3fd7dafeee740dc6c8d8a446ddedade8e", size = 8548242, upload-time = "2025-12-10T07:07:51.568Z" }, - { url = "https://files.pythonhosted.org/packages/49/d8/9be608c6024d021041c7f0b3928d4749a706f4e2c3832bbede4fb4f58c95/scikit_learn-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:5025ce924beccb28298246e589c691fe1b8c1c96507e6d27d12c5fadd85bfd76", size = 8079075, upload-time = "2025-12-10T07:07:53.697Z" }, - { url = "https://files.pythonhosted.org/packages/dd/47/f187b4636ff80cc63f21cd40b7b2d177134acaa10f6bb73746130ee8c2e5/scikit_learn-1.8.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4496bb2cf7a43ce1a2d7524a79e40bc5da45cf598dbf9545b7e8316ccba47bb4", size = 8660492, upload-time = "2025-12-10T07:07:55.574Z" }, - { url = "https://files.pythonhosted.org/packages/97/74/b7a304feb2b49df9fafa9382d4d09061a96ee9a9449a7cbea7988dda0828/scikit_learn-1.8.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a0bcfe4d0d14aec44921545fd2af2338c7471de9cb701f1da4c9d85906ab847a", size = 8931904, upload-time = "2025-12-10T07:07:57.666Z" }, - { url = "https://files.pythonhosted.org/packages/9f/c4/0ab22726a04ede56f689476b760f98f8f46607caecff993017ac1b64aa5d/scikit_learn-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:35c007dedb2ffe38fe3ee7d201ebac4a2deccd2408e8621d53067733e3c74809", size = 8019359, upload-time = "2025-12-10T07:07:59.838Z" }, - { url = "https://files.pythonhosted.org/packages/24/90/344a67811cfd561d7335c1b96ca21455e7e472d281c3c279c4d3f2300236/scikit_learn-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:8c497fff237d7b4e07e9ef1a640887fa4fb765647f86fbe00f969ff6280ce2bb", size = 7641898, upload-time = "2025-12-10T07:08:01.36Z" }, - { url = "https://files.pythonhosted.org/packages/03/aa/e22e0768512ce9255eba34775be2e85c2048da73da1193e841707f8f039c/scikit_learn-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0d6ae97234d5d7079dc0040990a6f7aeb97cb7fa7e8945f1999a429b23569e0a", size = 8513770, upload-time = "2025-12-10T07:08:03.251Z" }, - { url = "https://files.pythonhosted.org/packages/58/37/31b83b2594105f61a381fc74ca19e8780ee923be2d496fcd8d2e1147bd99/scikit_learn-1.8.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:edec98c5e7c128328124a029bceb09eda2d526997780fef8d65e9a69eead963e", size = 8044458, upload-time = "2025-12-10T07:08:05.336Z" }, - { url = "https://files.pythonhosted.org/packages/2d/5a/3f1caed8765f33eabb723596666da4ebbf43d11e96550fb18bdec42b467b/scikit_learn-1.8.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:74b66d8689d52ed04c271e1329f0c61635bcaf5b926db9b12d58914cdc01fe57", size = 8610341, upload-time = "2025-12-10T07:08:07.732Z" }, - { url = "https://files.pythonhosted.org/packages/38/cf/06896db3f71c75902a8e9943b444a56e727418f6b4b4a90c98c934f51ed4/scikit_learn-1.8.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8fdf95767f989b0cfedb85f7ed8ca215d4be728031f56ff5a519ee1e3276dc2e", size = 8900022, upload-time = "2025-12-10T07:08:09.862Z" }, - { url = "https://files.pythonhosted.org/packages/1c/f9/9b7563caf3ec8873e17a31401858efab6b39a882daf6c1bfa88879c0aa11/scikit_learn-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:2de443b9373b3b615aec1bb57f9baa6bb3a9bd093f1269ba95c17d870422b271", size = 7989409, upload-time = "2025-12-10T07:08:12.028Z" }, - { url = "https://files.pythonhosted.org/packages/49/bd/1f4001503650e72c4f6009ac0c4413cb17d2d601cef6f71c0453da2732fc/scikit_learn-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:eddde82a035681427cbedded4e6eff5e57fa59216c2e3e90b10b19ab1d0a65c3", size = 7619760, upload-time = "2025-12-10T07:08:13.688Z" }, - { url = "https://files.pythonhosted.org/packages/d2/7d/a630359fc9dcc95496588c8d8e3245cc8fd81980251079bc09c70d41d951/scikit_learn-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:7cc267b6108f0a1499a734167282c00c4ebf61328566b55ef262d48e9849c735", size = 8826045, upload-time = "2025-12-10T07:08:15.215Z" }, - { url = "https://files.pythonhosted.org/packages/cc/56/a0c86f6930cfcd1c7054a2bc417e26960bb88d32444fe7f71d5c2cfae891/scikit_learn-1.8.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:fe1c011a640a9f0791146011dfd3c7d9669785f9fed2b2a5f9e207536cf5c2fd", size = 8420324, upload-time = "2025-12-10T07:08:17.561Z" }, - { url = "https://files.pythonhosted.org/packages/46/1e/05962ea1cebc1cf3876667ecb14c283ef755bf409993c5946ade3b77e303/scikit_learn-1.8.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72358cce49465d140cc4e7792015bb1f0296a9742d5622c67e31399b75468b9e", size = 8680651, upload-time = "2025-12-10T07:08:19.952Z" }, - { url = "https://files.pythonhosted.org/packages/fe/56/a85473cd75f200c9759e3a5f0bcab2d116c92a8a02ee08ccd73b870f8bb4/scikit_learn-1.8.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:80832434a6cc114f5219211eec13dcbc16c2bac0e31ef64c6d346cde3cf054cb", size = 8925045, upload-time = "2025-12-10T07:08:22.11Z" }, - { url = "https://files.pythonhosted.org/packages/cc/b7/64d8cfa896c64435ae57f4917a548d7ac7a44762ff9802f75a79b77cb633/scikit_learn-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:ee787491dbfe082d9c3013f01f5991658b0f38aa8177e4cd4bf434c58f551702", size = 8507994, upload-time = "2025-12-10T07:08:23.943Z" }, - { url = "https://files.pythonhosted.org/packages/5e/37/e192ea709551799379958b4c4771ec507347027bb7c942662c7fbeba31cb/scikit_learn-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf97c10a3f5a7543f9b88cbf488d33d175e9146115a451ae34568597ba33dcde", size = 7869518, upload-time = "2025-12-10T07:08:25.71Z" }, - { url = "https://files.pythonhosted.org/packages/24/05/1af2c186174cc92dcab2233f327336058c077d38f6fe2aceb08e6ab4d509/scikit_learn-1.8.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c22a2da7a198c28dd1a6e1136f19c830beab7fdca5b3e5c8bba8394f8a5c45b3", size = 8528667, upload-time = "2025-12-10T07:08:27.541Z" }, - { url = "https://files.pythonhosted.org/packages/a8/25/01c0af38fe969473fb292bba9dc2b8f9b451f3112ff242c647fee3d0dfe7/scikit_learn-1.8.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:6b595b07a03069a2b1740dc08c2299993850ea81cce4fe19b2421e0c970de6b7", size = 8066524, upload-time = "2025-12-10T07:08:29.822Z" }, - { url = "https://files.pythonhosted.org/packages/be/ce/a0623350aa0b68647333940ee46fe45086c6060ec604874e38e9ab7d8e6c/scikit_learn-1.8.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:29ffc74089f3d5e87dfca4c2c8450f88bdc61b0fc6ed5d267f3988f19a1309f6", size = 8657133, upload-time = "2025-12-10T07:08:31.865Z" }, - { url = "https://files.pythonhosted.org/packages/b8/cb/861b41341d6f1245e6ca80b1c1a8c4dfce43255b03df034429089ca2a2c5/scikit_learn-1.8.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fb65db5d7531bccf3a4f6bec3462223bea71384e2cda41da0f10b7c292b9e7c4", size = 8923223, upload-time = "2025-12-10T07:08:34.166Z" }, - { url = "https://files.pythonhosted.org/packages/76/18/a8def8f91b18cd1ba6e05dbe02540168cb24d47e8dcf69e8d00b7da42a08/scikit_learn-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:56079a99c20d230e873ea40753102102734c5953366972a71d5cb39a32bc40c6", size = 8096518, upload-time = "2025-12-10T07:08:36.339Z" }, - { url = "https://files.pythonhosted.org/packages/d1/77/482076a678458307f0deb44e29891d6022617b2a64c840c725495bee343f/scikit_learn-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:3bad7565bc9cf37ce19a7c0d107742b320c1285df7aab1a6e2d28780df167242", size = 7754546, upload-time = "2025-12-10T07:08:38.128Z" }, - { url = "https://files.pythonhosted.org/packages/2d/d1/ef294ca754826daa043b2a104e59960abfab4cf653891037d19dd5b6f3cf/scikit_learn-1.8.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:4511be56637e46c25721e83d1a9cea9614e7badc7040c4d573d75fbe257d6fd7", size = 8848305, upload-time = "2025-12-10T07:08:41.013Z" }, - { url = "https://files.pythonhosted.org/packages/5b/e2/b1f8b05138ee813b8e1a4149f2f0d289547e60851fd1bb268886915adbda/scikit_learn-1.8.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:a69525355a641bf8ef136a7fa447672fb54fe8d60cab5538d9eb7c6438543fb9", size = 8432257, upload-time = "2025-12-10T07:08:42.873Z" }, - { url = "https://files.pythonhosted.org/packages/26/11/c32b2138a85dcb0c99f6afd13a70a951bfdff8a6ab42d8160522542fb647/scikit_learn-1.8.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c2656924ec73e5939c76ac4c8b026fc203b83d8900362eb2599d8aee80e4880f", size = 8678673, upload-time = "2025-12-10T07:08:45.362Z" }, - { url = "https://files.pythonhosted.org/packages/c7/57/51f2384575bdec454f4fe4e7a919d696c9ebce914590abf3e52d47607ab8/scikit_learn-1.8.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:15fc3b5d19cc2be65404786857f2e13c70c83dd4782676dd6814e3b89dc8f5b9", size = 8922467, upload-time = "2025-12-10T07:08:47.408Z" }, - { url = "https://files.pythonhosted.org/packages/35/4d/748c9e2872637a57981a04adc038dacaa16ba8ca887b23e34953f0b3f742/scikit_learn-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:00d6f1d66fbcf4eba6e356e1420d33cc06c70a45bb1363cd6f6a8e4ebbbdece2", size = 8774395, upload-time = "2025-12-10T07:08:49.337Z" }, - { url = "https://files.pythonhosted.org/packages/60/22/d7b2ebe4704a5e50790ba089d5c2ae308ab6bb852719e6c3bd4f04c3a363/scikit_learn-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:f28dd15c6bb0b66ba09728cf09fd8736c304be29409bd8445a080c1280619e8c", size = 8002647, upload-time = "2025-12-10T07:08:51.601Z" }, -] - -[[package]] -name = "scipy" -version = "1.15.3" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11' and sys_platform == 'darwin'", - "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux')", -] -dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0f/37/6964b830433e654ec7485e45a00fc9a27cf868d622838f6b6d9c5ec0d532/scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf", size = 59419214, upload-time = "2025-05-08T16:13:05.955Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/78/2f/4966032c5f8cc7e6a60f1b2e0ad686293b9474b65246b0c642e3ef3badd0/scipy-1.15.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:a345928c86d535060c9c2b25e71e87c39ab2f22fc96e9636bd74d1dbf9de448c", size = 38702770, upload-time = "2025-05-08T16:04:20.849Z" }, - { url = "https://files.pythonhosted.org/packages/a0/6e/0c3bf90fae0e910c274db43304ebe25a6b391327f3f10b5dcc638c090795/scipy-1.15.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:ad3432cb0f9ed87477a8d97f03b763fd1d57709f1bbde3c9369b1dff5503b253", size = 30094511, upload-time = "2025-05-08T16:04:27.103Z" }, - { url = "https://files.pythonhosted.org/packages/ea/b1/4deb37252311c1acff7f101f6453f0440794f51b6eacb1aad4459a134081/scipy-1.15.3-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:aef683a9ae6eb00728a542b796f52a5477b78252edede72b8327a886ab63293f", size = 22368151, upload-time = "2025-05-08T16:04:31.731Z" }, - { url = "https://files.pythonhosted.org/packages/38/7d/f457626e3cd3c29b3a49ca115a304cebb8cc6f31b04678f03b216899d3c6/scipy-1.15.3-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:1c832e1bd78dea67d5c16f786681b28dd695a8cb1fb90af2e27580d3d0967e92", size = 25121732, upload-time = "2025-05-08T16:04:36.596Z" }, - { url = "https://files.pythonhosted.org/packages/db/0a/92b1de4a7adc7a15dcf5bddc6e191f6f29ee663b30511ce20467ef9b82e4/scipy-1.15.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:263961f658ce2165bbd7b99fa5135195c3a12d9bef045345016b8b50c315cb82", size = 35547617, upload-time = "2025-05-08T16:04:43.546Z" }, - { url = "https://files.pythonhosted.org/packages/8e/6d/41991e503e51fc1134502694c5fa7a1671501a17ffa12716a4a9151af3df/scipy-1.15.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e2abc762b0811e09a0d3258abee2d98e0c703eee49464ce0069590846f31d40", size = 37662964, upload-time = "2025-05-08T16:04:49.431Z" }, - { url = "https://files.pythonhosted.org/packages/25/e1/3df8f83cb15f3500478c889be8fb18700813b95e9e087328230b98d547ff/scipy-1.15.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ed7284b21a7a0c8f1b6e5977ac05396c0d008b89e05498c8b7e8f4a1423bba0e", size = 37238749, upload-time = "2025-05-08T16:04:55.215Z" }, - { url = "https://files.pythonhosted.org/packages/93/3e/b3257cf446f2a3533ed7809757039016b74cd6f38271de91682aa844cfc5/scipy-1.15.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5380741e53df2c566f4d234b100a484b420af85deb39ea35a1cc1be84ff53a5c", size = 40022383, upload-time = "2025-05-08T16:05:01.914Z" }, - { url = "https://files.pythonhosted.org/packages/d1/84/55bc4881973d3f79b479a5a2e2df61c8c9a04fcb986a213ac9c02cfb659b/scipy-1.15.3-cp310-cp310-win_amd64.whl", hash = "sha256:9d61e97b186a57350f6d6fd72640f9e99d5a4a2b8fbf4b9ee9a841eab327dc13", size = 41259201, upload-time = "2025-05-08T16:05:08.166Z" }, - { url = "https://files.pythonhosted.org/packages/96/ab/5cc9f80f28f6a7dff646c5756e559823614a42b1939d86dd0ed550470210/scipy-1.15.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:993439ce220d25e3696d1b23b233dd010169b62f6456488567e830654ee37a6b", size = 38714255, upload-time = "2025-05-08T16:05:14.596Z" }, - { url = "https://files.pythonhosted.org/packages/4a/4a/66ba30abe5ad1a3ad15bfb0b59d22174012e8056ff448cb1644deccbfed2/scipy-1.15.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:34716e281f181a02341ddeaad584205bd2fd3c242063bd3423d61ac259ca7eba", size = 30111035, upload-time = "2025-05-08T16:05:20.152Z" }, - { url = "https://files.pythonhosted.org/packages/4b/fa/a7e5b95afd80d24313307f03624acc65801846fa75599034f8ceb9e2cbf6/scipy-1.15.3-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3b0334816afb8b91dab859281b1b9786934392aa3d527cd847e41bb6f45bee65", size = 22384499, upload-time = "2025-05-08T16:05:24.494Z" }, - { url = "https://files.pythonhosted.org/packages/17/99/f3aaddccf3588bb4aea70ba35328c204cadd89517a1612ecfda5b2dd9d7a/scipy-1.15.3-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:6db907c7368e3092e24919b5e31c76998b0ce1684d51a90943cb0ed1b4ffd6c1", size = 25152602, upload-time = "2025-05-08T16:05:29.313Z" }, - { url = "https://files.pythonhosted.org/packages/56/c5/1032cdb565f146109212153339f9cb8b993701e9fe56b1c97699eee12586/scipy-1.15.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:721d6b4ef5dc82ca8968c25b111e307083d7ca9091bc38163fb89243e85e3889", size = 35503415, upload-time = "2025-05-08T16:05:34.699Z" }, - { url = "https://files.pythonhosted.org/packages/bd/37/89f19c8c05505d0601ed5650156e50eb881ae3918786c8fd7262b4ee66d3/scipy-1.15.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39cb9c62e471b1bb3750066ecc3a3f3052b37751c7c3dfd0fd7e48900ed52982", size = 37652622, upload-time = "2025-05-08T16:05:40.762Z" }, - { url = "https://files.pythonhosted.org/packages/7e/31/be59513aa9695519b18e1851bb9e487de66f2d31f835201f1b42f5d4d475/scipy-1.15.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:795c46999bae845966368a3c013e0e00947932d68e235702b5c3f6ea799aa8c9", size = 37244796, upload-time = "2025-05-08T16:05:48.119Z" }, - { url = "https://files.pythonhosted.org/packages/10/c0/4f5f3eeccc235632aab79b27a74a9130c6c35df358129f7ac8b29f562ac7/scipy-1.15.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:18aaacb735ab38b38db42cb01f6b92a2d0d4b6aabefeb07f02849e47f8fb3594", size = 40047684, upload-time = "2025-05-08T16:05:54.22Z" }, - { url = "https://files.pythonhosted.org/packages/ab/a7/0ddaf514ce8a8714f6ed243a2b391b41dbb65251affe21ee3077ec45ea9a/scipy-1.15.3-cp311-cp311-win_amd64.whl", hash = "sha256:ae48a786a28412d744c62fd7816a4118ef97e5be0bee968ce8f0a2fba7acf3bb", size = 41246504, upload-time = "2025-05-08T16:06:00.437Z" }, - { url = "https://files.pythonhosted.org/packages/37/4b/683aa044c4162e10ed7a7ea30527f2cbd92e6999c10a8ed8edb253836e9c/scipy-1.15.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6ac6310fdbfb7aa6612408bd2f07295bcbd3fda00d2d702178434751fe48e019", size = 38766735, upload-time = "2025-05-08T16:06:06.471Z" }, - { url = "https://files.pythonhosted.org/packages/7b/7e/f30be3d03de07f25dc0ec926d1681fed5c732d759ac8f51079708c79e680/scipy-1.15.3-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:185cd3d6d05ca4b44a8f1595af87f9c372bb6acf9c808e99aa3e9aa03bd98cf6", size = 30173284, upload-time = "2025-05-08T16:06:11.686Z" }, - { url = "https://files.pythonhosted.org/packages/07/9c/0ddb0d0abdabe0d181c1793db51f02cd59e4901da6f9f7848e1f96759f0d/scipy-1.15.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:05dc6abcd105e1a29f95eada46d4a3f251743cfd7d3ae8ddb4088047f24ea477", size = 22446958, upload-time = "2025-05-08T16:06:15.97Z" }, - { url = "https://files.pythonhosted.org/packages/af/43/0bce905a965f36c58ff80d8bea33f1f9351b05fad4beaad4eae34699b7a1/scipy-1.15.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:06efcba926324df1696931a57a176c80848ccd67ce6ad020c810736bfd58eb1c", size = 25242454, upload-time = "2025-05-08T16:06:20.394Z" }, - { url = "https://files.pythonhosted.org/packages/56/30/a6f08f84ee5b7b28b4c597aca4cbe545535c39fe911845a96414700b64ba/scipy-1.15.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c05045d8b9bfd807ee1b9f38761993297b10b245f012b11b13b91ba8945f7e45", size = 35210199, upload-time = "2025-05-08T16:06:26.159Z" }, - { url = "https://files.pythonhosted.org/packages/0b/1f/03f52c282437a168ee2c7c14a1a0d0781a9a4a8962d84ac05c06b4c5b555/scipy-1.15.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:271e3713e645149ea5ea3e97b57fdab61ce61333f97cfae392c28ba786f9bb49", size = 37309455, upload-time = "2025-05-08T16:06:32.778Z" }, - { url = "https://files.pythonhosted.org/packages/89/b1/fbb53137f42c4bf630b1ffdfc2151a62d1d1b903b249f030d2b1c0280af8/scipy-1.15.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6cfd56fc1a8e53f6e89ba3a7a7251f7396412d655bca2aa5611c8ec9a6784a1e", size = 36885140, upload-time = "2025-05-08T16:06:39.249Z" }, - { url = "https://files.pythonhosted.org/packages/2e/2e/025e39e339f5090df1ff266d021892694dbb7e63568edcfe43f892fa381d/scipy-1.15.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0ff17c0bb1cb32952c09217d8d1eed9b53d1463e5f1dd6052c7857f83127d539", size = 39710549, upload-time = "2025-05-08T16:06:45.729Z" }, - { url = "https://files.pythonhosted.org/packages/e6/eb/3bf6ea8ab7f1503dca3a10df2e4b9c3f6b3316df07f6c0ded94b281c7101/scipy-1.15.3-cp312-cp312-win_amd64.whl", hash = "sha256:52092bc0472cfd17df49ff17e70624345efece4e1a12b23783a1ac59a1b728ed", size = 40966184, upload-time = "2025-05-08T16:06:52.623Z" }, - { url = "https://files.pythonhosted.org/packages/73/18/ec27848c9baae6e0d6573eda6e01a602e5649ee72c27c3a8aad673ebecfd/scipy-1.15.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2c620736bcc334782e24d173c0fdbb7590a0a436d2fdf39310a8902505008759", size = 38728256, upload-time = "2025-05-08T16:06:58.696Z" }, - { url = "https://files.pythonhosted.org/packages/74/cd/1aef2184948728b4b6e21267d53b3339762c285a46a274ebb7863c9e4742/scipy-1.15.3-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:7e11270a000969409d37ed399585ee530b9ef6aa99d50c019de4cb01e8e54e62", size = 30109540, upload-time = "2025-05-08T16:07:04.209Z" }, - { url = "https://files.pythonhosted.org/packages/5b/d8/59e452c0a255ec352bd0a833537a3bc1bfb679944c4938ab375b0a6b3a3e/scipy-1.15.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:8c9ed3ba2c8a2ce098163a9bdb26f891746d02136995df25227a20e71c396ebb", size = 22383115, upload-time = "2025-05-08T16:07:08.998Z" }, - { url = "https://files.pythonhosted.org/packages/08/f5/456f56bbbfccf696263b47095291040655e3cbaf05d063bdc7c7517f32ac/scipy-1.15.3-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:0bdd905264c0c9cfa74a4772cdb2070171790381a5c4d312c973382fc6eaf730", size = 25163884, upload-time = "2025-05-08T16:07:14.091Z" }, - { url = "https://files.pythonhosted.org/packages/a2/66/a9618b6a435a0f0c0b8a6d0a2efb32d4ec5a85f023c2b79d39512040355b/scipy-1.15.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79167bba085c31f38603e11a267d862957cbb3ce018d8b38f79ac043bc92d825", size = 35174018, upload-time = "2025-05-08T16:07:19.427Z" }, - { url = "https://files.pythonhosted.org/packages/b5/09/c5b6734a50ad4882432b6bb7c02baf757f5b2f256041da5df242e2d7e6b6/scipy-1.15.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c9deabd6d547aee2c9a81dee6cc96c6d7e9a9b1953f74850c179f91fdc729cb7", size = 37269716, upload-time = "2025-05-08T16:07:25.712Z" }, - { url = "https://files.pythonhosted.org/packages/77/0a/eac00ff741f23bcabd352731ed9b8995a0a60ef57f5fd788d611d43d69a1/scipy-1.15.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:dde4fc32993071ac0c7dd2d82569e544f0bdaff66269cb475e0f369adad13f11", size = 36872342, upload-time = "2025-05-08T16:07:31.468Z" }, - { url = "https://files.pythonhosted.org/packages/fe/54/4379be86dd74b6ad81551689107360d9a3e18f24d20767a2d5b9253a3f0a/scipy-1.15.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f77f853d584e72e874d87357ad70f44b437331507d1c311457bed8ed2b956126", size = 39670869, upload-time = "2025-05-08T16:07:38.002Z" }, - { url = "https://files.pythonhosted.org/packages/87/2e/892ad2862ba54f084ffe8cc4a22667eaf9c2bcec6d2bff1d15713c6c0703/scipy-1.15.3-cp313-cp313-win_amd64.whl", hash = "sha256:b90ab29d0c37ec9bf55424c064312930ca5f4bde15ee8619ee44e69319aab163", size = 40988851, upload-time = "2025-05-08T16:08:33.671Z" }, - { url = "https://files.pythonhosted.org/packages/1b/e9/7a879c137f7e55b30d75d90ce3eb468197646bc7b443ac036ae3fe109055/scipy-1.15.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3ac07623267feb3ae308487c260ac684b32ea35fd81e12845039952f558047b8", size = 38863011, upload-time = "2025-05-08T16:07:44.039Z" }, - { url = "https://files.pythonhosted.org/packages/51/d1/226a806bbd69f62ce5ef5f3ffadc35286e9fbc802f606a07eb83bf2359de/scipy-1.15.3-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:6487aa99c2a3d509a5227d9a5e889ff05830a06b2ce08ec30df6d79db5fcd5c5", size = 30266407, upload-time = "2025-05-08T16:07:49.891Z" }, - { url = "https://files.pythonhosted.org/packages/e5/9b/f32d1d6093ab9eeabbd839b0f7619c62e46cc4b7b6dbf05b6e615bbd4400/scipy-1.15.3-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:50f9e62461c95d933d5c5ef4a1f2ebf9a2b4e83b0db374cb3f1de104d935922e", size = 22540030, upload-time = "2025-05-08T16:07:54.121Z" }, - { url = "https://files.pythonhosted.org/packages/e7/29/c278f699b095c1a884f29fda126340fcc201461ee8bfea5c8bdb1c7c958b/scipy-1.15.3-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:14ed70039d182f411ffc74789a16df3835e05dc469b898233a245cdfd7f162cb", size = 25218709, upload-time = "2025-05-08T16:07:58.506Z" }, - { url = "https://files.pythonhosted.org/packages/24/18/9e5374b617aba742a990581373cd6b68a2945d65cc588482749ef2e64467/scipy-1.15.3-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a769105537aa07a69468a0eefcd121be52006db61cdd8cac8a0e68980bbb723", size = 34809045, upload-time = "2025-05-08T16:08:03.929Z" }, - { url = "https://files.pythonhosted.org/packages/e1/fe/9c4361e7ba2927074360856db6135ef4904d505e9b3afbbcb073c4008328/scipy-1.15.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9db984639887e3dffb3928d118145ffe40eff2fa40cb241a306ec57c219ebbbb", size = 36703062, upload-time = "2025-05-08T16:08:09.558Z" }, - { url = "https://files.pythonhosted.org/packages/b7/8e/038ccfe29d272b30086b25a4960f757f97122cb2ec42e62b460d02fe98e9/scipy-1.15.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:40e54d5c7e7ebf1aa596c374c49fa3135f04648a0caabcb66c52884b943f02b4", size = 36393132, upload-time = "2025-05-08T16:08:15.34Z" }, - { url = "https://files.pythonhosted.org/packages/10/7e/5c12285452970be5bdbe8352c619250b97ebf7917d7a9a9e96b8a8140f17/scipy-1.15.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5e721fed53187e71d0ccf382b6bf977644c533e506c4d33c3fb24de89f5c3ed5", size = 38979503, upload-time = "2025-05-08T16:08:21.513Z" }, - { url = "https://files.pythonhosted.org/packages/81/06/0a5e5349474e1cbc5757975b21bd4fad0e72ebf138c5592f191646154e06/scipy-1.15.3-cp313-cp313t-win_amd64.whl", hash = "sha256:76ad1fb5f8752eabf0fa02e4cc0336b4e8f021e2d5f061ed37d6d264db35e3ca", size = 40308097, upload-time = "2025-05-08T16:08:27.627Z" }, -] - -[[package]] -name = "scipy" -version = "1.17.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.14' and sys_platform == 'win32'", - "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version >= '3.14' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", - "python_full_version == '3.13.*' and sys_platform == 'win32'", - "python_full_version == '3.13.*' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version == '3.13.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", - "python_full_version == '3.12.*' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "python_full_version == '3.12.*' and sys_platform == 'win32'", - "python_full_version == '3.12.*' and sys_platform == 'emscripten'", - "(python_full_version == '3.12.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", - "python_full_version == '3.11.*' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "python_full_version == '3.11.*' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and sys_platform == 'emscripten'", - "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", -] -dependencies = [ - { name = "numpy", version = "2.4.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/df/75/b4ce781849931fef6fd529afa6b63711d5a733065722d0c3e2724af9e40a/scipy-1.17.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:1f95b894f13729334fb990162e911c9e5dc1ab390c58aa6cbecb389c5b5e28ec", size = 31613675, upload-time = "2026-02-23T00:16:00.13Z" }, - { url = "https://files.pythonhosted.org/packages/f7/58/bccc2861b305abdd1b8663d6130c0b3d7cc22e8d86663edbc8401bfd40d4/scipy-1.17.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:e18f12c6b0bc5a592ed23d3f7b891f68fd7f8241d69b7883769eb5d5dfb52696", size = 28162057, upload-time = "2026-02-23T00:16:09.456Z" }, - { url = "https://files.pythonhosted.org/packages/6d/ee/18146b7757ed4976276b9c9819108adbc73c5aad636e5353e20746b73069/scipy-1.17.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:a3472cfbca0a54177d0faa68f697d8ba4c80bbdc19908c3465556d9f7efce9ee", size = 20334032, upload-time = "2026-02-23T00:16:17.358Z" }, - { url = "https://files.pythonhosted.org/packages/ec/e6/cef1cf3557f0c54954198554a10016b6a03b2ec9e22a4e1df734936bd99c/scipy-1.17.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:766e0dc5a616d026a3a1cffa379af959671729083882f50307e18175797b3dfd", size = 22709533, upload-time = "2026-02-23T00:16:25.791Z" }, - { url = "https://files.pythonhosted.org/packages/4d/60/8804678875fc59362b0fb759ab3ecce1f09c10a735680318ac30da8cd76b/scipy-1.17.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:744b2bf3640d907b79f3fd7874efe432d1cf171ee721243e350f55234b4cec4c", size = 33062057, upload-time = "2026-02-23T00:16:36.931Z" }, - { url = "https://files.pythonhosted.org/packages/09/7d/af933f0f6e0767995b4e2d705a0665e454d1c19402aa7e895de3951ebb04/scipy-1.17.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43af8d1f3bea642559019edfe64e9b11192a8978efbd1539d7bc2aaa23d92de4", size = 35349300, upload-time = "2026-02-23T00:16:49.108Z" }, - { url = "https://files.pythonhosted.org/packages/b4/3d/7ccbbdcbb54c8fdc20d3b6930137c782a163fa626f0aef920349873421ba/scipy-1.17.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cd96a1898c0a47be4520327e01f874acfd61fb48a9420f8aa9f6483412ffa444", size = 35127333, upload-time = "2026-02-23T00:17:01.293Z" }, - { url = "https://files.pythonhosted.org/packages/e8/19/f926cb11c42b15ba08e3a71e376d816ac08614f769b4f47e06c3580c836a/scipy-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4eb6c25dd62ee8d5edf68a8e1c171dd71c292fdae95d8aeb3dd7d7de4c364082", size = 37741314, upload-time = "2026-02-23T00:17:12.576Z" }, - { url = "https://files.pythonhosted.org/packages/95/da/0d1df507cf574b3f224ccc3d45244c9a1d732c81dcb26b1e8a766ae271a8/scipy-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:d30e57c72013c2a4fe441c2fcb8e77b14e152ad48b5464858e07e2ad9fbfceff", size = 36607512, upload-time = "2026-02-23T00:17:23.424Z" }, - { url = "https://files.pythonhosted.org/packages/68/7f/bdd79ceaad24b671543ffe0ef61ed8e659440eb683b66f033454dcee90eb/scipy-1.17.1-cp311-cp311-win_arm64.whl", hash = "sha256:9ecb4efb1cd6e8c4afea0daa91a87fbddbce1b99d2895d151596716c0b2e859d", size = 24599248, upload-time = "2026-02-23T00:17:34.561Z" }, - { url = "https://files.pythonhosted.org/packages/35/48/b992b488d6f299dbe3f11a20b24d3dda3d46f1a635ede1c46b5b17a7b163/scipy-1.17.1-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:35c3a56d2ef83efc372eaec584314bd0ef2e2f0d2adb21c55e6ad5b344c0dcb8", size = 31610954, upload-time = "2026-02-23T00:17:49.855Z" }, - { url = "https://files.pythonhosted.org/packages/b2/02/cf107b01494c19dc100f1d0b7ac3cc08666e96ba2d64db7626066cee895e/scipy-1.17.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:fcb310ddb270a06114bb64bbe53c94926b943f5b7f0842194d585c65eb4edd76", size = 28172662, upload-time = "2026-02-23T00:18:01.64Z" }, - { url = "https://files.pythonhosted.org/packages/cf/a9/599c28631bad314d219cf9ffd40e985b24d603fc8a2f4ccc5ae8419a535b/scipy-1.17.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:cc90d2e9c7e5c7f1a482c9875007c095c3194b1cfedca3c2f3291cdc2bc7c086", size = 20344366, upload-time = "2026-02-23T00:18:12.015Z" }, - { url = "https://files.pythonhosted.org/packages/35/f5/906eda513271c8deb5af284e5ef0206d17a96239af79f9fa0aebfe0e36b4/scipy-1.17.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:c80be5ede8f3f8eded4eff73cc99a25c388ce98e555b17d31da05287015ffa5b", size = 22704017, upload-time = "2026-02-23T00:18:21.502Z" }, - { url = "https://files.pythonhosted.org/packages/da/34/16f10e3042d2f1d6b66e0428308ab52224b6a23049cb2f5c1756f713815f/scipy-1.17.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e19ebea31758fac5893a2ac360fedd00116cbb7628e650842a6691ba7ca28a21", size = 32927842, upload-time = "2026-02-23T00:18:35.367Z" }, - { url = "https://files.pythonhosted.org/packages/01/8e/1e35281b8ab6d5d72ebe9911edcdffa3f36b04ed9d51dec6dd140396e220/scipy-1.17.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02ae3b274fde71c5e92ac4d54bc06c42d80e399fec704383dcd99b301df37458", size = 35235890, upload-time = "2026-02-23T00:18:49.188Z" }, - { url = "https://files.pythonhosted.org/packages/c5/5c/9d7f4c88bea6e0d5a4f1bc0506a53a00e9fcb198de372bfe4d3652cef482/scipy-1.17.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8a604bae87c6195d8b1045eddece0514d041604b14f2727bbc2b3020172045eb", size = 35003557, upload-time = "2026-02-23T00:18:54.74Z" }, - { url = "https://files.pythonhosted.org/packages/65/94/7698add8f276dbab7a9de9fb6b0e02fc13ee61d51c7c3f85ac28b65e1239/scipy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f590cd684941912d10becc07325a3eeb77886fe981415660d9265c4c418d0bea", size = 37625856, upload-time = "2026-02-23T00:19:00.307Z" }, - { url = "https://files.pythonhosted.org/packages/a2/84/dc08d77fbf3d87d3ee27f6a0c6dcce1de5829a64f2eae85a0ecc1f0daa73/scipy-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:41b71f4a3a4cab9d366cd9065b288efc4d4f3c0b37a91a8e0947fb5bd7f31d87", size = 36549682, upload-time = "2026-02-23T00:19:07.67Z" }, - { url = "https://files.pythonhosted.org/packages/bc/98/fe9ae9ffb3b54b62559f52dedaebe204b408db8109a8c66fdd04869e6424/scipy-1.17.1-cp312-cp312-win_arm64.whl", hash = "sha256:f4115102802df98b2b0db3cce5cb9b92572633a1197c77b7553e5203f284a5b3", size = 24547340, upload-time = "2026-02-23T00:19:12.024Z" }, - { url = "https://files.pythonhosted.org/packages/76/27/07ee1b57b65e92645f219b37148a7e7928b82e2b5dbeccecb4dff7c64f0b/scipy-1.17.1-cp313-cp313-macosx_10_14_x86_64.whl", hash = "sha256:5e3c5c011904115f88a39308379c17f91546f77c1667cea98739fe0fccea804c", size = 31590199, upload-time = "2026-02-23T00:19:17.192Z" }, - { url = "https://files.pythonhosted.org/packages/ec/ae/db19f8ab842e9b724bf5dbb7db29302a91f1e55bc4d04b1025d6d605a2c5/scipy-1.17.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:6fac755ca3d2c3edcb22f479fceaa241704111414831ddd3bc6056e18516892f", size = 28154001, upload-time = "2026-02-23T00:19:22.241Z" }, - { url = "https://files.pythonhosted.org/packages/5b/58/3ce96251560107b381cbd6e8413c483bbb1228a6b919fa8652b0d4090e7f/scipy-1.17.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:7ff200bf9d24f2e4d5dc6ee8c3ac64d739d3a89e2326ba68aaf6c4a2b838fd7d", size = 20325719, upload-time = "2026-02-23T00:19:26.329Z" }, - { url = "https://files.pythonhosted.org/packages/b2/83/15087d945e0e4d48ce2377498abf5ad171ae013232ae31d06f336e64c999/scipy-1.17.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4b400bdc6f79fa02a4d86640310dde87a21fba0c979efff5248908c6f15fad1b", size = 22683595, upload-time = "2026-02-23T00:19:30.304Z" }, - { url = "https://files.pythonhosted.org/packages/b4/e0/e58fbde4a1a594c8be8114eb4aac1a55bcd6587047efc18a61eb1f5c0d30/scipy-1.17.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2b64ca7d4aee0102a97f3ba22124052b4bd2152522355073580bf4845e2550b6", size = 32896429, upload-time = "2026-02-23T00:19:35.536Z" }, - { url = "https://files.pythonhosted.org/packages/f5/5f/f17563f28ff03c7b6799c50d01d5d856a1d55f2676f537ca8d28c7f627cd/scipy-1.17.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:581b2264fc0aa555f3f435a5944da7504ea3a065d7029ad60e7c3d1ae09c5464", size = 35203952, upload-time = "2026-02-23T00:19:42.259Z" }, - { url = "https://files.pythonhosted.org/packages/8d/a5/9afd17de24f657fdfe4df9a3f1ea049b39aef7c06000c13db1530d81ccca/scipy-1.17.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:beeda3d4ae615106d7094f7e7cef6218392e4465cc95d25f900bebabfded0950", size = 34979063, upload-time = "2026-02-23T00:19:47.547Z" }, - { url = "https://files.pythonhosted.org/packages/8b/13/88b1d2384b424bf7c924f2038c1c409f8d88bb2a8d49d097861dd64a57b2/scipy-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6609bc224e9568f65064cfa72edc0f24ee6655b47575954ec6339534b2798369", size = 37598449, upload-time = "2026-02-23T00:19:53.238Z" }, - { url = "https://files.pythonhosted.org/packages/35/e5/d6d0e51fc888f692a35134336866341c08655d92614f492c6860dc45bb2c/scipy-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:37425bc9175607b0268f493d79a292c39f9d001a357bebb6b88fdfaff13f6448", size = 36510943, upload-time = "2026-02-23T00:20:50.89Z" }, - { url = "https://files.pythonhosted.org/packages/2a/fd/3be73c564e2a01e690e19cc618811540ba5354c67c8680dce3281123fb79/scipy-1.17.1-cp313-cp313-win_arm64.whl", hash = "sha256:5cf36e801231b6a2059bf354720274b7558746f3b1a4efb43fcf557ccd484a87", size = 24545621, upload-time = "2026-02-23T00:20:55.871Z" }, - { url = "https://files.pythonhosted.org/packages/6f/6b/17787db8b8114933a66f9dcc479a8272e4b4da75fe03b0c282f7b0ade8cd/scipy-1.17.1-cp313-cp313t-macosx_10_14_x86_64.whl", hash = "sha256:d59c30000a16d8edc7e64152e30220bfbd724c9bbb08368c054e24c651314f0a", size = 31936708, upload-time = "2026-02-23T00:19:58.694Z" }, - { url = "https://files.pythonhosted.org/packages/38/2e/524405c2b6392765ab1e2b722a41d5da33dc5c7b7278184a8ad29b6cb206/scipy-1.17.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:010f4333c96c9bb1a4516269e33cb5917b08ef2166d5556ca2fd9f082a9e6ea0", size = 28570135, upload-time = "2026-02-23T00:20:03.934Z" }, - { url = "https://files.pythonhosted.org/packages/fd/c3/5bd7199f4ea8556c0c8e39f04ccb014ac37d1468e6cfa6a95c6b3562b76e/scipy-1.17.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:2ceb2d3e01c5f1d83c4189737a42d9cb2fc38a6eeed225e7515eef71ad301dce", size = 20741977, upload-time = "2026-02-23T00:20:07.935Z" }, - { url = "https://files.pythonhosted.org/packages/d9/b8/8ccd9b766ad14c78386599708eb745f6b44f08400a5fd0ade7cf89b6fc93/scipy-1.17.1-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:844e165636711ef41f80b4103ed234181646b98a53c8f05da12ca5ca289134f6", size = 23029601, upload-time = "2026-02-23T00:20:12.161Z" }, - { url = "https://files.pythonhosted.org/packages/6d/a0/3cb6f4d2fb3e17428ad2880333cac878909ad1a89f678527b5328b93c1d4/scipy-1.17.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:158dd96d2207e21c966063e1635b1063cd7787b627b6f07305315dd73d9c679e", size = 33019667, upload-time = "2026-02-23T00:20:17.208Z" }, - { url = "https://files.pythonhosted.org/packages/f3/c3/2d834a5ac7bf3a0c806ad1508efc02dda3c8c61472a56132d7894c312dea/scipy-1.17.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74cbb80d93260fe2ffa334efa24cb8f2f0f622a9b9febf8b483c0b865bfb3475", size = 35264159, upload-time = "2026-02-23T00:20:23.087Z" }, - { url = "https://files.pythonhosted.org/packages/4d/77/d3ed4becfdbd217c52062fafe35a72388d1bd82c2d0ba5ca19d6fcc93e11/scipy-1.17.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:dbc12c9f3d185f5c737d801da555fb74b3dcfa1a50b66a1a93e09190f41fab50", size = 35102771, upload-time = "2026-02-23T00:20:28.636Z" }, - { url = "https://files.pythonhosted.org/packages/bd/12/d19da97efde68ca1ee5538bb261d5d2c062f0c055575128f11a2730e3ac1/scipy-1.17.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:94055a11dfebe37c656e70317e1996dc197e1a15bbcc351bcdd4610e128fe1ca", size = 37665910, upload-time = "2026-02-23T00:20:34.743Z" }, - { url = "https://files.pythonhosted.org/packages/06/1c/1172a88d507a4baaf72c5a09bb6c018fe2ae0ab622e5830b703a46cc9e44/scipy-1.17.1-cp313-cp313t-win_amd64.whl", hash = "sha256:e30bdeaa5deed6bc27b4cc490823cd0347d7dae09119b8803ae576ea0ce52e4c", size = 36562980, upload-time = "2026-02-23T00:20:40.575Z" }, - { url = "https://files.pythonhosted.org/packages/70/b0/eb757336e5a76dfa7911f63252e3b7d1de00935d7705cf772db5b45ec238/scipy-1.17.1-cp313-cp313t-win_arm64.whl", hash = "sha256:a720477885a9d2411f94a93d16f9d89bad0f28ca23c3f8daa521e2dcc3f44d49", size = 24856543, upload-time = "2026-02-23T00:20:45.313Z" }, - { url = "https://files.pythonhosted.org/packages/cf/83/333afb452af6f0fd70414dc04f898647ee1423979ce02efa75c3b0f2c28e/scipy-1.17.1-cp314-cp314-macosx_10_14_x86_64.whl", hash = "sha256:a48a72c77a310327f6a3a920092fa2b8fd03d7deaa60f093038f22d98e096717", size = 31584510, upload-time = "2026-02-23T00:21:01.015Z" }, - { url = "https://files.pythonhosted.org/packages/ed/a6/d05a85fd51daeb2e4ea71d102f15b34fedca8e931af02594193ae4fd25f7/scipy-1.17.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:45abad819184f07240d8a696117a7aacd39787af9e0b719d00285549ed19a1e9", size = 28170131, upload-time = "2026-02-23T00:21:05.888Z" }, - { url = "https://files.pythonhosted.org/packages/db/7b/8624a203326675d7746a254083a187398090a179335b2e4a20e2ddc46e83/scipy-1.17.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:3fd1fcdab3ea951b610dc4cef356d416d5802991e7e32b5254828d342f7b7e0b", size = 20342032, upload-time = "2026-02-23T00:21:09.904Z" }, - { url = "https://files.pythonhosted.org/packages/c9/35/2c342897c00775d688d8ff3987aced3426858fd89d5a0e26e020b660b301/scipy-1.17.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:7bdf2da170b67fdf10bca777614b1c7d96ae3ca5794fd9587dce41eb2966e866", size = 22678766, upload-time = "2026-02-23T00:21:14.313Z" }, - { url = "https://files.pythonhosted.org/packages/ef/f2/7cdb8eb308a1a6ae1e19f945913c82c23c0c442a462a46480ce487fdc0ac/scipy-1.17.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:adb2642e060a6549c343603a3851ba76ef0b74cc8c079a9a58121c7ec9fe2350", size = 32957007, upload-time = "2026-02-23T00:21:19.663Z" }, - { url = "https://files.pythonhosted.org/packages/0b/2e/7eea398450457ecb54e18e9d10110993fa65561c4f3add5e8eccd2b9cd41/scipy-1.17.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eee2cfda04c00a857206a4330f0c5e3e56535494e30ca445eb19ec624ae75118", size = 35221333, upload-time = "2026-02-23T00:21:25.278Z" }, - { url = "https://files.pythonhosted.org/packages/d9/77/5b8509d03b77f093a0d52e606d3c4f79e8b06d1d38c441dacb1e26cacf46/scipy-1.17.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d2650c1fb97e184d12d8ba010493ee7b322864f7d3d00d3f9bb97d9c21de4068", size = 35042066, upload-time = "2026-02-23T00:21:31.358Z" }, - { url = "https://files.pythonhosted.org/packages/f9/df/18f80fb99df40b4070328d5ae5c596f2f00fffb50167e31439e932f29e7d/scipy-1.17.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:08b900519463543aa604a06bec02461558a6e1cef8fdbb8098f77a48a83c8118", size = 37612763, upload-time = "2026-02-23T00:21:37.247Z" }, - { url = "https://files.pythonhosted.org/packages/4b/39/f0e8ea762a764a9dc52aa7dabcfad51a354819de1f0d4652b6a1122424d6/scipy-1.17.1-cp314-cp314-win_amd64.whl", hash = "sha256:3877ac408e14da24a6196de0ddcace62092bfc12a83823e92e49e40747e52c19", size = 37290984, upload-time = "2026-02-23T00:22:35.023Z" }, - { url = "https://files.pythonhosted.org/packages/7c/56/fe201e3b0f93d1a8bcf75d3379affd228a63d7e2d80ab45467a74b494947/scipy-1.17.1-cp314-cp314-win_arm64.whl", hash = "sha256:f8885db0bc2bffa59d5c1b72fad7a6a92d3e80e7257f967dd81abb553a90d293", size = 25192877, upload-time = "2026-02-23T00:22:39.798Z" }, - { url = "https://files.pythonhosted.org/packages/96/ad/f8c414e121f82e02d76f310f16db9899c4fcde36710329502a6b2a3c0392/scipy-1.17.1-cp314-cp314t-macosx_10_14_x86_64.whl", hash = "sha256:1cc682cea2ae55524432f3cdff9e9a3be743d52a7443d0cba9017c23c87ae2f6", size = 31949750, upload-time = "2026-02-23T00:21:42.289Z" }, - { url = "https://files.pythonhosted.org/packages/7c/b0/c741e8865d61b67c81e255f4f0a832846c064e426636cd7de84e74d209be/scipy-1.17.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:2040ad4d1795a0ae89bfc7e8429677f365d45aa9fd5e4587cf1ea737f927b4a1", size = 28585858, upload-time = "2026-02-23T00:21:47.706Z" }, - { url = "https://files.pythonhosted.org/packages/ed/1b/3985219c6177866628fa7c2595bfd23f193ceebbe472c98a08824b9466ff/scipy-1.17.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:131f5aaea57602008f9822e2115029b55d4b5f7c070287699fe45c661d051e39", size = 20757723, upload-time = "2026-02-23T00:21:52.039Z" }, - { url = "https://files.pythonhosted.org/packages/c0/19/2a04aa25050d656d6f7b9e7b685cc83d6957fb101665bfd9369ca6534563/scipy-1.17.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:9cdc1a2fcfd5c52cfb3045feb399f7b3ce822abdde3a193a6b9a60b3cb5854ca", size = 23043098, upload-time = "2026-02-23T00:21:56.185Z" }, - { url = "https://files.pythonhosted.org/packages/86/f1/3383beb9b5d0dbddd030335bf8a8b32d4317185efe495374f134d8be6cce/scipy-1.17.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e3dcd57ab780c741fde8dc68619de988b966db759a3c3152e8e9142c26295ad", size = 33030397, upload-time = "2026-02-23T00:22:01.404Z" }, - { url = "https://files.pythonhosted.org/packages/41/68/8f21e8a65a5a03f25a79165ec9d2b28c00e66dc80546cf5eb803aeeff35b/scipy-1.17.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a9956e4d4f4a301ebf6cde39850333a6b6110799d470dbbb1e25326ac447f52a", size = 35281163, upload-time = "2026-02-23T00:22:07.024Z" }, - { url = "https://files.pythonhosted.org/packages/84/8d/c8a5e19479554007a5632ed7529e665c315ae7492b4f946b0deb39870e39/scipy-1.17.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a4328d245944d09fd639771de275701ccadf5f781ba0ff092ad141e017eccda4", size = 35116291, upload-time = "2026-02-23T00:22:12.585Z" }, - { url = "https://files.pythonhosted.org/packages/52/52/e57eceff0e342a1f50e274264ed47497b59e6a4e3118808ee58ddda7b74a/scipy-1.17.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a77cbd07b940d326d39a1d1b37817e2ee4d79cb30e7338f3d0cddffae70fcaa2", size = 37682317, upload-time = "2026-02-23T00:22:18.513Z" }, - { url = "https://files.pythonhosted.org/packages/11/2f/b29eafe4a3fbc3d6de9662b36e028d5f039e72d345e05c250e121a230dd4/scipy-1.17.1-cp314-cp314t-win_amd64.whl", hash = "sha256:eb092099205ef62cd1782b006658db09e2fed75bffcae7cc0d44052d8aa0f484", size = 37345327, upload-time = "2026-02-23T00:22:24.442Z" }, - { url = "https://files.pythonhosted.org/packages/07/39/338d9219c4e87f3e708f18857ecd24d22a0c3094752393319553096b98af/scipy-1.17.1-cp314-cp314t-win_arm64.whl", hash = "sha256:200e1050faffacc162be6a486a984a0497866ec54149a01270adc8a59b7c7d21", size = 25489165, upload-time = "2026-02-23T00:22:29.563Z" }, -] - -[[package]] -name = "setuptools" -version = "81.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0d/1c/73e719955c59b8e424d015ab450f51c0af856ae46ea2da83eba51cc88de1/setuptools-81.0.0.tar.gz", hash = "sha256:487b53915f52501f0a79ccfd0c02c165ffe06631443a886740b91af4b7a5845a", size = 1198299, upload-time = "2026-02-06T21:10:39.601Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/e3/c164c88b2e5ce7b24d667b9bd83589cf4f3520d97cad01534cd3c4f55fdb/setuptools-81.0.0-py3-none-any.whl", hash = "sha256:fdd925d5c5d9f62e4b74b30d6dd7828ce236fd6ed998a08d81de62ce5a6310d6", size = 1062021, upload-time = "2026-02-06T21:10:37.175Z" }, -] - [[package]] name = "shellingham" version = "1.5.4" @@ -4818,15 +3968,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/44/48/d09817114dfaf4337a823aef52fefeef5d032118b2ab190d403bd8302d75/social_auth_core-4.5.6-py3-none-any.whl", hash = "sha256:43114bbc50f99789f7aadfd4943c2e15aee5e2f8c8612ad67fc4115b70b46ee1", size = 415386, upload-time = "2025-02-13T19:30:14.738Z" }, ] -[[package]] -name = "soupsieve" -version = "2.8.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7b/ae/2d9c981590ed9999a0d91755b47fc74f74de286b0f5cee14c9269041e6c4/soupsieve-2.8.3.tar.gz", hash = "sha256:3267f1eeea4251fb42728b6dfb746edc9acaffc4a45b27e19450b676586e8349", size = 118627, upload-time = "2026-01-20T04:27:02.457Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/46/2c/1462b1d0a634697ae9e55b3cecdcb64788e8b7d63f54d923fcd0bb140aed/soupsieve-2.8.3-py3-none-any.whl", hash = "sha256:ed64f2ba4eebeab06cc4962affce381647455978ffc1e36bb79a545b91f45a95", size = 37016, upload-time = "2026-01-20T04:27:01.012Z" }, -] - [[package]] name = "sqlalchemy" version = "2.0.48" @@ -4914,39 +4055,12 @@ wheels = [ ] [[package]] -name = "surya-ocr" -version = "0.17.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "einops" }, - { name = "filetype" }, - { name = "opencv-python-headless" }, - { name = "pillow" }, - { name = "platformdirs" }, - { name = "pre-commit" }, - { name = "pydantic" }, - { name = "pydantic-settings" }, - { name = "pypdfium2" }, - { name = "python-dotenv" }, - { name = "torch" }, - { name = "transformers" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b3/b4/b0f3afd024b4e6fcb4db6fdd17ee963d081e960c907aebd0556ff948abb6/surya_ocr-0.17.1.tar.gz", hash = "sha256:349d78d854c1ed5f816e583545ed6451aa0bc6992e283a805034799aacee8c24", size = 161854, upload-time = "2026-01-30T21:52:59.361Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/39/458fb9bc111e123faa2b5a0aade151c663f79dac4ad7e74f9c468b4b7786/surya_ocr-0.17.1-py3-none-any.whl", hash = "sha256:74c331ccb9be2d0c6a774122e572b85abdddf2982ecf6d11c1d83b3a0c9ae19d", size = 189881, upload-time = "2026-01-30T21:53:00.524Z" }, -] - -[[package]] -name = "sympy" -version = "1.14.0" +name = "tabulate" +version = "0.10.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mpmath" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } +sdist = { url = "https://files.pythonhosted.org/packages/46/58/8c37dea7bbf769b20d58e7ace7e5edfe65b849442b00ffcdd56be88697c6/tabulate-0.10.0.tar.gz", hash = "sha256:e2cfde8f79420f6deeffdeda9aaec3b6bc5abce947655d17ac662b126e48a60d", size = 91754, upload-time = "2026-03-04T18:55:34.402Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, + { url = "https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl", hash = "sha256:f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3", size = 39814, upload-time = "2026-03-04T18:55:31.284Z" }, ] [[package]] @@ -4958,15 +4072,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, ] -[[package]] -name = "threadpoolctl" -version = "3.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b7/4d/08c89e34946fce2aec4fbb45c9016efd5f4d7f24af8e5d93296e935631d8/threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e", size = 21274, upload-time = "2025-03-13T13:49:23.031Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" }, -] - [[package]] name = "tiktoken" version = "0.12.0" @@ -5112,99 +4217,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/23/d1/136eb2cb77520a31e1f64cbae9d33ec6df0d78bdf4160398e86eec8a8754/tomli-2.4.0-py3-none-any.whl", hash = "sha256:1f776e7d669ebceb01dee46484485f43a4048746235e683bcdffacdf1fb4785a", size = 14477, upload-time = "2026-01-11T11:22:37.446Z" }, ] -[[package]] -name = "torch" -version = "2.11.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-bindings", marker = "sys_platform == 'linux'" }, - { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" }, - { name = "filelock" }, - { name = "fsspec" }, - { name = "jinja2" }, - { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" }, - { name = "setuptools" }, - { name = "sympy" }, - { name = "triton", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/ac/f2/c1690994afe461aae2d0cac62251e6802a703dec0a6c549c02ecd0de92a9/torch-2.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2c0d7fcfbc0c4e8bb5ebc3907cbc0c6a0da1b8f82b1fc6e14e914fa0b9baf74e", size = 80526521, upload-time = "2026-03-23T18:12:06.86Z" }, - { url = "https://files.pythonhosted.org/packages/a4/f0/98ae802fa8c09d3149b0c8690741f3f5753c90e779bd28c9613257295945/torch-2.11.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:4cf8687f4aec3900f748d553483ef40e0ac38411c3c48d0a86a438f6d7a99b18", size = 419723025, upload-time = "2026-03-23T18:11:43.774Z" }, - { url = "https://files.pythonhosted.org/packages/f9/1e/18a9b10b4bd34f12d4e561c52b0ae7158707b8193c6cfc0aad2b48167090/torch-2.11.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:1b32ceda909818a03b112006709b02be1877240c31750a8d9c6b7bf5f2d8a6e5", size = 530589207, upload-time = "2026-03-23T18:11:23.756Z" }, - { url = "https://files.pythonhosted.org/packages/35/40/2d532e8c0e23705be9d1debce5bc37b68d59a39bda7584c26fe9668076fe/torch-2.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:b3c712ae6fb8e7a949051a953fc412fe0a6940337336c3b6f905e905dac5157f", size = 114518313, upload-time = "2026-03-23T18:11:58.281Z" }, - { url = "https://files.pythonhosted.org/packages/ae/0d/98b410492609e34a155fa8b121b55c7dca229f39636851c3a9ec20edea21/torch-2.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7b6a60d48062809f58595509c524b88e6ddec3ebe25833d6462eeab81e5f2ce4", size = 80529712, upload-time = "2026-03-23T18:12:02.608Z" }, - { url = "https://files.pythonhosted.org/packages/84/03/acea680005f098f79fd70c1d9d5ccc0cb4296ec2af539a0450108232fc0c/torch-2.11.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:d91aac77f24082809d2c5a93f52a5f085032740a1ebc9252a7b052ef5a4fddc6", size = 419718178, upload-time = "2026-03-23T18:10:46.675Z" }, - { url = "https://files.pythonhosted.org/packages/8c/8b/d7be22fbec9ffee6cff31a39f8750d4b3a65d349a286cf4aec74c2375662/torch-2.11.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:7aa2f9bbc6d4595ba72138026b2074be1233186150e9292865e04b7a63b8c67a", size = 530604548, upload-time = "2026-03-23T18:10:03.569Z" }, - { url = "https://files.pythonhosted.org/packages/d1/bd/9912d30b68845256aabbb4a40aeefeef3c3b20db5211ccda653544ada4b6/torch-2.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:73e24aaf8f36ab90d95cd1761208b2eb70841c2a9ca1a3f9061b39fc5331b708", size = 114519675, upload-time = "2026-03-23T18:11:52.995Z" }, - { url = "https://files.pythonhosted.org/packages/6f/8b/69e3008d78e5cee2b30183340cc425081b78afc5eff3d080daab0adda9aa/torch-2.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4b5866312ee6e52ea625cd211dcb97d6a2cdc1131a5f15cc0d87eec948f6dd34", size = 80606338, upload-time = "2026-03-23T18:11:34.781Z" }, - { url = "https://files.pythonhosted.org/packages/13/16/42e5915ebe4868caa6bac83a8ed59db57f12e9a61b7d749d584776ed53d5/torch-2.11.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:f99924682ef0aa6a4ab3b1b76f40dc6e273fca09f367d15a524266db100a723f", size = 419731115, upload-time = "2026-03-23T18:11:06.944Z" }, - { url = "https://files.pythonhosted.org/packages/1a/c9/82638ef24d7877510f83baf821f5619a61b45568ce21c0a87a91576510aa/torch-2.11.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:0f68f4ac6d95d12e896c3b7a912b5871619542ec54d3649cf48cc1edd4dd2756", size = 530712279, upload-time = "2026-03-23T18:10:31.481Z" }, - { url = "https://files.pythonhosted.org/packages/1c/ff/6756f1c7ee302f6d202120e0f4f05b432b839908f9071157302cedfc5232/torch-2.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:fbf39280699d1b869f55eac536deceaa1b60bd6788ba74f399cc67e60a5fab10", size = 114556047, upload-time = "2026-03-23T18:10:55.931Z" }, - { url = "https://files.pythonhosted.org/packages/87/89/5ea6722763acee56b045435fb84258db7375c48165ec8be7880ab2b281c5/torch-2.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1e6debd97ccd3205bbb37eb806a9d8219e1139d15419982c09e23ef7d4369d18", size = 80606801, upload-time = "2026-03-23T18:10:18.649Z" }, - { url = "https://files.pythonhosted.org/packages/32/d1/8ed2173589cbfe744ed54e5a73efc107c0085ba5777ee93a5f4c1ab90553/torch-2.11.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:63a68fa59de8f87acc7e85a5478bb2dddbb3392b7593ec3e78827c793c4b73fd", size = 419732382, upload-time = "2026-03-23T18:08:30.835Z" }, - { url = "https://files.pythonhosted.org/packages/3d/e1/b73f7c575a4b8f87a5928f50a1e35416b5e27295d8be9397d5293e7e8d4c/torch-2.11.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:cc89b9b173d9adfab59fd227f0ab5e5516d9a52b658ae41d64e59d2e55a418db", size = 530711509, upload-time = "2026-03-23T18:08:47.213Z" }, - { url = "https://files.pythonhosted.org/packages/66/82/3e3fcdd388fbe54e29fd3f991f36846ff4ac90b0d0181e9c8f7236565f82/torch-2.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:4dda3b3f52d121063a731ddb835f010dc137b920d7fec2778e52f60d8e4bf0cd", size = 114555842, upload-time = "2026-03-23T18:09:52.111Z" }, - { url = "https://files.pythonhosted.org/packages/db/38/8ac78069621b8c2b4979c2f96dc8409ef5e9c4189f6aac629189a78677ca/torch-2.11.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8b394322f49af4362d4f80e424bcaca7efcd049619af03a4cf4501520bdf0fb4", size = 80959574, upload-time = "2026-03-23T18:10:14.214Z" }, - { url = "https://files.pythonhosted.org/packages/6d/6c/56bfb37073e7136e6dd86bfc6af7339946dd684e0ecf2155ac0eee687ae1/torch-2.11.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:2658f34ce7e2dabf4ec73b45e2ca68aedad7a5be87ea756ad656eaf32bf1e1ea", size = 419732324, upload-time = "2026-03-23T18:09:36.604Z" }, - { url = "https://files.pythonhosted.org/packages/07/f4/1b666b6d61d3394cca306ea543ed03a64aad0a201b6cd159f1d41010aeb1/torch-2.11.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:98bb213c3084cfe176302949bdc360074b18a9da7ab59ef2edc9d9f742504778", size = 530596026, upload-time = "2026-03-23T18:09:20.842Z" }, - { url = "https://files.pythonhosted.org/packages/48/6b/30d1459fa7e4b67e9e3fe1685ca1d8bb4ce7c62ef436c3a615963c6c866c/torch-2.11.0-cp313-cp313t-win_amd64.whl", hash = "sha256:a97b94bbf62992949b4730c6cd2cc9aee7b335921ee8dc207d930f2ed09ae2db", size = 114793702, upload-time = "2026-03-23T18:09:47.304Z" }, - { url = "https://files.pythonhosted.org/packages/26/0d/8603382f61abd0db35841148ddc1ffd607bf3100b11c6e1dab6d2fc44e72/torch-2.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:01018087326984a33b64e04c8cb5c2795f9120e0d775ada1f6638840227b04d7", size = 80573442, upload-time = "2026-03-23T18:09:10.117Z" }, - { url = "https://files.pythonhosted.org/packages/c7/86/7cd7c66cb9cec6be330fff36db5bd0eef386d80c031b581ec81be1d4b26c/torch-2.11.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:2bb3cc54bd0dea126b0060bb1ec9de0f9c7f7342d93d436646516b0330cd5be7", size = 419749385, upload-time = "2026-03-23T18:07:33.77Z" }, - { url = "https://files.pythonhosted.org/packages/47/e8/b98ca2d39b2e0e4730c0ee52537e488e7008025bc77ca89552ff91021f7c/torch-2.11.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:4dc8b3809469b6c30b411bb8c4cad3828efd26236153d9beb6a3ec500f211a60", size = 530716756, upload-time = "2026-03-23T18:07:50.02Z" }, - { url = "https://files.pythonhosted.org/packages/78/88/d4a4cda8362f8a30d1ed428564878c3cafb0d87971fbd3947d4c84552095/torch-2.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:2b4e811728bd0cc58fb2b0948fe939a1ee2bf1422f6025be2fca4c7bd9d79718", size = 114552300, upload-time = "2026-03-23T18:09:05.617Z" }, - { url = "https://files.pythonhosted.org/packages/bf/46/4419098ed6d801750f26567b478fc185c3432e11e2cad712bc6b4c2ab0d0/torch-2.11.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8245477871c3700d4370352ffec94b103cfcb737229445cf9946cddb7b2ca7cd", size = 80959460, upload-time = "2026-03-23T18:09:00.818Z" }, - { url = "https://files.pythonhosted.org/packages/fd/66/54a56a4a6ceaffb567231994a9745821d3af922a854ed33b0b3a278e0a99/torch-2.11.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:ab9a8482f475f9ba20e12db84b0e55e2f58784bdca43a854a6ccd3fd4b9f75e6", size = 419735835, upload-time = "2026-03-23T18:07:18.974Z" }, - { url = "https://files.pythonhosted.org/packages/b1/e7/0b6665f533aa9e337662dc190425abc0af1fe3234088f4454c52393ded61/torch-2.11.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:563ed3d25542d7e7bbc5b235ccfacfeb97fb470c7fee257eae599adb8005c8a2", size = 530613405, upload-time = "2026-03-23T18:08:07.014Z" }, - { url = "https://files.pythonhosted.org/packages/cf/bf/c8d12a2c86dbfd7f40fb2f56fbf5a505ccf2d9ce131eb559dfc7c51e1a04/torch-2.11.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b2a43985ff5ef6ddd923bbcf99943e5f58059805787c5c9a2622bf05ca2965b0", size = 114792991, upload-time = "2026-03-23T18:08:19.216Z" }, -] - -[[package]] -name = "torchvision" -version = "0.26.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.4.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "pillow" }, - { name = "torch" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/74/b4/cdfee31e0402ea035135462cb0ab496e974d56fab6b4e7a1f0cbccb8cd28/torchvision-0.26.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a06d4772a8e13e772906ed736cc53ec6639e5e60554f8e5fa6ca165aabebc464", size = 1863503, upload-time = "2026-03-23T18:13:01.384Z" }, - { url = "https://files.pythonhosted.org/packages/e4/74/11fee109841e80ad14e5ca2d80bff6b10eb11b7838ff06f35bfeaa9f7251/torchvision-0.26.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:2adfbe438473236191ff077a4a9a0c767436879c89628aa97137e959b0c11a94", size = 7766423, upload-time = "2026-03-23T18:12:56.049Z" }, - { url = "https://files.pythonhosted.org/packages/5e/00/24d8c7845c3f270153fb81395a5135b2778e2538e81d14c6aea5106c689c/torchvision-0.26.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b6f9ad1ecc0eab52647298b379ee9426845f8903703e6127973f8f3d049a798b", size = 7518249, upload-time = "2026-03-23T18:12:51.743Z" }, - { url = "https://files.pythonhosted.org/packages/d7/ed/e53cd7c0da7ae002e5e929c1796ebbe7ec0c700c29f7a0a6696497fb3d8b/torchvision-0.26.0-cp310-cp310-win_amd64.whl", hash = "sha256:f13f12b3791a266de2d599cb8162925261622a037d87fc03132848343cf68f75", size = 3669784, upload-time = "2026-03-23T18:12:49.949Z" }, - { url = "https://files.pythonhosted.org/packages/b4/bd/d552a2521bade3295b2c6e7a4a0d1022261cab7ca7011f4e2a330dbb3caa/torchvision-0.26.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:55bd6ad4ae77be01ba67a410b05b51f53b0d0ee45f146eb6a0dfb9007e70ab3c", size = 1863499, upload-time = "2026-03-23T18:12:58.696Z" }, - { url = "https://files.pythonhosted.org/packages/33/bf/21b899792b08cae7a298551c68398a79e333697479ed311b3b067aab4bdc/torchvision-0.26.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:1c55dc8affbcc0eb2060fbabbe996ae9e5839b24bb6419777f17848945a411b1", size = 7767527, upload-time = "2026-03-23T18:12:44.348Z" }, - { url = "https://files.pythonhosted.org/packages/9a/45/57bbf9e216850d065e66dd31a50f57424b607f1d878ab8956e56a1f4e36b/torchvision-0.26.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:fd10b5f994c210f4f6d6761cf686f82d748554adf486cb0979770c3252868c8f", size = 7519925, upload-time = "2026-03-23T18:12:53.283Z" }, - { url = "https://files.pythonhosted.org/packages/10/58/ed8f7754299f3e91d6414b6dc09f62b3fa7c6e5d63dfe48d69ab81498a37/torchvision-0.26.0-cp311-cp311-win_amd64.whl", hash = "sha256:de6424b12887ad884f39a0ee446994ae3cd3b6a00a9cafe1bead85a031132af0", size = 3983834, upload-time = "2026-03-23T18:13:00.224Z" }, - { url = "https://files.pythonhosted.org/packages/ae/e7/56b47cc3b132aea90ccce22bcb8975dec688b002150012acc842846039d0/torchvision-0.26.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c409e1c3fdebec7a3834465086dbda8bf7680eff79abf7fd2f10c6b59520a7a4", size = 1863502, upload-time = "2026-03-23T18:12:57.326Z" }, - { url = "https://files.pythonhosted.org/packages/f4/ec/5c31c92c08b65662fe9604a4067ae8232582805949f11ddc042cebe818ed/torchvision-0.26.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:406557718e62fdf10f5706e88d8a5ec000f872da913bf629aab9297622585547", size = 7767944, upload-time = "2026-03-23T18:12:42.805Z" }, - { url = "https://files.pythonhosted.org/packages/f5/d8/cb6ccda1a1f35a6597645818641701207b3e8e13553e75fce5d86bac74b2/torchvision-0.26.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d61a5abb6b42a0c0c311996c2ac4b83a94418a97182c83b055a2a4ae985e05aa", size = 7522205, upload-time = "2026-03-23T18:12:54.654Z" }, - { url = "https://files.pythonhosted.org/packages/1c/a9/c272623a0f735c35f0f6cd6dc74784d4f970e800cf063bb76687895a2ab9/torchvision-0.26.0-cp312-cp312-win_amd64.whl", hash = "sha256:7993c01648e7c61d191b018e84d38fe0825c8fcb2720cd0f37caf7ba14404aa1", size = 4255155, upload-time = "2026-03-23T18:12:32.652Z" }, - { url = "https://files.pythonhosted.org/packages/da/80/0762f77f53605d10c9477be39bb47722cc8e383bbbc2531471ce0e396c07/torchvision-0.26.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:5d63dd43162691258b1b3529b9041bac7d54caa37eae0925f997108268cbf7c4", size = 1860809, upload-time = "2026-03-23T18:12:47.629Z" }, - { url = "https://files.pythonhosted.org/packages/e6/81/0b3e58d1478c660a5af4268713486b2df7203f35abd9195fea87348a5178/torchvision-0.26.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a39c7a26538c41fda453f9a9692b5ff9b35a5437db1d94f3027f6f509c160eac", size = 7727494, upload-time = "2026-03-23T18:12:46.062Z" }, - { url = "https://files.pythonhosted.org/packages/b6/dc/d9ab5d29115aa05e12e30f1397a3eeae1d88a511241dc3bce48dc4342675/torchvision-0.26.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:b7e6213620bbf97742e5f79832f9e9d769e6cf0f744c5b53dad80b76db633691", size = 7521747, upload-time = "2026-03-23T18:12:36.815Z" }, - { url = "https://files.pythonhosted.org/packages/a9/1b/f1bc86a918c5f6feab1eeff11982e2060f4704332e96185463d27855bdf5/torchvision-0.26.0-cp313-cp313-win_amd64.whl", hash = "sha256:4280c35ec8cba1fcc8294fb87e136924708726864c379e4c54494797d86bc474", size = 4319880, upload-time = "2026-03-23T18:12:38.168Z" }, - { url = "https://files.pythonhosted.org/packages/66/28/b4ad0a723ed95b003454caffcc41894b34bd8379df340848cae2c33871de/torchvision-0.26.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:358fc4726d0c08615b6d83b3149854f11efb2a564ed1acb6fce882e151412d23", size = 1951973, upload-time = "2026-03-23T18:12:48.781Z" }, - { url = "https://files.pythonhosted.org/packages/71/e2/7a89096e6cf2f3336353b5338ba925e0addf9d8601920340e6bdf47e8eb3/torchvision-0.26.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:3daf9cc149cf3cdcbd4df9c59dae69ffca86c6823250442c3bbfd63fc2e26c61", size = 7728679, upload-time = "2026-03-23T18:12:26.196Z" }, - { url = "https://files.pythonhosted.org/packages/69/1d/4e1eebc17d18ce080a11dcf3df3f8f717f0efdfa00983f06e8ba79259f61/torchvision-0.26.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:82c3965eca27e86a316e31e4c3e5a16d353e0bcbe0ef8efa2e66502c54493c4b", size = 7609138, upload-time = "2026-03-23T18:12:35.327Z" }, - { url = "https://files.pythonhosted.org/packages/f3/a4/f1155e943ae5b32400d7000adc81c79bb0392b16ceb33bcf13e02e48cced/torchvision-0.26.0-cp313-cp313t-win_amd64.whl", hash = "sha256:ebc043cc5a4f0bf22e7680806dbba37ffb19e70f6953bbb44ed1a90aeb5c9bea", size = 4248202, upload-time = "2026-03-23T18:12:41.423Z" }, - { url = "https://files.pythonhosted.org/packages/7f/c8/9bffa9c7f7bdf95b2a0a2dc535c290b9f1cc580c3fb3033ab1246ffffdeb/torchvision-0.26.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:eb61804eb9dbe88c5a2a6c4da8dec1d80d2d0a6f18c999c524e32266cb1ebcd3", size = 1860813, upload-time = "2026-03-23T18:12:39.636Z" }, - { url = "https://files.pythonhosted.org/packages/7b/ac/48f28ffd227991f2e14f4392dde7e8dc14352bb9428c1ef4a4bbf5f7ed85/torchvision-0.26.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:9a904f2131cbfadab4df828088a9f66291ad33f49ff853872aed1f86848ef776", size = 7727777, upload-time = "2026-03-23T18:12:22.549Z" }, - { url = "https://files.pythonhosted.org/packages/a4/21/a2266f7f1b0e58e624ff15fd6f01041f59182c49551ece0db9a183071329/torchvision-0.26.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:0f3e572efe62ad645017ea847e0b5e4f2f638d4e39f05bc011d1eb9ac68d4806", size = 7522174, upload-time = "2026-03-23T18:12:29.565Z" }, - { url = "https://files.pythonhosted.org/packages/fc/ba/1666f90bc0bdd77aaa11dcc42bb9f621a9c3668819c32430452e3d404730/torchvision-0.26.0-cp314-cp314-win_amd64.whl", hash = "sha256:114bec0c0e98aa4ba446f63e2fe7a2cbca37b39ac933987ee4804f65de121800", size = 4348469, upload-time = "2026-03-23T18:12:24.44Z" }, - { url = "https://files.pythonhosted.org/packages/45/8f/1f0402ac55c2ae15651ff831957d083fe70b2d12282e72612a30ba601512/torchvision-0.26.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:b7d3e295624a28b3b1769228ce1345d94cf4d390dd31136766f76f2d20f718da", size = 1860826, upload-time = "2026-03-23T18:12:34.1Z" }, - { url = "https://files.pythonhosted.org/packages/d2/6a/18a582fe3c5ee26f49b5c9fb21ad8016b4d1c06d10178894a58653946fda/torchvision-0.26.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:7058c5878262937e876f20c25867b33724586aa4499e2853b2d52b99a5e51953", size = 7729089, upload-time = "2026-03-23T18:12:31.394Z" }, - { url = "https://files.pythonhosted.org/packages/c5/9b/f7e119b59499edc00c55c03adc9ec3bd96144d9b81c46852c431f9c64a9a/torchvision-0.26.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:8008474855623c6ba52876589dc52df0aa66e518c25eca841445348e5f79844c", size = 7522704, upload-time = "2026-03-23T18:12:20.301Z" }, - { url = "https://files.pythonhosted.org/packages/d0/6a/09f3844c10643f6c0de5d95abc863420cfaf194c88c7dffd0ac523e2015f/torchvision-0.26.0-cp314-cp314t-win_amd64.whl", hash = "sha256:e9d0e022c19a78552fb055d0414d47fecb4a649309b9968573daea160ba6869c", size = 4454275, upload-time = "2026-03-23T18:12:27.487Z" }, -] - [[package]] name = "tqdm" version = "4.67.3" @@ -5217,49 +4229,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" }, ] -[[package]] -name = "transformers" -version = "4.57.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "filelock" }, - { name = "huggingface-hub" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.4.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "packaging" }, - { name = "pyyaml" }, - { name = "regex" }, - { name = "requests" }, - { name = "safetensors" }, - { name = "tokenizers" }, - { name = "tqdm" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c4/35/67252acc1b929dc88b6602e8c4a982e64f31e733b804c14bc24b47da35e6/transformers-4.57.6.tar.gz", hash = "sha256:55e44126ece9dc0a291521b7e5492b572e6ef2766338a610b9ab5afbb70689d3", size = 10134912, upload-time = "2026-01-16T10:38:39.284Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/03/b8/e484ef633af3887baeeb4b6ad12743363af7cce68ae51e938e00aaa0529d/transformers-4.57.6-py3-none-any.whl", hash = "sha256:4c9e9de11333ddfe5114bc872c9f370509198acf0b87a832a0ab9458e2bd0550", size = 11993498, upload-time = "2026-01-16T10:38:31.289Z" }, -] - -[[package]] -name = "triton" -version = "3.6.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/44/ba/b1b04f4b291a3205d95ebd24465de0e5bf010a2df27a4e58a9b5f039d8f2/triton-3.6.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c723cfb12f6842a0ae94ac307dba7e7a44741d720a40cf0e270ed4a4e3be781", size = 175972180, upload-time = "2026-01-20T16:15:53.664Z" }, - { url = "https://files.pythonhosted.org/packages/8c/f7/f1c9d3424ab199ac53c2da567b859bcddbb9c9e7154805119f8bd95ec36f/triton-3.6.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a6550fae429e0667e397e5de64b332d1e5695b73650ee75a6146e2e902770bea", size = 188105201, upload-time = "2026-01-20T16:00:29.272Z" }, - { url = "https://files.pythonhosted.org/packages/0f/2c/96f92f3c60387e14cc45aed49487f3486f89ea27106c1b1376913c62abe4/triton-3.6.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49df5ef37379c0c2b5c0012286f80174fcf0e073e5ade1ca9a86c36814553651", size = 176081190, upload-time = "2026-01-20T16:16:00.523Z" }, - { url = "https://files.pythonhosted.org/packages/e0/12/b05ba554d2c623bffa59922b94b0775673de251f468a9609bc9e45de95e9/triton-3.6.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8e323d608e3a9bfcc2d9efcc90ceefb764a82b99dea12a86d643c72539ad5d3", size = 188214640, upload-time = "2026-01-20T16:00:35.869Z" }, - { url = "https://files.pythonhosted.org/packages/17/5d/08201db32823bdf77a0e2b9039540080b2e5c23a20706ddba942924ebcd6/triton-3.6.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:374f52c11a711fd062b4bfbb201fd9ac0a5febd28a96fb41b4a0f51dde3157f4", size = 176128243, upload-time = "2026-01-20T16:16:07.857Z" }, - { url = "https://files.pythonhosted.org/packages/ab/a8/cdf8b3e4c98132f965f88c2313a4b493266832ad47fb52f23d14d4f86bb5/triton-3.6.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74caf5e34b66d9f3a429af689c1c7128daba1d8208df60e81106b115c00d6fca", size = 188266850, upload-time = "2026-01-20T16:00:43.041Z" }, - { url = "https://files.pythonhosted.org/packages/3c/12/34d71b350e89a204c2c7777a9bba0dcf2f19a5bfdd70b57c4dbc5ffd7154/triton-3.6.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:448e02fe6dc898e9e5aa89cf0ee5c371e99df5aa5e8ad976a80b93334f3494fd", size = 176133521, upload-time = "2026-01-20T16:16:13.321Z" }, - { url = "https://files.pythonhosted.org/packages/f9/0b/37d991d8c130ce81a8728ae3c25b6e60935838e9be1b58791f5997b24a54/triton-3.6.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10c7f76c6e72d2ef08df639e3d0d30729112f47a56b0c81672edc05ee5116ac9", size = 188289450, upload-time = "2026-01-20T16:00:49.136Z" }, - { url = "https://files.pythonhosted.org/packages/ce/4e/41b0c8033b503fd3cfcd12392cdd256945026a91ff02452bef40ec34bee7/triton-3.6.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1722e172d34e32abc3eb7711d0025bb69d7959ebea84e3b7f7a341cd7ed694d6", size = 176276087, upload-time = "2026-01-20T16:16:18.989Z" }, - { url = "https://files.pythonhosted.org/packages/35/f8/9c66bfc55361ec6d0e4040a0337fb5924ceb23de4648b8a81ae9d33b2b38/triton-3.6.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d002e07d7180fd65e622134fbd980c9a3d4211fb85224b56a0a0efbd422ab72f", size = 188400296, upload-time = "2026-01-20T16:00:56.042Z" }, - { url = "https://files.pythonhosted.org/packages/49/55/5ecf0dcaa0f2fbbd4420f7ef227ee3cb172e91e5fede9d0ecaddc43363b4/triton-3.6.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef5523241e7d1abca00f1d240949eebdd7c673b005edbbce0aca95b8191f1d43", size = 176138577, upload-time = "2026-01-20T16:16:25.426Z" }, - { url = "https://files.pythonhosted.org/packages/df/3d/9e7eee57b37c80cec63322c0231bb6da3cfe535a91d7a4d64896fcb89357/triton-3.6.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a17a5d5985f0ac494ed8a8e54568f092f7057ef60e1b0fa09d3fd1512064e803", size = 188273063, upload-time = "2026-01-20T16:01:07.278Z" }, - { url = "https://files.pythonhosted.org/packages/48/db/56ee649cab5eaff4757541325aca81f52d02d4a7cd3506776cad2451e060/triton-3.6.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b3a97e8ed304dfa9bd23bb41ca04cdf6b2e617d5e782a8653d616037a5d537d", size = 176274804, upload-time = "2026-01-20T16:16:31.528Z" }, - { url = "https://files.pythonhosted.org/packages/f6/56/6113c23ff46c00aae423333eb58b3e60bdfe9179d542781955a5e1514cb3/triton-3.6.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46bd1c1af4b6704e554cad2eeb3b0a6513a980d470ccfa63189737340c7746a7", size = 188397994, upload-time = "2026-01-20T16:01:14.236Z" }, -] - [[package]] name = "ty" version = "0.0.46" @@ -5364,6 +4333,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl", hash = "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f", size = 8827, upload-time = "2023-05-24T20:25:45.287Z" }, ] +[[package]] +name = "typing-inspection" +version = "0.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/26/b09b8010994eccc3c09092e6b34058f36a460eea2d4c3e8b910c695975a0/typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47", size = 76928, upload-time = "2026-08-12T12:37:25.997Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, +] + [[package]] name = "tzdata" version = "2025.3" @@ -5451,22 +4432,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e4/16/c1fd27e9549f3c4baf1dc9c20c456cd2f822dbf8de9f463824b0c0357e06/uvloop-0.22.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6cde23eeda1a25c75b2e07d39970f3374105d5eafbaab2a4482be82f272d5a5e", size = 4296730, upload-time = "2025-10-16T22:17:00.744Z" }, ] -[[package]] -name = "virtualenv" -version = "21.2.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "distlib" }, - { name = "filelock" }, - { name = "platformdirs" }, - { name = "python-discovery" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/97/c5/aff062c66b42e2183201a7ace10c6b2e959a9a16525c8e8ca8e59410d27a/virtualenv-21.2.1.tar.gz", hash = "sha256:b66ffe81301766c0d5e2208fc3576652c59d44e7b731fc5f5ed701c9b537fa78", size = 5844770, upload-time = "2026-04-09T18:47:11.482Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/20/0e/f083a76cb590e60dff3868779558eefefb8dfb7c9ed020babc7aa014ccbf/virtualenv-21.2.1-py3-none-any.whl", hash = "sha256:bd16b49c53562b28cf1a3ad2f36edb805ad71301dee70ddc449e5c88a9f919a2", size = 5828326, upload-time = "2026-04-09T18:47:09.331Z" }, -] - [[package]] name = "watchfiles" version = "1.1.1" @@ -5570,15 +4535,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6e/d4/ed38dd3b1767193de971e694aa544356e63353c33a85d948166b5ff58b9e/watchfiles-1.1.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e6f39af2eab0118338902798b5aa6664f46ff66bc0280de76fca67a7f262a49", size = 457546, upload-time = "2025-10-14T15:06:13.372Z" }, ] -[[package]] -name = "wcwidth" -version = "0.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/35/a2/8e3becb46433538a38726c948d3399905a4c7cabd0df578ede5dc51f0ec2/wcwidth-0.6.0.tar.gz", hash = "sha256:cdc4e4262d6ef9a1a57e018384cbeb1208d8abbc64176027e2c2455c81313159", size = 159684, upload-time = "2026-02-06T19:19:40.919Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/68/5a/199c59e0a824a3db2b89c5d2dade7ab5f9624dbf6448dc291b46d5ec94d3/wcwidth-0.6.0-py3-none-any.whl", hash = "sha256:1a3a1e510b553315f8e146c54764f4fb6264ffad731b3d78088cdb1478ffbdad", size = 94189, upload-time = "2026-02-06T19:19:39.646Z" }, -] - [[package]] name = "websockets" version = "14.2" From 5d702050d09ac86e007c824fce44b9b222b0982f Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Fri, 14 Aug 2026 19:12:48 -0700 Subject: [PATCH 02/10] docs(changelog): note the DoclingDocument layout contract --- extralit-server/CHANGELOG.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/extralit-server/CHANGELOG.md b/extralit-server/CHANGELOG.md index 9c92fb26b..651b0d98d 100644 --- a/extralit-server/CHANGELOG.md +++ b/extralit-server/CHANGELOG.md @@ -18,6 +18,20 @@ These are the section headers that we use: ### Added - Refactor document analysis and preprocessing job to support asynchronous s3 IO operations and large file processing. +- Adopted `DoclingDocument` as the canonical internal layout model, so every extracted item carries a `page_no` + `bbox` + `charspan` provenance triple that an extraction record can cite. +- Added swappable layout parsers behind `contexts/ocr/parsers`: `pdf_inspector` (MIT, required, the default) and `pymupdf` (AGPL, optional `pymupdf` extra, the only one yielding per-cell table geometry). +- Added `async_document_layout_job` on the OCR queue, persisting layout as canonical JSON plus a columnar Parquet projection in object storage. +- Added `GET /documents/{document_id}/layout`, with `pages` and `labels` filters, plus the matching frontend `DocumentLayout` entities and `DocumentRepository.getDocumentLayout`. +- Added `layout_parser` to `POST /workflows/start` to trigger layout extraction as part of the document workflow. + +### Changed + +- Rewrote `contexts/ocr/{text,tables,figures}` as single-item appenders driven by one ordered pass, fixing text inside tables and figures being emitted twice and restoring geometric reading order. + +### Removed + +- Removed the `marker` extra (`marker-pdf`, `torch`, `torchvision`, `transformers`) and the unused `async_marker_layout_job`. +- Removed the deprecated Unstructured-shaped `document/chunks.py` schemas (`Segments`, `TextSegment`, `TableSegment`, `FigureSegment`, `Coordinates`) and the unreferenced `GPU_QUEUE`. ### Fixed From dfab9a1d9b3e5f696163643c356a7c0a718c6756 Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Fri, 14 Aug 2026 19:28:41 -0700 Subject: [PATCH 03/10] fix(ocr): scope layout reads to workspace members and sequence after preprocessing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two HIGH findings from review #335. The layout route authorized with DocumentPolicy.get(), which carries an explicit unfixed TODO and returns true for any annotator. Since the document is loaded by UUID, an authenticated user from another workspace could read its extracted text, table HTML, and page geometry. Adds DocumentPolicy.get_by_workspace, following the existing WorkspacePolicy.get pattern, and applies it once the document's workspace is known. The shared get() is left alone — its TODO predates this work and affects sibling routes. The layout job was enqueued alongside analysis_and_preprocess_job, which runs OCRmyPDF rotation and overwrites the PDF at the same S3 path. Layout could therefore describe the pre-rotation file while clients render the post-rotation one, persisting bounding boxes for a PDF nobody displays. It now depends on the analysis job. The metadata_ read-modify-write also takes a row lock, since the external text-extraction job still writes that JSON column concurrently. Adds 10 regression tests covering both. --- .../api/handlers/v1/documents.py | 3 + .../api/policies/v1/document_policy.py | 9 ++ .../src/extralit_server/jobs/ocr_jobs.py | 4 + .../extralit_server/workflows/documents.py | 34 +++--- .../api/handlers/v1/test_document_layout.py | 60 ++++++++++- .../tests/unit/workflows/__init__.py | 0 .../test_document_workflow_layout.py | 102 ++++++++++++++++++ 7 files changed, 193 insertions(+), 19 deletions(-) create mode 100644 extralit-server/tests/unit/workflows/__init__.py create mode 100644 extralit-server/tests/unit/workflows/test_document_workflow_layout.py diff --git a/extralit-server/src/extralit_server/api/handlers/v1/documents.py b/extralit-server/src/extralit_server/api/handlers/v1/documents.py index 32ff39395..fb8cf080b 100644 --- a/extralit-server/src/extralit_server/api/handlers/v1/documents.py +++ b/extralit-server/src/extralit_server/api/handlers/v1/documents.py @@ -274,6 +274,9 @@ async def get_document_layout( detail=f"Document with id `{document_id}` not found", ) + # This route returns document *contents*, so membership is checked rather than role alone. + await authorize(current_user, DocumentPolicy.get_by_workspace(document.workspace_id)) + workspace = await Workspace.get(db, document.workspace_id) if workspace is None: raise HTTPException( diff --git a/extralit-server/src/extralit_server/api/policies/v1/document_policy.py b/extralit-server/src/extralit_server/api/policies/v1/document_policy.py index a3ae7b119..d2e76ec1d 100644 --- a/extralit-server/src/extralit_server/api/policies/v1/document_policy.py +++ b/extralit-server/src/extralit_server/api/policies/v1/document_policy.py @@ -20,6 +20,15 @@ async def is_allowed(actor: User) -> bool: return is_allowed + @classmethod + def get_by_workspace(cls, workspace_id: UUID) -> PolicyAction: + """Read a document's contents. Unlike `get`, this verifies workspace membership.""" + + async def is_allowed(actor: User) -> bool: + return actor.is_owner or await actor.is_member(workspace_id) + + return is_allowed + @classmethod def bulk_create(cls, workspace_id: UUID) -> PolicyAction: async def is_allowed(actor: User) -> bool: diff --git a/extralit-server/src/extralit_server/jobs/ocr_jobs.py b/extralit-server/src/extralit_server/jobs/ocr_jobs.py index 9e02b0b42..cb152a8a6 100644 --- a/extralit-server/src/extralit_server/jobs/ocr_jobs.py +++ b/extralit-server/src/extralit_server/jobs/ocr_jobs.py @@ -7,6 +7,7 @@ from rq import Retry, get_current_job from rq.decorators import job +from sqlalchemy import select from extralit_server.api.schemas.v1.document.metadata import DocumentProcessingMetadata, LayoutMetadata from extralit_server.contexts import files @@ -99,6 +100,9 @@ async def async_document_layout_job( ) async with AsyncSessionLocal() as db: + # Text extraction writes the same JSON column concurrently; serialize on the row so + # this read-modify-write does not clobber whatever it wrote. + await db.execute(select(Document.id).where(Document.id == document_id).with_for_update()) document = await db.get(Document, document_id) if document is not None: metadata = DocumentProcessingMetadata(**(document.metadata_ or {})) diff --git a/extralit-server/src/extralit_server/workflows/documents.py b/extralit-server/src/extralit_server/workflows/documents.py index 90f8ddf38..acd386117 100644 --- a/extralit-server/src/extralit_server/workflows/documents.py +++ b/extralit-server/src/extralit_server/workflows/documents.py @@ -82,26 +82,26 @@ async def create_document_workflow( }, ) - ocr_job_datas = [text_extraction_job_data] + analysis_jobs = group.enqueue_many(queue=DEFAULT_QUEUE, job_datas=[analysis_job_data]) + group.enqueue_many(queue=OCR_QUEUE, job_datas=[text_extraction_job_data]) if layout_parser: - ocr_job_datas.append( - OCR_QUEUE.prepare_data( - async_document_layout_job, - (document_id, s3_url, workspace_name, layout_parser), - timeout=1800, - job_id=f"document_layout_{document_id}", - meta={ - "document_id": str(document_id), - "reference": reference, - "workflow_step": "document_layout", - "workflow_id": str(workflow.id), - }, - ) + # Preprocessing rotates pages and overwrites the PDF at the same S3 path, so layout must + # run after it — otherwise the persisted bboxes describe a PDF nobody will render. + layout_job_data = OCR_QUEUE.prepare_data( + async_document_layout_job, + (document_id, s3_url, workspace_name, layout_parser), + timeout=1800, + job_id=f"document_layout_{document_id}", + depends_on=analysis_jobs[0] if analysis_jobs else None, + meta={ + "document_id": str(document_id), + "reference": reference, + "workflow_step": "document_layout", + "workflow_id": str(workflow.id), + }, ) - - group.enqueue_many(queue=DEFAULT_QUEUE, job_datas=[analysis_job_data]) - group.enqueue_many(queue=OCR_QUEUE, job_datas=ocr_job_datas) + group.enqueue_many(queue=OCR_QUEUE, job_datas=[layout_job_data]) # Step 6: Future table extraction job (conditional based on analysis results) # This will be added when table extraction is implemented diff --git a/extralit-server/tests/unit/api/handlers/v1/test_document_layout.py b/extralit-server/tests/unit/api/handlers/v1/test_document_layout.py index a2191cff9..aaa3aad11 100644 --- a/extralit-server/tests/unit/api/handlers/v1/test_document_layout.py +++ b/extralit-server/tests/unit/api/handlers/v1/test_document_layout.py @@ -9,6 +9,7 @@ from httpx import AsyncClient from sqlalchemy.ext.asyncio import AsyncSession +from extralit_server.constants import API_KEY_HEADER_NAME from extralit_server.contexts.ocr.docling_builder import ( LayoutBlock, PageContext, @@ -16,7 +17,13 @@ new_document, ) from extralit_server.models.database import Document -from tests.factories import DocumentFactory, WorkspaceFactory +from tests.factories import ( + AdminFactory, + AnnotatorFactory, + DocumentFactory, + WorkspaceFactory, + WorkspaceUserFactory, +) PAGE_WIDTH, PAGE_HEIGHT = 612.0, 792.0 @@ -47,7 +54,7 @@ def build_layout(): async def make_document(db: AsyncSession, layout_metadata: dict | None) -> Document: workspace = await WorkspaceFactory.create(name=f"ws-{uuid4().hex[:8]}") metadata = {"layout_metadata": layout_metadata} if layout_metadata else {} - return await DocumentFactory.create(workspace_id=workspace.id, metadata_=metadata) + return await DocumentFactory.create(workspace=workspace, metadata_=metadata) LAYOUT_METADATA = { @@ -198,3 +205,52 @@ async def test_requires_authentication(self, async_client: AsyncClient, db, load response = await async_client.get(f"/api/v1/documents/{document.id}/layout") assert response.status_code == 401 + + +@pytest.mark.asyncio +class TestDocumentLayoutAuthorization: + """This route returns document contents, so a role check alone is not enough.""" + + async def test_annotator_outside_the_workspace_is_denied(self, async_client: AsyncClient, db, load_layout): + document = await make_document(db, LAYOUT_METADATA) + outsider = await AnnotatorFactory.create() + + response = await async_client.get( + f"/api/v1/documents/{document.id}/layout", + headers={API_KEY_HEADER_NAME: outsider.api_key}, + ) + + assert response.status_code == 403 + + async def test_admin_outside_the_workspace_is_denied(self, async_client: AsyncClient, db, load_layout): + document = await make_document(db, LAYOUT_METADATA) + outsider = await AdminFactory.create() + + response = await async_client.get( + f"/api/v1/documents/{document.id}/layout", + headers={API_KEY_HEADER_NAME: outsider.api_key}, + ) + + assert response.status_code == 403 + + async def test_annotator_inside_the_workspace_is_allowed(self, async_client: AsyncClient, db, load_layout): + workspace = await WorkspaceFactory.create(name=f"ws-{uuid4().hex[:8]}") + document = await DocumentFactory.create(workspace=workspace, metadata_={"layout_metadata": LAYOUT_METADATA}) + member = await AnnotatorFactory.create() + await WorkspaceUserFactory.create(workspace_id=workspace.id, user_id=member.id) + + response = await async_client.get( + f"/api/v1/documents/{document.id}/layout", + headers={API_KEY_HEADER_NAME: member.api_key}, + ) + + assert response.status_code == 200 + + async def test_owner_is_allowed_across_workspaces( + self, async_client: AsyncClient, db, owner_auth_header, load_layout + ): + document = await make_document(db, LAYOUT_METADATA) + + response = await async_client.get(f"/api/v1/documents/{document.id}/layout", headers=owner_auth_header) + + assert response.status_code == 200 diff --git a/extralit-server/tests/unit/workflows/__init__.py b/extralit-server/tests/unit/workflows/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/extralit-server/tests/unit/workflows/test_document_workflow_layout.py b/extralit-server/tests/unit/workflows/test_document_workflow_layout.py new file mode 100644 index 000000000..64e4b9534 --- /dev/null +++ b/extralit-server/tests/unit/workflows/test_document_workflow_layout.py @@ -0,0 +1,102 @@ +"""Tests for how the layout job is sequenced into the document workflow.""" + +from unittest.mock import AsyncMock, MagicMock, patch +from uuid import uuid4 + +import pytest + +MODULE = "extralit_server.workflows.documents" + + +@pytest.fixture +def enqueued(): + """Capture prepare_data calls and enqueue_many batches without touching Redis.""" + calls = {"prepared": [], "batches": []} + + def prepare_data(func, args=None, **kwargs): + record = {"func": func, "args": args, **kwargs} + calls["prepared"].append(record) + return record + + def enqueue_many(queue=None, job_datas=None): + calls["batches"].append(job_datas) + return [MagicMock(name=f"job-{d.get('job_id')}", _data=d) for d in (job_datas or [])] + + group = MagicMock() + group.enqueue_many.side_effect = enqueue_many + + with ( + patch(f"{MODULE}.DEFAULT_QUEUE") as default_queue, + patch(f"{MODULE}.OCR_QUEUE") as ocr_queue, + patch(f"{MODULE}.Group", return_value=group), + patch(f"{MODULE}.AsyncSessionLocal") as session, + ): + default_queue.prepare_data.side_effect = prepare_data + ocr_queue.prepare_data.side_effect = prepare_data + db = MagicMock() + db.commit = AsyncMock() + db.refresh = AsyncMock() + session.return_value.__aenter__ = AsyncMock(return_value=db) + session.return_value.__aexit__ = AsyncMock(return_value=False) + yield calls + + +async def run_workflow(layout_parser=None): + from extralit_server.workflows.documents import create_document_workflow + + return await create_document_workflow( + document_id=uuid4(), + s3_url="/api/v1/file/ws/documents/doc.pdf", + reference="ref-1", + workspace_name="ws", + workspace_id=uuid4(), + layout_parser=layout_parser, + ) + + +def prepared_for(calls, step): + return next((c for c in calls["prepared"] if c.get("meta", {}).get("workflow_step") == step), None) + + +@pytest.mark.asyncio +class TestLayoutJobSequencing: + async def test_no_layout_job_is_enqueued_by_default(self, enqueued): + await run_workflow(layout_parser=None) + + assert prepared_for(enqueued, "document_layout") is None + + async def test_layout_job_is_enqueued_when_a_parser_is_named(self, enqueued): + await run_workflow(layout_parser="pdf_inspector") + + layout = prepared_for(enqueued, "document_layout") + assert layout is not None + assert layout["args"][3] == "pdf_inspector" + + async def test_layout_job_depends_on_preprocessing(self, enqueued): + # Preprocessing rotates pages and overwrites the PDF in place; running layout first + # would persist bboxes describing a PDF that nobody renders. + await run_workflow(layout_parser="pdf_inspector") + + layout = prepared_for(enqueued, "document_layout") + assert layout["depends_on"] is not None + + async def test_layout_depends_on_the_analysis_job_specifically(self, enqueued): + await run_workflow(layout_parser="pdf_inspector") + + analysis = prepared_for(enqueued, "analysis_and_preprocess") + layout = prepared_for(enqueued, "document_layout") + + assert layout["depends_on"]._data["job_id"] == analysis["job_id"] + + async def test_text_extraction_is_not_blocked_on_layout(self, enqueued): + await run_workflow(layout_parser="pdf_inspector") + + text_extraction = prepared_for(enqueued, "text_extraction") + assert text_extraction.get("depends_on") is None + + async def test_layout_is_enqueued_after_the_analysis_batch(self, enqueued): + await run_workflow(layout_parser="pdf_inspector") + + # The analysis job must already be enqueued before layout can depend on it. + batch_steps = [[d["meta"]["workflow_step"] for d in batch] for batch in enqueued["batches"]] + assert batch_steps.index(["analysis_and_preprocess"]) < batch_steps.index(["document_layout"]) From 28ae49bfb384af9af7a251ed078f031cdebede84 Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Sun, 16 Aug 2026 15:51:47 -0700 Subject: [PATCH 04/10] feat(deps): lance new_table_data_storage_version v2.2 --- extralit-server/pyproject.toml | 7 ++- .../extralit_server/index/lancedb_engine.py | 7 ++- extralit-server/uv.lock | 56 +++++++++++++------ 3 files changed, 49 insertions(+), 21 deletions(-) diff --git a/extralit-server/pyproject.toml b/extralit-server/pyproject.toml index 928aebdd9..0796e0d2b 100644 --- a/extralit-server/pyproject.toml +++ b/extralit-server/pyproject.toml @@ -76,8 +76,9 @@ dependencies = [ "ocrmypdf>=16.11.0", "pdf2image>=1.17.0", "opencv-python-headless>=4.11.0.86", - "pandera[io]>=0.20", - "lancedb>=0.34.0", + "pandera[io]>=0.32.0", + "lancedb>=0.37.1", + "pylance>=10.0.0", "duckdb>=1.5.4", "docling-core>=2.91.0,<3.0.0", "pyarrow>=23.0.1", @@ -91,7 +92,7 @@ postgresql = [ "asyncpg >= 0.30.0", ] # AGPL — must stay optional so the Apache-2.0 distribution never requires it -pymupdf = ["pymupdf4llm~=0.0.27"] +pymupdf = ["pymupdf4llm~=0.3.4"] [project.urls] homepage = "https://extralit.ai" diff --git a/extralit-server/src/extralit_server/index/lancedb_engine.py b/extralit-server/src/extralit_server/index/lancedb_engine.py index 720dda5ec..32a086411 100644 --- a/extralit-server/src/extralit_server/index/lancedb_engine.py +++ b/extralit-server/src/extralit_server/index/lancedb_engine.py @@ -35,6 +35,11 @@ def _sql_type_for(dtype: str) -> str: return _SQL_TYPE_BY_ARROW.get(arrow_type_for(dtype), "string") +# Lance file format 2.2 for tables created from here on; the async API exposes no per-table +# knob, and existing 2.0/2.1 tables keep their own version and stay readable. +_STORAGE_OPTIONS = {"new_table_data_storage_version": "2.2"} + + # Exact FTS totals are computed by materializing the match set; beyond this many matches # the reported total saturates at the ceiling (extraction tables are far smaller today). _FTS_TOTAL_CEILING = 10_000 @@ -96,7 +101,7 @@ async def new_instance(cls) -> "LanceIndexEngine": async def _conn(self) -> Any: if self._db is None: - self._db = await lancedb.connect_async(self._uri) + self._db = await lancedb.connect_async(self._uri, storage_options=_STORAGE_OPTIONS) return self._db async def close(self) -> None: diff --git a/extralit-server/uv.lock b/extralit-server/uv.lock index 0bf2e22bb..8dfad5d0d 100644 --- a/extralit-server/uv.lock +++ b/extralit-server/uv.lock @@ -1140,6 +1140,7 @@ dependencies = [ { name = "pyarrow" }, { name = "pydantic" }, { name = "pydantic-settings" }, + { name = "pylance" }, { name = "python-jose", extra = ["cryptography"] }, { name = "python-multipart" }, { name = "pyyaml" }, @@ -1199,7 +1200,7 @@ requires-dist = [ { name = "httpx", specifier = ">=0.27.0" }, { name = "huggingface-hub", specifier = ">=0.34.0" }, { name = "jinja2", specifier = ">=3.1.4" }, - { name = "lancedb", specifier = ">=0.34.0" }, + { name = "lancedb", specifier = ">=0.37.1" }, { name = "lazy-loader", specifier = ">=0.4" }, { name = "litellm", specifier = ">=1.80.0,<=1.82.6" }, { name = "numpy", specifier = ">=2.0.0,<3.0.0" }, @@ -1208,7 +1209,7 @@ requires-dist = [ { name = "opencv-python-headless", specifier = ">=4.11.0.86" }, { name = "opensearch-py", specifier = ">=2.0.0" }, { name = "packaging", specifier = ">=23.2" }, - { name = "pandera", extras = ["io"], specifier = ">=0.20" }, + { name = "pandera", extras = ["io"], specifier = ">=0.32.0" }, { name = "pdf-inspector", specifier = ">=1.14.2" }, { name = "pdf2image", specifier = ">=1.17.0" }, { name = "pillow", specifier = ">=10.1.0" }, @@ -1217,7 +1218,8 @@ requires-dist = [ { name = "pyarrow", specifier = ">=23.0.1" }, { name = "pydantic", specifier = ">=2.9.0" }, { name = "pydantic-settings", specifier = ">=2.6.0" }, - { name = "pymupdf4llm", marker = "extra == 'pymupdf'", specifier = "~=0.0.27" }, + { name = "pylance", specifier = ">=10.0.0" }, + { name = "pymupdf4llm", marker = "extra == 'pymupdf'", specifier = "~=0.3.4" }, { name = "python-jose", extras = ["cryptography"], specifier = ">=3.3.0" }, { name = "python-multipart", specifier = ">=0.0.16" }, { name = "pyyaml", specifier = ">=5.4.1,<6.1.0" }, @@ -1881,19 +1883,19 @@ wheels = [ [[package]] name = "lance-namespace" -version = "0.9.0" +version = "0.8.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "lance-namespace-urllib3-client" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/00/81/4cf8d0412e1f37b2bfa70d0aeb9c7ae4ab73607534e44d60b55efb485306/lance_namespace-0.9.0.tar.gz", hash = "sha256:f738b641cc615b17323baa4eb47900f184688739ee3d2ea9fe39396b9588e53d", size = 11637, upload-time = "2026-07-01T07:42:41.78Z" } +sdist = { url = "https://files.pythonhosted.org/packages/af/12/f7ab93b29be3edbf5fc3610714bf2d06088e7f4524bfb38dfd6852458b08/lance_namespace-0.8.6.tar.gz", hash = "sha256:18232e721c8188145f4ec9389cc2dfbeeabf54a619d94885ea1b3375bee9f4af", size = 11529, upload-time = "2026-06-12T17:36:41.651Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/fe/f38747c9610ade83dd9a99a0470b9432b6f21ce4e2bb5524edbe66f626fd/lance_namespace-0.9.0-py3-none-any.whl", hash = "sha256:f785ff10927e4ce0db69986576670fedd37f8a33521e8a4630c6be22db8061b2", size = 13501, upload-time = "2026-07-01T07:42:39.372Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1b/5b1668ee2dc8910965f390640359112a31157092fcf8e000b89c79b58708/lance_namespace-0.8.6-py3-none-any.whl", hash = "sha256:571eae34f9aad70e5b05020416c2860889b9ec82993ccd0eb015e7b39c3ea309", size = 13383, upload-time = "2026-06-12T17:36:43.456Z" }, ] [[package]] name = "lance-namespace-urllib3-client" -version = "0.9.0" +version = "0.8.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, @@ -1901,14 +1903,14 @@ dependencies = [ { name = "typing-extensions" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d3/c3/32d0e2618549ace857c80a457e5915ef3e1145661baff876c8a5ec27be5b/lance_namespace_urllib3_client-0.9.0.tar.gz", hash = "sha256:cf796fa5307fa4dde91fe4bec2af28b90ba79191852d4394e8fe44276538e40f", size = 235805, upload-time = "2026-07-01T07:42:42.563Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/80/fb224b4a89c1c1638cde949cb6cce6c3aca7759effbfea46a3d9c3960b21/lance_namespace_urllib3_client-0.8.6.tar.gz", hash = "sha256:b6fb1d306e74a7576e5309919020be744527de484a63dbf5eed10f8b368548df", size = 228772, upload-time = "2026-06-12T17:36:42.609Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cc/ab/c8754da0a1efc817f8480100cfe12b7e04034df834759de8ecf02beff3cc/lance_namespace_urllib3_client-0.9.0-py3-none-any.whl", hash = "sha256:be819c8cffb1e460a3a504dbf52d1ca009560a48e7202b8c4279998e4adf9fe4", size = 405586, upload-time = "2026-07-01T07:42:40.503Z" }, + { url = "https://files.pythonhosted.org/packages/c5/90/1e27de15cd1b16785a1c7312beb0a59e75c8344a815f600f58173a565bd1/lance_namespace_urllib3_client-0.8.6-py3-none-any.whl", hash = "sha256:9d78249c3fb15aa3d15d668f78f04a275af3d08d800a7027492f37996ac4968b", size = 369950, upload-time = "2026-06-12T17:36:40.438Z" }, ] [[package]] name = "lancedb" -version = "0.34.0" +version = "0.37.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "deprecation" }, @@ -1922,10 +1924,10 @@ dependencies = [ { name = "tqdm" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/df/f7/5262b9aa593f790757163c0165ab0da1dda054758901bea7e4f02c9cb633/lancedb-0.34.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:c462f2e6f933cad659fd0179394eaab578acbc9151fe2ef41bc29b36ecca5058", size = 52654213, upload-time = "2026-07-02T17:13:31.102Z" }, - { url = "https://files.pythonhosted.org/packages/69/99/05ea0d32229ebea695193ff20c15d6ecae25785ad82a9d4723d98832a284/lancedb-0.34.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:48829e88e708947d0520454ab9e4f8efa35f3e3626469eadd3a6e061b89cb223", size = 55434501, upload-time = "2026-07-02T17:13:34.81Z" }, - { url = "https://files.pythonhosted.org/packages/cd/4e/4325c13d5afa93c466428a5a0f168ad4d96f5eb4a77bbe7c5100d39c9897/lancedb-0.34.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:05ba8a5b58e064edfbe5be71b1abf2e411b4eaf295d1a173dcb1a55c5bfb5285", size = 58659359, upload-time = "2026-07-02T17:13:38.424Z" }, - { url = "https://files.pythonhosted.org/packages/d9/5d/8ca165f1386caf6c4d1c515afd52f345b66432264eecfdfb7fd33eefd9af/lancedb-0.34.0-cp39-abi3-win_amd64.whl", hash = "sha256:51cbc11808f9e3332819b9367c975b3a888541447a8e7bea09c57c852a279153", size = 63530726, upload-time = "2026-07-02T17:13:41.612Z" }, + { url = "https://files.pythonhosted.org/packages/23/2f/4ddcab82bb618c6c8de00725f3cf59585dcb9040964dde13cb0cae6ed3cd/lancedb-0.37.1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c15c46f23cf6959c79fb93cdba2c76536cf784d3134386662da03dc6ccac3c26", size = 58474767, upload-time = "2026-08-10T10:41:45.675Z" }, + { url = "https://files.pythonhosted.org/packages/e5/5e/dac0fd9478a21685444f23e6ec937babf4ff48b1616b68e8137778d6ecb9/lancedb-0.37.1-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:35d872d920cbfdc3771fbcd33f2c63bff6ff7203d6d2ba8fe4330e98eb859d12", size = 61666580, upload-time = "2026-08-10T10:41:49.535Z" }, + { url = "https://files.pythonhosted.org/packages/1f/fa/ee1cdb1e904872d75fa0ff44ad35cd23494e810299c33e68de0687de7a71/lancedb-0.37.1-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:86597f4dbc51a33a07341550dc77d21a1ddd1f7539266eda5bb64c4f3bd11cca", size = 64802395, upload-time = "2026-08-10T10:41:52.969Z" }, + { url = "https://files.pythonhosted.org/packages/42/55/65c69307373b7f05dea38465b7d3836c86390f0ebc79b5c56d2c0919f229/lancedb-0.37.1-cp310-abi3-win_amd64.whl", hash = "sha256:488eca15361dfc34439500c9e2607c4fb2b8bf190fa1003bd54b1d6eb40e0316", size = 70994296, upload-time = "2026-08-10T10:41:56.609Z" }, ] [[package]] @@ -3345,6 +3347,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/7a/8dd906bd22e79e47397a61742927f6747fe93242ef86645ee9092e610244/pyjwt-2.12.1-py3-none-any.whl", hash = "sha256:28ca37c070cad8ba8cd9790cd940535d40274d22f80ab87f3ac6a713e6e8454c", size = 29726, upload-time = "2026-03-13T19:27:35.677Z" }, ] +[[package]] +name = "pylance" +version = "10.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lance-namespace" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.4.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "pyarrow" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/b2/c81de196076c4c8d768f485324a0043e113ce0950a339977d83fee9783ef/pylance-10.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:d4bba56ae829202b7e9cdc82c92c00a4b52f03e679dc3edfad1db09d1285be2e", size = 69279797, upload-time = "2026-08-07T18:25:24.812Z" }, + { url = "https://files.pythonhosted.org/packages/d2/e9/af671a6225740bd70c1e70bd83fa08091d628b960397cbecc477f16aaaf1/pylance-10.0.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:489b944827c0271e16a62b4006f8c75acc3bd7fbc381f35388a2afaaae38d438", size = 72775384, upload-time = "2026-08-07T18:31:29.686Z" }, + { url = "https://files.pythonhosted.org/packages/5d/22/e07194195bb3bbdf062b0c31690fc92fccb2686d5e768efa1dc379a93350/pylance-10.0.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:018efe7d437d326b9049c1223458bd955e85e48509b4b0bbf8b2d3d94075dbff", size = 76632194, upload-time = "2026-08-07T18:44:29.001Z" }, + { url = "https://files.pythonhosted.org/packages/f2/71/ed9956cf657e86a5fee5d5ddcfa7bfed0d7c2968bee39ed85d12629d8c93/pylance-10.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:254bad5d765c14db6c4eddd5323133dee3c76ff6d40c4777afe0b0e91d4c04d2", size = 72804222, upload-time = "2026-08-07T18:31:24.264Z" }, + { url = "https://files.pythonhosted.org/packages/94/96/de449c246b2892df9d5a0af775b79a061e9897e6942bcee5c4ab6d6071f8/pylance-10.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9f0c089e30389d9b7765a4c16f3d9325b05ed4fbf6cc2bbc9418292983472e54", size = 76602015, upload-time = "2026-08-07T18:46:27.724Z" }, + { url = "https://files.pythonhosted.org/packages/53/81/4a5a9072b6d68c4dbb8c7b5530a381c7a6bea4ba92f9eca58ac885142722/pylance-10.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:9fecf46e6835dc64b2d71767d5eab4f73787543a852432e9ba8ecf8527b3cdab", size = 82799247, upload-time = "2026-08-07T18:47:46.855Z" }, +] + [[package]] name = "pymupdf" version = "1.28.2" @@ -3365,14 +3386,15 @@ wheels = [ [[package]] name = "pymupdf4llm" -version = "0.0.27" +version = "0.3.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pymupdf" }, + { name = "tabulate" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/03/99/2634b56ff2b68558e742c6e10cd798ed3b5b5af6cc659454b9f37e7b4ecf/pymupdf4llm-0.0.27.tar.gz", hash = "sha256:35cc8bd6e0968bc1300b1fe4500f52e47da6b570af19af6c193e5fb6ee367008", size = 29368, upload-time = "2025-07-19T11:52:01.059Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/9b/97c4fad642f0147af3e884126d80635a1fae02c836fde9c22f0efc053254/pymupdf4llm-0.3.4.tar.gz", hash = "sha256:48d396a5fb3c14351493c7f1dd25b2a843efdbdc4526e489ee100643a2cebec1", size = 74956, upload-time = "2026-02-14T10:22:24.423Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/c8/7eed2e902b61574b15b295017ddb5738c4970aa9ab76903fbaace28a522e/pymupdf4llm-0.0.27-py3-none-any.whl", hash = "sha256:2eaaf9419c35520efda38f3806a276f2ec6cd29564fbb60a5c9c53a49fedb13c", size = 30055, upload-time = "2025-07-19T11:52:04.089Z" }, + { url = "https://files.pythonhosted.org/packages/2b/67/4394de2e5967d80a9b01ec323049163410b3553185c42dea0c346fa27a41/pymupdf4llm-0.3.4-py3-none-any.whl", hash = "sha256:0517492f82af978541162ade20fc54649cdca52acd478e33b97cb6171d69956f", size = 78669, upload-time = "2026-02-14T10:22:27.096Z" }, ] [[package]] From 1dcde5a0ca7be122ddcca18b704c4477b25941e8 Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Sun, 16 Aug 2026 16:33:31 -0700 Subject: [PATCH 05/10] fix(workflows): sequence dependents after preprocessing and stop stale runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Text extraction and layout now depend on the preprocessing job with Dependency(allow_failure=True): preprocessing rewrites the PDF at the same S3 key as its last step and writes the margins both readers need, while rotation itself is best effort — without allow_failure a failed triage would strand both dependents in DEFERRED forever. Retry and result_ttl move onto Queue.prepare_data(); the @job decorator values are inert on this path, so finished jobs expired after RQ's 500s default and the derived workflow status decayed back to pending. Job ids carry the run suffix, so a forced restart no longer collides with the previous run's ids, and POST /workflows/start?force=true stops started jobs and cancels pending ones from the old group before enqueueing the new one. --- .../api/handlers/v1/workflows.py | 5 ++ .../src/extralit_server/contexts/workflows.py | 43 ++++++++++- .../extralit_server/workflows/documents.py | 35 ++++++--- .../integration/test_rq_groups_workflow.py | 4 + .../test_document_workflow_layout.py | 52 +++++++++++-- .../unit/workflows/test_stop_workflow_jobs.py | 73 +++++++++++++++++++ 6 files changed, 196 insertions(+), 16 deletions(-) create mode 100644 extralit-server/tests/unit/workflows/test_stop_workflow_jobs.py diff --git a/extralit-server/src/extralit_server/api/handlers/v1/workflows.py b/extralit-server/src/extralit_server/api/handlers/v1/workflows.py index 20140f453..c630d92e0 100644 --- a/extralit-server/src/extralit_server/api/handlers/v1/workflows.py +++ b/extralit-server/src/extralit_server/api/handlers/v1/workflows.py @@ -21,6 +21,7 @@ get_workflow_status, get_workflow_statuses_by_reference, restart_failed_jobs_in_workflow, + stop_workflow_jobs, ) from extralit_server.database import get_async_db from extralit_server.jobs.queues import REDIS_CONNECTION @@ -70,6 +71,10 @@ async def start_workflow( detail=f"Workflow already exists for document {request.document_id}. Use force=true to restart.", ) + if existing_workflow: + # The previous run writes to the same S3 keys and Lance rows as the new one. + stop_workflow_jobs(existing_workflow.group_id) + await create_document_workflow( document_id=request.document_id, s3_url=document.url, diff --git a/extralit-server/src/extralit_server/contexts/workflows.py b/extralit-server/src/extralit_server/contexts/workflows.py index 9f2966b90..50bc7b8a9 100644 --- a/extralit-server/src/extralit_server/contexts/workflows.py +++ b/extralit-server/src/extralit_server/contexts/workflows.py @@ -4,9 +4,10 @@ from typing import Any, Optional from uuid import UUID +from rq.command import send_stop_job_command from rq.exceptions import NoSuchJobError from rq.group import Group -from rq.job import Job +from rq.job import Job, JobStatus from sqlalchemy.ext.asyncio import AsyncSession from extralit_server.jobs.queues import REDIS_CONNECTION @@ -510,6 +511,46 @@ def get_failed_jobs_in_group(group_id: str) -> list[dict[str, Any]]: return [] +def stop_workflow_jobs(group_id: str) -> list[str]: + """ + Stop running jobs and cancel pending ones for a workflow group. + + Best effort: a job that has already finished, expired or vanished is skipped, and one + failure never aborts the sweep. Used before a forced re-run so the previous run cannot + keep writing artifacts underneath the new one. + + Args: + group_id: RQ group name of the run to stop + + Returns: + Ids of the jobs that were stopped or cancelled + """ + try: + group = Group.fetch(name=group_id, connection=REDIS_CONNECTION) + jobs = group.get_jobs() + except Exception as e: + _LOGGER.warning(f"Group {group_id} not found or expired, nothing to stop: {e}") + return [] + + stopped: list[str] = [] + for job in jobs: + try: + status = job.get_status(refresh=True) + if status == JobStatus.STARTED: + send_stop_job_command(connection=REDIS_CONNECTION, job_id=job.id) + elif status in (JobStatus.QUEUED, JobStatus.DEFERRED, JobStatus.SCHEDULED): + job.cancel() + else: + continue + stopped.append(job.id) + except Exception as e: + _LOGGER.warning(f"Failed to stop job {job.id} in group {group_id}: {e}") + + if stopped: + _LOGGER.info(f"Stopped {len(stopped)} jobs from previous workflow group {group_id}") + return stopped + + async def restart_failed_jobs_in_workflow(db: AsyncSession, workflow: DocumentWorkflow) -> dict[str, Any]: """ Restart failed jobs in the workflow group. diff --git a/extralit-server/src/extralit_server/workflows/documents.py b/extralit-server/src/extralit_server/workflows/documents.py index acd386117..17f1a8767 100644 --- a/extralit-server/src/extralit_server/workflows/documents.py +++ b/extralit-server/src/extralit_server/workflows/documents.py @@ -1,7 +1,9 @@ import logging from uuid import UUID, uuid4 +from rq import Retry from rq.group import Group +from rq.job import Dependency from extralit_server.database import AsyncSessionLocal from extralit_server.jobs.document_jobs import analysis_and_preprocess_job @@ -11,6 +13,9 @@ _LOGGER = logging.getLogger(__name__) +# RQ's 500s default drops finished jobs from the group, decaying derived workflow status to pending. +JOB_RESULT_TTL = 24 * 3600 + async def create_document_workflow( document_id: UUID, @@ -37,7 +42,8 @@ async def create_document_workflow( Returns: Dictionary containing workflow_id and group_id for tracking """ - group_id = f"document_workflow_{document_id}_{uuid4().hex[:8]}" + run_suffix = uuid4().hex[:8] + group_id = f"document_workflow_{document_id}_{run_suffix}" group = Group(REDIS_CONNECTION, name=group_id) # Step 3: Create DocumentWorkflow record for tracking @@ -55,12 +61,14 @@ async def create_document_workflow( await db.commit() await db.refresh(workflow) - # Step 4: Prepare jobs using Queue.prepare_data() + # Step 4: Prepare jobs using Queue.prepare_data(); the @job decorator kwargs are inert here. analysis_job_data = DEFAULT_QUEUE.prepare_data( analysis_and_preprocess_job, (document_id, s3_url, reference, workspace_name), timeout=600, - job_id=f"analysis_preprocess_{document_id}", + job_id=f"analysis_preprocess_{document_id}_{run_suffix}", + retry=Retry(max=3, interval=[10, 30, 60]), + result_ttl=JOB_RESULT_TTL, meta={ "document_id": str(document_id), "reference": reference, @@ -69,11 +77,21 @@ async def create_document_workflow( }, ) + analysis_jobs = group.enqueue_many(queue=DEFAULT_QUEUE, job_datas=[analysis_job_data]) + + # Preprocessing rewrites the PDF at the same S3 key as its last step and writes the margins + # every downstream reader needs, so dependents wait on it. Rotation is best effort, hence + # allow_failure: otherwise a failed triage strands them in DEFERRED forever. + on_analysis = Dependency(jobs=[analysis_jobs[0]], allow_failure=True) if analysis_jobs else None + text_extraction_job_data = OCR_QUEUE.prepare_data( "extralit_ocr.jobs.pymupdf_to_markdown_job", (document_id, s3_url, s3_url.split("/")[-1], {}, workspace_name), timeout=900, - job_id=f"text_extraction_{document_id}", + job_id=f"text_extraction_{document_id}_{run_suffix}", + depends_on=on_analysis, + retry=Retry(max=2, interval=[30, 60]), + result_ttl=JOB_RESULT_TTL, meta={ "document_id": str(document_id), "reference": reference, @@ -82,18 +100,17 @@ async def create_document_workflow( }, ) - analysis_jobs = group.enqueue_many(queue=DEFAULT_QUEUE, job_datas=[analysis_job_data]) group.enqueue_many(queue=OCR_QUEUE, job_datas=[text_extraction_job_data]) if layout_parser: - # Preprocessing rotates pages and overwrites the PDF at the same S3 path, so layout must - # run after it — otherwise the persisted bboxes describe a PDF nobody will render. layout_job_data = OCR_QUEUE.prepare_data( async_document_layout_job, (document_id, s3_url, workspace_name, layout_parser), timeout=1800, - job_id=f"document_layout_{document_id}", - depends_on=analysis_jobs[0] if analysis_jobs else None, + job_id=f"document_layout_{document_id}_{run_suffix}", + depends_on=on_analysis, + retry=Retry(max=2, interval=[30, 60]), + result_ttl=JOB_RESULT_TTL, meta={ "document_id": str(document_id), "reference": reference, diff --git a/extralit-server/tests/integration/test_rq_groups_workflow.py b/extralit-server/tests/integration/test_rq_groups_workflow.py index b847cb01d..d410c380b 100644 --- a/extralit-server/tests/integration/test_rq_groups_workflow.py +++ b/extralit-server/tests/integration/test_rq_groups_workflow.py @@ -121,6 +121,10 @@ async def test_create_document_workflow_with_rq_groups( # Mock RQ Group mock_group = MagicMock(spec=Group) mock_group.name = f"document_workflow_{test_document.id}_12345678" + # Dependents are wired with rq.job.Dependency, which type-checks the enqueued jobs. + mock_group.enqueue_many.side_effect = lambda queue=None, job_datas=None: [ + Job(id=data["job_id"], connection=MagicMock()) for data in (job_datas or []) + ] with patch("extralit_server.workflows.documents.Group", return_value=mock_group): group = await create_document_workflow( diff --git a/extralit-server/tests/unit/workflows/test_document_workflow_layout.py b/extralit-server/tests/unit/workflows/test_document_workflow_layout.py index 64e4b9534..d5d39f3d5 100644 --- a/extralit-server/tests/unit/workflows/test_document_workflow_layout.py +++ b/extralit-server/tests/unit/workflows/test_document_workflow_layout.py @@ -4,6 +4,7 @@ from uuid import uuid4 import pytest +from rq.job import Job MODULE = "extralit_server.workflows.documents" @@ -20,7 +21,8 @@ def prepare_data(func, args=None, **kwargs): def enqueue_many(queue=None, job_datas=None): calls["batches"].append(job_datas) - return [MagicMock(name=f"job-{d.get('job_id')}", _data=d) for d in (job_datas or [])] + # Real Job instances: rq.job.Dependency type-checks what it is given. + return [Job(id=d.get("job_id"), connection=MagicMock()) for d in (job_datas or [])] group = MagicMock() group.enqueue_many.side_effect = enqueue_many @@ -41,11 +43,11 @@ def enqueue_many(queue=None, job_datas=None): yield calls -async def run_workflow(layout_parser=None): +async def run_workflow(layout_parser=None, document_id=None): from extralit_server.workflows.documents import create_document_workflow return await create_document_workflow( - document_id=uuid4(), + document_id=document_id or uuid4(), s3_url="/api/v1/file/ws/documents/doc.pdf", reference="ref-1", workspace_name="ws", @@ -86,13 +88,23 @@ async def test_layout_depends_on_the_analysis_job_specifically(self, enqueued): analysis = prepared_for(enqueued, "analysis_and_preprocess") layout = prepared_for(enqueued, "document_layout") - assert layout["depends_on"]._data["job_id"] == analysis["job_id"] + assert layout["depends_on"].dependencies[0].id == analysis["job_id"] - async def test_text_extraction_is_not_blocked_on_layout(self, enqueued): + async def test_text_extraction_depends_on_preprocessing(self, enqueued): + # Text extraction reads margins written by preprocessing and must not race the PDF rewrite. await run_workflow(layout_parser="pdf_inspector") + analysis = prepared_for(enqueued, "analysis_and_preprocess") text_extraction = prepared_for(enqueued, "text_extraction") - assert text_extraction.get("depends_on") is None + + assert text_extraction["depends_on"].dependencies[0].id == analysis["job_id"] + + async def test_dependents_are_not_stranded_when_preprocessing_fails(self, enqueued): + # Rotation is best effort; without allow_failure RQ leaves dependents DEFERRED forever. + await run_workflow(layout_parser="pdf_inspector") + + for step in ("text_extraction", "document_layout"): + assert prepared_for(enqueued, step)["depends_on"].allow_failure is True async def test_layout_is_enqueued_after_the_analysis_batch(self, enqueued): await run_workflow(layout_parser="pdf_inspector") @@ -100,3 +112,31 @@ async def test_layout_is_enqueued_after_the_analysis_batch(self, enqueued): # The analysis job must already be enqueued before layout can depend on it. batch_steps = [[d["meta"]["workflow_step"] for d in batch] for batch in enqueued["batches"]] assert batch_steps.index(["analysis_and_preprocess"]) < batch_steps.index(["document_layout"]) + + +@pytest.mark.asyncio +class TestJobRetentionAndIdentity: + async def test_every_job_carries_retry_and_a_long_result_ttl(self, enqueued): + # @job decorator values are inert under Queue.prepare_data(); without these the default + # 500s result TTL expires finished jobs and the derived workflow status decays to pending. + await run_workflow(layout_parser="pdf_inspector") + + assert enqueued["prepared"] + for prepared in enqueued["prepared"]: + assert prepared["retry"] is not None + assert prepared["result_ttl"] >= 6 * 3600 + + async def test_job_ids_are_unique_per_workflow_run(self, enqueued): + document_id = uuid4() + + await run_workflow(layout_parser="pdf_inspector", document_id=document_id) + first = {c["meta"]["workflow_step"]: c["job_id"] for c in enqueued["prepared"]} + enqueued["prepared"].clear() + + await run_workflow(layout_parser="pdf_inspector", document_id=document_id) + second = {c["meta"]["workflow_step"]: c["job_id"] for c in enqueued["prepared"]} + + assert first.keys() == second.keys() + for step, job_id in first.items(): + assert str(document_id) in job_id + assert job_id != second[step] diff --git a/extralit-server/tests/unit/workflows/test_stop_workflow_jobs.py b/extralit-server/tests/unit/workflows/test_stop_workflow_jobs.py new file mode 100644 index 000000000..a7f392557 --- /dev/null +++ b/extralit-server/tests/unit/workflows/test_stop_workflow_jobs.py @@ -0,0 +1,73 @@ +"""Tests for stopping a previous workflow run before a forced restart.""" + +from unittest.mock import MagicMock, patch + +import pytest +from rq.job import Job, JobStatus + +MODULE = "extralit_server.contexts.workflows" + + +def make_job(job_id, status): + job = MagicMock(spec=Job) + job.id = job_id + job.get_status.return_value = status + return job + + +@pytest.fixture +def group(): + with patch(f"{MODULE}.Group.fetch") as fetch: + fetch.return_value = MagicMock(get_jobs=MagicMock(return_value=[])) + yield fetch.return_value + + +class TestStopWorkflowJobs: + def test_started_jobs_are_stopped_and_queued_jobs_cancelled(self, group): + from extralit_server.contexts.workflows import stop_workflow_jobs + + started = make_job("started", JobStatus.STARTED) + queued = make_job("queued", JobStatus.QUEUED) + deferred = make_job("deferred", JobStatus.DEFERRED) + group.get_jobs.return_value = [started, queued, deferred] + + with patch(f"{MODULE}.send_stop_job_command") as stop: + stopped = stop_workflow_jobs("group-1") + + stop.assert_called_once() + assert stop.call_args.kwargs["job_id"] == "started" + started.cancel.assert_not_called() + queued.cancel.assert_called_once() + deferred.cancel.assert_called_once() + assert set(stopped) == {"started", "queued", "deferred"} + + def test_finished_jobs_are_left_alone(self, group): + from extralit_server.contexts.workflows import stop_workflow_jobs + + finished = make_job("finished", JobStatus.FINISHED) + failed = make_job("failed", JobStatus.FAILED) + group.get_jobs.return_value = [finished, failed] + + with patch(f"{MODULE}.send_stop_job_command") as stop: + assert stop_workflow_jobs("group-1") == [] + + stop.assert_not_called() + finished.cancel.assert_not_called() + failed.cancel.assert_not_called() + + def test_a_failing_job_does_not_abort_the_rest(self, group): + from extralit_server.contexts.workflows import stop_workflow_jobs + + broken = make_job("broken", JobStatus.QUEUED) + broken.cancel.side_effect = RuntimeError("gone") + healthy = make_job("healthy", JobStatus.QUEUED) + group.get_jobs.return_value = [broken, healthy] + + with patch(f"{MODULE}.send_stop_job_command"): + assert stop_workflow_jobs("group-1") == ["healthy"] + + def test_missing_group_is_not_an_error(self): + from extralit_server.contexts.workflows import stop_workflow_jobs + + with patch(f"{MODULE}.Group.fetch", side_effect=RuntimeError("expired")): + assert stop_workflow_jobs("group-1") == [] From e596614cc1b285c9076b85705421bcde0a0d6cbb Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Sun, 16 Aug 2026 17:00:32 -0700 Subject: [PATCH 06/10] feat(ocr): aggregate layout into workspace-scoped Lance datasets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-document Parquet sidecars were shaped for writing one document at a time: a corpus question meant globbing thousands of ~7.6 KB objects, and a re-parse left both vintages' rows behind because nothing superseded them. Layout rows now go into one `layout/items.lance` and `layout/pages.lance` per workspace. A document's rows are deleted before they are appended, so a re-parse replaces rather than doubles, and the rows are queryable the moment the job commits — `duckdb_connection([workspace, ...])` exposes both as views with projection and filter pushdown. Because replacing is a delete commit plus an append commit, every writer holds a Redis lock keyed on the resolved workspace root; Lance's own commit conflict detection is the belt to that lock's braces. The layout job takes the lock, confirms the document row still exists, writes, releases, and only then updates `documents.metadata_` — through `update_processing_metadata`, which all writers of that JSON column now share, so no job can clobber another's slice. Where a workspace lives is resolved in exactly one place, `files.workspace_root`, which PDFs, thumbnails, layout JSON and the Lance roots all address through, so switching to one bucket with `{org}/{workspace}/` prefixes later is a change to that function rather than to every call site. Bucket versioning gets a lifecycle rule expiring noncurrent `layout/` versions, replacing the dead EXCLUDED_VERSIONING_PREFIXES. Measured (scripts/bench_layout_store.py, 200 docs x 250 items): local disk 24 ms/doc replace, 8 ms aggregate, 10 ms single-document read; MinIO 189 ms/doc replace (delete+append over four commits) plus an amortized 470 ms compaction, 16 ms aggregate, 31 ms single-document read. --- extralit-server/CHANGELOG.md | 5 +- extralit-server/scripts/bench_layout_store.py | 139 ++++++++++ .../api/handlers/v1/documents.py | 3 +- .../api/schemas/v1/document/metadata.py | 6 +- .../contexts/document/metadata.py | 41 +++ .../src/extralit_server/contexts/files.py | 71 ++++- .../contexts/ocr/layout_store.py | 261 ++++++++++++++++++ .../extralit_server/contexts/ocr/storage.py | 97 +++---- .../src/extralit_server/jobs/document_jobs.py | 23 +- .../src/extralit_server/jobs/ocr_jobs.py | 27 +- .../unit/api/handlers/v1/test_documents.py | 48 +++- .../tests/unit/contexts/document/__init__.py | 0 .../unit/contexts/document/test_metadata.py | 56 ++++ .../unit/contexts/ocr/test_layout_store.py | 260 +++++++++++++++++ .../tests/unit/contexts/ocr/test_storage.py | 90 ++++++ .../unit/contexts/test_files_artifacts.py | 46 +++ .../tests/unit/jobs/test_document_jobs.py | 6 +- .../tests/unit/jobs/test_ocr_jobs.py | 85 ++++++ 18 files changed, 1172 insertions(+), 92 deletions(-) create mode 100644 extralit-server/scripts/bench_layout_store.py create mode 100644 extralit-server/src/extralit_server/contexts/document/metadata.py create mode 100644 extralit-server/src/extralit_server/contexts/ocr/layout_store.py create mode 100644 extralit-server/tests/unit/contexts/document/__init__.py create mode 100644 extralit-server/tests/unit/contexts/document/test_metadata.py create mode 100644 extralit-server/tests/unit/contexts/ocr/test_layout_store.py create mode 100644 extralit-server/tests/unit/contexts/ocr/test_storage.py create mode 100644 extralit-server/tests/unit/contexts/test_files_artifacts.py create mode 100644 extralit-server/tests/unit/jobs/test_ocr_jobs.py diff --git a/extralit-server/CHANGELOG.md b/extralit-server/CHANGELOG.md index 651b0d98d..548738f2d 100644 --- a/extralit-server/CHANGELOG.md +++ b/extralit-server/CHANGELOG.md @@ -20,12 +20,15 @@ These are the section headers that we use: - Refactor document analysis and preprocessing job to support asynchronous s3 IO operations and large file processing. - Adopted `DoclingDocument` as the canonical internal layout model, so every extracted item carries a `page_no` + `bbox` + `charspan` provenance triple that an extraction record can cite. - Added swappable layout parsers behind `contexts/ocr/parsers`: `pdf_inspector` (MIT, required, the default) and `pymupdf` (AGPL, optional `pymupdf` extra, the only one yielding per-cell table geometry). -- Added `async_document_layout_job` on the OCR queue, persisting layout as canonical JSON plus a columnar Parquet projection in object storage. +- Added `async_document_layout_job` on the OCR queue, persisting layout as canonical JSON plus a columnar projection in the workspace's `layout/items.lance` and `layout/pages.lance` datasets, queryable through `LayoutStore` and its DuckDB views as soon as the job commits. - Added `GET /documents/{document_id}/layout`, with `pages` and `labels` filters, plus the matching frontend `DocumentLayout` entities and `DocumentRepository.getDocumentLayout`. - Added `layout_parser` to `POST /workflows/start` to trigger layout extraction as part of the document workflow. ### Changed +- Document workflow dependents (text extraction, layout) now wait on the analysis/preprocessing job with `allow_failure`, and carry their retry policy and a 24 h result TTL through `Queue.prepare_data()` — the `@job` decorator values never applied on that path. +- `POST /workflows/start?force=true` stops the previous run's jobs before enqueueing new ones, whose ids now carry the run suffix. +- Deleting a document removes its PDF, thumbnail, layout JSON and layout rows together, through `files.delete_document_artifacts`. - Rewrote `contexts/ocr/{text,tables,figures}` as single-item appenders driven by one ordered pass, fixing text inside tables and figures being emitted twice and restoring geometric reading order. ### Removed diff --git a/extralit-server/scripts/bench_layout_store.py b/extralit-server/scripts/bench_layout_store.py new file mode 100644 index 000000000..19f3f59d9 --- /dev/null +++ b/extralit-server/scripts/bench_layout_store.py @@ -0,0 +1,139 @@ +"""Compare per-document Parquet sidecars against the workspace Lance datasets. + + uv run python scripts/bench_layout_store.py --docs 500 --items 250 + EXTRALIT_S3_ENDPOINT=http://localhost:9000 uv run python scripts/bench_layout_store.py --workspace bench + +`--workspace` resolves the root through the same resolver production uses, so the S3 run measures +the real path (MinIO bucket `bench` must exist). Needs Redis for the workspace lock. +""" + +from __future__ import annotations + +import argparse +import io +import shutil +import time +from datetime import timedelta +from pathlib import Path +from uuid import uuid4 + +import duckdb +import pyarrow as pa +import pyarrow.parquet as pq + +from extralit_server.contexts.ocr import layout_store +from extralit_server.contexts.ocr.arrow import ITEM_SCHEMA, PAGE_SCHEMA +from extralit_server.contexts.ocr.layout_store import ITEMS_DATASET, PAGES_DATASET, LayoutStore + +LABELS = ["text", "section_header", "table", "picture", "caption"] + + +def synth(document_id: str, n_items: int) -> tuple[pa.Table, pa.Table]: + rows = [ + { + "document_id": document_id, + "self_ref": f"#/texts/{i}", + "parent_ref": "#/body", + "label": LABELS[i % len(LABELS)], + "content_layer": "body", + "level": 0, + "reading_order": i, + "prov_index": 0, + "page_no": i % 10 + 1, + "bbox": [10.0, float(i), 100.0, float(i) + 8.0], + "coord_origin": "TOPLEFT", + "charspan_start": 0, + "charspan_end": 12, + "text": f"item {i} of {document_id}", + "html": None, + } + for i in range(n_items) + ] + pages = [{"document_id": document_id, "page_no": page, "width": 612.0, "height": 792.0} for page in range(1, 11)] + return pa.Table.from_pylist(rows, schema=ITEM_SCHEMA), pa.Table.from_pylist(pages, schema=PAGE_SCHEMA) + + +def du(path: Path) -> int: + return sum(f.stat().st_size for f in path.rglob("*") if f.is_file()) + + +def timed(label: str, fn): + start = time.perf_counter() + result = fn() + print(f"{label:<44} {(time.perf_counter() - start) * 1000:8.1f} ms") + return result + + +def lance_connection(store: LayoutStore) -> duckdb.DuckDBPyConnection: + connection = duckdb.connect() + for name in (ITEMS_DATASET, PAGES_DATASET): + connection.register(f"_{name}", store.source(name)) + connection.execute(f"create view {name} as select * from _{name}") + return connection + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--docs", type=int, default=500) + parser.add_argument("--items", type=int, default=250) + parser.add_argument("--root", default="/tmp/bench-layout") + parser.add_argument("--workspace", help="resolve the Lance root for this workspace instead of --root") + args = parser.parse_args() + + root = Path(args.root) + shutil.rmtree(root, ignore_errors=True) + parquet_dir = root / "parquet" / "layout" + parquet_dir.mkdir(parents=True) + store = LayoutStore.for_workspace(args.workspace) if args.workspace else LayoutStore(str(root / "lance" / "layout")) + + documents = [ + (document_id, *synth(document_id, args.items)) for document_id in (str(uuid4()) for _ in range(args.docs)) + ] + print(f"{args.docs} documents x {args.items} items at {store.root_uri}\n") + + def write_parquet(): + for document_id, items, _pages in documents: + buffer = io.BytesIO() + pq.write_table(items, buffer, compression="zstd") + (parquet_dir / f"{document_id}.items.parquet").write_bytes(buffer.getvalue()) + + def write_lance(): + for document_id, items, pages in documents: + with store.locked_sync(): + store.replace_document(document_id, items, pages) + store.maybe_compact() + + timed("write: parquet sidecars", write_parquet) + elapsed = time.perf_counter() + write_lance() + print(f"{'write: lance replace + compaction':<44} {(time.perf_counter() - elapsed) * 1000 / args.docs:8.1f} ms/doc") + + glob = str(parquet_dir / "*.items.parquet") + one = documents[0][0] + connection = lance_connection(store) + timed( + "query: aggregate by label (parquet glob)", + lambda: duckdb.sql(f"select label, page_no, count(*) from read_parquet('{glob}') group by 1, 2").fetchall(), + ) + timed( + "query: aggregate by label (lance)", + lambda: connection.execute("select label, page_no, count(*) from items group by 1, 2").fetchall(), + ) + timed( + "query: one document (parquet glob)", + lambda: duckdb.sql(f"select count(*) from read_parquet('{glob}') where document_id = '{one}'").fetchall(), + ) + timed("query: one document (lance)", lambda: store.load_items(one).num_rows) + + print(f"\nfragments {store.fragment_count(ITEMS_DATASET)}") + print(f"bytes: parquet {du(parquet_dir):,} over {len(list(parquet_dir.iterdir())):,} objects") + if not args.workspace: + lance_dir = root / "lance" + print(f"bytes: lance {du(lance_dir):,} (retains {layout_store.CLEANUP_OLDER_THAN} of history)") + store.open(ITEMS_DATASET).cleanup_old_versions(older_than=timedelta(0)) + store.open(PAGES_DATASET).cleanup_old_versions(older_than=timedelta(0)) + print(f"bytes: lance {du(lance_dir):,} (history dropped)") + + +if __name__ == "__main__": + main() diff --git a/extralit-server/src/extralit_server/api/handlers/v1/documents.py b/extralit-server/src/extralit_server/api/handlers/v1/documents.py index fb8cf080b..a3987e9d9 100644 --- a/extralit-server/src/extralit_server/api/handlers/v1/documents.py +++ b/extralit-server/src/extralit_server/api/handlers/v1/documents.py @@ -231,8 +231,7 @@ async def delete_documents_by_workspace_id( _LOGGER.info(f"Deleting {len(documents)} documents") for document in documents: - object_path = files.get_pdf_s3_object_path(document.id) - await files.delete_object(s3_client, workspace.name, object_path) + await files.delete_document_artifacts(s3_client, workspace.name, document.id) return len(documents) diff --git a/extralit-server/src/extralit_server/api/schemas/v1/document/metadata.py b/extralit-server/src/extralit_server/api/schemas/v1/document/metadata.py index 54fa4aaa6..6ec0419a2 100644 --- a/extralit-server/src/extralit_server/api/schemas/v1/document/metadata.py +++ b/extralit-server/src/extralit_server/api/schemas/v1/document/metadata.py @@ -59,8 +59,10 @@ class LayoutMetadata(BaseModel): """ layout_url: str = Field(..., description="S3 object path of the canonical DoclingDocument JSON") - items_parquet_url: Optional[str] = Field(None, description="S3 object path of the per-item Parquet sidecar") - pages_parquet_url: Optional[str] = Field(None, description="S3 object path of the per-page Parquet sidecar") + items_uri: Optional[str] = Field(None, description="Lance dataset holding this workspace's layout items") + pages_uri: Optional[str] = Field(None, description="Lance dataset holding this workspace's page geometry") + items_version: Optional[int] = Field(None, description="Items dataset version after this document was written") + pages_version: Optional[int] = Field(None, description="Pages dataset version after this document was written") parser: str = Field(..., description="Name of the layout parser that produced the document") docling_version: str = Field(..., description="docling-core schema version the JSON was written with") num_items: int = Field(default=0, description="Number of layout items extracted") diff --git a/extralit-server/src/extralit_server/contexts/document/metadata.py b/extralit-server/src/extralit_server/contexts/document/metadata.py new file mode 100644 index 000000000..4ecf2c7b0 --- /dev/null +++ b/extralit-server/src/extralit_server/contexts/document/metadata.py @@ -0,0 +1,41 @@ +"""Serialized writes of `documents.metadata_`. + +Several jobs own different slices of the same JSON column, so a read-modify-write that is not +serialized silently drops whatever another job committed in between. Every writer takes the same +row lock through here. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Optional +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from extralit_server.api.schemas.v1.document.metadata import DocumentProcessingMetadata +from extralit_server.models.database import Document + + +async def update_processing_metadata( + db: AsyncSession, + document_id: UUID, + mutate: Callable[[DocumentProcessingMetadata], None], +) -> Optional[DocumentProcessingMetadata]: + """Apply `mutate` to the document's processing metadata under a row lock, and commit. + + Returns None when the document is gone, which is how a job learns its work was deleted. + """ + await db.execute(select(Document.id).where(Document.id == document_id).with_for_update()) + + # populate_existing: the lock is worthless if the session hands back its own stale copy. + document = await db.get(Document, document_id, populate_existing=True) + if document is None: + return None + + metadata = DocumentProcessingMetadata(**(document.metadata_ or {})) + mutate(metadata) + document.metadata_ = metadata.model_dump() + await db.commit() + return metadata diff --git a/extralit-server/src/extralit_server/contexts/files.py b/extralit-server/src/extralit_server/contexts/files.py index 2d2d4fe88..769a7a322 100644 --- a/extralit-server/src/extralit_server/contexts/files.py +++ b/extralit-server/src/extralit_server/contexts/files.py @@ -13,12 +13,28 @@ if TYPE_CHECKING: from types_aiobotocore_s3.client import S3Client -EXCLUDED_VERSIONING_PREFIXES = ["pdf"] CHUNK_LENGTH_MB = 10 * 1024 * 1024 +# Layout datasets rewrite whole files on every commit; keeping their noncurrent versions grows +# without bound and nothing reads them (the canonical JSON is the history). +LAYOUT_NONCURRENT_EXPIRATION_DAYS = 1 _LOGGER = logging.getLogger(__name__) +def workspace_root(workspace_name: str) -> tuple[str, str]: + """Resolve where a workspace's artifacts live, as `(bucket, key prefix)`. + + The single place that knows how a workspace maps onto storage: PDFs, thumbnails, layout JSON + and the Lance datasets all address through it, so moving to one bucket with `{org}/{workspace}/` + prefixes is a change here rather than at every call site. Today it is bucket-per-workspace, + which is why the prefix is empty and `files` can still pass `Bucket=workspace_name` directly. + """ + if not workspace_name: + raise ValueError("workspace_name cannot be empty") + + return workspace_name, "" + + async def get_s3_client() -> "S3Client": """Dependency function to get shared S3 client.""" s3_client = shared_resources.get("s3_client") @@ -346,6 +362,26 @@ async def delete_object(s3_client, bucket: str, object: str, version_id: str | N raise HTTPException(status_code=500, detail=f"Internal server error: {e!s}") +async def delete_document_artifacts(s3_client: "S3Client", workspace_name: str, document_id: UUID | str) -> None: + """Remove every artifact of a document: PDF, thumbnail, layout JSON and layout rows. + + Best effort — the DB rows are already gone by the time this runs, so a storage hiccup must + leave a leaked object rather than a document the caller cannot delete. + """ + from extralit_server.contexts.ocr import storage + + for object_path in (get_pdf_s3_object_path(document_id), get_thumbnail_s3_object_path(document_id)): + try: + await delete_object(s3_client, workspace_name, object_path) + except Exception as e: + _LOGGER.warning(f"Could not delete {object_path} for document {document_id}: {e}") + + try: + await storage.delete_layout(s3_client, workspace_name, document_id) + except Exception as e: + _LOGGER.warning(f"Could not delete layout artifacts for document {document_id}: {e}") + + async def bucket_exists(s3_client: "S3Client", bucket_name: str) -> bool: """Check if S3 bucket exists.""" try: @@ -378,28 +414,43 @@ async def get_bucket_versioning(s3_client: "S3Client", bucket_name: str) -> dict return None -async def create_bucket( - s3_client: "S3Client", - workspace_name: str, - excluded_prefixes: list[str] = EXCLUDED_VERSIONING_PREFIXES, -): - """Create S3 bucket.""" +async def create_bucket(s3_client: "S3Client", workspace_name: str): + """Create the workspace's bucket, versioned, with layout noncurrent versions expiring.""" + bucket, prefix = workspace_root(workspace_name) try: - await s3_client.create_bucket(Bucket=workspace_name) + await s3_client.create_bucket(Bucket=bucket) await s3_client.put_bucket_versioning( - Bucket=workspace_name, + Bucket=bucket, VersioningConfiguration={ "Status": "Enabled", "MFADelete": "Disabled", }, ) + try: + await s3_client.put_bucket_lifecycle_configuration( + Bucket=bucket, + LifecycleConfiguration={ + "Rules": [ + { + "ID": "expire-noncurrent-layout-versions", + "Status": "Enabled", + "Filter": {"Prefix": f"{prefix}layout/"}, + "NoncurrentVersionExpiration": {"NoncurrentDays": LAYOUT_NONCURRENT_EXPIRATION_DAYS}, + } + ] + }, + ) + except Exception as e: + # A backend without lifecycle support costs storage, never correctness. + _LOGGER.warning(f"Could not set the layout lifecycle rule on bucket {bucket}: {e}") + except ClientError as e: if e.response["Error"]["Code"] in ["BucketAlreadyOwnedByYou", "BucketAlreadyExists"]: pass # Bucket already exists, that's fine else: - _LOGGER.error(f"Error creating bucket {workspace_name}: {e}") + _LOGGER.error(f"Error creating bucket {bucket}: {e}") raise HTTPException(status_code=500, detail=f"Error creating bucket: {e!s}") except Exception as e: _LOGGER.error(f"Error creating bucket {workspace_name}: {e}") diff --git a/extralit-server/src/extralit_server/contexts/ocr/layout_store.py b/extralit-server/src/extralit_server/contexts/ocr/layout_store.py new file mode 100644 index 000000000..5dcfa1dbc --- /dev/null +++ b/extralit-server/src/extralit_server/contexts/ocr/layout_store.py @@ -0,0 +1,261 @@ +"""Workspace-scoped Lance datasets for extracted layout. + +One `items`/`pages` dataset per workspace, so a corpus-wide question is a single scan instead of a +glob over one small object per document. A document's rows are deleted before they are appended, +which is what keeps a re-parse from leaving two vintages behind — and because that is two commits, +every writer holds the workspace lock across the pair. +""" + +from __future__ import annotations + +import logging +from collections.abc import Iterator, Sequence +from contextlib import asynccontextmanager, contextmanager +from datetime import timedelta +from pathlib import Path +from typing import TYPE_CHECKING, Any, Optional +from uuid import UUID + +import lance +import pyarrow as pa +from anyio import to_thread +from lance.commit import CommitConflictError + +from extralit_server.contexts.files import workspace_root +from extralit_server.contexts.ocr.arrow import ITEM_SCHEMA, PAGE_SCHEMA +from extralit_server.jobs.queues import REDIS_CONNECTION +from extralit_server.settings import settings + +if TYPE_CHECKING: + import duckdb + +_LOGGER = logging.getLogger("extralit_server.contexts.ocr.layout_store") + +LAYOUT_PREFIX = "layout" +ITEMS_DATASET = "items" +PAGES_DATASET = "pages" +COMPACT_FRAGMENT_THRESHOLD = 32 +# Bounds how much settled data a compaction rewrites, at the cost of more files per workspace. +TARGET_ROWS_PER_FRAGMENT = 250_000 +# Long enough that a running query never loses its files; the canonical JSON is the real history. +CLEANUP_OLDER_THAN = timedelta(minutes=15) +# Must outlast a compaction, which is the longest thing done under the lock. +LOCK_TTL_SECONDS = 600 +LOCK_WAIT_SECONDS = 300 +REPLACE_ATTEMPTS = 3 + +_SCHEMAS = {ITEMS_DATASET: ITEM_SCHEMA, PAGES_DATASET: PAGE_SCHEMA} + + +def _s3_storage_options() -> dict[str, str]: + endpoint = settings.s3_endpoint or "" + return { + "aws_access_key_id": settings.s3_access_key or "", + "aws_secret_access_key": settings.s3_secret_key or "", + "aws_endpoint": endpoint, + "aws_region": settings.s3_region or "us-east-1", + "allow_http": str(endpoint.startswith("http://")).lower(), + "aws_virtual_hosted_style_request": "false", + } + + +def layout_root(workspace_name: str) -> tuple[str, Optional[dict[str, str]]]: + """Root of the workspace's datasets, resolved exactly like every other artifact of it.""" + bucket, prefix = workspace_root(workspace_name) + if all([settings.s3_endpoint, settings.s3_access_key, settings.s3_secret_key]): + return f"s3://{bucket}/{prefix}{LAYOUT_PREFIX}", _s3_storage_options() + return str(Path(settings.home_path) / bucket / f"{prefix}{LAYOUT_PREFIX}"), None + + +def _document_filter(document_id: UUID | str) -> str: + """A UUID round-trip is the whole sanitisation — nothing else reaches the filter string.""" + return f"document_id = '{UUID(str(document_id))}'" + + +class LayoutStore: + """Reader and writer of one workspace's layout datasets. + + Writers must hold `locked()`/`locked_sync()`: replacing a document is a delete commit followed + by an append commit, and two interleaved replaces would otherwise drop or double rows. Lance's + own commit conflict detection is the belt to that lock's braces. + """ + + def __init__(self, root_uri: str, storage_options: Optional[dict[str, str]] = None) -> None: + self.root_uri = root_uri.rstrip("/") + self.storage_options = storage_options + self._lock_depth = 0 + self._lock = None + + @classmethod + def for_workspace(cls, workspace_name: str) -> LayoutStore: + return cls(*layout_root(workspace_name)) + + def uri(self, name: str) -> str: + return f"{self.root_uri}/{name}.lance" + + def items_uri(self) -> str: + return self.uri(ITEMS_DATASET) + + def pages_uri(self) -> str: + return self.uri(PAGES_DATASET) + + # --- locking ------------------------------------------------------------------------------- + + def _acquire(self) -> None: + if self._lock_depth == 0: + # The resolved root, not the display name: a renamed workspace must not fork the lock. + lock = REDIS_CONNECTION.lock( + f"extralit:layout:{self.root_uri}", + timeout=LOCK_TTL_SECONDS, + blocking_timeout=LOCK_WAIT_SECONDS, + thread_local=False, + ) + if not lock.acquire(): + raise TimeoutError(f"Timed out waiting for the layout lock on {self.root_uri}") + self._lock = lock + self._lock_depth += 1 + + def _release(self) -> None: + self._lock_depth -= 1 + if self._lock_depth == 0 and self._lock is not None: + lock, self._lock = self._lock, None + try: + lock.release() + except Exception as error: + # Expired by TTL, or already taken over: the write itself has already committed. + _LOGGER.warning(f"Could not release the layout lock on {self.root_uri}: {error}") + + @contextmanager + def locked_sync(self) -> Iterator[None]: + """Serialize this workspace's writers. Reentrant per store instance.""" + self._acquire() + try: + yield + finally: + self._release() + + @asynccontextmanager + async def locked(self): + """`locked_sync` for async callers; acquisition blocks, so it happens off the event loop.""" + await to_thread.run_sync(self._acquire) + try: + yield + finally: + await to_thread.run_sync(self._release) + + # --- writes -------------------------------------------------------------------------------- + + def open(self, name: str) -> Optional[lance.LanceDataset]: + try: + return lance.dataset(self.uri(name), storage_options=self.storage_options) + except (ValueError, FileNotFoundError): + return None + + def _write(self, name: str, data: pa.Table, mode: str) -> int: + return lance.write_dataset(data, self.uri(name), mode=mode, storage_options=self.storage_options).version + + def _replace_one(self, name: str, document_id: UUID | str, data: pa.Table) -> int: + dataset = self.open(name) + if dataset is None: + try: + return self._write(name, data, mode="create") + except OSError: + # Another worker created it between the open and the write; join it instead. + dataset = lance.dataset(self.uri(name), storage_options=self.storage_options) + + dataset.delete(_document_filter(document_id)) + if data.num_rows == 0: + return lance.dataset(self.uri(name), storage_options=self.storage_options).version + return self._write(name, data, mode="append") + + def replace_document(self, document_id: UUID | str, items: pa.Table, pages: pa.Table) -> dict[str, int]: + """Swap this document's rows for `items`/`pages`. Caller must hold the workspace lock.""" + versions = {} + for name, data in ((ITEMS_DATASET, items), (PAGES_DATASET, pages)): + for attempt in range(1, REPLACE_ATTEMPTS + 1): + try: + versions[f"{name}_version"] = self._replace_one(name, document_id, data) + break + except CommitConflictError: + # Retry the delete and the append together; half a replace is a corrupt vintage. + if attempt == REPLACE_ATTEMPTS: + raise + _LOGGER.warning(f"Commit conflict on {self.uri(name)}, retrying the whole replace") + return versions + + def delete_document(self, document_id: UUID | str) -> None: + """Drop this document's rows. Caller must hold the workspace lock.""" + condition = _document_filter(document_id) + for name in (ITEMS_DATASET, PAGES_DATASET): + dataset = self.open(name) + if dataset is not None: + dataset.delete(condition) + + # --- reads --------------------------------------------------------------------------------- + + def _read( + self, + name: str, + document_id: UUID | str, + columns: Optional[Sequence[str]] = None, + where: Optional[str] = None, + ) -> pa.Table: + condition = _document_filter(document_id) + projection = list(columns) if columns else None + dataset = self.open(name) + if dataset is None: + empty = pa.Table.from_pylist([], schema=_SCHEMAS[name]) + return empty.select(projection) if projection else empty + return dataset.to_table(filter=f"({condition}) AND ({where})" if where else condition, columns=projection) + + def load_items(self, document_id: UUID | str, columns=None, where: Optional[str] = None) -> pa.Table: + return self._read(ITEMS_DATASET, document_id, columns=columns, where=where) + + def load_pages(self, document_id: UUID | str, columns=None, where: Optional[str] = None) -> pa.Table: + return self._read(PAGES_DATASET, document_id, columns=columns, where=where) + + def source(self, name: str) -> Any: + """The dataset, or an empty table of the right schema when nothing has been written yet.""" + dataset = self.open(name) + return dataset if dataset is not None else pa.Table.from_pylist([], schema=_SCHEMAS[name]) + + # --- maintenance --------------------------------------------------------------------------- + + def fragment_count(self, name: str) -> int: + dataset = self.open(name) + return 0 if dataset is None else len(dataset.get_fragments()) + + def maybe_compact(self) -> None: + """Best effort: a failed compaction must never fail the extraction that triggered it.""" + for name in (ITEMS_DATASET, PAGES_DATASET): + try: + if self.fragment_count(name) <= COMPACT_FRAGMENT_THRESHOLD: + continue + dataset = self.open(name) + dataset.optimize.compact_files(target_rows_per_fragment=TARGET_ROWS_PER_FRAGMENT) + self.open(name).cleanup_old_versions(older_than=CLEANUP_OLDER_THAN) + except Exception as error: + _LOGGER.warning(f"Layout compaction of {name} at {self.root_uri} failed: {error}") + + +@contextmanager +def duckdb_connection(workspaces: Sequence[str]) -> Iterator[duckdb.DuckDBPyConnection]: + """`items` and `pages` as DuckDB views over one or more workspaces. + + Lance datasets take the projection and the filter, so `select label, count(*)` never reads the + text column. Aggregating N workspaces is a loop over their roots, not a catalog. + """ + import duckdb + + connection = duckdb.connect() + try: + for name in (ITEMS_DATASET, PAGES_DATASET): + branches = [] + for index, workspace_name in enumerate(workspaces): + alias = f"_{name}_{index}" + connection.register(alias, LayoutStore.for_workspace(workspace_name).source(name)) + branches.append(f"select * from {alias}") + connection.execute(f"create view {name} as {' union all '.join(branches)}") + yield connection + finally: + connection.close() diff --git a/extralit-server/src/extralit_server/contexts/ocr/storage.py b/extralit-server/src/extralit_server/contexts/ocr/storage.py index bc549b1de..2e47a2ac5 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/storage.py +++ b/extralit-server/src/extralit_server/contexts/ocr/storage.py @@ -1,88 +1,91 @@ """Object-storage layout for extracted document layout. -Canonical JSON is the source of truth; the Parquet sidecars are a columnar projection for -analytics and cross-document provenance search. Both live in S3 rather than `documents.metadata_`, -which is returned in full by every document listing. +Canonical JSON is the source of truth; the columnar rows live in the workspace's Lance datasets +(see `layout_store`) so a corpus-wide question is one scan. Both live in S3 rather than +`documents.metadata_`, which is returned in full by every document listing. """ from __future__ import annotations -import io import json -from typing import TYPE_CHECKING, Optional +import logging +from typing import TYPE_CHECKING, Any, Optional from uuid import UUID -import pyarrow as pa -import pyarrow.parquet as pq +from anyio import to_thread from docling_core.types.doc import DoclingDocument from extralit_server.contexts import files from extralit_server.contexts.ocr.arrow import items_table, pages_table +from extralit_server.contexts.ocr.layout_store import LAYOUT_PREFIX, LayoutStore if TYPE_CHECKING: from types_aiobotocore_s3 import S3Client -LAYOUT_PREFIX = "layout" +_LOGGER = logging.getLogger("extralit_server.contexts.ocr.storage") def layout_object_path(document_id: UUID | str) -> str: return f"{LAYOUT_PREFIX}/{document_id}.docling.json" -def items_object_path(document_id: UUID | str) -> str: - return f"{LAYOUT_PREFIX}/{document_id}.items.parquet" - - -def pages_object_path(document_id: UUID | str) -> str: - return f"{LAYOUT_PREFIX}/{document_id}.pages.parquet" - - -def _to_parquet(table: pa.Table) -> bytes: - buffer = io.BytesIO() - pq.write_table(table, buffer, compression="zstd") - return buffer.getvalue() - - async def store_layout( s3_client: S3Client, workspace_name: str, document_id: UUID | str, doc: DoclingDocument, -) -> dict[str, str]: - """Write the canonical JSON and both Parquet sidecars. Returns their object paths.""" + store: Optional[LayoutStore] = None, +) -> dict[str, Any]: + """Write the canonical JSON, then replace this document's rows in the workspace datasets. + + `store` lets a caller that already holds the workspace lock pass its own handle through; the + lock is reentrant per store instance, so the replace still runs serialized either way. + """ document_id = str(document_id) - paths = { - "layout_url": layout_object_path(document_id), - "items_parquet_url": items_object_path(document_id), - "pages_parquet_url": pages_object_path(document_id), - } + layout_url = layout_object_path(document_id) + store = store or LayoutStore.for_workspace(workspace_name) await files.put_object( s3_client, workspace_name, - paths["layout_url"], + layout_url, json.dumps(doc.export_to_dict(), ensure_ascii=False), content_type="application/json", metadata={"docling_version": doc.version, "document_id": document_id}, ) - await files.put_object( - s3_client, - workspace_name, - paths["items_parquet_url"], - _to_parquet(items_table(doc, document_id)), - content_type="application/vnd.apache.parquet", - metadata={"document_id": document_id}, - ) - await files.put_object( - s3_client, - workspace_name, - paths["pages_parquet_url"], - _to_parquet(pages_table(doc, document_id)), - content_type="application/vnd.apache.parquet", - metadata={"document_id": document_id}, - ) - return paths + items = items_table(doc, document_id) + pages = pages_table(doc, document_id) + async with store.locked(): + versions = await to_thread.run_sync(store.replace_document, document_id, items, pages) + await to_thread.run_sync(store.maybe_compact) + + return { + "layout_url": layout_url, + "items_uri": store.items_uri(), + "pages_uri": store.pages_uri(), + **versions, + } + + +async def delete_layout( + s3_client: S3Client, + workspace_name: str, + document_id: UUID | str, + store: Optional[LayoutStore] = None, +) -> None: + """Drop both artifacts. Orphaned rows would silently skew every workspace aggregate.""" + try: + await files.delete_object(s3_client, workspace_name, layout_object_path(document_id)) + except Exception as error: + _LOGGER.warning(f"Could not delete layout JSON for document {document_id}: {error}") + + try: + store = store or LayoutStore.for_workspace(workspace_name) + async with store.locked(): + await to_thread.run_sync(store.delete_document, document_id) + except Exception as error: + _LOGGER.warning(f"Could not delete layout rows for document {document_id}: {error}") async def load_layout( diff --git a/extralit-server/src/extralit_server/jobs/document_jobs.py b/extralit-server/src/extralit_server/jobs/document_jobs.py index cbf25f796..6e60df977 100644 --- a/extralit-server/src/extralit_server/jobs/document_jobs.py +++ b/extralit-server/src/extralit_server/jobs/document_jobs.py @@ -2,7 +2,6 @@ import asyncio import logging -from datetime import datetime, timezone from typing import Any from uuid import UUID @@ -13,10 +12,10 @@ from extralit_server.contexts import files from extralit_server.contexts.document.analysis import PDFOCRLayerDetector from extralit_server.contexts.document.margin import PDFAnalyzer +from extralit_server.contexts.document.metadata import update_processing_metadata from extralit_server.contexts.document.preprocessing import PDFPreprocessingSettings, PDFPreprocessor from extralit_server.database import AsyncSessionLocal from extralit_server.jobs.queues import DEFAULT_QUEUE, REDIS_CONNECTION -from extralit_server.models.database import Document _LOGGER = logging.getLogger(__name__) @@ -146,21 +145,13 @@ async def analysis_and_preprocess_job( }, } - # Store combined results in document.metadata_ using async database operations + # The layout job writes the same JSON column concurrently; both go through the row lock. + def apply(metadata: DocumentProcessingMetadata) -> None: + metadata.update_analysis_results(analysis_result) + metadata.update_preprocessing_results(combined_result["preprocessing_result"]) + async with AsyncSessionLocal() as db: - document = await db.get(Document, document_id) - if document: - # Initialize or update document metadata - if document.metadata_ is None: - document.metadata_ = DocumentProcessingMetadata( - workflow_started_at=datetime.now(timezone.utc) - ).model_dump() - - metadata = DocumentProcessingMetadata(**document.metadata_) - metadata.update_analysis_results(analysis_result) - metadata.update_preprocessing_results(combined_result["preprocessing_result"]) - document.metadata_ = metadata.model_dump() - await db.commit() + await update_processing_metadata(db, document_id, apply) # Store results for dependent jobs current_job.meta["needs_ocr"] = analysis_result["needs_ocr"] diff --git a/extralit-server/src/extralit_server/jobs/ocr_jobs.py b/extralit-server/src/extralit_server/jobs/ocr_jobs.py index cb152a8a6..842e4a0b8 100644 --- a/extralit-server/src/extralit_server/jobs/ocr_jobs.py +++ b/extralit-server/src/extralit_server/jobs/ocr_jobs.py @@ -9,9 +9,11 @@ from rq.decorators import job from sqlalchemy import select -from extralit_server.api.schemas.v1.document.metadata import DocumentProcessingMetadata, LayoutMetadata +from extralit_server.api.schemas.v1.document.metadata import LayoutMetadata from extralit_server.contexts import files +from extralit_server.contexts.document.metadata import update_processing_metadata from extralit_server.contexts.ocr import storage +from extralit_server.contexts.ocr.layout_store import LayoutStore from extralit_server.contexts.ocr.parsers import default_parser_name, get_parser from extralit_server.contexts.ocr.parsers.pdf_inspector import classify from extralit_server.database import AsyncSessionLocal @@ -88,7 +90,17 @@ async def async_document_layout_job( filename=s3_url.split("/")[-1], ) - paths = await storage.store_layout(s3_client, workspace_name, document_id, doc) + # Ordering closes the delete-vs-running-job race: nothing may write rows for a document + # whose delete already ran, and the workspace lock is held across the check and the write. + store = LayoutStore.for_workspace(workspace_name) + async with store.locked(): + async with AsyncSessionLocal() as db: + still_exists = await db.scalar(select(Document.id).where(Document.id == document_id)) + if still_exists is None: + _LOGGER.info(f"Document {document_id} was deleted before its layout was stored") + return {"document_id": str(document_id), "parser": parser_name, "skipped": "document deleted"} + + paths = await storage.store_layout(s3_client, workspace_name, document_id, doc, store=store) layout = LayoutMetadata( **paths, @@ -99,16 +111,9 @@ async def async_document_layout_job( pages_needing_ocr=classification.get("pages_needing_ocr", []), ) + # Outside the workspace lock: the row lock only serializes writers of this JSON column. async with AsyncSessionLocal() as db: - # Text extraction writes the same JSON column concurrently; serialize on the row so - # this read-modify-write does not clobber whatever it wrote. - await db.execute(select(Document.id).where(Document.id == document_id).with_for_update()) - document = await db.get(Document, document_id) - if document is not None: - metadata = DocumentProcessingMetadata(**(document.metadata_ or {})) - metadata.layout_metadata = layout - document.metadata_ = metadata.model_dump() - await db.commit() + await update_processing_metadata(db, document_id, lambda m: setattr(m, "layout_metadata", layout)) result = { "document_id": str(document_id), diff --git a/extralit-server/tests/unit/api/handlers/v1/test_documents.py b/extralit-server/tests/unit/api/handlers/v1/test_documents.py index bf6c5a997..3e93e982d 100644 --- a/extralit-server/tests/unit/api/handlers/v1/test_documents.py +++ b/extralit-server/tests/unit/api/handlers/v1/test_documents.py @@ -1,5 +1,5 @@ import json -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch from uuid import UUID, uuid4 import pytest @@ -251,3 +251,49 @@ async def test_list_documents(async_client: "AsyncClient", db: "AsyncSession", o assert response.status_code == 200 assert len(response.json()) == 2 assert response.json()[0]["id"] == str(doc_id) + + +@pytest.mark.asyncio +async def test_delete_documents_removes_every_artifact( + async_client: AsyncClient, db: AsyncSession, owner_auth_header: dict +): + workspace = await WorkspaceFactory.create() + document = await DocumentFactory.create(workspace=workspace) + + with patch("extralit_server.contexts.files.delete_document_artifacts", new_callable=AsyncMock) as fan_out: + response = await async_client.request( + "DELETE", + f"/api/v1/documents/workspace/{workspace.id}", + json={"id": str(document.id)}, + headers=owner_auth_header, + ) + + assert response.status_code == 200, response.json() + assert fan_out.await_args.args[1:] == (workspace.name, document.id) + + +@pytest.mark.asyncio +async def test_delete_documents_succeeds_when_layout_cleanup_fails( + async_client: AsyncClient, db: AsyncSession, owner_auth_header: dict +): + # The rows are already gone; a stuck Lance dataset must not make the document undeletable. + workspace = await WorkspaceFactory.create() + document = await DocumentFactory.create(workspace=workspace) + + with ( + patch("extralit_server.contexts.files.delete_object", new_callable=AsyncMock), + patch( + "extralit_server.contexts.ocr.storage.delete_layout", + new_callable=AsyncMock, + side_effect=RuntimeError("lance down"), + ), + ): + response = await async_client.request( + "DELETE", + f"/api/v1/documents/workspace/{workspace.id}", + json={"id": str(document.id)}, + headers=owner_auth_header, + ) + + assert response.status_code == 200, response.json() + assert response.json() == 1 diff --git a/extralit-server/tests/unit/contexts/document/__init__.py b/extralit-server/tests/unit/contexts/document/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/extralit-server/tests/unit/contexts/document/test_metadata.py b/extralit-server/tests/unit/contexts/document/test_metadata.py new file mode 100644 index 000000000..b0ec6a453 --- /dev/null +++ b/extralit-server/tests/unit/contexts/document/test_metadata.py @@ -0,0 +1,56 @@ +"""Tests for the serialized read-modify-write of `documents.metadata_`.""" + +import pytest +from sqlalchemy import update +from sqlalchemy.ext.asyncio import AsyncSession + +from extralit_server.api.schemas.v1.document.metadata import DocumentProcessingMetadata, LayoutMetadata +from extralit_server.contexts.document.metadata import update_processing_metadata +from extralit_server.models.database import Document +from tests.factories import DocumentFactory, WorkspaceFactory + + +def _layout(parser: str = "pdf_inspector") -> LayoutMetadata: + return LayoutMetadata(layout_url="layout/x.docling.json", parser=parser, docling_version="1.7.0") + + +@pytest.mark.asyncio +class TestUpdateProcessingMetadata: + async def test_locks_the_row_before_reading(self, db: AsyncSession, mocker): + workspace = await WorkspaceFactory.create() + document = await DocumentFactory.create(workspace=workspace) + statements = [] + original = db.execute + + async def record(statement, *args, **kwargs): + statements.append(str(statement)) + return await original(statement, *args, **kwargs) + + mocker.patch.object(db, "execute", side_effect=record) + + await update_processing_metadata(db, document.id, lambda m: setattr(m, "workflow_status", "done")) + + assert any("FOR UPDATE" in statement for statement in statements) + + async def test_keeps_keys_written_since_this_session_last_read(self, db: AsyncSession): + workspace = await WorkspaceFactory.create() + document = await DocumentFactory.create(workspace=workspace) + await db.get(Document, document.id) # this session now holds a stale copy + + await db.execute( + update(Document) + .where(Document.id == document.id) + .values(metadata_=DocumentProcessingMetadata(layout_metadata=_layout()).model_dump()) + ) + + metadata = await update_processing_metadata( + db, document.id, lambda m: setattr(m, "workflow_status", "completed") + ) + + assert metadata.layout_metadata is not None + assert metadata.workflow_status == "completed" + + async def test_returns_none_when_the_document_is_gone(self, db: AsyncSession): + from uuid import uuid4 + + assert await update_processing_metadata(db, uuid4(), lambda m: None) is None diff --git a/extralit-server/tests/unit/contexts/ocr/test_layout_store.py b/extralit-server/tests/unit/contexts/ocr/test_layout_store.py new file mode 100644 index 000000000..8ed72e182 --- /dev/null +++ b/extralit-server/tests/unit/contexts/ocr/test_layout_store.py @@ -0,0 +1,260 @@ +"""Tests for the workspace-scoped Lance layout store.""" + +from __future__ import annotations + +import threading +import time +from uuid import uuid4 + +import pyarrow as pa +import pytest + +from extralit_server.contexts import files +from extralit_server.contexts.ocr.arrow import ITEM_SCHEMA, PAGE_SCHEMA +from extralit_server.contexts.ocr.layout_store import ( + ITEMS_DATASET, + LAYOUT_PREFIX, + LayoutStore, + duckdb_connection, +) +from extralit_server.settings import settings + +WORKSPACE = "ws-layout" + + +def items(document_id: str, count: int = 3, label: str = "text") -> pa.Table: + return pa.Table.from_pylist( + [ + { + "document_id": document_id, + "self_ref": f"#/texts/{i}", + "parent_ref": "#/body", + "label": label, + "content_layer": "body", + "level": 1, + "reading_order": i, + "prov_index": 0, + "page_no": 1, + "bbox": [0.0, 0.0, 10.0, 10.0], + "coord_origin": "TOPLEFT", + "charspan_start": 0, + "charspan_end": 5, + "text": f"row {i}", + "html": None, + } + for i in range(count) + ], + schema=ITEM_SCHEMA, + ) + + +def pages(document_id: str, count: int = 1) -> pa.Table: + return pa.Table.from_pylist( + [{"document_id": document_id, "page_no": i + 1, "width": 612.0, "height": 792.0} for i in range(count)], + schema=PAGE_SCHEMA, + ) + + +@pytest.fixture +def local_store(tmp_path, monkeypatch): + monkeypatch.setattr(settings, "home_path", str(tmp_path)) + monkeypatch.setattr(settings, "s3_endpoint", None) + return LayoutStore.for_workspace(WORKSPACE) + + +class TestAddressing: + def test_local_root_matches_where_the_local_file_client_puts_objects(self, tmp_path, monkeypatch): + monkeypatch.setattr(settings, "home_path", str(tmp_path)) + monkeypatch.setattr(settings, "s3_endpoint", None) + + bucket, prefix = files.workspace_root(WORKSPACE) + store = LayoutStore.for_workspace(WORKSPACE) + + # `{home_path}/{bucket}/{key}` is exactly LocalFileClient's object path. + assert store.root_uri == f"{tmp_path}/{bucket}/{prefix}{LAYOUT_PREFIX}" + assert store.items_uri().endswith(f"{LAYOUT_PREFIX}/{ITEMS_DATASET}.lance") + + def test_s3_root_matches_the_bucket_and_prefix_files_resolve(self, monkeypatch): + monkeypatch.setattr(settings, "s3_endpoint", "http://localhost:9000") + monkeypatch.setattr(settings, "s3_access_key", "minio") + monkeypatch.setattr(settings, "s3_secret_key", "secret") + + bucket, prefix = files.workspace_root(WORKSPACE) + store = LayoutStore.for_workspace(WORKSPACE) + + assert store.root_uri == f"s3://{bucket}/{prefix}{LAYOUT_PREFIX}" + assert store.storage_options["allow_http"] == "true" + + def test_pdf_and_thumbnail_keys_share_the_layout_root(self, monkeypatch): + # A later single-bucket mode must move every artifact of a workspace together. + monkeypatch.setattr(settings, "s3_endpoint", "http://localhost:9000") + monkeypatch.setattr(settings, "s3_access_key", "minio") + monkeypatch.setattr(settings, "s3_secret_key", "secret") + document_id = uuid4() + + bucket, prefix = files.workspace_root(WORKSPACE) + store = LayoutStore.for_workspace(WORKSPACE) + + for key in (files.get_pdf_s3_object_path(document_id), files.get_thumbnail_s3_object_path(document_id)): + assert store.root_uri.startswith(f"s3://{bucket}/{prefix}") + assert not key.startswith("/") + + def test_the_resolver_agrees_with_the_bucket_files_addresses_today(self): + # files.py still passes `Bucket=workspace_name`; the resolver must not silently disagree. + assert files.workspace_root(WORKSPACE) == (WORKSPACE, "") + + +class TestReplace: + def test_a_second_replace_leaves_one_vintage(self, local_store): + document_id = str(uuid4()) + + local_store.replace_document(document_id, items(document_id, 3), pages(document_id, 1)) + local_store.replace_document(document_id, items(document_id, 5), pages(document_id, 2)) + + assert local_store.load_items(document_id).num_rows == 5 + assert local_store.load_pages(document_id).num_rows == 2 + + def test_other_documents_are_untouched(self, local_store): + keeper, replaced = str(uuid4()), str(uuid4()) + + local_store.replace_document(keeper, items(keeper, 4), pages(keeper)) + local_store.replace_document(replaced, items(replaced, 2), pages(replaced)) + local_store.replace_document(replaced, items(replaced, 1), pages(replaced)) + + assert local_store.load_items(keeper).num_rows == 4 + assert local_store.load_items(replaced).num_rows == 1 + + def test_a_zero_row_document_still_clears_its_old_rows(self, local_store): + document_id = str(uuid4()) + local_store.replace_document(document_id, items(document_id, 3), pages(document_id)) + + empty_items = pa.Table.from_pylist([], schema=ITEM_SCHEMA) + empty_pages = pa.Table.from_pylist([], schema=PAGE_SCHEMA) + local_store.replace_document(document_id, empty_items, empty_pages) + + assert local_store.load_items(document_id).num_rows == 0 + assert local_store.load_pages(document_id).num_rows == 0 + + def test_replace_reports_the_dataset_versions(self, local_store): + document_id = str(uuid4()) + + versions = local_store.replace_document(document_id, items(document_id), pages(document_id)) + + assert versions["items_version"] >= 1 + assert versions["pages_version"] >= 1 + + def test_concurrent_replaces_of_one_document_neither_lose_nor_duplicate_rows(self, local_store, monkeypatch): + # Replacing is a delete commit then an append commit. Both writers are held at the append + # until the other has passed its delete, which is exactly the interleaving that doubles + # rows when the workspace lock is not held. + document_id = str(uuid4()) + local_store.replace_document(document_id, items(document_id, 2), pages(document_id)) + original_write = LayoutStore._write + arrivals = {"count": 0} + counter_lock = threading.Lock() + + def synchronized_write(self, name, data, mode): + if mode == "append" and name == ITEMS_DATASET: + with counter_lock: + arrivals["count"] += 1 + deadline = time.monotonic() + 2.0 + while arrivals["count"] < 2 and time.monotonic() < deadline: + time.sleep(0.01) + return original_write(self, name, data, mode) + + monkeypatch.setattr(LayoutStore, "_write", synchronized_write) + barrier = threading.Barrier(2) + errors: list[Exception] = [] + + def replace(count: int): + store = LayoutStore.for_workspace(WORKSPACE) + try: + barrier.wait(timeout=30) + with store.locked_sync(): + store.replace_document(document_id, items(document_id, count), pages(document_id)) + except Exception as error: # surfaced below; a raise here would be swallowed + errors.append(error) + + threads = [threading.Thread(target=replace, args=(7,)) for _ in range(2)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=60) + + assert errors == [] + assert local_store.load_items(document_id).num_rows == 7 + + +class TestDelete: + def test_delete_removes_items_and_pages(self, local_store): + document_id, other = str(uuid4()), str(uuid4()) + local_store.replace_document(document_id, items(document_id, 3), pages(document_id, 2)) + local_store.replace_document(other, items(other, 1), pages(other, 1)) + + local_store.delete_document(document_id) + + assert local_store.load_items(document_id).num_rows == 0 + assert local_store.load_pages(document_id).num_rows == 0 + assert local_store.load_items(other).num_rows == 1 + + def test_delete_on_a_missing_dataset_is_a_no_op(self, local_store): + local_store.delete_document(str(uuid4())) + + +class TestReads: + def test_missing_datasets_read_as_empty_tables_with_the_right_schema(self, local_store): + table = local_store.load_items(str(uuid4())) + + assert table.num_rows == 0 + assert table.schema.names == ITEM_SCHEMA.names + + def test_projection_and_filter_are_pushed_down(self, local_store): + document_id = str(uuid4()) + local_store.replace_document(document_id, items(document_id, 4), pages(document_id)) + + table = local_store.load_items(document_id, columns=["self_ref", "page_no"], where="reading_order < 2") + + assert table.schema.names == ["self_ref", "page_no"] + assert table.num_rows == 2 + + def test_duckdb_views_aggregate_across_documents(self, local_store): + first, second = str(uuid4()), str(uuid4()) + local_store.replace_document(first, items(first, 3, label="text"), pages(first)) + local_store.replace_document(second, items(second, 2, label="table"), pages(second)) + + with duckdb_connection([WORKSPACE]) as connection: + counts = dict(connection.execute("select label, count(*) from items group by label").fetchall()) + page_count = connection.execute("select count(*) from pages").fetchone()[0] + + assert counts == {"text": 3, "table": 2} + assert page_count == 2 + + def test_duckdb_views_exist_before_anything_is_written(self, local_store): + with duckdb_connection([WORKSPACE]) as connection: + assert connection.execute("select count(*) from items").fetchone()[0] == 0 + assert connection.execute("select count(*) from pages").fetchone()[0] == 0 + + +class TestCompaction: + def test_compaction_fires_past_the_threshold_and_preserves_rows(self, local_store, monkeypatch): + monkeypatch.setattr("extralit_server.contexts.ocr.layout_store.COMPACT_FRAGMENT_THRESHOLD", 3) + document_ids = [str(uuid4()) for _ in range(6)] + for document_id in document_ids: + local_store.replace_document(document_id, items(document_id, 2), pages(document_id)) + + before = local_store.fragment_count(ITEMS_DATASET) + local_store.maybe_compact() + + assert before > 3 + assert local_store.fragment_count(ITEMS_DATASET) < before + for document_id in document_ids: + assert local_store.load_items(document_id).num_rows == 2 + + def test_a_broken_compaction_never_reaches_the_caller(self, local_store, monkeypatch): + document_id = str(uuid4()) + local_store.replace_document(document_id, items(document_id), pages(document_id)) + monkeypatch.setattr( + LayoutStore, "fragment_count", lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("boom")) + ) + + local_store.maybe_compact() diff --git a/extralit-server/tests/unit/contexts/ocr/test_storage.py b/extralit-server/tests/unit/contexts/ocr/test_storage.py new file mode 100644 index 000000000..5144bc648 --- /dev/null +++ b/extralit-server/tests/unit/contexts/ocr/test_storage.py @@ -0,0 +1,90 @@ +"""Tests for the layout artifacts written per document.""" + +from unittest.mock import AsyncMock, patch + +import pytest +from docling_core.types.doc import BoundingBox, CoordOrigin, DocItemLabel, Size + +from extralit_server.contexts.ocr import storage +from extralit_server.contexts.ocr.docling_builder import LayoutBlock, PageContext, append_blocks, new_document +from extralit_server.contexts.ocr.layout_store import LayoutStore + +DOCUMENT_ID = "11111111-2222-3333-4444-555555555555" +WORKSPACE = "ws1" + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +def doc(): + document = new_document("sample") + ctx = PageContext(page_no=1, size=Size(width=612, height=792)) + append_blocks( + document, + ctx, + [ + LayoutBlock( + label=DocItemLabel.TEXT, + bbox=BoundingBox(l=10, t=10, r=100, b=30, coord_origin=CoordOrigin.TOPLEFT), + text="Body", + ) + ], + ) + return document + + +@pytest.fixture +def store(tmp_path): + layout_store = LayoutStore(str(tmp_path / WORKSPACE / "layout")) + with patch.object(storage.LayoutStore, "for_workspace", return_value=layout_store): + yield layout_store + + +class TestStoreLayout: + async def test_writes_canonical_json_and_returns_lance_uris(self, doc, store): + with patch.object(storage.files, "put_object", AsyncMock()) as put_object: + paths = await storage.store_layout(AsyncMock(), WORKSPACE, DOCUMENT_ID, doc) + + assert paths["layout_url"] == f"layout/{DOCUMENT_ID}.docling.json" + assert paths["items_uri"] == store.items_uri() + assert paths["pages_uri"] == store.pages_uri() + assert paths["items_version"] == 1 and paths["pages_version"] == 1 + assert put_object.await_count == 1 + assert put_object.await_args.kwargs["content_type"] == "application/json" + + async def test_rows_land_in_the_workspace_datasets(self, doc, store): + with patch.object(storage.files, "put_object", AsyncMock()): + await storage.store_layout(AsyncMock(), WORKSPACE, DOCUMENT_ID, doc) + + assert store.load_items(DOCUMENT_ID).num_rows == 1 + assert store.load_pages(DOCUMENT_ID).num_rows == 1 + + async def test_a_caller_holding_the_lock_can_pass_its_own_store(self, doc, store): + with patch.object(storage.files, "put_object", AsyncMock()): + async with store.locked(): + await storage.store_layout(AsyncMock(), WORKSPACE, DOCUMENT_ID, doc, store=store) + + assert store.load_items(DOCUMENT_ID).num_rows == 1 + + +class TestDeleteLayout: + async def test_removes_json_and_rows(self, doc, store): + with patch.object(storage.files, "put_object", AsyncMock()): + await storage.store_layout(AsyncMock(), WORKSPACE, DOCUMENT_ID, doc) + + with patch.object(storage.files, "delete_object", AsyncMock()) as delete_object: + await storage.delete_layout(AsyncMock(), WORKSPACE, DOCUMENT_ID) + + assert delete_object.await_args.args[2] == f"layout/{DOCUMENT_ID}.docling.json" + assert store.load_items(DOCUMENT_ID).num_rows == 0 + + async def test_survives_a_missing_object(self, store): + with patch.object(storage.files, "delete_object", AsyncMock(side_effect=RuntimeError("gone"))): + await storage.delete_layout(AsyncMock(), WORKSPACE, DOCUMENT_ID) + + async def test_survives_a_failing_dataset(self, store): + with ( + patch.object(storage.files, "delete_object", AsyncMock()), + patch.object(LayoutStore, "delete_document", side_effect=RuntimeError("lance down")), + ): + await storage.delete_layout(AsyncMock(), WORKSPACE, DOCUMENT_ID) diff --git a/extralit-server/tests/unit/contexts/test_files_artifacts.py b/extralit-server/tests/unit/contexts/test_files_artifacts.py new file mode 100644 index 000000000..894969312 --- /dev/null +++ b/extralit-server/tests/unit/contexts/test_files_artifacts.py @@ -0,0 +1,46 @@ +"""Tests for the per-document artifact fan-out.""" + +from unittest.mock import AsyncMock, patch +from uuid import uuid4 + +import pytest + +from extralit_server.contexts import files + +pytestmark = pytest.mark.asyncio + +WORKSPACE = "ws1" + + +class TestDeleteDocumentArtifacts: + async def test_every_artifact_of_the_document_is_removed(self): + document_id = uuid4() + + with ( + patch.object(files, "delete_object", AsyncMock()) as delete_object, + patch("extralit_server.contexts.ocr.storage.delete_layout", AsyncMock()) as delete_layout, + ): + await files.delete_document_artifacts(AsyncMock(), WORKSPACE, document_id) + + deleted = [call.args[2] for call in delete_object.await_args_list] + assert deleted == [f"pdf/{document_id}", f"thumbnails/{document_id}"] + assert delete_layout.await_args.args[1:] == (WORKSPACE, document_id) + + async def test_a_failing_artifact_does_not_stop_the_others(self): + document_id = uuid4() + + with ( + patch.object(files, "delete_object", AsyncMock(side_effect=RuntimeError("gone"))) as delete_object, + patch("extralit_server.contexts.ocr.storage.delete_layout", AsyncMock()) as delete_layout, + ): + await files.delete_document_artifacts(AsyncMock(), WORKSPACE, document_id) + + assert delete_object.await_count == 2 + assert delete_layout.await_count == 1 + + async def test_a_failing_layout_delete_is_swallowed(self): + with ( + patch.object(files, "delete_object", AsyncMock()), + patch("extralit_server.contexts.ocr.storage.delete_layout", AsyncMock(side_effect=RuntimeError("boom"))), + ): + await files.delete_document_artifacts(AsyncMock(), WORKSPACE, uuid4()) diff --git a/extralit-server/tests/unit/jobs/test_document_jobs.py b/extralit-server/tests/unit/jobs/test_document_jobs.py index f3fda7f38..7803584fa 100644 --- a/extralit-server/tests/unit/jobs/test_document_jobs.py +++ b/extralit-server/tests/unit/jobs/test_document_jobs.py @@ -114,8 +114,10 @@ async def test_analysis_and_preprocess_job_success( mock_analyzer.analyze_pdf_layout.assert_called_once() mock_preprocessor.preprocess.assert_called_once() - # Verify database operations - mock_db.get.assert_called_once_with(Document, document_id) + # Verify database operations: the metadata write goes through the row lock. + mock_db.execute.assert_awaited_once() + assert "FOR UPDATE" in str(mock_db.execute.await_args.args[0]) + mock_db.get.assert_called_once_with(Document, document_id, populate_existing=True) mock_db.commit.assert_called_once() @patch("extralit_server.jobs.document_jobs.files") diff --git a/extralit-server/tests/unit/jobs/test_ocr_jobs.py b/extralit-server/tests/unit/jobs/test_ocr_jobs.py new file mode 100644 index 000000000..39d59fa69 --- /dev/null +++ b/extralit-server/tests/unit/jobs/test_ocr_jobs.py @@ -0,0 +1,85 @@ +"""Tests for how the layout job orders its writes.""" + +from unittest.mock import AsyncMock, MagicMock, patch +from uuid import uuid4 + +import pytest + +MODULE = "extralit_server.jobs.ocr_jobs" + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +def job_context(tmp_path, monkeypatch): + """Everything the layout job touches outside its own logic, with the document present.""" + from extralit_server.contexts.ocr.layout_store import LayoutStore + + calls = [] + store = LayoutStore(str(tmp_path / "ws" / "layout")) + doc = MagicMock(version="1.7.0", pages={1: MagicMock()}) + doc.iterate_items.return_value = [] + + async def store_layout(s3_client, workspace_name, document_id, document, store=None): + calls.append(("store_layout", store._lock_depth)) + return {"layout_url": "layout/x.docling.json", "items_uri": "items", "pages_uri": "pages"} + + async def update_metadata(db, document_id, mutate): + calls.append(("update_metadata", store._lock_depth)) + return None + + db = MagicMock() + db.scalar = AsyncMock(return_value=uuid4()) + + with ( + patch(f"{MODULE}.files.get_s3_client", AsyncMock(return_value=AsyncMock())), + patch(f"{MODULE}.files.download_file_content", AsyncMock(return_value=b"%PDF-1.4")), + patch(f"{MODULE}.route_parser", return_value=("pdf_inspector", {"pages_needing_ocr": [2]})), + patch(f"{MODULE}.get_parser", return_value=lambda *args, **kwargs: doc), + patch(f"{MODULE}.LayoutStore.for_workspace", return_value=store), + patch(f"{MODULE}.storage.store_layout", store_layout), + patch(f"{MODULE}.update_processing_metadata", update_metadata), + patch(f"{MODULE}.get_current_job", return_value=None), + patch(f"{MODULE}.AsyncSessionLocal") as session, + ): + session.return_value.__aenter__ = AsyncMock(return_value=db) + session.return_value.__aexit__ = AsyncMock(return_value=False) + yield {"calls": calls, "db": db, "store": store} + + +async def run_job(document_id=None): + from extralit_server.jobs.ocr_jobs import async_document_layout_job + + return await async_document_layout_job( + document_id or uuid4(), + "/api/v1/file/ws/pdf/doc.pdf", + "ws", + "pdf_inspector", + ) + + +class TestLayoutJobOrdering: + async def test_rows_are_written_under_the_workspace_lock(self, job_context): + await run_job() + + assert ("store_layout", 1) in job_context["calls"] + + async def test_metadata_is_updated_after_the_lock_is_released(self, job_context): + await run_job() + + steps = [name for name, _ in job_context["calls"]] + assert steps == ["store_layout", "update_metadata"] + assert ("update_metadata", 0) in job_context["calls"] + + async def test_a_document_deleted_mid_parse_is_not_resurrected(self, job_context): + job_context["db"].scalar = AsyncMock(return_value=None) + + result = await run_job() + + assert result["skipped"] == "document deleted" + assert job_context["calls"] == [] + + async def test_pages_needing_ocr_are_surfaced(self, job_context): + result = await run_job() + + assert result["pages_needing_ocr"] == [2] From 8245608170566309929a1417a2cce66f73214f47 Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Sun, 16 Aug 2026 17:19:03 -0700 Subject: [PATCH 07/10] feat(pdf): add PDF preprocessing configuration guide and integrate triage functionality This commit introduces a new documentation file detailing the PDF preprocessing configuration for the Extralit Server, specifically focusing on the rotation-only processing using OCRmyPDF. It outlines the settings available via environment variables, the workflow during PDF uploads, and troubleshooting tips. Additionally, a new triage functionality is implemented to classify PDF structures without performing OCR, identifying pages that require OCR and providing metadata about the document's layout. This enhances the overall document processing workflow by ensuring that structural analysis is performed efficiently before any preprocessing steps. The changes also include updates to the metadata models to accommodate the new triage results and adjustments in the document processing jobs to integrate the triage step effectively. --- docs/admin_guide/pdf_preprocessing_config.md | 120 ++++++ extralit-server/CHANGELOG.md | 3 + .../api/schemas/v1/document/metadata.py | 37 +- .../api/schemas/v1/document/preprocessing.py | 4 +- .../contexts/document/analysis.py | 172 -------- .../contexts/document/margin.py | 17 +- .../contexts/document/preprocessing.py | 194 +++------- .../extralit_server/contexts/ocr/triage.py | 37 ++ .../src/extralit_server/jobs/document_jobs.py | 108 ++---- .../src/extralit_server/jobs/preload.py | 2 +- .../extralit_server/workflows/documents.py | 39 +- .../integration/test_rq_groups_workflow.py | 4 +- .../contexts/document/test_preprocessing.py | 79 ++++ .../contexts/ocr/test_pdf_inspector_parser.py | 2 + .../tests/unit/contexts/ocr/test_triage.py | 38 ++ .../tests/unit/jobs/test_document_jobs.py | 366 ++++++++---------- .../test_document_workflow_layout.py | 10 +- 17 files changed, 606 insertions(+), 626 deletions(-) create mode 100644 docs/admin_guide/pdf_preprocessing_config.md delete mode 100644 extralit-server/src/extralit_server/contexts/document/analysis.py create mode 100644 extralit-server/src/extralit_server/contexts/ocr/triage.py create mode 100644 extralit-server/tests/unit/contexts/document/test_preprocessing.py create mode 100644 extralit-server/tests/unit/contexts/ocr/test_triage.py diff --git a/docs/admin_guide/pdf_preprocessing_config.md b/docs/admin_guide/pdf_preprocessing_config.md new file mode 100644 index 000000000..c8310f226 --- /dev/null +++ b/docs/admin_guide/pdf_preprocessing_config.md @@ -0,0 +1,120 @@ +# PDF Preprocessing Configuration Guide + +Extralit Server runs [OCRmyPDF](https://github.com/ocrmypdf/ocrmypdf) over every uploaded PDF for +**page rotation only**. No OCR text is produced: `tesseract_timeout=0` kills the tesseract OCR +spawn and `skip_text` leaves pages that already have text untouched. The one thing tesseract is +still asked for is OSD — deciding whether a page is sideways — which is why its budget is bounded. + +Settings live in `PDFPreprocessingSettings` and are configurable via `PREPROCESSING_`-prefixed +environment variables. + +## What runs on an upload + +| Step | Component | Output | +|---|---|---| +| Triage | `contexts/ocr/triage.py` (pdf-inspector) | `pdf_type`, `pages_needing_ocr`, `pages_with_tables`, `pages_with_columns`, encoding issues | +| Margins + thumbnail | `contexts/document/margin.py` (`PDFAnalyzer`) | `analysis_metadata.layout_analysis.margin_analysis`, thumbnail object | +| Rotation | `contexts/document/preprocessing.py` (ocrmypdf) | the PDF rewritten at the same key | +| Layout | `jobs/ocr_jobs.py` | canonical `DoclingDocument` JSON + the workspace's Lance rows | + +Triage classifies which pages have no usable text, but pdf-inspector bundles no OCR engine and +tesseract OCR is off, so those pages are **surfaced, not fixed**: they appear in +`analysis_metadata.triage.pages_needing_ocr` and `layout_metadata.pages_needing_ocr` and stay an +explicit gap until an OCR job exists. + +## Configuration Reference + +#### `PREPROCESSING_ENABLED` +- **Type**: `bool` — **Default**: `true` +- Master switch. When `false`, the PDF is passed through byte-identical; triage, margins and the + thumbnail still run (they are the analysis job's own work, not the preprocessor's). + +#### `PREPROCESSING_ROTATE_PAGES` +- **Type**: `bool` — **Default**: `true` +- Auto-rotate pages whose text is not upright. + +#### `PREPROCESSING_ROTATE_PAGES_THRESHOLD` +- **Type**: `float` — **Default**: `2.0` +- Confidence OSD must reach before a page is rotated. Lower (1.0–1.5) rotates more eagerly; + higher (3.0+) avoids false rotations. + +#### `PREPROCESSING_TESSERACT_NON_OCR_TIMEOUT` +- **Type**: `float` (seconds per page) — **Default**: `30.0` +- Budget for OSD, the only tesseract call made here. OCRmyPDF's own default is 180 s per page, + which dominates the runtime on image-heavy PDFs. + +#### `PREPROCESSING_JOBS` +- **Type**: `int` — **Default**: `1` +- Worker processes for ocrmypdf. Keep at `1` in containers with limited CPU to avoid + oversubscription; `2–4` on a multi-core host. + +#### `PREPROCESSING_PROGRESS_BAR` +- **Type**: `bool` — **Default**: `false` +- Useful interactively, noise in background jobs. + +### Fixed, not configurable + +`skip_text=True`, `tesseract_timeout=0`, `clean=False`, `optimize=0`. These are what make the pass +rotation-only: `clean` (unpaper) and `optimize` only pay off alongside OCR output, and the +alternatives that would OCR image pages are destructive — `force_ocr` rasterizes the existing text +layer, and `redo_ocr` strips invisible OCR text it cannot regenerate with tesseract disabled. + +### Known limit + +Verified in `ocrmypdf/_pipeline.py::is_ocr_required`: under `skip_text`, OSD only runs on pages +ocrmypdf would process, i.e. image-only pages. A born-digital page keeps whatever `/Rotate` it +already has. A text-only PDF still pays pdfinfo, a re-save and the S3 rewrite — cheap, and nothing +is rasterized. + +## Troubleshooting + +### Rotation is slow on scanned PDFs + +OSD is the cost. Lower `PREPROCESSING_TESSERACT_NON_OCR_TIMEOUT`, or set +`PREPROCESSING_ROTATE_PAGES=false` to skip orientation detection entirely. + +### A page is rotated the wrong way + +Raise `PREPROCESSING_ROTATE_PAGES_THRESHOLD` so OSD needs more confidence before acting. + +### Rotation failed + +The job records it and keeps going: the original bytes are stored, and +`preprocessing_metadata.rotation_ran` is `false` with the reason in `preprocessing_metadata.error`. +Nothing downstream is blocked, because layout and text extraction depend on this job with +`allow_failure`. + +### High memory usage + +Set `PREPROCESSING_JOBS=1`. Rotation itself holds one page image at a time. + +## Integration Example + +```python +from extralit_server.contexts.document.preprocessing import ( + PDFPreprocessingSettings, + PDFPreprocessor, +) + +settings = PDFPreprocessingSettings(rotate_pages=True, tesseract_non_ocr_timeout=15.0, jobs=2) +result = PDFPreprocessor(settings).preprocess(pdf_bytes, "document.pdf") + +print(result.metadata.rotation_ran, result.metadata.processing_time, result.metadata.error) +processed_pdf = result.processed_data +``` + +## Related Components + +| File | Purpose | +|------|---------| +| [`preprocessing.py`](../../extralit-server/src/extralit_server/contexts/document/preprocessing.py) | The rotation pass and its settings | +| [`triage.py`](../../extralit-server/src/extralit_server/contexts/ocr/triage.py) | Structural classification (pdf-inspector) | +| [`margin.py`](../../extralit-server/src/extralit_server/contexts/document/margin.py) | Margin detection and thumbnail, over the leading pages | +| [`document/metadata.py`](../../extralit-server/src/extralit_server/api/schemas/v1/document/metadata.py) | What lands in `documents.metadata_` | + +## Further Reading + +- [OCRmyPDF Documentation](https://ocrmypdf.readthedocs.io/) +- [Extralit](https://github.com/Extralit/extralit) +- [Extralit HF Space](https://github.com/Extralit/extralit-hf-space) +- [Papers OCR Benchmarks](https://github.com/Extralit/papers-ocr-benchmarks) diff --git a/extralit-server/CHANGELOG.md b/extralit-server/CHANGELOG.md index 548738f2d..dc8700df5 100644 --- a/extralit-server/CHANGELOG.md +++ b/extralit-server/CHANGELOG.md @@ -26,6 +26,8 @@ These are the section headers that we use: ### Changed +- The analysis job now triages PDFs with pdf-inspector (`pdf_type`, `pages_needing_ocr`, tables, columns) instead of the pdfminer OCR-layer detector, estimates margins over the leading five pages instead of rendering every page, and runs ocrmypdf for page rotation only, with tesseract OCR disabled and a bounded OSD budget. A failed rotation is recorded and the job still succeeds. +- Layout extraction is enqueued on upload by default; `layout_parser` remains an override. - Document workflow dependents (text extraction, layout) now wait on the analysis/preprocessing job with `allow_failure`, and carry their retry policy and a 24 h result TTL through `Queue.prepare_data()` — the `@job` decorator values never applied on that path. - `POST /workflows/start?force=true` stops the previous run's jobs before enqueueing new ones, whose ids now carry the run suffix. - Deleting a document removes its PDF, thumbnail, layout JSON and layout rows together, through `files.delete_document_artifacts`. @@ -33,6 +35,7 @@ These are the section headers that we use: ### Removed +- Removed `contexts/document/analysis.py` (`PDFOCRLayerDetector`) and the OCR-only preprocessing knobs (`language`, `force_ocr`, `redo_ocr`, `skip_big`, `pdf_renderer`, `output_type`, `fast_web_view`, `deskew`, `enable_analysis`); `has_ocr_text_layer`, `needs_ocr` and `ocr_quality` are no longer written. - Removed the `marker` extra (`marker-pdf`, `torch`, `torchvision`, `transformers`) and the unused `async_marker_layout_job`. - Removed the deprecated Unstructured-shaped `document/chunks.py` schemas (`Segments`, `TextSegment`, `TableSegment`, `FigureSegment`, `Coordinates`) and the unreferenced `GPU_QUEUE`. diff --git a/extralit-server/src/extralit_server/api/schemas/v1/document/metadata.py b/extralit-server/src/extralit_server/api/schemas/v1/document/metadata.py index 6ec0419a2..2bef5c758 100644 --- a/extralit-server/src/extralit_server/api/schemas/v1/document/metadata.py +++ b/extralit-server/src/extralit_server/api/schemas/v1/document/metadata.py @@ -24,12 +24,33 @@ class LayoutAnalysisMetadata(BaseModel): margin_analysis: dict[str, Any] = Field(default_factory=dict, description="Margin analysis results") +class TriageMetadata(BaseModel): + """Structural classification of the PDF, from pdf-inspector's page objects. + + `pages_needing_ocr` fires on any image-bearing page under ~1400 characters, so figure-heavy + papers are false positives. Good enough to gate rotation and to surface the OCR gap; nothing + is skipped on the strength of it. + """ + + pdf_type: str = Field(..., description="text_based, image_based, mixed or unknown") + confidence: float = Field(default=0.0, description="Classifier confidence, 0.0-1.0") + page_count: int = Field(default=0, description="Number of pages") + pages_needing_ocr: list[int] = Field(default_factory=list, description="1-indexed pages with no usable text") + ocr_reasons_by_page: dict[str, list[str]] = Field( + default_factory=dict, description="Why each of those pages was flagged, keyed by page number" + ) + pages_with_tables: list[int] = Field(default_factory=list, description="1-indexed pages carrying tables") + pages_with_columns: list[int] = Field(default_factory=list, description="1-indexed multi-column pages") + has_encoding_issues: bool = Field(default=False, description="Whether text decodes to mojibake") + + class AnalysisMetadata(BaseModel): """Analysis job results stored in documents.metadata_.""" - has_ocr_text_layer: Optional[bool] = Field(None, description="Whether PDF has OCR text layer") - needs_ocr: Optional[bool] = Field(None, description="Whether additional OCR processing is needed") - ocr_quality: OCRQualityMetadata = Field(..., description="OCR quality analysis") + triage: Optional[TriageMetadata] = Field(None, description="Structural classification of the PDF") + has_ocr_text_layer: Optional[bool] = Field(None, description="Deprecated, no longer written") + needs_ocr: Optional[bool] = Field(None, description="Deprecated, no longer written") + ocr_quality: Optional[OCRQualityMetadata] = Field(None, description="Deprecated, no longer written") layout_analysis: LayoutAnalysisMetadata = Field(..., description="Layout analysis results") thumbnail_generated: Optional[bool] = Field( None, description="Whether a thumbnail was generated during layout analysis" @@ -95,18 +116,18 @@ def update_analysis_results(self, analysis_result: dict) -> None: elif "estimated_margins" in layout_data: margin_analysis = layout_data["estimated_margins"] + triage = analysis_result.get("triage") self.analysis_metadata = AnalysisMetadata( thumbnail_generated=analysis_result.get("thumbnail_generated"), - has_ocr_text_layer=analysis_result.get("has_ocr_text_layer"), - needs_ocr=analysis_result.get("needs_ocr"), - ocr_quality=OCRQualityMetadata(**analysis_result.get("analysis_metadata", {})), + triage=TriageMetadata(**triage) if isinstance(triage, dict) else triage, layout_analysis=LayoutAnalysisMetadata( - page_count=layout_data.get("page_count"), + page_count=analysis_result.get("page_count") or layout_data.get("page_count"), + has_tables=bool(triage.get("pages_with_tables")) if isinstance(triage, dict) else False, margin_analysis=margin_analysis, **{ k: v for k, v in layout_data.items() - if k not in ["layout_analysis", "estimated_margins", "page_count"] + if k not in ["layout_analysis", "estimated_margins", "page_count", "pages_sampled", "has_tables"] }, ), ) diff --git a/extralit-server/src/extralit_server/api/schemas/v1/document/preprocessing.py b/extralit-server/src/extralit_server/api/schemas/v1/document/preprocessing.py index 2b44a417a..32232f012 100644 --- a/extralit-server/src/extralit_server/api/schemas/v1/document/preprocessing.py +++ b/extralit-server/src/extralit_server/api/schemas/v1/document/preprocessing.py @@ -37,10 +37,10 @@ class PDFMetadata(BaseModel): filename: str processing_time: float + rotation_ran: bool = False + error: str | None = None page_count: int | None = None - language_detected: list[str] | None = None processing_settings: dict | None = None - analysis_results: dict | None = None def model_dump(self, **kwargs) -> dict[str, Any]: """ diff --git a/extralit-server/src/extralit_server/contexts/document/analysis.py b/extralit-server/src/extralit_server/contexts/document/analysis.py deleted file mode 100644 index 328f48cad..000000000 --- a/extralit-server/src/extralit_server/contexts/document/analysis.py +++ /dev/null @@ -1,172 +0,0 @@ -from io import BytesIO - -from pdfminer.converter import PDFPageAggregator -from pdfminer.layout import LAParams, LTChar, LTTextBox -from pdfminer.pdfinterp import PDFPageInterpreter, PDFResourceManager -from pdfminer.pdfpage import PDFPage - - -class PDFOCRLayerDetector: - def __init__(self): - self.resource_manager = PDFResourceManager() - self.laparams = LAParams() - self.device = PDFPageAggregator(self.resource_manager, laparams=self.laparams) - self.interpreter = PDFPageInterpreter(self.resource_manager, self.device) - - def has_ocr_text_layer(self, pdf_bytes: bytes, threshold: float = 0.5, verbose=False) -> bool: - """ - Detect if PDF has OCR text layer by analyzing font resources per page. - Returns True if more than 50% of pages have font resources (indicating searchable text). - - Args: - pdf_bytes: PDF file content as bytes - - Returns: - bool: True if PDF has OCR text layer, False otherwise - """ - page_info = self._check_font_resources_per_page(pdf_bytes) - - if not page_info: - return False - - if verbose: - print(f"Total pages: {len(page_info)}") - print(page_info) - - pages_with_fonts = sum(1 for page in page_info if page.get("has_fonts", False)) - page_count = len(page_info) - - # Return True if more than 50% of pages have fonts - return pages_with_fonts > (page_count * threshold) - - def _check_font_resources_per_page(self, pdf_bytes: bytes) -> list[dict]: - """ - Check each page for font resources - indicates searchable text - """ - page_info = [] - - pdf_stream = BytesIO(pdf_bytes) - for page_num, page in enumerate(PDFPage.get_pages(pdf_stream)): - page_data = { - "page_number": page_num + 1, - "has_fonts": False, - "font_count": 0, - "has_images": False, - "resource_types": [], - } - - if hasattr(page, "resources") and page.resources: - resources = page.resources - - if "Font" in resources: - page_data["has_fonts"] = True - font_resource = resources["Font"] - try: - page_data["font_count"] = len(font_resource) # type: ignore - except (TypeError, AttributeError): - page_data["font_count"] = 1 - - if "XObject" in resources: - page_data["has_images"] = True - - page_data["resource_types"] = list(resources.keys()) - - page_info.append(page_data) - - return page_info - - def analyze_character_quality(self, pdf_bytes: bytes) -> dict: - char_stats = { - "total_chars": 0, - "font_variations": set(), - "suspicious_patterns": 0, - "ocr_artifacts": 0, - "avg_char_size": 0, - "size_variations": [], - } - - pdf_stream = BytesIO(pdf_bytes) - for page in PDFPage.get_pages(pdf_stream): - self.interpreter.process_page(page) - layout = self.device.get_result() - - for element in layout: - if isinstance(element, LTTextBox): - for line in element: - for char in line: - if isinstance(char, LTChar): - char_stats["total_chars"] += 1 - - if self._is_ocr_artifact(char): - char_stats["ocr_artifacts"] += 1 - - if self._is_suspicious_char(char): - char_stats["suspicious_patterns"] += 1 - - char_stats["ocr_quality_score"] = self._calculate_quality_score(char_stats) - - return char_stats - - def _is_ocr_artifact(self, char: LTChar) -> bool: - if "hidden" in char.fontname.lower() or "ocr" in char.fontname.lower(): - return True - - char_text = char.get_text() - if len(char_text) == 1: - # Look for replacement characters or unusual Unicode - if ord(char_text) > 65535 or char_text in ["�", "□", "▯"]: - return True - - return False - - def _is_suspicious_char(self, char: LTChar) -> bool: - char_text = char.get_text() - - # Single character that's not alphanumeric or common punctuation - if len(char_text) == 1 and not (char_text.isalnum() or char_text in ".,!?;: "): - return True - - # Very small font size (might indicate hidden text) - if char.size < 1.0: - return True - - return False - - def _calculate_quality_score(self, char_stats: dict) -> float: - if char_stats["total_chars"] == 0: - return 0.0 - - score = 1.0 - - # Penalize OCR artifacts - artifact_ratio = char_stats["ocr_artifacts"] / char_stats["total_chars"] - score -= artifact_ratio * 0.5 - - # Penalize suspicious patterns - suspicious_ratio = char_stats["suspicious_patterns"] / char_stats["total_chars"] - score -= suspicious_ratio * 0.3 - - return max(0.0, min(1.0, score)) - - -if __name__ == "__main__": - import sys - from pathlib import Path - - if len(sys.argv) != 2: - print("Usage: python analysis.py ") - sys.exit(1) - - pdf_path = sys.argv[1] - if not Path(pdf_path).is_file(): - print(f"File not found: {pdf_path}") - sys.exit(1) - - with open(pdf_path, "rb") as f: - pdf_bytes = f.read() - - ocr_detector = PDFOCRLayerDetector() - has_ocr = ocr_detector.has_ocr_text_layer(pdf_bytes) - print(f"PDF has_ocr_text_layer: {has_ocr}") - ocr_quality = ocr_detector.analyze_character_quality(pdf_bytes) - print(f"PDF analyze_character_quality: {ocr_quality}") diff --git a/extralit-server/src/extralit_server/contexts/document/margin.py b/extralit-server/src/extralit_server/contexts/document/margin.py index 5eeb7f2c6..fac7f675f 100644 --- a/extralit-server/src/extralit_server/contexts/document/margin.py +++ b/extralit-server/src/extralit_server/contexts/document/margin.py @@ -25,6 +25,9 @@ _LOGGER = logging.getLogger(__name__) +#: Margins are estimated by comparing the leading pages; the rest are never rendered. +MARGIN_SAMPLE_PAGES = 5 + def pil_to_cv(image: "Image") -> "NDArray": """Convert PIL Image to OpenCV format.""" @@ -120,24 +123,28 @@ def find_horizontal_bands(mask: "Image", min_height: int = 15, min_ratio: float class PDFAnalyzer: - def analyze_pdf_layout(self, pdf_data: bytes, filename: str) -> tuple[dict[str, Any], Optional[bytes]]: + def analyze_pdf_layout( + self, pdf_data: bytes, filename: str, max_pages: int = MARGIN_SAMPLE_PAGES + ) -> tuple[dict[str, Any], Optional[bytes]]: """ Analyze PDF layout to extract margin and region information. Args: pdf_data: PDF file data as bytes filename: Filename for logging + max_pages: how many leading pages to render; only these are ever compared Returns: Tuple of (dictionary containing layout analysis metadata, thumbnail bytes or None) """ try: - images = pdf2image.convert_from_bytes(pdf_data, dpi=150) # type: ignore + # Rendering every page at 150 DPI cost minutes on long PDFs for a five-page comparison. + images = pdf2image.convert_from_bytes(pdf_data, dpi=150, first_page=1, last_page=max_pages) # type: ignore if not images: return {"error": "No pages found"}, None - _LOGGER.info(f"Analyzing layout for {filename} with {len(images)} pages") + _LOGGER.info(f"Analyzing layout for {filename} with {len(images)} of its leading pages") # Generate thumbnail from first page thumbnail_bytes = None @@ -152,7 +159,7 @@ def analyze_pdf_layout(self, pdf_data: bytes, filename: str) -> tuple[dict[str, layout_data = self._analyze_page_layout(images) layout_result = { - "page_count": len(images), + "pages_sampled": len(images), "page_dimensions": {"width": images[0].size[0], "height": images[0].size[1]} if images else {}, **layout_data, } @@ -174,7 +181,7 @@ def _analyze_page_layout(self, images: list["Image"]) -> dict[str, Any]: reference_img = images[0].convert("RGB") margin_data = [] - for i in range(1, min(len(images), 5)): # Analyze up to 5 pages for efficiency + for i in range(1, len(images)): compare_img = images[i].convert("RGB") page_margins = self._compare_pages_for_margins(reference_img, compare_img) if page_margins: diff --git a/extralit-server/src/extralit_server/contexts/document/preprocessing.py b/extralit-server/src/extralit_server/contexts/document/preprocessing.py index 72010beb1..a67b1da4c 100644 --- a/extralit-server/src/extralit_server/contexts/document/preprocessing.py +++ b/extralit-server/src/extralit_server/contexts/document/preprocessing.py @@ -1,4 +1,9 @@ -"""Document preprocessing utilities.""" +"""Rotation-only ocrmypdf pass. + +No OCR is produced here: `tesseract_timeout=0` kills the tesseract spawn, and `skip_text` leaves +text pages untouched. OSD (page orientation) is the one thing tesseract is still asked for, which +is why its budget is bounded. Margins and the thumbnail belong to the analysis job. +""" import logging import os @@ -13,7 +18,6 @@ from pydantic_settings import BaseSettings from extralit_server.api.schemas.v1.document.preprocessing import PDFMetadata -from extralit_server.contexts.document.margin import PDFAnalyzer ocrmypdf = lazy.load("ocrmypdf") @@ -32,96 +36,47 @@ class PDFProcessingResponse: class PDFPreprocessingSettings(BaseSettings): """ - PDF preprocessing settings that can be configured via environment variables. - - All settings have the PREPROCESSING_ prefix. + PDF preprocessing settings, configurable via `PREPROCESSING_`-prefixed environment variables. """ class Config: env_prefix = "PREPROCESSING_" - enabled: bool = Field( - default=True, description="Enable PDF preprocessing with OCRmyPDF. Set to False to disable all processing." - ) - - enable_analysis: bool = Field(default=True, description="Enable PDF layout analysis and margin detection") - - language: list[str] = Field( - default=["eng"], description="List of languages for OCR processing (e.g., ['eng', 'spa', 'fra'])" - ) + enabled: bool = Field(default=True, description="Run ocrmypdf at all. False leaves the PDF byte-identical.") - rotate_pages: bool = Field(default=True, description="Auto-rotate pages with horizontal text") + rotate_pages: bool = Field(default=True, description="Auto-rotate pages whose text is not upright") rotate_pages_threshold: float = Field( - default=2.0, - description="Threshold for auto-rotation", + default=2.0, description="Confidence tesseract's OSD must reach before a page is rotated" ) - deskew: bool = Field(default=False, description="Fix skewed text") - - clean: bool = Field(default=True, description="Use `unpaper` to clean up artifacts") - - optimize: int = Field( - default=1, description="Optimize output file size (0=none, 1=lossless, 2=lossy, 3=aggressive)" - ) - - pdf_renderer: str = Field(default="hocr", description="PDF renderer: 'auto', 'hocr', 'sandwich'") - - force_ocr: bool = Field(default=False, description="Force OCR on all pages, even if they already have text") - - skip_text: bool = Field(default=True, description="Skip text-based operations (OCR only for images)") - - redo_ocr: bool = Field(default=False, description="Redo OCR on pages that already have OCR") - - tesseract_timeout: int = Field( - default=0, description="Timeout for Tesseract OCR processing in seconds (0 to skip Tesseract OCR)" + tesseract_non_ocr_timeout: float = Field( + default=30.0, + description="Per-page budget for OSD, the only tesseract call made here (ocrmypdf's own default is 180s)", ) progress_bar: bool = Field(default=False, description="Show progress bar during processing") - output_type: str = Field( - default="pdf", - description="Output type for OCRmyPDF. Set to 'pdf' to skip PDF/A conversion.", - ) - - fast_web_view: int = Field( - default=999999, - description="Fast web view optimization. Set to 999999 to disable fast web view optimization.", - ) - - skip_big: float = Field( - default=100.0, - description="Image size threshold in MB to skip OCR processing.", - ) - jobs: int = Field( default=1, - description="Number of worker processes to use for OCR. Set to 1 for Docker containers with limited CPU to avoid oversubscription.", + description="Worker processes for ocrmypdf. 1 in containers with limited CPU, to avoid oversubscription.", ) def get_ocrmypdf_args(self) -> dict: - """ - Get OCRmyPDF arguments as a dictionary for use with **kwargs. + """Arguments for `ocrmypdf.ocr`, with everything OCR-shaped nailed shut. - Returns: - Dictionary of OCRmyPDF arguments excluding input/output parameters. + `clean` (unpaper) and `optimize` only pay off alongside OCR output, and rasterizing + alternatives (`force_ocr`, `redo_ocr`) would destroy the text layer this pipeline relies on. """ return { - "language": self.language, "rotate_pages": self.rotate_pages, "rotate_pages_threshold": self.rotate_pages_threshold, - "deskew": self.deskew, - "clean": self.clean, - "optimize": self.optimize, - "pdf_renderer": self.pdf_renderer, - "force_ocr": self.force_ocr, - "skip_text": self.skip_text, - "tesseract_timeout": self.tesseract_timeout, - "redo_ocr": self.redo_ocr, + "skip_text": True, + "tesseract_timeout": 0, + "tesseract_non_ocr_timeout": self.tesseract_non_ocr_timeout, + "clean": False, + "optimize": 0, "progress_bar": self.progress_bar, - "output_type": self.output_type, - "fast_web_view": self.fast_web_view, - "skip_big": self.skip_big, "jobs": self.jobs, } @@ -130,86 +85,49 @@ def get_ocrmypdf_args(self) -> dict: class PDFPreprocessor: - """ - PDF preprocessor that uses OCRmyPDF for rotation, OCR, and optimization. - Also performs layout analysis to extract margin and structure information. - - Can be configured with environment variables using the PDFPreprocessingSettings. - """ + """Runs ocrmypdf over a PDF for page rotation only.""" def __init__(self, settings: PDFPreprocessingSettings = settings): - """ - Initialize the PDF preprocessor. - - Args: - settings: Optional PDFPreprocessingSettings instance. If None, loads from environment. - """ self.settings = settings - if self.settings.enable_analysis: - self.analyzer = PDFAnalyzer() - else: - self.analyzer = None - def preprocess(self, file_data: bytes, filename: str) -> PDFProcessingResponse: - """ - Preprocess PDF with OCRmyPDF and analyze layout structure. + """Rotate pages, best effort. - Args: - file_data: PDF file data as bytes - filename: Original filename for logging purposes - - Returns: - PDFProcessingResult containing processed data and layout analysis metadata + Returns the original bytes with `rotation_ran=False` and the reason in `error` when + ocrmypdf fails — a failed rotation must not cost the caller its document. """ - # Initialize metadata variables - analysis_results = None - processing_time = 0.0 - processed_data = file_data - - # Handle non-PDF files - if not filename.lower().endswith(".pdf"): - pass # Use default values - - # Handle disabled preprocessing - elif not self.settings.enabled: - if self.analyzer: - analysis_results = self.analyzer.analyze_pdf_layout(file_data, filename) - - # Handle PDF processing - else: - try: - start_time = time.time() - - # Step 1: Analyze original PDF layout (if enabled) - if self.analyzer: - analysis_results = self.analyzer.analyze_pdf_layout(file_data, filename) - - # Step 2: OCR preprocessing - try: - input_buffer = BytesIO(file_data) - output_buffer = BytesIO() - - ocrmypdf.ocr(input_buffer, output_buffer, **self.settings.get_ocrmypdf_args()) # type: ignore - - processed_data = output_buffer.getvalue() - output_buffer.close() - input_buffer.close() - - except Exception as buffer_error: - _LOGGER.debug(f"BytesIO approach failed for {filename}, falling back to temp files: {buffer_error}") - processed_data = self._preprocess_with_temp_files(file_data, filename) - - processing_time = time.time() - start_time - print(filename, analysis_results) - - except Exception: - # Use default values on error - pass + if not filename.lower().endswith(".pdf") or not self.settings.enabled: + return PDFProcessingResponse( + processed_data=file_data, + metadata=PDFMetadata(filename=filename, processing_time=0.0), + ) - # Single PDFMetadata initialization for all code paths - metadata = PDFMetadata(filename=filename, processing_time=processing_time, analysis_results=analysis_results) + start_time = time.time() + processed_data, rotation_ran, error = file_data, False, None + try: + try: + input_buffer = BytesIO(file_data) + output_buffer = BytesIO() + ocrmypdf.ocr(input_buffer, output_buffer, **self.settings.get_ocrmypdf_args()) # type: ignore + processed_data = output_buffer.getvalue() + output_buffer.close() + input_buffer.close() + except TypeError as buffer_error: + # Some ocrmypdf paths insist on real files; a genuine failure re-raises below. + _LOGGER.debug(f"BytesIO approach failed for {filename}, falling back to temp files: {buffer_error}") + processed_data = self._preprocess_with_temp_files(file_data, filename) + rotation_ran = True + except Exception as e: + _LOGGER.warning(f"Rotation failed for {filename}, keeping the original: {e}") + processed_data, error = file_data, str(e) + + metadata = PDFMetadata( + filename=filename, + processing_time=time.time() - start_time, + rotation_ran=rotation_ran, + error=error, + ) return PDFProcessingResponse(processed_data=processed_data, metadata=metadata) def _preprocess_with_temp_files(self, file_data: bytes, filename: str) -> bytes: diff --git a/extralit-server/src/extralit_server/contexts/ocr/triage.py b/extralit-server/src/extralit_server/contexts/ocr/triage.py new file mode 100644 index 000000000..778b6d354 --- /dev/null +++ b/extralit-server/src/extralit_server/contexts/ocr/triage.py @@ -0,0 +1,37 @@ +"""Cheap, structural triage of a PDF (~53 ms over 300 pages). + +pdf-inspector reads the page objects; it never rasterizes and bundles no OCR engine, so it can say +which pages *need* OCR but cannot produce text for them. `pages_needing_ocr` is therefore surfaced +as an explicit gap rather than acted on. +""" + +from __future__ import annotations + +import logging + +import pdf_inspector + +from extralit_server.api.schemas.v1.document.metadata import TriageMetadata + +_LOGGER = logging.getLogger(__name__) + + +def triage_pdf(pdf_bytes: bytes) -> TriageMetadata: + """Classify a PDF's structure. Never raises: an unreadable PDF is `pdf_type="unknown"`.""" + try: + result = pdf_inspector.detect_pdf_bytes(pdf_bytes) + except Exception as error: + _LOGGER.warning(f"PDF triage failed: {error}") + return TriageMetadata(pdf_type="unknown") + + return TriageMetadata( + pdf_type=str(result.pdf_type), + confidence=result.confidence, + page_count=result.page_count, + # detect_pdf_bytes reports 1-indexed pages (classify_pdf_bytes does not). + pages_needing_ocr=sorted(result.pages_needing_ocr or []), + ocr_reasons_by_page={str(entry.page): list(entry.reasons) for entry in (result.ocr_reasons_by_page or [])}, + pages_with_tables=sorted(result.pages_with_tables or []), + pages_with_columns=sorted(result.pages_with_columns or []), + has_encoding_issues=bool(result.has_encoding_issues), + ) diff --git a/extralit-server/src/extralit_server/jobs/document_jobs.py b/extralit-server/src/extralit_server/jobs/document_jobs.py index 6e60df977..be41b7162 100644 --- a/extralit-server/src/extralit_server/jobs/document_jobs.py +++ b/extralit-server/src/extralit_server/jobs/document_jobs.py @@ -1,6 +1,5 @@ """Document upload job functions.""" -import asyncio import logging from typing import Any from uuid import UUID @@ -10,10 +9,10 @@ from extralit_server.api.schemas.v1.document.metadata import DocumentProcessingMetadata from extralit_server.contexts import files -from extralit_server.contexts.document.analysis import PDFOCRLayerDetector from extralit_server.contexts.document.margin import PDFAnalyzer from extralit_server.contexts.document.metadata import update_processing_metadata -from extralit_server.contexts.document.preprocessing import PDFPreprocessingSettings, PDFPreprocessor +from extralit_server.contexts.document.preprocessing import PDFPreprocessor +from extralit_server.contexts.ocr.triage import triage_pdf from extralit_server.database import AsyncSessionLocal from extralit_server.jobs.queues import DEFAULT_QUEUE, REDIS_CONNECTION @@ -25,12 +24,15 @@ async def analysis_and_preprocess_job( document_id: UUID, s3_url: str, reference: str, workspace_name: str ) -> dict[str, Any]: """ - Analyze PDF structure and content, then preprocess using existing modules. + Triage a PDF, estimate its margins, rotate it, and record what was found. - This job combines PDFOCRLayerDetector, PDFAnalyzer, and PDFPreprocessor to: - 1. Analyze original PDF structure and content - 2. Preprocess PDF using OCRmyPDF for page rotation (overwrites same S3 path) - 3. Store combined results in documents.metadata_ using DocumentProcessingMetadata schema + Order matters: everything is computed before the PDF is rewritten, because that rewrite is what + dependents wait on, and a reader that sees the new bytes must also see the new metadata. + + 1. Triage (pdf-inspector): pdf_type, pages needing OCR, tables, columns — structural only. + 2. Margins + thumbnail over the leading pages. + 3. Rotation: ocrmypdf on every PDF with OCR disabled; best effort. + 4. Rewrite the PDF at the same key, then the thumbnail, then the metadata. Args: document_id: UUID of the document to process @@ -60,88 +62,60 @@ async def analysis_and_preprocess_job( pdf_data = await files.download_file_content(s3_client, s3_url) filename = s3_url.split("/")[-1] - # Step 1: Analyze original PDF structure and content - ocr_detector = PDFOCRLayerDetector() - has_ocr_text_layer = ocr_detector.has_ocr_text_layer(pdf_data) - ocr_quality = ocr_detector.analyze_character_quality(pdf_data) + triage = triage_pdf(pdf_data) - pdf_analyzer = PDFAnalyzer() - layout_analysis, thumbnail_data = pdf_analyzer.analyze_pdf_layout(pdf_data, filename) + layout_analysis, thumbnail_data = PDFAnalyzer().analyze_pdf_layout(pdf_data, filename) analysis_result = { "document_id": str(document_id), - "has_ocr_text_layer": has_ocr_text_layer, - "ocr_quality_score": ocr_quality.get("ocr_quality_score", 0.0), + "triage": triage.model_dump(), + "page_count": triage.page_count, "layout_analysis": layout_analysis, - "needs_ocr": not has_ocr_text_layer or ocr_quality.get("ocr_quality_score", 0.0) < 0.7, - "analysis_metadata": { - "total_chars": ocr_quality.get("total_chars", 0), - "ocr_artifacts": ocr_quality.get("ocr_artifacts", 0), - "suspicious_patterns": ocr_quality.get("suspicious_patterns", 0), - "ocr_quality_score": ocr_quality.get("ocr_quality_score", 0.0), - }, + "thumbnail_generated": False, } - # Step 2: Preprocess PDF (OCRmyPDF for page rotation, overwrites same S3 path) - settings = PDFPreprocessingSettings(enable_analysis=False) # Analysis already done - preprocessor = PDFPreprocessor(settings) - processing_response = preprocessor.preprocess(pdf_data, filename) + # Rotation runs on every PDF: ocrmypdf's OSD is the only thing that knows which scanned + # pages are sideways, and under skip_text a born-digital page passes through untouched. + processing_response = PDFPreprocessor().preprocess(pdf_data, filename) + if not processing_response.metadata.rotation_ran: + _LOGGER.warning(f"Rotation did not run for document {document_id}: {processing_response.metadata.error}") - # Step 2.5: Prepare concurrent file uploads - # OCRmyPDF overwrites the same S3 object path, so we upload back to same location + # The PDF rewrite is the last S3 write of this job — dependents key on it. object_path = s3_url.replace(f"/api/v1/file/{workspace_name}/", "") - upload_tasks = [ - files.put_object( - s3_client, - workspace_name, - object_path, - processing_response.processed_data, - content_type="application/pdf", - metadata={"processing_applied": "ocrmypdf_rotation", "original_filename": filename}, - ) - ] - - # Step 3: Add thumbnail upload task if thumbnail data exists - analysis_result["thumbnail_generated"] = False if thumbnail_data is not None: - thumbnail_object_path = files.get_thumbnail_s3_object_path(document_id) - upload_tasks.append( - files.put_object( + try: + await files.put_object( s3_client, workspace_name, - thumbnail_object_path, + files.get_thumbnail_s3_object_path(document_id), thumbnail_data, content_type="image/png", metadata={"original_filename": filename}, ) - ) - - # Execute uploads concurrently - try: - await asyncio.gather(*upload_tasks) - _LOGGER.info(f"Successfully uploaded processed PDF for document {document_id}") - if thumbnail_data is not None: - _LOGGER.info(f"Generated and stored thumbnail for document {document_id}") analysis_result["thumbnail_generated"] = True - except Exception as e: - _LOGGER.warning(f"Failed to upload files for document {document_id}: {e}") - if thumbnail_data is not None: - analysis_result["thumbnail_generated"] = False - # Re-raise the exception as this is a critical failure - raise - - if thumbnail_data is None: + except Exception as e: + _LOGGER.warning(f"Failed to store the thumbnail for document {document_id}: {e}") + else: _LOGGER.warning(f"No thumbnail data available for document {document_id}") - # Combine results + await files.put_object( + s3_client, + workspace_name, + object_path, + processing_response.processed_data, + content_type="application/pdf", + metadata={"processing_applied": "ocrmypdf_rotation", "original_filename": filename}, + ) + combined_result = { "document_id": str(document_id), "analysis_result": analysis_result, "preprocessing_result": { "processing_time": processing_response.metadata.processing_time, - "ocr_applied": getattr(processing_response.metadata, "ocr_applied", False), - "preprocessing_metadata": processing_response.metadata.model_dump(), + "ocr_applied": False, + "rotation_ran": processing_response.metadata.rotation_ran, + "error": processing_response.metadata.error, }, } @@ -153,8 +127,8 @@ def apply(metadata: DocumentProcessingMetadata) -> None: async with AsyncSessionLocal() as db: await update_processing_metadata(db, document_id, apply) - # Store results for dependent jobs - current_job.meta["needs_ocr"] = analysis_result["needs_ocr"] + # Read by the scheduler branch that will enqueue an OCR job once an engine exists. + current_job.meta["pages_needing_ocr"] = triage.pages_needing_ocr current_job.meta["analysis_complete"] = True current_job.meta["preprocessing_complete"] = True current_job.save_meta() diff --git a/extralit-server/src/extralit_server/jobs/preload.py b/extralit-server/src/extralit_server/jobs/preload.py index 60bb95ea7..d45809665 100644 --- a/extralit-server/src/extralit_server/jobs/preload.py +++ b/extralit-server/src/extralit_server/jobs/preload.py @@ -7,9 +7,9 @@ from extralit_server.api.schemas.v1.document.metadata import DocumentProcessingMetadata # noqa: F401 from extralit_server.contexts import files, imports, search # noqa: F401 -from extralit_server.contexts.document.analysis import PDFOCRLayerDetector # noqa: F401 from extralit_server.contexts.document.margin import PDFAnalyzer # noqa: F401 from extralit_server.contexts.document.preprocessing import PDFPreprocessingSettings, PDFPreprocessor # noqa: F401 +from extralit_server.contexts.ocr.triage import triage_pdf # noqa: F401 from extralit_server.database import AsyncSessionLocal, async_engine # noqa: F401 from extralit_server.helpers import create_s3_client # noqa: F401 from extralit_server.jobs import ( # noqa: F401 diff --git a/extralit-server/src/extralit_server/workflows/documents.py b/extralit-server/src/extralit_server/workflows/documents.py index 17f1a8767..43bd496c4 100644 --- a/extralit-server/src/extralit_server/workflows/documents.py +++ b/extralit-server/src/extralit_server/workflows/documents.py @@ -5,6 +5,7 @@ from rq.group import Group from rq.job import Dependency +from extralit_server.contexts.ocr.parsers import default_parser_name from extralit_server.database import AsyncSessionLocal from extralit_server.jobs.document_jobs import analysis_and_preprocess_job from extralit_server.jobs.ocr_jobs import async_document_layout_job @@ -37,7 +38,7 @@ async def create_document_workflow( reference: Reference key for tracking workspace_name: Workspace name for job context workspace_id: UUID of the workspace - layout_parser: Name of a layout parser to run; None skips layout extraction + layout_parser: Layout parser to run; None uses the default parser Returns: Dictionary containing workflow_id and group_id for tracking @@ -102,23 +103,25 @@ async def create_document_workflow( group.enqueue_many(queue=OCR_QUEUE, job_datas=[text_extraction_job_data]) - if layout_parser: - layout_job_data = OCR_QUEUE.prepare_data( - async_document_layout_job, - (document_id, s3_url, workspace_name, layout_parser), - timeout=1800, - job_id=f"document_layout_{document_id}_{run_suffix}", - depends_on=on_analysis, - retry=Retry(max=2, interval=[30, 60]), - result_ttl=JOB_RESULT_TTL, - meta={ - "document_id": str(document_id), - "reference": reference, - "workflow_step": "document_layout", - "workflow_id": str(workflow.id), - }, - ) - group.enqueue_many(queue=OCR_QUEUE, job_datas=[layout_job_data]) + # Deferred OCR branch: once an OCR engine is configured, an `ocr_job` is enqueued here when + # triage reports pages needing OCR, and layout depends on it instead of on triage. pdf-inspector + # classifies those pages but bundles no engine, so today they stay an explicitly surfaced gap. + layout_job_data = OCR_QUEUE.prepare_data( + async_document_layout_job, + (document_id, s3_url, workspace_name, layout_parser or default_parser_name()), + timeout=1800, + job_id=f"document_layout_{document_id}_{run_suffix}", + depends_on=on_analysis, + retry=Retry(max=2, interval=[30, 60]), + result_ttl=JOB_RESULT_TTL, + meta={ + "document_id": str(document_id), + "reference": reference, + "workflow_step": "document_layout", + "workflow_id": str(workflow.id), + }, + ) + group.enqueue_many(queue=OCR_QUEUE, job_datas=[layout_job_data]) # Step 6: Future table extraction job (conditional based on analysis results) # This will be added when table extraction is implemented diff --git a/extralit-server/tests/integration/test_rq_groups_workflow.py b/extralit-server/tests/integration/test_rq_groups_workflow.py index d410c380b..45a3a1043 100644 --- a/extralit-server/tests/integration/test_rq_groups_workflow.py +++ b/extralit-server/tests/integration/test_rq_groups_workflow.py @@ -146,9 +146,9 @@ async def test_create_document_workflow_with_rq_groups( assert workflow.status == "running" assert workflow.group_id.startswith(f"document_workflow_{test_document.id}") - # Verify jobs were prepared and enqueued + # Verify jobs were prepared and enqueued: analysis on default, text + layout on ocr. mock_default_queue.prepare_data.assert_called_once() - mock_ocr_queue.prepare_data.assert_called_once() + assert mock_ocr_queue.prepare_data.call_count == 2 mock_group.enqueue_many.assert_called() # The workflow must persist from its own session. Without this the test passes diff --git a/extralit-server/tests/unit/contexts/document/test_preprocessing.py b/extralit-server/tests/unit/contexts/document/test_preprocessing.py new file mode 100644 index 000000000..854d8543c --- /dev/null +++ b/extralit-server/tests/unit/contexts/document/test_preprocessing.py @@ -0,0 +1,79 @@ +"""Tests for the rotation-only ocrmypdf pass.""" + +from unittest.mock import MagicMock, patch + +import pytest + +from extralit_server.contexts.document.preprocessing import PDFPreprocessingSettings, PDFPreprocessor + +MODULE = "extralit_server.contexts.document.preprocessing" + + +class TestOcrmypdfArgs: + def test_ocr_is_off_and_only_rotation_is_paid_for(self): + args = PDFPreprocessingSettings().get_ocrmypdf_args() + + assert args == { + "rotate_pages": True, + "rotate_pages_threshold": 2.0, + "skip_text": True, + "tesseract_timeout": 0, + "tesseract_non_ocr_timeout": 30.0, + "clean": False, + "optimize": 0, + "progress_bar": False, + "jobs": 1, + } + + def test_no_ocr_only_knob_survives(self): + fields = PDFPreprocessingSettings.model_fields + + for dropped in ("language", "force_ocr", "redo_ocr", "skip_big", "pdf_renderer", "output_type", "deskew"): + assert dropped not in fields + + +class TestPreprocess: + def test_rotation_runs_on_every_pdf(self): + settings = PDFPreprocessingSettings() + + with patch(f"{MODULE}.ocrmypdf") as ocrmypdf: + ocrmypdf.ocr = MagicMock() + response = PDFPreprocessor(settings).preprocess(b"%PDF-1.5", "paper.pdf") + + assert ocrmypdf.ocr.call_args.kwargs == settings.get_ocrmypdf_args() + assert response.metadata.rotation_ran is True + assert response.metadata.error is None + + def test_a_failed_rotation_returns_the_original_bytes(self): + with patch(f"{MODULE}.ocrmypdf") as ocrmypdf: + ocrmypdf.ocr = MagicMock(side_effect=RuntimeError("ghostscript died")) + response = PDFPreprocessor().preprocess(b"%PDF-1.5 original", "paper.pdf") + + assert response.processed_data == b"%PDF-1.5 original" + assert response.metadata.rotation_ran is False + assert "ghostscript died" in response.metadata.error + + def test_a_non_pdf_is_left_alone(self): + with patch(f"{MODULE}.ocrmypdf") as ocrmypdf: + ocrmypdf.ocr = MagicMock() + response = PDFPreprocessor().preprocess(b"not a pdf", "notes.txt") + + ocrmypdf.ocr.assert_not_called() + assert response.processed_data == b"not a pdf" + assert response.metadata.rotation_ran is False + + def test_disabling_preprocessing_skips_ocrmypdf(self): + with patch(f"{MODULE}.ocrmypdf") as ocrmypdf: + ocrmypdf.ocr = MagicMock() + response = PDFPreprocessor(PDFPreprocessingSettings(enabled=False)).preprocess(b"%PDF", "paper.pdf") + + ocrmypdf.ocr.assert_not_called() + assert response.metadata.rotation_ran is False + + +@pytest.mark.parametrize("attribute", ["analyzer", "enable_analysis"]) +def test_the_preprocessor_no_longer_analyzes(attribute): + # Margins and the thumbnail are the analysis job's business; the in-class path returned a + # tuple where the metadata expected a dict. + assert not hasattr(PDFPreprocessor(), attribute) + assert attribute not in PDFPreprocessingSettings.model_fields diff --git a/extralit-server/tests/unit/contexts/ocr/test_pdf_inspector_parser.py b/extralit-server/tests/unit/contexts/ocr/test_pdf_inspector_parser.py index 2f902add9..a79092a23 100644 --- a/extralit-server/tests/unit/contexts/ocr/test_pdf_inspector_parser.py +++ b/extralit-server/tests/unit/contexts/ocr/test_pdf_inspector_parser.py @@ -185,6 +185,8 @@ def test_reports_page_count_and_normalized_ocr_pages(self, untagged_bytes): assert result["page_count"] == 1 # classify_pdf reports 0-indexed pages; we normalize to docling's 1-indexed page_no. + # `sample.pdf` is a known false positive: any image-bearing page under ~1400 characters + # is flagged, which is why nothing is skipped on the strength of this list. assert result["pages_needing_ocr"] == [1] def test_reports_a_pdf_type(self, untagged_bytes): diff --git a/extralit-server/tests/unit/contexts/ocr/test_triage.py b/extralit-server/tests/unit/contexts/ocr/test_triage.py new file mode 100644 index 000000000..2a99325b2 --- /dev/null +++ b/extralit-server/tests/unit/contexts/ocr/test_triage.py @@ -0,0 +1,38 @@ +"""Tests for the structural triage pass.""" + +from pathlib import Path + +import pytest + +from extralit_server.contexts.ocr.triage import triage_pdf + +FIXTURES = Path(__file__).parents[3] / "fixtures" / "pdf" + + +@pytest.fixture +def pdf_bytes(): + return (FIXTURES / "sample.pdf").read_bytes() + + +class TestTriagePdf: + def test_reports_structure_with_one_indexed_pages(self, pdf_bytes): + result = triage_pdf(pdf_bytes) + + assert result.page_count == 1 + assert result.pdf_type + assert 0 not in result.pages_needing_ocr + assert all(page >= 1 for page in result.pages_needing_ocr) + + def test_ocr_reasons_are_keyed_by_page(self, pdf_bytes): + result = triage_pdf(pdf_bytes) + + for page, reasons in result.ocr_reasons_by_page.items(): + assert int(page) >= 1 + assert reasons + + def test_an_unreadable_pdf_is_unknown_rather_than_an_error(self): + result = triage_pdf(b"not a pdf at all") + + assert result.pdf_type == "unknown" + assert result.page_count == 0 + assert result.pages_needing_ocr == [] diff --git a/extralit-server/tests/unit/jobs/test_document_jobs.py b/extralit-server/tests/unit/jobs/test_document_jobs.py index 7803584fa..b9f67fd5d 100644 --- a/extralit-server/tests/unit/jobs/test_document_jobs.py +++ b/extralit-server/tests/unit/jobs/test_document_jobs.py @@ -1,219 +1,165 @@ +"""Tests for the triage + margins + rotation job.""" + from unittest.mock import AsyncMock, MagicMock, patch from uuid import uuid4 import pytest +from extralit_server.api.schemas.v1.document.metadata import TriageMetadata from extralit_server.jobs.document_jobs import analysis_and_preprocess_job -from extralit_server.models.database import Document - - -class TestDocumentJobs: - """Test suite for document job functions.""" - - @patch("extralit_server.jobs.document_jobs.files") - @patch("extralit_server.jobs.document_jobs.PDFPreprocessor") - @patch("extralit_server.jobs.document_jobs.PDFAnalyzer") - @patch("extralit_server.jobs.document_jobs.PDFOCRLayerDetector") - @patch("extralit_server.jobs.document_jobs.AsyncSessionLocal") - @patch("extralit_server.jobs.document_jobs.get_current_job") - async def test_analysis_and_preprocess_job_success( - self, - mock_get_current_job, - mock_session, - mock_ocr_detector_class, - mock_analyzer_class, - mock_preprocessor_class, - mock_files, - ): - """Test successful analysis and preprocess job.""" - # Setup test data - document_id = uuid4() - s3_url = f"/api/v1/file/test-workspace/documents/{document_id}/test.pdf" - reference = "test_ref" - workspace_name = "test-workspace" - - # Mock current job - mock_job = MagicMock() - mock_job.meta = {} - mock_get_current_job.return_value = mock_job - - # Mock file operations - mock_client = MagicMock() - mock_files.get_s3_client = AsyncMock(return_value=mock_client) - mock_files.download_file_content = AsyncMock(return_value=b"%PDF-1.5 test pdf content") - mock_files.get_thumbnail_s3_object_path.return_value = f"thumbnails/{document_id}" - mock_files.put_object = AsyncMock() - - # Mock OCR detector - mock_ocr_detector = MagicMock() - mock_ocr_detector.has_ocr_text_layer.return_value = True - mock_ocr_detector.analyze_character_quality.return_value = { - "ocr_quality_score": 0.8, - "total_chars": 1000, - "ocr_artifacts": 5, - "suspicious_patterns": 2, - } - mock_ocr_detector_class.return_value = mock_ocr_detector - - # Mock PDF analyzer - now returns tuple (layout_analysis, thumbnail_data) - mock_analyzer = MagicMock() - layout_analysis = { - "page_count": 1, - "page_dimensions": {"width": 612, "height": 792}, - "layout_analysis": {"analysis_method": "single_page_default"}, + +MODULE = "extralit_server.jobs.document_jobs" + +DOCUMENT_ID = uuid4() +S3_URL = f"/api/v1/file/test-workspace/documents/{DOCUMENT_ID}/test.pdf" +WORKSPACE = "test-workspace" +LAYOUT_ANALYSIS = { + "pages_sampled": 5, + "page_dimensions": {"width": 612, "height": 792}, + "layout_analysis": {"estimated_margins": {"left_px": 40, "top_px": 60, "right_px": 40, "bottom_px": 60}}, +} + + +def triage(**overrides) -> TriageMetadata: + return TriageMetadata( + **{ + "pdf_type": "text_based", + "confidence": 0.9, + "page_count": 12, + "pages_needing_ocr": [], + "pages_with_tables": [3], + **overrides, } - thumbnail_data = b"mock_thumbnail_data" - mock_analyzer.analyze_pdf_layout.return_value = (layout_analysis, thumbnail_data) - mock_analyzer_class.return_value = mock_analyzer - - # Mock preprocessor - mock_preprocessor = MagicMock() - mock_processing_response = MagicMock() - mock_processing_response.processed_data = b"%PDF-1.5 processed content" - mock_processing_response.metadata.processing_time = 5.0 - mock_processing_response.metadata.model_dump.return_value = {"processing_time": 5.0} - mock_preprocessor.preprocess.return_value = mock_processing_response - mock_preprocessor_class.return_value = mock_preprocessor - - # Mock database session - mock_db = AsyncMock() - mock_document = MagicMock() - mock_document.metadata_ = None - mock_db.get = AsyncMock(return_value=mock_document) - mock_db.commit = AsyncMock() - mock_session.return_value.__aenter__ = AsyncMock(return_value=mock_db) - mock_session.return_value.__aexit__ = AsyncMock(return_value=None) - - # Execute job - result = await analysis_and_preprocess_job(document_id, s3_url, reference, workspace_name) - - # Verify result structure - assert "document_id" in result - assert "analysis_result" in result - assert "preprocessing_result" in result - assert result["document_id"] == str(document_id) - - # Verify analysis result - analysis_result = result["analysis_result"] - assert analysis_result["has_ocr_text_layer"] is True - assert analysis_result["ocr_quality_score"] == 0.8 - assert analysis_result["layout_analysis"] == layout_analysis - assert analysis_result["thumbnail_generated"] is True - - # Verify preprocessing result - preprocessing_result = result["preprocessing_result"] - assert preprocessing_result["processing_time"] == 5.0 - - # Verify file operations were called - mock_files.download_file_content.assert_called_once() - mock_files.put_object.assert_called() # Called for both processed PDF and thumbnail - - # Verify analyzers were called correctly - mock_ocr_detector.has_ocr_text_layer.assert_called_once() - mock_ocr_detector.analyze_character_quality.assert_called_once() - mock_analyzer.analyze_pdf_layout.assert_called_once() - mock_preprocessor.preprocess.assert_called_once() - - # Verify database operations: the metadata write goes through the row lock. - mock_db.execute.assert_awaited_once() - assert "FOR UPDATE" in str(mock_db.execute.await_args.args[0]) - mock_db.get.assert_called_once_with(Document, document_id, populate_existing=True) - mock_db.commit.assert_called_once() - - @patch("extralit_server.jobs.document_jobs.files") - @patch("extralit_server.jobs.document_jobs.get_current_job") - async def test_analysis_and_preprocess_job_no_client(self, mock_get_current_job, mock_files): - """Test analysis and preprocess job when storage client is not available.""" - # Setup test data - document_id = uuid4() - s3_url = f"/api/v1/file/test-workspace/documents/{document_id}/test.pdf" - reference = "test_ref" - workspace_name = "test-workspace" - - # Mock current job - mock_job = MagicMock() - mock_job.meta = {} - mock_get_current_job.return_value = mock_job - - # Mock file operations - no client available - mock_files.get_s3_client = AsyncMock(return_value=None) - - # Execute job and expect exception - with pytest.raises(TypeError, match=r"object.*can't be used in 'await' expression"): - await analysis_and_preprocess_job(document_id, s3_url, reference, workspace_name) - # Verify job meta was updated with error - assert "error" in mock_job.meta - - @patch("extralit_server.jobs.document_jobs.files") - @patch("extralit_server.jobs.document_jobs.PDFAnalyzer") - @patch("extralit_server.jobs.document_jobs.PDFOCRLayerDetector") - @patch("extralit_server.jobs.document_jobs.get_current_job") - async def test_analysis_and_preprocess_job_no_thumbnail( - self, mock_get_current_job, mock_ocr_detector_class, mock_analyzer_class, mock_files + ) + + +@pytest.fixture +def job_context(): + """Everything the job talks to, with a well-behaved PDF.""" + current_job = MagicMock(meta={}) + analyzer = MagicMock() + analyzer.analyze_pdf_layout.return_value = (LAYOUT_ANALYSIS, b"thumbnail-bytes") + response = MagicMock() + response.processed_data = b"%PDF rotated" + response.metadata.processing_time = 3.0 + response.metadata.rotation_ran = True + response.metadata.error = None + preprocessor = MagicMock() + preprocessor.preprocess.return_value = response + written: list[dict] = [] + + async def put_object(client, workspace, key, data, **kwargs): + written.append({"key": key, "data": data}) + + with ( + patch(f"{MODULE}.files") as files, + patch(f"{MODULE}.PDFAnalyzer", return_value=analyzer), + patch(f"{MODULE}.PDFPreprocessor", return_value=preprocessor), + patch(f"{MODULE}.triage_pdf", return_value=triage()) as triage_pdf, + patch(f"{MODULE}.update_processing_metadata", AsyncMock()) as update_metadata, + patch(f"{MODULE}.get_current_job", return_value=current_job), + patch(f"{MODULE}.AsyncSessionLocal") as session, ): - """Test analysis and preprocess job when thumbnail generation fails.""" - # Setup test data - document_id = uuid4() - s3_url = f"/api/v1/file/test-workspace/documents/{document_id}/test.pdf" - reference = "test_ref" - workspace_name = "test-workspace" - - # Mock current job - mock_job = MagicMock() - mock_job.meta = {} - mock_get_current_job.return_value = mock_job - - # Mock file operations - mock_client = MagicMock() - mock_files.get_s3_client = AsyncMock(return_value=mock_client) - mock_files.download_file_content = AsyncMock(return_value=b"%PDF-1.5 test pdf content") - mock_files.put_object = AsyncMock() - - # Mock OCR detector - mock_ocr_detector = MagicMock() - mock_ocr_detector.has_ocr_text_layer.return_value = False - mock_ocr_detector.analyze_character_quality.return_value = { - "ocr_quality_score": 0.3, - "total_chars": 500, - "ocr_artifacts": 50, - "suspicious_patterns": 20, - } - mock_ocr_detector_class.return_value = mock_ocr_detector - - # Mock PDF analyzer - returns no thumbnail data - mock_analyzer = MagicMock() - layout_analysis = { - "page_count": 1, - "page_dimensions": {"width": 612, "height": 792}, - "layout_analysis": {"analysis_method": "single_page_default"}, + files.get_s3_client = AsyncMock(return_value=MagicMock()) + files.download_file_content = AsyncMock(return_value=b"%PDF original") + files.get_thumbnail_s3_object_path.return_value = f"thumbnails/{DOCUMENT_ID}" + files.put_object = AsyncMock(side_effect=put_object) + session.return_value.__aenter__ = AsyncMock(return_value=AsyncMock()) + session.return_value.__aexit__ = AsyncMock(return_value=None) + yield { + "job": current_job, + "files": files, + "analyzer": analyzer, + "preprocessor": preprocessor, + "response": response, + "triage_pdf": triage_pdf, + "update_metadata": update_metadata, + "written": written, } - mock_analyzer.analyze_pdf_layout.return_value = (layout_analysis, None) # No thumbnail - mock_analyzer_class.return_value = mock_analyzer - - # Mock preprocessor to skip it for this test by raising exception early - with patch("extralit_server.jobs.document_jobs.PDFPreprocessor") as mock_preprocessor_class: - mock_preprocessor = MagicMock() - mock_processing_response = MagicMock() - mock_processing_response.processed_data = b"%PDF-1.5 processed content" - mock_processing_response.metadata.processing_time = 3.0 - mock_processing_response.metadata.model_dump.return_value = {"processing_time": 3.0} - mock_preprocessor.preprocess.return_value = mock_processing_response - mock_preprocessor_class.return_value = mock_preprocessor - - # Mock database - with patch("extralit_server.jobs.document_jobs.AsyncSessionLocal") as mock_session: - mock_db = AsyncMock() - mock_document = MagicMock() - mock_document.metadata_ = None - mock_db.get = AsyncMock(return_value=mock_document) - mock_db.commit = AsyncMock() - mock_session.return_value.__aenter__ = AsyncMock(return_value=mock_db) - mock_session.return_value.__aexit__ = AsyncMock(return_value=None) - - # Execute job - result = await analysis_and_preprocess_job(document_id, s3_url, reference, workspace_name) - - # Verify that thumbnail was not generated - analysis_result = result["analysis_result"] - assert analysis_result["thumbnail_generated"] is False - assert analysis_result["needs_ocr"] is True # Low quality score and no OCR layer + + +async def run_job(): + return await analysis_and_preprocess_job(DOCUMENT_ID, S3_URL, "test_ref", WORKSPACE) + + +@pytest.mark.asyncio +class TestTriage: + async def test_triage_is_persisted_and_returned(self, job_context): + result = await run_job() + + analysis = result["analysis_result"] + assert analysis["triage"]["pdf_type"] == "text_based" + assert analysis["triage"]["pages_with_tables"] == [3] + assert analysis["page_count"] == 12 + + async def test_pages_needing_ocr_reach_the_job_meta(self, job_context): + job_context["triage_pdf"].return_value = triage(pdf_type="image_based", pages_needing_ocr=[1, 2]) + + await run_job() + + assert job_context["job"].meta["pages_needing_ocr"] == [1, 2] + + async def test_margins_are_estimated_from_the_leading_pages_only(self, job_context): + from extralit_server.contexts.document.margin import MARGIN_SAMPLE_PAGES + + await run_job() + + _pdf, filename = job_context["analyzer"].analyze_pdf_layout.call_args.args + assert filename == "test.pdf" + assert MARGIN_SAMPLE_PAGES == 5 + + async def test_metadata_is_written_under_the_row_lock(self, job_context): + await run_job() + + assert job_context["update_metadata"].await_count == 1 + + +@pytest.mark.asyncio +class TestRotation: + async def test_rotation_runs_on_every_pdf(self, job_context): + # Not gated on triage: ocrmypdf's OSD is the only thing that can see a sideways page. + job_context["triage_pdf"].return_value = triage(pdf_type="text_based", pages_needing_ocr=[]) + + await run_job() + + job_context["preprocessor"].preprocess.assert_called_once() + + async def test_the_pdf_rewrite_is_the_last_object_written(self, job_context): + await run_job() + + keys = [write["key"] for write in job_context["written"]] + assert keys[-1] == f"documents/{DOCUMENT_ID}/test.pdf" + assert f"thumbnails/{DOCUMENT_ID}" in keys + assert job_context["written"][-1]["data"] == b"%PDF rotated" + + async def test_a_failed_rotation_is_recorded_and_the_job_still_succeeds(self, job_context): + job_context["response"].metadata.rotation_ran = False + job_context["response"].metadata.error = "ghostscript died" + job_context["response"].processed_data = b"%PDF original" + + result = await run_job() + + preprocessing = result["preprocessing_result"] + assert preprocessing["rotation_ran"] is False + assert preprocessing["error"] == "ghostscript died" + assert preprocessing["ocr_applied"] is False + assert job_context["written"][-1]["data"] == b"%PDF original" + + async def test_a_missing_thumbnail_does_not_fail_the_job(self, job_context): + job_context["analyzer"].analyze_pdf_layout.return_value = (LAYOUT_ANALYSIS, None) + + result = await run_job() + + assert result["analysis_result"]["thumbnail_generated"] is False + assert [write["key"] for write in job_context["written"]] == [f"documents/{DOCUMENT_ID}/test.pdf"] + + +@pytest.mark.asyncio +class TestFailures: + async def test_a_storage_failure_surfaces_on_the_job(self, job_context): + job_context["files"].download_file_content = AsyncMock(side_effect=RuntimeError("s3 down")) + + with pytest.raises(RuntimeError, match="s3 down"): + await run_job() + + assert job_context["job"].meta["error"] == "s3 down" diff --git a/extralit-server/tests/unit/workflows/test_document_workflow_layout.py b/extralit-server/tests/unit/workflows/test_document_workflow_layout.py index d5d39f3d5..567fe496b 100644 --- a/extralit-server/tests/unit/workflows/test_document_workflow_layout.py +++ b/extralit-server/tests/unit/workflows/test_document_workflow_layout.py @@ -62,12 +62,16 @@ def prepared_for(calls, step): @pytest.mark.asyncio class TestLayoutJobSequencing: - async def test_no_layout_job_is_enqueued_by_default(self, enqueued): + async def test_layout_runs_by_default_with_the_default_parser(self, enqueued): + from extralit_server.contexts.ocr.parsers import default_parser_name + await run_workflow(layout_parser=None) - assert prepared_for(enqueued, "document_layout") is None + layout = prepared_for(enqueued, "document_layout") + assert layout is not None + assert layout["args"][3] == default_parser_name() - async def test_layout_job_is_enqueued_when_a_parser_is_named(self, enqueued): + async def test_a_named_parser_overrides_the_default(self, enqueued): await run_workflow(layout_parser="pdf_inspector") layout = prepared_for(enqueued, "document_layout") From 9ca175ef385ab0a5682054061baa0a15b519e348 Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Sun, 16 Aug 2026 17:38:53 -0700 Subject: [PATCH 08/10] fix(workflows): let a superseded run write nothing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit send_stop_job_command only asks a worker to stop, so `force=true` could enqueue a replacement while the previous run was still alive — and that stale run would then overwrite the new one's rotated PDF, layout JSON, Lance rows and metadata. Every artifact writer now checks a generation token before it writes: the job's own workflow_id against the document's newest DocumentWorkflow row. The layout job checks it inside the workspace lock next to its existence check; the analysis job checks it just before the thumbnail and the PDF rewrite. A superseded run returns early and writes nothing. A job with no workflow in its meta is always current, so direct calls are unaffected. Fixes roborev #341 (HIGH). #342 (swallowed artifact-cleanup failures) is answered in its comment: the leak is negligible and the durable-outbox fix is deferred by plan, so the docstrings now say the failure is ignorable. --- .../src/extralit_server/contexts/files.py | 6 ++-- .../extralit_server/contexts/ocr/storage.py | 3 +- .../src/extralit_server/contexts/workflows.py | 16 +++++++++ .../src/extralit_server/jobs/document_jobs.py | 8 +++++ .../src/extralit_server/jobs/ocr_jobs.py | 7 ++++ .../tests/unit/jobs/test_document_jobs.py | 14 ++++++++ .../tests/unit/jobs/test_ocr_jobs.py | 14 +++++++- .../workflows/test_workflow_generation.py | 35 +++++++++++++++++++ 8 files changed, 99 insertions(+), 4 deletions(-) create mode 100644 extralit-server/tests/unit/workflows/test_workflow_generation.py diff --git a/extralit-server/src/extralit_server/contexts/files.py b/extralit-server/src/extralit_server/contexts/files.py index 769a7a322..2784e8e41 100644 --- a/extralit-server/src/extralit_server/contexts/files.py +++ b/extralit-server/src/extralit_server/contexts/files.py @@ -365,8 +365,10 @@ async def delete_object(s3_client, bucket: str, object: str, version_id: str | N async def delete_document_artifacts(s3_client: "S3Client", workspace_name: str, document_id: UUID | str) -> None: """Remove every artifact of a document: PDF, thumbnail, layout JSON and layout rows. - Best effort — the DB rows are already gone by the time this runs, so a storage hiccup must - leave a leaked object rather than a document the caller cannot delete. + Best effort — the DB rows are already gone by the time this runs, so a storage hiccup leaves a + leaked object rather than a document the caller cannot delete. A leak here is negligible and can + be ignored: the objects are unreachable and layout rows only survive until the document is + re-parsed or a sweeper runs. """ from extralit_server.contexts.ocr import storage diff --git a/extralit-server/src/extralit_server/contexts/ocr/storage.py b/extralit-server/src/extralit_server/contexts/ocr/storage.py index 2e47a2ac5..6f71b7ddd 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/storage.py +++ b/extralit-server/src/extralit_server/contexts/ocr/storage.py @@ -74,7 +74,8 @@ async def delete_layout( document_id: UUID | str, store: Optional[LayoutStore] = None, ) -> None: - """Drop both artifacts. Orphaned rows would silently skew every workspace aggregate.""" + """Drop both artifacts. Orphaned rows would skew workspace aggregates, but a failure here is + negligible: they are superseded on the next parse of that document.""" try: await files.delete_object(s3_client, workspace_name, layout_object_path(document_id)) except Exception as error: diff --git a/extralit-server/src/extralit_server/contexts/workflows.py b/extralit-server/src/extralit_server/contexts/workflows.py index 50bc7b8a9..d57cff7a8 100644 --- a/extralit-server/src/extralit_server/contexts/workflows.py +++ b/extralit-server/src/extralit_server/contexts/workflows.py @@ -511,6 +511,22 @@ def get_failed_jobs_in_group(group_id: str) -> list[dict[str, Any]]: return [] +async def is_current_workflow_run(db: AsyncSession, document_id: UUID, workflow_id: Optional[str]) -> bool: + """Whether this job still belongs to the document's newest workflow run. + + `send_stop_job_command` only *asks* a worker to stop, so a forced restart can leave the previous + run alive long enough to overwrite the new one's PDF, layout or metadata. Every writer checks + this generation token before it writes; the workflow row is the token. + + A job with no workflow in its meta (direct call, test, ad-hoc enqueue) is always current. + """ + if not workflow_id: + return True + + workflow = await DocumentWorkflow.get_by_document_id(db, document_id) + return workflow is None or str(workflow.id) == str(workflow_id) + + def stop_workflow_jobs(group_id: str) -> list[str]: """ Stop running jobs and cancel pending ones for a workflow group. diff --git a/extralit-server/src/extralit_server/jobs/document_jobs.py b/extralit-server/src/extralit_server/jobs/document_jobs.py index be41b7162..08d6f15e0 100644 --- a/extralit-server/src/extralit_server/jobs/document_jobs.py +++ b/extralit-server/src/extralit_server/jobs/document_jobs.py @@ -13,6 +13,7 @@ from extralit_server.contexts.document.metadata import update_processing_metadata from extralit_server.contexts.document.preprocessing import PDFPreprocessor from extralit_server.contexts.ocr.triage import triage_pdf +from extralit_server.contexts.workflows import is_current_workflow_run from extralit_server.database import AsyncSessionLocal from extralit_server.jobs.queues import DEFAULT_QUEUE, REDIS_CONNECTION @@ -80,6 +81,13 @@ async def analysis_and_preprocess_job( if not processing_response.metadata.rotation_ran: _LOGGER.warning(f"Rotation did not run for document {document_id}: {processing_response.metadata.error}") + # A forced restart may already be running; its rotation and metadata must not lose to this + # one's, because stopping a started job is only a request. + async with AsyncSessionLocal() as db: + if not await is_current_workflow_run(db, document_id, current_job.meta.get("workflow_id")): + _LOGGER.info(f"Analysis run for document {document_id} was superseded before it could store") + return {"document_id": str(document_id), "skipped": "workflow superseded"} + # The PDF rewrite is the last S3 write of this job — dependents key on it. object_path = s3_url.replace(f"/api/v1/file/{workspace_name}/", "") diff --git a/extralit-server/src/extralit_server/jobs/ocr_jobs.py b/extralit-server/src/extralit_server/jobs/ocr_jobs.py index 842e4a0b8..8671dbb6e 100644 --- a/extralit-server/src/extralit_server/jobs/ocr_jobs.py +++ b/extralit-server/src/extralit_server/jobs/ocr_jobs.py @@ -16,6 +16,7 @@ from extralit_server.contexts.ocr.layout_store import LayoutStore from extralit_server.contexts.ocr.parsers import default_parser_name, get_parser from extralit_server.contexts.ocr.parsers.pdf_inspector import classify +from extralit_server.contexts.workflows import is_current_workflow_run from extralit_server.database import AsyncSessionLocal from extralit_server.jobs.queues import OCR_QUEUE, REDIS_CONNECTION from extralit_server.models.database import Document @@ -92,13 +93,19 @@ async def async_document_layout_job( # Ordering closes the delete-vs-running-job race: nothing may write rows for a document # whose delete already ran, and the workspace lock is held across the check and the write. + workflow_id = (current_job.meta or {}).get("workflow_id") if current_job is not None else None store = LayoutStore.for_workspace(workspace_name) async with store.locked(): async with AsyncSessionLocal() as db: still_exists = await db.scalar(select(Document.id).where(Document.id == document_id)) + superseded = not await is_current_workflow_run(db, document_id, workflow_id) if still_exists is None: _LOGGER.info(f"Document {document_id} was deleted before its layout was stored") return {"document_id": str(document_id), "parser": parser_name, "skipped": "document deleted"} + if superseded: + # A forced restart already began; its layout must not lose to this one's. + _LOGGER.info(f"Layout run for document {document_id} was superseded before it could store") + return {"document_id": str(document_id), "parser": parser_name, "skipped": "workflow superseded"} paths = await storage.store_layout(s3_client, workspace_name, document_id, doc, store=store) diff --git a/extralit-server/tests/unit/jobs/test_document_jobs.py b/extralit-server/tests/unit/jobs/test_document_jobs.py index b9f67fd5d..1ace996eb 100644 --- a/extralit-server/tests/unit/jobs/test_document_jobs.py +++ b/extralit-server/tests/unit/jobs/test_document_jobs.py @@ -57,6 +57,7 @@ async def put_object(client, workspace, key, data, **kwargs): patch(f"{MODULE}.PDFPreprocessor", return_value=preprocessor), patch(f"{MODULE}.triage_pdf", return_value=triage()) as triage_pdf, patch(f"{MODULE}.update_processing_metadata", AsyncMock()) as update_metadata, + patch(f"{MODULE}.is_current_workflow_run", AsyncMock(return_value=True)) as is_current, patch(f"{MODULE}.get_current_job", return_value=current_job), patch(f"{MODULE}.AsyncSessionLocal") as session, ): @@ -75,6 +76,7 @@ async def put_object(client, workspace, key, data, **kwargs): "triage_pdf": triage_pdf, "update_metadata": update_metadata, "written": written, + "is_current": is_current, } @@ -163,3 +165,15 @@ async def test_a_storage_failure_surfaces_on_the_job(self, job_context): await run_job() assert job_context["job"].meta["error"] == "s3 down" + + +@pytest.mark.asyncio +class TestSupersededRuns: + async def test_a_superseded_run_rewrites_nothing(self, job_context): + job_context["is_current"].return_value = False + + result = await run_job() + + assert result["skipped"] == "workflow superseded" + assert job_context["written"] == [] + assert job_context["update_metadata"].await_count == 0 diff --git a/extralit-server/tests/unit/jobs/test_ocr_jobs.py b/extralit-server/tests/unit/jobs/test_ocr_jobs.py index 39d59fa69..83bf664d8 100644 --- a/extralit-server/tests/unit/jobs/test_ocr_jobs.py +++ b/extralit-server/tests/unit/jobs/test_ocr_jobs.py @@ -40,11 +40,12 @@ async def update_metadata(db, document_id, mutate): patch(f"{MODULE}.storage.store_layout", store_layout), patch(f"{MODULE}.update_processing_metadata", update_metadata), patch(f"{MODULE}.get_current_job", return_value=None), + patch(f"{MODULE}.is_current_workflow_run", AsyncMock(return_value=True)) as is_current, patch(f"{MODULE}.AsyncSessionLocal") as session, ): session.return_value.__aenter__ = AsyncMock(return_value=db) session.return_value.__aexit__ = AsyncMock(return_value=False) - yield {"calls": calls, "db": db, "store": store} + yield {"calls": calls, "db": db, "store": store, "is_current": is_current} async def run_job(document_id=None): @@ -83,3 +84,14 @@ async def test_pages_needing_ocr_are_surfaced(self, job_context): result = await run_job() assert result["pages_needing_ocr"] == [2] + + +class TestSupersededRuns: + async def test_a_superseded_run_does_not_write(self, job_context): + # Stopping a started job is only a request, so a forced restart can overlap this run. + job_context["is_current"].return_value = False + + result = await run_job() + + assert result["skipped"] == "workflow superseded" + assert job_context["calls"] == [] diff --git a/extralit-server/tests/unit/workflows/test_workflow_generation.py b/extralit-server/tests/unit/workflows/test_workflow_generation.py new file mode 100644 index 000000000..bc1978b31 --- /dev/null +++ b/extralit-server/tests/unit/workflows/test_workflow_generation.py @@ -0,0 +1,35 @@ +"""Tests for the workflow-generation guard every artifact writer checks.""" + +from unittest.mock import AsyncMock, MagicMock, patch +from uuid import uuid4 + +import pytest + +from extralit_server.contexts.workflows import is_current_workflow_run + +MODULE = "extralit_server.contexts.workflows" + +pytestmark = pytest.mark.asyncio + + +class TestIsCurrentWorkflowRun: + async def test_the_newest_run_is_current(self): + workflow = MagicMock(id=uuid4()) + + with patch(f"{MODULE}.DocumentWorkflow.get_by_document_id", AsyncMock(return_value=workflow)): + assert await is_current_workflow_run(AsyncMock(), uuid4(), str(workflow.id)) is True + + async def test_a_superseded_run_is_not_current(self): + with patch(f"{MODULE}.DocumentWorkflow.get_by_document_id", AsyncMock(return_value=MagicMock(id=uuid4()))): + assert await is_current_workflow_run(AsyncMock(), uuid4(), str(uuid4())) is False + + async def test_a_job_without_a_workflow_is_always_current(self): + # Direct calls and ad-hoc enqueues carry no workflow; they must not be gated on one. + with patch(f"{MODULE}.DocumentWorkflow.get_by_document_id", AsyncMock()) as lookup: + assert await is_current_workflow_run(AsyncMock(), uuid4(), None) is True + + lookup.assert_not_awaited() + + async def test_a_document_without_a_workflow_row_is_current(self): + with patch(f"{MODULE}.DocumentWorkflow.get_by_document_id", AsyncMock(return_value=None)): + assert await is_current_workflow_run(AsyncMock(), uuid4(), str(uuid4())) is True From 43b1deacfa8fc54530a6779c4a8292e0ffc74888 Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Sun, 16 Aug 2026 18:50:50 -0700 Subject: [PATCH 09/10] fix(ocr): address review findings on the layout contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Silent-failure fixes: - Layout filters never reached the server. Axios serializes arrays as `pages[]=2`, but the endpoint declares `pages: list[int] = Query(...)`, which binds only repeated bare keys, so FastAPI dropped every filter without erroring. `paramsSerializer: { indexes: null }` restores `pages=2&pages=3`. - `binary_hash` used `hash(pdf_bytes)`, which Python salts per process, so the same PDF hashed differently across workers. Replaced with a shared `content_hash()` on xxh3_64 — ~10x faster than sha256 and, unlike the truncated digest it replaces, fills `DocumentOrigin.binary_hash`'s full Uint64. - `create_bucket` let `BucketAlreadyOwnedByYou` escape to the outer handler, so pre-existing workspace buckets never picked up versioning or the layout lifecycle rule — exactly the version growth the rule was added to bound. - `PreprocessingMetadata` dropped `rotation_ran` and `error`, which the job computes and the admin guide documents, leaving a failed rotation invisible. - `analysis_and_preprocess_job` wrote the PDF, thumbnail and metadata for a document deleted mid-run, then reported success. Now guarded on both sides, mirroring the pattern already in `ocr_jobs.py`. Also: `filename` added to the `LayoutParser` protocol (both parsers take it and the job passes it); unknown `layout_parser` rejected at the API boundary rather than deep inside the queued job; single-pass tagged/untagged partition in pdf_inspector, replacing an O(n^2) scan that also dropped spans comparing equal; explicit `pyarrow.compute` import that only worked via a docling_core side effect; fixture path and heading levels in docs. Not adopted from the review: frontend bbox origin normalization (the builder already flips everything to TOPLEFT before it becomes provenance), and running the layout job through an async-aware worker (RQ 2.4.1's `Job._execute` already awaits coroutine job functions). --- .gitignore | 2 +- docs/admin_guide/pdf_preprocessing_config.md | 12 ++-- .../repositories/DocumentRepository.test.ts | 11 +++- .../repositories/DocumentRepository.ts | 6 +- extralit-server/pyproject.toml | 2 + .../api/schemas/v1/document/metadata.py | 4 ++ .../api/schemas/v1/workflows.py | 11 +++- .../src/extralit_server/contexts/files.py | 14 ++-- .../contexts/ocr/docling_builder.py | 6 ++ .../contexts/ocr/parsers/__init__.py | 1 + .../contexts/ocr/parsers/pdf_inspector.py | 19 ++++-- .../contexts/ocr/parsers/pymupdf.py | 10 ++- .../src/extralit_server/jobs/document_jobs.py | 9 ++- .../api/schemas/v1/test_document_metadata.py | 35 ++++++++++ .../unit/api/schemas/v1/test_workflows.py | 27 ++++++++ .../tests/unit/contexts/ocr/test_arrow.py | 3 +- extralit-server/uv.lock | 66 +++++++++++++++++++ 17 files changed, 213 insertions(+), 25 deletions(-) create mode 100644 extralit-server/tests/unit/api/schemas/v1/test_document_metadata.py create mode 100644 extralit-server/tests/unit/api/schemas/v1/test_workflows.py diff --git a/.gitignore b/.gitignore index d502620ad..5fcc30066 100644 --- a/.gitignore +++ b/.gitignore @@ -165,7 +165,7 @@ extralit/site # Development files **/*.db **/*.pdf -# Layout parser test fixtures are committed; regenerate with tests/fixtures/pdf/generate.py +# Layout parser test fixtures are committed; regenerate with extralit-server/tests/fixtures/pdf/generate.py !extralit-server/tests/fixtures/pdf/*.pdf output/ .output/ diff --git a/docs/admin_guide/pdf_preprocessing_config.md b/docs/admin_guide/pdf_preprocessing_config.md index c8310f226..8cdb96062 100644 --- a/docs/admin_guide/pdf_preprocessing_config.md +++ b/docs/admin_guide/pdf_preprocessing_config.md @@ -24,31 +24,31 @@ explicit gap until an OCR job exists. ## Configuration Reference -#### `PREPROCESSING_ENABLED` +### `PREPROCESSING_ENABLED` - **Type**: `bool` — **Default**: `true` - Master switch. When `false`, the PDF is passed through byte-identical; triage, margins and the thumbnail still run (they are the analysis job's own work, not the preprocessor's). -#### `PREPROCESSING_ROTATE_PAGES` +### `PREPROCESSING_ROTATE_PAGES` - **Type**: `bool` — **Default**: `true` - Auto-rotate pages whose text is not upright. -#### `PREPROCESSING_ROTATE_PAGES_THRESHOLD` +### `PREPROCESSING_ROTATE_PAGES_THRESHOLD` - **Type**: `float` — **Default**: `2.0` - Confidence OSD must reach before a page is rotated. Lower (1.0–1.5) rotates more eagerly; higher (3.0+) avoids false rotations. -#### `PREPROCESSING_TESSERACT_NON_OCR_TIMEOUT` +### `PREPROCESSING_TESSERACT_NON_OCR_TIMEOUT` - **Type**: `float` (seconds per page) — **Default**: `30.0` - Budget for OSD, the only tesseract call made here. OCRmyPDF's own default is 180 s per page, which dominates the runtime on image-heavy PDFs. -#### `PREPROCESSING_JOBS` +### `PREPROCESSING_JOBS` - **Type**: `int` — **Default**: `1` - Worker processes for ocrmypdf. Keep at `1` in containers with limited CPU to avoid oversubscription; `2–4` on a multi-core host. -#### `PREPROCESSING_PROGRESS_BAR` +### `PREPROCESSING_PROGRESS_BAR` - **Type**: `bool` — **Default**: `false` - Useful interactively, noise in background jobs. diff --git a/extralit-frontend/v1/infrastructure/repositories/DocumentRepository.test.ts b/extralit-frontend/v1/infrastructure/repositories/DocumentRepository.test.ts index bf4dfcbb9..6dc0d230d 100644 --- a/extralit-frontend/v1/infrastructure/repositories/DocumentRepository.test.ts +++ b/extralit-frontend/v1/infrastructure/repositories/DocumentRepository.test.ts @@ -72,7 +72,10 @@ describe("DocumentRepository", () => { const layout = await repository.getDocumentLayout("d-1"); - expect(axios.get).toHaveBeenCalledWith("/v1/documents/d-1/layout", { params: {} }); + expect(axios.get).toHaveBeenCalledWith("/v1/documents/d-1/layout", { + params: {}, + paramsSerializer: { indexes: null }, + }); expect(layout.documentId).toBe("d-1"); expect(layout.doclingVersion).toBe("1.10.0"); expect(layout.numPages).toBe(2); @@ -102,6 +105,7 @@ describe("DocumentRepository", () => { expect(axios.get).toHaveBeenCalledWith("/v1/documents/d-1/layout", { params: { pages: [2], labels: ["table"] }, + paramsSerializer: { indexes: null }, }); }); @@ -111,7 +115,10 @@ describe("DocumentRepository", () => { await repository.getDocumentLayout("d-1", { pages: [], labels: [] }); - expect(axios.get).toHaveBeenCalledWith("/v1/documents/d-1/layout", { params: {} }); + expect(axios.get).toHaveBeenCalledWith("/v1/documents/d-1/layout", { + params: {}, + paramsSerializer: { indexes: null }, + }); }); it("throws a typed error when the request fails", async () => { diff --git a/extralit-frontend/v1/infrastructure/repositories/DocumentRepository.ts b/extralit-frontend/v1/infrastructure/repositories/DocumentRepository.ts index 4dce1bc52..cf02ec7b5 100644 --- a/extralit-frontend/v1/infrastructure/repositories/DocumentRepository.ts +++ b/extralit-frontend/v1/infrastructure/repositories/DocumentRepository.ts @@ -147,7 +147,11 @@ export class DocumentRepository { if (filters?.pages?.length) params.pages = filters.pages; if (filters?.labels?.length) params.labels = filters.labels; - const { data } = await this.axios.get(`/v1/documents/${documentId}/layout`, { params }); + // FastAPI binds repeated bare keys; axios' default `pages[]=` bracket form is silently dropped. + const { data } = await this.axios.get(`/v1/documents/${documentId}/layout`, { + params, + paramsSerializer: { indexes: null }, + }); return toDocumentLayout(data); } catch (error) { diff --git a/extralit-server/pyproject.toml b/extralit-server/pyproject.toml index 0796e0d2b..a398f14b8 100644 --- a/extralit-server/pyproject.toml +++ b/extralit-server/pyproject.toml @@ -83,6 +83,8 @@ dependencies = [ "docling-core>=2.91.0,<3.0.0", "pyarrow>=23.0.1", "pdf-inspector>=1.14.2", + "xxhash>=3.6.0", + "obstore>=0.11.0", ] [project.optional-dependencies] diff --git a/extralit-server/src/extralit_server/api/schemas/v1/document/metadata.py b/extralit-server/src/extralit_server/api/schemas/v1/document/metadata.py index 2bef5c758..da6d88383 100644 --- a/extralit-server/src/extralit_server/api/schemas/v1/document/metadata.py +++ b/extralit-server/src/extralit_server/api/schemas/v1/document/metadata.py @@ -63,6 +63,8 @@ class PreprocessingMetadata(BaseModel): processing_time: float = Field(..., description="Processing time in seconds") ocr_applied: bool = Field(..., description="Whether OCR was applied during preprocessing") processed_s3_url: Optional[str] = Field(None, description="S3 URL of processed PDF") + rotation_ran: Optional[bool] = Field(None, description="Whether page rotation completed") + error: Optional[str] = Field(None, description="Why preprocessing fell back to the original PDF") class TextExtractionMetadata(BaseModel): @@ -138,6 +140,8 @@ def update_preprocessing_results(self, preprocess_result: dict) -> None: processing_time=preprocess_result["processing_time"], ocr_applied=preprocess_result.get("ocr_applied", False), processed_s3_url=preprocess_result.get("processed_s3_url"), + rotation_ran=preprocess_result.get("rotation_ran"), + error=preprocess_result.get("error"), ) def is_workflow_complete(self) -> bool: diff --git a/extralit-server/src/extralit_server/api/schemas/v1/workflows.py b/extralit-server/src/extralit_server/api/schemas/v1/workflows.py index 6e8943fd2..9ceff77c3 100644 --- a/extralit-server/src/extralit_server/api/schemas/v1/workflows.py +++ b/extralit-server/src/extralit_server/api/schemas/v1/workflows.py @@ -2,7 +2,9 @@ from typing import Any, Optional from uuid import UUID -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, field_validator + +from extralit_server.contexts.ocr.parsers import list_parsers class StartWorkflowRequest(BaseModel): @@ -18,6 +20,13 @@ class StartWorkflowRequest(BaseModel): None, description="Layout parser to run (e.g. `pdf_inspector`, `pymupdf`); omit to skip layout extraction" ) + @field_validator("layout_parser") + @classmethod + def _known_parser(cls, value: Optional[str]) -> Optional[str]: + if value is not None and value not in list_parsers(): + raise ValueError(f"unknown layout parser {value!r}; available: {list_parsers()}") + return value + class StartWorkflowResponse(BaseModel): """Response schema for starting a workflow.""" diff --git a/extralit-server/src/extralit_server/contexts/files.py b/extralit-server/src/extralit_server/contexts/files.py index 2784e8e41..e60db50a4 100644 --- a/extralit-server/src/extralit_server/contexts/files.py +++ b/extralit-server/src/extralit_server/contexts/files.py @@ -420,7 +420,12 @@ async def create_bucket(s3_client: "S3Client", workspace_name: str): """Create the workspace's bucket, versioned, with layout noncurrent versions expiring.""" bucket, prefix = workspace_root(workspace_name) try: - await s3_client.create_bucket(Bucket=bucket) + try: + await s3_client.create_bucket(Bucket=bucket) + except ClientError as e: + # An existing bucket must still pick up versioning and the lifecycle rule. + if e.response["Error"]["Code"] not in ["BucketAlreadyOwnedByYou", "BucketAlreadyExists"]: + raise await s3_client.put_bucket_versioning( Bucket=bucket, @@ -449,11 +454,8 @@ async def create_bucket(s3_client: "S3Client", workspace_name: str): _LOGGER.warning(f"Could not set the layout lifecycle rule on bucket {bucket}: {e}") except ClientError as e: - if e.response["Error"]["Code"] in ["BucketAlreadyOwnedByYou", "BucketAlreadyExists"]: - pass # Bucket already exists, that's fine - else: - _LOGGER.error(f"Error creating bucket {bucket}: {e}") - raise HTTPException(status_code=500, detail=f"Error creating bucket: {e!s}") + _LOGGER.error(f"Error creating bucket {bucket}: {e}") + raise HTTPException(status_code=500, detail=f"Error creating bucket: {e!s}") except Exception as e: _LOGGER.error(f"Error creating bucket {workspace_name}: {e}") raise HTTPException(status_code=500, detail=f"Internal server error: {e!s}") diff --git a/extralit-server/src/extralit_server/contexts/ocr/docling_builder.py b/extralit-server/src/extralit_server/contexts/ocr/docling_builder.py index a13abc133..cf2bc03b0 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/docling_builder.py +++ b/extralit-server/src/extralit_server/contexts/ocr/docling_builder.py @@ -9,6 +9,7 @@ from dataclasses import dataclass from typing import Optional +import xxhash from docling_core.types.doc import ( BoundingBox, CoordOrigin, @@ -62,6 +63,11 @@ def width(self) -> float: return self.size.width +def content_hash(pdf_bytes: bytes) -> int: + """Fill `DocumentOrigin.binary_hash`. `hash()` is salted per process, so it cannot be used.""" + return xxhash.xxh3_64_intdigest(pdf_bytes) + + def new_document( name: str, *, diff --git a/extralit-server/src/extralit_server/contexts/ocr/parsers/__init__.py b/extralit-server/src/extralit_server/contexts/ocr/parsers/__init__.py index f6000942f..72fb50181 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/parsers/__init__.py +++ b/extralit-server/src/extralit_server/contexts/ocr/parsers/__init__.py @@ -24,6 +24,7 @@ def __call__( *, name: str, pages: Optional[Sequence[int]] = None, + filename: Optional[str] = None, ) -> DoclingDocument: ... diff --git a/extralit-server/src/extralit_server/contexts/ocr/parsers/pdf_inspector.py b/extralit-server/src/extralit_server/contexts/ocr/parsers/pdf_inspector.py index 200fbb181..0bcd94404 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/parsers/pdf_inspector.py +++ b/extralit-server/src/extralit_server/contexts/ocr/parsers/pdf_inspector.py @@ -19,7 +19,13 @@ import pikepdf from docling_core.types.doc import BoundingBox, CoordOrigin, DocItemLabel, DoclingDocument, Size, TableCell -from extralit_server.contexts.ocr.docling_builder import LayoutBlock, PageContext, append_blocks, new_document +from extralit_server.contexts.ocr.docling_builder import ( + LayoutBlock, + PageContext, + append_blocks, + content_hash, + new_document, +) from extralit_server.contexts.ocr.tables import make_cell _LOGGER = logging.getLogger(__name__) @@ -202,8 +208,13 @@ def _blocks_for_page( images = [i for i in items if i.item_type != "text"] text_items = [i for i in items if i.item_type == "text"] - tagged = [i for i in text_items if i.mcid is not None and (page_no, i.mcid) in roles] - untagged = [i for i in text_items if i not in tagged] + tagged: list[Any] = [] + untagged: list[Any] = [] + for item in text_items: + if item.mcid is not None and (page_no, item.mcid) in roles: + tagged.append(item) + else: + untagged.append(item) for image in images: role = roles.get((page_no, image.mcid)) if image.mcid is not None else None @@ -251,7 +262,7 @@ def parse( filename: Optional[str] = None, ) -> DoclingDocument: """Parse PDF bytes into a `DoclingDocument`. `pages` is a 1-indexed allowlist.""" - doc = new_document(name, filename=filename, binary_hash=hash(pdf_bytes) & 0xFFFFFFFF) + doc = new_document(name, filename=filename, binary_hash=content_hash(pdf_bytes)) sizes = page_sizes(pdf_bytes) roles = _structure_roles(pdf_bytes) diff --git a/extralit-server/src/extralit_server/contexts/ocr/parsers/pymupdf.py b/extralit-server/src/extralit_server/contexts/ocr/parsers/pymupdf.py index 38f7391dd..fd86aed7c 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/parsers/pymupdf.py +++ b/extralit-server/src/extralit_server/contexts/ocr/parsers/pymupdf.py @@ -14,7 +14,13 @@ import pymupdf4llm from docling_core.types.doc import BoundingBox, CoordOrigin, DocItemLabel, DoclingDocument, Size, TableCell -from extralit_server.contexts.ocr.docling_builder import LayoutBlock, PageContext, append_blocks, new_document +from extralit_server.contexts.ocr.docling_builder import ( + LayoutBlock, + PageContext, + append_blocks, + content_hash, + new_document, +) from extralit_server.contexts.ocr.tables import make_cell _LOGGER = logging.getLogger(__name__) @@ -154,7 +160,7 @@ def parse( filename: Optional[str] = None, ) -> DoclingDocument: """Parse PDF bytes into a `DoclingDocument`. `pages` is a 1-indexed allowlist.""" - doc = new_document(name, filename=filename, binary_hash=hash(pdf_bytes) & 0xFFFFFFFF) + doc = new_document(name, filename=filename, binary_hash=content_hash(pdf_bytes)) wanted = set(pages) if pages else None with pymupdf.open(stream=pdf_bytes, filetype="pdf") as pdf: diff --git a/extralit-server/src/extralit_server/jobs/document_jobs.py b/extralit-server/src/extralit_server/jobs/document_jobs.py index 08d6f15e0..89295e666 100644 --- a/extralit-server/src/extralit_server/jobs/document_jobs.py +++ b/extralit-server/src/extralit_server/jobs/document_jobs.py @@ -6,6 +6,7 @@ from rq import Retry, get_current_job from rq.decorators import job +from sqlalchemy import select from extralit_server.api.schemas.v1.document.metadata import DocumentProcessingMetadata from extralit_server.contexts import files @@ -16,6 +17,7 @@ from extralit_server.contexts.workflows import is_current_workflow_run from extralit_server.database import AsyncSessionLocal from extralit_server.jobs.queues import DEFAULT_QUEUE, REDIS_CONNECTION +from extralit_server.models.database import Document _LOGGER = logging.getLogger(__name__) @@ -84,6 +86,9 @@ async def analysis_and_preprocess_job( # A forced restart may already be running; its rotation and metadata must not lose to this # one's, because stopping a started job is only a request. async with AsyncSessionLocal() as db: + if await db.scalar(select(Document.id).where(Document.id == document_id)) is None: + _LOGGER.info(f"Document {document_id} was deleted before its analysis could store") + return {"document_id": str(document_id), "skipped": "document deleted"} if not await is_current_workflow_run(db, document_id, current_job.meta.get("workflow_id")): _LOGGER.info(f"Analysis run for document {document_id} was superseded before it could store") return {"document_id": str(document_id), "skipped": "workflow superseded"} @@ -133,7 +138,9 @@ def apply(metadata: DocumentProcessingMetadata) -> None: metadata.update_preprocessing_results(combined_result["preprocessing_result"]) async with AsyncSessionLocal() as db: - await update_processing_metadata(db, document_id, apply) + if await update_processing_metadata(db, document_id, apply) is None: + _LOGGER.info(f"Document {document_id} was deleted before its analysis could store") + return {"document_id": str(document_id), "skipped": "document deleted"} # Read by the scheduler branch that will enqueue an OCR job once an engine exists. current_job.meta["pages_needing_ocr"] = triage.pages_needing_ocr diff --git a/extralit-server/tests/unit/api/schemas/v1/test_document_metadata.py b/extralit-server/tests/unit/api/schemas/v1/test_document_metadata.py new file mode 100644 index 000000000..bcd4b5767 --- /dev/null +++ b/extralit-server/tests/unit/api/schemas/v1/test_document_metadata.py @@ -0,0 +1,35 @@ +from extralit_server.api.schemas.v1.document.metadata import DocumentProcessingMetadata + +FAILED_ROTATION = { + "processing_time": 1.5, + "ocr_applied": False, + "rotation_ran": False, + "error": "ocrmypdf exited with 2", +} + + +class TestUpdatePreprocessingResults: + def test_retains_the_rotation_outcome(self): + metadata = DocumentProcessingMetadata() + + metadata.update_preprocessing_results(FAILED_ROTATION) + + assert metadata.preprocessing_metadata.rotation_ran is False + assert metadata.preprocessing_metadata.error == "ocrmypdf exited with 2" + + def test_the_outcome_survives_serialization_into_documents_metadata(self): + metadata = DocumentProcessingMetadata() + metadata.update_preprocessing_results(FAILED_ROTATION) + + stored = metadata.model_dump()["preprocessing_metadata"] + + assert stored["rotation_ran"] is False + assert stored["error"] == "ocrmypdf exited with 2" + + def test_a_job_result_without_rotation_fields_leaves_them_unset(self): + metadata = DocumentProcessingMetadata() + + metadata.update_preprocessing_results({"processing_time": 0.2, "ocr_applied": False}) + + assert metadata.preprocessing_metadata.rotation_ran is None + assert metadata.preprocessing_metadata.error is None diff --git a/extralit-server/tests/unit/api/schemas/v1/test_workflows.py b/extralit-server/tests/unit/api/schemas/v1/test_workflows.py new file mode 100644 index 000000000..7a6bf817f --- /dev/null +++ b/extralit-server/tests/unit/api/schemas/v1/test_workflows.py @@ -0,0 +1,27 @@ +from uuid import uuid4 + +import pytest +from pydantic import ValidationError + +from extralit_server.api.schemas.v1.workflows import StartWorkflowRequest +from extralit_server.contexts.ocr.parsers import list_parsers + + +def _request(**overrides) -> StartWorkflowRequest: + return StartWorkflowRequest(document_id=uuid4(), workspace_name="ws", **overrides) + + +class TestStartWorkflowRequest: + def test_accepts_a_registered_parser(self): + assert _request(layout_parser="pdf_inspector").layout_parser == "pdf_inspector" + + def test_omitting_the_parser_skips_layout_extraction(self): + assert _request().layout_parser is None + + def test_rejects_an_unregistered_parser(self): + with pytest.raises(ValidationError, match="unknown layout parser"): + _request(layout_parser="marker") + + def test_error_lists_the_installed_parsers(self): + with pytest.raises(ValidationError, match=str(list_parsers())): + _request(layout_parser="marker") diff --git a/extralit-server/tests/unit/contexts/ocr/test_arrow.py b/extralit-server/tests/unit/contexts/ocr/test_arrow.py index 547ece47d..71e8c431e 100644 --- a/extralit-server/tests/unit/contexts/ocr/test_arrow.py +++ b/extralit-server/tests/unit/contexts/ocr/test_arrow.py @@ -1,6 +1,7 @@ """Tests for the columnar projection of a `DoclingDocument`.""" import pyarrow as pa +import pyarrow.compute as pc import pyarrow.parquet as pq import pytest from docling_core.types.doc import BoundingBox, CoordOrigin, DocItemLabel, ProvenanceItem, Size @@ -72,7 +73,7 @@ def test_multi_prov_item_expands_to_multiple_rows(self, doc): ) table = items_table(doc, DOCUMENT_ID) - rows = table.filter(pa.compute.equal(table["self_ref"], "#/tables/0")).to_pylist() + rows = table.filter(pc.equal(table["self_ref"], "#/tables/0")).to_pylist() assert [r["prov_index"] for r in rows] == [0, 1] assert [r["page_no"] for r in rows] == [1, 2] diff --git a/extralit-server/uv.lock b/extralit-server/uv.lock index 8dfad5d0d..82fff85e5 100644 --- a/extralit-server/uv.lock +++ b/extralit-server/uv.lock @@ -1128,6 +1128,7 @@ dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "numpy", version = "2.4.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "oauthlib" }, + { name = "obstore" }, { name = "ocrmypdf" }, { name = "opencv-python-headless" }, { name = "opensearch-py" }, @@ -1153,6 +1154,7 @@ dependencies = [ { name = "typer" }, { name = "types-aiobotocore-s3" }, { name = "uvicorn", extra = ["standard"] }, + { name = "xxhash" }, ] [package.optional-dependencies] @@ -1205,6 +1207,7 @@ requires-dist = [ { name = "litellm", specifier = ">=1.80.0,<=1.82.6" }, { name = "numpy", specifier = ">=2.0.0,<3.0.0" }, { name = "oauthlib", specifier = ">=3.2.0" }, + { name = "obstore", specifier = ">=0.11.0" }, { name = "ocrmypdf", specifier = ">=16.11.0" }, { name = "opencv-python-headless", specifier = ">=4.11.0.86" }, { name = "opensearch-py", specifier = ">=2.0.0" }, @@ -1232,6 +1235,7 @@ requires-dist = [ { name = "typer", specifier = ">=0.19.1" }, { name = "types-aiobotocore-s3", specifier = "==2.24.2" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.32.0" }, + { name = "xxhash", specifier = ">=3.6.0" }, ] provides-extras = ["postgresql", "pymupdf"] @@ -2558,6 +2562,68 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/80/cab10959dc1faead58dc8384a781dfbf93cb4d33d50988f7a69f1b7c9bbe/oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca", size = 151688, upload-time = "2022-10-17T20:04:24.037Z" }, ] +[[package]] +name = "obstore" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2e/2f/f83afaab7945509d72245b2b00af0b4834ce78fdd2d9ae9f0ad1a3036a91/obstore-0.11.0.tar.gz", hash = "sha256:a2f55163bcd348b4a60d12e6893eac50eddc742bad8032a1705d49140b992204", size = 130565, upload-time = "2026-06-25T18:29:49.405Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1d/ea/3792fd91107068987d96e65b616ce2e7b461b3a04665d69ef91babd3094f/obstore-0.11.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:fc61226fecb74125f4bfdb5f8c29e440538814968e304a6df5a41e8becc21f6d", size = 5491962, upload-time = "2026-06-25T18:28:07.095Z" }, + { url = "https://files.pythonhosted.org/packages/ad/a5/60f84c4fdbe8c6f769820544b7dbc7c0595e47bb610fdb32acb87b0240e3/obstore-0.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2427bf1c18d49c04c5f08763e921ba382b615655dd0ec6544a6f696462c9ecc6", size = 4675452, upload-time = "2026-06-25T18:28:09.5Z" }, + { url = "https://files.pythonhosted.org/packages/0b/d1/558a2500ca0a7808691b13d39c39752b48987105c0f73d1c36b343ab61fb/obstore-0.11.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:931f58915ee9b71d82951e8c4e438a6823ba7d362f6edde394089bec07c9dd65", size = 5069608, upload-time = "2026-06-25T18:28:11.203Z" }, + { url = "https://files.pythonhosted.org/packages/76/85/1211b6b132a216381c1af7368be6ef7df52294209046de5ec41007c33c8e/obstore-0.11.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:86055c89ebae043dedbbe7b02b07f68f3af7e9d70eedadc5d79805572ffc2543", size = 5298950, upload-time = "2026-06-25T18:28:13.22Z" }, + { url = "https://files.pythonhosted.org/packages/01/f1/78c069aa1372b2d274895cb5b3dec097135a9a981d7ca6335bd7500ae0d5/obstore-0.11.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:51c5e265910bb6502b6246bef9f9f1f9151b8889d8bac549f94ab30eccae67d5", size = 5492459, upload-time = "2026-06-25T18:28:15.222Z" }, + { url = "https://files.pythonhosted.org/packages/6c/f7/103df5872a229fa7619a88c4ee5f181bc02a64dba7424098fece44be9058/obstore-0.11.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:82e59348de18fa332306dd4aceee14af69d64d6626cd3244de3869041dfed461", size = 5361500, upload-time = "2026-06-25T18:28:16.837Z" }, + { url = "https://files.pythonhosted.org/packages/55/40/7b03d2a2ae89b6703e70dafb1b8498b8a615601c9ebe01fbb125f635392d/obstore-0.11.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d9d7411de9eef9007c971059ca714e50ef1bff24ba95c4ca762c9155b569af4b", size = 5635311, upload-time = "2026-06-25T18:28:18.86Z" }, + { url = "https://files.pythonhosted.org/packages/2b/a9/552ca7e52813dfe26cbddff964d7807c8ac720c6d915066adbc61d8d2638/obstore-0.11.0-cp310-cp310-manylinux_2_24_aarch64.whl", hash = "sha256:9be99562b09b32e7b1e51305f9871fa6d96ecf26fa05b7df8f6064be7036e77f", size = 5415966, upload-time = "2026-06-25T18:28:20.641Z" }, + { url = "https://files.pythonhosted.org/packages/21/93/35f974522a5d325e56b2fe5d9d306231f73384216a26d5595382141b8b0e/obstore-0.11.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d40161c924435ae45c89b3b32f6329ad247b76feec17f3123f92621defa533ed", size = 5622823, upload-time = "2026-06-25T18:28:22.587Z" }, + { url = "https://files.pythonhosted.org/packages/66/b6/99580e8b8a3d26130a299b9f7f5b15ed9fcfaf57a6b69becf59d6e169842/obstore-0.11.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:b69b9e6d5fd5f36063fb17eada334fc6e52e3309daed20b04986945dbc0915a2", size = 5297560, upload-time = "2026-06-25T18:28:24.558Z" }, + { url = "https://files.pythonhosted.org/packages/73/e6/e3b70c0472c7848d57c273e172853c0d20ab97ec6491abc034da2ea8c327/obstore-0.11.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:2b1d9568db8af6341c7ca9ecf15bc444c1ac748933b0e0e93fb9f2797a16ea37", size = 5427004, upload-time = "2026-06-25T18:28:26.366Z" }, + { url = "https://files.pythonhosted.org/packages/de/4d/1fa2abe05f9ba3fa010aadb3695ce02e97ab3da6b8e12e46b309ca71f249/obstore-0.11.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2a569f81c835c28b247a531849b357249d96753a30b45b4f4c3b442f24d988fc", size = 5865056, upload-time = "2026-06-25T18:28:28.231Z" }, + { url = "https://files.pythonhosted.org/packages/df/ba/ddc091b18baa49afdfc8b270ac8c16bdd69b05bf3e75acd2e20bd5970d9d/obstore-0.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:545b23c2d13f2a5911a3b1a2c0b9dbf307b430684d171c5a7905c3e514a80662", size = 5323295, upload-time = "2026-06-25T18:28:30.107Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b2/00c213e7e5ca8065f97e37e55294adab836e3f6a88b23e4029069aaecf95/obstore-0.11.0-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:42f36546c7ac44dbab1173d2330a8a1b1a3f0e37950e553b8c904e3dd0744b25", size = 5491935, upload-time = "2026-06-25T18:28:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/ac/37/6a6b9a5e15a8a37c24d14317a87648097c4888593b588510c03c030d2e90/obstore-0.11.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:687bb9d3962d568b7c439c5d0c6fea19b2749862a8e5c8eebd0c058c4eccde9e", size = 4672619, upload-time = "2026-06-25T18:28:33.852Z" }, + { url = "https://files.pythonhosted.org/packages/28/f9/6745ce8c4f7bfac19dc14a4438b48a2e93a689b92b0cecfc695e41a4e8b1/obstore-0.11.0-cp311-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:010b51578c7514a41719d795cdb7a1e6529be509dac3772e477187a59422bb97", size = 5072806, upload-time = "2026-06-25T18:28:36.127Z" }, + { url = "https://files.pythonhosted.org/packages/6c/18/991d3b3cdd851c0225e55f3dc45b47fd9e249827d188995011469f805132/obstore-0.11.0-cp311-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cfaa8129a3f5d8518a3a75184d4b02348db0f6263177cd1f0951f6568243cc9e", size = 5303777, upload-time = "2026-06-25T18:28:37.89Z" }, + { url = "https://files.pythonhosted.org/packages/8d/e9/90e56015a45b5e56a84fc3188c4e5fb088b288d41992c73a629e10df6760/obstore-0.11.0-cp311-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c790a5cb9ff2970d1f464a6a708d734dce9939e9f668cb6708c5dba5d61589b2", size = 5493871, upload-time = "2026-06-25T18:28:39.981Z" }, + { url = "https://files.pythonhosted.org/packages/66/02/f1744091d59ce71c5523174eb860fbb298275c901e89b9ea6fbf3e654a33/obstore-0.11.0-cp311-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:827113e12fe8088e0281a9d57b90b2b8dbc8a6ffe3b15dadb9baa5feb3d266c1", size = 5361913, upload-time = "2026-06-25T18:28:42.089Z" }, + { url = "https://files.pythonhosted.org/packages/5d/59/3f47822683ee2b6db8685faa25829946d6343a561251ec2704548455d946/obstore-0.11.0-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a2ff6d3ed553298828fb760b4aef6347fbcc7b5c5e3ce3f8381ce805c370021a", size = 5638724, upload-time = "2026-06-25T18:28:43.897Z" }, + { url = "https://files.pythonhosted.org/packages/23/50/1df335fdf9b527b3933f1e94ab6fc720ad314260fab8591cb0b6668ff192/obstore-0.11.0-cp311-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:39d04b324fcf984e7050734ebda77b81764025b0c011750201a0d8954087f7aa", size = 5413508, upload-time = "2026-06-25T18:28:45.624Z" }, + { url = "https://files.pythonhosted.org/packages/de/dc/a259aba149b841ca7c91fea177df9972a60a636b54077beed1a35b254994/obstore-0.11.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:37c0d15d775b1370ef5204ee3919a5ddf7e2592d11815213105f8db031f2ab8d", size = 5619995, upload-time = "2026-06-25T18:28:47.599Z" }, + { url = "https://files.pythonhosted.org/packages/e5/b4/ec25fdb4d6b060bc6eea647fc0e88f75fcc20fe8d16d67fb0dbe999d323b/obstore-0.11.0-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:7f468caf9b6e0f12ff151e5fe618de5fc9192befa9bd02734b06de4efd2e49f6", size = 5299512, upload-time = "2026-06-25T18:28:49.629Z" }, + { url = "https://files.pythonhosted.org/packages/a8/e5/29be060d06ec13e2af3d1b6cfb77b7c37f8be6c56b77295c945fefad73e4/obstore-0.11.0-cp311-abi3-musllinux_1_2_i686.whl", hash = "sha256:42d8e8fad85be8ee488c1a9a9b7c6a42128abb84e67175da40d3d1165c1846df", size = 5427026, upload-time = "2026-06-25T18:28:51.317Z" }, + { url = "https://files.pythonhosted.org/packages/57/b7/577a965f440e9ea64243518663f9d16be7df8eafc7123818e8e841fa21ce/obstore-0.11.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9c8fd2a544e2e0b926669c47fcfb8d2314e234abc240ea165dae04ee42e1d7ac", size = 5869187, upload-time = "2026-06-25T18:28:53.166Z" }, + { url = "https://files.pythonhosted.org/packages/e2/18/8fdbaee22bfd5b9c44e1fdff8ca0508e2fe60c42bf9fc85f0c9c27b4ecf2/obstore-0.11.0-cp311-abi3-win_amd64.whl", hash = "sha256:6fb3d4678c0f4242d3109362e9b1df5d7b27765f43d5aacb2e81af53a75cb9ef", size = 5329384, upload-time = "2026-06-25T18:28:55.305Z" }, + { url = "https://files.pythonhosted.org/packages/8b/8b/7555e48ec768728fcfc71a051c6b28d6ddaf1bececf492ce5ef995aab5f0/obstore-0.11.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:f3132393eff9f3f2b543ecbb3bcc12319a7c433fef06493b4350d6854d505a14", size = 5515763, upload-time = "2026-06-25T18:28:57.527Z" }, + { url = "https://files.pythonhosted.org/packages/23/8f/94d83f3336421cbb5e436ab0ae5695eae72f7c82990d6b1ac090712c8052/obstore-0.11.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a3a8da19b47af4c14ecc694209b3c18ac6d89f96be5656ee3a19b77947c14155", size = 4649491, upload-time = "2026-06-25T18:28:59.386Z" }, + { url = "https://files.pythonhosted.org/packages/fe/86/11f4e1f51a8c6cf21a5915c018d2357201ad3c5799d418f0c6529fafaab2/obstore-0.11.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e91298b9b6c3a0408c28eece62bca6c5b6cda2f6350351d84e07b4dc8fb2631f", size = 5060659, upload-time = "2026-06-25T18:29:01.139Z" }, + { url = "https://files.pythonhosted.org/packages/49/e7/fd3036b0923d10e878e2073020f1ef692a618ed1cc3980d3e4a468c93713/obstore-0.11.0-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3422c532486671dfb5e3e739bf15ec9ca2a8da544a3c23b74ae3857dcab1c6a8", size = 5277058, upload-time = "2026-06-25T18:29:02.96Z" }, + { url = "https://files.pythonhosted.org/packages/2f/a7/b016c3ac6857ac856326dc0a292e3871b8500d03f25f3b90e168b05de357/obstore-0.11.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3de027ce46cf0592c2b41654e2c27dbc52a4a726f6fbc511380acc0da3a9f658", size = 5475852, upload-time = "2026-06-25T18:29:05.032Z" }, + { url = "https://files.pythonhosted.org/packages/85/9e/644ffe8db7757de7f71f94a036ab24222bccb4de290d3ca69f76547e812d/obstore-0.11.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1a80a95548678210bc336b866e37139c293565c3b163eb3fef2433d5d6640a33", size = 5363082, upload-time = "2026-06-25T18:29:06.878Z" }, + { url = "https://files.pythonhosted.org/packages/ae/0b/26af6b5fa6ba96af84086f44f78b4e5b0af1729c31402d7b28c68989d174/obstore-0.11.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acaa261dc15efb95bbeca06f8fe9b47ee23d7302a6aa1fa3a9654baab8b23d7c", size = 5629116, upload-time = "2026-06-25T18:29:08.771Z" }, + { url = "https://files.pythonhosted.org/packages/ae/ce/f30d502991c6719b2fbd7b8385ef3e39da07bfca099108bcc5eeed8b9c20/obstore-0.11.0-cp314-cp314t-manylinux_2_24_aarch64.whl", hash = "sha256:a3300cabbc3129670987b3723629c791d83c117ef1b6a0c670c2043648e000a1", size = 5404534, upload-time = "2026-06-25T18:29:11.294Z" }, + { url = "https://files.pythonhosted.org/packages/52/20/d5bf5f816e868717ba647ed9a2109e800deb402d0265d410456b3fcb4376/obstore-0.11.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f4e6a9480843645cd4ee122c41d5c5a46a56f1e9cdda85638826f2e0e439fe5c", size = 5613159, upload-time = "2026-06-25T18:29:13.414Z" }, + { url = "https://files.pythonhosted.org/packages/db/b4/6d4c1c211e3b06cc8554189e0d4406e8fa1f98ed55f9213b8e398a11599f/obstore-0.11.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:9fb2b1814c4314b8903f4e2ebbe8c3365fea6543669615ee9a0288b0d3a2edeb", size = 5286279, upload-time = "2026-06-25T18:29:15.424Z" }, + { url = "https://files.pythonhosted.org/packages/02/5e/d7b5589424a56171b16ab94cf92eb493490c300aaa044913bbdd94cace68/obstore-0.11.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:63fb9b072815eafe4705f617f567d896b1c858adcb390795fa1e269367791031", size = 5401780, upload-time = "2026-06-25T18:29:17.514Z" }, + { url = "https://files.pythonhosted.org/packages/c4/18/841baea8936e51a18b0e5d4c51f09c0a7798cb73b027e9794be2362a0f0b/obstore-0.11.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:086fafba314ff98cfab1c4bf7814699e862513e8889720cf6f7462296cb32787", size = 5853618, upload-time = "2026-06-25T18:29:19.354Z" }, + { url = "https://files.pythonhosted.org/packages/83/9a/d6127f5422b78e0222b0a9eadcfd7a5aa8d873a9498da7d4a77d4ac8ce2e/obstore-0.11.0-cp314-cp314t-win_amd64.whl", hash = "sha256:676d1154f6f08721110f9b7d14ee3a3c0293abaf9da135bb90f54e276dca1cac", size = 5314113, upload-time = "2026-06-25T18:29:21.209Z" }, + { url = "https://files.pythonhosted.org/packages/61/17/6ddc3e035a0adc3397bc2b9ea4ebe52db6711ad87ddf0180b7675fa25d6a/obstore-0.11.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c30aec09acff505e27b7392eb5e4b7bb7073d3f21e44ea43a64913369d83cba0", size = 5500652, upload-time = "2026-06-25T18:29:23.423Z" }, + { url = "https://files.pythonhosted.org/packages/7c/66/0e8ebfebf7151b401dee19373b3a699b179c3687bea5d684adbef2c7c67d/obstore-0.11.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:32ec11db91b85e4482baf2c8c0f43b469e8788765cd844839c84468516446181", size = 4681675, upload-time = "2026-06-25T18:29:25.766Z" }, + { url = "https://files.pythonhosted.org/packages/fa/84/717505cdf8e453ef16be74e6ca4204629722227c1cbecf3719a17513dd9a/obstore-0.11.0-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0a7f574cf222156f95846fda755365085e8c825be48359f32fa57935fa79f172", size = 5078712, upload-time = "2026-06-25T18:29:27.502Z" }, + { url = "https://files.pythonhosted.org/packages/e2/e4/a3a87a9ef6973cc59f8cd3543b74cde3bac81ff18bfdab80a460c95d5df0/obstore-0.11.0-pp311-pypy311_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:193d8af924c24ab60c342d64c55896f8cae026851182ade6f1b650789d17b84a", size = 5309116, upload-time = "2026-06-25T18:29:29.912Z" }, + { url = "https://files.pythonhosted.org/packages/98/40/af6699c140cd4f5fb638ccad2053b096ebaa8b5fcd2a2c5176113c1bc847/obstore-0.11.0-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ed68e3653f12a995bca4372cdccf8663bd3df2ce9750132d11c4c04489e2dc61", size = 5503037, upload-time = "2026-06-25T18:29:32.396Z" }, + { url = "https://files.pythonhosted.org/packages/22/0c/e83123e8e2d2075dd686ed1c13e462c19b9c57ec0a67c316bdde862c6def/obstore-0.11.0-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ee8d5aa13158e39d20e76730fdc81a50fd80f6d415a1dc327cf317e5c4e2e878", size = 5368241, upload-time = "2026-06-25T18:29:34.366Z" }, + { url = "https://files.pythonhosted.org/packages/8a/94/d1cba6347ed6e540765e6db2c2a262f8aba50ae40a0d47749465c42e94a0/obstore-0.11.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca0a6bef07fc26990828528090c6b41860ca949d3cf1b81b24854674173b47ba", size = 5644399, upload-time = "2026-06-25T18:29:36.482Z" }, + { url = "https://files.pythonhosted.org/packages/d0/24/a9033feffb423d5f6a7bdc85041e9b9e1f67b2835fd484427d005ed17187/obstore-0.11.0-pp311-pypy311_pp73-manylinux_2_24_aarch64.whl", hash = "sha256:c9c6ccc06801afb0fc0e8f0e8c957f643c2c4f7e349eb36a736167e280709c6c", size = 5423124, upload-time = "2026-06-25T18:29:38.513Z" }, + { url = "https://files.pythonhosted.org/packages/69/e8/2ee75c66bc703670bbf1aa51c320a4d282efa85adf8e9fa3f7467c02513e/obstore-0.11.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:9555f36a0724d34d747b38cda0cb55b3f650a22e8671614037c52190bf4da6cc", size = 5630049, upload-time = "2026-06-25T18:29:40.647Z" }, + { url = "https://files.pythonhosted.org/packages/f4/72/76b3099664d997747e5e7b11b322926d486d1471266941fc0103d912b666/obstore-0.11.0-pp311-pypy311_pp73-musllinux_1_2_armv7l.whl", hash = "sha256:f0c1527daf1c75d3f70ee3f6bef2767d8b07005cd7a5b43dc8c96a1323f3ac54", size = 5308482, upload-time = "2026-06-25T18:29:42.643Z" }, + { url = "https://files.pythonhosted.org/packages/34/96/03594ac63d7b1a0e773b0c604c38003ee828e6d5348488eaeb2c57bdc29e/obstore-0.11.0-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:ed5910bde525d7d936d36dc78c7d4c07b57b6c9b55e402357b107103595d56ff", size = 5433377, upload-time = "2026-06-25T18:29:45.323Z" }, + { url = "https://files.pythonhosted.org/packages/2c/53/07226c948760264fb7ba253825ecbe5941abfac9875e6d1b2679cc82e3ef/obstore-0.11.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:65c0208ead516c1a79afced16f2cda9b5542886123e53ea0a8a7d5d56b86fba5", size = 5868708, upload-time = "2026-06-25T18:29:47.668Z" }, +] + [[package]] name = "ocrmypdf" version = "16.13.0" From f22d9677ec498558fadb90212f9c854cc0b07fec Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Mon, 17 Aug 2026 18:30:26 -0700 Subject: [PATCH 10/10] test(ocr): assert the layout job forwards its workflow token The fixture returned `None` from `get_current_job`, so the job under test computed `workflow_id=None` and the superseded assertion passed on a mock forced to return False regardless of its arguments. Dropping the token entirely would still have gone green; it now fails. --- extralit-server/tests/unit/jobs/test_ocr_jobs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extralit-server/tests/unit/jobs/test_ocr_jobs.py b/extralit-server/tests/unit/jobs/test_ocr_jobs.py index 83bf664d8..0f6d01ca8 100644 --- a/extralit-server/tests/unit/jobs/test_ocr_jobs.py +++ b/extralit-server/tests/unit/jobs/test_ocr_jobs.py @@ -39,7 +39,7 @@ async def update_metadata(db, document_id, mutate): patch(f"{MODULE}.LayoutStore.for_workspace", return_value=store), patch(f"{MODULE}.storage.store_layout", store_layout), patch(f"{MODULE}.update_processing_metadata", update_metadata), - patch(f"{MODULE}.get_current_job", return_value=None), + patch(f"{MODULE}.get_current_job", return_value=MagicMock(meta={"workflow_id": "wf-1"})), patch(f"{MODULE}.is_current_workflow_run", AsyncMock(return_value=True)) as is_current, patch(f"{MODULE}.AsyncSessionLocal") as session, ): @@ -93,5 +93,6 @@ async def test_a_superseded_run_does_not_write(self, job_context): result = await run_job() + assert job_context["is_current"].await_args.args[2] == "wf-1" assert result["skipped"] == "workflow superseded" assert job_context["calls"] == []