diff --git a/benchmark/detection.py b/benchmark/detection.py index 9ff2f98..6e23a9b 100644 --- a/benchmark/detection.py +++ b/benchmark/detection.py @@ -2,26 +2,32 @@ import collections import copy import json +import os +import time + +import datasets +from tabulate import tabulate from surya.benchmark.bbox import get_pdf_lines from surya.benchmark.metrics import precision_recall -from surya.benchmark.tesseract import tesseract_bboxes, tesseract_parallel -from surya.model.segformer import load_model, load_processor -from surya.model.processing import open_pdf, get_page_images +from surya.benchmark.tesseract import tesseract_parallel from surya.detection import batch_inference +from surya.model.processing import get_page_images, open_pdf +from surya.model.segformer import load_model, load_processor from surya.postprocessing.heatmap import draw_bboxes_on_image from surya.postprocessing.util import rescale_bbox from surya.settings import settings -import os -import time -from tabulate import tabulate -import datasets def main(): parser = argparse.ArgumentParser(description="Detect bboxes in a PDF.") parser.add_argument("--pdf_path", type=str, help="Path to PDF to detect bboxes in.", default=None) - parser.add_argument("--results_dir", type=str, help="Path to JSON file with OCR results.", default=os.path.join(settings.RESULT_DIR, "benchmark")) + parser.add_argument( + "--results_dir", + type=str, + help="Path to JSON file with OCR results.", + default=os.path.join(settings.RESULT_DIR, "benchmark"), + ) parser.add_argument("--max", type=int, help="Maximum number of pdf pages to OCR.", default=100) parser.add_argument("--debug", action="store_true", help="Run in debug mode.", default=False) args = parser.parse_args() @@ -34,7 +40,7 @@ def main(): doc = open_pdf(args.pdf_path) page_count = len(doc) page_indices = list(range(page_count)) - page_indices = page_indices[:args.max] + page_indices = page_indices[: args.max] images = get_page_images(doc, page_indices) doc.close() @@ -72,10 +78,7 @@ def main(): surya_metrics = precision_recall(surya_boxes, cb) tess_metrics = precision_recall(tb, cb) - page_metrics[idx] = { - "surya": surya_metrics, - "tesseract": tess_metrics - } + page_metrics[idx] = {"surya": surya_metrics, "tesseract": tess_metrics} if args.debug: bbox_image = draw_bboxes_on_image(surya_boxes, copy.deepcopy(images[idx])) @@ -93,12 +96,9 @@ def main(): mean_metrics[k][m] = sum(metric) / len(metric) out_data = { - "times": { - "surya": surya_time, - "tesseract": tess_time - }, + "times": {"surya": surya_time, "tesseract": tess_time}, "metrics": mean_metrics, - "page_metrics": page_metrics + "page_metrics": page_metrics, } with open(os.path.join(result_path, "results.json"), "w+") as f: @@ -107,11 +107,13 @@ def main(): table_headers = ["Model", "Time (s)", "Time per page (s)"] + metric_types table_data = [ ["surya", surya_time, surya_time / len(images)] + [mean_metrics["surya"][m] for m in metric_types], - ["tesseract", tess_time, tess_time / len(images)] + [mean_metrics["tesseract"][m] for m in metric_types] + ["tesseract", tess_time, tess_time / len(images)] + [mean_metrics["tesseract"][m] for m in metric_types], ] print(tabulate(table_data, headers=table_headers, tablefmt="github")) - print("Precision and recall are over the mutual coverage of the detected boxes and the ground truth boxes at a .5 threshold. There is a precision penalty for multiple boxes overlapping reference lines.") + print( + "Precision and recall are over the mutual coverage of the detected boxes and the ground truth boxes at a .5 threshold. There is a precision penalty for multiple boxes overlapping reference lines." + ) print(f"Wrote results to {result_path}") diff --git a/benchmark/pymupdf_test.py b/benchmark/pymupdf_test.py index dee2cec..bc1fb91 100644 --- a/benchmark/pymupdf_test.py +++ b/benchmark/pymupdf_test.py @@ -2,16 +2,20 @@ import os from surya.benchmark.bbox import get_pdf_lines +from surya.model.processing import get_page_images, open_pdf from surya.postprocessing.heatmap import draw_bboxes_on_image - -from surya.model.processing import open_pdf, get_page_images from surya.settings import settings def main(): parser = argparse.ArgumentParser(description="Draw pymupdf line bboxes on images.") parser.add_argument("pdf_path", type=str, help="Path to PDF to detect bboxes in.") - parser.add_argument("--results_dir", type=str, help="Path to JSON file with OCR results.", default=os.path.join(settings.RESULT_DIR, "pymupdf")) + parser.add_argument( + "--results_dir", + type=str, + help="Path to JSON file with OCR results.", + default=os.path.join(settings.RESULT_DIR, "pymupdf"), + ) args = parser.parse_args() doc = open_pdf(args.pdf_path) @@ -34,6 +38,6 @@ def main(): print(f"Wrote results to {result_path}") + if __name__ == "__main__": main() - diff --git a/benchmark/tesseract_test.py b/benchmark/tesseract_test.py index d39d4e6..070f2a0 100644 --- a/benchmark/tesseract_test.py +++ b/benchmark/tesseract_test.py @@ -2,16 +2,20 @@ import os from surya.benchmark.tesseract import tesseract_bboxes +from surya.model.processing import get_page_images, open_pdf from surya.postprocessing.heatmap import draw_bboxes_on_image - -from surya.model.processing import open_pdf, get_page_images from surya.settings import settings def main(): parser = argparse.ArgumentParser(description="Draw tesseract bboxes on imagese.") parser.add_argument("pdf_path", type=str, help="Path to PDF to detect bboxes in.") - parser.add_argument("--results_dir", type=str, help="Path to JSON file with OCR results.", default=os.path.join(settings.RESULT_DIR, "tesseract")) + parser.add_argument( + "--results_dir", + type=str, + help="Path to JSON file with OCR results.", + default=os.path.join(settings.RESULT_DIR, "tesseract"), + ) args = parser.parse_args() doc = open_pdf(args.pdf_path) @@ -33,6 +37,6 @@ def main(): print(f"Wrote results to {result_path}") + if __name__ == "__main__": main() - diff --git a/detect_text.py b/detect_text.py index a339fc8..d770ada 100644 --- a/detect_text.py +++ b/detect_text.py @@ -1,18 +1,18 @@ import argparse import copy import json +import os from collections import defaultdict +import filetype from PIL import Image -from surya.model.segformer import load_model, load_processor -from surya.model.processing import open_pdf, get_page_images from surya.detection import batch_inference +from surya.model.processing import get_page_images, open_pdf +from surya.model.segformer import load_model, load_processor from surya.postprocessing.affinity import draw_lines_on_image from surya.postprocessing.heatmap import draw_bboxes_on_image from surya.settings import settings -import os -import filetype def get_name_from_path(path): @@ -68,7 +68,12 @@ def load_from_folder(folder_path, max_pages=None): def main(): parser = argparse.ArgumentParser(description="Detect bboxes in an input file or folder (PDFs or image).") parser.add_argument("input_path", type=str, help="Path to pdf or image file to detect bboxes in.") - parser.add_argument("--results_dir", type=str, help="Path to JSON file with OCR results.", default=os.path.join(settings.RESULT_DIR, "surya")) + parser.add_argument( + "--results_dir", + type=str, + help="Path to JSON file with OCR results.", + default=os.path.join(settings.RESULT_DIR, "surya"), + ) parser.add_argument("--max", type=int, help="Maximum number of pages to process.", default=None) parser.add_argument("--images", action="store_true", help="Save images of detected bboxes.", default=False) parser.add_argument("--debug", action="store_true", help="Run in debug mode.", default=False) @@ -121,10 +126,3 @@ def main(): if __name__ == "__main__": main() - - - - - - - diff --git a/poetry.lock b/poetry.lock index 17a385f..4b01456 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,9 +1,10 @@ -# This file is automatically @generated by Poetry 1.5.0 and should not be changed by hand. +# This file is automatically @generated by Poetry and should not be changed by hand. [[package]] name = "aiohttp" version = "3.9.1" description = "Async http client/server framework (asyncio)" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -100,6 +101,7 @@ speedups = ["Brotli", "aiodns", "brotlicffi"] name = "aiosignal" version = "1.3.1" description = "aiosignal: a list of registered asynchronous callbacks" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -114,6 +116,7 @@ frozenlist = ">=1.1.0" name = "annotated-types" version = "0.6.0" description = "Reusable constraint types to use with typing.Annotated" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -125,6 +128,7 @@ files = [ name = "anyio" version = "4.2.0" description = "High level compatibility layer for multiple asynchronous event loop implementations" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -147,6 +151,7 @@ trio = ["trio (>=0.23)"] name = "appnope" version = "0.1.3" description = "Disable App Nap on macOS >= 10.9" +category = "dev" optional = false python-versions = "*" files = [ @@ -158,6 +163,7 @@ files = [ name = "argon2-cffi" version = "23.1.0" description = "Argon2 for Python" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -178,6 +184,7 @@ typing = ["mypy"] name = "argon2-cffi-bindings" version = "21.2.0" description = "Low-level CFFI bindings for Argon2" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -215,6 +222,7 @@ tests = ["pytest"] name = "arrow" version = "1.3.0" description = "Better dates & times for Python" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -228,12 +236,13 @@ types-python-dateutil = ">=2.8.10" [package.extras] doc = ["doc8", "sphinx (>=7.0.0)", "sphinx-autobuild", "sphinx-autodoc-typehints", "sphinx_rtd_theme (>=1.3.0)"] -test = ["dateparser (==1.*)", "pre-commit", "pytest", "pytest-cov", "pytest-mock", "pytz (==2021.1)", "simplejson (==3.*)"] +test = ["dateparser (>=1.0.0,<2.0.0)", "pre-commit", "pytest", "pytest-cov", "pytest-mock", "pytz (==2021.1)", "simplejson (>=3.0.0,<4.0.0)"] [[package]] name = "asttokens" version = "2.4.1" description = "Annotate AST trees with source code positions" +category = "dev" optional = false python-versions = "*" files = [ @@ -252,6 +261,7 @@ test = ["astroid (>=1,<2)", "astroid (>=2,<4)", "pytest"] name = "async-lru" version = "2.0.4" description = "Simple LRU cache for asyncio" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -266,6 +276,7 @@ typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""} name = "async-timeout" version = "4.0.3" description = "Timeout context manager for asyncio programs" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -277,6 +288,7 @@ files = [ name = "attrs" version = "23.2.0" description = "Classes Without Boilerplate" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -296,6 +308,7 @@ tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "p name = "babel" version = "2.14.0" description = "Internationalization utilities" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -310,6 +323,7 @@ dev = ["freezegun (>=1.0,<2.0)", "pytest (>=6.0)", "pytest-cov"] name = "beautifulsoup4" version = "4.12.2" description = "Screen-scraping library" +category = "dev" optional = false python-versions = ">=3.6.0" files = [ @@ -328,6 +342,7 @@ lxml = ["lxml"] name = "bleach" version = "6.1.0" description = "An easy safelist-based HTML-sanitizing tool." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -346,6 +361,7 @@ css = ["tinycss2 (>=1.1.0,<1.3)"] name = "certifi" version = "2023.11.17" description = "Python package for providing Mozilla's CA Bundle." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -357,6 +373,7 @@ files = [ name = "cffi" version = "1.16.0" description = "Foreign Function Interface for Python calling C code." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -421,6 +438,7 @@ pycparser = "*" name = "charset-normalizer" version = "3.3.2" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -520,6 +538,7 @@ files = [ name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -531,6 +550,7 @@ files = [ name = "comm" version = "0.2.1" description = "Jupyter Python Comm implementation, for usage in ipykernel, xeus-python etc." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -548,6 +568,7 @@ test = ["pytest"] name = "datasets" version = "2.16.1" description = "HuggingFace community-driven open-source library of datasets" +category = "dev" optional = false python-versions = ">=3.8.0" files = [ @@ -592,6 +613,7 @@ vision = ["Pillow (>=6.2.1)"] name = "debugpy" version = "1.8.0" description = "An implementation of the Debug Adapter Protocol for Python" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -619,6 +641,7 @@ files = [ name = "decorator" version = "5.1.1" description = "Decorators for Humans" +category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -630,6 +653,7 @@ files = [ name = "defusedxml" version = "0.7.1" description = "XML bomb protection for Python stdlib modules" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -641,6 +665,7 @@ files = [ name = "dill" version = "0.3.7" description = "serialize all of Python" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -655,6 +680,7 @@ graph = ["objgraph (>=1.7.2)"] name = "exceptiongroup" version = "1.2.0" description = "Backport of PEP 654 (exception groups)" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -669,6 +695,7 @@ test = ["pytest (>=6)"] name = "executing" version = "2.0.1" description = "Get the currently executing AST node of a frame, and other information" +category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -683,6 +710,7 @@ tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipyth name = "fastjsonschema" version = "2.19.1" description = "Fastest Python implementation of JSON schema" +category = "dev" optional = false python-versions = "*" files = [ @@ -697,6 +725,7 @@ devel = ["colorama", "json-spec", "jsonschema", "pylint", "pytest", "pytest-benc name = "filelock" version = "3.13.1" description = "A platform independent file lock." +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -713,6 +742,7 @@ typing = ["typing-extensions (>=4.8)"] name = "filetype" version = "1.2.0" description = "Infer file type and MIME type of any file/buffer. No external dependencies." +category = "main" optional = false python-versions = "*" files = [ @@ -724,6 +754,7 @@ files = [ name = "fqdn" version = "1.5.1" description = "Validates fully-qualified domain names against RFC 1123, so that they are acceptable to modern bowsers" +category = "dev" optional = false python-versions = ">=2.7, !=3.0, !=3.1, !=3.2, !=3.3, !=3.4, <4" files = [ @@ -735,6 +766,7 @@ files = [ name = "frozenlist" version = "1.4.1" description = "A list-like structure which implements collections.abc.MutableSequence" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -821,6 +853,7 @@ files = [ name = "fsspec" version = "2023.10.0" description = "File-system specification" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -860,6 +893,7 @@ tqdm = ["tqdm"] name = "huggingface-hub" version = "0.20.2" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" +category = "main" optional = false python-versions = ">=3.8.0" files = [ @@ -892,6 +926,7 @@ typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "t name = "idna" version = "3.6" description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -903,6 +938,7 @@ files = [ name = "importlib-metadata" version = "7.0.1" description = "Read metadata from Python packages" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -922,6 +958,7 @@ testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs name = "ipykernel" version = "6.28.0" description = "IPython Kernel for Jupyter" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -935,7 +972,7 @@ comm = ">=0.1.1" debugpy = ">=1.6.5" ipython = ">=7.23.1" jupyter-client = ">=6.1.12" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0.0 || >=5.1.0" matplotlib-inline = ">=0.1" nest-asyncio = "*" packaging = "*" @@ -955,6 +992,7 @@ test = ["flaky", "ipyparallel", "pre-commit", "pytest (>=7.0)", "pytest-asyncio" name = "ipython" version = "8.18.1" description = "IPython: Productive Interactive Computing" +category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -992,6 +1030,7 @@ test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.22)", "pa name = "ipywidgets" version = "8.1.1" description = "Jupyter interactive widgets" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1013,6 +1052,7 @@ test = ["ipykernel", "jsonschema", "pytest (>=3.6.0)", "pytest-cov", "pytz"] name = "isoduration" version = "20.11.0" description = "Operations with ISO 8601 durations" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1027,6 +1067,7 @@ arrow = ">=0.15.0" name = "jedi" version = "0.19.1" description = "An autocompletion tool for Python that can be used for text editors." +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1046,6 +1087,7 @@ testing = ["Django", "attrs", "colorama", "docopt", "pytest (<7.0.0)"] name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1063,6 +1105,7 @@ i18n = ["Babel (>=2.7)"] name = "json5" version = "0.9.14" description = "A Python implementation of the JSON5 data format." +category = "dev" optional = false python-versions = "*" files = [ @@ -1077,6 +1120,7 @@ dev = ["hypothesis"] name = "jsonpointer" version = "2.4" description = "Identify specific nodes in a JSON document (RFC 6901)" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" files = [ @@ -1088,6 +1132,7 @@ files = [ name = "jsonschema" version = "4.20.0" description = "An implementation of JSON Schema validation for Python" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1117,6 +1162,7 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339- name = "jsonschema-specifications" version = "2023.12.1" description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1131,6 +1177,7 @@ referencing = ">=0.31.0" name = "jupyter" version = "1.0.0" description = "Jupyter metapackage. Install all the Jupyter components in one go." +category = "dev" optional = false python-versions = "*" files = [ @@ -1151,6 +1198,7 @@ qtconsole = "*" name = "jupyter-client" version = "8.6.0" description = "Jupyter protocol implementation and client libraries" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1160,7 +1208,7 @@ files = [ [package.dependencies] importlib-metadata = {version = ">=4.8.3", markers = "python_version < \"3.10\""} -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0.0 || >=5.1.0" python-dateutil = ">=2.8.2" pyzmq = ">=23.0" tornado = ">=6.2" @@ -1174,6 +1222,7 @@ test = ["coverage", "ipykernel (>=6.14)", "mypy", "paramiko", "pre-commit", "pyt name = "jupyter-console" version = "6.6.3" description = "Jupyter terminal console" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1185,7 +1234,7 @@ files = [ ipykernel = ">=6.14" ipython = "*" jupyter-client = ">=7.0.0" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0.0 || >=5.1.0" prompt-toolkit = ">=3.0.30" pygments = "*" pyzmq = ">=17" @@ -1198,6 +1247,7 @@ test = ["flaky", "pexpect", "pytest"] name = "jupyter-core" version = "5.7.1" description = "Jupyter core package. A base package on which Jupyter projects rely." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1218,6 +1268,7 @@ test = ["ipykernel", "pre-commit", "pytest", "pytest-cov", "pytest-timeout"] name = "jupyter-events" version = "0.9.0" description = "Jupyter Event System library" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1243,6 +1294,7 @@ test = ["click", "pre-commit", "pytest (>=7.0)", "pytest-asyncio (>=0.19.0)", "p name = "jupyter-lsp" version = "2.2.1" description = "Multi-Language Server WebSocket proxy for Jupyter Notebook/Lab server" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1258,6 +1310,7 @@ jupyter-server = ">=1.1.2" name = "jupyter-server" version = "2.12.3" description = "The backend—i.e. core services, APIs, and REST endpoints—to Jupyter web applications." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1270,7 +1323,7 @@ anyio = ">=3.1.0" argon2-cffi = "*" jinja2 = "*" jupyter-client = ">=7.4.4" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0.0 || >=5.1.0" jupyter-events = ">=0.9.0" jupyter-server-terminals = "*" nbconvert = ">=6.4.4" @@ -1294,6 +1347,7 @@ test = ["flaky", "ipykernel", "pre-commit", "pytest (>=7.0)", "pytest-console-sc name = "jupyter-server-terminals" version = "0.5.1" description = "A Jupyter Server Extension Providing Terminals." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1313,6 +1367,7 @@ test = ["jupyter-server (>=2.0.0)", "pytest (>=7.0)", "pytest-jupyter[server] (> name = "jupyterlab" version = "4.0.10" description = "JupyterLab computational environment" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1345,6 +1400,7 @@ test = ["coverage", "pytest (>=7.0)", "pytest-check-links (>=0.7)", "pytest-cons name = "jupyterlab-pygments" version = "0.3.0" description = "Pygments theme using JupyterLab CSS variables" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1356,6 +1412,7 @@ files = [ name = "jupyterlab-server" version = "2.25.2" description = "A set of server components for JupyterLab and JupyterLab like applications." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1382,6 +1439,7 @@ test = ["hatch", "ipykernel", "openapi-core (>=0.18.0,<0.19.0)", "openapi-spec-v name = "jupyterlab-widgets" version = "3.0.9" description = "Jupyter interactive widgets for JupyterLab" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1393,6 +1451,7 @@ files = [ name = "markupsafe" version = "2.1.3" description = "Safely add untrusted strings to HTML/XML markup." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1416,6 +1475,16 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -1452,6 +1521,7 @@ files = [ name = "matplotlib-inline" version = "0.1.6" description = "Inline Matplotlib backend for Jupyter" +category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1466,6 +1536,7 @@ traitlets = "*" name = "mistune" version = "3.0.2" description = "A sane and fast Markdown parser with useful plugins and renderers" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1477,6 +1548,7 @@ files = [ name = "mpmath" version = "1.3.0" description = "Python library for arbitrary-precision floating-point arithmetic" +category = "main" optional = false python-versions = "*" files = [ @@ -1494,6 +1566,7 @@ tests = ["pytest (>=4.6)"] name = "multidict" version = "6.0.4" description = "multidict implementation" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1577,6 +1650,7 @@ files = [ name = "multiprocess" version = "0.70.15" description = "better multiprocessing and multithreading in Python" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1605,6 +1679,7 @@ dill = ">=0.3.7" name = "nbclient" version = "0.9.0" description = "A client library for executing notebooks. Formerly nbconvert's ExecutePreprocessor." +category = "dev" optional = false python-versions = ">=3.8.0" files = [ @@ -1614,7 +1689,7 @@ files = [ [package.dependencies] jupyter-client = ">=6.1.12" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0.0 || >=5.1.0" nbformat = ">=5.1" traitlets = ">=5.4" @@ -1627,6 +1702,7 @@ test = ["flaky", "ipykernel (>=6.19.3)", "ipython", "ipywidgets", "nbconvert (>= name = "nbconvert" version = "7.14.0" description = "Converting Jupyter Notebooks" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1665,6 +1741,7 @@ webpdf = ["playwright"] name = "nbformat" version = "5.9.2" description = "The Jupyter Notebook format" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1686,6 +1763,7 @@ test = ["pep440", "pre-commit", "pytest", "testpath"] name = "nest-asyncio" version = "1.5.8" description = "Patch asyncio to allow nested event loops" +category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1697,6 +1775,7 @@ files = [ name = "networkx" version = "3.2.1" description = "Python package for creating and manipulating graphs and networks" +category = "main" optional = false python-versions = ">=3.9" files = [ @@ -1715,6 +1794,7 @@ test = ["pytest (>=7.2)", "pytest-cov (>=4.0)"] name = "notebook" version = "7.0.6" description = "Jupyter Notebook - A web-based notebook environment for interactive computing" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1738,6 +1818,7 @@ test = ["importlib-resources (>=5.0)", "ipykernel", "jupyter-server[test] (>=2.4 name = "notebook-shim" version = "0.2.3" description = "A shim layer for notebook traits and config" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1755,6 +1836,7 @@ test = ["pytest", "pytest-console-scripts", "pytest-jupyter", "pytest-tornasync" name = "numpy" version = "1.26.3" description = "Fundamental package for array computing in Python" +category = "main" optional = false python-versions = ">=3.9" files = [ @@ -1800,6 +1882,7 @@ files = [ name = "nvidia-cublas-cu12" version = "12.1.3.1" description = "CUBLAS native runtime libraries" +category = "main" optional = false python-versions = ">=3" files = [ @@ -1811,6 +1894,7 @@ files = [ name = "nvidia-cuda-cupti-cu12" version = "12.1.105" description = "CUDA profiling tools runtime libs." +category = "main" optional = false python-versions = ">=3" files = [ @@ -1822,6 +1906,7 @@ files = [ name = "nvidia-cuda-nvrtc-cu12" version = "12.1.105" description = "NVRTC native runtime libraries" +category = "main" optional = false python-versions = ">=3" files = [ @@ -1833,6 +1918,7 @@ files = [ name = "nvidia-cuda-runtime-cu12" version = "12.1.105" description = "CUDA Runtime native Libraries" +category = "main" optional = false python-versions = ">=3" files = [ @@ -1844,6 +1930,7 @@ files = [ name = "nvidia-cudnn-cu12" version = "8.9.2.26" description = "cuDNN runtime libraries" +category = "main" optional = false python-versions = ">=3" files = [ @@ -1857,6 +1944,7 @@ nvidia-cublas-cu12 = "*" name = "nvidia-cufft-cu12" version = "11.0.2.54" description = "CUFFT native runtime libraries" +category = "main" optional = false python-versions = ">=3" files = [ @@ -1868,6 +1956,7 @@ files = [ name = "nvidia-curand-cu12" version = "10.3.2.106" description = "CURAND native runtime libraries" +category = "main" optional = false python-versions = ">=3" files = [ @@ -1879,6 +1968,7 @@ files = [ name = "nvidia-cusolver-cu12" version = "11.4.5.107" description = "CUDA solver native runtime libraries" +category = "main" optional = false python-versions = ">=3" files = [ @@ -1895,6 +1985,7 @@ nvidia-nvjitlink-cu12 = "*" name = "nvidia-cusparse-cu12" version = "12.1.0.106" description = "CUSPARSE native runtime libraries" +category = "main" optional = false python-versions = ">=3" files = [ @@ -1909,6 +2000,7 @@ nvidia-nvjitlink-cu12 = "*" name = "nvidia-nccl-cu12" version = "2.18.1" description = "NVIDIA Collective Communication Library (NCCL) Runtime" +category = "main" optional = false python-versions = ">=3" files = [ @@ -1919,6 +2011,7 @@ files = [ name = "nvidia-nvjitlink-cu12" version = "12.3.101" description = "Nvidia JIT LTO Library" +category = "main" optional = false python-versions = ">=3" files = [ @@ -1930,6 +2023,7 @@ files = [ name = "nvidia-nvtx-cu12" version = "12.1.105" description = "NVIDIA Tools Extension" +category = "main" optional = false python-versions = ">=3" files = [ @@ -1941,6 +2035,7 @@ files = [ name = "opencv-python" version = "4.9.0.80" description = "Wrapper package for OpenCV python bindings." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1969,6 +2064,7 @@ numpy = [ name = "overrides" version = "7.4.0" description = "A decorator to automatically detect mismatch when overriding a method." +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1980,6 +2076,7 @@ files = [ name = "packaging" version = "23.2" description = "Core utilities for Python packages" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1991,6 +2088,7 @@ files = [ name = "pandas" version = "2.1.4" description = "Powerful data structures for data analysis, time series, and statistics" +category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -2059,6 +2157,7 @@ xml = ["lxml (>=4.8.0)"] name = "pandocfilters" version = "1.5.0" description = "Utilities for writing pandoc filters in python" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -2070,6 +2169,7 @@ files = [ name = "parso" version = "0.8.3" description = "A Python Parser" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2085,6 +2185,7 @@ testing = ["docopt", "pytest (<6.0.0)"] name = "pexpect" version = "4.9.0" description = "Pexpect allows easy control of interactive console applications." +category = "dev" optional = false python-versions = "*" files = [ @@ -2099,6 +2200,7 @@ ptyprocess = ">=0.5" name = "pillow" version = "10.2.0" description = "Python Imaging Library (Fork)" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2184,6 +2286,7 @@ xmp = ["defusedxml"] name = "platformdirs" version = "4.1.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2199,6 +2302,7 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-co name = "prometheus-client" version = "0.19.0" description = "Python client for the Prometheus monitoring system." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2213,6 +2317,7 @@ twisted = ["twisted"] name = "prompt-toolkit" version = "3.0.43" description = "Library for building powerful interactive command lines in Python" +category = "dev" optional = false python-versions = ">=3.7.0" files = [ @@ -2227,6 +2332,7 @@ wcwidth = "*" name = "psutil" version = "5.9.7" description = "Cross-platform lib for process and system monitoring in Python." +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ @@ -2255,6 +2361,7 @@ test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] name = "ptyprocess" version = "0.7.0" description = "Run a subprocess in a pseudo terminal" +category = "dev" optional = false python-versions = "*" files = [ @@ -2266,6 +2373,7 @@ files = [ name = "pure-eval" version = "0.2.2" description = "Safely evaluate AST nodes without side effects" +category = "dev" optional = false python-versions = "*" files = [ @@ -2280,6 +2388,7 @@ tests = ["pytest"] name = "pyarrow" version = "14.0.2" description = "Python library for Apache Arrow" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2328,6 +2437,7 @@ numpy = ">=1.16.6" name = "pyarrow-hotfix" version = "0.6" description = "" +category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -2339,6 +2449,7 @@ files = [ name = "pycparser" version = "2.21" description = "C parser in Python" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -2350,6 +2461,7 @@ files = [ name = "pydantic" version = "2.5.3" description = "Data validation using Python type hints" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2369,6 +2481,7 @@ email = ["email-validator (>=2.0.0)"] name = "pydantic-core" version = "2.14.6" description = "" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2486,6 +2599,7 @@ typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" name = "pydantic-settings" version = "2.1.0" description = "Settings management using Pydantic" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2501,6 +2615,7 @@ python-dotenv = ">=0.21.0" name = "pygments" version = "2.17.2" description = "Pygments is a syntax highlighting package written in Python." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2516,6 +2631,7 @@ windows-terminal = ["colorama (>=0.4.6)"] name = "pymupdf" version = "1.23.8" description = "A high performance Python library for data extraction, analysis, conversion & manipulation of PDF (and other) documents." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2559,6 +2675,7 @@ PyMuPDFb = "1.23.7" name = "pymupdfb" version = "1.23.7" description = "MuPDF shared libraries for PyMuPDF." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2574,6 +2691,7 @@ files = [ name = "pypdfium2" version = "4.25.0" description = "Python bindings to PDFium" +category = "main" optional = false python-versions = ">= 3.6" files = [ @@ -2596,6 +2714,7 @@ files = [ name = "pytesseract" version = "0.3.10" description = "Python-tesseract is a python wrapper for Google's Tesseract-OCR" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2611,6 +2730,7 @@ Pillow = ">=8.0.0" name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" +category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -2625,6 +2745,7 @@ six = ">=1.5" name = "python-dotenv" version = "1.0.0" description = "Read key-value pairs from a .env file and set them as environment variables" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2639,6 +2760,7 @@ cli = ["click (>=5.0)"] name = "python-json-logger" version = "2.0.7" description = "A python library adding a json log formatter" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2650,6 +2772,7 @@ files = [ name = "pytz" version = "2023.3.post1" description = "World timezone definitions, modern and historical" +category = "dev" optional = false python-versions = "*" files = [ @@ -2661,6 +2784,7 @@ files = [ name = "pywin32" version = "306" description = "Python for Window Extensions" +category = "dev" optional = false python-versions = "*" files = [ @@ -2684,6 +2808,7 @@ files = [ name = "pywinpty" version = "2.0.12" description = "Pseudo terminal support for Windows from Python." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2699,6 +2824,7 @@ files = [ name = "pyyaml" version = "6.0.1" description = "YAML parser and emitter for Python" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2707,6 +2833,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -2714,8 +2841,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -2732,6 +2866,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -2739,6 +2874,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -2748,6 +2884,7 @@ files = [ name = "pyzmq" version = "25.1.2" description = "Python bindings for 0MQ" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2853,6 +2990,7 @@ cffi = {version = "*", markers = "implementation_name == \"pypy\""} name = "qtconsole" version = "5.5.1" description = "Jupyter Qt console" +category = "dev" optional = false python-versions = ">= 3.8" files = [ @@ -2878,6 +3016,7 @@ test = ["flaky", "pytest", "pytest-qt"] name = "qtpy" version = "2.4.1" description = "Provides an abstraction layer on top of the various Qt bindings (PyQt5/6 and PySide2/6)." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2895,6 +3034,7 @@ test = ["pytest (>=6,!=7.0.0,!=7.0.1)", "pytest-cov (>=3.0.0)", "pytest-qt"] name = "referencing" version = "0.32.1" description = "JSON Referencing + Python" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2910,6 +3050,7 @@ rpds-py = ">=0.7.0" name = "regex" version = "2023.12.25" description = "Alternative regular expression module, to replace re." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3012,6 +3153,7 @@ files = [ name = "requests" version = "2.31.0" description = "Python HTTP for Humans." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3033,6 +3175,7 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "rfc3339-validator" version = "0.1.4" description = "A pure python RFC3339 validator" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -3047,6 +3190,7 @@ six = "*" name = "rfc3986-validator" version = "0.1.1" description = "Pure python rfc3986 validator" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -3058,6 +3202,7 @@ files = [ name = "rpds-py" version = "0.16.2" description = "Python bindings to Rust's persistent data structures (rpds)" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -3162,10 +3307,38 @@ files = [ {file = "rpds_py-0.16.2.tar.gz", hash = "sha256:781ef8bfc091b19960fc0142a23aedadafa826bc32b433fdfe6fd7f964d7ef44"}, ] +[[package]] +name = "ruff" +version = "0.1.13" +description = "An extremely fast Python linter and code formatter, written in Rust." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruff-0.1.13-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:e3fd36e0d48aeac672aa850045e784673449ce619afc12823ea7868fcc41d8ba"}, + {file = "ruff-0.1.13-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9fb6b3b86450d4ec6a6732f9f60c4406061b6851c4b29f944f8c9d91c3611c7a"}, + {file = "ruff-0.1.13-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b13ba5d7156daaf3fd08b6b993360a96060500aca7e307d95ecbc5bb47a69296"}, + {file = "ruff-0.1.13-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9ebb40442f7b531e136d334ef0851412410061e65d61ca8ce90d894a094feb22"}, + {file = "ruff-0.1.13-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:226b517f42d59a543d6383cfe03cccf0091e3e0ed1b856c6824be03d2a75d3b6"}, + {file = "ruff-0.1.13-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:5f0312ba1061e9b8c724e9a702d3c8621e3c6e6c2c9bd862550ab2951ac75c16"}, + {file = "ruff-0.1.13-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2f59bcf5217c661254bd6bc42d65a6fd1a8b80c48763cb5c2293295babd945dd"}, + {file = "ruff-0.1.13-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e6894b00495e00c27b6ba61af1fc666f17de6140345e5ef27dd6e08fb987259d"}, + {file = "ruff-0.1.13-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a1600942485c6e66119da294c6294856b5c86fd6df591ce293e4a4cc8e72989"}, + {file = "ruff-0.1.13-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:ee3febce7863e231a467f90e681d3d89210b900d49ce88723ce052c8761be8c7"}, + {file = "ruff-0.1.13-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:dcaab50e278ff497ee4d1fe69b29ca0a9a47cd954bb17963628fa417933c6eb1"}, + {file = "ruff-0.1.13-py3-none-musllinux_1_2_i686.whl", hash = "sha256:f57de973de4edef3ad3044d6a50c02ad9fc2dff0d88587f25f1a48e3f72edf5e"}, + {file = "ruff-0.1.13-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:7a36fa90eb12208272a858475ec43ac811ac37e91ef868759770b71bdabe27b6"}, + {file = "ruff-0.1.13-py3-none-win32.whl", hash = "sha256:a623349a505ff768dad6bd57087e2461be8db58305ebd5577bd0e98631f9ae69"}, + {file = "ruff-0.1.13-py3-none-win_amd64.whl", hash = "sha256:f988746e3c3982bea7f824c8fa318ce7f538c4dfefec99cd09c8770bd33e6539"}, + {file = "ruff-0.1.13-py3-none-win_arm64.whl", hash = "sha256:6bbbc3042075871ec17f28864808540a26f0f79a4478c357d3e3d2284e832998"}, + {file = "ruff-0.1.13.tar.gz", hash = "sha256:e261f1baed6291f434ffb1d5c6bd8051d1c2a26958072d38dfbec39b3dda7352"}, +] + [[package]] name = "safetensors" version = "0.4.1" description = "" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3285,6 +3458,7 @@ torch = ["safetensors[numpy]", "torch (>=1.10)"] name = "send2trash" version = "1.8.2" description = "Send file to trash natively under Mac OS X, Windows and Linux" +category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" files = [ @@ -3301,6 +3475,7 @@ win32 = ["pywin32"] name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -3312,6 +3487,7 @@ files = [ name = "snakeviz" version = "2.2.0" description = "A web-based viewer for Python profiler output" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3326,6 +3502,7 @@ tornado = ">=2.0" name = "sniffio" version = "1.3.0" description = "Sniff out which async library your code is running under" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3337,6 +3514,7 @@ files = [ name = "soupsieve" version = "2.5" description = "A modern CSS selector implementation for Beautiful Soup." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -3348,6 +3526,7 @@ files = [ name = "stack-data" version = "0.6.3" description = "Extract data from python stack frames and tracebacks for informative displays" +category = "dev" optional = false python-versions = "*" files = [ @@ -3367,6 +3546,7 @@ tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"] name = "sympy" version = "1.12" description = "Computer algebra system (CAS) in Python" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -3381,6 +3561,7 @@ mpmath = ">=0.19" name = "tabulate" version = "0.9.0" description = "Pretty-print tabular data" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3395,6 +3576,7 @@ widechars = ["wcwidth"] name = "terminado" version = "0.18.0" description = "Tornado websocket backend for the Xterm.js Javascript terminal emulator library." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -3416,6 +3598,7 @@ typing = ["mypy (>=1.6,<2.0)", "traitlets (>=5.11.1)"] name = "tinycss2" version = "1.2.1" description = "A tiny CSS parser" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3434,6 +3617,7 @@ test = ["flake8", "isort", "pytest"] name = "tokenizers" version = "0.15.0" description = "" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3549,6 +3733,7 @@ testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] name = "tomli" version = "2.0.1" description = "A lil' TOML parser" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3560,6 +3745,7 @@ files = [ name = "torch" version = "2.1.2" description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" +category = "main" optional = false python-versions = ">=3.8.0" files = [ @@ -3613,6 +3799,7 @@ opt-einsum = ["opt-einsum (>=3.3)"] name = "tornado" version = "6.4" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." +category = "dev" optional = false python-versions = ">= 3.8" files = [ @@ -3633,6 +3820,7 @@ files = [ name = "tqdm" version = "4.66.1" description = "Fast, Extensible Progress Meter" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3653,6 +3841,7 @@ telegram = ["requests"] name = "traitlets" version = "5.14.1" description = "Traitlets Python configuration system" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -3668,6 +3857,7 @@ test = ["argcomplete (>=3.0.3)", "mypy (>=1.7.0)", "pre-commit", "pytest (>=7.0, name = "transformers" version = "4.36.2" description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" +category = "main" optional = false python-versions = ">=3.8.0" files = [ @@ -3736,6 +3926,7 @@ vision = ["Pillow (>=10.0.1,<=15.0)"] name = "triton" version = "2.1.0" description = "A language and compiler for custom Deep Learning operations" +category = "main" optional = false python-versions = "*" files = [ @@ -3761,6 +3952,7 @@ tutorials = ["matplotlib", "pandas", "tabulate"] name = "types-python-dateutil" version = "2.8.19.20240106" description = "Typing stubs for python-dateutil" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -3772,6 +3964,7 @@ files = [ name = "typing-extensions" version = "4.9.0" description = "Backported and Experimental Type Hints for Python 3.8+" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -3783,6 +3976,7 @@ files = [ name = "tzdata" version = "2023.4" description = "Provider of IANA time zone data" +category = "dev" optional = false python-versions = ">=2" files = [ @@ -3794,6 +3988,7 @@ files = [ name = "uri-template" version = "1.3.0" description = "RFC 6570 URI Template Processor" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3808,6 +4003,7 @@ dev = ["flake8", "flake8-annotations", "flake8-bandit", "flake8-bugbear", "flake name = "urllib3" version = "2.1.0" description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -3824,6 +4020,7 @@ zstd = ["zstandard (>=0.18.0)"] name = "wcwidth" version = "0.2.13" description = "Measures the displayed width of unicode strings in a terminal" +category = "dev" optional = false python-versions = "*" files = [ @@ -3835,6 +4032,7 @@ files = [ name = "webcolors" version = "1.13" description = "A library for working with the color formats defined by HTML and CSS." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3850,6 +4048,7 @@ tests = ["pytest", "pytest-cov"] name = "webencodings" version = "0.5.1" description = "Character encoding aliases for legacy web content" +category = "dev" optional = false python-versions = "*" files = [ @@ -3861,6 +4060,7 @@ files = [ name = "websocket-client" version = "1.7.0" description = "WebSocket client for Python with low level API options" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -3877,6 +4077,7 @@ test = ["websockets"] name = "widgetsnbextension" version = "4.0.9" description = "Jupyter interactive widgets for Jupyter Notebook" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3888,6 +4089,7 @@ files = [ name = "xxhash" version = "3.4.1" description = "Python binding for xxHash" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -4005,6 +4207,7 @@ files = [ name = "yarl" version = "1.9.4" description = "Yet another URL library" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -4108,6 +4311,7 @@ multidict = ">=4.0" name = "zipp" version = "3.17.0" description = "Backport of pathlib-compatible object wrapper for zip files" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -4122,4 +4326,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "0f74bd4a7248e35dcd9e55c45a531b9027b0d3037b1283566ce5f12d44d1278b" +content-hash = "ae72cb24c5a9a41aaa745461625e8fb696d8f7f5521ccddfbc0e0c1e89cf2b0b" diff --git a/pyproject.toml b/pyproject.toml index c8a0948..63fefb6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,10 +33,30 @@ pytesseract = "^0.3.10" pymupdf = "^1.23.8" snakeviz = "^2.2.0" datasets = "^2.16.1" +ruff = "^0.1.13" [tool.poetry.scripts] surya_detect = "detect_text:main" +[tool.ruff] +select = [ + # Pyflakes + "F", + # pycodestyle + "E", + "W", + # isort + "I", + # pyupgrade + "UP", +] +ignore = ["E501", "E203"] +line-length = 120 +target-version = "py39" + +[tool.ruff.isort] +known-first-party = ["surya"] + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" diff --git a/scripts/verify_benchmark_scores.py b/scripts/verify_benchmark_scores.py index 7f17b34..bf4d8f7 100644 --- a/scripts/verify_benchmark_scores.py +++ b/scripts/verify_benchmark_scores.py @@ -1,9 +1,9 @@ -import json import argparse +import json def verify_scores(file_path): - with open(file_path, 'r') as file: + with open(file_path) as file: data = json.load(file) scores = data["metrics"]["surya"] diff --git a/surya/benchmark/bbox.py b/surya/benchmark/bbox.py index b7593e8..3c441c8 100644 --- a/surya/benchmark/bbox.py +++ b/surya/benchmark/bbox.py @@ -1,4 +1,5 @@ import fitz as pymupdf + from surya.postprocessing.util import rescale_bbox @@ -7,7 +8,11 @@ def get_pdf_lines(pdf_path, img_sizes): page_lines = [] for idx, img_size in enumerate(img_sizes): page = doc[idx] - blocks = page.get_text("dict", sort=True, flags=pymupdf.TEXTFLAGS_DICT & ~pymupdf.TEXT_PRESERVE_LIGATURES & ~pymupdf.TEXT_PRESERVE_IMAGES)["blocks"] + blocks = page.get_text( + "dict", + sort=True, + flags=pymupdf.TEXTFLAGS_DICT & ~pymupdf.TEXT_PRESERVE_LIGATURES & ~pymupdf.TEXT_PRESERVE_IMAGES, + )["blocks"] line_boxes = [] for block_idx, block in enumerate(blocks): @@ -19,4 +24,4 @@ def get_pdf_lines(pdf_path, img_sizes): line_boxes = [rescale_bbox(bbox, (pwidth, pheight), img_size) for bbox in line_boxes] page_lines.append(line_boxes) - return page_lines \ No newline at end of file + return page_lines diff --git a/surya/benchmark/metrics.py b/surya/benchmark/metrics.py index 6ac1821..1e43e6c 100644 --- a/surya/benchmark/metrics.py +++ b/surya/benchmark/metrics.py @@ -1,8 +1,9 @@ +from concurrent.futures import ProcessPoolExecutor from functools import partial from itertools import repeat import numpy as np -from concurrent.futures import ProcessPoolExecutor + def intersection_area(box1, box2): x_left = max(box1[0], box2[0]) @@ -55,7 +56,7 @@ def calculate_coverage(box, other_boxes, penalize_double=False): return covered_pixels_count / box_area -def precision_recall(preds, references, threshold=.5, workers=8): +def precision_recall(preds, references, threshold=0.5, workers=8): if len(references) == 0: return { "precision": 1, @@ -100,4 +101,4 @@ def mean_coverage(preds, references): if len(coverages) == 0: return 0 coverage = sum(coverages) / len(coverages) - return {"coverage": coverage} \ No newline at end of file + return {"coverage": coverage} diff --git a/surya/benchmark/tesseract.py b/surya/benchmark/tesseract.py index 8530089..9530f6a 100644 --- a/surya/benchmark/tesseract.py +++ b/surya/benchmark/tesseract.py @@ -1,9 +1,11 @@ +import os +from concurrent.futures import ProcessPoolExecutor + import numpy as np import pytesseract from pytesseract import Output + from surya.settings import settings -import os -from concurrent.futures import ProcessPoolExecutor def tesseract_bboxes(img): @@ -11,10 +13,10 @@ def tesseract_bboxes(img): ocr = pytesseract.image_to_data(arr_img, output_type=Output.DICT) bboxes = [] - n_boxes = len(ocr['level']) + n_boxes = len(ocr["level"]) for i in range(n_boxes): # It is possible to merge by line here with line number, but it gives bad results. - _, x, y, w, h = ocr['text'][i], ocr['left'][i], ocr['top'][i], ocr['width'][i], ocr['height'][i] + _, x, y, w, h = ocr["text"][i], ocr["left"][i], ocr["top"][i], ocr["width"][i], ocr["height"][i] bbox = (x, y, x + w, y + h) bboxes.append(bbox) @@ -33,4 +35,4 @@ def tesseract_parallel(imgs): with ProcessPoolExecutor(max_workers=tess_parallel) as executor: tess_bboxes = executor.map(tesseract_bboxes, imgs) tess_bboxes = list(tess_bboxes) - return tess_bboxes \ No newline at end of file + return tess_bboxes diff --git a/surya/benchmark/util.py b/surya/benchmark/util.py index a32f470..928b840 100644 --- a/surya/benchmark/util.py +++ b/surya/benchmark/util.py @@ -5,7 +5,7 @@ def merge_boxes(box1, box2): def join_lines(bboxes, max_gap=5): to_merge = {} for i, box1 in bboxes: - for z, box2 in bboxes[i + 1:]: + for z, box2 in bboxes[i + 1 :]: j = i + z + 1 if box1 == box2: continue diff --git a/surya/detection.py b/surya/detection.py index 6e92ad2..8e3ef74 100644 --- a/surya/detection.py +++ b/surya/detection.py @@ -1,17 +1,15 @@ -from typing import List - import cv2 -import torch -from torch import nn import numpy as np +import torch from PIL import Image -from surya.postprocessing.heatmap import get_and_clean_boxes -from surya.postprocessing.affinity import get_vertical_lines, get_horizontal_lines + from surya.model.processing import prepare_image, split_image +from surya.postprocessing.affinity import get_horizontal_lines, get_vertical_lines +from surya.postprocessing.heatmap import get_and_clean_boxes from surya.settings import settings -def batch_inference(images: List, model, processor): +def batch_inference(images: list, model, processor): assert all([isinstance(image, Image.Image) for image in images]) images = [image.copy().convert("RGB") for image in images] @@ -29,7 +27,7 @@ def batch_inference(images: List, model, processor): pred_parts = [] for i in range(0, len(image_splits), settings.DETECTOR_BATCH_SIZE): - batch = image_splits[i:i+settings.DETECTOR_BATCH_SIZE] + batch = image_splits[i : i + settings.DETECTOR_BATCH_SIZE] # Batch images in dim 0 batch = torch.stack(batch, dim=0) batch = batch.to(model.dtype) @@ -45,7 +43,7 @@ def batch_inference(images: List, model, processor): heatmap_shape = list(heatmap.shape) correct_shape = [processor.size["height"], processor.size["width"]] - cv2_size = list(reversed(correct_shape)) # opencv uses (width, height) instead of (height, width) + cv2_size = list(reversed(correct_shape)) # opencv uses (width, height) instead of (height, width) if heatmap_shape != correct_shape: heatmap = cv2.resize(heatmap, cv2_size, interpolation=cv2.INTER_LINEAR) @@ -87,18 +85,14 @@ def batch_inference(images: List, model, processor): vertical_lines = get_vertical_lines(affinity_map, affinity_size, orig_sizes[i]) horizontal_lines = get_horizontal_lines(affinity_map, affinity_size, orig_sizes[i]) - results.append({ - "bboxes": bboxes, - "vertical_lines": vertical_lines, - "horizontal_lines": horizontal_lines, - "heatmap": heat_img, - "affinity_map": aff_img, - }) + results.append( + { + "bboxes": bboxes, + "vertical_lines": vertical_lines, + "horizontal_lines": horizontal_lines, + "heatmap": heat_img, + "affinity_map": aff_img, + } + ) return results - - - - - - diff --git a/surya/model/processing.py b/surya/model/processing.py index 07f08d8..42c9dae 100644 --- a/surya/model/processing.py +++ b/surya/model/processing.py @@ -1,10 +1,10 @@ -from typing import List +import math import numpy as np -import math import pypdfium2 -from PIL import Image, ImageOps import torch +from PIL import Image, ImageOps + from surya.settings import settings @@ -34,8 +34,8 @@ def split_image(img, processor): def prepare_image(img, processor): new_size = (processor.size["width"], processor.size["height"]) - img.thumbnail(new_size, Image.Resampling.LANCZOS) # Shrink largest dimension to fit new size - img = img.resize(new_size, Image.Resampling.LANCZOS) # Stretch smaller dimension to fit new size + img.thumbnail(new_size, Image.Resampling.LANCZOS) # Shrink largest dimension to fit new size + img = img.resize(new_size, Image.Resampling.LANCZOS) # Stretch smaller dimension to fit new size img = np.asarray(img, dtype=np.uint8) img = processor(img)["pixel_values"][0] @@ -47,7 +47,7 @@ def open_pdf(pdf_filepath): return pypdfium2.PdfDocument(pdf_filepath) -def get_page_images(doc, indices: List, dpi=96): +def get_page_images(doc, indices: list, dpi=96): renderer = doc.render( pypdfium2.PdfBitmap.to_pil, page_indices=indices, @@ -55,4 +55,4 @@ def get_page_images(doc, indices: List, dpi=96): ) images = list(renderer) images = [image.convert("RGB") for image in images] - return images \ No newline at end of file + return images diff --git a/surya/model/segformer.py b/surya/model/segformer.py index b539e2e..380e9a5 100644 --- a/surya/model/segformer.py +++ b/surya/model/segformer.py @@ -1,19 +1,28 @@ -from typing import Optional, Tuple, Union +from typing import Optional, Union -from transformers import SegformerConfig, SegformerForSemanticSegmentation, SegformerImageProcessor, \ - SegformerDecodeHead, SegformerModel import torch from torch import nn - +from transformers import ( + SegformerConfig, + SegformerDecodeHead, + SegformerForSemanticSegmentation, + SegformerImageProcessor, + SegformerModel, +) from transformers.modeling_outputs import SemanticSegmenterOutput + from surya.settings import settings -def load_model(checkpoint=settings.DETECTOR_MODEL_CHECKPOINT, device=settings.TORCH_DEVICE_MODEL, dtype=settings.MODEL_DTYPE): +def load_model( + checkpoint=settings.DETECTOR_MODEL_CHECKPOINT, device=settings.TORCH_DEVICE_MODEL, dtype=settings.MODEL_DTYPE +): config = SegformerConfig.from_pretrained(checkpoint) model = SegformerForRegressionMask.from_pretrained(checkpoint, torch_dtype=dtype, config=config) if "mps" in device: - print("Warning: MPS may have poor results. This is a bug with MPS, see here - https://github.com/pytorch/pytorch/issues/84936") + print( + "Warning: MPS may have poor results. This is a bug with MPS, see here - https://github.com/pytorch/pytorch/issues/84936" + ) model = model.to(device) model = model.eval() return model @@ -79,7 +88,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, SemanticSegmenterOutput]: + ) -> Union[tuple, SemanticSegmenterOutput]: return_dict = return_dict if return_dict is not None else self.config.use_return_dict output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -103,4 +112,4 @@ def forward( logits=logits, hidden_states=outputs.hidden_states if output_hidden_states else None, attentions=outputs.attentions, - ) \ No newline at end of file + ) diff --git a/surya/postprocessing/affinity.py b/surya/postprocessing/affinity.py index ea3750d..fd81664 100644 --- a/surya/postprocessing/affinity.py +++ b/surya/postprocessing/affinity.py @@ -1,7 +1,6 @@ import cv2 import numpy as np - -from PIL import Image, ImageDraw +from PIL import ImageDraw from surya.postprocessing.util import get_line_angle, rescale_bbox @@ -17,7 +16,6 @@ def get_detected_lines_sobel(image, vertical=True): sobelx = cv2.Sobel(image, cv2.CV_32F, dx, dy, ksize=3) - # Absolute Sobel (to capture both edges) abs_sobelx = np.absolute(sobelx) @@ -119,8 +117,12 @@ def get_vertical_lines(image, new_size, orig_size, divisor=20, x_tolerance=40, y if line["bbox"][0] != line2["bbox"][0]: continue - expanded_line1 = [line["bbox"][0], line["bbox"][1] - y_tolerance, line["bbox"][2], - line["bbox"][3] + y_tolerance] + expanded_line1 = [ + line["bbox"][0], + line["bbox"][1] - y_tolerance, + line["bbox"][2], + line["bbox"][3] + y_tolerance, + ] line1_points = set(range(expanded_line1[1], expanded_line1[3])) line2_points = set(range(line2["bbox"][1], line2["bbox"][3])) @@ -166,8 +168,9 @@ def get_vertical_lines(image, new_size, orig_size, divisor=20, x_tolerance=40, y return vertical_lines + def get_horizontal_lines(affinity_map, processor_size, image_size): horizontal_lines = get_detected_lines(affinity_map, horizontal=True) for line in horizontal_lines: line["bbox"] = rescale_bbox(line["bbox"], processor_size, image_size) - return horizontal_lines \ No newline at end of file + return horizontal_lines diff --git a/surya/postprocessing/heatmap.py b/surya/postprocessing/heatmap.py index e1b1897..4897a1a 100644 --- a/surya/postprocessing/heatmap.py +++ b/surya/postprocessing/heatmap.py @@ -1,6 +1,7 @@ -import numpy as np -import cv2 import math + +import cv2 +import numpy as np from PIL import ImageDraw from surya.postprocessing.util import rescale_bbox @@ -31,7 +32,9 @@ def detect_boxes(linemap, text_threshold, low_text): ret, text_score = cv2.threshold(linemap, low_text, 1, 0) text_score_comb = np.clip(text_score, 0, 1) - label_count, labels, stats, centroids = cv2.connectedComponentsWithStats(text_score_comb.astype(np.uint8), connectivity=4) + label_count, labels, stats, centroids = cv2.connectedComponentsWithStats( + text_score_comb.astype(np.uint8), connectivity=4 + ) det = [] for k in range(1, label_count): @@ -62,11 +65,11 @@ def detect_boxes(linemap, text_threshold, low_text): if ey >= img_h: ey = img_h - kernel = cv2.getStructuringElement(cv2.MORPH_RECT,(1 + niter, 1 + niter)) + kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1 + niter, 1 + niter)) segmap[sy:ey, sx:ex] = cv2.dilate(segmap[sy:ey, sx:ex], kernel) # make box - np_contours = np.roll(np.array(np.where(segmap != 0)),1, axis=0).transpose().reshape(-1,2) + np_contours = np.roll(np.array(np.where(segmap != 0)), 1, axis=0).transpose().reshape(-1, 2) rectangle = cv2.minAreaRect(np_contours) box = cv2.boxPoints(rectangle) @@ -80,7 +83,7 @@ def detect_boxes(linemap, text_threshold, low_text): # make clock-wise order startidx = box.sum(axis=1).argmin() - box = np.roll(box, 4-startidx, 0) + box = np.roll(box, 4 - startidx, 0) box = np.array(box) det.append(box) @@ -88,15 +91,14 @@ def detect_boxes(linemap, text_threshold, low_text): return det, labels -def get_detected_boxes(textmap, text_threshold=settings.DETECTOR_TEXT_THRESHOLD, low_text=settings.DETECTOR_NMS_THRESHOLD): +def get_detected_boxes( + textmap, text_threshold=settings.DETECTOR_TEXT_THRESHOLD, low_text=settings.DETECTOR_NMS_THRESHOLD +): textmap = textmap.copy() textmap = textmap.astype(np.float32) boxes, labels = detect_boxes(textmap, text_threshold, low_text) # From point form to box form - boxes = [ - [box[0][0], box[0][1], box[1][0], box[2][1]] - for box in boxes - ] + boxes = [[box[0][0], box[0][1], box[1][0], box[2][1]] for box in boxes] # Ensure correct box format for box in boxes: @@ -121,4 +123,3 @@ def draw_bboxes_on_image(bboxes, image): draw.rectangle(bbox, outline="red", width=1) return image - diff --git a/surya/postprocessing/util.py b/surya/postprocessing/util.py index 95de0b0..6236bb6 100644 --- a/surya/postprocessing/util.py +++ b/surya/postprocessing/util.py @@ -1,5 +1,5 @@ -import math import copy +import math def get_line_angle(x1, y1, x2, y2): @@ -23,4 +23,4 @@ def rescale_bbox(bbox, processor_size, image_size): new_bbox[1] = int(new_bbox[1] * height_scaler) new_bbox[2] = int(new_bbox[2] * width_scaler) new_bbox[3] = int(new_bbox[3] * height_scaler) - return new_bbox \ No newline at end of file + return new_bbox diff --git a/surya/settings.py b/surya/settings.py index 2d0d1f2..8124c5f 100644 --- a/surya/settings.py +++ b/surya/settings.py @@ -1,10 +1,9 @@ -from typing import Dict, Optional +from typing import Optional +import torch from dotenv import find_dotenv from pydantic import computed_field from pydantic_settings import BaseSettings -import torch -import os class Settings(BaseSettings): @@ -27,9 +26,9 @@ def TORCH_DEVICE_MODEL(self) -> str: DETECTOR_BATCH_SIZE: int = 2 if TORCH_DEVICE_MODEL == "cpu" else 32 DETECTOR_MODEL_CHECKPOINT: str = "vikp/line_detector" BENCH_DATASET_NAME: str = "vikp/doclaynet_bench" - DETECTOR_IMAGE_CHUNK_HEIGHT: int = 1200 # Height at which to slice images vertically - DETECTOR_TEXT_THRESHOLD: float = 0.6 # Threshold for text detection - DETECTOR_NMS_THRESHOLD: float = 0.35 # Threshold for non-maximum suppression + DETECTOR_IMAGE_CHUNK_HEIGHT: int = 1200 # Height at which to slice images vertically + DETECTOR_TEXT_THRESHOLD: float = 0.6 # Threshold for text detection + DETECTOR_NMS_THRESHOLD: float = 0.35 # Threshold for non-maximum suppression # Paths DATA_DIR: str = "data" @@ -45,4 +44,4 @@ class Config: extra = "ignore" -settings = Settings() \ No newline at end of file +settings = Settings()