diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index f12fbcb..e13e299 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -6,7 +6,10 @@ // Sets the run context to one level up instead of the .devcontainer folder. "context": "..", // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. - "dockerfile": "../Dockerfile" + "dockerfile": "../Dockerfile", + "args": { + "INSTALL_GIT": "true" + } }, // Features to add to the dev container. More info: https://containers.dev/features. diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..5ace460 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index d3f2789..321f823 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -5,9 +5,9 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: "3.x" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8aa6d18..c4dbdcf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,8 +5,8 @@ jobs: tests: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: | 3.10 @@ -14,7 +14,7 @@ jobs: 3.12 - name: Set up pip cache if: runner.os == 'Linux' - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }} diff --git a/Dockerfile b/Dockerfile index f9c0bef..0072d9e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,10 +2,15 @@ FROM python:3.13-slim-bullseye USER root +ARG INSTALL_GIT=false +RUN if [ "$INSTALL_GIT" = "true" ]; then \ + apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*; \ + fi + # Runtime dependency RUN apt-get update && apt-get install -y --no-install-recommends \ ffmpeg \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* RUN pip install markitdown diff --git a/README.md b/README.md index 6dc096c..d2314c3 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ [![PyPI](https://img.shields.io/pypi/v/markitdown.svg)](https://pypi.org/project/markitdown/) ![PyPI - Downloads](https://img.shields.io/pypi/dd/markitdown) +[![Built by AutoGen Team](https://img.shields.io/badge/Built%20by-AutoGen%20Team-blue)](https://github.com/microsoft/autogen) MarkItDown is a utility for converting various files to Markdown (e.g., for indexing, text analysis, etc). @@ -29,6 +30,12 @@ To install MarkItDown, use pip: `pip install markitdown`. Alternatively, you can markitdown path-to-file.pdf > document.md ``` +Or use `-o` to specify the output file: + +```bash +markitdown path-to-file.pdf -o document.md +``` + You can also pipe content: ```bash @@ -65,7 +72,43 @@ print(result.text_content) docker build -t markitdown:latest . docker run --rm -i markitdown:latest < ~/your-file.pdf > output.md ``` +
+ +Batch Processing Multiple Files + +This example shows how to convert multiple files to markdown format in a single run. The script processes all supported files in a directory and creates corresponding markdown files. + +```python convert.py +from markitdown import MarkItDown +from openai import OpenAI +import os +client = OpenAI(api_key="your-api-key-here") +md = MarkItDown(llm_client=client, llm_model="gpt-4o-2024-11-20") +supported_extensions = ('.pptx', '.docx', '.pdf', '.jpg', '.jpeg', '.png') +files_to_convert = [f for f in os.listdir('.') if f.lower().endswith(supported_extensions)] +for file in files_to_convert: + print(f"\nConverting {file}...") + try: + md_file = os.path.splitext(file)[0] + '.md' + result = md.convert(file) + with open(md_file, 'w') as f: + f.write(result.text_content) + + print(f"Successfully converted {file} to {md_file}") + except Exception as e: + print(f"Error converting {file}: {str(e)}") + +print("\nAll conversions completed!") +``` +2. Place the script in the same directory as your files +3. Install required packages: like openai +4. Run script ```bash python convert.py ``` + +Note that original files will remain unchanged and new markdown files are created with the same base name. + +
+ ## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a @@ -80,6 +123,20 @@ This project has adopted the [Microsoft Open Source Code of Conduct](https://ope For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. +### How to Contribute + +You can help by looking at issues or helping review PRs. Any issue or PR is welcome, but we have also marked some as 'open for contribution' and 'open for reviewing' to help facilitate community contributions. These are ofcourse just suggestions and you are welcome to contribute in any way you like. + + +
+ +| | All | Especially Needs Help from Community | +|-----------------------|------------------------------------------|------------------------------------------------------------------------------------------| +| **Issues** | [All Issues](https://github.com/microsoft/markitdown/issues) | [Issues open for contribution](https://github.com/microsoft/markitdown/issues?q=is%3Aissue+is%3Aopen+label%3A%22open+for+contribution%22) | +| **PRs** | [All PRs](https://github.com/microsoft/markitdown/pulls) | [PRs open for reviewing](https://github.com/microsoft/markitdown/pulls?q=is%3Apr+is%3Aopen+label%3A%22open+for+reviewing%22) | + +
+ ### Running Tests and Checks - Install `hatch` in your environment and run tests: diff --git a/src/markitdown/__main__.py b/src/markitdown/__main__.py index be2a0f2..b6cf963 100644 --- a/src/markitdown/__main__.py +++ b/src/markitdown/__main__.py @@ -1,48 +1,80 @@ # SPDX-FileCopyrightText: 2024-present Adam Fourney # # SPDX-License-Identifier: MIT -import sys import argparse +import sys from textwrap import dedent -from ._markitdown import MarkItDown +from .__about__ import __version__ +from ._markitdown import MarkItDown, DocumentConverterResult def main(): parser = argparse.ArgumentParser( description="Convert various file formats to markdown.", + prog="markitdown", formatter_class=argparse.RawDescriptionHelpFormatter, usage=dedent( """ - SYNTAX: - + SYNTAX: + markitdown If FILENAME is empty, markitdown reads from stdin. - + EXAMPLE: - + markitdown example.pdf - + OR - + cat example.pdf | markitdown - - OR - + + OR + markitdown < example.pdf + + OR to save to a file use + + markitdown example.pdf -o example.md + + OR + + markitdown example.pdf > example.md """ ).strip(), ) + parser.add_argument( + "-v", + "--version", + action="version", + version=f"%(prog)s {__version__}", + help="show the version number and exit", + ) + parser.add_argument("filename", nargs="?") + parser.add_argument( + "-o", + "--output", + help="Output file name. If not provided, output is written to stdout.", + ) args = parser.parse_args() if args.filename is None: markitdown = MarkItDown() result = markitdown.convert_stream(sys.stdin.buffer) - print(result.text_content) + _handle_output(args, result) else: markitdown = MarkItDown() result = markitdown.convert(args.filename) + _handle_output(args, result) + + +def _handle_output(args, result: DocumentConverterResult): + """Handle output to stdout or file""" + if args.output: + with open(args.output, "w", encoding="utf-8") as f: + f.write(result.text_content) + else: print(result.text_content) diff --git a/src/markitdown/_markitdown.py b/src/markitdown/_markitdown.py index 1552e95..674ea77 100644 --- a/src/markitdown/_markitdown.py +++ b/src/markitdown/_markitdown.py @@ -16,6 +16,7 @@ import zipfile from xml.dom import minidom from typing import Any, Dict, List, Optional, Union +from pathlib import Path from urllib.parse import parse_qs, quote, unquote, urlparse, urlunparse from warnings import warn, resetwarnings, catch_warnings @@ -1345,11 +1346,11 @@ def __init__( self.register_page_converter(ZipConverter()) def convert( - self, source: Union[str, requests.Response], **kwargs: Any + self, source: Union[str, requests.Response, Path], **kwargs: Any ) -> DocumentConverterResult: # TODO: deal with kwargs """ Args: - - source: can be a string representing a path or url, or a requests.response object + - source: can be a string representing a path either as string pathlib path object or url, or a requests.response object - extension: specifies the file extension to use when interpreting the file. If None, infer from source (path, uri, content-type, etc.) """ @@ -1366,10 +1367,14 @@ def convert( # Request response elif isinstance(source, requests.Response): return self.convert_response(source, **kwargs) + elif isinstance(source, Path): + return self.convert_local(source, **kwargs) def convert_local( - self, path: str, **kwargs: Any + self, path: Union[str, Path], **kwargs: Any ) -> DocumentConverterResult: # TODO: deal with kwargs + if isinstance(path, Path): + path = str(path) # Prepare a list of extensions to try (in order of priority) ext = kwargs.get("file_extension") extensions = [ext] if ext is not None else [] diff --git a/src/markitdown/py.typed b/src/markitdown/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_markitdown.py b/tests/test_markitdown.py index 14d5f88..1e29479 100644 --- a/tests/test_markitdown.py +++ b/tests/test_markitdown.py @@ -133,6 +133,17 @@ ] +# --- Helper Functions --- +def validate_strings(result, expected_strings, exclude_strings=None): + """Validate presence or absence of specific strings.""" + text_content = result.text_content.replace("\\", "") + for string in expected_strings: + assert string in text_content + if exclude_strings: + for string in exclude_strings: + assert string not in text_content + + @pytest.mark.skipif( skip_remote, reason="do not run tests that query external urls", @@ -177,73 +188,53 @@ def test_markitdown_local(use_mlm = False) -> None: # Test XLSX processing result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.xlsx")) - for test_string in XLSX_TEST_STRINGS: - text_content = result.text_content.replace("\\", "") - assert test_string in text_content + validate_strings(result, XLSX_TEST_STRINGS) # Test DOCX processing result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.docx")) - for test_string in DOCX_TEST_STRINGS: - text_content = result.text_content.replace("\\", "") - assert test_string in text_content + validate_strings(result, DOCX_TEST_STRINGS) # Test DOCX processing, with comments result = markitdown.convert( os.path.join(TEST_FILES_DIR, "test_with_comment.docx"), style_map="comment-reference => ", ) - for test_string in DOCX_COMMENT_TEST_STRINGS: - text_content = result.text_content.replace("\\", "") - assert test_string in text_content + validate_strings(result, DOCX_COMMENT_TEST_STRINGS) # Test DOCX processing, with comments and setting style_map on init markitdown_with_style_map = MarkItDown(style_map="comment-reference => ") result = markitdown_with_style_map.convert( os.path.join(TEST_FILES_DIR, "test_with_comment.docx") ) - for test_string in DOCX_COMMENT_TEST_STRINGS: - text_content = result.text_content.replace("\\", "") - assert test_string in text_content + validate_strings(result, DOCX_COMMENT_TEST_STRINGS) # Test PPTX processing result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.pptx")) - for test_string in PPTX_TEST_STRINGS: - text_content = result.text_content.replace("\\", "") - assert test_string in text_content + validate_strings(result, PPTX_TEST_STRINGS) # Test HTML processing result = markitdown.convert( os.path.join(TEST_FILES_DIR, "test_blog.html"), url=BLOG_TEST_URL ) - for test_string in BLOG_TEST_STRINGS: - text_content = result.text_content.replace("\\", "") - assert test_string in text_content + validate_strings(result, BLOG_TEST_STRINGS) # Test ZIP file processing result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test_files.zip")) - for test_string in DOCX_TEST_STRINGS: - text_content = result.text_content.replace("\\", "") - assert test_string in text_content + validate_strings(result, XLSX_TEST_STRINGS) # Test Wikipedia processing result = markitdown.convert( os.path.join(TEST_FILES_DIR, "test_wikipedia.html"), url=WIKIPEDIA_TEST_URL ) text_content = result.text_content.replace("\\", "") - for test_string in WIKIPEDIA_TEST_EXCLUDES: - assert test_string not in text_content - for test_string in WIKIPEDIA_TEST_STRINGS: - assert test_string in text_content + validate_strings(result, WIKIPEDIA_TEST_STRINGS, WIKIPEDIA_TEST_EXCLUDES) # Test Bing processing result = markitdown.convert( os.path.join(TEST_FILES_DIR, "test_serp.html"), url=SERP_TEST_URL ) text_content = result.text_content.replace("\\", "") - for test_string in SERP_TEST_EXCLUDES: - assert test_string not in text_content - for test_string in SERP_TEST_STRINGS: - assert test_string in text_content + validate_strings(result, SERP_TEST_STRINGS, SERP_TEST_EXCLUDES) # Test RSS processing result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test_rss.xml")) @@ -253,9 +244,7 @@ def test_markitdown_local(use_mlm = False) -> None: ## Test non-UTF-8 encoding result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test_mskanji.csv")) - text_content = result.text_content.replace("\\", "") - for test_string in CSV_CP932_TEST_STRINGS: - assert test_string in text_content + validate_strings(result, CSV_CP932_TEST_STRINGS) @pytest.mark.skipif(