From f398f3d4434ef1664250e1032b7f2733944dd6d5 Mon Sep 17 00:00:00 2001 From: "Petr@AP Consulting" Date: Tue, 17 Dec 2024 10:26:09 +0100 Subject: [PATCH 01/23] Update README.md I added description and script for batch of files processing --- README.md | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/README.md b/README.md index 7079dbf..01ceb71 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,51 @@ You can also use the project as Docker Image: docker build -t markitdown:latest . docker run --rm -i markitdown:latest < ~/your-file.pdf > output.md ``` +Batch Processing Multiple Files +This extension allows you to convert multiple files to markdown format in a single run. The script processes all supported files in a directory and creates corresponding markdown files. + +Features + +- Converts multiple files in one operation +- Supports various file formats (.pptx, .docx, .pdf, .jpg, .jpeg, .png etc. you can change it) +- Maintains original filenames (changes extension to .md) +- Includes GPT-4o-latest image descriptions when available +- Continues processing if individual files fail + +Usage +1. Create a Python script (e.g., convert.py): +```python +from markitdown import MarkItDown +from openai import OpenAI +import os +client = OpenAI(api_key="your-api-key-here") +md = MarkItDown(mlm_client=client, mlm_model="gpt-4o-2024-11-20") +supported_extensions = ('.pptx', '.docx', '.pdf', '.jpg', '.jpeg', '.png') +files_to_convert = [f for f in os.listdir('.') if f.lower().endswith(supported_extensions)] +for file in files_to_convert: + print(f"\nConverting {file}...") + try: + md_file = os.path.splitext(file)[0] + '.md' + result = md.convert(file) + with open(md_file, 'w') as f: + f.write(result.text_content) + + print(f"Successfully converted {file} to {md_file}") + except Exception as e: + print(f"Error converting {file}: {str(e)}") + +print("\nAll conversions completed!") +``` +2. Place the script in the same directory as your files +3. Install required packages: like openai +4. Run script ```bash python3 convert.py ``` + +- The script processes all supported files in the current directory +- Original files remain unchanged +- New markdown files are created with the same base name +- Progress and any errors are displayed during conversion + ## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a From 224f1df0fc33e83d49825e3d8b947d945787ad7d Mon Sep 17 00:00:00 2001 From: "Petr@AP Consulting" Date: Wed, 18 Dec 2024 09:28:18 +0100 Subject: [PATCH 02/23] Update README.md I collapsed section about batch processing as was suggested --- README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 01ceb71..669caa2 100644 --- a/README.md +++ b/README.md @@ -78,11 +78,13 @@ You can also use the project as Docker Image: docker build -t markitdown:latest . docker run --rm -i markitdown:latest < ~/your-file.pdf > output.md ``` -Batch Processing Multiple Files +
+ +Batch Processing Multiple Files This extension allows you to convert multiple files to markdown format in a single run. The script processes all supported files in a directory and creates corresponding markdown files. -Features +### Features - Converts multiple files in one operation - Supports various file formats (.pptx, .docx, .pdf, .jpg, .jpeg, .png etc. you can change it) @@ -90,7 +92,7 @@ Features - Includes GPT-4o-latest image descriptions when available - Continues processing if individual files fail -Usage +### Usage 1. Create a Python script (e.g., convert.py): ```python from markitdown import MarkItDown @@ -122,6 +124,8 @@ print("\nAll conversions completed!") - Original files remain unchanged - New markdown files are created with the same base name - Progress and any errors are displayed during conversion + +
## Contributing From 233ba679b88389fb53aded8a15f7b967f93f5af3 Mon Sep 17 00:00:00 2001 From: "Petr@AP Consulting" <173082609+PetrAPConsulting@users.noreply.github.com> Date: Wed, 18 Dec 2024 21:05:04 +0100 Subject: [PATCH 03/23] Update README.md Co-authored-by: gagb --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ebeca7f..f160d86 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ docker run --rm -i markitdown:latest < ~/your-file.pdf > output.md Batch Processing Multiple Files -This extension allows you to convert multiple files to markdown format in a single run. The script processes all supported files in a directory and creates corresponding markdown files. +This example shows how to convert multiple files to markdown format in a single run. The script processes all supported files in a directory and creates corresponding markdown files. ### Features From bb929629f3c573adef8fcdf15ae7112f052299d5 Mon Sep 17 00:00:00 2001 From: "Petr@AP Consulting" <173082609+PetrAPConsulting@users.noreply.github.com> Date: Wed, 18 Dec 2024 21:05:36 +0100 Subject: [PATCH 04/23] Update README.md Co-authored-by: gagb --- README.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/README.md b/README.md index f160d86..627243d 100644 --- a/README.md +++ b/README.md @@ -66,13 +66,6 @@ docker run --rm -i markitdown:latest < ~/your-file.pdf > output.md This example shows how to convert multiple files to markdown format in a single run. The script processes all supported files in a directory and creates corresponding markdown files. -### Features - -- Converts multiple files in one operation -- Supports various file formats (.pptx, .docx, .pdf, .jpg, .jpeg, .png etc. you can change it) -- Maintains original filenames (changes extension to .md) -- Includes GPT-4o-latest image descriptions when available -- Continues processing if individual files fail ### Usage 1. Create a Python script (e.g., convert.py): From 088007338d1567299a6654cf95fb4616413f7131 Mon Sep 17 00:00:00 2001 From: "Petr@AP Consulting" <173082609+PetrAPConsulting@users.noreply.github.com> Date: Wed, 18 Dec 2024 21:07:55 +0100 Subject: [PATCH 05/23] Update README.md Co-authored-by: gagb --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 627243d..62de2e6 100644 --- a/README.md +++ b/README.md @@ -67,8 +67,6 @@ docker run --rm -i markitdown:latest < ~/your-file.pdf > output.md This example shows how to convert multiple files to markdown format in a single run. The script processes all supported files in a directory and creates corresponding markdown files. -### Usage -1. Create a Python script (e.g., convert.py): ```python from markitdown import MarkItDown from openai import OpenAI From f4471d96e2b61de672e3e1f4bf95222191844274 Mon Sep 17 00:00:00 2001 From: "Petr@AP Consulting" <173082609+PetrAPConsulting@users.noreply.github.com> Date: Wed, 18 Dec 2024 21:08:10 +0100 Subject: [PATCH 06/23] Update README.md Co-authored-by: gagb --- README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index 62de2e6..70a1b5c 100644 --- a/README.md +++ b/README.md @@ -93,10 +93,7 @@ print("\nAll conversions completed!") 3. Install required packages: like openai 4. Run script ```bash python3 convert.py ``` -- The script processes all supported files in the current directory -- Original files remain unchanged -- New markdown files are created with the same base name -- Progress and any errors are displayed during conversion +Note that original files will remain unchanged and new markdown files are created with the same base name. From f6e75c46d4f08a073f5fc07dd0bc122138f52436 Mon Sep 17 00:00:00 2001 From: "Petr@AP Consulting" <173082609+PetrAPConsulting@users.noreply.github.com> Date: Wed, 18 Dec 2024 21:17:47 +0100 Subject: [PATCH 07/23] Update README.md I changed command for running script from Mac version (python3) to Windows version (python) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 70a1b5c..b82e5fc 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ print("\nAll conversions completed!") ``` 2. Place the script in the same directory as your files 3. Install required packages: like openai -4. Run script ```bash python3 convert.py ``` +4. Run script ```bash python convert.py ``` Note that original files will remain unchanged and new markdown files are created with the same base name. From b28f380a4768bfb88f9bd209cadc97ae73b7a5b8 Mon Sep 17 00:00:00 2001 From: "Petr@AP Consulting" <173082609+PetrAPConsulting@users.noreply.github.com> Date: Thu, 19 Dec 2024 09:23:15 +0100 Subject: [PATCH 08/23] Update README.md Co-authored-by: gagb --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b82e5fc..d0201d4 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ docker run --rm -i markitdown:latest < ~/your-file.pdf > output.md This example shows how to convert multiple files to markdown format in a single run. The script processes all supported files in a directory and creates corresponding markdown files. -```python +```python convert.py from markitdown import MarkItDown from openai import OpenAI import os From 5c776bda70619e1a59ec8178fae1e3bdb12ff17b Mon Sep 17 00:00:00 2001 From: gagb Date: Thu, 19 Dec 2024 10:30:53 -0800 Subject: [PATCH 09/23] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a91768a..6ffe8ff 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ from markitdown import MarkItDown from openai import OpenAI import os client = OpenAI(api_key="your-api-key-here") -md = MarkItDown(mlm_client=client, mlm_model="gpt-4o-2024-11-20") +md = MarkItDown(llm_client=client, llm_model="gpt-4o-2024-11-20") supported_extensions = ('.pptx', '.docx', '.pdf', '.jpg', '.jpeg', '.png') files_to_convert = [f for f in os.listdir('.') if f.lower().endswith(supported_extensions)] for file in files_to_convert: From 613825d5b3e07353083be66196159bfeb2778ad8 Mon Sep 17 00:00:00 2001 From: Sugato Ray Date: Fri, 20 Dec 2024 02:12:24 +0000 Subject: [PATCH 10/23] [feat]: add support for type-hinting for PEP-561 --- src/markitdown/py.typed | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/markitdown/py.typed diff --git a/src/markitdown/py.typed b/src/markitdown/py.typed new file mode 100644 index 0000000..e69de29 From 8921fe7304b5ac3117e6445935d8a8eff9104ebb Mon Sep 17 00:00:00 2001 From: Sugato Ray Date: Fri, 20 Dec 2024 02:18:14 +0000 Subject: [PATCH 11/23] ignore .vscode folder - avoid local developer vscode editor settings --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 82f9275..e6c8f2e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +.vscode + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] From 52d73080c7ae3d7662d04f5cc520a09de1908878 Mon Sep 17 00:00:00 2001 From: lumin <71011125+l-lumin@users.noreply.github.com> Date: Sat, 21 Dec 2024 04:42:32 +0900 Subject: [PATCH 12/23] refactor(tests): add helper function for tests (#87) * refactor(tests): simplify string validation in tests Introduce a helper function `validate_strings` to streamline the validation of expected and excluded strings in test cases. Replace repetitive string assertions in the `test_markitdown_local` function with calls to this new helper, improving code readability and maintainability. * run pre-commit --------- Co-authored-by: lumin <71011125+l-melon@users.noreply.github.com> Co-authored-by: gagb --- tests/test_markitdown.py | 53 ++++++++++++++++------------------------ 1 file changed, 21 insertions(+), 32 deletions(-) diff --git a/tests/test_markitdown.py b/tests/test_markitdown.py index 316e670..4a981bd 100644 --- a/tests/test_markitdown.py +++ b/tests/test_markitdown.py @@ -131,6 +131,17 @@ ] +# --- Helper Functions --- +def validate_strings(result, expected_strings, exclude_strings=None): + """Validate presence or absence of specific strings.""" + text_content = result.text_content.replace("\\", "") + for string in expected_strings: + assert string in text_content + if exclude_strings: + for string in exclude_strings: + assert string not in text_content + + @pytest.mark.skipif( skip_remote, reason="do not run tests that query external urls", @@ -163,73 +174,53 @@ def test_markitdown_local() -> None: # Test XLSX processing result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.xlsx")) - for test_string in XLSX_TEST_STRINGS: - text_content = result.text_content.replace("\\", "") - assert test_string in text_content + validate_strings(result, XLSX_TEST_STRINGS) # Test DOCX processing result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.docx")) - for test_string in DOCX_TEST_STRINGS: - text_content = result.text_content.replace("\\", "") - assert test_string in text_content + validate_strings(result, DOCX_TEST_STRINGS) # Test DOCX processing, with comments result = markitdown.convert( os.path.join(TEST_FILES_DIR, "test_with_comment.docx"), style_map="comment-reference => ", ) - for test_string in DOCX_COMMENT_TEST_STRINGS: - text_content = result.text_content.replace("\\", "") - assert test_string in text_content + validate_strings(result, DOCX_COMMENT_TEST_STRINGS) # Test DOCX processing, with comments and setting style_map on init markitdown_with_style_map = MarkItDown(style_map="comment-reference => ") result = markitdown_with_style_map.convert( os.path.join(TEST_FILES_DIR, "test_with_comment.docx") ) - for test_string in DOCX_COMMENT_TEST_STRINGS: - text_content = result.text_content.replace("\\", "") - assert test_string in text_content + validate_strings(result, DOCX_COMMENT_TEST_STRINGS) # Test PPTX processing result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.pptx")) - for test_string in PPTX_TEST_STRINGS: - text_content = result.text_content.replace("\\", "") - assert test_string in text_content + validate_strings(result, PPTX_TEST_STRINGS) # Test HTML processing result = markitdown.convert( os.path.join(TEST_FILES_DIR, "test_blog.html"), url=BLOG_TEST_URL ) - for test_string in BLOG_TEST_STRINGS: - text_content = result.text_content.replace("\\", "") - assert test_string in text_content + validate_strings(result, BLOG_TEST_STRINGS) # Test ZIP file processing result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test_files.zip")) - for test_string in DOCX_TEST_STRINGS: - text_content = result.text_content.replace("\\", "") - assert test_string in text_content + validate_strings(result, XLSX_TEST_STRINGS) # Test Wikipedia processing result = markitdown.convert( os.path.join(TEST_FILES_DIR, "test_wikipedia.html"), url=WIKIPEDIA_TEST_URL ) text_content = result.text_content.replace("\\", "") - for test_string in WIKIPEDIA_TEST_EXCLUDES: - assert test_string not in text_content - for test_string in WIKIPEDIA_TEST_STRINGS: - assert test_string in text_content + validate_strings(result, WIKIPEDIA_TEST_STRINGS, WIKIPEDIA_TEST_EXCLUDES) # Test Bing processing result = markitdown.convert( os.path.join(TEST_FILES_DIR, "test_serp.html"), url=SERP_TEST_URL ) text_content = result.text_content.replace("\\", "") - for test_string in SERP_TEST_EXCLUDES: - assert test_string not in text_content - for test_string in SERP_TEST_STRINGS: - assert test_string in text_content + validate_strings(result, SERP_TEST_STRINGS, SERP_TEST_EXCLUDES) # Test RSS processing result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test_rss.xml")) @@ -239,9 +230,7 @@ def test_markitdown_local() -> None: ## Test non-UTF-8 encoding result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test_mskanji.csv")) - text_content = result.text_content.replace("\\", "") - for test_string in CSV_CP932_TEST_STRINGS: - assert test_string in text_content + validate_strings(result, CSV_CP932_TEST_STRINGS) @pytest.mark.skipif( From 7e6c36c5d4af0eb51b3b404b837391a8c6bb549e Mon Sep 17 00:00:00 2001 From: gagb Date: Fri, 20 Dec 2024 14:08:58 -0800 Subject: [PATCH 13/23] docs: add contribution guidelines to README (#176) --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 7040311..be26560 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,20 @@ This project has adopted the [Microsoft Open Source Code of Conduct](https://ope For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. +### How to Contribute + +You can help by looking at issues or helping review PRs. Any issue or PR is welcome, but we have also marked some as 'open for contribution' and 'open for reviewing' to help faciliate community contributions. These are ofcourse just suggestions and you are welcome to contribute in any way you like. + + +
+ +| | All | Especially Needs Help from Community | +|-----------------------|------------------------------------------|------------------------------------------------------------------------------------------| +| **Issues** | [All Issues](https://github.com/microsoft/markitdown/issues) | [Issues open for contribution](https://github.com/microsoft/markitdown/issues?q=is%3Aissue+is%3Aopen+label%3A%22open+for+contribution%22) | +| **PRs** | [All PRs](https://github.com/microsoft/markitdown/pulls) | [PRs open for reviewing](https://github.com/microsoft/markitdown/pulls?q=is%3Apr+is%3Aopen+label%3A%22open+for+reviewing%22) | + +
+ ### Running Tests and Checks - Install `hatch` in your environment and run tests: From 5276616ba1476f663c42c00a0136c9b6257219b6 Mon Sep 17 00:00:00 2001 From: SigireddyBalasai Date: Sat, 21 Dec 2024 03:42:48 +0530 Subject: [PATCH 14/23] Added support to use Pathlib (#93) * Add support for Path objects in MarkItDown conversion methods * Remove unnecessary blank line in test_markitdown_exiftool function * Remove unnecessary blank line in test_markitdown_exiftool function * remove pathlib path in test file --------- Co-authored-by: afourney Co-authored-by: gagb --- src/markitdown/_markitdown.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/markitdown/_markitdown.py b/src/markitdown/_markitdown.py index 040a586..789c1e5 100644 --- a/src/markitdown/_markitdown.py +++ b/src/markitdown/_markitdown.py @@ -15,6 +15,7 @@ import zipfile from xml.dom import minidom from typing import Any, Dict, List, Optional, Union +from pathlib import Path from urllib.parse import parse_qs, quote, unquote, urlparse, urlunparse from warnings import warn, resetwarnings, catch_warnings @@ -1286,11 +1287,11 @@ def __init__( self.register_page_converter(ZipConverter()) def convert( - self, source: Union[str, requests.Response], **kwargs: Any + self, source: Union[str, requests.Response, Path], **kwargs: Any ) -> DocumentConverterResult: # TODO: deal with kwargs """ Args: - - source: can be a string representing a path or url, or a requests.response object + - source: can be a string representing a path either as string pathlib path object or url, or a requests.response object - extension: specifies the file extension to use when interpreting the file. If None, infer from source (path, uri, content-type, etc.) """ @@ -1307,10 +1308,14 @@ def convert( # Request response elif isinstance(source, requests.Response): return self.convert_response(source, **kwargs) + elif isinstance(source, Path): + return self.convert_local(source, **kwargs) def convert_local( - self, path: str, **kwargs: Any + self, path: Union[str, Path], **kwargs: Any ) -> DocumentConverterResult: # TODO: deal with kwargs + if isinstance(path, Path): + path = str(path) # Prepare a list of extensions to try (in order of priority) ext = kwargs.get("file_extension") extensions = [ext] if ext is not None else [] From c1a0d3deaf3d3794a1f670ad68dd00f0b9cc1ddf Mon Sep 17 00:00:00 2001 From: lumin <71011125+l-lumin@users.noreply.github.com> Date: Sat, 21 Dec 2024 07:28:55 +0900 Subject: [PATCH 15/23] chore: configure Dependabot for GitHub Actions updates (#112) Sets up Dependabot to automatically check for updates to GitHub Actions on a weekly basis, ensuring that the project remains up-to-date with the latest dependencies and security fixes. Co-authored-by: gagb --- .github/dependabot.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..5ace460 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" From 377a7eaa7d6ed3eb0bb125db0e87c15d43699dcc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Dec 2024 14:36:48 -0800 Subject: [PATCH 16/23] Bump actions/checkout from 2 to 4 (#177) Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pre-commit.yml | 2 +- .github/workflows/tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index d3f2789..6128f9b 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -5,7 +5,7 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v2 with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8aa6d18..fe35e4f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,7 +5,7 @@ jobs: tests: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: | From 1123392306bf14c40610bef2773d7a7b01ffbc05 Mon Sep 17 00:00:00 2001 From: Soulter <37870767+Soulter@users.noreply.github.com> Date: Sat, 21 Dec 2024 06:43:00 +0800 Subject: [PATCH 17/23] fix: support -o param to avoid encoding issues (#116) * perf: cli supports -o param * doc: update README --------- Co-authored-by: gagb --- README.md | 6 ++++++ src/markitdown/__main__.py | 26 ++++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index be26560..0fae1e6 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,12 @@ To install MarkItDown, use pip: `pip install markitdown`. Alternatively, you can markitdown path-to-file.pdf > document.md ``` +Or use `-o` to specify the output file: + +```bash +markitdown path-to-file.pdf -o document.md +``` + You can also pipe content: ```bash diff --git a/src/markitdown/__main__.py b/src/markitdown/__main__.py index be2a0f2..3193ae7 100644 --- a/src/markitdown/__main__.py +++ b/src/markitdown/__main__.py @@ -4,7 +4,7 @@ import sys import argparse from textwrap import dedent -from ._markitdown import MarkItDown +from ._markitdown import MarkItDown, DocumentConverterResult def main(): @@ -29,20 +29,42 @@ def main(): OR markitdown < example.pdf + + OR to save to a file use + + markitdown example.pdf -o example.md + + OR + + markitdown example.pdf > example.md """ ).strip(), ) parser.add_argument("filename", nargs="?") + parser.add_argument( + "-o", + "--output", + help="Output file name. If not provided, output is written to stdout.", + ) args = parser.parse_args() if args.filename is None: markitdown = MarkItDown() result = markitdown.convert_stream(sys.stdin.buffer) - print(result.text_content) + _handle_output(args, result) else: markitdown = MarkItDown() result = markitdown.convert(args.filename) + _handle_output(args, result) + + +def _handle_output(args, result: DocumentConverterResult): + """Handle output to stdout or file""" + if args.output: + with open(args.output, "w", encoding="utf-8") as f: + f.write(result.text_content) + else: print(result.text_content) From 857a2d160d0c44310a1804c03a2ef8856f7f673d Mon Sep 17 00:00:00 2001 From: gagb Date: Fri, 20 Dec 2024 14:49:20 -0800 Subject: [PATCH 18/23] Update README.md (#180) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0fae1e6..6b51415 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ [![PyPI](https://img.shields.io/pypi/v/markitdown.svg)](https://pypi.org/project/markitdown/) ![PyPI - Downloads](https://img.shields.io/pypi/dd/markitdown) +[![Built by AutoGen Team](https://img.shields.io/badge/Built%20by-AutoGen%20Team-blue)](https://github.com/microsoft/autogen) MarkItDown is a utility for converting various files to Markdown (e.g., for indexing, text analysis, etc). From 9b6946777253c15739111e05a906d304c67fd267 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Dec 2024 16:17:43 -0800 Subject: [PATCH 19/23] Bump actions/cache from 3 to 4 (#178) Bumps [actions/cache](https://github.com/actions/cache) from 3 to 4. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: gagb Co-authored-by: afourney --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fe35e4f..678995a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,7 +14,7 @@ jobs: 3.12 - name: Set up pip cache if: runner.os == 'Linux' - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }} From 73161982fff4bf6755e706496e4d090f4cafe77c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Dec 2024 16:20:22 -0800 Subject: [PATCH 20/23] Bump actions/setup-python from 2 to 5 (#179) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: afourney --- .github/workflows/pre-commit.yml | 2 +- .github/workflows/tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 6128f9b..321f823 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -7,7 +7,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: "3.x" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 678995a..c4dbdcf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: | 3.10 From cfd2319c14a1ed0be6f5a89e145f18878803fa7e Mon Sep 17 00:00:00 2001 From: lumin <71011125+l-lumin@users.noreply.github.com> Date: Sat, 21 Dec 2024 09:24:45 +0900 Subject: [PATCH 21/23] feat: add version option to markitdown CLI (#172) Add a `--version` option to the markitdown command-line interface that displays the current version number. --- src/markitdown/__main__.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/src/markitdown/__main__.py b/src/markitdown/__main__.py index 3193ae7..b6cf963 100644 --- a/src/markitdown/__main__.py +++ b/src/markitdown/__main__.py @@ -1,33 +1,35 @@ # SPDX-FileCopyrightText: 2024-present Adam Fourney # # SPDX-License-Identifier: MIT -import sys import argparse +import sys from textwrap import dedent +from .__about__ import __version__ from ._markitdown import MarkItDown, DocumentConverterResult def main(): parser = argparse.ArgumentParser( description="Convert various file formats to markdown.", + prog="markitdown", formatter_class=argparse.RawDescriptionHelpFormatter, usage=dedent( """ - SYNTAX: - + SYNTAX: + markitdown If FILENAME is empty, markitdown reads from stdin. - + EXAMPLE: - + markitdown example.pdf - + OR - + cat example.pdf | markitdown - - OR - + + OR + markitdown < example.pdf OR to save to a file use @@ -41,6 +43,14 @@ def main(): ).strip(), ) + parser.add_argument( + "-v", + "--version", + action="version", + version=f"%(prog)s {__version__}", + help="show the version number and exit", + ) + parser.add_argument("filename", nargs="?") parser.add_argument( "-o", From f94d09990ef6ccb9d7d94800dbec4b5068504055 Mon Sep 17 00:00:00 2001 From: numekudi <51479021+numekudi@users.noreply.github.com> Date: Sat, 21 Dec 2024 11:09:17 +0900 Subject: [PATCH 22/23] feat: enable Git support in devcontainer (#136) Co-authored-by: gagb --- .devcontainer/devcontainer.json | 5 ++++- Dockerfile | 7 ++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index f12fbcb..e13e299 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -6,7 +6,10 @@ // Sets the run context to one level up instead of the .devcontainer folder. "context": "..", // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. - "dockerfile": "../Dockerfile" + "dockerfile": "../Dockerfile", + "args": { + "INSTALL_GIT": "true" + } }, // Features to add to the dev container. More info: https://containers.dev/features. diff --git a/Dockerfile b/Dockerfile index f9c0bef..0072d9e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,10 +2,15 @@ FROM python:3.13-slim-bullseye USER root +ARG INSTALL_GIT=false +RUN if [ "$INSTALL_GIT" = "true" ]; then \ + apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*; \ + fi + # Runtime dependency RUN apt-get update && apt-get install -y --no-install-recommends \ ffmpeg \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* RUN pip install markitdown From 125e206047dd4635a71c036d52c56d1655636c50 Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Sat, 21 Dec 2024 18:51:30 +0900 Subject: [PATCH 23/23] docs: update README.md (#182) faciliate -> facilitate --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6b51415..d2314c3 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additio ### How to Contribute -You can help by looking at issues or helping review PRs. Any issue or PR is welcome, but we have also marked some as 'open for contribution' and 'open for reviewing' to help faciliate community contributions. These are ofcourse just suggestions and you are welcome to contribute in any way you like. +You can help by looking at issues or helping review PRs. Any issue or PR is welcome, but we have also marked some as 'open for contribution' and 'open for reviewing' to help facilitate community contributions. These are ofcourse just suggestions and you are welcome to contribute in any way you like.