Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature: Support XLSM, XLSB & Replace excel engine with faster calamine #259

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Feature: Support XLSM, XLSB & Replace excel engine with faster calamine #259

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b953121
combine xlsx and xls to excel, replace openpxyl/xlrd with calamine
yeungadrian Jan 3, 2025
bbacf89
remove pandas, use calamine + tabulate
yeungadrian Jan 4, 2025
1c0362f
rm todo
yeungadrian Jan 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,8 @@ dependencies = [
"markdownify",
"numpy",
"python-pptx",
"pandas",
"openpyxl",
"xlrd",
"python-calamine",
"tabulate",
"pdfminer.six",
"puremagic",
"pydub",
Expand Down
46 changes: 12 additions & 34 deletions src/markitdown/_markitdown.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,11 @@
import mammoth
import markdownify
import olefile
import pandas as pd
import pdfminer
import pdfminer.high_level
import pptx
from python_calamine import load_workbook
from tabulate import tabulate

# File-format detection
import puremagic
Expand Down Expand Up @@ -715,46 +716,24 @@ def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:
return result


class XlsxConverter(HtmlConverter):
class ExcelConverter(HtmlConverter):
"""
Converts XLSX files to Markdown, with each sheet presented as a separate Markdown table.
Converts Excel (XLSX, XLS, XLSM or XLSB) files to Markdown, with each sheet presented as a separate Markdown table.
"""

def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:
# Bail if not a XLSX
# Bail if not a XLSX, XLS, XLSM or XLSB
extension = kwargs.get("file_extension", "")
if extension.lower() != ".xlsx":
if extension.lower() not in [".xlsx", ".xls", ".xlsb", ".xlsm"]:
return None

sheets = pd.read_excel(local_path, sheet_name=None, engine="openpyxl")
workbook = load_workbook(local_path)
md_content = ""
for s in sheets:
for s in workbook.sheet_names:
sheet = workbook.get_sheet_by_name(s)
tabular_data = sheet.to_python(skip_empty_area=False)
md_content += f"## {s}\n"
html_content = sheets[s].to_html(index=False)
md_content += self._convert(html_content).text_content.strip() + "\n\n"

return DocumentConverterResult(
title=None,
text_content=md_content.strip(),
)


class XlsConverter(HtmlConverter):
"""
Converts XLS files to Markdown, with each sheet presented as a separate Markdown table.
"""

def convert(self, local_path, **kwargs) -> Union[None, DocumentConverterResult]:
# Bail if not a XLS
extension = kwargs.get("file_extension", "")
if extension.lower() != ".xls":
return None

sheets = pd.read_excel(local_path, sheet_name=None, engine="xlrd")
md_content = ""
for s in sheets:
md_content += f"## {s}\n"
html_content = sheets[s].to_html(index=False)
html_content = tabulate(tabular_data, tablefmt="html")
md_content += self._convert(html_content).text_content.strip() + "\n\n"

return DocumentConverterResult(
Expand Down Expand Up @@ -1376,8 +1355,7 @@ def __init__(
self.register_page_converter(YouTubeConverter())
self.register_page_converter(BingSerpConverter())
self.register_page_converter(DocxConverter())
self.register_page_converter(XlsxConverter())
self.register_page_converter(XlsConverter())
self.register_page_converter(ExcelConverter())
self.register_page_converter(PptxConverter())
self.register_page_converter(WavConverter())
self.register_page_converter(Mp3Converter())
Expand Down
Binary file added tests/test_files/test.xlsb
View file
Open in desktop
Binary file not shown.
Binary file added tests/test_files/test.xlsm
View file
Open in desktop
Binary file not shown.
18 changes: 9 additions & 9 deletions tests/test_markitdown.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,12 +54,6 @@
"affc7dad-52dc-4b98-9b5d-51e65d8a8ad0",
]

XLS_TEST_STRINGS = [
"## 09060124-b5e7-4717-9d07-3c046eb",
"6ff4173b-42a5-4784-9b19-f49caff4d93d",
"affc7dad-52dc-4b98-9b5d-51e65d8a8ad0",
]

DOCX_TEST_STRINGS = [
"314b0a30-5b04-470b-b9f7-eed2c2bec74a",
"49e168b7-d2ae-407f-a055-2167576f39a1",
Expand Down Expand Up @@ -191,11 +185,17 @@ def test_markitdown_local() -> None:
result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.xlsx"))
validate_strings(result, XLSX_TEST_STRINGS)

# Test XLSB processing
result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.xlsb"))
validate_strings(result, XLSX_TEST_STRINGS)

# Test XLSM processing
result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.xlsm"))
validate_strings(result, XLSX_TEST_STRINGS)

# Test XLS processing
result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.xls"))
for test_string in XLS_TEST_STRINGS:
text_content = result.text_content.replace("\\", "")
assert test_string in text_content
validate_strings(result, XLSX_TEST_STRINGS)

# Test DOCX processing
result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.docx"))
Expand Down