from pathlib import Path
import json
import re
from pdfminer.high_level import extract_text
from haystack import Document
from haystack.components.preprocessors import DocumentSplitter, DocumentCleaner


def clean_text(text: str) -> str:
    """Remove unwanted characters and normalize whitespace."""
    text = text.replace("\f", " ")
    text = re.sub(r"[\r\n]+", " ", text)
    text = re.sub(r"\s+", " ", text)
    return text.strip()


def knowledgebase_file_to_docs(file_path: str) -> list[Document]:
    """
    Given a KnowledgeBase file path (currently PDF), return final docs ready for embedding.
    """
    # 1. Extract raw text
    text = extract_text(file_path)

    # 2. Wrap into a single Document
    raw_doc = Document(content=text.strip(), meta={"filename": Path(file_path).name})

    # 3. Split
    splitter = DocumentSplitter(
        split_by="word",
        split_length=2000,
        split_overlap=200,
    )
    split_docs = splitter.run([raw_doc])["documents"]

    # 4. Clean
    cleaner = DocumentCleaner(
        remove_empty_lines=True,
        remove_extra_whitespaces=True,
        remove_substrings=["..."],
    )
    cleaned_docs = cleaner.run(split_docs)["documents"]

    # 5. Final cleanup
    final_docs = []
    for doc in cleaned_docs:
        doc.content = clean_text(doc.content)
        if len(doc.content) > 10:
            final_docs.append(doc)

    return final_docs
