from pathlib import Path
import os
from dotenv import load_dotenv
from openai import OpenAI

class EmbeddingAPI:
    def __init__(self):
        # 动态锁定绝对路径
        env_path = Path(__file__).resolve().parent / ".env"
        load_dotenv(dotenv_path=env_path)
        
        # 直接在内部实例化好 client，不对外暴露脏细节
        self.client = OpenAI(
            api_key=os.getenv("DASHSCOPE_API_KEY"),
            base_url=os.getenv("DASHSCOPE_BASE_URL")
        )

    def generate_embedding(self, text: list[str], model: str = "text-embedding-v4") -> list[list[float]]:
        # 直接使用内部的 self.client
        completion = self.client.embeddings.create(
            model=model,
            input=text,
            dimensions=1024,
            encoding_format="float"
        )
        return [e.embedding for e in completion.data]

if __name__ == "__main__":
    # 外部调用变得极其丝滑和纯净
    embedding_api = EmbeddingAPI()
    
    text = ['你好', '你好吗', '你好啊']
    embeddings = embedding_api.generate_embedding(text)
    
    for t, e in zip(text, embeddings):
        print(f"文本：{t} | 向量前10维：{e[:10]} | 总维度：{len(e)}\n")

import os
import re
import sys
import json
import argparse
from pathlib import Path
from typing import Dict, Any, Literal, Optional

try:
    from magic_pdf.pipe.UNIPipe import UNIPipe
    from magic_pdf.rw.DiskReaderWriter import DiskReaderWriter
    HAS_MINERU = True
except ImportError:
    HAS_MINERU = False

try:
    import fitz
    HAS_PYMUPDF = True
except ImportError:
    HAS_PYMUPDF = False

OutputFormat = Literal["md", "txt"]

SUPPORTED_FORMATS = {
    ".pdf":  "PDF",
    ".png":  "图片",
    ".jpg":  "图片",
    ".jpeg": "图片",
    ".md":   "Markdown",
    ".txt":  "纯文本",
    ".docx": "Word",
}


class DocumentCleaner:
    @staticmethod def remove_noise(text: str) -> str:
        text = re.sub(r'^\s*—?\s*\d+\s*—?\s*$', '', text, flags=re.MULTILINE)
        text = re.sub(r'(?i)Page\s*\d+\s*(of\s*\d+)?', '', text)
        text = re.sub(r'第\s*\d+\s*页\s*(/\s*共\s*\d+\s*页)?', '', text)
        text = re.sub(r'(?im)^[•·]\s*\d+\s*$', '', text)
        text = re.sub(r'(?im)^(目录|CONTENTS|Table\s+of\s+Contents)\s*$', '', text)
        text = re.sub(r'\n{3,}', '\n\n', text)
        return text.strip()

    @staticmethod def to_plain_text(md: str) -> str:
        text = re.sub(r'!\[.*?\]\(.*?\)', '', md)
        text = re.sub(r'\[([^\]]*?)\]\(.*?\)', r'\1', text)
        text = re.sub(r'#{1,6}\s+', '', text)
        text = re.sub(r'[*_~`]', '', text)
        text = re.sub(r'\|', ' ', text)
        text = re.sub(r'^[\s\-]+$', '', text, flags=re.MULTILINE)
        text = re.sub(r'\n{3,}', '\n\n', text)
        return text.strip()


_SCRIPT_DIR = Path(__file__).parent

class RAGDocumentExtractor:
    def __init__(self, image_dir: str | None = None):
        self._image_dir = Path(image_dir or (_SCRIPT_DIR / "extracted_images"))
        self._image_dir.mkdir(exist_ok=True, parents=True)

    @staticmethod def _resolve(path: str) -> Path:
        p = Path(path)
        if p.is_absolute():
            return p
        if p.exists():
            return p
        alt = _SCRIPT_DIR / p
        if alt.exists():
            return alt
        return p

    def extract(
        self,
        file_path: str,
        fmt: OutputFormat = "md",
        clean: bool = True,
        encoding: str | None = None,
    ) -> Dict[str, Any]:
        path = self._resolve(file_path)
        if not path.exists():
            raise FileNotFoundError(f"文档不存在: {file_path}（已尝试基于脚本目录和当前目录查找）")

        suffix = path.suffix.lower()
        if suffix not in SUPPORTED_FORMATS:
            raise ValueError(f"不支持的格式: {suffix}")

        raw, meta = self._parse(path, suffix, encoding)
        content = DocumentCleaner.remove_noise(raw) if clean else raw

        if fmt == "txt":
            content = DocumentCleaner.to_plain_text(content)

        return {
            "filename": path.name,
            "format": SUPPORTED_FORMATS[suffix],
            "extension": suffix,
            "content": content,
            "length": len(content),
            "output_format": fmt,
            **meta,
        }

    def _parse(self, path: Path, suffix: str, encoding: str | None) -> tuple[str, dict]:
        if suffix in (".pdf", ".png", ".jpg", ".jpeg"):
            return self._parse_visual(path, suffix)
        if suffix in (".md", ".txt"):
            return self._parse_text(path, encoding)
        if suffix == ".docx":
            return self._parse_docx(path)
        return "", {}

    def _parse_visual(self, path: Path, suffix: str) -> tuple[str, dict]:
        err_ctx = suffix

        if HAS_MINERU:
            try:
                writer = DiskReaderWriter(os.fsdecode(self._image_dir))
                pipe = UNIPipe(path.read_bytes(), {"_index_": 0}, writer)
                pipe.pipe_classify()
                pipe.pipe_parse()
                md = pipe.pipe_to_markdown()
                return md, {"parser": "mineru"}
            except Exception as e:
                if suffix != ".pdf":
                    raise RuntimeError(f"MinerU 解析失败: {e}") from e
                err_ctx = f"mineru({e})" if suffix == ".pdf" and HAS_PYMUPDF:
            try:
                doc = fitz.open(path)
                pages = [page.get_text() for page in doc]
                err_ctx = None return "\n\n".join(pages), {"parser": "pymupdf", "pages": len(doc)}
            except Exception as e:
                err_ctx = f"pymupdf({e})" if err_ctx:
            raise RuntimeError(f"解析失败 [{err_ctx}]")
        raise ImportError("请安装解析依赖: pip install magic-pdf 或 pip install pymupdf")

    def _parse_text(self, path: Path, encoding: str | None) -> tuple[str, dict]:
        enc = encoding or self._detect_encoding(path)
        text = path.read_text(encoding=enc)
        return text, {"encoding": enc}

    def _parse_docx(self, path: Path) -> tuple[str, dict]:
        try:
            import docx
        except ImportError:
            raise ImportError("Word 解析需要: pip install python-docx")
        doc = docx.Document(path)
        paras = [p.text for p in doc.paragraphs]
        return "\n\n".join(paras), {"parser": "python-docx"}

    @staticmethod def _detect_encoding(path: Path) -> str:
        raw = path.read_bytes()
        if raw.startswith(b'\xef\xbb\xbf'):
            return "utf-8-sig" if raw.startswith(b'\xff\xfe'):
            return "utf-16-le" if raw.startswith(b'\xfe\xff'):
            return "utf-16-be" try:
            raw.decode("utf-8")
            return "utf-8" except UnicodeDecodeError:
            try:
                raw.decode("gbk")
                return "gbk" except UnicodeDecodeError:
                return "utf-8"


def extract(
    file_path: str,
    fmt: OutputFormat = "md",
    clean: bool = True,
    encoding: str | None = None,
    image_dir: str | None = None,
) -> Dict[str, Any]:
    return RAGDocumentExtractor(image_dir=image_dir).extract(
        file_path, fmt=fmt, clean=clean, encoding=encoding
    )


def get_text(
    file_path: str,
    clean: bool = True,
    encoding: str | None = None,
    image_dir: str | None = None,
) -> str:
    return extract(file_path, fmt="txt", clean=clean, encoding=encoding, image_dir=image_dir)["content"]


def get_markdown(
    file_path: str,
    clean: bool = True,
    encoding: str | None = None,
    image_dir: str | None = None,
) -> str:
    return extract(file_path, fmt="md", clean=clean, encoding=encoding, image_dir=image_dir)["content"]


def convert_file(
    file_path: str,
    output_path: str,
    fmt: OutputFormat = "md",
    clean: bool = True,
    encoding: str | None = None,
    image_dir: str | None = None,
    as_json: bool = False,
) -> str:
    result = extract(file_path, fmt=fmt, clean=clean, encoding=encoding, image_dir=image_dir)
    content = json.dumps(result, ensure_ascii=False, indent=2) if as_json else result["content"]
    Path(output_path).write_text(content, encoding="utf-8")
    return output_path


def main():
    parser = argparse.ArgumentParser(
        description="RAG 文档预处理 — 解析 → 清洗 → 输出",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=(
            "示例:\n" "  %(prog)s demo.pdf                           # 默认 Markdown 输出\n" "  %(prog)s demo.pdf -f txt                     # 纯文本（适合直接切块）\n" "  %(prog)s demo.docx -o out.md                 # 保存到文件\n" "  %(prog)s demo.pdf --no-clean                  # 跳过噪音清洗\n" "  %(prog)s demo.txt --encoding gbk              # 指定文本编码\n" "  %(prog)s demo.pdf --format json               # JSON 格式输出\n" "  %(prog)s --demo                               # 运行内置演示\n"
        ),
    )
    parser.add_argument("file", nargs="?", help="文档路径")
    parser.add_argument("-o", "--output", help="输出文件路径")
    parser.add_argument("-f", "--format", choices=["md", "txt"], default="md",
                        help="输出格式: md（Markdown，默认）| txt（纯文本）")
    parser.add_argument("--encoding", default=None, help="文本编码（默认自动检测）")
    parser.add_argument("--no-clean", action="store_true", help="跳过噪音清洗")
    parser.add_argument("--image-dir", default=None, help="图片提取目录")
    parser.add_argument("--json", action="store_true", dest="as_json", help="JSON 格式输出（含元数据）")
    parser.add_argument("--demo", action="store_true", help="运行内置演示")

    args = parser.parse_args()

    if args.demo:
        test_file = Path(__file__).parent / "docs" / "上海交通大学学生生存手册.pdf" if not test_file.exists():
            print(f"演示文件不存在: {test_file}", file=sys.stderr)
            sys.exit(1)
        args.file = str(test_file)
        if not args.output:
            args.as_json = True if not args.file:
        parser.print_help()
        sys.exit(1)

    result = extract(
        file_path=args.file,
        fmt=args.format,
        clean=not args.no_clean,
        encoding=args.encoding,
        image_dir=args.image_dir,
    )

    output = json.dumps(result, ensure_ascii=False, indent=2) if args.as_json else result["content"]

    if args.output:
        Path(args.output).write_text(output, encoding="utf-8")
        print(f"已写入: {args.output}", file=sys.stderr)
    else:
        print(output)


if __name__ == "__main__":
    txt = get_text("./docs/上海交通大学学生生存手册.pdf")
    print(txt)
    print(len(txt))