Orchestrator/privat/NoteGraph/ingestion/pipeline.py

"""Main processing pipeline orchestrating discovery → extraction → enrichment → linking → output.

Handles:
  - Progress tracking (current file / total)
  - Per-file error handling (log + skip + continue)
  - Summary reporting (processed, success, failed counts)
  - Dry-run mode
"""

import logging
from dataclasses import dataclass, field
from pathlib import Path

from ingestion.config import IngestionConfig
from ingestion.discovery import discover_files
from ingestion.enrichment.agent import EnrichmentAgent
from ingestion.enrichment.models import EnrichmentResult
from ingestion.extraction import get_extractor
from ingestion.integrations.git import commit_imported_notes
from ingestion.integrations.orgmylife import OrgMyLifeClient
from ingestion.linking.linker import generate_wiki_links
from ingestion.linking.stubs import create_stub_pages
from ingestion.output.naming import generate_filename, resolve_collision
from ingestion.output.renderer import render_note
from ingestion.output.router import route_to_folder

logger = logging.getLogger(__name__)


@dataclass
class ProcessResult:
    """Result of processing a single file."""

    source_file: Path
    output_file: Path | None = None
    success: bool = False
    error: str | None = None
    stub_pages: list[Path] = field(default_factory=list)


@dataclass
class BatchResult:
    """Summary of a batch processing run."""

    total: int = 0
    success: int = 0
    failed: int = 0
    results: list[ProcessResult] = field(default_factory=list)


def process_file(
    file_path: Path,
    config: IngestionConfig,
    enrichment_agent: EnrichmentAgent | None = None,
    dry_run: bool = False,
    no_commit: bool = False,
    create_tasks: bool = False,
    output_dir: Path | None = None,
) -> ProcessResult:
    """Process a single file through the full ingestion pipeline.

    Steps: extract → enrich → link → render → write → (optional) git commit

    Args:
        file_path: Path to the source file.
        config: Ingestion configuration.
        enrichment_agent: Optional pre-configured enrichment agent.
        dry_run: If True, don't write any files.
        no_commit: If True, skip git commit.
        create_tasks: If True, create OrgMyLife tasks for action items.
        output_dir: Optional override for output directory.

    Returns:
        ProcessResult with success status and output path.
    """
    result = ProcessResult(source_file=file_path)

    try:
        # 1. Extract text
        extractor = get_extractor(file_path, config)
        if extractor is None:
            result.error = f"No extractor available for {file_path.suffix}"
            return result

        extraction = extractor.extract(file_path)
        if not extraction.text.strip():
            result.error = "No text extracted from file"
            return result

        # 2. Enrich with LLM
        if enrichment_agent is None:
            enrichment_agent = EnrichmentAgent(config)

        enrichment = enrichment_agent.enrich(extraction.text)

        # 3. Generate wiki-links
        linked_text = generate_wiki_links(extraction.text, enrichment.entities)

        # 4. Render output
        rendered = render_note(enrichment, linked_text, file_path)

        # 5. Determine output location
        notes_dir = Path(config.notes_dir)
        target_folder = route_to_folder(enrichment.category, notes_dir, output_dir)

        # 6. Generate filename
        date_entities = [e for e in enrichment.entities if e.type == "date"]
        date_str = date_entities[0].value if date_entities else None
        filename = generate_filename(enrichment.title, date_str)
        output_path = resolve_collision(target_folder, filename)

        if dry_run:
            logger.info("[DRY RUN] Would write: %s", output_path)
            result.output_file = output_path
            result.success = True
            return result

        # 7. Write file
        output_path.parent.mkdir(parents=True, exist_ok=True)
        output_path.write_text(rendered, encoding="utf-8")
        result.output_file = output_path
        logger.info("Written: %s", output_path)

        # 8. Create stub pages
        stubs = create_stub_pages(enrichment.entities, notes_dir)
        result.stub_pages = stubs

        # 9. Create OrgMyLife tasks if requested
        if create_tasks and enrichment.action_items and config.orgmylife_api_key:
            _create_tasks(enrichment, output_path, config)

        result.success = True

    except Exception as e:
        logger.error("Failed to process %s: %s", file_path, e)
        result.error = str(e)

    return result


def process_batch(
    paths: list[Path],
    config: IngestionConfig,
    dry_run: bool = False,
    no_commit: bool = False,
    create_tasks: bool = False,
    output_dir: Path | None = None,
    progress_callback=None,
) -> BatchResult:
    """Process a batch of files through the pipeline.

    Args:
        paths: List of file paths to process.
        config: Ingestion configuration.
        dry_run: If True, don't write any files.
        no_commit: If True, skip git commit.
        create_tasks: If True, create OrgMyLife tasks.
        output_dir: Optional override for output directory.
        progress_callback: Optional callable(current, total, filename).

    Returns:
        BatchResult with summary counts and per-file results.
    """
    batch = BatchResult(total=len(paths))
    enrichment_agent = EnrichmentAgent(config)
    written_files: list[Path] = []

    for i, file_path in enumerate(paths, 1):
        if progress_callback:
            progress_callback(i, len(paths), file_path.name)

        logger.info("Processing [%d/%d]: %s", i, len(paths), file_path.name)

        result = process_file(
            file_path=file_path,
            config=config,
            enrichment_agent=enrichment_agent,
            dry_run=dry_run,
            no_commit=True,  # Commit once at end
            create_tasks=create_tasks,
            output_dir=output_dir,
        )

        batch.results.append(result)
        if result.success:
            batch.success += 1
            if result.output_file:
                written_files.append(result.output_file)
            written_files.extend(result.stub_pages)
        else:
            batch.failed += 1
            logger.warning("Skipped %s: %s", file_path.name, result.error)

    # Git commit all at once (unless dry-run or no-commit)
    if not dry_run and not no_commit and config.auto_commit and written_files:
        commit_imported_notes(written_files, source_type="batch-import")

    return batch


def _create_tasks(
    enrichment: EnrichmentResult, note_path: Path, config: IngestionConfig
) -> None:
    """Create OrgMyLife tasks for action items."""
    try:
        client = OrgMyLifeClient(
            base_url=config.orgmylife_api_url,
            api_key=config.orgmylife_api_key,
        )
        for item in enrichment.action_items:
            client.create_task(
                title=item.description,
                source_url=str(note_path),
                description=f"From note: {enrichment.title}",
            )
    except Exception as e:
        logger.warning("Failed to create OrgMyLife tasks: %s", e)