"""Main processing pipeline orchestrating discovery → extraction → enrichment → linking → output. Handles: - Progress tracking (current file / total) - Per-file error handling (log + skip + continue) - Summary reporting (processed, success, failed counts) - Dry-run mode """ import logging from dataclasses import dataclass, field from pathlib import Path from ingestion.config import IngestionConfig from ingestion.discovery import discover_files from ingestion.enrichment.agent import EnrichmentAgent from ingestion.enrichment.models import EnrichmentResult from ingestion.extraction import get_extractor from ingestion.integrations.git import commit_imported_notes from ingestion.integrations.orgmylife import OrgMyLifeClient from ingestion.linking.linker import generate_wiki_links from ingestion.linking.stubs import create_stub_pages from ingestion.output.naming import generate_filename, resolve_collision from ingestion.output.renderer import render_note from ingestion.output.router import route_to_folder logger = logging.getLogger(__name__) @dataclass class ProcessResult: """Result of processing a single file.""" source_file: Path output_file: Path | None = None success: bool = False error: str | None = None stub_pages: list[Path] = field(default_factory=list) @dataclass class BatchResult: """Summary of a batch processing run.""" total: int = 0 success: int = 0 failed: int = 0 results: list[ProcessResult] = field(default_factory=list) def process_file( file_path: Path, config: IngestionConfig, enrichment_agent: EnrichmentAgent | None = None, dry_run: bool = False, no_commit: bool = False, create_tasks: bool = False, output_dir: Path | None = None, ) -> ProcessResult: """Process a single file through the full ingestion pipeline. Steps: extract → enrich → link → render → write → (optional) git commit Args: file_path: Path to the source file. config: Ingestion configuration. enrichment_agent: Optional pre-configured enrichment agent. dry_run: If True, don't write any files. no_commit: If True, skip git commit. create_tasks: If True, create OrgMyLife tasks for action items. output_dir: Optional override for output directory. Returns: ProcessResult with success status and output path. """ result = ProcessResult(source_file=file_path) try: # 1. Extract text extractor = get_extractor(file_path, config) if extractor is None: result.error = f"No extractor available for {file_path.suffix}" return result extraction = extractor.extract(file_path) if not extraction.text.strip(): result.error = "No text extracted from file" return result # 2. Enrich with LLM if enrichment_agent is None: enrichment_agent = EnrichmentAgent(config) enrichment = enrichment_agent.enrich(extraction.text) # 3. Generate wiki-links linked_text = generate_wiki_links(extraction.text, enrichment.entities) # 4. Render output rendered = render_note(enrichment, linked_text, file_path) # 5. Determine output location notes_dir = Path(config.notes_dir) target_folder = route_to_folder(enrichment.category, notes_dir, output_dir) # 6. Generate filename date_entities = [e for e in enrichment.entities if e.type == "date"] date_str = date_entities[0].value if date_entities else None filename = generate_filename(enrichment.title, date_str) output_path = resolve_collision(target_folder, filename) if dry_run: logger.info("[DRY RUN] Would write: %s", output_path) result.output_file = output_path result.success = True return result # 7. Write file output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_text(rendered, encoding="utf-8") result.output_file = output_path logger.info("Written: %s", output_path) # 8. Create stub pages stubs = create_stub_pages(enrichment.entities, notes_dir) result.stub_pages = stubs # 9. Create OrgMyLife tasks if requested if create_tasks and enrichment.action_items and config.orgmylife_api_key: _create_tasks(enrichment, output_path, config) result.success = True except Exception as e: logger.error("Failed to process %s: %s", file_path, e) result.error = str(e) return result def process_batch( paths: list[Path], config: IngestionConfig, dry_run: bool = False, no_commit: bool = False, create_tasks: bool = False, output_dir: Path | None = None, progress_callback=None, ) -> BatchResult: """Process a batch of files through the pipeline. Args: paths: List of file paths to process. config: Ingestion configuration. dry_run: If True, don't write any files. no_commit: If True, skip git commit. create_tasks: If True, create OrgMyLife tasks. output_dir: Optional override for output directory. progress_callback: Optional callable(current, total, filename). Returns: BatchResult with summary counts and per-file results. """ batch = BatchResult(total=len(paths)) enrichment_agent = EnrichmentAgent(config) written_files: list[Path] = [] for i, file_path in enumerate(paths, 1): if progress_callback: progress_callback(i, len(paths), file_path.name) logger.info("Processing [%d/%d]: %s", i, len(paths), file_path.name) result = process_file( file_path=file_path, config=config, enrichment_agent=enrichment_agent, dry_run=dry_run, no_commit=True, # Commit once at end create_tasks=create_tasks, output_dir=output_dir, ) batch.results.append(result) if result.success: batch.success += 1 if result.output_file: written_files.append(result.output_file) written_files.extend(result.stub_pages) else: batch.failed += 1 logger.warning("Skipped %s: %s", file_path.name, result.error) # Git commit all at once (unless dry-run or no-commit) if not dry_run and not no_commit and config.auto_commit and written_files: commit_imported_notes(written_files, source_type="batch-import") return batch def _create_tasks( enrichment: EnrichmentResult, note_path: Path, config: IngestionConfig ) -> None: """Create OrgMyLife tasks for action items.""" try: client = OrgMyLifeClient( base_url=config.orgmylife_api_url, api_key=config.orgmylife_api_key, ) for item in enrichment.action_items: client.create_task( title=item.description, source_url=str(note_path), description=f"From note: {enrichment.title}", ) except Exception as e: logger.warning("Failed to create OrgMyLife tasks: %s", e)