feat: group private apps into andreknie-privat
This commit is contained in:
@@ -0,0 +1,217 @@
|
||||
"""Main processing pipeline orchestrating discovery → extraction → enrichment → linking → output.
|
||||
|
||||
Handles:
|
||||
- Progress tracking (current file / total)
|
||||
- Per-file error handling (log + skip + continue)
|
||||
- Summary reporting (processed, success, failed counts)
|
||||
- Dry-run mode
|
||||
"""
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from ingestion.config import IngestionConfig
|
||||
from ingestion.discovery import discover_files
|
||||
from ingestion.enrichment.agent import EnrichmentAgent
|
||||
from ingestion.enrichment.models import EnrichmentResult
|
||||
from ingestion.extraction import get_extractor
|
||||
from ingestion.integrations.git import commit_imported_notes
|
||||
from ingestion.integrations.orgmylife import OrgMyLifeClient
|
||||
from ingestion.linking.linker import generate_wiki_links
|
||||
from ingestion.linking.stubs import create_stub_pages
|
||||
from ingestion.output.naming import generate_filename, resolve_collision
|
||||
from ingestion.output.renderer import render_note
|
||||
from ingestion.output.router import route_to_folder
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProcessResult:
|
||||
"""Result of processing a single file."""
|
||||
|
||||
source_file: Path
|
||||
output_file: Path | None = None
|
||||
success: bool = False
|
||||
error: str | None = None
|
||||
stub_pages: list[Path] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BatchResult:
|
||||
"""Summary of a batch processing run."""
|
||||
|
||||
total: int = 0
|
||||
success: int = 0
|
||||
failed: int = 0
|
||||
results: list[ProcessResult] = field(default_factory=list)
|
||||
|
||||
|
||||
def process_file(
|
||||
file_path: Path,
|
||||
config: IngestionConfig,
|
||||
enrichment_agent: EnrichmentAgent | None = None,
|
||||
dry_run: bool = False,
|
||||
no_commit: bool = False,
|
||||
create_tasks: bool = False,
|
||||
output_dir: Path | None = None,
|
||||
) -> ProcessResult:
|
||||
"""Process a single file through the full ingestion pipeline.
|
||||
|
||||
Steps: extract → enrich → link → render → write → (optional) git commit
|
||||
|
||||
Args:
|
||||
file_path: Path to the source file.
|
||||
config: Ingestion configuration.
|
||||
enrichment_agent: Optional pre-configured enrichment agent.
|
||||
dry_run: If True, don't write any files.
|
||||
no_commit: If True, skip git commit.
|
||||
create_tasks: If True, create OrgMyLife tasks for action items.
|
||||
output_dir: Optional override for output directory.
|
||||
|
||||
Returns:
|
||||
ProcessResult with success status and output path.
|
||||
"""
|
||||
result = ProcessResult(source_file=file_path)
|
||||
|
||||
try:
|
||||
# 1. Extract text
|
||||
extractor = get_extractor(file_path, config)
|
||||
if extractor is None:
|
||||
result.error = f"No extractor available for {file_path.suffix}"
|
||||
return result
|
||||
|
||||
extraction = extractor.extract(file_path)
|
||||
if not extraction.text.strip():
|
||||
result.error = "No text extracted from file"
|
||||
return result
|
||||
|
||||
# 2. Enrich with LLM
|
||||
if enrichment_agent is None:
|
||||
enrichment_agent = EnrichmentAgent(config)
|
||||
|
||||
enrichment = enrichment_agent.enrich(extraction.text)
|
||||
|
||||
# 3. Generate wiki-links
|
||||
linked_text = generate_wiki_links(extraction.text, enrichment.entities)
|
||||
|
||||
# 4. Render output
|
||||
rendered = render_note(enrichment, linked_text, file_path)
|
||||
|
||||
# 5. Determine output location
|
||||
notes_dir = Path(config.notes_dir)
|
||||
target_folder = route_to_folder(enrichment.category, notes_dir, output_dir)
|
||||
|
||||
# 6. Generate filename
|
||||
date_entities = [e for e in enrichment.entities if e.type == "date"]
|
||||
date_str = date_entities[0].value if date_entities else None
|
||||
filename = generate_filename(enrichment.title, date_str)
|
||||
output_path = resolve_collision(target_folder, filename)
|
||||
|
||||
if dry_run:
|
||||
logger.info("[DRY RUN] Would write: %s", output_path)
|
||||
result.output_file = output_path
|
||||
result.success = True
|
||||
return result
|
||||
|
||||
# 7. Write file
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(rendered, encoding="utf-8")
|
||||
result.output_file = output_path
|
||||
logger.info("Written: %s", output_path)
|
||||
|
||||
# 8. Create stub pages
|
||||
stubs = create_stub_pages(enrichment.entities, notes_dir)
|
||||
result.stub_pages = stubs
|
||||
|
||||
# 9. Create OrgMyLife tasks if requested
|
||||
if create_tasks and enrichment.action_items and config.orgmylife_api_key:
|
||||
_create_tasks(enrichment, output_path, config)
|
||||
|
||||
result.success = True
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Failed to process %s: %s", file_path, e)
|
||||
result.error = str(e)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def process_batch(
|
||||
paths: list[Path],
|
||||
config: IngestionConfig,
|
||||
dry_run: bool = False,
|
||||
no_commit: bool = False,
|
||||
create_tasks: bool = False,
|
||||
output_dir: Path | None = None,
|
||||
progress_callback=None,
|
||||
) -> BatchResult:
|
||||
"""Process a batch of files through the pipeline.
|
||||
|
||||
Args:
|
||||
paths: List of file paths to process.
|
||||
config: Ingestion configuration.
|
||||
dry_run: If True, don't write any files.
|
||||
no_commit: If True, skip git commit.
|
||||
create_tasks: If True, create OrgMyLife tasks.
|
||||
output_dir: Optional override for output directory.
|
||||
progress_callback: Optional callable(current, total, filename).
|
||||
|
||||
Returns:
|
||||
BatchResult with summary counts and per-file results.
|
||||
"""
|
||||
batch = BatchResult(total=len(paths))
|
||||
enrichment_agent = EnrichmentAgent(config)
|
||||
written_files: list[Path] = []
|
||||
|
||||
for i, file_path in enumerate(paths, 1):
|
||||
if progress_callback:
|
||||
progress_callback(i, len(paths), file_path.name)
|
||||
|
||||
logger.info("Processing [%d/%d]: %s", i, len(paths), file_path.name)
|
||||
|
||||
result = process_file(
|
||||
file_path=file_path,
|
||||
config=config,
|
||||
enrichment_agent=enrichment_agent,
|
||||
dry_run=dry_run,
|
||||
no_commit=True, # Commit once at end
|
||||
create_tasks=create_tasks,
|
||||
output_dir=output_dir,
|
||||
)
|
||||
|
||||
batch.results.append(result)
|
||||
if result.success:
|
||||
batch.success += 1
|
||||
if result.output_file:
|
||||
written_files.append(result.output_file)
|
||||
written_files.extend(result.stub_pages)
|
||||
else:
|
||||
batch.failed += 1
|
||||
logger.warning("Skipped %s: %s", file_path.name, result.error)
|
||||
|
||||
# Git commit all at once (unless dry-run or no-commit)
|
||||
if not dry_run and not no_commit and config.auto_commit and written_files:
|
||||
commit_imported_notes(written_files, source_type="batch-import")
|
||||
|
||||
return batch
|
||||
|
||||
|
||||
def _create_tasks(
|
||||
enrichment: EnrichmentResult, note_path: Path, config: IngestionConfig
|
||||
) -> None:
|
||||
"""Create OrgMyLife tasks for action items."""
|
||||
try:
|
||||
client = OrgMyLifeClient(
|
||||
base_url=config.orgmylife_api_url,
|
||||
api_key=config.orgmylife_api_key,
|
||||
)
|
||||
for item in enrichment.action_items:
|
||||
client.create_task(
|
||||
title=item.description,
|
||||
source_url=str(note_path),
|
||||
description=f"From note: {enrichment.title}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to create OrgMyLife tasks: %s", e)
|
||||
Reference in New Issue
Block a user