feat: group private apps into andreknie-privat
This commit is contained in:
@@ -0,0 +1,33 @@
|
||||
FROM python:3.11-slim
|
||||
|
||||
# Install system dependencies for OCR
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
tesseract-ocr \
|
||||
tesseract-ocr-deu \
|
||||
git \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy pyproject.toml first for dependency caching
|
||||
COPY pyproject.toml .
|
||||
|
||||
# Install dependencies only (not the package itself yet)
|
||||
RUN pip install --no-cache-dir \
|
||||
click watchdog pytesseract pdfplumber python-docx litellm httpx \
|
||||
pydantic pydantic-settings fastapi uvicorn python-multipart pyyaml python-slugify
|
||||
|
||||
# Copy source code into /app/ingestion/ so imports work
|
||||
COPY . /app/ingestion/
|
||||
|
||||
# Set PYTHONPATH so "import ingestion" resolves to /app/ingestion/
|
||||
ENV PYTHONPATH=/app
|
||||
|
||||
# Create inbox directory
|
||||
RUN mkdir -p /app/inbox
|
||||
|
||||
# Expose upload server port
|
||||
EXPOSE 8001
|
||||
|
||||
# Default: run the upload web server
|
||||
CMD ["uvicorn", "ingestion.web.upload:app", "--host", "0.0.0.0", "--port", "8001"]
|
||||
@@ -0,0 +1,3 @@
|
||||
"""NoteGraph document ingestion and AI-powered auto-referencing pipeline."""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
@@ -0,0 +1,5 @@
|
||||
"""Entry point for `python -m ingestion`."""
|
||||
|
||||
from ingestion.cli import cli
|
||||
|
||||
cli()
|
||||
@@ -0,0 +1,169 @@
|
||||
"""Click-based CLI for the NoteGraph ingestion pipeline.
|
||||
|
||||
Subcommands:
|
||||
import - Import files or directories into NoteGraph
|
||||
watch - Start drop-folder watcher for continuous ingestion
|
||||
sync - One-shot Nextcloud inbox sync
|
||||
upload - Start web upload server
|
||||
"""
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
|
||||
from ingestion.config import IngestionConfig
|
||||
|
||||
|
||||
def _setup_logging(verbose: bool) -> None:
|
||||
"""Configure logging based on verbosity."""
|
||||
level = logging.DEBUG if verbose else logging.INFO
|
||||
logging.basicConfig(
|
||||
level=level,
|
||||
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
||||
datefmt="%H:%M:%S",
|
||||
)
|
||||
|
||||
|
||||
@click.group()
|
||||
@click.version_option(version="0.1.0", prog_name="notegraph-ingest")
|
||||
def cli():
|
||||
"""NoteGraph Ingestion — import documents into your knowledge base."""
|
||||
pass
|
||||
|
||||
|
||||
@cli.command("import")
|
||||
@click.argument("path", type=click.Path(exists=True))
|
||||
@click.option("--output-dir", "-o", type=click.Path(), default=None, help="Override output directory")
|
||||
@click.option("--create-tasks", is_flag=True, help="Create OrgMyLife tasks for action items")
|
||||
@click.option("--provider", type=str, default=None, help="Override LLM provider (openai, anthropic, etc.)")
|
||||
@click.option("--dry-run", is_flag=True, help="Show what would be created without writing files")
|
||||
@click.option("--no-commit", is_flag=True, help="Skip git auto-commit")
|
||||
@click.option("--verbose", "-v", is_flag=True, help="Enable debug logging")
|
||||
def import_cmd(path, output_dir, create_tasks, provider, dry_run, no_commit, verbose):
|
||||
"""Import files or directories into NoteGraph.
|
||||
|
||||
PATH can be a single file or a directory to scan recursively.
|
||||
"""
|
||||
_setup_logging(verbose)
|
||||
|
||||
config = IngestionConfig()
|
||||
if provider:
|
||||
config.llm_provider = provider
|
||||
|
||||
config.validate_api_key()
|
||||
|
||||
from ingestion.discovery import discover_files
|
||||
from ingestion.pipeline import process_batch
|
||||
|
||||
source_path = Path(path)
|
||||
files = discover_files(source_path)
|
||||
|
||||
if not files:
|
||||
click.echo("No supported files found.")
|
||||
sys.exit(0)
|
||||
|
||||
click.echo(f"Found {len(files)} file(s) to process.")
|
||||
|
||||
if dry_run:
|
||||
click.echo("[DRY RUN] No files will be written.\n")
|
||||
|
||||
def progress(current, total, filename):
|
||||
click.echo(f" [{current}/{total}] {filename}")
|
||||
|
||||
result = process_batch(
|
||||
paths=files,
|
||||
config=config,
|
||||
dry_run=dry_run,
|
||||
no_commit=no_commit,
|
||||
create_tasks=create_tasks,
|
||||
output_dir=Path(output_dir) if output_dir else None,
|
||||
progress_callback=progress,
|
||||
)
|
||||
|
||||
# Summary
|
||||
click.echo("")
|
||||
click.echo(f"Done! Processed {result.total} file(s):")
|
||||
click.echo(f" ✓ Success: {result.success}")
|
||||
if result.failed:
|
||||
click.echo(f" ✗ Failed: {result.failed}")
|
||||
|
||||
|
||||
@cli.command("watch")
|
||||
@click.option("--verbose", "-v", is_flag=True, help="Enable debug logging")
|
||||
def watch_cmd(verbose):
|
||||
"""Start drop-folder watcher for continuous ingestion."""
|
||||
_setup_logging(verbose)
|
||||
|
||||
from ingestion.watcher import start_watcher
|
||||
|
||||
config = IngestionConfig()
|
||||
config.validate_api_key()
|
||||
|
||||
click.echo(f"Watching inbox: {config.inbox_dir}")
|
||||
click.echo("Press Ctrl+C to stop.\n")
|
||||
|
||||
try:
|
||||
start_watcher(config)
|
||||
except KeyboardInterrupt:
|
||||
click.echo("\nWatcher stopped.")
|
||||
|
||||
|
||||
@cli.command("sync")
|
||||
@click.option("--verbose", "-v", is_flag=True, help="Enable debug logging")
|
||||
def sync_cmd(verbose):
|
||||
"""One-shot Nextcloud inbox sync."""
|
||||
_setup_logging(verbose)
|
||||
|
||||
from ingestion.integrations.nextcloud import NextcloudInboxSync
|
||||
|
||||
config = IngestionConfig()
|
||||
|
||||
if not config.nextcloud_inbox_url:
|
||||
click.echo("ERROR: NEXTCLOUD_INBOX_URL not configured.")
|
||||
sys.exit(1)
|
||||
|
||||
config.validate_api_key()
|
||||
|
||||
sync = NextcloudInboxSync(config)
|
||||
downloaded = sync.poll()
|
||||
|
||||
if not downloaded:
|
||||
click.echo("No new files in Nextcloud inbox.")
|
||||
return
|
||||
|
||||
click.echo(f"Downloaded {len(downloaded)} file(s) from Nextcloud.")
|
||||
|
||||
# Process downloaded files
|
||||
from ingestion.pipeline import process_batch
|
||||
|
||||
result = process_batch(
|
||||
paths=downloaded,
|
||||
config=config,
|
||||
)
|
||||
|
||||
# Mark processed files on Nextcloud
|
||||
for r in result.results:
|
||||
if r.success:
|
||||
sync.mark_processed(r.source_file.name)
|
||||
|
||||
click.echo(f"Processed: {result.success} success, {result.failed} failed.")
|
||||
|
||||
|
||||
@cli.command("upload")
|
||||
@click.option("--host", default="0.0.0.0", help="Bind host")
|
||||
@click.option("--port", default=8001, type=int, help="Bind port")
|
||||
@click.option("--verbose", "-v", is_flag=True, help="Enable debug logging")
|
||||
def upload_cmd(host, port, verbose):
|
||||
"""Start web upload server for mobile file ingestion."""
|
||||
_setup_logging(verbose)
|
||||
|
||||
import uvicorn
|
||||
|
||||
click.echo(f"Starting upload server on {host}:{port}")
|
||||
uvicorn.run("ingestion.web.upload:app", host=host, port=port, log_level="info")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Configuration for the NoteGraph ingestion service."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic_settings import BaseSettings
|
||||
|
||||
|
||||
class IngestionConfig(BaseSettings):
|
||||
"""All configuration loaded from environment variables / .env file."""
|
||||
|
||||
# LLM Provider
|
||||
llm_provider: str = "openai"
|
||||
llm_model: str = "gpt-4o"
|
||||
openai_api_key: str = ""
|
||||
anthropic_api_key: str = ""
|
||||
google_api_key: str = ""
|
||||
mistral_api_key: str = ""
|
||||
ollama_base_url: str = "http://localhost:11434"
|
||||
|
||||
# OrgMyLife Integration
|
||||
orgmylife_api_url: str = "https://api.andreknie.de"
|
||||
orgmylife_api_key: str = ""
|
||||
|
||||
# Nextcloud Inbox
|
||||
nextcloud_inbox_url: str = ""
|
||||
nextcloud_inbox_user: str = ""
|
||||
nextcloud_inbox_password: str = ""
|
||||
|
||||
# Paths
|
||||
notes_dir: str = "./notes"
|
||||
inbox_dir: str = "./ingestion/inbox"
|
||||
|
||||
# OCR
|
||||
ocr_enabled: bool = True
|
||||
ocr_language: str = "deu+eng"
|
||||
|
||||
# Behavior
|
||||
confidence_threshold: float = 0.7
|
||||
auto_commit: bool = True
|
||||
|
||||
# Web Upload Auth (same as SilverBullet)
|
||||
sb_user: str = "admin:changeme"
|
||||
|
||||
model_config = {"env_file": ".env", "env_prefix": "", "extra": "ignore"}
|
||||
|
||||
@property
|
||||
def llm_model_string(self) -> str:
|
||||
"""LiteLLM model string (e.g., 'openai/gpt-4o', 'ollama/llama3')."""
|
||||
if self.llm_provider == "ollama":
|
||||
return f"ollama/{self.llm_model}"
|
||||
return f"{self.llm_provider}/{self.llm_model}"
|
||||
|
||||
def validate_api_key(self) -> None:
|
||||
"""Exit with a clear error if the required API key is missing."""
|
||||
key_map = {
|
||||
"openai": ("openai_api_key", "OPENAI_API_KEY"),
|
||||
"anthropic": ("anthropic_api_key", "ANTHROPIC_API_KEY"),
|
||||
"google": ("google_api_key", "GOOGLE_API_KEY"),
|
||||
"mistral": ("mistral_api_key", "MISTRAL_API_KEY"),
|
||||
}
|
||||
if self.llm_provider in key_map:
|
||||
attr, env_name = key_map[self.llm_provider]
|
||||
if not getattr(self, attr):
|
||||
raise SystemExit(
|
||||
f"ERROR: {env_name} is required for provider '{self.llm_provider}' but not set."
|
||||
)
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Recursive file discovery with extension filtering.
|
||||
|
||||
Discovers files for ingestion by walking directories and filtering
|
||||
to supported document types.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SUPPORTED_EXTENSIONS = {".pdf", ".jpg", ".jpeg", ".png", ".md", ".txt", ".docx"}
|
||||
|
||||
|
||||
def discover_files(path: Path) -> list[Path]:
|
||||
"""Discover supported files from a path (file or directory).
|
||||
|
||||
Args:
|
||||
path: A single file path or a directory to scan recursively.
|
||||
|
||||
Returns:
|
||||
Sorted list of Path objects with supported extensions.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If the path does not exist.
|
||||
"""
|
||||
path = Path(path)
|
||||
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Path does not exist: {path}")
|
||||
|
||||
if path.is_file():
|
||||
if path.suffix.lower() in SUPPORTED_EXTENSIONS:
|
||||
return [path]
|
||||
logger.warning("Unsupported file type: %s", path)
|
||||
return []
|
||||
|
||||
# Recursive directory traversal
|
||||
files: list[Path] = []
|
||||
for item in sorted(path.rglob("*")):
|
||||
if item.is_file() and item.suffix.lower() in SUPPORTED_EXTENSIONS:
|
||||
files.append(item)
|
||||
|
||||
logger.info("Discovered %d supported files in %s", len(files), path)
|
||||
return files
|
||||
@@ -0,0 +1 @@
|
||||
"""LLM-powered enrichment: entity detection, categorization, and action item extraction."""
|
||||
@@ -0,0 +1,122 @@
|
||||
"""Enrichment agent orchestrating LLM calls for entity detection.
|
||||
|
||||
Calls the LLM with extracted text, parses the structured response,
|
||||
filters entities by confidence threshold, and handles parse errors with retry.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from ingestion.config import IngestionConfig
|
||||
from ingestion.integrations.llm import LLMClient, LLMError
|
||||
|
||||
from .models import EnrichmentResult, Entity
|
||||
from .prompts import STRICT_RETRY_PROMPT, build_system_prompt, build_user_prompt
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EnrichmentError(Exception):
|
||||
"""Raised when enrichment fails after retries."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class EnrichmentAgent:
|
||||
"""Orchestrates LLM-based document enrichment."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: IngestionConfig,
|
||||
llm_client: LLMClient | None = None,
|
||||
known_projects: list[str] | None = None,
|
||||
known_people: list[str] | None = None,
|
||||
):
|
||||
self.config = config
|
||||
self.llm = llm_client or LLMClient(config)
|
||||
self.confidence_threshold = config.confidence_threshold
|
||||
self.system_prompt = build_system_prompt(known_projects, known_people)
|
||||
|
||||
def enrich(self, text: str) -> EnrichmentResult:
|
||||
"""Enrich extracted text with LLM-detected entities and metadata.
|
||||
|
||||
Args:
|
||||
text: The extracted document text to analyze.
|
||||
|
||||
Returns:
|
||||
EnrichmentResult with entities filtered by confidence threshold.
|
||||
|
||||
Raises:
|
||||
EnrichmentError: If LLM call and retry both fail.
|
||||
"""
|
||||
messages = [
|
||||
{"role": "system", "content": self.system_prompt},
|
||||
{"role": "user", "content": build_user_prompt(text)},
|
||||
]
|
||||
|
||||
# First attempt
|
||||
try:
|
||||
response_text = self.llm.complete(messages)
|
||||
result = self._parse_response(response_text)
|
||||
except (json.JSONDecodeError, ValueError) as parse_err:
|
||||
logger.warning(
|
||||
"First LLM response parse failed (%s), retrying with strict prompt",
|
||||
parse_err,
|
||||
)
|
||||
# Retry with stricter prompt
|
||||
result = self._retry_with_strict_prompt(messages, response_text)
|
||||
except LLMError as e:
|
||||
raise EnrichmentError(f"LLM call failed: {e}") from e
|
||||
|
||||
# Filter entities by confidence threshold
|
||||
result.entities = self._filter_by_confidence(result.entities)
|
||||
|
||||
return result
|
||||
|
||||
def _retry_with_strict_prompt(
|
||||
self, original_messages: list[dict[str, str]], failed_response: str
|
||||
) -> EnrichmentResult:
|
||||
"""Retry LLM call with a stricter prompt after parse failure."""
|
||||
retry_messages = original_messages + [
|
||||
{"role": "assistant", "content": failed_response},
|
||||
{"role": "user", "content": STRICT_RETRY_PROMPT},
|
||||
]
|
||||
|
||||
try:
|
||||
response_text = self.llm.complete(retry_messages)
|
||||
return self._parse_response(response_text)
|
||||
except (json.JSONDecodeError, ValueError, LLMError) as e:
|
||||
logger.error("Retry also failed (%s), using defaults", e)
|
||||
return self._default_result()
|
||||
|
||||
def _parse_response(self, response_text: str) -> EnrichmentResult:
|
||||
"""Parse LLM response text into an EnrichmentResult.
|
||||
|
||||
Handles responses that may be wrapped in markdown code fences.
|
||||
"""
|
||||
cleaned = response_text.strip()
|
||||
|
||||
# Strip markdown code fences if present
|
||||
if cleaned.startswith("```"):
|
||||
lines = cleaned.split("\n")
|
||||
# Remove first line (```json) and last line (```)
|
||||
lines = [l for l in lines[1:] if not l.strip() == "```"]
|
||||
cleaned = "\n".join(lines)
|
||||
|
||||
data = json.loads(cleaned)
|
||||
return EnrichmentResult.model_validate(data)
|
||||
|
||||
def _filter_by_confidence(self, entities: list[Entity]) -> list[Entity]:
|
||||
"""Keep only entities meeting the confidence threshold."""
|
||||
return [e for e in entities if e.confidence >= self.confidence_threshold]
|
||||
|
||||
def _default_result(self) -> EnrichmentResult:
|
||||
"""Return a minimal default result when all parsing fails."""
|
||||
return EnrichmentResult(
|
||||
title="Untitled Note",
|
||||
category="inbox",
|
||||
tags=[],
|
||||
entities=[],
|
||||
action_items=[],
|
||||
summary=None,
|
||||
)
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Pydantic models for LLM enrichment responses.
|
||||
|
||||
Defines the structured output format expected from the enrichment agent:
|
||||
entities, action items, and the overall enrichment result.
|
||||
"""
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class Entity(BaseModel):
|
||||
"""A detected entity in the document text."""
|
||||
|
||||
type: Literal["person", "project", "date", "action_item"]
|
||||
value: str
|
||||
confidence: float = Field(ge=0.0, le=1.0)
|
||||
position: int | None = None # character offset in source text
|
||||
|
||||
|
||||
class ActionItem(BaseModel):
|
||||
"""An extracted action item / task."""
|
||||
|
||||
description: str
|
||||
assignee: str | None = None
|
||||
deadline: str | None = None # ISO 8601 date string
|
||||
|
||||
|
||||
class EnrichmentResult(BaseModel):
|
||||
"""Complete enrichment output from the LLM."""
|
||||
|
||||
title: str
|
||||
category: Literal["meeting", "project", "decision", "inbox"]
|
||||
tags: list[str] = Field(default_factory=list)
|
||||
entities: list[Entity] = Field(default_factory=list)
|
||||
action_items: list[ActionItem] = Field(default_factory=list)
|
||||
summary: str | None = None
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Structured LLM prompts for document enrichment.
|
||||
|
||||
Provides the system prompt that instructs the LLM to return structured JSON
|
||||
with entities, categories, tags, and action items.
|
||||
"""
|
||||
|
||||
ENRICHMENT_SYSTEM_PROMPT = """\
|
||||
You are a knowledge management assistant. Analyze the following text extracted \
|
||||
from a document and return structured metadata as JSON.
|
||||
|
||||
Return a JSON object with exactly these fields:
|
||||
- "title": A concise, descriptive title for this note (max 80 chars)
|
||||
- "category": One of "meeting", "project", "decision", "inbox"
|
||||
- "tags": List of relevant topic tags (lowercase, no # prefix)
|
||||
- "entities": List of detected entities, each with:
|
||||
- "type": One of "person", "project", "date", "action_item"
|
||||
- "value": The extracted text value
|
||||
- "confidence": Float between 0.0 and 1.0 indicating detection confidence
|
||||
- "position": Approximate character offset where entity appears (integer or null)
|
||||
- "action_items": List of action items, each with:
|
||||
- "description": What needs to be done
|
||||
- "assignee": Person responsible (string or null)
|
||||
- "deadline": Date in ISO 8601 format, e.g. "2024-03-22" (string or null)
|
||||
- "summary": A 1-2 sentence summary of the document (string or null)
|
||||
|
||||
Rules:
|
||||
- Be precise with entity detection. Only mark entities you are confident about.
|
||||
- For people, use their full name as the value.
|
||||
- For projects, use the project name as commonly referenced.
|
||||
- For dates, use ISO 8601 format (YYYY-MM-DD) as the value.
|
||||
- Assign confidence scores honestly: 0.9+ for clearly stated entities, 0.7-0.9 for \
|
||||
likely entities, below 0.7 for uncertain ones.
|
||||
- Category "meeting" for meeting notes, agendas, minutes.
|
||||
- Category "project" for project-specific documentation.
|
||||
- Category "decision" for recorded decisions, ADRs.
|
||||
- Category "inbox" for anything that doesn't clearly fit the above.
|
||||
|
||||
Context: This is a personal knowledge base for a project manager working across \
|
||||
multiple business projects.{context_section}
|
||||
"""
|
||||
|
||||
STRICT_RETRY_PROMPT = """\
|
||||
Your previous response was not valid JSON. Please respond with ONLY a valid JSON \
|
||||
object matching the schema described in the system prompt. No markdown, no code fences, \
|
||||
no explanation — just the raw JSON object.
|
||||
"""
|
||||
|
||||
|
||||
def build_system_prompt(
|
||||
known_projects: list[str] | None = None,
|
||||
known_people: list[str] | None = None,
|
||||
) -> str:
|
||||
"""Build the system prompt with optional context about known entities.
|
||||
|
||||
Args:
|
||||
known_projects: List of known project names for context.
|
||||
known_people: List of known people names for context.
|
||||
|
||||
Returns:
|
||||
Complete system prompt string.
|
||||
"""
|
||||
context_parts: list[str] = []
|
||||
|
||||
if known_projects:
|
||||
projects_str = ", ".join(known_projects)
|
||||
context_parts.append(f"Known projects: {projects_str}.")
|
||||
|
||||
if known_people:
|
||||
people_str = ", ".join(known_people)
|
||||
context_parts.append(f"Known people: {people_str}.")
|
||||
|
||||
if context_parts:
|
||||
context_section = "\n" + " ".join(context_parts)
|
||||
else:
|
||||
context_section = ""
|
||||
|
||||
return ENRICHMENT_SYSTEM_PROMPT.format(context_section=context_section)
|
||||
|
||||
|
||||
def build_user_prompt(extracted_text: str) -> str:
|
||||
"""Build the user message containing the extracted document text.
|
||||
|
||||
Args:
|
||||
extracted_text: The raw text extracted from the document.
|
||||
|
||||
Returns:
|
||||
User prompt string.
|
||||
"""
|
||||
return f"Analyze this document and return the structured JSON:\n\n{extracted_text}"
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Text extraction layer for various document formats."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from .base import ExtractionResult, TextExtractor
|
||||
from .docx import DocxExtractor
|
||||
from .ocr import OCRExtractor
|
||||
from .pdf import PDFExtractor
|
||||
from .text import PlainTextExtractor
|
||||
|
||||
|
||||
def get_extractor(file_path: Path, config=None):
|
||||
"""Get the appropriate extractor for a file.
|
||||
|
||||
Args:
|
||||
file_path: Path to the file to extract.
|
||||
config: Optional IngestionConfig for LLM-based extraction.
|
||||
|
||||
Returns:
|
||||
An extractor instance that can handle the file, or None.
|
||||
"""
|
||||
suffix = file_path.suffix.lower()
|
||||
|
||||
if suffix in (".md", ".txt"):
|
||||
return PlainTextExtractor()
|
||||
elif suffix == ".pdf":
|
||||
if config:
|
||||
return PDFExtractor(
|
||||
ocr_enabled=config.ocr_enabled,
|
||||
ocr_language=config.ocr_language,
|
||||
llm_provider=config.llm_provider,
|
||||
llm_model=config.llm_model,
|
||||
mistral_api_key=config.mistral_api_key,
|
||||
)
|
||||
return PDFExtractor()
|
||||
elif suffix in (".jpg", ".jpeg", ".png"):
|
||||
return OCRExtractor()
|
||||
elif suffix == ".docx":
|
||||
return DocxExtractor()
|
||||
return None
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ExtractionResult",
|
||||
"TextExtractor",
|
||||
"PlainTextExtractor",
|
||||
"PDFExtractor",
|
||||
"OCRExtractor",
|
||||
"DocxExtractor",
|
||||
"get_extractor",
|
||||
]
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Base protocol and data structures for text extraction."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExtractionResult:
|
||||
"""Result of extracting text from a document."""
|
||||
|
||||
text: str
|
||||
source_file: Path
|
||||
extraction_method: str # "direct", "ocr", "mixed"
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
class TextExtractor(Protocol):
|
||||
"""Protocol that all text extractors must implement."""
|
||||
|
||||
def can_handle(self, file_path: Path) -> bool:
|
||||
"""Return True if this extractor can process the given file."""
|
||||
...
|
||||
|
||||
def extract(self, file_path: Path) -> ExtractionResult:
|
||||
"""Extract text content from the file."""
|
||||
...
|
||||
@@ -0,0 +1,57 @@
|
||||
"""DOCX text extraction using python-docx.
|
||||
|
||||
Preserves heading structure and paragraph breaks.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from .base import ExtractionResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DocxExtractor:
|
||||
"""Extracts text from .docx files preserving structure."""
|
||||
|
||||
def can_handle(self, file_path: Path) -> bool:
|
||||
"""Return True for .docx files."""
|
||||
return file_path.suffix.lower() == ".docx"
|
||||
|
||||
def extract(self, file_path: Path) -> ExtractionResult:
|
||||
"""Extract text from DOCX preserving headings and paragraphs."""
|
||||
try:
|
||||
from docx import Document
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"python-docx is required for DOCX extraction. "
|
||||
"Install with: pip install python-docx"
|
||||
)
|
||||
|
||||
doc = Document(str(file_path))
|
||||
parts: list[str] = []
|
||||
|
||||
for paragraph in doc.paragraphs:
|
||||
text = paragraph.text.strip()
|
||||
if not text:
|
||||
continue
|
||||
|
||||
# Preserve heading structure with markdown-style headers
|
||||
style_name = paragraph.style.name.lower() if paragraph.style else ""
|
||||
if "heading 1" in style_name:
|
||||
parts.append(f"# {text}")
|
||||
elif "heading 2" in style_name:
|
||||
parts.append(f"## {text}")
|
||||
elif "heading 3" in style_name:
|
||||
parts.append(f"### {text}")
|
||||
elif "heading 4" in style_name:
|
||||
parts.append(f"#### {text}")
|
||||
else:
|
||||
parts.append(text)
|
||||
|
||||
return ExtractionResult(
|
||||
text="\n\n".join(parts),
|
||||
source_file=file_path,
|
||||
extraction_method="direct",
|
||||
metadata={"paragraph_count": len(parts)},
|
||||
)
|
||||
@@ -0,0 +1,48 @@
|
||||
"""OCR extraction for images using pytesseract.
|
||||
|
||||
Handles .jpg, .jpeg, .png files. Configured for German + English (deu+eng).
|
||||
Returns empty text (not error) when OCR cannot extract readable content.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from .base import ExtractionResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SUPPORTED_EXTENSIONS = {".jpg", ".jpeg", ".png"}
|
||||
|
||||
|
||||
class OCRExtractor:
|
||||
"""Extracts text from images using Tesseract OCR."""
|
||||
|
||||
def __init__(self, language: str = "deu+eng"):
|
||||
self.language = language
|
||||
|
||||
def can_handle(self, file_path: Path) -> bool:
|
||||
"""Return True for .jpg, .jpeg, .png files."""
|
||||
return file_path.suffix.lower() in SUPPORTED_EXTENSIONS
|
||||
|
||||
def extract(self, file_path: Path) -> ExtractionResult:
|
||||
"""Run OCR on an image file. Returns empty text on failure."""
|
||||
text = self._run_ocr(file_path)
|
||||
return ExtractionResult(
|
||||
text=text,
|
||||
source_file=file_path,
|
||||
extraction_method="ocr",
|
||||
metadata={"ocr_language": self.language},
|
||||
)
|
||||
|
||||
def _run_ocr(self, file_path: Path) -> str:
|
||||
"""Execute pytesseract on the image. Returns empty string on failure."""
|
||||
try:
|
||||
import pytesseract
|
||||
from PIL import Image
|
||||
|
||||
image = Image.open(file_path)
|
||||
text = pytesseract.image_to_string(image, lang=self.language)
|
||||
return text.strip()
|
||||
except Exception as e:
|
||||
logger.warning("OCR extraction failed for %s: %s", file_path, e)
|
||||
return ""
|
||||
@@ -0,0 +1,156 @@
|
||||
"""PDF text extraction using pdfplumber.
|
||||
|
||||
Detects scanned pages (no extractable text) and delegates to:
|
||||
1. Mistral Vision API (preferred — handles handwriting well)
|
||||
2. Tesseract OCR (fallback if no LLM configured)
|
||||
"""
|
||||
|
||||
import base64
|
||||
import io
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import pdfplumber
|
||||
|
||||
from .base import ExtractionResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PDFExtractor:
|
||||
"""Extracts text from PDF files, with Vision LLM or OCR fallback for scanned pages."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ocr_enabled: bool = True,
|
||||
ocr_language: str = "deu+eng",
|
||||
llm_provider: str = "mistral",
|
||||
llm_model: str = "mistral-small-latest",
|
||||
mistral_api_key: str = "",
|
||||
):
|
||||
self.ocr_enabled = ocr_enabled
|
||||
self.ocr_language = ocr_language
|
||||
self.llm_provider = llm_provider
|
||||
self.llm_model = llm_model
|
||||
self.mistral_api_key = mistral_api_key
|
||||
self._use_vision = bool(mistral_api_key)
|
||||
|
||||
def can_handle(self, file_path: Path) -> bool:
|
||||
"""Return True for .pdf files."""
|
||||
return file_path.suffix.lower() == ".pdf"
|
||||
|
||||
def extract(self, file_path: Path) -> ExtractionResult:
|
||||
"""Extract text from PDF, using Vision LLM or OCR for scanned pages."""
|
||||
pages_text: list[str] = []
|
||||
has_direct_text = False
|
||||
has_vision_text = False
|
||||
|
||||
with pdfplumber.open(file_path) as pdf:
|
||||
for page_num, page in enumerate(pdf.pages, start=1):
|
||||
text = page.extract_text() or ""
|
||||
text = text.strip()
|
||||
|
||||
if text:
|
||||
pages_text.append(text)
|
||||
has_direct_text = True
|
||||
elif self.ocr_enabled:
|
||||
# Scanned page — try Vision LLM first, then Tesseract
|
||||
logger.debug(
|
||||
"Page %d of %s has no text, attempting vision OCR",
|
||||
page_num, file_path,
|
||||
)
|
||||
ocr_text = self._vision_ocr_page(page, page_num, file_path)
|
||||
if ocr_text:
|
||||
pages_text.append(ocr_text)
|
||||
has_vision_text = True
|
||||
|
||||
# Determine extraction method
|
||||
if has_direct_text and has_vision_text:
|
||||
method = "mixed"
|
||||
elif has_vision_text:
|
||||
method = "vision-ocr"
|
||||
else:
|
||||
method = "direct"
|
||||
|
||||
return ExtractionResult(
|
||||
text="\n\n".join(pages_text),
|
||||
source_file=file_path,
|
||||
extraction_method=method,
|
||||
metadata={"page_count": len(pages_text)},
|
||||
)
|
||||
|
||||
def _vision_ocr_page(self, page, page_num: int, file_path: Path) -> str:
|
||||
"""OCR a PDF page using Mistral Vision API, with Tesseract fallback."""
|
||||
if self._use_vision:
|
||||
try:
|
||||
return self._mistral_vision_ocr(page, page_num)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Vision OCR failed for page %d of %s: %s, trying Tesseract",
|
||||
page_num, file_path, e,
|
||||
)
|
||||
|
||||
# Fallback to Tesseract
|
||||
return self._tesseract_ocr(page, page_num, file_path)
|
||||
|
||||
def _mistral_vision_ocr(self, page, page_num: int) -> str:
|
||||
"""Send page image to Mistral Vision for text extraction."""
|
||||
import httpx
|
||||
|
||||
# Convert page to PNG image
|
||||
image = page.to_image(resolution=200)
|
||||
img_buffer = io.BytesIO()
|
||||
image.original.save(img_buffer, format="PNG")
|
||||
img_bytes = img_buffer.getvalue()
|
||||
img_b64 = base64.b64encode(img_bytes).decode("utf-8")
|
||||
|
||||
# Call Mistral Vision API via LiteLLM-compatible format
|
||||
response = httpx.post(
|
||||
"https://api.mistral.ai/v1/chat/completions",
|
||||
headers={
|
||||
"Authorization": f"Bearer {self.mistral_api_key}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
json={
|
||||
"model": self.llm_model,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Extract ALL text from this image. This is a handwritten note or document. "
|
||||
"Transcribe everything you can read, preserving structure (paragraphs, lists, headings). "
|
||||
"If text is in German, keep it in German. Return ONLY the extracted text, no commentary.",
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": f"data:image/png;base64,{img_b64}",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
"max_tokens": 4000,
|
||||
},
|
||||
timeout=60.0,
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
text = data["choices"][0]["message"]["content"]
|
||||
logger.info("Vision OCR page %d: extracted %d chars", page_num, len(text))
|
||||
return text.strip()
|
||||
|
||||
def _tesseract_ocr(self, page, page_num: int, file_path: Path) -> str:
|
||||
"""Fallback: OCR a PDF page using Tesseract."""
|
||||
try:
|
||||
import pytesseract
|
||||
image = page.to_image(resolution=300)
|
||||
text = pytesseract.image_to_string(image.original, lang=self.ocr_language)
|
||||
return text.strip()
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Tesseract OCR failed for page %d of %s: %s", page_num, file_path, e
|
||||
)
|
||||
return ""
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Plain text and markdown file extractor.
|
||||
|
||||
Handles .md and .txt files. Reads as UTF-8 by default, with fallback
|
||||
charset detection for non-UTF-8 encoded files.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from .base import ExtractionResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SUPPORTED_EXTENSIONS = {".md", ".txt"}
|
||||
|
||||
|
||||
class PlainTextExtractor:
|
||||
"""Extracts text from plain text and markdown files."""
|
||||
|
||||
def can_handle(self, file_path: Path) -> bool:
|
||||
"""Return True for .md and .txt files."""
|
||||
return file_path.suffix.lower() in SUPPORTED_EXTENSIONS
|
||||
|
||||
def extract(self, file_path: Path) -> ExtractionResult:
|
||||
"""Read file as UTF-8, falling back to charset detection."""
|
||||
text = self._read_with_fallback(file_path)
|
||||
return ExtractionResult(
|
||||
text=text,
|
||||
source_file=file_path,
|
||||
extraction_method="direct",
|
||||
metadata={"encoding": "utf-8"},
|
||||
)
|
||||
|
||||
def _read_with_fallback(self, file_path: Path) -> str:
|
||||
"""Try UTF-8 first, then attempt charset detection."""
|
||||
try:
|
||||
return file_path.read_text(encoding="utf-8")
|
||||
except UnicodeDecodeError:
|
||||
logger.warning(
|
||||
"UTF-8 decode failed for %s, attempting charset detection", file_path
|
||||
)
|
||||
return self._read_with_detection(file_path)
|
||||
|
||||
def _read_with_detection(self, file_path: Path) -> str:
|
||||
"""Detect encoding and read file."""
|
||||
raw = file_path.read_bytes()
|
||||
|
||||
# Try common encodings before giving up
|
||||
for encoding in ("latin-1", "cp1252", "iso-8859-1"):
|
||||
try:
|
||||
return raw.decode(encoding)
|
||||
except (UnicodeDecodeError, LookupError):
|
||||
continue
|
||||
|
||||
# Last resort: decode with replacement characters
|
||||
logger.warning("Could not detect encoding for %s, using replacement", file_path)
|
||||
return raw.decode("utf-8", errors="replace")
|
||||
@@ -0,0 +1 @@
|
||||
"""External service integrations: LLM providers, OrgMyLife, Git, Nextcloud."""
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Git commit operations for auto-committing imported notes.
|
||||
|
||||
Stages new markdown files and stub pages, commits with message format:
|
||||
ingestion: import N notes from [source-type]
|
||||
|
||||
Supports --no-commit flag. Handles git failures gracefully.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def commit_imported_notes(files: list[Path], source_type: str = "import") -> bool:
|
||||
"""Stage and commit imported note files.
|
||||
|
||||
Args:
|
||||
files: List of file paths to stage and commit.
|
||||
source_type: Source description for commit message (e.g., 'batch-import', 'upload', 'nextcloud').
|
||||
|
||||
Returns:
|
||||
True if commit succeeded, False otherwise.
|
||||
"""
|
||||
if not files:
|
||||
logger.debug("No files to commit.")
|
||||
return False
|
||||
|
||||
try:
|
||||
# Stage files
|
||||
file_strs = [str(f) for f in files]
|
||||
subprocess.run(
|
||||
["git", "add"] + file_strs,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
|
||||
# Commit
|
||||
count = len(files)
|
||||
message = f"ingestion: import {count} notes from {source_type}"
|
||||
|
||||
result = subprocess.run(
|
||||
["git", "commit", "-m", message],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
logger.info("Git commit: %s", message)
|
||||
return True
|
||||
else:
|
||||
# Nothing to commit (files already tracked/unchanged)
|
||||
logger.debug("Git commit returned %d: %s", result.returncode, result.stderr.strip())
|
||||
return False
|
||||
|
||||
except FileNotFoundError:
|
||||
logger.warning("Git not found in PATH — skipping commit.")
|
||||
return False
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.warning("Git operation failed: %s", e.stderr.strip() if e.stderr else str(e))
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.warning("Unexpected git error: %s", e)
|
||||
return False
|
||||
@@ -0,0 +1,112 @@
|
||||
"""LiteLLM wrapper for model-agnostic LLM access.
|
||||
|
||||
Supports OpenAI, Anthropic, Google, Mistral, and Ollama via a unified interface.
|
||||
Implements retry logic with exponential backoff for transient failures.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ingestion.config import IngestionConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Retry configuration
|
||||
MAX_RETRIES = 3
|
||||
BASE_DELAY_SECONDS = 1.0
|
||||
MAX_DELAY_SECONDS = 30.0
|
||||
BACKOFF_FACTOR = 2.0
|
||||
|
||||
|
||||
class LLMError(Exception):
|
||||
"""Raised when an LLM call fails after all retries."""
|
||||
|
||||
def __init__(self, provider: str, model: str, message: str):
|
||||
self.provider = provider
|
||||
self.model = model
|
||||
super().__init__(
|
||||
f"LLM call failed [{provider}/{model}]: {message}"
|
||||
)
|
||||
|
||||
|
||||
class LLMClient:
|
||||
"""Model-agnostic LLM client using LiteLLM."""
|
||||
|
||||
def __init__(self, config: IngestionConfig):
|
||||
self.model = config.llm_model_string
|
||||
self.provider = config.llm_provider
|
||||
self.config = config
|
||||
|
||||
def complete(
|
||||
self,
|
||||
messages: list[dict[str, str]],
|
||||
response_format: type[BaseModel] | None = None,
|
||||
) -> str:
|
||||
"""Send a completion request with retry logic.
|
||||
|
||||
Args:
|
||||
messages: List of message dicts with 'role' and 'content' keys.
|
||||
response_format: Optional Pydantic model for structured output.
|
||||
|
||||
Returns:
|
||||
The assistant's response content as a string.
|
||||
|
||||
Raises:
|
||||
LLMError: If all retries are exhausted.
|
||||
"""
|
||||
import litellm
|
||||
|
||||
last_error: Exception | None = None
|
||||
|
||||
for attempt in range(1, MAX_RETRIES + 1):
|
||||
try:
|
||||
kwargs: dict[str, Any] = {
|
||||
"model": self.model,
|
||||
"messages": messages,
|
||||
}
|
||||
|
||||
if response_format is not None:
|
||||
kwargs["response_format"] = response_format
|
||||
|
||||
response = litellm.completion(**kwargs)
|
||||
content = response.choices[0].message.content
|
||||
return content or ""
|
||||
|
||||
except (
|
||||
litellm.RateLimitError,
|
||||
litellm.Timeout,
|
||||
litellm.ServiceUnavailableError,
|
||||
litellm.InternalServerError,
|
||||
) as e:
|
||||
last_error = e
|
||||
delay = min(
|
||||
BASE_DELAY_SECONDS * (BACKOFF_FACTOR ** (attempt - 1)),
|
||||
MAX_DELAY_SECONDS,
|
||||
)
|
||||
logger.warning(
|
||||
"LLM call attempt %d/%d failed (%s: %s), retrying in %.1fs",
|
||||
attempt,
|
||||
MAX_RETRIES,
|
||||
type(e).__name__,
|
||||
str(e),
|
||||
delay,
|
||||
)
|
||||
time.sleep(delay)
|
||||
|
||||
except Exception as e:
|
||||
# Non-retryable error
|
||||
raise LLMError(
|
||||
provider=self.provider,
|
||||
model=self.model,
|
||||
message=f"{type(e).__name__}: {e}",
|
||||
) from e
|
||||
|
||||
# All retries exhausted
|
||||
raise LLMError(
|
||||
provider=self.provider,
|
||||
model=self.model,
|
||||
message=f"All {MAX_RETRIES} retries exhausted. Last error: {last_error}",
|
||||
)
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Nextcloud WebDAV inbox sync.
|
||||
|
||||
Polls a Nextcloud folder via WebDAV (PROPFIND/GET) for new files,
|
||||
downloads them for processing, and moves processed files to a processed/ subfolder.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
from urllib.parse import unquote, urljoin
|
||||
|
||||
import httpx
|
||||
|
||||
from ingestion.config import IngestionConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# WebDAV XML namespace
|
||||
DAV_NS = "DAV:"
|
||||
|
||||
|
||||
class NextcloudInboxSync:
|
||||
"""Sync files from a Nextcloud WebDAV folder."""
|
||||
|
||||
def __init__(self, config: IngestionConfig):
|
||||
self.base_url = config.nextcloud_inbox_url.rstrip("/") + "/"
|
||||
self.user = config.nextcloud_inbox_user
|
||||
self.password = config.nextcloud_inbox_password
|
||||
self.local_inbox = Path(config.inbox_dir)
|
||||
self.local_inbox.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self._client = httpx.Client(
|
||||
auth=(self.user, self.password),
|
||||
timeout=30.0,
|
||||
)
|
||||
|
||||
def poll(self) -> list[Path]:
|
||||
"""Poll Nextcloud inbox for files and download them.
|
||||
|
||||
Returns:
|
||||
List of local paths to downloaded files.
|
||||
"""
|
||||
try:
|
||||
filenames = self._list_files()
|
||||
except Exception as e:
|
||||
logger.error("Failed to list Nextcloud inbox: %s", e)
|
||||
return []
|
||||
|
||||
if not filenames:
|
||||
logger.info("Nextcloud inbox is empty.")
|
||||
return []
|
||||
|
||||
downloaded: list[Path] = []
|
||||
for filename in filenames:
|
||||
local_path = self.download(filename)
|
||||
if local_path:
|
||||
downloaded.append(local_path)
|
||||
|
||||
logger.info("Downloaded %d file(s) from Nextcloud.", len(downloaded))
|
||||
return downloaded
|
||||
|
||||
def _list_files(self) -> list[str]:
|
||||
"""PROPFIND to list files in the inbox folder."""
|
||||
propfind_body = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<d:propfind xmlns:d="DAV:">
|
||||
<d:prop>
|
||||
<d:resourcetype/>
|
||||
<d:getcontentlength/>
|
||||
</d:prop>
|
||||
</d:propfind>"""
|
||||
|
||||
response = self._client.request(
|
||||
"PROPFIND",
|
||||
self.base_url,
|
||||
content=propfind_body,
|
||||
headers={
|
||||
"Content-Type": "application/xml",
|
||||
"Depth": "1",
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
# Parse WebDAV XML response
|
||||
root = ET.fromstring(response.text)
|
||||
filenames: list[str] = []
|
||||
|
||||
for resp_elem in root.findall(f"{{{DAV_NS}}}response"):
|
||||
href = resp_elem.findtext(f"{{{DAV_NS}}}href", "")
|
||||
# Skip the folder itself (collections)
|
||||
resourcetype = resp_elem.find(
|
||||
f".//{{{DAV_NS}}}resourcetype/{{{DAV_NS}}}collection"
|
||||
)
|
||||
if resourcetype is not None:
|
||||
continue
|
||||
|
||||
# Extract filename from href
|
||||
filename = unquote(href.rstrip("/").split("/")[-1])
|
||||
if filename and not filename.startswith("."):
|
||||
filenames.append(filename)
|
||||
|
||||
return filenames
|
||||
|
||||
def download(self, filename: str) -> Path | None:
|
||||
"""Download a file from Nextcloud inbox.
|
||||
|
||||
Args:
|
||||
filename: Name of the file to download.
|
||||
|
||||
Returns:
|
||||
Local path to the downloaded file, or None on failure.
|
||||
"""
|
||||
url = urljoin(self.base_url, filename)
|
||||
try:
|
||||
response = self._client.get(url)
|
||||
response.raise_for_status()
|
||||
|
||||
local_path = self.local_inbox / filename
|
||||
local_path.write_bytes(response.content)
|
||||
logger.info("Downloaded: %s", filename)
|
||||
return local_path
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Failed to download %s: %s", filename, e)
|
||||
return None
|
||||
|
||||
def mark_processed(self, filename: str) -> bool:
|
||||
"""Move a processed file to the processed/ subfolder on Nextcloud.
|
||||
|
||||
Args:
|
||||
filename: Name of the file to move.
|
||||
|
||||
Returns:
|
||||
True if move succeeded, False otherwise.
|
||||
"""
|
||||
source_url = urljoin(self.base_url, filename)
|
||||
dest_url = urljoin(self.base_url, f"processed/{filename}")
|
||||
|
||||
try:
|
||||
# Ensure processed/ folder exists (MKCOL)
|
||||
processed_url = urljoin(self.base_url, "processed/")
|
||||
self._client.request("MKCOL", processed_url)
|
||||
|
||||
# MOVE the file
|
||||
response = self._client.request(
|
||||
"MOVE",
|
||||
source_url,
|
||||
headers={"Destination": dest_url, "Overwrite": "T"},
|
||||
)
|
||||
if response.status_code in (201, 204):
|
||||
logger.info("Marked processed: %s", filename)
|
||||
return True
|
||||
else:
|
||||
logger.warning(
|
||||
"MOVE returned %d for %s", response.status_code, filename
|
||||
)
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("Failed to mark %s as processed: %s", filename, e)
|
||||
return False
|
||||
@@ -0,0 +1,74 @@
|
||||
"""OrgMyLife API client for creating tasks from extracted action items.
|
||||
|
||||
POST to /api/tasks with title, source_url, description.
|
||||
Handles unreachable API gracefully (log warning, continue).
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OrgMyLifeClient:
|
||||
"""HTTP client for OrgMyLife task creation."""
|
||||
|
||||
def __init__(self, base_url: str, api_key: str):
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.api_key = api_key
|
||||
|
||||
def create_task(
|
||||
self,
|
||||
title: str,
|
||||
source_url: str,
|
||||
description: str = "",
|
||||
priority: int = 4,
|
||||
) -> dict:
|
||||
"""Create a task in OrgMyLife.
|
||||
|
||||
Args:
|
||||
title: Task title (action item description).
|
||||
source_url: Reference back to the source note.
|
||||
description: Optional additional description.
|
||||
priority: Task priority (1-5, default 4).
|
||||
|
||||
Returns:
|
||||
Response data dict, or empty dict on failure.
|
||||
"""
|
||||
url = f"{self.base_url}/api/tasks"
|
||||
payload = {
|
||||
"title": title,
|
||||
"source_url": source_url,
|
||||
"description": description,
|
||||
"priority": priority,
|
||||
}
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
try:
|
||||
response = httpx.post(
|
||||
url,
|
||||
json=payload,
|
||||
headers=headers,
|
||||
timeout=10.0,
|
||||
)
|
||||
response.raise_for_status()
|
||||
logger.info("Created OrgMyLife task: %s", title[:50])
|
||||
return response.json()
|
||||
|
||||
except httpx.ConnectError:
|
||||
logger.warning(
|
||||
"OrgMyLife API unreachable at %s — skipping task creation", self.base_url
|
||||
)
|
||||
return {}
|
||||
except httpx.HTTPStatusError as e:
|
||||
logger.warning(
|
||||
"OrgMyLife API error %d: %s", e.response.status_code, e.response.text[:200]
|
||||
)
|
||||
return {}
|
||||
except Exception as e:
|
||||
logger.warning("OrgMyLife task creation failed: %s", e)
|
||||
return {}
|
||||
@@ -0,0 +1 @@
|
||||
"""Wiki-link generation and stub page creation."""
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Wiki-link generation from detected entities.
|
||||
|
||||
Converts entity references to SilverBullet-compatible wiki-links:
|
||||
- Person → [[people/firstname-lastname]]
|
||||
- Project → [[projects/project-slug]]
|
||||
|
||||
Only the first occurrence of each entity is linked; subsequent mentions remain plain text.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from slugify import slugify
|
||||
|
||||
from ingestion.enrichment.models import Entity
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _entity_to_wikilink(entity: Entity) -> str:
|
||||
"""Convert an entity to its wiki-link representation."""
|
||||
slug = slugify(entity.value)
|
||||
if entity.type == "person":
|
||||
return f"[[people/{slug}]]"
|
||||
elif entity.type == "project":
|
||||
return f"[[projects/{slug}]]"
|
||||
return ""
|
||||
|
||||
|
||||
def generate_wiki_links(text: str, entities: list[Entity]) -> str:
|
||||
"""Insert wiki-links into text for person and project entities.
|
||||
|
||||
Only the first occurrence of each entity's value is converted to a wiki-link.
|
||||
Dates and action_items are skipped (handled by frontmatter/renderer).
|
||||
|
||||
Args:
|
||||
text: The document body text.
|
||||
entities: List of detected entities from enrichment.
|
||||
|
||||
Returns:
|
||||
Text with wiki-links inserted at first occurrence of each entity.
|
||||
"""
|
||||
# Filter to linkable entity types
|
||||
linkable = [e for e in entities if e.type in ("person", "project")]
|
||||
|
||||
# Track which entity values we've already linked (case-insensitive)
|
||||
linked_values: set[str] = set()
|
||||
|
||||
for entity in linkable:
|
||||
lower_val = entity.value.lower()
|
||||
if lower_val in linked_values:
|
||||
continue
|
||||
|
||||
# Find first occurrence and replace it
|
||||
idx = text.find(entity.value)
|
||||
if idx == -1:
|
||||
# Try case-insensitive search
|
||||
idx = text.lower().find(lower_val)
|
||||
if idx == -1:
|
||||
continue
|
||||
# Use the actual text at that position for replacement
|
||||
original_text = text[idx : idx + len(entity.value)]
|
||||
else:
|
||||
original_text = entity.value
|
||||
|
||||
wikilink = _entity_to_wikilink(entity)
|
||||
if wikilink:
|
||||
# Replace only the first occurrence
|
||||
text = text[:idx] + wikilink + text[idx + len(original_text) :]
|
||||
linked_values.add(lower_val)
|
||||
|
||||
return text
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Stub page creation for entities that don't yet have a NoteGraph page.
|
||||
|
||||
Creates minimal markdown pages in the appropriate directory (people/ or projects/)
|
||||
so that wiki-links resolve immediately after import.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from slugify import slugify
|
||||
|
||||
from ingestion.enrichment.models import Entity
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def create_stub_pages(entities: list[Entity], notes_dir: Path) -> list[Path]:
|
||||
"""Create stub pages for entities that don't have existing pages.
|
||||
|
||||
Args:
|
||||
entities: List of entities to check/create stubs for.
|
||||
notes_dir: Root notes directory (e.g., ./notes).
|
||||
|
||||
Returns:
|
||||
List of paths to newly created stub pages.
|
||||
"""
|
||||
notes_dir = Path(notes_dir)
|
||||
created: list[Path] = []
|
||||
|
||||
linkable = [e for e in entities if e.type in ("person", "project")]
|
||||
seen_slugs: set[str] = set()
|
||||
|
||||
for entity in linkable:
|
||||
slug = slugify(entity.value)
|
||||
if slug in seen_slugs:
|
||||
continue
|
||||
seen_slugs.add(slug)
|
||||
|
||||
if entity.type == "person":
|
||||
target = notes_dir / "people" / f"{slug}.md"
|
||||
elif entity.type == "project":
|
||||
target = notes_dir / "projects" / f"{slug}.md"
|
||||
else:
|
||||
continue
|
||||
|
||||
if target.exists():
|
||||
logger.debug("Page already exists: %s", target)
|
||||
continue
|
||||
|
||||
# Create stub page
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(f"# {entity.value}\n", encoding="utf-8")
|
||||
logger.info("Created stub page: %s", target)
|
||||
created.append(target)
|
||||
|
||||
return created
|
||||
@@ -0,0 +1 @@
|
||||
"""Output rendering, folder routing, and filename generation."""
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Filename generation with collision avoidance.
|
||||
|
||||
Pattern: YYYY-MM-DD-slugified-title.md
|
||||
Slug: lowercase alphanumeric + hyphens only.
|
||||
Appends numeric suffix (-2, -3, ...) if file already exists.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from slugify import slugify
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def generate_filename(title: str, date: str | None = None) -> str:
|
||||
"""Generate a filename from title and optional date.
|
||||
|
||||
Args:
|
||||
title: The note title to slugify.
|
||||
date: Optional date string (YYYY-MM-DD). Uses today if not provided.
|
||||
|
||||
Returns:
|
||||
Filename in format YYYY-MM-DD-slugified-title.md
|
||||
"""
|
||||
if date is None:
|
||||
date = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
|
||||
# Ensure date is just the date portion (handle ISO datetime strings)
|
||||
date_part = date[:10] if len(date) >= 10 else date
|
||||
|
||||
slug = slugify(title, max_length=80)
|
||||
if not slug:
|
||||
slug = "untitled"
|
||||
|
||||
return f"{date_part}-{slug}.md"
|
||||
|
||||
|
||||
def resolve_collision(target_dir: Path, filename: str) -> Path:
|
||||
"""Resolve filename collisions by appending numeric suffix.
|
||||
|
||||
Args:
|
||||
target_dir: Directory where the file will be written.
|
||||
filename: Desired filename (e.g., 2024-03-15-meeting-notes.md).
|
||||
|
||||
Returns:
|
||||
Full path with unique filename (appends -2, -3, etc. if needed).
|
||||
"""
|
||||
target = Path(target_dir) / filename
|
||||
|
||||
if not target.exists():
|
||||
return target
|
||||
|
||||
# Split filename to insert suffix before .md
|
||||
stem = filename.rsplit(".md", 1)[0]
|
||||
suffix_num = 2
|
||||
|
||||
while True:
|
||||
new_filename = f"{stem}-{suffix_num}.md"
|
||||
new_target = Path(target_dir) / new_filename
|
||||
if not new_target.exists():
|
||||
logger.debug("Collision resolved: %s → %s", filename, new_filename)
|
||||
return new_target
|
||||
suffix_num += 1
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Markdown + YAML frontmatter rendering for processed notes.
|
||||
|
||||
Renders:
|
||||
- Frontmatter with title, date, tags, people, projects, source
|
||||
- Body with wiki-links inserted
|
||||
- Action Items section at the end
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from ingestion.enrichment.models import EnrichmentResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def render_note(
|
||||
enrichment: EnrichmentResult, body_text: str, source_file: Path
|
||||
) -> str:
|
||||
"""Render a complete markdown note with YAML frontmatter.
|
||||
|
||||
Args:
|
||||
enrichment: The enrichment result with metadata and entities.
|
||||
body_text: The document body (with wiki-links already inserted).
|
||||
source_file: Original source file path for provenance.
|
||||
|
||||
Returns:
|
||||
Complete markdown string with frontmatter, body, and action items.
|
||||
"""
|
||||
# Build frontmatter data
|
||||
frontmatter: dict = {
|
||||
"title": enrichment.title,
|
||||
}
|
||||
|
||||
# Extract date from entities or use today
|
||||
date_entities = [e for e in enrichment.entities if e.type == "date"]
|
||||
if date_entities:
|
||||
frontmatter["date"] = date_entities[0].value
|
||||
else:
|
||||
frontmatter["date"] = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
|
||||
if enrichment.tags:
|
||||
frontmatter["tags"] = enrichment.tags
|
||||
|
||||
# People and projects from entities
|
||||
people = list(
|
||||
{e.value for e in enrichment.entities if e.type == "person"}
|
||||
)
|
||||
projects = list(
|
||||
{e.value for e in enrichment.entities if e.type == "project"}
|
||||
)
|
||||
|
||||
if people:
|
||||
frontmatter["people"] = sorted(people)
|
||||
if projects:
|
||||
frontmatter["projects"] = sorted(projects)
|
||||
|
||||
frontmatter["source"] = {
|
||||
"file": source_file.name,
|
||||
"imported": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S"),
|
||||
}
|
||||
|
||||
# Render YAML frontmatter
|
||||
yaml_str = yaml.dump(
|
||||
frontmatter,
|
||||
default_flow_style=False,
|
||||
allow_unicode=True,
|
||||
sort_keys=False,
|
||||
)
|
||||
|
||||
# Build the full document
|
||||
parts = [
|
||||
"---",
|
||||
yaml_str.rstrip(),
|
||||
"---",
|
||||
"",
|
||||
body_text.rstrip(),
|
||||
]
|
||||
|
||||
# Add action items section if any
|
||||
if enrichment.action_items:
|
||||
parts.append("")
|
||||
parts.append("## Action Items")
|
||||
parts.append("")
|
||||
for item in enrichment.action_items:
|
||||
line = f"- [ ] {item.description}"
|
||||
if item.assignee:
|
||||
line += f" ({item.assignee})"
|
||||
if item.deadline:
|
||||
line += f" (deadline: {item.deadline})"
|
||||
parts.append(line)
|
||||
|
||||
parts.append("") # trailing newline
|
||||
return "\n".join(parts)
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Category-based folder routing for output files.
|
||||
|
||||
Routes notes to the appropriate NoteGraph folder based on detected category:
|
||||
- meeting → notes/meetings/
|
||||
- project → notes/projects/
|
||||
- decision → notes/decisions/
|
||||
- default → notes/inbox/
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Category to folder mapping
|
||||
CATEGORY_FOLDERS: dict[str, str] = {
|
||||
"meeting": "meetings",
|
||||
"project": "projects",
|
||||
"decision": "decisions",
|
||||
}
|
||||
|
||||
|
||||
def route_to_folder(
|
||||
category: str, notes_dir: Path, override: Path | None = None
|
||||
) -> Path:
|
||||
"""Determine the output folder for a note based on its category.
|
||||
|
||||
Args:
|
||||
category: The detected category (meeting, project, decision, inbox).
|
||||
notes_dir: Root notes directory (e.g., ./notes).
|
||||
override: Optional explicit output directory override (--output-dir).
|
||||
|
||||
Returns:
|
||||
Path to the target folder (created if it doesn't exist).
|
||||
"""
|
||||
if override is not None:
|
||||
target = Path(override)
|
||||
else:
|
||||
folder_name = CATEGORY_FOLDERS.get(category, "inbox")
|
||||
target = Path(notes_dir) / folder_name
|
||||
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
logger.debug("Routing category '%s' to folder: %s", category, target)
|
||||
return target
|
||||
@@ -0,0 +1,217 @@
|
||||
"""Main processing pipeline orchestrating discovery → extraction → enrichment → linking → output.
|
||||
|
||||
Handles:
|
||||
- Progress tracking (current file / total)
|
||||
- Per-file error handling (log + skip + continue)
|
||||
- Summary reporting (processed, success, failed counts)
|
||||
- Dry-run mode
|
||||
"""
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from ingestion.config import IngestionConfig
|
||||
from ingestion.discovery import discover_files
|
||||
from ingestion.enrichment.agent import EnrichmentAgent
|
||||
from ingestion.enrichment.models import EnrichmentResult
|
||||
from ingestion.extraction import get_extractor
|
||||
from ingestion.integrations.git import commit_imported_notes
|
||||
from ingestion.integrations.orgmylife import OrgMyLifeClient
|
||||
from ingestion.linking.linker import generate_wiki_links
|
||||
from ingestion.linking.stubs import create_stub_pages
|
||||
from ingestion.output.naming import generate_filename, resolve_collision
|
||||
from ingestion.output.renderer import render_note
|
||||
from ingestion.output.router import route_to_folder
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProcessResult:
|
||||
"""Result of processing a single file."""
|
||||
|
||||
source_file: Path
|
||||
output_file: Path | None = None
|
||||
success: bool = False
|
||||
error: str | None = None
|
||||
stub_pages: list[Path] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BatchResult:
|
||||
"""Summary of a batch processing run."""
|
||||
|
||||
total: int = 0
|
||||
success: int = 0
|
||||
failed: int = 0
|
||||
results: list[ProcessResult] = field(default_factory=list)
|
||||
|
||||
|
||||
def process_file(
|
||||
file_path: Path,
|
||||
config: IngestionConfig,
|
||||
enrichment_agent: EnrichmentAgent | None = None,
|
||||
dry_run: bool = False,
|
||||
no_commit: bool = False,
|
||||
create_tasks: bool = False,
|
||||
output_dir: Path | None = None,
|
||||
) -> ProcessResult:
|
||||
"""Process a single file through the full ingestion pipeline.
|
||||
|
||||
Steps: extract → enrich → link → render → write → (optional) git commit
|
||||
|
||||
Args:
|
||||
file_path: Path to the source file.
|
||||
config: Ingestion configuration.
|
||||
enrichment_agent: Optional pre-configured enrichment agent.
|
||||
dry_run: If True, don't write any files.
|
||||
no_commit: If True, skip git commit.
|
||||
create_tasks: If True, create OrgMyLife tasks for action items.
|
||||
output_dir: Optional override for output directory.
|
||||
|
||||
Returns:
|
||||
ProcessResult with success status and output path.
|
||||
"""
|
||||
result = ProcessResult(source_file=file_path)
|
||||
|
||||
try:
|
||||
# 1. Extract text
|
||||
extractor = get_extractor(file_path, config)
|
||||
if extractor is None:
|
||||
result.error = f"No extractor available for {file_path.suffix}"
|
||||
return result
|
||||
|
||||
extraction = extractor.extract(file_path)
|
||||
if not extraction.text.strip():
|
||||
result.error = "No text extracted from file"
|
||||
return result
|
||||
|
||||
# 2. Enrich with LLM
|
||||
if enrichment_agent is None:
|
||||
enrichment_agent = EnrichmentAgent(config)
|
||||
|
||||
enrichment = enrichment_agent.enrich(extraction.text)
|
||||
|
||||
# 3. Generate wiki-links
|
||||
linked_text = generate_wiki_links(extraction.text, enrichment.entities)
|
||||
|
||||
# 4. Render output
|
||||
rendered = render_note(enrichment, linked_text, file_path)
|
||||
|
||||
# 5. Determine output location
|
||||
notes_dir = Path(config.notes_dir)
|
||||
target_folder = route_to_folder(enrichment.category, notes_dir, output_dir)
|
||||
|
||||
# 6. Generate filename
|
||||
date_entities = [e for e in enrichment.entities if e.type == "date"]
|
||||
date_str = date_entities[0].value if date_entities else None
|
||||
filename = generate_filename(enrichment.title, date_str)
|
||||
output_path = resolve_collision(target_folder, filename)
|
||||
|
||||
if dry_run:
|
||||
logger.info("[DRY RUN] Would write: %s", output_path)
|
||||
result.output_file = output_path
|
||||
result.success = True
|
||||
return result
|
||||
|
||||
# 7. Write file
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(rendered, encoding="utf-8")
|
||||
result.output_file = output_path
|
||||
logger.info("Written: %s", output_path)
|
||||
|
||||
# 8. Create stub pages
|
||||
stubs = create_stub_pages(enrichment.entities, notes_dir)
|
||||
result.stub_pages = stubs
|
||||
|
||||
# 9. Create OrgMyLife tasks if requested
|
||||
if create_tasks and enrichment.action_items and config.orgmylife_api_key:
|
||||
_create_tasks(enrichment, output_path, config)
|
||||
|
||||
result.success = True
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Failed to process %s: %s", file_path, e)
|
||||
result.error = str(e)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def process_batch(
|
||||
paths: list[Path],
|
||||
config: IngestionConfig,
|
||||
dry_run: bool = False,
|
||||
no_commit: bool = False,
|
||||
create_tasks: bool = False,
|
||||
output_dir: Path | None = None,
|
||||
progress_callback=None,
|
||||
) -> BatchResult:
|
||||
"""Process a batch of files through the pipeline.
|
||||
|
||||
Args:
|
||||
paths: List of file paths to process.
|
||||
config: Ingestion configuration.
|
||||
dry_run: If True, don't write any files.
|
||||
no_commit: If True, skip git commit.
|
||||
create_tasks: If True, create OrgMyLife tasks.
|
||||
output_dir: Optional override for output directory.
|
||||
progress_callback: Optional callable(current, total, filename).
|
||||
|
||||
Returns:
|
||||
BatchResult with summary counts and per-file results.
|
||||
"""
|
||||
batch = BatchResult(total=len(paths))
|
||||
enrichment_agent = EnrichmentAgent(config)
|
||||
written_files: list[Path] = []
|
||||
|
||||
for i, file_path in enumerate(paths, 1):
|
||||
if progress_callback:
|
||||
progress_callback(i, len(paths), file_path.name)
|
||||
|
||||
logger.info("Processing [%d/%d]: %s", i, len(paths), file_path.name)
|
||||
|
||||
result = process_file(
|
||||
file_path=file_path,
|
||||
config=config,
|
||||
enrichment_agent=enrichment_agent,
|
||||
dry_run=dry_run,
|
||||
no_commit=True, # Commit once at end
|
||||
create_tasks=create_tasks,
|
||||
output_dir=output_dir,
|
||||
)
|
||||
|
||||
batch.results.append(result)
|
||||
if result.success:
|
||||
batch.success += 1
|
||||
if result.output_file:
|
||||
written_files.append(result.output_file)
|
||||
written_files.extend(result.stub_pages)
|
||||
else:
|
||||
batch.failed += 1
|
||||
logger.warning("Skipped %s: %s", file_path.name, result.error)
|
||||
|
||||
# Git commit all at once (unless dry-run or no-commit)
|
||||
if not dry_run and not no_commit and config.auto_commit and written_files:
|
||||
commit_imported_notes(written_files, source_type="batch-import")
|
||||
|
||||
return batch
|
||||
|
||||
|
||||
def _create_tasks(
|
||||
enrichment: EnrichmentResult, note_path: Path, config: IngestionConfig
|
||||
) -> None:
|
||||
"""Create OrgMyLife tasks for action items."""
|
||||
try:
|
||||
client = OrgMyLifeClient(
|
||||
base_url=config.orgmylife_api_url,
|
||||
api_key=config.orgmylife_api_key,
|
||||
)
|
||||
for item in enrichment.action_items:
|
||||
client.create_task(
|
||||
title=item.description,
|
||||
source_url=str(note_path),
|
||||
description=f"From note: {enrichment.title}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to create OrgMyLife tasks: %s", e)
|
||||
@@ -0,0 +1,40 @@
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "notegraph-ingestion"
|
||||
version = "0.1.0"
|
||||
description = "NoteGraph document ingestion and AI-powered auto-referencing pipeline"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
"click>=8.1",
|
||||
"watchdog>=4.0",
|
||||
"pytesseract>=0.3.10",
|
||||
"pdfplumber>=0.11",
|
||||
"python-docx>=1.1",
|
||||
"litellm>=1.40",
|
||||
"httpx>=0.27",
|
||||
"pydantic>=2.7",
|
||||
"pydantic-settings>=2.3",
|
||||
"fastapi>=0.111",
|
||||
"uvicorn>=0.30",
|
||||
"python-multipart>=0.0.9",
|
||||
"pyyaml>=6.0",
|
||||
"python-slugify>=8.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest>=8.0",
|
||||
"pytest-asyncio>=0.23",
|
||||
"hypothesis>=6.100",
|
||||
"pytest-mock>=3.12",
|
||||
"respx>=0.21",
|
||||
]
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["ingestion"]
|
||||
|
||||
[project.scripts]
|
||||
notegraph-ingest = "ingestion.cli:cli"
|
||||
@@ -0,0 +1 @@
|
||||
"""Tests for the NoteGraph ingestion pipeline."""
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Shared test fixtures for the ingestion test suite."""
|
||||
|
||||
import pytest
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from ingestion.config import IngestionConfig
|
||||
from ingestion.enrichment.models import ActionItem, EnrichmentResult, Entity
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tmp_notes_dir(tmp_path: Path) -> Path:
|
||||
"""Create a temporary notes directory structure."""
|
||||
notes = tmp_path / "notes"
|
||||
notes.mkdir()
|
||||
(notes / "inbox").mkdir()
|
||||
(notes / "meetings").mkdir()
|
||||
(notes / "projects").mkdir()
|
||||
(notes / "people").mkdir()
|
||||
(notes / "decisions").mkdir()
|
||||
return notes
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tmp_inbox(tmp_path: Path) -> Path:
|
||||
"""Create a temporary inbox directory."""
|
||||
inbox = tmp_path / "inbox"
|
||||
inbox.mkdir()
|
||||
return inbox
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_config(tmp_path: Path) -> IngestionConfig:
|
||||
"""Create a test configuration with temporary paths."""
|
||||
notes_dir = tmp_path / "notes"
|
||||
notes_dir.mkdir(exist_ok=True)
|
||||
inbox_dir = tmp_path / "inbox"
|
||||
inbox_dir.mkdir(exist_ok=True)
|
||||
return IngestionConfig(
|
||||
llm_provider="openai",
|
||||
llm_model="gpt-4o",
|
||||
openai_api_key="test-key-fake",
|
||||
notes_dir=str(notes_dir),
|
||||
inbox_dir=str(inbox_dir),
|
||||
auto_commit=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_enrichment_result() -> EnrichmentResult:
|
||||
"""Create a sample enrichment result for testing."""
|
||||
return EnrichmentResult(
|
||||
title="Meeting with Beier GmbH",
|
||||
category="meeting",
|
||||
tags=["meeting", "customer"],
|
||||
entities=[
|
||||
Entity(type="person", value="Thomas Beier", confidence=0.95),
|
||||
Entity(type="project", value="Druckluft", confidence=0.9),
|
||||
Entity(type="date", value="2024-03-15", confidence=0.99),
|
||||
],
|
||||
action_items=[
|
||||
ActionItem(description="Send updated proposal", assignee="André", deadline="2024-03-22"),
|
||||
ActionItem(description="Schedule follow-up meeting"),
|
||||
],
|
||||
summary="Discussion about the Druckluft project.",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_llm_client():
|
||||
"""Create a mock LLM client that returns a valid enrichment JSON."""
|
||||
import json
|
||||
|
||||
mock = MagicMock()
|
||||
mock.complete.return_value = json.dumps({
|
||||
"title": "Test Note",
|
||||
"category": "inbox",
|
||||
"tags": ["test"],
|
||||
"entities": [],
|
||||
"action_items": [],
|
||||
"summary": "A test note.",
|
||||
})
|
||||
return mock
|
||||
@@ -0,0 +1,806 @@
|
||||
"""Property-based tests for NoteGraph Ingestion using Hypothesis.
|
||||
|
||||
All 18 correctness properties from the design document are tested here.
|
||||
Each test uses @settings(max_examples=100) as specified.
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import string
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from hypothesis import given, settings, assume, HealthCheck
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from ingestion.config import IngestionConfig
|
||||
from ingestion.discovery import SUPPORTED_EXTENSIONS, discover_files
|
||||
from ingestion.enrichment.models import ActionItem, EnrichmentResult, Entity
|
||||
from ingestion.extraction.text import PlainTextExtractor
|
||||
from ingestion.integrations.git import commit_imported_notes
|
||||
from ingestion.integrations.orgmylife import OrgMyLifeClient
|
||||
from ingestion.linking.linker import generate_wiki_links, _entity_to_wikilink
|
||||
from ingestion.output.naming import generate_filename, resolve_collision
|
||||
from ingestion.output.renderer import render_note
|
||||
from ingestion.output.router import CATEGORY_FOLDERS, route_to_folder
|
||||
from ingestion.watcher import should_ignore
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Common settings for all property tests
|
||||
# ---------------------------------------------------------------------------
|
||||
pbt_settings = settings(
|
||||
max_examples=100,
|
||||
suppress_health_check=[HealthCheck.function_scoped_fixture],
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Strategy for supported file extensions
|
||||
supported_ext_st = st.sampled_from(sorted(SUPPORTED_EXTENSIONS))
|
||||
|
||||
# Strategy for unsupported file extensions
|
||||
unsupported_ext_st = st.text(
|
||||
alphabet=string.ascii_lowercase, min_size=2, max_size=5
|
||||
).map(lambda s: f".{s}").filter(lambda ext: ext not in SUPPORTED_EXTENSIONS)
|
||||
|
||||
# Strategy for valid filenames (no path separators, no null bytes, no reserved chars)
|
||||
filename_chars = st.characters(
|
||||
whitelist_categories=("L", "N"),
|
||||
)
|
||||
filename_st = st.text(alphabet=filename_chars, min_size=1, max_size=20).map(
|
||||
lambda s: s.strip() or "file"
|
||||
)
|
||||
|
||||
# Strategy for valid UTF-8 text content (no null bytes, non-empty)
|
||||
utf8_text_st = st.text(
|
||||
alphabet=st.characters(blacklist_categories=("Cs",), blacklist_characters="\x00"),
|
||||
min_size=1,
|
||||
max_size=2000,
|
||||
)
|
||||
|
||||
# Strategy for entity types
|
||||
entity_type_st = st.sampled_from(["person", "project", "date", "action_item"])
|
||||
|
||||
# Strategy for confidence scores
|
||||
confidence_st = st.floats(min_value=0.0, max_value=1.0, allow_nan=False)
|
||||
|
||||
# Strategy for person/project names (non-empty, printable, with at least one letter)
|
||||
name_st = st.text(
|
||||
alphabet=st.characters(whitelist_categories=("L", "N", "Zs")),
|
||||
min_size=2,
|
||||
max_size=40,
|
||||
).filter(lambda s: s.strip() and any(c.isalpha() for c in s))
|
||||
|
||||
# Strategy for valid providers
|
||||
provider_st = st.sampled_from(["openai", "anthropic", "google", "mistral", "ollama"])
|
||||
|
||||
# Strategy for providers requiring API keys
|
||||
keyed_provider_st = st.sampled_from(["openai", "anthropic", "google", "mistral"])
|
||||
|
||||
# Strategy for categories
|
||||
category_st = st.sampled_from(["meeting", "project", "decision", "inbox"])
|
||||
|
||||
# Strategy for dates in YYYY-MM-DD format
|
||||
date_st = st.dates().map(lambda d: d.strftime("%Y-%m-%d"))
|
||||
|
||||
# Strategy for note titles (must produce a non-empty slug)
|
||||
title_st = st.text(
|
||||
alphabet=st.characters(whitelist_categories=("L", "N", "Zs")),
|
||||
min_size=2,
|
||||
max_size=60,
|
||||
).filter(lambda s: s.strip() and any(c.isalnum() for c in s))
|
||||
|
||||
# Strategy for action item descriptions
|
||||
action_desc_st = st.text(
|
||||
alphabet=st.characters(whitelist_categories=("L", "N", "Zs", "P")),
|
||||
min_size=3,
|
||||
max_size=100,
|
||||
).filter(lambda s: s.strip())
|
||||
|
||||
# Strategy for source types
|
||||
source_type_st = st.text(
|
||||
alphabet=string.ascii_lowercase + "-",
|
||||
min_size=3,
|
||||
max_size=20,
|
||||
).filter(lambda s: s.strip("-") and not s.startswith("-") and not s.endswith("-"))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 1 (Task 3.2): File discovery filtering
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 1: File discovery returns only supported extensions
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(
|
||||
supported_files=st.lists(
|
||||
st.tuples(filename_st, supported_ext_st), min_size=0, max_size=10
|
||||
),
|
||||
unsupported_files=st.lists(
|
||||
st.tuples(filename_st, unsupported_ext_st), min_size=0, max_size=10
|
||||
),
|
||||
)
|
||||
def test_discovery_returns_only_supported_extensions(
|
||||
tmp_path: Path, supported_files, unsupported_files
|
||||
):
|
||||
"""For any directory tree with arbitrary extensions, discovery returns exactly
|
||||
those with supported extensions (.pdf, .jpg, .jpeg, .png, .md, .txt, .docx)."""
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
td_path = Path(td)
|
||||
|
||||
# Create files with supported extensions
|
||||
expected = set()
|
||||
for i, (name, ext) in enumerate(supported_files):
|
||||
fp = td_path / f"s{i}_{name}{ext}"
|
||||
fp.write_text("content", encoding="utf-8")
|
||||
expected.add(fp)
|
||||
|
||||
# Create files with unsupported extensions
|
||||
for i, (name, ext) in enumerate(unsupported_files):
|
||||
fp = td_path / f"u{i}_{name}{ext}"
|
||||
fp.write_text("content", encoding="utf-8")
|
||||
|
||||
result = discover_files(td_path)
|
||||
result_set = set(result)
|
||||
|
||||
# All returned files must have supported extensions
|
||||
for f in result:
|
||||
assert f.suffix.lower() in SUPPORTED_EXTENSIONS
|
||||
|
||||
# All supported files we created must be in the result
|
||||
for f in expected:
|
||||
assert f in result_set
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 2 (Task 9.2): Bulk import resilience
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 2: Failed files do not block remaining processing
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(
|
||||
fail_indices=st.lists(st.integers(min_value=0, max_value=4), unique=True, max_size=3),
|
||||
)
|
||||
def test_bulk_import_resilience(tmp_path: Path, fail_indices, mock_config):
|
||||
"""For any batch where a subset fails, remaining files are processed successfully."""
|
||||
from ingestion.pipeline import process_batch
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
td_path = Path(td)
|
||||
|
||||
# Create 5 text files
|
||||
files = []
|
||||
for i in range(5):
|
||||
f = td_path / f"note_{i}.txt"
|
||||
f.write_text(f"Content of note {i}", encoding="utf-8")
|
||||
files.append(f)
|
||||
|
||||
# Normalize fail indices to valid range
|
||||
fail_set = {i for i in fail_indices if 0 <= i < 5}
|
||||
|
||||
# Mock enrichment agent that fails for specific indices
|
||||
call_count = {"n": 0}
|
||||
|
||||
def mock_enrich(text):
|
||||
idx = call_count["n"]
|
||||
call_count["n"] += 1
|
||||
if idx in fail_set:
|
||||
raise RuntimeError(f"Simulated failure for file {idx}")
|
||||
return EnrichmentResult(
|
||||
title=f"Note {idx}",
|
||||
category="inbox",
|
||||
tags=[],
|
||||
entities=[],
|
||||
action_items=[],
|
||||
summary=None,
|
||||
)
|
||||
|
||||
mock_agent = MagicMock()
|
||||
mock_agent.enrich.side_effect = mock_enrich
|
||||
|
||||
# Set up output directory
|
||||
notes_dir = td_path / "notes"
|
||||
notes_dir.mkdir()
|
||||
(notes_dir / "inbox").mkdir()
|
||||
mock_config.notes_dir = str(notes_dir)
|
||||
|
||||
with patch("ingestion.pipeline.EnrichmentAgent", return_value=mock_agent):
|
||||
result = process_batch(
|
||||
paths=files,
|
||||
config=mock_config,
|
||||
dry_run=False,
|
||||
no_commit=True,
|
||||
)
|
||||
|
||||
expected_success = 5 - len(fail_set)
|
||||
assert result.success == expected_success
|
||||
assert result.failed == len(fail_set)
|
||||
assert result.total == 5
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 3 (Task 2.6): Text extraction identity
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 3: Text extraction preserves content for text-based formats
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(content=utf8_text_st)
|
||||
def test_text_extraction_identity(tmp_path: Path, content: str):
|
||||
"""For any valid UTF-8 string written to a .md or .txt file, extraction returns
|
||||
identical content (modulo platform newline normalization)."""
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
td_path = Path(td)
|
||||
extractor = PlainTextExtractor()
|
||||
|
||||
for ext in (".md", ".txt"):
|
||||
fp = td_path / f"test{ext}"
|
||||
fp.write_text(content, encoding="utf-8")
|
||||
result = extractor.extract(fp)
|
||||
# Python's read_text uses universal newlines (translates \r\n and \r to \n)
|
||||
# This is expected platform behavior, so we compare after normalization
|
||||
expected = content.replace("\r\n", "\n").replace("\r", "\n")
|
||||
assert result.text == expected
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 4 (Task 5.4): Entity parsing valid structure
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 4: Entity parsing extracts all entities with valid structure
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(
|
||||
entities=st.lists(
|
||||
st.fixed_dictionaries({
|
||||
"type": entity_type_st,
|
||||
"value": name_st,
|
||||
"confidence": confidence_st,
|
||||
}),
|
||||
min_size=1,
|
||||
max_size=10,
|
||||
),
|
||||
)
|
||||
def test_entity_parsing_valid_structure(entities):
|
||||
"""For any valid enrichment response JSON, each parsed entity has valid type,
|
||||
non-empty value, confidence in [0.0, 1.0]."""
|
||||
data = {
|
||||
"title": "Test",
|
||||
"category": "inbox",
|
||||
"tags": [],
|
||||
"entities": entities,
|
||||
"action_items": [],
|
||||
}
|
||||
|
||||
result = EnrichmentResult.model_validate(data)
|
||||
|
||||
for entity in result.entities:
|
||||
assert entity.type in ("person", "project", "date", "action_item")
|
||||
assert len(entity.value) > 0
|
||||
assert 0.0 <= entity.confidence <= 1.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 5 (Task 5.4): Confidence threshold filtering
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 5: Confidence threshold filtering
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(
|
||||
scores=st.lists(confidence_st, min_size=1, max_size=20),
|
||||
)
|
||||
def test_confidence_threshold_filtering(scores):
|
||||
"""For any list of entities with arbitrary confidence scores, filtered output
|
||||
contains exactly those with confidence >= 0.7."""
|
||||
entities = [
|
||||
Entity(type="person", value=f"Person {i}", confidence=score)
|
||||
for i, score in enumerate(scores)
|
||||
]
|
||||
|
||||
threshold = 0.7
|
||||
filtered = [e for e in entities if e.confidence >= threshold]
|
||||
|
||||
# Verify the filter matches what the enrichment agent does
|
||||
from ingestion.enrichment.agent import EnrichmentAgent
|
||||
|
||||
agent = MagicMock(spec=EnrichmentAgent)
|
||||
agent.confidence_threshold = threshold
|
||||
agent._filter_by_confidence = EnrichmentAgent._filter_by_confidence.__get__(agent)
|
||||
|
||||
result = agent._filter_by_confidence(entities)
|
||||
|
||||
assert len(result) == len(filtered)
|
||||
for entity in result:
|
||||
assert entity.confidence >= threshold
|
||||
|
||||
# Verify no entity below threshold is included
|
||||
for entity in entities:
|
||||
if entity.confidence < threshold:
|
||||
assert entity not in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 6 (Task 7.3): Wiki-link format per entity type
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 6: Wiki-link generation follows correct format per entity type
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(name=name_st)
|
||||
def test_wikilink_format_person(name: str):
|
||||
"""For any person name → [[people/slugified-name]]."""
|
||||
from slugify import slugify
|
||||
|
||||
entity = Entity(type="person", value=name, confidence=0.9)
|
||||
link = _entity_to_wikilink(entity)
|
||||
|
||||
expected_slug = slugify(name)
|
||||
assume(expected_slug) # skip if slugify produces empty string
|
||||
|
||||
assert link == f"[[people/{expected_slug}]]"
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(name=name_st)
|
||||
def test_wikilink_format_project(name: str):
|
||||
"""For any project name → [[projects/slugified-name]]."""
|
||||
from slugify import slugify
|
||||
|
||||
entity = Entity(type="project", value=name, confidence=0.9)
|
||||
link = _entity_to_wikilink(entity)
|
||||
|
||||
expected_slug = slugify(name)
|
||||
assume(expected_slug) # skip if slugify produces empty string
|
||||
|
||||
assert link == f"[[projects/{expected_slug}]]"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 7 (Task 7.3): Only first occurrence linked
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 7: Only first entity occurrence is linked
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(
|
||||
name=name_st,
|
||||
n_occurrences=st.integers(min_value=2, max_value=5),
|
||||
)
|
||||
def test_only_first_occurrence_linked(name: str, n_occurrences: int):
|
||||
"""For any text with N >= 2 occurrences of the same entity, only the first
|
||||
is converted to a wiki-link."""
|
||||
from slugify import slugify
|
||||
|
||||
slug = slugify(name)
|
||||
assume(slug) # skip if slugify produces empty string
|
||||
|
||||
# Build text with multiple occurrences separated by unique text
|
||||
separator = ". Then we discussed "
|
||||
text = separator.join([name] * n_occurrences)
|
||||
|
||||
entity = Entity(type="person", value=name, confidence=0.9)
|
||||
result = generate_wiki_links(text, [entity])
|
||||
|
||||
expected_link = f"[[people/{slug}]]"
|
||||
|
||||
# First occurrence should be a wiki-link
|
||||
assert expected_link in result
|
||||
|
||||
# Count wiki-links — should be exactly 1
|
||||
link_count = result.count(expected_link)
|
||||
assert link_count == 1, f"Expected 1 wiki-link, found {link_count}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 8 (Task 8.4): Frontmatter round-trip validity
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 8: Frontmatter round-trip validity
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(
|
||||
title=title_st,
|
||||
category=category_st,
|
||||
tags=st.lists(st.text(alphabet=string.ascii_lowercase, min_size=2, max_size=10), max_size=5),
|
||||
people=st.lists(name_st, min_size=0, max_size=3),
|
||||
projects=st.lists(name_st, min_size=0, max_size=3),
|
||||
)
|
||||
def test_frontmatter_roundtrip_validity(
|
||||
tmp_path: Path, title, category, tags, people, projects
|
||||
):
|
||||
"""For any valid EnrichmentResult, render to markdown then parse frontmatter →
|
||||
valid YAML with required fields."""
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
td_path = Path(td)
|
||||
|
||||
entities = []
|
||||
for p in people:
|
||||
entities.append(Entity(type="person", value=p, confidence=0.9))
|
||||
for proj in projects:
|
||||
entities.append(Entity(type="project", value=proj, confidence=0.9))
|
||||
entities.append(Entity(type="date", value="2024-06-15", confidence=0.99))
|
||||
|
||||
enrichment = EnrichmentResult(
|
||||
title=title,
|
||||
category=category,
|
||||
tags=tags,
|
||||
entities=entities,
|
||||
action_items=[],
|
||||
summary=None,
|
||||
)
|
||||
|
||||
source_file = td_path / "source.pdf"
|
||||
source_file.write_text("", encoding="utf-8")
|
||||
|
||||
rendered = render_note(enrichment, "Body text here.", source_file)
|
||||
|
||||
# Parse frontmatter
|
||||
assert rendered.startswith("---\n")
|
||||
parts = rendered.split("---\n", 2)
|
||||
assert len(parts) >= 3, "Expected YAML frontmatter delimiters"
|
||||
|
||||
frontmatter_yaml = parts[1]
|
||||
parsed = yaml.safe_load(frontmatter_yaml)
|
||||
|
||||
# Required fields
|
||||
assert "title" in parsed
|
||||
assert "date" in parsed
|
||||
assert "source" in parsed
|
||||
assert "file" in parsed["source"]
|
||||
assert "imported" in parsed["source"]
|
||||
|
||||
# Title matches
|
||||
assert parsed["title"] == title
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 9 (Task 4.2): Provider routing from configuration
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 9: Provider routing from configuration
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(provider=provider_st)
|
||||
def test_provider_routing_from_configuration(provider: str):
|
||||
"""For any valid provider name (openai, anthropic, google, mistral, ollama),
|
||||
configuring LLM_PROVIDER produces the correct model string."""
|
||||
config = IngestionConfig(
|
||||
llm_provider=provider,
|
||||
llm_model="test-model",
|
||||
openai_api_key="fake",
|
||||
anthropic_api_key="fake",
|
||||
google_api_key="fake",
|
||||
mistral_api_key="fake",
|
||||
)
|
||||
|
||||
model_string = config.llm_model_string
|
||||
|
||||
if provider == "ollama":
|
||||
assert model_string == "ollama/test-model"
|
||||
else:
|
||||
assert model_string == f"{provider}/test-model"
|
||||
|
||||
# Verify the provider prefix is correct
|
||||
assert model_string.startswith(f"{provider}/")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 10 (Task 4.3): Missing API key error
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 10: Missing API key produces descriptive error
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(provider=keyed_provider_st)
|
||||
def test_missing_api_key_error(provider: str):
|
||||
"""For any provider requiring an API key, when the key is empty,
|
||||
validate_api_key() raises SystemExit naming the missing variable."""
|
||||
config = IngestionConfig(
|
||||
llm_provider=provider,
|
||||
llm_model="test-model",
|
||||
openai_api_key="",
|
||||
anthropic_api_key="",
|
||||
google_api_key="",
|
||||
mistral_api_key="",
|
||||
)
|
||||
|
||||
env_var_map = {
|
||||
"openai": "OPENAI_API_KEY",
|
||||
"anthropic": "ANTHROPIC_API_KEY",
|
||||
"google": "GOOGLE_API_KEY",
|
||||
"mistral": "MISTRAL_API_KEY",
|
||||
}
|
||||
|
||||
with pytest.raises(SystemExit) as exc_info:
|
||||
config.validate_api_key()
|
||||
|
||||
error_msg = str(exc_info.value)
|
||||
assert env_var_map[provider] in error_msg
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 11 (Task 8.4): Action items in output
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 11: Action items appear in output markdown section
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(
|
||||
descriptions=st.lists(action_desc_st, min_size=1, max_size=5),
|
||||
)
|
||||
def test_action_items_in_output(tmp_path: Path, descriptions):
|
||||
"""For any list of action items, rendered markdown contains `## Action Items`
|
||||
section listing every item."""
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
td_path = Path(td)
|
||||
|
||||
action_items = [ActionItem(description=desc) for desc in descriptions]
|
||||
|
||||
enrichment = EnrichmentResult(
|
||||
title="Test Note",
|
||||
category="inbox",
|
||||
tags=[],
|
||||
entities=[Entity(type="date", value="2024-01-01", confidence=0.99)],
|
||||
action_items=action_items,
|
||||
summary=None,
|
||||
)
|
||||
|
||||
source_file = td_path / "source.txt"
|
||||
source_file.write_text("", encoding="utf-8")
|
||||
|
||||
rendered = render_note(enrichment, "Body text.", source_file)
|
||||
|
||||
assert "## Action Items" in rendered
|
||||
|
||||
for desc in descriptions:
|
||||
assert desc in rendered
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 12 (Task 15.2): OrgMyLife task payload
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 12: OrgMyLife task payload correctness
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(
|
||||
description=action_desc_st,
|
||||
note_path=st.text(
|
||||
alphabet=string.ascii_lowercase + "/.-_", min_size=5, max_size=50
|
||||
).filter(lambda s: s.strip() and not s.startswith("/")),
|
||||
)
|
||||
def test_orgmylife_task_payload(description: str, note_path: str):
|
||||
"""For any action item description and note path, the task creation payload
|
||||
has correct title and source_url."""
|
||||
import httpx
|
||||
|
||||
client = OrgMyLifeClient(base_url="https://api.example.com", api_key="test-key")
|
||||
|
||||
# Capture the payload sent to httpx.post
|
||||
captured_payload = {}
|
||||
|
||||
def mock_post(url, json=None, headers=None, timeout=None):
|
||||
captured_payload.update(json)
|
||||
response = MagicMock()
|
||||
response.status_code = 201
|
||||
response.raise_for_status = MagicMock()
|
||||
response.json.return_value = {"id": 1, "title": json["title"]}
|
||||
return response
|
||||
|
||||
with patch.object(httpx, "post", side_effect=mock_post):
|
||||
client.create_task(title=description, source_url=note_path)
|
||||
|
||||
assert captured_payload["title"] == description
|
||||
assert captured_payload["source_url"] == note_path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 13 (Task 8.4): Category-based folder routing
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 13: Category-based folder routing
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(category=category_st)
|
||||
def test_category_based_folder_routing(tmp_path: Path, category: str):
|
||||
"""For any category (meeting/project/decision/inbox), output path is within
|
||||
the correct folder."""
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
notes_dir = Path(td) / "notes"
|
||||
notes_dir.mkdir()
|
||||
|
||||
result = route_to_folder(category, notes_dir)
|
||||
|
||||
expected_folder = CATEGORY_FOLDERS.get(category, "inbox")
|
||||
assert result == notes_dir / expected_folder
|
||||
assert result.exists()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 14 (Task 8.4): Filename date-slug pattern
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 14: Output filename follows date-slug pattern
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(date=date_st, title=title_st)
|
||||
def test_filename_date_slug_pattern(date: str, title: str):
|
||||
"""For any date and title, filename matches YYYY-MM-DD-slugified-title.md."""
|
||||
filename = generate_filename(title, date)
|
||||
|
||||
# Must end with .md
|
||||
assert filename.endswith(".md")
|
||||
|
||||
# Must start with the date
|
||||
assert filename.startswith(date)
|
||||
|
||||
# Pattern: YYYY-MM-DD-slug.md (slug is lowercase alphanumeric + hyphens)
|
||||
pattern = r"^\d{4}-\d{2}-\d{2}-[a-z0-9]([a-z0-9-]*[a-z0-9])?\.md$"
|
||||
assert re.match(pattern, filename), f"Filename '{filename}' doesn't match pattern"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 15 (Task 8.4): Filename collision avoidance
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 15: Filename collision avoidance
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(
|
||||
title=title_st,
|
||||
n_existing=st.integers(min_value=1, max_value=5),
|
||||
)
|
||||
def test_filename_collision_avoidance(tmp_path: Path, title: str, n_existing: int):
|
||||
"""When a file with the same name exists, a numeric suffix is appended."""
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
td_path = Path(td)
|
||||
|
||||
filename = generate_filename(title, "2024-01-15")
|
||||
|
||||
# Create the original file
|
||||
(td_path / filename).write_text("existing", encoding="utf-8")
|
||||
|
||||
# Create collision files with suffixes
|
||||
stem = filename.rsplit(".md", 1)[0]
|
||||
for i in range(2, n_existing + 1):
|
||||
(td_path / f"{stem}-{i}.md").write_text("existing", encoding="utf-8")
|
||||
|
||||
# Resolve collision
|
||||
resolved = resolve_collision(td_path, filename)
|
||||
|
||||
# The resolved path must not exist yet
|
||||
assert not resolved.exists()
|
||||
|
||||
# The resolved path must have a numeric suffix
|
||||
expected_suffix = f"-{n_existing + 1}.md"
|
||||
assert str(resolved).endswith(expected_suffix)
|
||||
|
||||
# The resolved path must be in the same directory
|
||||
assert resolved.parent == td_path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 16 (Task 14.2): Temporary file filtering
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 16: Temporary file filtering
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(
|
||||
filename=st.text(
|
||||
alphabet=st.characters(
|
||||
whitelist_categories=("L", "N", "P"),
|
||||
blacklist_characters="/\\:\x00",
|
||||
),
|
||||
min_size=1,
|
||||
max_size=50,
|
||||
),
|
||||
)
|
||||
def test_temporary_file_filtering(filename: str):
|
||||
"""For any filename, should_ignore returns true iff name starts with '.'
|
||||
or ends with '.tmp'."""
|
||||
result = should_ignore(filename)
|
||||
expected = filename.startswith(".") or filename.endswith(".tmp")
|
||||
assert result == expected, f"should_ignore('{filename}') = {result}, expected {expected}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 17 (Task 9.2): Dry-run no file writes
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 17: Dry-run produces no file writes
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(
|
||||
n_files=st.integers(min_value=1, max_value=5),
|
||||
)
|
||||
def test_dry_run_no_file_writes(tmp_path: Path, n_files: int, mock_config):
|
||||
"""For any input with dry_run=True, zero files are created."""
|
||||
from ingestion.pipeline import process_batch
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
td_path = Path(td)
|
||||
|
||||
# Create input files
|
||||
files = []
|
||||
for i in range(n_files):
|
||||
f = td_path / f"input_{i}.txt"
|
||||
f.write_text(f"Content {i}", encoding="utf-8")
|
||||
files.append(f)
|
||||
|
||||
# Set up output directory
|
||||
notes_dir = td_path / "output_notes"
|
||||
notes_dir.mkdir()
|
||||
(notes_dir / "inbox").mkdir()
|
||||
mock_config.notes_dir = str(notes_dir)
|
||||
|
||||
# Mock the enrichment agent
|
||||
def mock_enrich(text):
|
||||
return EnrichmentResult(
|
||||
title="Test",
|
||||
category="inbox",
|
||||
tags=[],
|
||||
entities=[],
|
||||
action_items=[],
|
||||
summary=None,
|
||||
)
|
||||
|
||||
mock_agent = MagicMock()
|
||||
mock_agent.enrich.side_effect = mock_enrich
|
||||
|
||||
with patch("ingestion.pipeline.EnrichmentAgent", return_value=mock_agent):
|
||||
result = process_batch(
|
||||
paths=files,
|
||||
config=mock_config,
|
||||
dry_run=True,
|
||||
no_commit=True,
|
||||
)
|
||||
|
||||
# Count files in the output notes directory (should be 0 new markdown files)
|
||||
output_files = list(notes_dir.rglob("*.md"))
|
||||
assert len(output_files) == 0, f"Expected 0 files written, found {len(output_files)}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 18 (Task 16.2): Git commit message format
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 18: Git commit message format
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(
|
||||
count=st.integers(min_value=1, max_value=1000),
|
||||
source_type=source_type_st,
|
||||
)
|
||||
def test_git_commit_message_format(count: int, source_type: str):
|
||||
"""For any count N and source type, message matches
|
||||
'ingestion: import N notes from [source-type]'."""
|
||||
files = [Path(f"/fake/note_{i}.md") for i in range(count)]
|
||||
captured_messages = []
|
||||
|
||||
def mock_run(cmd, **kwargs):
|
||||
if cmd[0] == "git" and cmd[1] == "commit":
|
||||
# Extract -m argument
|
||||
msg_idx = cmd.index("-m") + 1
|
||||
captured_messages.append(cmd[msg_idx])
|
||||
result = MagicMock()
|
||||
result.returncode = 0
|
||||
result.stdout = ""
|
||||
result.stderr = ""
|
||||
return result
|
||||
|
||||
with patch("subprocess.run", side_effect=mock_run):
|
||||
commit_imported_notes(files, source_type=source_type)
|
||||
|
||||
assert len(captured_messages) == 1
|
||||
expected = f"ingestion: import {count} notes from {source_type}"
|
||||
assert captured_messages[0] == expected
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Drop-folder watcher using watchdog for continuous file ingestion.
|
||||
|
||||
Monitors the inbox folder for new files and triggers the processing pipeline.
|
||||
Moves processed files to archive/ and failed files to failed/ subdirectory.
|
||||
Ignores temporary files (names starting with '.' or ending with '.tmp').
|
||||
"""
|
||||
|
||||
import logging
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from watchdog.events import FileCreatedEvent, FileSystemEventHandler
|
||||
from watchdog.observers import Observer
|
||||
|
||||
from ingestion.config import IngestionConfig
|
||||
from ingestion.discovery import SUPPORTED_EXTENSIONS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def should_ignore(filename: str) -> bool:
|
||||
"""Check if a file should be ignored by the watcher.
|
||||
|
||||
Ignores files starting with '.' or ending with '.tmp'.
|
||||
"""
|
||||
return filename.startswith(".") or filename.endswith(".tmp")
|
||||
|
||||
|
||||
class InboxHandler(FileSystemEventHandler):
|
||||
"""Handle new files in the inbox directory."""
|
||||
|
||||
def __init__(self, config: IngestionConfig):
|
||||
self.config = config
|
||||
self.inbox_dir = Path(config.inbox_dir)
|
||||
self.archive_dir = self.inbox_dir / "archive"
|
||||
self.failed_dir = self.inbox_dir / "failed"
|
||||
self.archive_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.failed_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def on_created(self, event: FileCreatedEvent):
|
||||
"""Triggered when a new file is created in the inbox."""
|
||||
if event.is_directory:
|
||||
return
|
||||
|
||||
file_path = Path(event.src_path)
|
||||
self._process_file(file_path)
|
||||
|
||||
def _process_file(self, file_path: Path) -> None:
|
||||
"""Process a single file and move to archive or failed."""
|
||||
if should_ignore(file_path.name):
|
||||
logger.debug("Ignoring temporary file: %s", file_path.name)
|
||||
return
|
||||
|
||||
if file_path.suffix.lower() not in SUPPORTED_EXTENSIONS:
|
||||
logger.debug("Ignoring unsupported file: %s", file_path.name)
|
||||
return
|
||||
|
||||
# Small delay to ensure file is fully written
|
||||
time.sleep(0.5)
|
||||
|
||||
logger.info("Processing new file: %s", file_path.name)
|
||||
|
||||
try:
|
||||
from ingestion.pipeline import process_file
|
||||
|
||||
result = process_file(file_path=file_path, config=self.config)
|
||||
|
||||
if result.success:
|
||||
# Move to archive
|
||||
dest = self.archive_dir / file_path.name
|
||||
shutil.move(str(file_path), str(dest))
|
||||
logger.info("Archived: %s → %s", file_path.name, dest)
|
||||
else:
|
||||
# Move to failed
|
||||
dest = self.failed_dir / file_path.name
|
||||
shutil.move(str(file_path), str(dest))
|
||||
logger.warning(
|
||||
"Failed: %s → %s (error: %s)",
|
||||
file_path.name,
|
||||
dest,
|
||||
result.error,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Unhandled error processing %s: %s", file_path.name, e)
|
||||
dest = self.failed_dir / file_path.name
|
||||
try:
|
||||
shutil.move(str(file_path), str(dest))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def start_watcher(config: IngestionConfig) -> None:
|
||||
"""Start the drop-folder watcher.
|
||||
|
||||
Processes existing files first, then watches for new ones.
|
||||
Blocks until interrupted (Ctrl+C).
|
||||
"""
|
||||
inbox_dir = Path(config.inbox_dir)
|
||||
inbox_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
handler = InboxHandler(config)
|
||||
|
||||
# Process existing files on startup
|
||||
existing = sorted(inbox_dir.iterdir())
|
||||
for f in existing:
|
||||
if f.is_file() and not should_ignore(f.name):
|
||||
handler._process_file(f)
|
||||
|
||||
# Start watching
|
||||
observer = Observer()
|
||||
observer.schedule(handler, str(inbox_dir), recursive=False)
|
||||
observer.start()
|
||||
|
||||
logger.info("Watcher started on: %s", inbox_dir)
|
||||
|
||||
try:
|
||||
while True:
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
observer.stop()
|
||||
|
||||
observer.join()
|
||||
@@ -0,0 +1 @@
|
||||
"""FastAPI web upload endpoint for mobile file ingestion."""
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,233 @@
|
||||
"""FastAPI web upload endpoint for mobile file ingestion.
|
||||
|
||||
Auth: Token-based via URL query param (?token=secret) or header (X-Upload-Token).
|
||||
Processing: File is saved immediately, enrichment runs in background.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import secrets
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI, File, HTTPException, Query, Request, UploadFile
|
||||
from fastapi.responses import HTMLResponse
|
||||
|
||||
from ingestion.config import IngestionConfig
|
||||
from ingestion.web.browse import router as browse_router
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
app = FastAPI(title="NoteGraph", version="0.1.0")
|
||||
app.include_router(browse_router)
|
||||
|
||||
|
||||
@app.get("/")
|
||||
async def root(request: Request):
|
||||
"""Redirect root to browse page."""
|
||||
from fastapi.responses import RedirectResponse
|
||||
token = request.query_params.get("token", "")
|
||||
return RedirectResponse(url=f"/browse?token={token}" if token else "/browse")
|
||||
|
||||
|
||||
def _get_config() -> IngestionConfig:
|
||||
return IngestionConfig()
|
||||
|
||||
|
||||
def _verify_token(request: Request, token: str = Query(default=None)):
|
||||
"""Verify upload token from query param or X-Upload-Token header."""
|
||||
config = _get_config()
|
||||
# Token is the password part of SB_USER (after the colon)
|
||||
sb_user = config.sb_user.strip()
|
||||
parts = sb_user.split(":", 1)
|
||||
expected_token = parts[1].strip() if len(parts) == 2 else sb_user
|
||||
|
||||
# Check query param
|
||||
if token and secrets.compare_digest(token.encode(), expected_token.encode()):
|
||||
return True
|
||||
|
||||
# Check header
|
||||
header_token = request.headers.get("X-Upload-Token", "")
|
||||
if header_token and secrets.compare_digest(header_token.encode(), expected_token.encode()):
|
||||
return True
|
||||
|
||||
raise HTTPException(status_code=401, detail="Invalid or missing token. Use ?token=YOUR_PASSWORD")
|
||||
|
||||
|
||||
def _process_in_background(file_path: Path, config: IngestionConfig):
|
||||
"""Run the enrichment pipeline in a background thread."""
|
||||
try:
|
||||
from ingestion.pipeline import process_file
|
||||
result = process_file(file_path=file_path, config=config)
|
||||
if result.success:
|
||||
logger.info("Background processing complete: %s → %s", file_path.name, result.output_file)
|
||||
else:
|
||||
logger.warning("Background processing failed for %s: %s", file_path.name, result.error)
|
||||
except Exception as e:
|
||||
logger.error("Background processing error for %s: %s", file_path.name, e)
|
||||
|
||||
|
||||
UPLOAD_HTML = """<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>NoteGraph Upload</title>
|
||||
<style>
|
||||
* { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
body {
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
||||
background: #1a1a2e;
|
||||
color: #eee;
|
||||
min-height: 100vh;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 1rem;
|
||||
}
|
||||
.container {
|
||||
background: #16213e;
|
||||
border-radius: 12px;
|
||||
padding: 2rem;
|
||||
width: 100%;
|
||||
max-width: 480px;
|
||||
box-shadow: 0 8px 32px rgba(0,0,0,0.3);
|
||||
}
|
||||
h1 { text-align: center; margin-bottom: 1.5rem; font-size: 1.5rem; color: #4fc3f7; }
|
||||
.drop-zone {
|
||||
border: 2px dashed #4fc3f7;
|
||||
border-radius: 8px;
|
||||
padding: 2rem 1rem;
|
||||
text-align: center;
|
||||
cursor: pointer;
|
||||
transition: all 0.3s;
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
.drop-zone:hover, .drop-zone.dragover {
|
||||
background: rgba(79, 195, 247, 0.1);
|
||||
border-color: #81d4fa;
|
||||
}
|
||||
.drop-zone p { margin-bottom: 0.5rem; font-size: 1.1rem; }
|
||||
.drop-zone small { color: #999; }
|
||||
input[type="file"] { display: none; }
|
||||
.btn {
|
||||
display: block; width: 100%; padding: 0.9rem;
|
||||
background: #4fc3f7; color: #1a1a2e; border: none;
|
||||
border-radius: 8px; font-size: 1rem; font-weight: 600;
|
||||
cursor: pointer; transition: background 0.3s;
|
||||
}
|
||||
.btn:hover { background: #81d4fa; }
|
||||
.btn:disabled { background: #555; cursor: not-allowed; }
|
||||
.status { margin-top: 1rem; padding: 0.8rem; border-radius: 6px; text-align: center; display: none; }
|
||||
.status.success { display: block; background: #1b5e20; color: #a5d6a7; }
|
||||
.status.error { display: block; background: #b71c1c; color: #ef9a9a; }
|
||||
.status.loading { display: block; background: #0d47a1; color: #90caf9; }
|
||||
.filename { margin-top: 0.5rem; font-size: 0.9rem; color: #aaa; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<h1>📄 NoteGraph Upload</h1>
|
||||
<form id="uploadForm" enctype="multipart/form-data">
|
||||
<div class="drop-zone" id="dropZone">
|
||||
<p>Drop file here or tap to select</p>
|
||||
<small>PDF, JPG, PNG, DOCX, TXT, MD</small>
|
||||
</div>
|
||||
<input type="file" id="fileInput" name="file" accept=".pdf,.jpg,.jpeg,.png,.docx,.txt,.md">
|
||||
<div class="filename" id="fileName"></div>
|
||||
<button type="submit" class="btn" id="submitBtn" disabled>Upload & Process</button>
|
||||
</form>
|
||||
<div class="status" id="status"></div>
|
||||
</div>
|
||||
<script>
|
||||
const dropZone = document.getElementById('dropZone');
|
||||
const fileInput = document.getElementById('fileInput');
|
||||
const form = document.getElementById('uploadForm');
|
||||
const submitBtn = document.getElementById('submitBtn');
|
||||
const statusEl = document.getElementById('status');
|
||||
const fileNameEl = document.getElementById('fileName');
|
||||
|
||||
// Get token from URL
|
||||
const urlParams = new URLSearchParams(window.location.search);
|
||||
const token = urlParams.get('token') || '';
|
||||
|
||||
dropZone.addEventListener('click', () => fileInput.click());
|
||||
dropZone.addEventListener('dragover', (e) => { e.preventDefault(); dropZone.classList.add('dragover'); });
|
||||
dropZone.addEventListener('dragleave', () => dropZone.classList.remove('dragover'));
|
||||
dropZone.addEventListener('drop', (e) => {
|
||||
e.preventDefault(); dropZone.classList.remove('dragover');
|
||||
if (e.dataTransfer.files.length) { fileInput.files = e.dataTransfer.files; updateFileName(); }
|
||||
});
|
||||
fileInput.addEventListener('change', updateFileName);
|
||||
|
||||
function updateFileName() {
|
||||
if (fileInput.files.length) { fileNameEl.textContent = fileInput.files[0].name; submitBtn.disabled = false; }
|
||||
}
|
||||
|
||||
form.addEventListener('submit', async (e) => {
|
||||
e.preventDefault();
|
||||
if (!fileInput.files.length) return;
|
||||
submitBtn.disabled = true;
|
||||
statusEl.className = 'status loading';
|
||||
statusEl.textContent = 'Uploading...';
|
||||
statusEl.style.display = 'block';
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append('file', fileInput.files[0]);
|
||||
|
||||
try {
|
||||
const resp = await fetch('/upload?token=' + encodeURIComponent(token), { method: 'POST', body: formData });
|
||||
const data = await resp.json();
|
||||
if (resp.ok) {
|
||||
statusEl.className = 'status success';
|
||||
statusEl.textContent = data.message || 'Upload successful!';
|
||||
} else {
|
||||
statusEl.className = 'status error';
|
||||
statusEl.textContent = data.detail || 'Upload failed';
|
||||
}
|
||||
} catch (err) {
|
||||
statusEl.className = 'status error';
|
||||
statusEl.textContent = 'Network error: ' + err.message;
|
||||
}
|
||||
submitBtn.disabled = false;
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
|
||||
@app.get("/", response_class=HTMLResponse)
|
||||
@app.get("/upload", response_class=HTMLResponse)
|
||||
async def upload_form(request: Request, token: str = Query(default=None)):
|
||||
"""Serve the upload form. Token required."""
|
||||
_verify_token(request, token)
|
||||
return UPLOAD_HTML
|
||||
|
||||
|
||||
@app.post("/upload")
|
||||
async def upload_file(
|
||||
request: Request,
|
||||
file: UploadFile = File(...),
|
||||
token: str = Query(default=None),
|
||||
):
|
||||
"""Accept file upload, save immediately, process in background."""
|
||||
_verify_token(request, token)
|
||||
|
||||
config = _get_config()
|
||||
inbox = Path(config.inbox_dir)
|
||||
inbox.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Save uploaded file
|
||||
target = inbox / file.filename
|
||||
content = await file.read()
|
||||
target.write_bytes(content)
|
||||
logger.info("Uploaded file saved: %s (%d bytes)", target, len(content))
|
||||
|
||||
# Process in background (don't block the response)
|
||||
thread = threading.Thread(
|
||||
target=_process_in_background,
|
||||
args=(target, config),
|
||||
daemon=True,
|
||||
)
|
||||
thread.start()
|
||||
|
||||
return {"message": f"File received: {file.filename}. Processing in background — check SilverBullet in ~30s."}
|
||||
Reference in New Issue
Block a user