feat: group private apps into andreknie-privat

This commit is contained in:
2026-07-25 15:47:01 +02:00
parent 4c7d48ae6d
commit 2e83750cb7
109 changed files with 0 additions and 0 deletions
@@ -0,0 +1,33 @@
FROM python:3.11-slim
# Install system dependencies for OCR
RUN apt-get update && apt-get install -y --no-install-recommends \
tesseract-ocr \
tesseract-ocr-deu \
git \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
# Copy pyproject.toml first for dependency caching
COPY pyproject.toml .
# Install dependencies only (not the package itself yet)
RUN pip install --no-cache-dir \
click watchdog pytesseract pdfplumber python-docx litellm httpx \
pydantic pydantic-settings fastapi uvicorn python-multipart pyyaml python-slugify
# Copy source code into /app/ingestion/ so imports work
COPY . /app/ingestion/
# Set PYTHONPATH so "import ingestion" resolves to /app/ingestion/
ENV PYTHONPATH=/app
# Create inbox directory
RUN mkdir -p /app/inbox
# Expose upload server port
EXPOSE 8001
# Default: run the upload web server
CMD ["uvicorn", "ingestion.web.upload:app", "--host", "0.0.0.0", "--port", "8001"]
@@ -0,0 +1,3 @@
"""NoteGraph document ingestion and AI-powered auto-referencing pipeline."""
__version__ = "0.1.0"
@@ -0,0 +1,5 @@
"""Entry point for `python -m ingestion`."""
from ingestion.cli import cli
cli()
@@ -0,0 +1,169 @@
"""Click-based CLI for the NoteGraph ingestion pipeline.
Subcommands:
import - Import files or directories into NoteGraph
watch - Start drop-folder watcher for continuous ingestion
sync - One-shot Nextcloud inbox sync
upload - Start web upload server
"""
import logging
import sys
from pathlib import Path
import click
from ingestion.config import IngestionConfig
def _setup_logging(verbose: bool) -> None:
"""Configure logging based on verbosity."""
level = logging.DEBUG if verbose else logging.INFO
logging.basicConfig(
level=level,
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
datefmt="%H:%M:%S",
)
@click.group()
@click.version_option(version="0.1.0", prog_name="notegraph-ingest")
def cli():
"""NoteGraph Ingestion — import documents into your knowledge base."""
pass
@cli.command("import")
@click.argument("path", type=click.Path(exists=True))
@click.option("--output-dir", "-o", type=click.Path(), default=None, help="Override output directory")
@click.option("--create-tasks", is_flag=True, help="Create OrgMyLife tasks for action items")
@click.option("--provider", type=str, default=None, help="Override LLM provider (openai, anthropic, etc.)")
@click.option("--dry-run", is_flag=True, help="Show what would be created without writing files")
@click.option("--no-commit", is_flag=True, help="Skip git auto-commit")
@click.option("--verbose", "-v", is_flag=True, help="Enable debug logging")
def import_cmd(path, output_dir, create_tasks, provider, dry_run, no_commit, verbose):
"""Import files or directories into NoteGraph.
PATH can be a single file or a directory to scan recursively.
"""
_setup_logging(verbose)
config = IngestionConfig()
if provider:
config.llm_provider = provider
config.validate_api_key()
from ingestion.discovery import discover_files
from ingestion.pipeline import process_batch
source_path = Path(path)
files = discover_files(source_path)
if not files:
click.echo("No supported files found.")
sys.exit(0)
click.echo(f"Found {len(files)} file(s) to process.")
if dry_run:
click.echo("[DRY RUN] No files will be written.\n")
def progress(current, total, filename):
click.echo(f" [{current}/{total}] {filename}")
result = process_batch(
paths=files,
config=config,
dry_run=dry_run,
no_commit=no_commit,
create_tasks=create_tasks,
output_dir=Path(output_dir) if output_dir else None,
progress_callback=progress,
)
# Summary
click.echo("")
click.echo(f"Done! Processed {result.total} file(s):")
click.echo(f" ✓ Success: {result.success}")
if result.failed:
click.echo(f" ✗ Failed: {result.failed}")
@cli.command("watch")
@click.option("--verbose", "-v", is_flag=True, help="Enable debug logging")
def watch_cmd(verbose):
"""Start drop-folder watcher for continuous ingestion."""
_setup_logging(verbose)
from ingestion.watcher import start_watcher
config = IngestionConfig()
config.validate_api_key()
click.echo(f"Watching inbox: {config.inbox_dir}")
click.echo("Press Ctrl+C to stop.\n")
try:
start_watcher(config)
except KeyboardInterrupt:
click.echo("\nWatcher stopped.")
@cli.command("sync")
@click.option("--verbose", "-v", is_flag=True, help="Enable debug logging")
def sync_cmd(verbose):
"""One-shot Nextcloud inbox sync."""
_setup_logging(verbose)
from ingestion.integrations.nextcloud import NextcloudInboxSync
config = IngestionConfig()
if not config.nextcloud_inbox_url:
click.echo("ERROR: NEXTCLOUD_INBOX_URL not configured.")
sys.exit(1)
config.validate_api_key()
sync = NextcloudInboxSync(config)
downloaded = sync.poll()
if not downloaded:
click.echo("No new files in Nextcloud inbox.")
return
click.echo(f"Downloaded {len(downloaded)} file(s) from Nextcloud.")
# Process downloaded files
from ingestion.pipeline import process_batch
result = process_batch(
paths=downloaded,
config=config,
)
# Mark processed files on Nextcloud
for r in result.results:
if r.success:
sync.mark_processed(r.source_file.name)
click.echo(f"Processed: {result.success} success, {result.failed} failed.")
@cli.command("upload")
@click.option("--host", default="0.0.0.0", help="Bind host")
@click.option("--port", default=8001, type=int, help="Bind port")
@click.option("--verbose", "-v", is_flag=True, help="Enable debug logging")
def upload_cmd(host, port, verbose):
"""Start web upload server for mobile file ingestion."""
_setup_logging(verbose)
import uvicorn
click.echo(f"Starting upload server on {host}:{port}")
uvicorn.run("ingestion.web.upload:app", host=host, port=port, log_level="info")
if __name__ == "__main__":
cli()
@@ -0,0 +1,66 @@
"""Configuration for the NoteGraph ingestion service."""
from pathlib import Path
from pydantic_settings import BaseSettings
class IngestionConfig(BaseSettings):
"""All configuration loaded from environment variables / .env file."""
# LLM Provider
llm_provider: str = "openai"
llm_model: str = "gpt-4o"
openai_api_key: str = ""
anthropic_api_key: str = ""
google_api_key: str = ""
mistral_api_key: str = ""
ollama_base_url: str = "http://localhost:11434"
# OrgMyLife Integration
orgmylife_api_url: str = "https://api.andreknie.de"
orgmylife_api_key: str = ""
# Nextcloud Inbox
nextcloud_inbox_url: str = ""
nextcloud_inbox_user: str = ""
nextcloud_inbox_password: str = ""
# Paths
notes_dir: str = "./notes"
inbox_dir: str = "./ingestion/inbox"
# OCR
ocr_enabled: bool = True
ocr_language: str = "deu+eng"
# Behavior
confidence_threshold: float = 0.7
auto_commit: bool = True
# Web Upload Auth (same as SilverBullet)
sb_user: str = "admin:changeme"
model_config = {"env_file": ".env", "env_prefix": "", "extra": "ignore"}
@property
def llm_model_string(self) -> str:
"""LiteLLM model string (e.g., 'openai/gpt-4o', 'ollama/llama3')."""
if self.llm_provider == "ollama":
return f"ollama/{self.llm_model}"
return f"{self.llm_provider}/{self.llm_model}"
def validate_api_key(self) -> None:
"""Exit with a clear error if the required API key is missing."""
key_map = {
"openai": ("openai_api_key", "OPENAI_API_KEY"),
"anthropic": ("anthropic_api_key", "ANTHROPIC_API_KEY"),
"google": ("google_api_key", "GOOGLE_API_KEY"),
"mistral": ("mistral_api_key", "MISTRAL_API_KEY"),
}
if self.llm_provider in key_map:
attr, env_name = key_map[self.llm_provider]
if not getattr(self, attr):
raise SystemExit(
f"ERROR: {env_name} is required for provider '{self.llm_provider}' but not set."
)
@@ -0,0 +1,45 @@
"""Recursive file discovery with extension filtering.
Discovers files for ingestion by walking directories and filtering
to supported document types.
"""
import logging
from pathlib import Path
logger = logging.getLogger(__name__)
SUPPORTED_EXTENSIONS = {".pdf", ".jpg", ".jpeg", ".png", ".md", ".txt", ".docx"}
def discover_files(path: Path) -> list[Path]:
"""Discover supported files from a path (file or directory).
Args:
path: A single file path or a directory to scan recursively.
Returns:
Sorted list of Path objects with supported extensions.
Raises:
FileNotFoundError: If the path does not exist.
"""
path = Path(path)
if not path.exists():
raise FileNotFoundError(f"Path does not exist: {path}")
if path.is_file():
if path.suffix.lower() in SUPPORTED_EXTENSIONS:
return [path]
logger.warning("Unsupported file type: %s", path)
return []
# Recursive directory traversal
files: list[Path] = []
for item in sorted(path.rglob("*")):
if item.is_file() and item.suffix.lower() in SUPPORTED_EXTENSIONS:
files.append(item)
logger.info("Discovered %d supported files in %s", len(files), path)
return files
@@ -0,0 +1 @@
"""LLM-powered enrichment: entity detection, categorization, and action item extraction."""
@@ -0,0 +1,122 @@
"""Enrichment agent orchestrating LLM calls for entity detection.
Calls the LLM with extracted text, parses the structured response,
filters entities by confidence threshold, and handles parse errors with retry.
"""
import json
import logging
from ingestion.config import IngestionConfig
from ingestion.integrations.llm import LLMClient, LLMError
from .models import EnrichmentResult, Entity
from .prompts import STRICT_RETRY_PROMPT, build_system_prompt, build_user_prompt
logger = logging.getLogger(__name__)
class EnrichmentError(Exception):
"""Raised when enrichment fails after retries."""
pass
class EnrichmentAgent:
"""Orchestrates LLM-based document enrichment."""
def __init__(
self,
config: IngestionConfig,
llm_client: LLMClient | None = None,
known_projects: list[str] | None = None,
known_people: list[str] | None = None,
):
self.config = config
self.llm = llm_client or LLMClient(config)
self.confidence_threshold = config.confidence_threshold
self.system_prompt = build_system_prompt(known_projects, known_people)
def enrich(self, text: str) -> EnrichmentResult:
"""Enrich extracted text with LLM-detected entities and metadata.
Args:
text: The extracted document text to analyze.
Returns:
EnrichmentResult with entities filtered by confidence threshold.
Raises:
EnrichmentError: If LLM call and retry both fail.
"""
messages = [
{"role": "system", "content": self.system_prompt},
{"role": "user", "content": build_user_prompt(text)},
]
# First attempt
try:
response_text = self.llm.complete(messages)
result = self._parse_response(response_text)
except (json.JSONDecodeError, ValueError) as parse_err:
logger.warning(
"First LLM response parse failed (%s), retrying with strict prompt",
parse_err,
)
# Retry with stricter prompt
result = self._retry_with_strict_prompt(messages, response_text)
except LLMError as e:
raise EnrichmentError(f"LLM call failed: {e}") from e
# Filter entities by confidence threshold
result.entities = self._filter_by_confidence(result.entities)
return result
def _retry_with_strict_prompt(
self, original_messages: list[dict[str, str]], failed_response: str
) -> EnrichmentResult:
"""Retry LLM call with a stricter prompt after parse failure."""
retry_messages = original_messages + [
{"role": "assistant", "content": failed_response},
{"role": "user", "content": STRICT_RETRY_PROMPT},
]
try:
response_text = self.llm.complete(retry_messages)
return self._parse_response(response_text)
except (json.JSONDecodeError, ValueError, LLMError) as e:
logger.error("Retry also failed (%s), using defaults", e)
return self._default_result()
def _parse_response(self, response_text: str) -> EnrichmentResult:
"""Parse LLM response text into an EnrichmentResult.
Handles responses that may be wrapped in markdown code fences.
"""
cleaned = response_text.strip()
# Strip markdown code fences if present
if cleaned.startswith("```"):
lines = cleaned.split("\n")
# Remove first line (```json) and last line (```)
lines = [l for l in lines[1:] if not l.strip() == "```"]
cleaned = "\n".join(lines)
data = json.loads(cleaned)
return EnrichmentResult.model_validate(data)
def _filter_by_confidence(self, entities: list[Entity]) -> list[Entity]:
"""Keep only entities meeting the confidence threshold."""
return [e for e in entities if e.confidence >= self.confidence_threshold]
def _default_result(self) -> EnrichmentResult:
"""Return a minimal default result when all parsing fails."""
return EnrichmentResult(
title="Untitled Note",
category="inbox",
tags=[],
entities=[],
action_items=[],
summary=None,
)
@@ -0,0 +1,37 @@
"""Pydantic models for LLM enrichment responses.
Defines the structured output format expected from the enrichment agent:
entities, action items, and the overall enrichment result.
"""
from typing import Literal
from pydantic import BaseModel, Field
class Entity(BaseModel):
"""A detected entity in the document text."""
type: Literal["person", "project", "date", "action_item"]
value: str
confidence: float = Field(ge=0.0, le=1.0)
position: int | None = None # character offset in source text
class ActionItem(BaseModel):
"""An extracted action item / task."""
description: str
assignee: str | None = None
deadline: str | None = None # ISO 8601 date string
class EnrichmentResult(BaseModel):
"""Complete enrichment output from the LLM."""
title: str
category: Literal["meeting", "project", "decision", "inbox"]
tags: list[str] = Field(default_factory=list)
entities: list[Entity] = Field(default_factory=list)
action_items: list[ActionItem] = Field(default_factory=list)
summary: str | None = None
@@ -0,0 +1,89 @@
"""Structured LLM prompts for document enrichment.
Provides the system prompt that instructs the LLM to return structured JSON
with entities, categories, tags, and action items.
"""
ENRICHMENT_SYSTEM_PROMPT = """\
You are a knowledge management assistant. Analyze the following text extracted \
from a document and return structured metadata as JSON.
Return a JSON object with exactly these fields:
- "title": A concise, descriptive title for this note (max 80 chars)
- "category": One of "meeting", "project", "decision", "inbox"
- "tags": List of relevant topic tags (lowercase, no # prefix)
- "entities": List of detected entities, each with:
- "type": One of "person", "project", "date", "action_item"
- "value": The extracted text value
- "confidence": Float between 0.0 and 1.0 indicating detection confidence
- "position": Approximate character offset where entity appears (integer or null)
- "action_items": List of action items, each with:
- "description": What needs to be done
- "assignee": Person responsible (string or null)
- "deadline": Date in ISO 8601 format, e.g. "2024-03-22" (string or null)
- "summary": A 1-2 sentence summary of the document (string or null)
Rules:
- Be precise with entity detection. Only mark entities you are confident about.
- For people, use their full name as the value.
- For projects, use the project name as commonly referenced.
- For dates, use ISO 8601 format (YYYY-MM-DD) as the value.
- Assign confidence scores honestly: 0.9+ for clearly stated entities, 0.7-0.9 for \
likely entities, below 0.7 for uncertain ones.
- Category "meeting" for meeting notes, agendas, minutes.
- Category "project" for project-specific documentation.
- Category "decision" for recorded decisions, ADRs.
- Category "inbox" for anything that doesn't clearly fit the above.
Context: This is a personal knowledge base for a project manager working across \
multiple business projects.{context_section}
"""
STRICT_RETRY_PROMPT = """\
Your previous response was not valid JSON. Please respond with ONLY a valid JSON \
object matching the schema described in the system prompt. No markdown, no code fences, \
no explanation — just the raw JSON object.
"""
def build_system_prompt(
known_projects: list[str] | None = None,
known_people: list[str] | None = None,
) -> str:
"""Build the system prompt with optional context about known entities.
Args:
known_projects: List of known project names for context.
known_people: List of known people names for context.
Returns:
Complete system prompt string.
"""
context_parts: list[str] = []
if known_projects:
projects_str = ", ".join(known_projects)
context_parts.append(f"Known projects: {projects_str}.")
if known_people:
people_str = ", ".join(known_people)
context_parts.append(f"Known people: {people_str}.")
if context_parts:
context_section = "\n" + " ".join(context_parts)
else:
context_section = ""
return ENRICHMENT_SYSTEM_PROMPT.format(context_section=context_section)
def build_user_prompt(extracted_text: str) -> str:
"""Build the user message containing the extracted document text.
Args:
extracted_text: The raw text extracted from the document.
Returns:
User prompt string.
"""
return f"Analyze this document and return the structured JSON:\n\n{extracted_text}"
@@ -0,0 +1,51 @@
"""Text extraction layer for various document formats."""
from pathlib import Path
from .base import ExtractionResult, TextExtractor
from .docx import DocxExtractor
from .ocr import OCRExtractor
from .pdf import PDFExtractor
from .text import PlainTextExtractor
def get_extractor(file_path: Path, config=None):
"""Get the appropriate extractor for a file.
Args:
file_path: Path to the file to extract.
config: Optional IngestionConfig for LLM-based extraction.
Returns:
An extractor instance that can handle the file, or None.
"""
suffix = file_path.suffix.lower()
if suffix in (".md", ".txt"):
return PlainTextExtractor()
elif suffix == ".pdf":
if config:
return PDFExtractor(
ocr_enabled=config.ocr_enabled,
ocr_language=config.ocr_language,
llm_provider=config.llm_provider,
llm_model=config.llm_model,
mistral_api_key=config.mistral_api_key,
)
return PDFExtractor()
elif suffix in (".jpg", ".jpeg", ".png"):
return OCRExtractor()
elif suffix == ".docx":
return DocxExtractor()
return None
__all__ = [
"ExtractionResult",
"TextExtractor",
"PlainTextExtractor",
"PDFExtractor",
"OCRExtractor",
"DocxExtractor",
"get_extractor",
]
@@ -0,0 +1,27 @@
"""Base protocol and data structures for text extraction."""
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Protocol
@dataclass
class ExtractionResult:
"""Result of extracting text from a document."""
text: str
source_file: Path
extraction_method: str # "direct", "ocr", "mixed"
metadata: dict[str, Any] = field(default_factory=dict)
class TextExtractor(Protocol):
"""Protocol that all text extractors must implement."""
def can_handle(self, file_path: Path) -> bool:
"""Return True if this extractor can process the given file."""
...
def extract(self, file_path: Path) -> ExtractionResult:
"""Extract text content from the file."""
...
@@ -0,0 +1,57 @@
"""DOCX text extraction using python-docx.
Preserves heading structure and paragraph breaks.
"""
import logging
from pathlib import Path
from .base import ExtractionResult
logger = logging.getLogger(__name__)
class DocxExtractor:
"""Extracts text from .docx files preserving structure."""
def can_handle(self, file_path: Path) -> bool:
"""Return True for .docx files."""
return file_path.suffix.lower() == ".docx"
def extract(self, file_path: Path) -> ExtractionResult:
"""Extract text from DOCX preserving headings and paragraphs."""
try:
from docx import Document
except ImportError:
raise ImportError(
"python-docx is required for DOCX extraction. "
"Install with: pip install python-docx"
)
doc = Document(str(file_path))
parts: list[str] = []
for paragraph in doc.paragraphs:
text = paragraph.text.strip()
if not text:
continue
# Preserve heading structure with markdown-style headers
style_name = paragraph.style.name.lower() if paragraph.style else ""
if "heading 1" in style_name:
parts.append(f"# {text}")
elif "heading 2" in style_name:
parts.append(f"## {text}")
elif "heading 3" in style_name:
parts.append(f"### {text}")
elif "heading 4" in style_name:
parts.append(f"#### {text}")
else:
parts.append(text)
return ExtractionResult(
text="\n\n".join(parts),
source_file=file_path,
extraction_method="direct",
metadata={"paragraph_count": len(parts)},
)
@@ -0,0 +1,48 @@
"""OCR extraction for images using pytesseract.
Handles .jpg, .jpeg, .png files. Configured for German + English (deu+eng).
Returns empty text (not error) when OCR cannot extract readable content.
"""
import logging
from pathlib import Path
from .base import ExtractionResult
logger = logging.getLogger(__name__)
SUPPORTED_EXTENSIONS = {".jpg", ".jpeg", ".png"}
class OCRExtractor:
"""Extracts text from images using Tesseract OCR."""
def __init__(self, language: str = "deu+eng"):
self.language = language
def can_handle(self, file_path: Path) -> bool:
"""Return True for .jpg, .jpeg, .png files."""
return file_path.suffix.lower() in SUPPORTED_EXTENSIONS
def extract(self, file_path: Path) -> ExtractionResult:
"""Run OCR on an image file. Returns empty text on failure."""
text = self._run_ocr(file_path)
return ExtractionResult(
text=text,
source_file=file_path,
extraction_method="ocr",
metadata={"ocr_language": self.language},
)
def _run_ocr(self, file_path: Path) -> str:
"""Execute pytesseract on the image. Returns empty string on failure."""
try:
import pytesseract
from PIL import Image
image = Image.open(file_path)
text = pytesseract.image_to_string(image, lang=self.language)
return text.strip()
except Exception as e:
logger.warning("OCR extraction failed for %s: %s", file_path, e)
return ""
@@ -0,0 +1,156 @@
"""PDF text extraction using pdfplumber.
Detects scanned pages (no extractable text) and delegates to:
1. Mistral Vision API (preferred — handles handwriting well)
2. Tesseract OCR (fallback if no LLM configured)
"""
import base64
import io
import logging
from pathlib import Path
import pdfplumber
from .base import ExtractionResult
logger = logging.getLogger(__name__)
class PDFExtractor:
"""Extracts text from PDF files, with Vision LLM or OCR fallback for scanned pages."""
def __init__(
self,
ocr_enabled: bool = True,
ocr_language: str = "deu+eng",
llm_provider: str = "mistral",
llm_model: str = "mistral-small-latest",
mistral_api_key: str = "",
):
self.ocr_enabled = ocr_enabled
self.ocr_language = ocr_language
self.llm_provider = llm_provider
self.llm_model = llm_model
self.mistral_api_key = mistral_api_key
self._use_vision = bool(mistral_api_key)
def can_handle(self, file_path: Path) -> bool:
"""Return True for .pdf files."""
return file_path.suffix.lower() == ".pdf"
def extract(self, file_path: Path) -> ExtractionResult:
"""Extract text from PDF, using Vision LLM or OCR for scanned pages."""
pages_text: list[str] = []
has_direct_text = False
has_vision_text = False
with pdfplumber.open(file_path) as pdf:
for page_num, page in enumerate(pdf.pages, start=1):
text = page.extract_text() or ""
text = text.strip()
if text:
pages_text.append(text)
has_direct_text = True
elif self.ocr_enabled:
# Scanned page — try Vision LLM first, then Tesseract
logger.debug(
"Page %d of %s has no text, attempting vision OCR",
page_num, file_path,
)
ocr_text = self._vision_ocr_page(page, page_num, file_path)
if ocr_text:
pages_text.append(ocr_text)
has_vision_text = True
# Determine extraction method
if has_direct_text and has_vision_text:
method = "mixed"
elif has_vision_text:
method = "vision-ocr"
else:
method = "direct"
return ExtractionResult(
text="\n\n".join(pages_text),
source_file=file_path,
extraction_method=method,
metadata={"page_count": len(pages_text)},
)
def _vision_ocr_page(self, page, page_num: int, file_path: Path) -> str:
"""OCR a PDF page using Mistral Vision API, with Tesseract fallback."""
if self._use_vision:
try:
return self._mistral_vision_ocr(page, page_num)
except Exception as e:
logger.warning(
"Vision OCR failed for page %d of %s: %s, trying Tesseract",
page_num, file_path, e,
)
# Fallback to Tesseract
return self._tesseract_ocr(page, page_num, file_path)
def _mistral_vision_ocr(self, page, page_num: int) -> str:
"""Send page image to Mistral Vision for text extraction."""
import httpx
# Convert page to PNG image
image = page.to_image(resolution=200)
img_buffer = io.BytesIO()
image.original.save(img_buffer, format="PNG")
img_bytes = img_buffer.getvalue()
img_b64 = base64.b64encode(img_bytes).decode("utf-8")
# Call Mistral Vision API via LiteLLM-compatible format
response = httpx.post(
"https://api.mistral.ai/v1/chat/completions",
headers={
"Authorization": f"Bearer {self.mistral_api_key}",
"Content-Type": "application/json",
},
json={
"model": self.llm_model,
"messages": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "Extract ALL text from this image. This is a handwritten note or document. "
"Transcribe everything you can read, preserving structure (paragraphs, lists, headings). "
"If text is in German, keep it in German. Return ONLY the extracted text, no commentary.",
},
{
"type": "image_url",
"image_url": {
"url": f"data:image/png;base64,{img_b64}",
},
},
],
}
],
"max_tokens": 4000,
},
timeout=60.0,
)
response.raise_for_status()
data = response.json()
text = data["choices"][0]["message"]["content"]
logger.info("Vision OCR page %d: extracted %d chars", page_num, len(text))
return text.strip()
def _tesseract_ocr(self, page, page_num: int, file_path: Path) -> str:
"""Fallback: OCR a PDF page using Tesseract."""
try:
import pytesseract
image = page.to_image(resolution=300)
text = pytesseract.image_to_string(image.original, lang=self.ocr_language)
return text.strip()
except Exception as e:
logger.warning(
"Tesseract OCR failed for page %d of %s: %s", page_num, file_path, e
)
return ""
@@ -0,0 +1,57 @@
"""Plain text and markdown file extractor.
Handles .md and .txt files. Reads as UTF-8 by default, with fallback
charset detection for non-UTF-8 encoded files.
"""
import logging
from pathlib import Path
from .base import ExtractionResult
logger = logging.getLogger(__name__)
SUPPORTED_EXTENSIONS = {".md", ".txt"}
class PlainTextExtractor:
"""Extracts text from plain text and markdown files."""
def can_handle(self, file_path: Path) -> bool:
"""Return True for .md and .txt files."""
return file_path.suffix.lower() in SUPPORTED_EXTENSIONS
def extract(self, file_path: Path) -> ExtractionResult:
"""Read file as UTF-8, falling back to charset detection."""
text = self._read_with_fallback(file_path)
return ExtractionResult(
text=text,
source_file=file_path,
extraction_method="direct",
metadata={"encoding": "utf-8"},
)
def _read_with_fallback(self, file_path: Path) -> str:
"""Try UTF-8 first, then attempt charset detection."""
try:
return file_path.read_text(encoding="utf-8")
except UnicodeDecodeError:
logger.warning(
"UTF-8 decode failed for %s, attempting charset detection", file_path
)
return self._read_with_detection(file_path)
def _read_with_detection(self, file_path: Path) -> str:
"""Detect encoding and read file."""
raw = file_path.read_bytes()
# Try common encodings before giving up
for encoding in ("latin-1", "cp1252", "iso-8859-1"):
try:
return raw.decode(encoding)
except (UnicodeDecodeError, LookupError):
continue
# Last resort: decode with replacement characters
logger.warning("Could not detect encoding for %s, using replacement", file_path)
return raw.decode("utf-8", errors="replace")
@@ -0,0 +1 @@
"""External service integrations: LLM providers, OrgMyLife, Git, Nextcloud."""
@@ -0,0 +1,66 @@
"""Git commit operations for auto-committing imported notes.
Stages new markdown files and stub pages, commits with message format:
ingestion: import N notes from [source-type]
Supports --no-commit flag. Handles git failures gracefully.
"""
import logging
import subprocess
from pathlib import Path
logger = logging.getLogger(__name__)
def commit_imported_notes(files: list[Path], source_type: str = "import") -> bool:
"""Stage and commit imported note files.
Args:
files: List of file paths to stage and commit.
source_type: Source description for commit message (e.g., 'batch-import', 'upload', 'nextcloud').
Returns:
True if commit succeeded, False otherwise.
"""
if not files:
logger.debug("No files to commit.")
return False
try:
# Stage files
file_strs = [str(f) for f in files]
subprocess.run(
["git", "add"] + file_strs,
capture_output=True,
text=True,
check=True,
)
# Commit
count = len(files)
message = f"ingestion: import {count} notes from {source_type}"
result = subprocess.run(
["git", "commit", "-m", message],
capture_output=True,
text=True,
)
if result.returncode == 0:
logger.info("Git commit: %s", message)
return True
else:
# Nothing to commit (files already tracked/unchanged)
logger.debug("Git commit returned %d: %s", result.returncode, result.stderr.strip())
return False
except FileNotFoundError:
logger.warning("Git not found in PATH — skipping commit.")
return False
except subprocess.CalledProcessError as e:
logger.warning("Git operation failed: %s", e.stderr.strip() if e.stderr else str(e))
return False
except Exception as e:
logger.warning("Unexpected git error: %s", e)
return False
@@ -0,0 +1,112 @@
"""LiteLLM wrapper for model-agnostic LLM access.
Supports OpenAI, Anthropic, Google, Mistral, and Ollama via a unified interface.
Implements retry logic with exponential backoff for transient failures.
"""
import logging
import time
from typing import Any
from pydantic import BaseModel
from ingestion.config import IngestionConfig
logger = logging.getLogger(__name__)
# Retry configuration
MAX_RETRIES = 3
BASE_DELAY_SECONDS = 1.0
MAX_DELAY_SECONDS = 30.0
BACKOFF_FACTOR = 2.0
class LLMError(Exception):
"""Raised when an LLM call fails after all retries."""
def __init__(self, provider: str, model: str, message: str):
self.provider = provider
self.model = model
super().__init__(
f"LLM call failed [{provider}/{model}]: {message}"
)
class LLMClient:
"""Model-agnostic LLM client using LiteLLM."""
def __init__(self, config: IngestionConfig):
self.model = config.llm_model_string
self.provider = config.llm_provider
self.config = config
def complete(
self,
messages: list[dict[str, str]],
response_format: type[BaseModel] | None = None,
) -> str:
"""Send a completion request with retry logic.
Args:
messages: List of message dicts with 'role' and 'content' keys.
response_format: Optional Pydantic model for structured output.
Returns:
The assistant's response content as a string.
Raises:
LLMError: If all retries are exhausted.
"""
import litellm
last_error: Exception | None = None
for attempt in range(1, MAX_RETRIES + 1):
try:
kwargs: dict[str, Any] = {
"model": self.model,
"messages": messages,
}
if response_format is not None:
kwargs["response_format"] = response_format
response = litellm.completion(**kwargs)
content = response.choices[0].message.content
return content or ""
except (
litellm.RateLimitError,
litellm.Timeout,
litellm.ServiceUnavailableError,
litellm.InternalServerError,
) as e:
last_error = e
delay = min(
BASE_DELAY_SECONDS * (BACKOFF_FACTOR ** (attempt - 1)),
MAX_DELAY_SECONDS,
)
logger.warning(
"LLM call attempt %d/%d failed (%s: %s), retrying in %.1fs",
attempt,
MAX_RETRIES,
type(e).__name__,
str(e),
delay,
)
time.sleep(delay)
except Exception as e:
# Non-retryable error
raise LLMError(
provider=self.provider,
model=self.model,
message=f"{type(e).__name__}: {e}",
) from e
# All retries exhausted
raise LLMError(
provider=self.provider,
model=self.model,
message=f"All {MAX_RETRIES} retries exhausted. Last error: {last_error}",
)
@@ -0,0 +1,160 @@
"""Nextcloud WebDAV inbox sync.
Polls a Nextcloud folder via WebDAV (PROPFIND/GET) for new files,
downloads them for processing, and moves processed files to a processed/ subfolder.
"""
import logging
import xml.etree.ElementTree as ET
from pathlib import Path
from urllib.parse import unquote, urljoin
import httpx
from ingestion.config import IngestionConfig
logger = logging.getLogger(__name__)
# WebDAV XML namespace
DAV_NS = "DAV:"
class NextcloudInboxSync:
"""Sync files from a Nextcloud WebDAV folder."""
def __init__(self, config: IngestionConfig):
self.base_url = config.nextcloud_inbox_url.rstrip("/") + "/"
self.user = config.nextcloud_inbox_user
self.password = config.nextcloud_inbox_password
self.local_inbox = Path(config.inbox_dir)
self.local_inbox.mkdir(parents=True, exist_ok=True)
self._client = httpx.Client(
auth=(self.user, self.password),
timeout=30.0,
)
def poll(self) -> list[Path]:
"""Poll Nextcloud inbox for files and download them.
Returns:
List of local paths to downloaded files.
"""
try:
filenames = self._list_files()
except Exception as e:
logger.error("Failed to list Nextcloud inbox: %s", e)
return []
if not filenames:
logger.info("Nextcloud inbox is empty.")
return []
downloaded: list[Path] = []
for filename in filenames:
local_path = self.download(filename)
if local_path:
downloaded.append(local_path)
logger.info("Downloaded %d file(s) from Nextcloud.", len(downloaded))
return downloaded
def _list_files(self) -> list[str]:
"""PROPFIND to list files in the inbox folder."""
propfind_body = """<?xml version="1.0" encoding="UTF-8"?>
<d:propfind xmlns:d="DAV:">
<d:prop>
<d:resourcetype/>
<d:getcontentlength/>
</d:prop>
</d:propfind>"""
response = self._client.request(
"PROPFIND",
self.base_url,
content=propfind_body,
headers={
"Content-Type": "application/xml",
"Depth": "1",
},
)
response.raise_for_status()
# Parse WebDAV XML response
root = ET.fromstring(response.text)
filenames: list[str] = []
for resp_elem in root.findall(f"{{{DAV_NS}}}response"):
href = resp_elem.findtext(f"{{{DAV_NS}}}href", "")
# Skip the folder itself (collections)
resourcetype = resp_elem.find(
f".//{{{DAV_NS}}}resourcetype/{{{DAV_NS}}}collection"
)
if resourcetype is not None:
continue
# Extract filename from href
filename = unquote(href.rstrip("/").split("/")[-1])
if filename and not filename.startswith("."):
filenames.append(filename)
return filenames
def download(self, filename: str) -> Path | None:
"""Download a file from Nextcloud inbox.
Args:
filename: Name of the file to download.
Returns:
Local path to the downloaded file, or None on failure.
"""
url = urljoin(self.base_url, filename)
try:
response = self._client.get(url)
response.raise_for_status()
local_path = self.local_inbox / filename
local_path.write_bytes(response.content)
logger.info("Downloaded: %s", filename)
return local_path
except Exception as e:
logger.error("Failed to download %s: %s", filename, e)
return None
def mark_processed(self, filename: str) -> bool:
"""Move a processed file to the processed/ subfolder on Nextcloud.
Args:
filename: Name of the file to move.
Returns:
True if move succeeded, False otherwise.
"""
source_url = urljoin(self.base_url, filename)
dest_url = urljoin(self.base_url, f"processed/{filename}")
try:
# Ensure processed/ folder exists (MKCOL)
processed_url = urljoin(self.base_url, "processed/")
self._client.request("MKCOL", processed_url)
# MOVE the file
response = self._client.request(
"MOVE",
source_url,
headers={"Destination": dest_url, "Overwrite": "T"},
)
if response.status_code in (201, 204):
logger.info("Marked processed: %s", filename)
return True
else:
logger.warning(
"MOVE returned %d for %s", response.status_code, filename
)
return False
except Exception as e:
logger.warning("Failed to mark %s as processed: %s", filename, e)
return False
@@ -0,0 +1,74 @@
"""OrgMyLife API client for creating tasks from extracted action items.
POST to /api/tasks with title, source_url, description.
Handles unreachable API gracefully (log warning, continue).
"""
import logging
import httpx
logger = logging.getLogger(__name__)
class OrgMyLifeClient:
"""HTTP client for OrgMyLife task creation."""
def __init__(self, base_url: str, api_key: str):
self.base_url = base_url.rstrip("/")
self.api_key = api_key
def create_task(
self,
title: str,
source_url: str,
description: str = "",
priority: int = 4,
) -> dict:
"""Create a task in OrgMyLife.
Args:
title: Task title (action item description).
source_url: Reference back to the source note.
description: Optional additional description.
priority: Task priority (1-5, default 4).
Returns:
Response data dict, or empty dict on failure.
"""
url = f"{self.base_url}/api/tasks"
payload = {
"title": title,
"source_url": source_url,
"description": description,
"priority": priority,
}
headers = {
"Authorization": f"Bearer {self.api_key}",
"Content-Type": "application/json",
}
try:
response = httpx.post(
url,
json=payload,
headers=headers,
timeout=10.0,
)
response.raise_for_status()
logger.info("Created OrgMyLife task: %s", title[:50])
return response.json()
except httpx.ConnectError:
logger.warning(
"OrgMyLife API unreachable at %s — skipping task creation", self.base_url
)
return {}
except httpx.HTTPStatusError as e:
logger.warning(
"OrgMyLife API error %d: %s", e.response.status_code, e.response.text[:200]
)
return {}
except Exception as e:
logger.warning("OrgMyLife task creation failed: %s", e)
return {}
@@ -0,0 +1 @@
"""Wiki-link generation and stub page creation."""
@@ -0,0 +1,71 @@
"""Wiki-link generation from detected entities.
Converts entity references to SilverBullet-compatible wiki-links:
- Person → [[people/firstname-lastname]]
- Project → [[projects/project-slug]]
Only the first occurrence of each entity is linked; subsequent mentions remain plain text.
"""
import logging
from slugify import slugify
from ingestion.enrichment.models import Entity
logger = logging.getLogger(__name__)
def _entity_to_wikilink(entity: Entity) -> str:
"""Convert an entity to its wiki-link representation."""
slug = slugify(entity.value)
if entity.type == "person":
return f"[[people/{slug}]]"
elif entity.type == "project":
return f"[[projects/{slug}]]"
return ""
def generate_wiki_links(text: str, entities: list[Entity]) -> str:
"""Insert wiki-links into text for person and project entities.
Only the first occurrence of each entity's value is converted to a wiki-link.
Dates and action_items are skipped (handled by frontmatter/renderer).
Args:
text: The document body text.
entities: List of detected entities from enrichment.
Returns:
Text with wiki-links inserted at first occurrence of each entity.
"""
# Filter to linkable entity types
linkable = [e for e in entities if e.type in ("person", "project")]
# Track which entity values we've already linked (case-insensitive)
linked_values: set[str] = set()
for entity in linkable:
lower_val = entity.value.lower()
if lower_val in linked_values:
continue
# Find first occurrence and replace it
idx = text.find(entity.value)
if idx == -1:
# Try case-insensitive search
idx = text.lower().find(lower_val)
if idx == -1:
continue
# Use the actual text at that position for replacement
original_text = text[idx : idx + len(entity.value)]
else:
original_text = entity.value
wikilink = _entity_to_wikilink(entity)
if wikilink:
# Replace only the first occurrence
text = text[:idx] + wikilink + text[idx + len(original_text) :]
linked_values.add(lower_val)
return text
@@ -0,0 +1,56 @@
"""Stub page creation for entities that don't yet have a NoteGraph page.
Creates minimal markdown pages in the appropriate directory (people/ or projects/)
so that wiki-links resolve immediately after import.
"""
import logging
from pathlib import Path
from slugify import slugify
from ingestion.enrichment.models import Entity
logger = logging.getLogger(__name__)
def create_stub_pages(entities: list[Entity], notes_dir: Path) -> list[Path]:
"""Create stub pages for entities that don't have existing pages.
Args:
entities: List of entities to check/create stubs for.
notes_dir: Root notes directory (e.g., ./notes).
Returns:
List of paths to newly created stub pages.
"""
notes_dir = Path(notes_dir)
created: list[Path] = []
linkable = [e for e in entities if e.type in ("person", "project")]
seen_slugs: set[str] = set()
for entity in linkable:
slug = slugify(entity.value)
if slug in seen_slugs:
continue
seen_slugs.add(slug)
if entity.type == "person":
target = notes_dir / "people" / f"{slug}.md"
elif entity.type == "project":
target = notes_dir / "projects" / f"{slug}.md"
else:
continue
if target.exists():
logger.debug("Page already exists: %s", target)
continue
# Create stub page
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(f"# {entity.value}\n", encoding="utf-8")
logger.info("Created stub page: %s", target)
created.append(target)
return created
@@ -0,0 +1 @@
"""Output rendering, folder routing, and filename generation."""
@@ -0,0 +1,65 @@
"""Filename generation with collision avoidance.
Pattern: YYYY-MM-DD-slugified-title.md
Slug: lowercase alphanumeric + hyphens only.
Appends numeric suffix (-2, -3, ...) if file already exists.
"""
import logging
from datetime import datetime, timezone
from pathlib import Path
from slugify import slugify
logger = logging.getLogger(__name__)
def generate_filename(title: str, date: str | None = None) -> str:
"""Generate a filename from title and optional date.
Args:
title: The note title to slugify.
date: Optional date string (YYYY-MM-DD). Uses today if not provided.
Returns:
Filename in format YYYY-MM-DD-slugified-title.md
"""
if date is None:
date = datetime.now(timezone.utc).strftime("%Y-%m-%d")
# Ensure date is just the date portion (handle ISO datetime strings)
date_part = date[:10] if len(date) >= 10 else date
slug = slugify(title, max_length=80)
if not slug:
slug = "untitled"
return f"{date_part}-{slug}.md"
def resolve_collision(target_dir: Path, filename: str) -> Path:
"""Resolve filename collisions by appending numeric suffix.
Args:
target_dir: Directory where the file will be written.
filename: Desired filename (e.g., 2024-03-15-meeting-notes.md).
Returns:
Full path with unique filename (appends -2, -3, etc. if needed).
"""
target = Path(target_dir) / filename
if not target.exists():
return target
# Split filename to insert suffix before .md
stem = filename.rsplit(".md", 1)[0]
suffix_num = 2
while True:
new_filename = f"{stem}-{suffix_num}.md"
new_target = Path(target_dir) / new_filename
if not new_target.exists():
logger.debug("Collision resolved: %s%s", filename, new_filename)
return new_target
suffix_num += 1
@@ -0,0 +1,97 @@
"""Markdown + YAML frontmatter rendering for processed notes.
Renders:
- Frontmatter with title, date, tags, people, projects, source
- Body with wiki-links inserted
- Action Items section at the end
"""
import logging
from datetime import datetime, timezone
from pathlib import Path
import yaml
from ingestion.enrichment.models import EnrichmentResult
logger = logging.getLogger(__name__)
def render_note(
enrichment: EnrichmentResult, body_text: str, source_file: Path
) -> str:
"""Render a complete markdown note with YAML frontmatter.
Args:
enrichment: The enrichment result with metadata and entities.
body_text: The document body (with wiki-links already inserted).
source_file: Original source file path for provenance.
Returns:
Complete markdown string with frontmatter, body, and action items.
"""
# Build frontmatter data
frontmatter: dict = {
"title": enrichment.title,
}
# Extract date from entities or use today
date_entities = [e for e in enrichment.entities if e.type == "date"]
if date_entities:
frontmatter["date"] = date_entities[0].value
else:
frontmatter["date"] = datetime.now(timezone.utc).strftime("%Y-%m-%d")
if enrichment.tags:
frontmatter["tags"] = enrichment.tags
# People and projects from entities
people = list(
{e.value for e in enrichment.entities if e.type == "person"}
)
projects = list(
{e.value for e in enrichment.entities if e.type == "project"}
)
if people:
frontmatter["people"] = sorted(people)
if projects:
frontmatter["projects"] = sorted(projects)
frontmatter["source"] = {
"file": source_file.name,
"imported": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S"),
}
# Render YAML frontmatter
yaml_str = yaml.dump(
frontmatter,
default_flow_style=False,
allow_unicode=True,
sort_keys=False,
)
# Build the full document
parts = [
"---",
yaml_str.rstrip(),
"---",
"",
body_text.rstrip(),
]
# Add action items section if any
if enrichment.action_items:
parts.append("")
parts.append("## Action Items")
parts.append("")
for item in enrichment.action_items:
line = f"- [ ] {item.description}"
if item.assignee:
line += f" ({item.assignee})"
if item.deadline:
line += f" (deadline: {item.deadline})"
parts.append(line)
parts.append("") # trailing newline
return "\n".join(parts)
@@ -0,0 +1,44 @@
"""Category-based folder routing for output files.
Routes notes to the appropriate NoteGraph folder based on detected category:
- meeting → notes/meetings/
- project → notes/projects/
- decision → notes/decisions/
- default → notes/inbox/
"""
import logging
from pathlib import Path
logger = logging.getLogger(__name__)
# Category to folder mapping
CATEGORY_FOLDERS: dict[str, str] = {
"meeting": "meetings",
"project": "projects",
"decision": "decisions",
}
def route_to_folder(
category: str, notes_dir: Path, override: Path | None = None
) -> Path:
"""Determine the output folder for a note based on its category.
Args:
category: The detected category (meeting, project, decision, inbox).
notes_dir: Root notes directory (e.g., ./notes).
override: Optional explicit output directory override (--output-dir).
Returns:
Path to the target folder (created if it doesn't exist).
"""
if override is not None:
target = Path(override)
else:
folder_name = CATEGORY_FOLDERS.get(category, "inbox")
target = Path(notes_dir) / folder_name
target.mkdir(parents=True, exist_ok=True)
logger.debug("Routing category '%s' to folder: %s", category, target)
return target
@@ -0,0 +1,217 @@
"""Main processing pipeline orchestrating discovery → extraction → enrichment → linking → output.
Handles:
- Progress tracking (current file / total)
- Per-file error handling (log + skip + continue)
- Summary reporting (processed, success, failed counts)
- Dry-run mode
"""
import logging
from dataclasses import dataclass, field
from pathlib import Path
from ingestion.config import IngestionConfig
from ingestion.discovery import discover_files
from ingestion.enrichment.agent import EnrichmentAgent
from ingestion.enrichment.models import EnrichmentResult
from ingestion.extraction import get_extractor
from ingestion.integrations.git import commit_imported_notes
from ingestion.integrations.orgmylife import OrgMyLifeClient
from ingestion.linking.linker import generate_wiki_links
from ingestion.linking.stubs import create_stub_pages
from ingestion.output.naming import generate_filename, resolve_collision
from ingestion.output.renderer import render_note
from ingestion.output.router import route_to_folder
logger = logging.getLogger(__name__)
@dataclass
class ProcessResult:
"""Result of processing a single file."""
source_file: Path
output_file: Path | None = None
success: bool = False
error: str | None = None
stub_pages: list[Path] = field(default_factory=list)
@dataclass
class BatchResult:
"""Summary of a batch processing run."""
total: int = 0
success: int = 0
failed: int = 0
results: list[ProcessResult] = field(default_factory=list)
def process_file(
file_path: Path,
config: IngestionConfig,
enrichment_agent: EnrichmentAgent | None = None,
dry_run: bool = False,
no_commit: bool = False,
create_tasks: bool = False,
output_dir: Path | None = None,
) -> ProcessResult:
"""Process a single file through the full ingestion pipeline.
Steps: extract → enrich → link → render → write → (optional) git commit
Args:
file_path: Path to the source file.
config: Ingestion configuration.
enrichment_agent: Optional pre-configured enrichment agent.
dry_run: If True, don't write any files.
no_commit: If True, skip git commit.
create_tasks: If True, create OrgMyLife tasks for action items.
output_dir: Optional override for output directory.
Returns:
ProcessResult with success status and output path.
"""
result = ProcessResult(source_file=file_path)
try:
# 1. Extract text
extractor = get_extractor(file_path, config)
if extractor is None:
result.error = f"No extractor available for {file_path.suffix}"
return result
extraction = extractor.extract(file_path)
if not extraction.text.strip():
result.error = "No text extracted from file"
return result
# 2. Enrich with LLM
if enrichment_agent is None:
enrichment_agent = EnrichmentAgent(config)
enrichment = enrichment_agent.enrich(extraction.text)
# 3. Generate wiki-links
linked_text = generate_wiki_links(extraction.text, enrichment.entities)
# 4. Render output
rendered = render_note(enrichment, linked_text, file_path)
# 5. Determine output location
notes_dir = Path(config.notes_dir)
target_folder = route_to_folder(enrichment.category, notes_dir, output_dir)
# 6. Generate filename
date_entities = [e for e in enrichment.entities if e.type == "date"]
date_str = date_entities[0].value if date_entities else None
filename = generate_filename(enrichment.title, date_str)
output_path = resolve_collision(target_folder, filename)
if dry_run:
logger.info("[DRY RUN] Would write: %s", output_path)
result.output_file = output_path
result.success = True
return result
# 7. Write file
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(rendered, encoding="utf-8")
result.output_file = output_path
logger.info("Written: %s", output_path)
# 8. Create stub pages
stubs = create_stub_pages(enrichment.entities, notes_dir)
result.stub_pages = stubs
# 9. Create OrgMyLife tasks if requested
if create_tasks and enrichment.action_items and config.orgmylife_api_key:
_create_tasks(enrichment, output_path, config)
result.success = True
except Exception as e:
logger.error("Failed to process %s: %s", file_path, e)
result.error = str(e)
return result
def process_batch(
paths: list[Path],
config: IngestionConfig,
dry_run: bool = False,
no_commit: bool = False,
create_tasks: bool = False,
output_dir: Path | None = None,
progress_callback=None,
) -> BatchResult:
"""Process a batch of files through the pipeline.
Args:
paths: List of file paths to process.
config: Ingestion configuration.
dry_run: If True, don't write any files.
no_commit: If True, skip git commit.
create_tasks: If True, create OrgMyLife tasks.
output_dir: Optional override for output directory.
progress_callback: Optional callable(current, total, filename).
Returns:
BatchResult with summary counts and per-file results.
"""
batch = BatchResult(total=len(paths))
enrichment_agent = EnrichmentAgent(config)
written_files: list[Path] = []
for i, file_path in enumerate(paths, 1):
if progress_callback:
progress_callback(i, len(paths), file_path.name)
logger.info("Processing [%d/%d]: %s", i, len(paths), file_path.name)
result = process_file(
file_path=file_path,
config=config,
enrichment_agent=enrichment_agent,
dry_run=dry_run,
no_commit=True, # Commit once at end
create_tasks=create_tasks,
output_dir=output_dir,
)
batch.results.append(result)
if result.success:
batch.success += 1
if result.output_file:
written_files.append(result.output_file)
written_files.extend(result.stub_pages)
else:
batch.failed += 1
logger.warning("Skipped %s: %s", file_path.name, result.error)
# Git commit all at once (unless dry-run or no-commit)
if not dry_run and not no_commit and config.auto_commit and written_files:
commit_imported_notes(written_files, source_type="batch-import")
return batch
def _create_tasks(
enrichment: EnrichmentResult, note_path: Path, config: IngestionConfig
) -> None:
"""Create OrgMyLife tasks for action items."""
try:
client = OrgMyLifeClient(
base_url=config.orgmylife_api_url,
api_key=config.orgmylife_api_key,
)
for item in enrichment.action_items:
client.create_task(
title=item.description,
source_url=str(note_path),
description=f"From note: {enrichment.title}",
)
except Exception as e:
logger.warning("Failed to create OrgMyLife tasks: %s", e)
@@ -0,0 +1,40 @@
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project]
name = "notegraph-ingestion"
version = "0.1.0"
description = "NoteGraph document ingestion and AI-powered auto-referencing pipeline"
requires-python = ">=3.11"
dependencies = [
"click>=8.1",
"watchdog>=4.0",
"pytesseract>=0.3.10",
"pdfplumber>=0.11",
"python-docx>=1.1",
"litellm>=1.40",
"httpx>=0.27",
"pydantic>=2.7",
"pydantic-settings>=2.3",
"fastapi>=0.111",
"uvicorn>=0.30",
"python-multipart>=0.0.9",
"pyyaml>=6.0",
"python-slugify>=8.0",
]
[project.optional-dependencies]
dev = [
"pytest>=8.0",
"pytest-asyncio>=0.23",
"hypothesis>=6.100",
"pytest-mock>=3.12",
"respx>=0.21",
]
[tool.hatch.build.targets.wheel]
packages = ["ingestion"]
[project.scripts]
notegraph-ingest = "ingestion.cli:cli"
@@ -0,0 +1 @@
"""Tests for the NoteGraph ingestion pipeline."""
@@ -0,0 +1,83 @@
"""Shared test fixtures for the ingestion test suite."""
import pytest
from pathlib import Path
from unittest.mock import MagicMock
from ingestion.config import IngestionConfig
from ingestion.enrichment.models import ActionItem, EnrichmentResult, Entity
@pytest.fixture
def tmp_notes_dir(tmp_path: Path) -> Path:
"""Create a temporary notes directory structure."""
notes = tmp_path / "notes"
notes.mkdir()
(notes / "inbox").mkdir()
(notes / "meetings").mkdir()
(notes / "projects").mkdir()
(notes / "people").mkdir()
(notes / "decisions").mkdir()
return notes
@pytest.fixture
def tmp_inbox(tmp_path: Path) -> Path:
"""Create a temporary inbox directory."""
inbox = tmp_path / "inbox"
inbox.mkdir()
return inbox
@pytest.fixture
def mock_config(tmp_path: Path) -> IngestionConfig:
"""Create a test configuration with temporary paths."""
notes_dir = tmp_path / "notes"
notes_dir.mkdir(exist_ok=True)
inbox_dir = tmp_path / "inbox"
inbox_dir.mkdir(exist_ok=True)
return IngestionConfig(
llm_provider="openai",
llm_model="gpt-4o",
openai_api_key="test-key-fake",
notes_dir=str(notes_dir),
inbox_dir=str(inbox_dir),
auto_commit=False,
)
@pytest.fixture
def sample_enrichment_result() -> EnrichmentResult:
"""Create a sample enrichment result for testing."""
return EnrichmentResult(
title="Meeting with Beier GmbH",
category="meeting",
tags=["meeting", "customer"],
entities=[
Entity(type="person", value="Thomas Beier", confidence=0.95),
Entity(type="project", value="Druckluft", confidence=0.9),
Entity(type="date", value="2024-03-15", confidence=0.99),
],
action_items=[
ActionItem(description="Send updated proposal", assignee="André", deadline="2024-03-22"),
ActionItem(description="Schedule follow-up meeting"),
],
summary="Discussion about the Druckluft project.",
)
@pytest.fixture
def mock_llm_client():
"""Create a mock LLM client that returns a valid enrichment JSON."""
import json
mock = MagicMock()
mock.complete.return_value = json.dumps({
"title": "Test Note",
"category": "inbox",
"tags": ["test"],
"entities": [],
"action_items": [],
"summary": "A test note.",
})
return mock
@@ -0,0 +1,806 @@
"""Property-based tests for NoteGraph Ingestion using Hypothesis.
All 18 correctness properties from the design document are tested here.
Each test uses @settings(max_examples=100) as specified.
"""
import json
import re
import string
import tempfile
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
import yaml
from hypothesis import given, settings, assume, HealthCheck
from hypothesis import strategies as st
from ingestion.config import IngestionConfig
from ingestion.discovery import SUPPORTED_EXTENSIONS, discover_files
from ingestion.enrichment.models import ActionItem, EnrichmentResult, Entity
from ingestion.extraction.text import PlainTextExtractor
from ingestion.integrations.git import commit_imported_notes
from ingestion.integrations.orgmylife import OrgMyLifeClient
from ingestion.linking.linker import generate_wiki_links, _entity_to_wikilink
from ingestion.output.naming import generate_filename, resolve_collision
from ingestion.output.renderer import render_note
from ingestion.output.router import CATEGORY_FOLDERS, route_to_folder
from ingestion.watcher import should_ignore
# ---------------------------------------------------------------------------
# Common settings for all property tests
# ---------------------------------------------------------------------------
pbt_settings = settings(
max_examples=100,
suppress_health_check=[HealthCheck.function_scoped_fixture],
)
# ---------------------------------------------------------------------------
# Strategies
# ---------------------------------------------------------------------------
# Strategy for supported file extensions
supported_ext_st = st.sampled_from(sorted(SUPPORTED_EXTENSIONS))
# Strategy for unsupported file extensions
unsupported_ext_st = st.text(
alphabet=string.ascii_lowercase, min_size=2, max_size=5
).map(lambda s: f".{s}").filter(lambda ext: ext not in SUPPORTED_EXTENSIONS)
# Strategy for valid filenames (no path separators, no null bytes, no reserved chars)
filename_chars = st.characters(
whitelist_categories=("L", "N"),
)
filename_st = st.text(alphabet=filename_chars, min_size=1, max_size=20).map(
lambda s: s.strip() or "file"
)
# Strategy for valid UTF-8 text content (no null bytes, non-empty)
utf8_text_st = st.text(
alphabet=st.characters(blacklist_categories=("Cs",), blacklist_characters="\x00"),
min_size=1,
max_size=2000,
)
# Strategy for entity types
entity_type_st = st.sampled_from(["person", "project", "date", "action_item"])
# Strategy for confidence scores
confidence_st = st.floats(min_value=0.0, max_value=1.0, allow_nan=False)
# Strategy for person/project names (non-empty, printable, with at least one letter)
name_st = st.text(
alphabet=st.characters(whitelist_categories=("L", "N", "Zs")),
min_size=2,
max_size=40,
).filter(lambda s: s.strip() and any(c.isalpha() for c in s))
# Strategy for valid providers
provider_st = st.sampled_from(["openai", "anthropic", "google", "mistral", "ollama"])
# Strategy for providers requiring API keys
keyed_provider_st = st.sampled_from(["openai", "anthropic", "google", "mistral"])
# Strategy for categories
category_st = st.sampled_from(["meeting", "project", "decision", "inbox"])
# Strategy for dates in YYYY-MM-DD format
date_st = st.dates().map(lambda d: d.strftime("%Y-%m-%d"))
# Strategy for note titles (must produce a non-empty slug)
title_st = st.text(
alphabet=st.characters(whitelist_categories=("L", "N", "Zs")),
min_size=2,
max_size=60,
).filter(lambda s: s.strip() and any(c.isalnum() for c in s))
# Strategy for action item descriptions
action_desc_st = st.text(
alphabet=st.characters(whitelist_categories=("L", "N", "Zs", "P")),
min_size=3,
max_size=100,
).filter(lambda s: s.strip())
# Strategy for source types
source_type_st = st.text(
alphabet=string.ascii_lowercase + "-",
min_size=3,
max_size=20,
).filter(lambda s: s.strip("-") and not s.startswith("-") and not s.endswith("-"))
# ---------------------------------------------------------------------------
# Property 1 (Task 3.2): File discovery filtering
# ---------------------------------------------------------------------------
# Feature: notegraph-ingestion, Property 1: File discovery returns only supported extensions
@pbt_settings
@given(
supported_files=st.lists(
st.tuples(filename_st, supported_ext_st), min_size=0, max_size=10
),
unsupported_files=st.lists(
st.tuples(filename_st, unsupported_ext_st), min_size=0, max_size=10
),
)
def test_discovery_returns_only_supported_extensions(
tmp_path: Path, supported_files, unsupported_files
):
"""For any directory tree with arbitrary extensions, discovery returns exactly
those with supported extensions (.pdf, .jpg, .jpeg, .png, .md, .txt, .docx)."""
with tempfile.TemporaryDirectory() as td:
td_path = Path(td)
# Create files with supported extensions
expected = set()
for i, (name, ext) in enumerate(supported_files):
fp = td_path / f"s{i}_{name}{ext}"
fp.write_text("content", encoding="utf-8")
expected.add(fp)
# Create files with unsupported extensions
for i, (name, ext) in enumerate(unsupported_files):
fp = td_path / f"u{i}_{name}{ext}"
fp.write_text("content", encoding="utf-8")
result = discover_files(td_path)
result_set = set(result)
# All returned files must have supported extensions
for f in result:
assert f.suffix.lower() in SUPPORTED_EXTENSIONS
# All supported files we created must be in the result
for f in expected:
assert f in result_set
# ---------------------------------------------------------------------------
# Property 2 (Task 9.2): Bulk import resilience
# ---------------------------------------------------------------------------
# Feature: notegraph-ingestion, Property 2: Failed files do not block remaining processing
@pbt_settings
@given(
fail_indices=st.lists(st.integers(min_value=0, max_value=4), unique=True, max_size=3),
)
def test_bulk_import_resilience(tmp_path: Path, fail_indices, mock_config):
"""For any batch where a subset fails, remaining files are processed successfully."""
from ingestion.pipeline import process_batch
with tempfile.TemporaryDirectory() as td:
td_path = Path(td)
# Create 5 text files
files = []
for i in range(5):
f = td_path / f"note_{i}.txt"
f.write_text(f"Content of note {i}", encoding="utf-8")
files.append(f)
# Normalize fail indices to valid range
fail_set = {i for i in fail_indices if 0 <= i < 5}
# Mock enrichment agent that fails for specific indices
call_count = {"n": 0}
def mock_enrich(text):
idx = call_count["n"]
call_count["n"] += 1
if idx in fail_set:
raise RuntimeError(f"Simulated failure for file {idx}")
return EnrichmentResult(
title=f"Note {idx}",
category="inbox",
tags=[],
entities=[],
action_items=[],
summary=None,
)
mock_agent = MagicMock()
mock_agent.enrich.side_effect = mock_enrich
# Set up output directory
notes_dir = td_path / "notes"
notes_dir.mkdir()
(notes_dir / "inbox").mkdir()
mock_config.notes_dir = str(notes_dir)
with patch("ingestion.pipeline.EnrichmentAgent", return_value=mock_agent):
result = process_batch(
paths=files,
config=mock_config,
dry_run=False,
no_commit=True,
)
expected_success = 5 - len(fail_set)
assert result.success == expected_success
assert result.failed == len(fail_set)
assert result.total == 5
# ---------------------------------------------------------------------------
# Property 3 (Task 2.6): Text extraction identity
# ---------------------------------------------------------------------------
# Feature: notegraph-ingestion, Property 3: Text extraction preserves content for text-based formats
@pbt_settings
@given(content=utf8_text_st)
def test_text_extraction_identity(tmp_path: Path, content: str):
"""For any valid UTF-8 string written to a .md or .txt file, extraction returns
identical content (modulo platform newline normalization)."""
with tempfile.TemporaryDirectory() as td:
td_path = Path(td)
extractor = PlainTextExtractor()
for ext in (".md", ".txt"):
fp = td_path / f"test{ext}"
fp.write_text(content, encoding="utf-8")
result = extractor.extract(fp)
# Python's read_text uses universal newlines (translates \r\n and \r to \n)
# This is expected platform behavior, so we compare after normalization
expected = content.replace("\r\n", "\n").replace("\r", "\n")
assert result.text == expected
# ---------------------------------------------------------------------------
# Property 4 (Task 5.4): Entity parsing valid structure
# ---------------------------------------------------------------------------
# Feature: notegraph-ingestion, Property 4: Entity parsing extracts all entities with valid structure
@pbt_settings
@given(
entities=st.lists(
st.fixed_dictionaries({
"type": entity_type_st,
"value": name_st,
"confidence": confidence_st,
}),
min_size=1,
max_size=10,
),
)
def test_entity_parsing_valid_structure(entities):
"""For any valid enrichment response JSON, each parsed entity has valid type,
non-empty value, confidence in [0.0, 1.0]."""
data = {
"title": "Test",
"category": "inbox",
"tags": [],
"entities": entities,
"action_items": [],
}
result = EnrichmentResult.model_validate(data)
for entity in result.entities:
assert entity.type in ("person", "project", "date", "action_item")
assert len(entity.value) > 0
assert 0.0 <= entity.confidence <= 1.0
# ---------------------------------------------------------------------------
# Property 5 (Task 5.4): Confidence threshold filtering
# ---------------------------------------------------------------------------
# Feature: notegraph-ingestion, Property 5: Confidence threshold filtering
@pbt_settings
@given(
scores=st.lists(confidence_st, min_size=1, max_size=20),
)
def test_confidence_threshold_filtering(scores):
"""For any list of entities with arbitrary confidence scores, filtered output
contains exactly those with confidence >= 0.7."""
entities = [
Entity(type="person", value=f"Person {i}", confidence=score)
for i, score in enumerate(scores)
]
threshold = 0.7
filtered = [e for e in entities if e.confidence >= threshold]
# Verify the filter matches what the enrichment agent does
from ingestion.enrichment.agent import EnrichmentAgent
agent = MagicMock(spec=EnrichmentAgent)
agent.confidence_threshold = threshold
agent._filter_by_confidence = EnrichmentAgent._filter_by_confidence.__get__(agent)
result = agent._filter_by_confidence(entities)
assert len(result) == len(filtered)
for entity in result:
assert entity.confidence >= threshold
# Verify no entity below threshold is included
for entity in entities:
if entity.confidence < threshold:
assert entity not in result
# ---------------------------------------------------------------------------
# Property 6 (Task 7.3): Wiki-link format per entity type
# ---------------------------------------------------------------------------
# Feature: notegraph-ingestion, Property 6: Wiki-link generation follows correct format per entity type
@pbt_settings
@given(name=name_st)
def test_wikilink_format_person(name: str):
"""For any person name → [[people/slugified-name]]."""
from slugify import slugify
entity = Entity(type="person", value=name, confidence=0.9)
link = _entity_to_wikilink(entity)
expected_slug = slugify(name)
assume(expected_slug) # skip if slugify produces empty string
assert link == f"[[people/{expected_slug}]]"
@pbt_settings
@given(name=name_st)
def test_wikilink_format_project(name: str):
"""For any project name → [[projects/slugified-name]]."""
from slugify import slugify
entity = Entity(type="project", value=name, confidence=0.9)
link = _entity_to_wikilink(entity)
expected_slug = slugify(name)
assume(expected_slug) # skip if slugify produces empty string
assert link == f"[[projects/{expected_slug}]]"
# ---------------------------------------------------------------------------
# Property 7 (Task 7.3): Only first occurrence linked
# ---------------------------------------------------------------------------
# Feature: notegraph-ingestion, Property 7: Only first entity occurrence is linked
@pbt_settings
@given(
name=name_st,
n_occurrences=st.integers(min_value=2, max_value=5),
)
def test_only_first_occurrence_linked(name: str, n_occurrences: int):
"""For any text with N >= 2 occurrences of the same entity, only the first
is converted to a wiki-link."""
from slugify import slugify
slug = slugify(name)
assume(slug) # skip if slugify produces empty string
# Build text with multiple occurrences separated by unique text
separator = ". Then we discussed "
text = separator.join([name] * n_occurrences)
entity = Entity(type="person", value=name, confidence=0.9)
result = generate_wiki_links(text, [entity])
expected_link = f"[[people/{slug}]]"
# First occurrence should be a wiki-link
assert expected_link in result
# Count wiki-links — should be exactly 1
link_count = result.count(expected_link)
assert link_count == 1, f"Expected 1 wiki-link, found {link_count}"
# ---------------------------------------------------------------------------
# Property 8 (Task 8.4): Frontmatter round-trip validity
# ---------------------------------------------------------------------------
# Feature: notegraph-ingestion, Property 8: Frontmatter round-trip validity
@pbt_settings
@given(
title=title_st,
category=category_st,
tags=st.lists(st.text(alphabet=string.ascii_lowercase, min_size=2, max_size=10), max_size=5),
people=st.lists(name_st, min_size=0, max_size=3),
projects=st.lists(name_st, min_size=0, max_size=3),
)
def test_frontmatter_roundtrip_validity(
tmp_path: Path, title, category, tags, people, projects
):
"""For any valid EnrichmentResult, render to markdown then parse frontmatter →
valid YAML with required fields."""
with tempfile.TemporaryDirectory() as td:
td_path = Path(td)
entities = []
for p in people:
entities.append(Entity(type="person", value=p, confidence=0.9))
for proj in projects:
entities.append(Entity(type="project", value=proj, confidence=0.9))
entities.append(Entity(type="date", value="2024-06-15", confidence=0.99))
enrichment = EnrichmentResult(
title=title,
category=category,
tags=tags,
entities=entities,
action_items=[],
summary=None,
)
source_file = td_path / "source.pdf"
source_file.write_text("", encoding="utf-8")
rendered = render_note(enrichment, "Body text here.", source_file)
# Parse frontmatter
assert rendered.startswith("---\n")
parts = rendered.split("---\n", 2)
assert len(parts) >= 3, "Expected YAML frontmatter delimiters"
frontmatter_yaml = parts[1]
parsed = yaml.safe_load(frontmatter_yaml)
# Required fields
assert "title" in parsed
assert "date" in parsed
assert "source" in parsed
assert "file" in parsed["source"]
assert "imported" in parsed["source"]
# Title matches
assert parsed["title"] == title
# ---------------------------------------------------------------------------
# Property 9 (Task 4.2): Provider routing from configuration
# ---------------------------------------------------------------------------
# Feature: notegraph-ingestion, Property 9: Provider routing from configuration
@pbt_settings
@given(provider=provider_st)
def test_provider_routing_from_configuration(provider: str):
"""For any valid provider name (openai, anthropic, google, mistral, ollama),
configuring LLM_PROVIDER produces the correct model string."""
config = IngestionConfig(
llm_provider=provider,
llm_model="test-model",
openai_api_key="fake",
anthropic_api_key="fake",
google_api_key="fake",
mistral_api_key="fake",
)
model_string = config.llm_model_string
if provider == "ollama":
assert model_string == "ollama/test-model"
else:
assert model_string == f"{provider}/test-model"
# Verify the provider prefix is correct
assert model_string.startswith(f"{provider}/")
# ---------------------------------------------------------------------------
# Property 10 (Task 4.3): Missing API key error
# ---------------------------------------------------------------------------
# Feature: notegraph-ingestion, Property 10: Missing API key produces descriptive error
@pbt_settings
@given(provider=keyed_provider_st)
def test_missing_api_key_error(provider: str):
"""For any provider requiring an API key, when the key is empty,
validate_api_key() raises SystemExit naming the missing variable."""
config = IngestionConfig(
llm_provider=provider,
llm_model="test-model",
openai_api_key="",
anthropic_api_key="",
google_api_key="",
mistral_api_key="",
)
env_var_map = {
"openai": "OPENAI_API_KEY",
"anthropic": "ANTHROPIC_API_KEY",
"google": "GOOGLE_API_KEY",
"mistral": "MISTRAL_API_KEY",
}
with pytest.raises(SystemExit) as exc_info:
config.validate_api_key()
error_msg = str(exc_info.value)
assert env_var_map[provider] in error_msg
# ---------------------------------------------------------------------------
# Property 11 (Task 8.4): Action items in output
# ---------------------------------------------------------------------------
# Feature: notegraph-ingestion, Property 11: Action items appear in output markdown section
@pbt_settings
@given(
descriptions=st.lists(action_desc_st, min_size=1, max_size=5),
)
def test_action_items_in_output(tmp_path: Path, descriptions):
"""For any list of action items, rendered markdown contains `## Action Items`
section listing every item."""
with tempfile.TemporaryDirectory() as td:
td_path = Path(td)
action_items = [ActionItem(description=desc) for desc in descriptions]
enrichment = EnrichmentResult(
title="Test Note",
category="inbox",
tags=[],
entities=[Entity(type="date", value="2024-01-01", confidence=0.99)],
action_items=action_items,
summary=None,
)
source_file = td_path / "source.txt"
source_file.write_text("", encoding="utf-8")
rendered = render_note(enrichment, "Body text.", source_file)
assert "## Action Items" in rendered
for desc in descriptions:
assert desc in rendered
# ---------------------------------------------------------------------------
# Property 12 (Task 15.2): OrgMyLife task payload
# ---------------------------------------------------------------------------
# Feature: notegraph-ingestion, Property 12: OrgMyLife task payload correctness
@pbt_settings
@given(
description=action_desc_st,
note_path=st.text(
alphabet=string.ascii_lowercase + "/.-_", min_size=5, max_size=50
).filter(lambda s: s.strip() and not s.startswith("/")),
)
def test_orgmylife_task_payload(description: str, note_path: str):
"""For any action item description and note path, the task creation payload
has correct title and source_url."""
import httpx
client = OrgMyLifeClient(base_url="https://api.example.com", api_key="test-key")
# Capture the payload sent to httpx.post
captured_payload = {}
def mock_post(url, json=None, headers=None, timeout=None):
captured_payload.update(json)
response = MagicMock()
response.status_code = 201
response.raise_for_status = MagicMock()
response.json.return_value = {"id": 1, "title": json["title"]}
return response
with patch.object(httpx, "post", side_effect=mock_post):
client.create_task(title=description, source_url=note_path)
assert captured_payload["title"] == description
assert captured_payload["source_url"] == note_path
# ---------------------------------------------------------------------------
# Property 13 (Task 8.4): Category-based folder routing
# ---------------------------------------------------------------------------
# Feature: notegraph-ingestion, Property 13: Category-based folder routing
@pbt_settings
@given(category=category_st)
def test_category_based_folder_routing(tmp_path: Path, category: str):
"""For any category (meeting/project/decision/inbox), output path is within
the correct folder."""
with tempfile.TemporaryDirectory() as td:
notes_dir = Path(td) / "notes"
notes_dir.mkdir()
result = route_to_folder(category, notes_dir)
expected_folder = CATEGORY_FOLDERS.get(category, "inbox")
assert result == notes_dir / expected_folder
assert result.exists()
# ---------------------------------------------------------------------------
# Property 14 (Task 8.4): Filename date-slug pattern
# ---------------------------------------------------------------------------
# Feature: notegraph-ingestion, Property 14: Output filename follows date-slug pattern
@pbt_settings
@given(date=date_st, title=title_st)
def test_filename_date_slug_pattern(date: str, title: str):
"""For any date and title, filename matches YYYY-MM-DD-slugified-title.md."""
filename = generate_filename(title, date)
# Must end with .md
assert filename.endswith(".md")
# Must start with the date
assert filename.startswith(date)
# Pattern: YYYY-MM-DD-slug.md (slug is lowercase alphanumeric + hyphens)
pattern = r"^\d{4}-\d{2}-\d{2}-[a-z0-9]([a-z0-9-]*[a-z0-9])?\.md$"
assert re.match(pattern, filename), f"Filename '{filename}' doesn't match pattern"
# ---------------------------------------------------------------------------
# Property 15 (Task 8.4): Filename collision avoidance
# ---------------------------------------------------------------------------
# Feature: notegraph-ingestion, Property 15: Filename collision avoidance
@pbt_settings
@given(
title=title_st,
n_existing=st.integers(min_value=1, max_value=5),
)
def test_filename_collision_avoidance(tmp_path: Path, title: str, n_existing: int):
"""When a file with the same name exists, a numeric suffix is appended."""
with tempfile.TemporaryDirectory() as td:
td_path = Path(td)
filename = generate_filename(title, "2024-01-15")
# Create the original file
(td_path / filename).write_text("existing", encoding="utf-8")
# Create collision files with suffixes
stem = filename.rsplit(".md", 1)[0]
for i in range(2, n_existing + 1):
(td_path / f"{stem}-{i}.md").write_text("existing", encoding="utf-8")
# Resolve collision
resolved = resolve_collision(td_path, filename)
# The resolved path must not exist yet
assert not resolved.exists()
# The resolved path must have a numeric suffix
expected_suffix = f"-{n_existing + 1}.md"
assert str(resolved).endswith(expected_suffix)
# The resolved path must be in the same directory
assert resolved.parent == td_path
# ---------------------------------------------------------------------------
# Property 16 (Task 14.2): Temporary file filtering
# ---------------------------------------------------------------------------
# Feature: notegraph-ingestion, Property 16: Temporary file filtering
@pbt_settings
@given(
filename=st.text(
alphabet=st.characters(
whitelist_categories=("L", "N", "P"),
blacklist_characters="/\\:\x00",
),
min_size=1,
max_size=50,
),
)
def test_temporary_file_filtering(filename: str):
"""For any filename, should_ignore returns true iff name starts with '.'
or ends with '.tmp'."""
result = should_ignore(filename)
expected = filename.startswith(".") or filename.endswith(".tmp")
assert result == expected, f"should_ignore('{filename}') = {result}, expected {expected}"
# ---------------------------------------------------------------------------
# Property 17 (Task 9.2): Dry-run no file writes
# ---------------------------------------------------------------------------
# Feature: notegraph-ingestion, Property 17: Dry-run produces no file writes
@pbt_settings
@given(
n_files=st.integers(min_value=1, max_value=5),
)
def test_dry_run_no_file_writes(tmp_path: Path, n_files: int, mock_config):
"""For any input with dry_run=True, zero files are created."""
from ingestion.pipeline import process_batch
with tempfile.TemporaryDirectory() as td:
td_path = Path(td)
# Create input files
files = []
for i in range(n_files):
f = td_path / f"input_{i}.txt"
f.write_text(f"Content {i}", encoding="utf-8")
files.append(f)
# Set up output directory
notes_dir = td_path / "output_notes"
notes_dir.mkdir()
(notes_dir / "inbox").mkdir()
mock_config.notes_dir = str(notes_dir)
# Mock the enrichment agent
def mock_enrich(text):
return EnrichmentResult(
title="Test",
category="inbox",
tags=[],
entities=[],
action_items=[],
summary=None,
)
mock_agent = MagicMock()
mock_agent.enrich.side_effect = mock_enrich
with patch("ingestion.pipeline.EnrichmentAgent", return_value=mock_agent):
result = process_batch(
paths=files,
config=mock_config,
dry_run=True,
no_commit=True,
)
# Count files in the output notes directory (should be 0 new markdown files)
output_files = list(notes_dir.rglob("*.md"))
assert len(output_files) == 0, f"Expected 0 files written, found {len(output_files)}"
# ---------------------------------------------------------------------------
# Property 18 (Task 16.2): Git commit message format
# ---------------------------------------------------------------------------
# Feature: notegraph-ingestion, Property 18: Git commit message format
@pbt_settings
@given(
count=st.integers(min_value=1, max_value=1000),
source_type=source_type_st,
)
def test_git_commit_message_format(count: int, source_type: str):
"""For any count N and source type, message matches
'ingestion: import N notes from [source-type]'."""
files = [Path(f"/fake/note_{i}.md") for i in range(count)]
captured_messages = []
def mock_run(cmd, **kwargs):
if cmd[0] == "git" and cmd[1] == "commit":
# Extract -m argument
msg_idx = cmd.index("-m") + 1
captured_messages.append(cmd[msg_idx])
result = MagicMock()
result.returncode = 0
result.stdout = ""
result.stderr = ""
return result
with patch("subprocess.run", side_effect=mock_run):
commit_imported_notes(files, source_type=source_type)
assert len(captured_messages) == 1
expected = f"ingestion: import {count} notes from {source_type}"
assert captured_messages[0] == expected
@@ -0,0 +1,124 @@
"""Drop-folder watcher using watchdog for continuous file ingestion.
Monitors the inbox folder for new files and triggers the processing pipeline.
Moves processed files to archive/ and failed files to failed/ subdirectory.
Ignores temporary files (names starting with '.' or ending with '.tmp').
"""
import logging
import shutil
import time
from pathlib import Path
from watchdog.events import FileCreatedEvent, FileSystemEventHandler
from watchdog.observers import Observer
from ingestion.config import IngestionConfig
from ingestion.discovery import SUPPORTED_EXTENSIONS
logger = logging.getLogger(__name__)
def should_ignore(filename: str) -> bool:
"""Check if a file should be ignored by the watcher.
Ignores files starting with '.' or ending with '.tmp'.
"""
return filename.startswith(".") or filename.endswith(".tmp")
class InboxHandler(FileSystemEventHandler):
"""Handle new files in the inbox directory."""
def __init__(self, config: IngestionConfig):
self.config = config
self.inbox_dir = Path(config.inbox_dir)
self.archive_dir = self.inbox_dir / "archive"
self.failed_dir = self.inbox_dir / "failed"
self.archive_dir.mkdir(parents=True, exist_ok=True)
self.failed_dir.mkdir(parents=True, exist_ok=True)
def on_created(self, event: FileCreatedEvent):
"""Triggered when a new file is created in the inbox."""
if event.is_directory:
return
file_path = Path(event.src_path)
self._process_file(file_path)
def _process_file(self, file_path: Path) -> None:
"""Process a single file and move to archive or failed."""
if should_ignore(file_path.name):
logger.debug("Ignoring temporary file: %s", file_path.name)
return
if file_path.suffix.lower() not in SUPPORTED_EXTENSIONS:
logger.debug("Ignoring unsupported file: %s", file_path.name)
return
# Small delay to ensure file is fully written
time.sleep(0.5)
logger.info("Processing new file: %s", file_path.name)
try:
from ingestion.pipeline import process_file
result = process_file(file_path=file_path, config=self.config)
if result.success:
# Move to archive
dest = self.archive_dir / file_path.name
shutil.move(str(file_path), str(dest))
logger.info("Archived: %s%s", file_path.name, dest)
else:
# Move to failed
dest = self.failed_dir / file_path.name
shutil.move(str(file_path), str(dest))
logger.warning(
"Failed: %s%s (error: %s)",
file_path.name,
dest,
result.error,
)
except Exception as e:
logger.error("Unhandled error processing %s: %s", file_path.name, e)
dest = self.failed_dir / file_path.name
try:
shutil.move(str(file_path), str(dest))
except Exception:
pass
def start_watcher(config: IngestionConfig) -> None:
"""Start the drop-folder watcher.
Processes existing files first, then watches for new ones.
Blocks until interrupted (Ctrl+C).
"""
inbox_dir = Path(config.inbox_dir)
inbox_dir.mkdir(parents=True, exist_ok=True)
handler = InboxHandler(config)
# Process existing files on startup
existing = sorted(inbox_dir.iterdir())
for f in existing:
if f.is_file() and not should_ignore(f.name):
handler._process_file(f)
# Start watching
observer = Observer()
observer.schedule(handler, str(inbox_dir), recursive=False)
observer.start()
logger.info("Watcher started on: %s", inbox_dir)
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
@@ -0,0 +1 @@
"""FastAPI web upload endpoint for mobile file ingestion."""
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,233 @@
"""FastAPI web upload endpoint for mobile file ingestion.
Auth: Token-based via URL query param (?token=secret) or header (X-Upload-Token).
Processing: File is saved immediately, enrichment runs in background.
"""
import logging
import secrets
import threading
from pathlib import Path
from fastapi import FastAPI, File, HTTPException, Query, Request, UploadFile
from fastapi.responses import HTMLResponse
from ingestion.config import IngestionConfig
from ingestion.web.browse import router as browse_router
logger = logging.getLogger(__name__)
app = FastAPI(title="NoteGraph", version="0.1.0")
app.include_router(browse_router)
@app.get("/")
async def root(request: Request):
"""Redirect root to browse page."""
from fastapi.responses import RedirectResponse
token = request.query_params.get("token", "")
return RedirectResponse(url=f"/browse?token={token}" if token else "/browse")
def _get_config() -> IngestionConfig:
return IngestionConfig()
def _verify_token(request: Request, token: str = Query(default=None)):
"""Verify upload token from query param or X-Upload-Token header."""
config = _get_config()
# Token is the password part of SB_USER (after the colon)
sb_user = config.sb_user.strip()
parts = sb_user.split(":", 1)
expected_token = parts[1].strip() if len(parts) == 2 else sb_user
# Check query param
if token and secrets.compare_digest(token.encode(), expected_token.encode()):
return True
# Check header
header_token = request.headers.get("X-Upload-Token", "")
if header_token and secrets.compare_digest(header_token.encode(), expected_token.encode()):
return True
raise HTTPException(status_code=401, detail="Invalid or missing token. Use ?token=YOUR_PASSWORD")
def _process_in_background(file_path: Path, config: IngestionConfig):
"""Run the enrichment pipeline in a background thread."""
try:
from ingestion.pipeline import process_file
result = process_file(file_path=file_path, config=config)
if result.success:
logger.info("Background processing complete: %s%s", file_path.name, result.output_file)
else:
logger.warning("Background processing failed for %s: %s", file_path.name, result.error)
except Exception as e:
logger.error("Background processing error for %s: %s", file_path.name, e)
UPLOAD_HTML = """<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>NoteGraph Upload</title>
<style>
* { box-sizing: border-box; margin: 0; padding: 0; }
body {
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
background: #1a1a2e;
color: #eee;
min-height: 100vh;
display: flex;
align-items: center;
justify-content: center;
padding: 1rem;
}
.container {
background: #16213e;
border-radius: 12px;
padding: 2rem;
width: 100%;
max-width: 480px;
box-shadow: 0 8px 32px rgba(0,0,0,0.3);
}
h1 { text-align: center; margin-bottom: 1.5rem; font-size: 1.5rem; color: #4fc3f7; }
.drop-zone {
border: 2px dashed #4fc3f7;
border-radius: 8px;
padding: 2rem 1rem;
text-align: center;
cursor: pointer;
transition: all 0.3s;
margin-bottom: 1rem;
}
.drop-zone:hover, .drop-zone.dragover {
background: rgba(79, 195, 247, 0.1);
border-color: #81d4fa;
}
.drop-zone p { margin-bottom: 0.5rem; font-size: 1.1rem; }
.drop-zone small { color: #999; }
input[type="file"] { display: none; }
.btn {
display: block; width: 100%; padding: 0.9rem;
background: #4fc3f7; color: #1a1a2e; border: none;
border-radius: 8px; font-size: 1rem; font-weight: 600;
cursor: pointer; transition: background 0.3s;
}
.btn:hover { background: #81d4fa; }
.btn:disabled { background: #555; cursor: not-allowed; }
.status { margin-top: 1rem; padding: 0.8rem; border-radius: 6px; text-align: center; display: none; }
.status.success { display: block; background: #1b5e20; color: #a5d6a7; }
.status.error { display: block; background: #b71c1c; color: #ef9a9a; }
.status.loading { display: block; background: #0d47a1; color: #90caf9; }
.filename { margin-top: 0.5rem; font-size: 0.9rem; color: #aaa; }
</style>
</head>
<body>
<div class="container">
<h1>📄 NoteGraph Upload</h1>
<form id="uploadForm" enctype="multipart/form-data">
<div class="drop-zone" id="dropZone">
<p>Drop file here or tap to select</p>
<small>PDF, JPG, PNG, DOCX, TXT, MD</small>
</div>
<input type="file" id="fileInput" name="file" accept=".pdf,.jpg,.jpeg,.png,.docx,.txt,.md">
<div class="filename" id="fileName"></div>
<button type="submit" class="btn" id="submitBtn" disabled>Upload & Process</button>
</form>
<div class="status" id="status"></div>
</div>
<script>
const dropZone = document.getElementById('dropZone');
const fileInput = document.getElementById('fileInput');
const form = document.getElementById('uploadForm');
const submitBtn = document.getElementById('submitBtn');
const statusEl = document.getElementById('status');
const fileNameEl = document.getElementById('fileName');
// Get token from URL
const urlParams = new URLSearchParams(window.location.search);
const token = urlParams.get('token') || '';
dropZone.addEventListener('click', () => fileInput.click());
dropZone.addEventListener('dragover', (e) => { e.preventDefault(); dropZone.classList.add('dragover'); });
dropZone.addEventListener('dragleave', () => dropZone.classList.remove('dragover'));
dropZone.addEventListener('drop', (e) => {
e.preventDefault(); dropZone.classList.remove('dragover');
if (e.dataTransfer.files.length) { fileInput.files = e.dataTransfer.files; updateFileName(); }
});
fileInput.addEventListener('change', updateFileName);
function updateFileName() {
if (fileInput.files.length) { fileNameEl.textContent = fileInput.files[0].name; submitBtn.disabled = false; }
}
form.addEventListener('submit', async (e) => {
e.preventDefault();
if (!fileInput.files.length) return;
submitBtn.disabled = true;
statusEl.className = 'status loading';
statusEl.textContent = 'Uploading...';
statusEl.style.display = 'block';
const formData = new FormData();
formData.append('file', fileInput.files[0]);
try {
const resp = await fetch('/upload?token=' + encodeURIComponent(token), { method: 'POST', body: formData });
const data = await resp.json();
if (resp.ok) {
statusEl.className = 'status success';
statusEl.textContent = data.message || 'Upload successful!';
} else {
statusEl.className = 'status error';
statusEl.textContent = data.detail || 'Upload failed';
}
} catch (err) {
statusEl.className = 'status error';
statusEl.textContent = 'Network error: ' + err.message;
}
submitBtn.disabled = false;
});
</script>
</body>
</html>"""
@app.get("/", response_class=HTMLResponse)
@app.get("/upload", response_class=HTMLResponse)
async def upload_form(request: Request, token: str = Query(default=None)):
"""Serve the upload form. Token required."""
_verify_token(request, token)
return UPLOAD_HTML
@app.post("/upload")
async def upload_file(
request: Request,
file: UploadFile = File(...),
token: str = Query(default=None),
):
"""Accept file upload, save immediately, process in background."""
_verify_token(request, token)
config = _get_config()
inbox = Path(config.inbox_dir)
inbox.mkdir(parents=True, exist_ok=True)
# Save uploaded file
target = inbox / file.filename
content = await file.read()
target.write_bytes(content)
logger.info("Uploaded file saved: %s (%d bytes)", target, len(content))
# Process in background (don't block the response)
thread = threading.Thread(
target=_process_in_background,
args=(target, config),
daemon=True,
)
thread.start()
return {"message": f"File received: {file.filename}. Processing in background — check SilverBullet in ~30s."}