feat: group private apps into andreknie-privat
This commit is contained in:
@@ -0,0 +1,56 @@
|
||||
name: Deploy NoteGraph to VPS
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'privat/NoteGraph/**'
|
||||
workflow_dispatch: {}
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Deploy to VPS
|
||||
uses: appleboy/ssh-action@v1
|
||||
env:
|
||||
ENV_CONTENT: |
|
||||
SB_USER=${{ secrets.SB_USER }}
|
||||
DOMAIN=notes.andreknie.de
|
||||
GIT_AUTOCOMMIT=true
|
||||
LLM_PROVIDER=${{ secrets.LLM_PROVIDER }}
|
||||
LLM_MODEL=${{ secrets.LLM_MODEL }}
|
||||
OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}
|
||||
ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }}
|
||||
GOOGLE_API_KEY=${{ secrets.GOOGLE_API_KEY }}
|
||||
MISTRAL_API_KEY=${{ secrets.MISTRAL_API_KEY }}
|
||||
NEXTCLOUD_INBOX_URL=${{ secrets.NEXTCLOUD_INBOX_URL }}
|
||||
NEXTCLOUD_INBOX_USER=${{ secrets.NEXTCLOUD_INBOX_USER }}
|
||||
NEXTCLOUD_INBOX_PASSWORD=${{ secrets.NEXTCLOUD_INBOX_PASSWORD }}
|
||||
ORGMYLIFE_API_URL=${{ secrets.ORGMYLIFE_API_URL }}
|
||||
ORGMYLIFE_API_KEY=${{ secrets.ORGMYLIFE_API_KEY }}
|
||||
with:
|
||||
host: 217.160.174.2
|
||||
username: root
|
||||
key: ${{ secrets.VPS_SSH_KEY }}
|
||||
envs: ENV_CONTENT
|
||||
script: |
|
||||
# Ensure directory exists
|
||||
if [ ! -d /opt/NoteGraph ]; then
|
||||
cd /opt
|
||||
git clone https://github.com/DoctoDre/NoteGraph.git || true
|
||||
fi
|
||||
|
||||
cd /opt/NoteGraph
|
||||
git pull origin main || true
|
||||
|
||||
# Write .env from GitHub secrets
|
||||
printf '%s\n' "$ENV_CONTENT" > .env
|
||||
|
||||
# Update Caddy config and reload
|
||||
cp /opt/NoteGraph/Caddyfile /etc/caddy/Caddyfile.d/notegraph.conf 2>/dev/null || \
|
||||
cp /opt/NoteGraph/Caddyfile /etc/caddy/Caddyfile
|
||||
systemctl reload caddy || true
|
||||
|
||||
docker compose up -d --build
|
||||
echo "NoteGraph deploy complete"
|
||||
@@ -0,0 +1,57 @@
|
||||
# =============================================================================
|
||||
# NoteGraph Configuration
|
||||
# =============================================================================
|
||||
|
||||
# --- SilverBullet ---
|
||||
# Format: "username:password" for basic auth
|
||||
SB_USER=admin:changeme
|
||||
|
||||
# Domain (for reverse proxy / Caddy)
|
||||
DOMAIN=notes.andreknie.de
|
||||
|
||||
# Git auto-commit (optional)
|
||||
GIT_AUTOCOMMIT=true
|
||||
GIT_REMOTE=https://github.com/DoctoDre/NoteGraph.git
|
||||
|
||||
# --- LLM Provider ---
|
||||
# Supported: openai, anthropic, google, mistral, ollama
|
||||
LLM_PROVIDER=openai
|
||||
LLM_MODEL=gpt-4o
|
||||
|
||||
# Provider API keys (set the one matching your LLM_PROVIDER)
|
||||
OPENAI_API_KEY=
|
||||
ANTHROPIC_API_KEY=
|
||||
GOOGLE_API_KEY=
|
||||
MISTRAL_API_KEY=
|
||||
|
||||
# Ollama (local models, no API key needed)
|
||||
OLLAMA_BASE_URL=http://localhost:11434
|
||||
|
||||
# --- OrgMyLife Integration ---
|
||||
# API endpoint and key for creating tasks from action items
|
||||
ORGMYLIFE_API_URL=https://api.andreknie.de
|
||||
ORGMYLIFE_API_KEY=
|
||||
|
||||
# --- Nextcloud Inbox ---
|
||||
# WebDAV URL to poll for new documents (e.g., https://cloud.example.com/remote.php/dav/files/user/Inbox/)
|
||||
NEXTCLOUD_INBOX_URL=
|
||||
NEXTCLOUD_INBOX_USER=
|
||||
NEXTCLOUD_INBOX_PASSWORD=
|
||||
|
||||
# --- Paths ---
|
||||
# Directory containing the NoteGraph markdown files
|
||||
NOTES_DIR=./notes
|
||||
# Directory where incoming files are dropped for processing
|
||||
INBOX_DIR=./ingestion/inbox
|
||||
|
||||
# --- OCR ---
|
||||
# Enable/disable OCR for scanned documents and images
|
||||
OCR_ENABLED=true
|
||||
# Tesseract language codes (+ separated)
|
||||
OCR_LANGUAGE=deu+eng
|
||||
|
||||
# --- Behavior ---
|
||||
# Minimum confidence score (0.0-1.0) for entity detection
|
||||
CONFIDENCE_THRESHOLD=0.7
|
||||
# Automatically git commit after processing
|
||||
AUTO_COMMIT=true
|
||||
@@ -0,0 +1,19 @@
|
||||
# Python
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.egg-info/
|
||||
.pytest_cache/
|
||||
|
||||
# Environment
|
||||
.env
|
||||
|
||||
# OS files
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Editor
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# SilverBullet internal
|
||||
notes/.silverbullet/
|
||||
@@ -0,0 +1,16 @@
|
||||
# Component: NoteGraph
|
||||
|
||||
## Description
|
||||
Knowledge base with SilverBullet frontend and Python ingestion pipeline
|
||||
|
||||
## Metadata
|
||||
- **Deployment Target:** vps-docker
|
||||
- **Upstream URL:** https://github.com/DoctoDre/NoteGraph
|
||||
- **Status:** needs-redo
|
||||
|
||||
## Interconnections
|
||||
- (to be documented)
|
||||
|
||||
## Notes
|
||||
- Part of privat context in the Orchestrator monorepo
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
notes.andreknie.de {
|
||||
# Primary: ingestion service (browse, upload, API)
|
||||
reverse_proxy localhost:8001
|
||||
}
|
||||
|
||||
api.andreknie.de {
|
||||
reverse_proxy localhost:8000
|
||||
}
|
||||
|
||||
andreknie.de {
|
||||
handle {
|
||||
respond "Nothing here" 404
|
||||
}
|
||||
}
|
||||
|
||||
unikat.andreknie.de {
|
||||
reverse_proxy localhost:3002
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
# Deploying NoteGraph
|
||||
|
||||
## VPS Setup (First Time)
|
||||
|
||||
### 1. SSH into VPS and set up
|
||||
|
||||
```bash
|
||||
ssh root@217.160.174.2
|
||||
|
||||
cd /opt
|
||||
git clone https://github.com/DoctoDre/NoteGraph.git
|
||||
cd NoteGraph
|
||||
|
||||
# Configure
|
||||
cp .env.example .env
|
||||
nano .env
|
||||
# Set: SB_USER=admin:yourpassword
|
||||
```
|
||||
|
||||
### 2. Add Caddy reverse proxy entry
|
||||
|
||||
Add to `/etc/caddy/Caddyfile`:
|
||||
|
||||
```
|
||||
notes.andreknie.de {
|
||||
reverse_proxy localhost:3000
|
||||
}
|
||||
```
|
||||
|
||||
Then reload:
|
||||
```bash
|
||||
systemctl reload caddy
|
||||
```
|
||||
|
||||
### 3. Start
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
### 4. DNS
|
||||
|
||||
Add an A record: `notes.andreknie.de` → `217.160.174.2`
|
||||
|
||||
### 5. Access
|
||||
|
||||
Open `https://notes.andreknie.de` — log in with the credentials from `.env`.
|
||||
|
||||
## Subsequent Deploys
|
||||
|
||||
Push to `main` → GitHub Actions deploys automatically (pulls + restarts containers).
|
||||
|
||||
## Git-Backed Notes
|
||||
|
||||
The `git-sync` container auto-commits note changes every 5 minutes. To push these to GitHub, set up a deploy key or token on the VPS:
|
||||
|
||||
```bash
|
||||
cd /opt/NoteGraph/notes
|
||||
git remote add origin https://github.com/DoctoDre/NoteGraph.git
|
||||
# Or use SSH key for push access
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
- **SilverBullet** runs on port 3000 (Docker)
|
||||
- **Caddy** handles HTTPS + reverse proxy
|
||||
- **Notes** stored as markdown in `./notes/` (mounted volume)
|
||||
- **Git-sync** container auto-commits changes
|
||||
@@ -0,0 +1,89 @@
|
||||
# NoteGraph
|
||||
|
||||
A personal knowledge infrastructure for consolidating notes, thoughts, decisions, and contextual information across projects.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ NoteGraph │
|
||||
├─────────────────────────────────────────────────────────────┤
|
||||
│ UI Layer │ SilverBullet (self-hosted PWA) │
|
||||
│ Storage Layer │ Markdown files in Git │
|
||||
│ Import Layer │ PDF/OCR/AI ingestion service │
|
||||
│ Graph Layer │ Relationship index + search │
|
||||
│ Integration Layer │ OrgMyLife API bridge (tasks ↔ notes) │
|
||||
│ AI Layer │ Auto-tagging, enrichment, suggestions │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Deploy on VPS (Docker)
|
||||
|
||||
```bash
|
||||
cd NoteGraph
|
||||
cp .env.example .env
|
||||
# Edit .env with your credentials
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
SilverBullet will be available at `https://notes.andreknie.de` (or whatever domain you configure).
|
||||
|
||||
### Local Development
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
# Access at http://localhost:3000
|
||||
```
|
||||
|
||||
## Structure
|
||||
|
||||
```
|
||||
NoteGraph/
|
||||
├── docker-compose.yml # SilverBullet + reverse proxy
|
||||
├── notes/ # Markdown knowledge base (git-backed)
|
||||
│ ├── inbox/ # Quick captures, unsorted
|
||||
│ ├── projects/ # Project-specific notes
|
||||
│ ├── people/ # Person-specific context
|
||||
│ ├── meetings/ # Meeting notes
|
||||
│ ├── decisions/ # Recorded decisions
|
||||
│ └── templates/ # Note templates
|
||||
├── ingestion/ # Document import service (Phase 2)
|
||||
├── graph-service/ # Relationship index (Phase 3)
|
||||
└── .github/workflows/ # Auto-commit, backup
|
||||
```
|
||||
|
||||
## Features
|
||||
|
||||
### Phase 1 (Current)
|
||||
- [x] SilverBullet deployment (Docker)
|
||||
- [x] Git-backed markdown storage
|
||||
- [x] Wiki-links (`[[note title]]`) and backlinks
|
||||
- [x] Full-text search
|
||||
- [x] Tags and metadata
|
||||
- [x] Offline PWA (mobile access)
|
||||
- [x] Auto-commit to Git
|
||||
|
||||
### Phase 2 (Planned)
|
||||
- [ ] PDF text extraction
|
||||
- [ ] Handwritten note OCR
|
||||
- [ ] AI summarization on import
|
||||
- [ ] Auto-tagging
|
||||
|
||||
### Phase 3 (Planned)
|
||||
- [ ] Relationship graph index
|
||||
- [ ] AI-assisted enrichment
|
||||
- [ ] Graph visualization
|
||||
- [ ] OrgMyLife deep integration (task ↔ note linking)
|
||||
|
||||
## Integration with OrgMyLife
|
||||
|
||||
Notes can be linked to OrgMyLife tasks via tags:
|
||||
- Tag a note with `#task/123` to link it to task #123
|
||||
- The OrgMyLife dashboard can surface relevant notes for upcoming meetings
|
||||
- Tasks created from notes maintain traceability
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
@@ -0,0 +1,18 @@
|
||||
# NoteGraph – Komplett-Neustart nach Monorepo-Konsolidierung
|
||||
|
||||
**Status:** Muss nach der Monorepo-Konsolidierung komplett neu aufgebaut werden.
|
||||
|
||||
## Warum?
|
||||
|
||||
Das NoteGraph-Konzept wird durch die neue Monorepo-Architektur ersetzt:
|
||||
- Der **Wissensspeicher** (shared/knowledge-store/) übernimmt die ETL-Pipeline und Indexierung
|
||||
- Die **Scope-basierte Ordnerstruktur** ersetzt die bisherige NoteGraph-Verzeichnisstruktur
|
||||
- Der **YAML-Index mit Progressive Disclosure** ersetzt die bisherige Suchlogik
|
||||
- Die **Kontextbrücke** ermöglicht kontextübergreifenden Wissenstransfer
|
||||
|
||||
## Nächste Schritte
|
||||
|
||||
1. NoteGraph-Daten in den Wissensspeicher migrieren (ETL-Pipeline: `NoteGraphMigrator`)
|
||||
2. Bestehende Verzeichnisstruktur (decisions, inbox, meetings, people, projects) als Scopes abbilden
|
||||
3. Verknüpfungen als Graph-Kanten im YAML-Frontmatter übernehmen
|
||||
4. NoteGraph-Repo anschließend archivieren oder entfernen
|
||||
@@ -0,0 +1,57 @@
|
||||
services:
|
||||
silverbullet:
|
||||
image: zefhemel/silverbullet:latest
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
SB_USER: "${SB_USER:-admin:changeme}"
|
||||
volumes:
|
||||
- ./notes:/space
|
||||
ports:
|
||||
- "3000:3000"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:3000"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
# Auto-commit notes to git every 5 minutes
|
||||
git-sync:
|
||||
image: alpine/git:latest
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- ./notes:/space
|
||||
- ./.git:/repo-git:ro
|
||||
entrypoint: /bin/sh
|
||||
command: |
|
||||
-c '
|
||||
cd /space
|
||||
git init 2>/dev/null || true
|
||||
git config user.name "NoteGraph Auto-Sync"
|
||||
git config user.email "notegraph@andreknie.de"
|
||||
while true; do
|
||||
sleep 300
|
||||
cd /space
|
||||
git add -A
|
||||
if ! git diff --cached --quiet 2>/dev/null; then
|
||||
git commit -m "auto: sync notes $(date +%Y-%m-%d_%H:%M)"
|
||||
fi
|
||||
done
|
||||
'
|
||||
|
||||
# Ingestion pipeline: web upload + drop-folder watcher
|
||||
ingestion:
|
||||
build:
|
||||
context: ./ingestion
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "8001:8001"
|
||||
volumes:
|
||||
- ./notes:/app/notes
|
||||
- ./ingestion/inbox:/app/inbox
|
||||
env_file:
|
||||
- .env
|
||||
environment:
|
||||
NOTES_DIR: /app/notes
|
||||
INBOX_DIR: /app/inbox
|
||||
depends_on:
|
||||
- silverbullet
|
||||
@@ -0,0 +1,33 @@
|
||||
FROM python:3.11-slim
|
||||
|
||||
# Install system dependencies for OCR
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
tesseract-ocr \
|
||||
tesseract-ocr-deu \
|
||||
git \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy pyproject.toml first for dependency caching
|
||||
COPY pyproject.toml .
|
||||
|
||||
# Install dependencies only (not the package itself yet)
|
||||
RUN pip install --no-cache-dir \
|
||||
click watchdog pytesseract pdfplumber python-docx litellm httpx \
|
||||
pydantic pydantic-settings fastapi uvicorn python-multipart pyyaml python-slugify
|
||||
|
||||
# Copy source code into /app/ingestion/ so imports work
|
||||
COPY . /app/ingestion/
|
||||
|
||||
# Set PYTHONPATH so "import ingestion" resolves to /app/ingestion/
|
||||
ENV PYTHONPATH=/app
|
||||
|
||||
# Create inbox directory
|
||||
RUN mkdir -p /app/inbox
|
||||
|
||||
# Expose upload server port
|
||||
EXPOSE 8001
|
||||
|
||||
# Default: run the upload web server
|
||||
CMD ["uvicorn", "ingestion.web.upload:app", "--host", "0.0.0.0", "--port", "8001"]
|
||||
@@ -0,0 +1,3 @@
|
||||
"""NoteGraph document ingestion and AI-powered auto-referencing pipeline."""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
@@ -0,0 +1,5 @@
|
||||
"""Entry point for `python -m ingestion`."""
|
||||
|
||||
from ingestion.cli import cli
|
||||
|
||||
cli()
|
||||
@@ -0,0 +1,169 @@
|
||||
"""Click-based CLI for the NoteGraph ingestion pipeline.
|
||||
|
||||
Subcommands:
|
||||
import - Import files or directories into NoteGraph
|
||||
watch - Start drop-folder watcher for continuous ingestion
|
||||
sync - One-shot Nextcloud inbox sync
|
||||
upload - Start web upload server
|
||||
"""
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
|
||||
from ingestion.config import IngestionConfig
|
||||
|
||||
|
||||
def _setup_logging(verbose: bool) -> None:
|
||||
"""Configure logging based on verbosity."""
|
||||
level = logging.DEBUG if verbose else logging.INFO
|
||||
logging.basicConfig(
|
||||
level=level,
|
||||
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
||||
datefmt="%H:%M:%S",
|
||||
)
|
||||
|
||||
|
||||
@click.group()
|
||||
@click.version_option(version="0.1.0", prog_name="notegraph-ingest")
|
||||
def cli():
|
||||
"""NoteGraph Ingestion — import documents into your knowledge base."""
|
||||
pass
|
||||
|
||||
|
||||
@cli.command("import")
|
||||
@click.argument("path", type=click.Path(exists=True))
|
||||
@click.option("--output-dir", "-o", type=click.Path(), default=None, help="Override output directory")
|
||||
@click.option("--create-tasks", is_flag=True, help="Create OrgMyLife tasks for action items")
|
||||
@click.option("--provider", type=str, default=None, help="Override LLM provider (openai, anthropic, etc.)")
|
||||
@click.option("--dry-run", is_flag=True, help="Show what would be created without writing files")
|
||||
@click.option("--no-commit", is_flag=True, help="Skip git auto-commit")
|
||||
@click.option("--verbose", "-v", is_flag=True, help="Enable debug logging")
|
||||
def import_cmd(path, output_dir, create_tasks, provider, dry_run, no_commit, verbose):
|
||||
"""Import files or directories into NoteGraph.
|
||||
|
||||
PATH can be a single file or a directory to scan recursively.
|
||||
"""
|
||||
_setup_logging(verbose)
|
||||
|
||||
config = IngestionConfig()
|
||||
if provider:
|
||||
config.llm_provider = provider
|
||||
|
||||
config.validate_api_key()
|
||||
|
||||
from ingestion.discovery import discover_files
|
||||
from ingestion.pipeline import process_batch
|
||||
|
||||
source_path = Path(path)
|
||||
files = discover_files(source_path)
|
||||
|
||||
if not files:
|
||||
click.echo("No supported files found.")
|
||||
sys.exit(0)
|
||||
|
||||
click.echo(f"Found {len(files)} file(s) to process.")
|
||||
|
||||
if dry_run:
|
||||
click.echo("[DRY RUN] No files will be written.\n")
|
||||
|
||||
def progress(current, total, filename):
|
||||
click.echo(f" [{current}/{total}] {filename}")
|
||||
|
||||
result = process_batch(
|
||||
paths=files,
|
||||
config=config,
|
||||
dry_run=dry_run,
|
||||
no_commit=no_commit,
|
||||
create_tasks=create_tasks,
|
||||
output_dir=Path(output_dir) if output_dir else None,
|
||||
progress_callback=progress,
|
||||
)
|
||||
|
||||
# Summary
|
||||
click.echo("")
|
||||
click.echo(f"Done! Processed {result.total} file(s):")
|
||||
click.echo(f" ✓ Success: {result.success}")
|
||||
if result.failed:
|
||||
click.echo(f" ✗ Failed: {result.failed}")
|
||||
|
||||
|
||||
@cli.command("watch")
|
||||
@click.option("--verbose", "-v", is_flag=True, help="Enable debug logging")
|
||||
def watch_cmd(verbose):
|
||||
"""Start drop-folder watcher for continuous ingestion."""
|
||||
_setup_logging(verbose)
|
||||
|
||||
from ingestion.watcher import start_watcher
|
||||
|
||||
config = IngestionConfig()
|
||||
config.validate_api_key()
|
||||
|
||||
click.echo(f"Watching inbox: {config.inbox_dir}")
|
||||
click.echo("Press Ctrl+C to stop.\n")
|
||||
|
||||
try:
|
||||
start_watcher(config)
|
||||
except KeyboardInterrupt:
|
||||
click.echo("\nWatcher stopped.")
|
||||
|
||||
|
||||
@cli.command("sync")
|
||||
@click.option("--verbose", "-v", is_flag=True, help="Enable debug logging")
|
||||
def sync_cmd(verbose):
|
||||
"""One-shot Nextcloud inbox sync."""
|
||||
_setup_logging(verbose)
|
||||
|
||||
from ingestion.integrations.nextcloud import NextcloudInboxSync
|
||||
|
||||
config = IngestionConfig()
|
||||
|
||||
if not config.nextcloud_inbox_url:
|
||||
click.echo("ERROR: NEXTCLOUD_INBOX_URL not configured.")
|
||||
sys.exit(1)
|
||||
|
||||
config.validate_api_key()
|
||||
|
||||
sync = NextcloudInboxSync(config)
|
||||
downloaded = sync.poll()
|
||||
|
||||
if not downloaded:
|
||||
click.echo("No new files in Nextcloud inbox.")
|
||||
return
|
||||
|
||||
click.echo(f"Downloaded {len(downloaded)} file(s) from Nextcloud.")
|
||||
|
||||
# Process downloaded files
|
||||
from ingestion.pipeline import process_batch
|
||||
|
||||
result = process_batch(
|
||||
paths=downloaded,
|
||||
config=config,
|
||||
)
|
||||
|
||||
# Mark processed files on Nextcloud
|
||||
for r in result.results:
|
||||
if r.success:
|
||||
sync.mark_processed(r.source_file.name)
|
||||
|
||||
click.echo(f"Processed: {result.success} success, {result.failed} failed.")
|
||||
|
||||
|
||||
@cli.command("upload")
|
||||
@click.option("--host", default="0.0.0.0", help="Bind host")
|
||||
@click.option("--port", default=8001, type=int, help="Bind port")
|
||||
@click.option("--verbose", "-v", is_flag=True, help="Enable debug logging")
|
||||
def upload_cmd(host, port, verbose):
|
||||
"""Start web upload server for mobile file ingestion."""
|
||||
_setup_logging(verbose)
|
||||
|
||||
import uvicorn
|
||||
|
||||
click.echo(f"Starting upload server on {host}:{port}")
|
||||
uvicorn.run("ingestion.web.upload:app", host=host, port=port, log_level="info")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Configuration for the NoteGraph ingestion service."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic_settings import BaseSettings
|
||||
|
||||
|
||||
class IngestionConfig(BaseSettings):
|
||||
"""All configuration loaded from environment variables / .env file."""
|
||||
|
||||
# LLM Provider
|
||||
llm_provider: str = "openai"
|
||||
llm_model: str = "gpt-4o"
|
||||
openai_api_key: str = ""
|
||||
anthropic_api_key: str = ""
|
||||
google_api_key: str = ""
|
||||
mistral_api_key: str = ""
|
||||
ollama_base_url: str = "http://localhost:11434"
|
||||
|
||||
# OrgMyLife Integration
|
||||
orgmylife_api_url: str = "https://api.andreknie.de"
|
||||
orgmylife_api_key: str = ""
|
||||
|
||||
# Nextcloud Inbox
|
||||
nextcloud_inbox_url: str = ""
|
||||
nextcloud_inbox_user: str = ""
|
||||
nextcloud_inbox_password: str = ""
|
||||
|
||||
# Paths
|
||||
notes_dir: str = "./notes"
|
||||
inbox_dir: str = "./ingestion/inbox"
|
||||
|
||||
# OCR
|
||||
ocr_enabled: bool = True
|
||||
ocr_language: str = "deu+eng"
|
||||
|
||||
# Behavior
|
||||
confidence_threshold: float = 0.7
|
||||
auto_commit: bool = True
|
||||
|
||||
# Web Upload Auth (same as SilverBullet)
|
||||
sb_user: str = "admin:changeme"
|
||||
|
||||
model_config = {"env_file": ".env", "env_prefix": "", "extra": "ignore"}
|
||||
|
||||
@property
|
||||
def llm_model_string(self) -> str:
|
||||
"""LiteLLM model string (e.g., 'openai/gpt-4o', 'ollama/llama3')."""
|
||||
if self.llm_provider == "ollama":
|
||||
return f"ollama/{self.llm_model}"
|
||||
return f"{self.llm_provider}/{self.llm_model}"
|
||||
|
||||
def validate_api_key(self) -> None:
|
||||
"""Exit with a clear error if the required API key is missing."""
|
||||
key_map = {
|
||||
"openai": ("openai_api_key", "OPENAI_API_KEY"),
|
||||
"anthropic": ("anthropic_api_key", "ANTHROPIC_API_KEY"),
|
||||
"google": ("google_api_key", "GOOGLE_API_KEY"),
|
||||
"mistral": ("mistral_api_key", "MISTRAL_API_KEY"),
|
||||
}
|
||||
if self.llm_provider in key_map:
|
||||
attr, env_name = key_map[self.llm_provider]
|
||||
if not getattr(self, attr):
|
||||
raise SystemExit(
|
||||
f"ERROR: {env_name} is required for provider '{self.llm_provider}' but not set."
|
||||
)
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Recursive file discovery with extension filtering.
|
||||
|
||||
Discovers files for ingestion by walking directories and filtering
|
||||
to supported document types.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SUPPORTED_EXTENSIONS = {".pdf", ".jpg", ".jpeg", ".png", ".md", ".txt", ".docx"}
|
||||
|
||||
|
||||
def discover_files(path: Path) -> list[Path]:
|
||||
"""Discover supported files from a path (file or directory).
|
||||
|
||||
Args:
|
||||
path: A single file path or a directory to scan recursively.
|
||||
|
||||
Returns:
|
||||
Sorted list of Path objects with supported extensions.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If the path does not exist.
|
||||
"""
|
||||
path = Path(path)
|
||||
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Path does not exist: {path}")
|
||||
|
||||
if path.is_file():
|
||||
if path.suffix.lower() in SUPPORTED_EXTENSIONS:
|
||||
return [path]
|
||||
logger.warning("Unsupported file type: %s", path)
|
||||
return []
|
||||
|
||||
# Recursive directory traversal
|
||||
files: list[Path] = []
|
||||
for item in sorted(path.rglob("*")):
|
||||
if item.is_file() and item.suffix.lower() in SUPPORTED_EXTENSIONS:
|
||||
files.append(item)
|
||||
|
||||
logger.info("Discovered %d supported files in %s", len(files), path)
|
||||
return files
|
||||
@@ -0,0 +1 @@
|
||||
"""LLM-powered enrichment: entity detection, categorization, and action item extraction."""
|
||||
@@ -0,0 +1,122 @@
|
||||
"""Enrichment agent orchestrating LLM calls for entity detection.
|
||||
|
||||
Calls the LLM with extracted text, parses the structured response,
|
||||
filters entities by confidence threshold, and handles parse errors with retry.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from ingestion.config import IngestionConfig
|
||||
from ingestion.integrations.llm import LLMClient, LLMError
|
||||
|
||||
from .models import EnrichmentResult, Entity
|
||||
from .prompts import STRICT_RETRY_PROMPT, build_system_prompt, build_user_prompt
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EnrichmentError(Exception):
|
||||
"""Raised when enrichment fails after retries."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class EnrichmentAgent:
|
||||
"""Orchestrates LLM-based document enrichment."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: IngestionConfig,
|
||||
llm_client: LLMClient | None = None,
|
||||
known_projects: list[str] | None = None,
|
||||
known_people: list[str] | None = None,
|
||||
):
|
||||
self.config = config
|
||||
self.llm = llm_client or LLMClient(config)
|
||||
self.confidence_threshold = config.confidence_threshold
|
||||
self.system_prompt = build_system_prompt(known_projects, known_people)
|
||||
|
||||
def enrich(self, text: str) -> EnrichmentResult:
|
||||
"""Enrich extracted text with LLM-detected entities and metadata.
|
||||
|
||||
Args:
|
||||
text: The extracted document text to analyze.
|
||||
|
||||
Returns:
|
||||
EnrichmentResult with entities filtered by confidence threshold.
|
||||
|
||||
Raises:
|
||||
EnrichmentError: If LLM call and retry both fail.
|
||||
"""
|
||||
messages = [
|
||||
{"role": "system", "content": self.system_prompt},
|
||||
{"role": "user", "content": build_user_prompt(text)},
|
||||
]
|
||||
|
||||
# First attempt
|
||||
try:
|
||||
response_text = self.llm.complete(messages)
|
||||
result = self._parse_response(response_text)
|
||||
except (json.JSONDecodeError, ValueError) as parse_err:
|
||||
logger.warning(
|
||||
"First LLM response parse failed (%s), retrying with strict prompt",
|
||||
parse_err,
|
||||
)
|
||||
# Retry with stricter prompt
|
||||
result = self._retry_with_strict_prompt(messages, response_text)
|
||||
except LLMError as e:
|
||||
raise EnrichmentError(f"LLM call failed: {e}") from e
|
||||
|
||||
# Filter entities by confidence threshold
|
||||
result.entities = self._filter_by_confidence(result.entities)
|
||||
|
||||
return result
|
||||
|
||||
def _retry_with_strict_prompt(
|
||||
self, original_messages: list[dict[str, str]], failed_response: str
|
||||
) -> EnrichmentResult:
|
||||
"""Retry LLM call with a stricter prompt after parse failure."""
|
||||
retry_messages = original_messages + [
|
||||
{"role": "assistant", "content": failed_response},
|
||||
{"role": "user", "content": STRICT_RETRY_PROMPT},
|
||||
]
|
||||
|
||||
try:
|
||||
response_text = self.llm.complete(retry_messages)
|
||||
return self._parse_response(response_text)
|
||||
except (json.JSONDecodeError, ValueError, LLMError) as e:
|
||||
logger.error("Retry also failed (%s), using defaults", e)
|
||||
return self._default_result()
|
||||
|
||||
def _parse_response(self, response_text: str) -> EnrichmentResult:
|
||||
"""Parse LLM response text into an EnrichmentResult.
|
||||
|
||||
Handles responses that may be wrapped in markdown code fences.
|
||||
"""
|
||||
cleaned = response_text.strip()
|
||||
|
||||
# Strip markdown code fences if present
|
||||
if cleaned.startswith("```"):
|
||||
lines = cleaned.split("\n")
|
||||
# Remove first line (```json) and last line (```)
|
||||
lines = [l for l in lines[1:] if not l.strip() == "```"]
|
||||
cleaned = "\n".join(lines)
|
||||
|
||||
data = json.loads(cleaned)
|
||||
return EnrichmentResult.model_validate(data)
|
||||
|
||||
def _filter_by_confidence(self, entities: list[Entity]) -> list[Entity]:
|
||||
"""Keep only entities meeting the confidence threshold."""
|
||||
return [e for e in entities if e.confidence >= self.confidence_threshold]
|
||||
|
||||
def _default_result(self) -> EnrichmentResult:
|
||||
"""Return a minimal default result when all parsing fails."""
|
||||
return EnrichmentResult(
|
||||
title="Untitled Note",
|
||||
category="inbox",
|
||||
tags=[],
|
||||
entities=[],
|
||||
action_items=[],
|
||||
summary=None,
|
||||
)
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Pydantic models for LLM enrichment responses.
|
||||
|
||||
Defines the structured output format expected from the enrichment agent:
|
||||
entities, action items, and the overall enrichment result.
|
||||
"""
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class Entity(BaseModel):
|
||||
"""A detected entity in the document text."""
|
||||
|
||||
type: Literal["person", "project", "date", "action_item"]
|
||||
value: str
|
||||
confidence: float = Field(ge=0.0, le=1.0)
|
||||
position: int | None = None # character offset in source text
|
||||
|
||||
|
||||
class ActionItem(BaseModel):
|
||||
"""An extracted action item / task."""
|
||||
|
||||
description: str
|
||||
assignee: str | None = None
|
||||
deadline: str | None = None # ISO 8601 date string
|
||||
|
||||
|
||||
class EnrichmentResult(BaseModel):
|
||||
"""Complete enrichment output from the LLM."""
|
||||
|
||||
title: str
|
||||
category: Literal["meeting", "project", "decision", "inbox"]
|
||||
tags: list[str] = Field(default_factory=list)
|
||||
entities: list[Entity] = Field(default_factory=list)
|
||||
action_items: list[ActionItem] = Field(default_factory=list)
|
||||
summary: str | None = None
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Structured LLM prompts for document enrichment.
|
||||
|
||||
Provides the system prompt that instructs the LLM to return structured JSON
|
||||
with entities, categories, tags, and action items.
|
||||
"""
|
||||
|
||||
ENRICHMENT_SYSTEM_PROMPT = """\
|
||||
You are a knowledge management assistant. Analyze the following text extracted \
|
||||
from a document and return structured metadata as JSON.
|
||||
|
||||
Return a JSON object with exactly these fields:
|
||||
- "title": A concise, descriptive title for this note (max 80 chars)
|
||||
- "category": One of "meeting", "project", "decision", "inbox"
|
||||
- "tags": List of relevant topic tags (lowercase, no # prefix)
|
||||
- "entities": List of detected entities, each with:
|
||||
- "type": One of "person", "project", "date", "action_item"
|
||||
- "value": The extracted text value
|
||||
- "confidence": Float between 0.0 and 1.0 indicating detection confidence
|
||||
- "position": Approximate character offset where entity appears (integer or null)
|
||||
- "action_items": List of action items, each with:
|
||||
- "description": What needs to be done
|
||||
- "assignee": Person responsible (string or null)
|
||||
- "deadline": Date in ISO 8601 format, e.g. "2024-03-22" (string or null)
|
||||
- "summary": A 1-2 sentence summary of the document (string or null)
|
||||
|
||||
Rules:
|
||||
- Be precise with entity detection. Only mark entities you are confident about.
|
||||
- For people, use their full name as the value.
|
||||
- For projects, use the project name as commonly referenced.
|
||||
- For dates, use ISO 8601 format (YYYY-MM-DD) as the value.
|
||||
- Assign confidence scores honestly: 0.9+ for clearly stated entities, 0.7-0.9 for \
|
||||
likely entities, below 0.7 for uncertain ones.
|
||||
- Category "meeting" for meeting notes, agendas, minutes.
|
||||
- Category "project" for project-specific documentation.
|
||||
- Category "decision" for recorded decisions, ADRs.
|
||||
- Category "inbox" for anything that doesn't clearly fit the above.
|
||||
|
||||
Context: This is a personal knowledge base for a project manager working across \
|
||||
multiple business projects.{context_section}
|
||||
"""
|
||||
|
||||
STRICT_RETRY_PROMPT = """\
|
||||
Your previous response was not valid JSON. Please respond with ONLY a valid JSON \
|
||||
object matching the schema described in the system prompt. No markdown, no code fences, \
|
||||
no explanation — just the raw JSON object.
|
||||
"""
|
||||
|
||||
|
||||
def build_system_prompt(
|
||||
known_projects: list[str] | None = None,
|
||||
known_people: list[str] | None = None,
|
||||
) -> str:
|
||||
"""Build the system prompt with optional context about known entities.
|
||||
|
||||
Args:
|
||||
known_projects: List of known project names for context.
|
||||
known_people: List of known people names for context.
|
||||
|
||||
Returns:
|
||||
Complete system prompt string.
|
||||
"""
|
||||
context_parts: list[str] = []
|
||||
|
||||
if known_projects:
|
||||
projects_str = ", ".join(known_projects)
|
||||
context_parts.append(f"Known projects: {projects_str}.")
|
||||
|
||||
if known_people:
|
||||
people_str = ", ".join(known_people)
|
||||
context_parts.append(f"Known people: {people_str}.")
|
||||
|
||||
if context_parts:
|
||||
context_section = "\n" + " ".join(context_parts)
|
||||
else:
|
||||
context_section = ""
|
||||
|
||||
return ENRICHMENT_SYSTEM_PROMPT.format(context_section=context_section)
|
||||
|
||||
|
||||
def build_user_prompt(extracted_text: str) -> str:
|
||||
"""Build the user message containing the extracted document text.
|
||||
|
||||
Args:
|
||||
extracted_text: The raw text extracted from the document.
|
||||
|
||||
Returns:
|
||||
User prompt string.
|
||||
"""
|
||||
return f"Analyze this document and return the structured JSON:\n\n{extracted_text}"
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Text extraction layer for various document formats."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from .base import ExtractionResult, TextExtractor
|
||||
from .docx import DocxExtractor
|
||||
from .ocr import OCRExtractor
|
||||
from .pdf import PDFExtractor
|
||||
from .text import PlainTextExtractor
|
||||
|
||||
|
||||
def get_extractor(file_path: Path, config=None):
|
||||
"""Get the appropriate extractor for a file.
|
||||
|
||||
Args:
|
||||
file_path: Path to the file to extract.
|
||||
config: Optional IngestionConfig for LLM-based extraction.
|
||||
|
||||
Returns:
|
||||
An extractor instance that can handle the file, or None.
|
||||
"""
|
||||
suffix = file_path.suffix.lower()
|
||||
|
||||
if suffix in (".md", ".txt"):
|
||||
return PlainTextExtractor()
|
||||
elif suffix == ".pdf":
|
||||
if config:
|
||||
return PDFExtractor(
|
||||
ocr_enabled=config.ocr_enabled,
|
||||
ocr_language=config.ocr_language,
|
||||
llm_provider=config.llm_provider,
|
||||
llm_model=config.llm_model,
|
||||
mistral_api_key=config.mistral_api_key,
|
||||
)
|
||||
return PDFExtractor()
|
||||
elif suffix in (".jpg", ".jpeg", ".png"):
|
||||
return OCRExtractor()
|
||||
elif suffix == ".docx":
|
||||
return DocxExtractor()
|
||||
return None
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ExtractionResult",
|
||||
"TextExtractor",
|
||||
"PlainTextExtractor",
|
||||
"PDFExtractor",
|
||||
"OCRExtractor",
|
||||
"DocxExtractor",
|
||||
"get_extractor",
|
||||
]
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Base protocol and data structures for text extraction."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExtractionResult:
|
||||
"""Result of extracting text from a document."""
|
||||
|
||||
text: str
|
||||
source_file: Path
|
||||
extraction_method: str # "direct", "ocr", "mixed"
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
class TextExtractor(Protocol):
|
||||
"""Protocol that all text extractors must implement."""
|
||||
|
||||
def can_handle(self, file_path: Path) -> bool:
|
||||
"""Return True if this extractor can process the given file."""
|
||||
...
|
||||
|
||||
def extract(self, file_path: Path) -> ExtractionResult:
|
||||
"""Extract text content from the file."""
|
||||
...
|
||||
@@ -0,0 +1,57 @@
|
||||
"""DOCX text extraction using python-docx.
|
||||
|
||||
Preserves heading structure and paragraph breaks.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from .base import ExtractionResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DocxExtractor:
|
||||
"""Extracts text from .docx files preserving structure."""
|
||||
|
||||
def can_handle(self, file_path: Path) -> bool:
|
||||
"""Return True for .docx files."""
|
||||
return file_path.suffix.lower() == ".docx"
|
||||
|
||||
def extract(self, file_path: Path) -> ExtractionResult:
|
||||
"""Extract text from DOCX preserving headings and paragraphs."""
|
||||
try:
|
||||
from docx import Document
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"python-docx is required for DOCX extraction. "
|
||||
"Install with: pip install python-docx"
|
||||
)
|
||||
|
||||
doc = Document(str(file_path))
|
||||
parts: list[str] = []
|
||||
|
||||
for paragraph in doc.paragraphs:
|
||||
text = paragraph.text.strip()
|
||||
if not text:
|
||||
continue
|
||||
|
||||
# Preserve heading structure with markdown-style headers
|
||||
style_name = paragraph.style.name.lower() if paragraph.style else ""
|
||||
if "heading 1" in style_name:
|
||||
parts.append(f"# {text}")
|
||||
elif "heading 2" in style_name:
|
||||
parts.append(f"## {text}")
|
||||
elif "heading 3" in style_name:
|
||||
parts.append(f"### {text}")
|
||||
elif "heading 4" in style_name:
|
||||
parts.append(f"#### {text}")
|
||||
else:
|
||||
parts.append(text)
|
||||
|
||||
return ExtractionResult(
|
||||
text="\n\n".join(parts),
|
||||
source_file=file_path,
|
||||
extraction_method="direct",
|
||||
metadata={"paragraph_count": len(parts)},
|
||||
)
|
||||
@@ -0,0 +1,48 @@
|
||||
"""OCR extraction for images using pytesseract.
|
||||
|
||||
Handles .jpg, .jpeg, .png files. Configured for German + English (deu+eng).
|
||||
Returns empty text (not error) when OCR cannot extract readable content.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from .base import ExtractionResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SUPPORTED_EXTENSIONS = {".jpg", ".jpeg", ".png"}
|
||||
|
||||
|
||||
class OCRExtractor:
|
||||
"""Extracts text from images using Tesseract OCR."""
|
||||
|
||||
def __init__(self, language: str = "deu+eng"):
|
||||
self.language = language
|
||||
|
||||
def can_handle(self, file_path: Path) -> bool:
|
||||
"""Return True for .jpg, .jpeg, .png files."""
|
||||
return file_path.suffix.lower() in SUPPORTED_EXTENSIONS
|
||||
|
||||
def extract(self, file_path: Path) -> ExtractionResult:
|
||||
"""Run OCR on an image file. Returns empty text on failure."""
|
||||
text = self._run_ocr(file_path)
|
||||
return ExtractionResult(
|
||||
text=text,
|
||||
source_file=file_path,
|
||||
extraction_method="ocr",
|
||||
metadata={"ocr_language": self.language},
|
||||
)
|
||||
|
||||
def _run_ocr(self, file_path: Path) -> str:
|
||||
"""Execute pytesseract on the image. Returns empty string on failure."""
|
||||
try:
|
||||
import pytesseract
|
||||
from PIL import Image
|
||||
|
||||
image = Image.open(file_path)
|
||||
text = pytesseract.image_to_string(image, lang=self.language)
|
||||
return text.strip()
|
||||
except Exception as e:
|
||||
logger.warning("OCR extraction failed for %s: %s", file_path, e)
|
||||
return ""
|
||||
@@ -0,0 +1,156 @@
|
||||
"""PDF text extraction using pdfplumber.
|
||||
|
||||
Detects scanned pages (no extractable text) and delegates to:
|
||||
1. Mistral Vision API (preferred — handles handwriting well)
|
||||
2. Tesseract OCR (fallback if no LLM configured)
|
||||
"""
|
||||
|
||||
import base64
|
||||
import io
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import pdfplumber
|
||||
|
||||
from .base import ExtractionResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PDFExtractor:
|
||||
"""Extracts text from PDF files, with Vision LLM or OCR fallback for scanned pages."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ocr_enabled: bool = True,
|
||||
ocr_language: str = "deu+eng",
|
||||
llm_provider: str = "mistral",
|
||||
llm_model: str = "mistral-small-latest",
|
||||
mistral_api_key: str = "",
|
||||
):
|
||||
self.ocr_enabled = ocr_enabled
|
||||
self.ocr_language = ocr_language
|
||||
self.llm_provider = llm_provider
|
||||
self.llm_model = llm_model
|
||||
self.mistral_api_key = mistral_api_key
|
||||
self._use_vision = bool(mistral_api_key)
|
||||
|
||||
def can_handle(self, file_path: Path) -> bool:
|
||||
"""Return True for .pdf files."""
|
||||
return file_path.suffix.lower() == ".pdf"
|
||||
|
||||
def extract(self, file_path: Path) -> ExtractionResult:
|
||||
"""Extract text from PDF, using Vision LLM or OCR for scanned pages."""
|
||||
pages_text: list[str] = []
|
||||
has_direct_text = False
|
||||
has_vision_text = False
|
||||
|
||||
with pdfplumber.open(file_path) as pdf:
|
||||
for page_num, page in enumerate(pdf.pages, start=1):
|
||||
text = page.extract_text() or ""
|
||||
text = text.strip()
|
||||
|
||||
if text:
|
||||
pages_text.append(text)
|
||||
has_direct_text = True
|
||||
elif self.ocr_enabled:
|
||||
# Scanned page — try Vision LLM first, then Tesseract
|
||||
logger.debug(
|
||||
"Page %d of %s has no text, attempting vision OCR",
|
||||
page_num, file_path,
|
||||
)
|
||||
ocr_text = self._vision_ocr_page(page, page_num, file_path)
|
||||
if ocr_text:
|
||||
pages_text.append(ocr_text)
|
||||
has_vision_text = True
|
||||
|
||||
# Determine extraction method
|
||||
if has_direct_text and has_vision_text:
|
||||
method = "mixed"
|
||||
elif has_vision_text:
|
||||
method = "vision-ocr"
|
||||
else:
|
||||
method = "direct"
|
||||
|
||||
return ExtractionResult(
|
||||
text="\n\n".join(pages_text),
|
||||
source_file=file_path,
|
||||
extraction_method=method,
|
||||
metadata={"page_count": len(pages_text)},
|
||||
)
|
||||
|
||||
def _vision_ocr_page(self, page, page_num: int, file_path: Path) -> str:
|
||||
"""OCR a PDF page using Mistral Vision API, with Tesseract fallback."""
|
||||
if self._use_vision:
|
||||
try:
|
||||
return self._mistral_vision_ocr(page, page_num)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Vision OCR failed for page %d of %s: %s, trying Tesseract",
|
||||
page_num, file_path, e,
|
||||
)
|
||||
|
||||
# Fallback to Tesseract
|
||||
return self._tesseract_ocr(page, page_num, file_path)
|
||||
|
||||
def _mistral_vision_ocr(self, page, page_num: int) -> str:
|
||||
"""Send page image to Mistral Vision for text extraction."""
|
||||
import httpx
|
||||
|
||||
# Convert page to PNG image
|
||||
image = page.to_image(resolution=200)
|
||||
img_buffer = io.BytesIO()
|
||||
image.original.save(img_buffer, format="PNG")
|
||||
img_bytes = img_buffer.getvalue()
|
||||
img_b64 = base64.b64encode(img_bytes).decode("utf-8")
|
||||
|
||||
# Call Mistral Vision API via LiteLLM-compatible format
|
||||
response = httpx.post(
|
||||
"https://api.mistral.ai/v1/chat/completions",
|
||||
headers={
|
||||
"Authorization": f"Bearer {self.mistral_api_key}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
json={
|
||||
"model": self.llm_model,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Extract ALL text from this image. This is a handwritten note or document. "
|
||||
"Transcribe everything you can read, preserving structure (paragraphs, lists, headings). "
|
||||
"If text is in German, keep it in German. Return ONLY the extracted text, no commentary.",
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": f"data:image/png;base64,{img_b64}",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
"max_tokens": 4000,
|
||||
},
|
||||
timeout=60.0,
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
text = data["choices"][0]["message"]["content"]
|
||||
logger.info("Vision OCR page %d: extracted %d chars", page_num, len(text))
|
||||
return text.strip()
|
||||
|
||||
def _tesseract_ocr(self, page, page_num: int, file_path: Path) -> str:
|
||||
"""Fallback: OCR a PDF page using Tesseract."""
|
||||
try:
|
||||
import pytesseract
|
||||
image = page.to_image(resolution=300)
|
||||
text = pytesseract.image_to_string(image.original, lang=self.ocr_language)
|
||||
return text.strip()
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Tesseract OCR failed for page %d of %s: %s", page_num, file_path, e
|
||||
)
|
||||
return ""
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Plain text and markdown file extractor.
|
||||
|
||||
Handles .md and .txt files. Reads as UTF-8 by default, with fallback
|
||||
charset detection for non-UTF-8 encoded files.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from .base import ExtractionResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SUPPORTED_EXTENSIONS = {".md", ".txt"}
|
||||
|
||||
|
||||
class PlainTextExtractor:
|
||||
"""Extracts text from plain text and markdown files."""
|
||||
|
||||
def can_handle(self, file_path: Path) -> bool:
|
||||
"""Return True for .md and .txt files."""
|
||||
return file_path.suffix.lower() in SUPPORTED_EXTENSIONS
|
||||
|
||||
def extract(self, file_path: Path) -> ExtractionResult:
|
||||
"""Read file as UTF-8, falling back to charset detection."""
|
||||
text = self._read_with_fallback(file_path)
|
||||
return ExtractionResult(
|
||||
text=text,
|
||||
source_file=file_path,
|
||||
extraction_method="direct",
|
||||
metadata={"encoding": "utf-8"},
|
||||
)
|
||||
|
||||
def _read_with_fallback(self, file_path: Path) -> str:
|
||||
"""Try UTF-8 first, then attempt charset detection."""
|
||||
try:
|
||||
return file_path.read_text(encoding="utf-8")
|
||||
except UnicodeDecodeError:
|
||||
logger.warning(
|
||||
"UTF-8 decode failed for %s, attempting charset detection", file_path
|
||||
)
|
||||
return self._read_with_detection(file_path)
|
||||
|
||||
def _read_with_detection(self, file_path: Path) -> str:
|
||||
"""Detect encoding and read file."""
|
||||
raw = file_path.read_bytes()
|
||||
|
||||
# Try common encodings before giving up
|
||||
for encoding in ("latin-1", "cp1252", "iso-8859-1"):
|
||||
try:
|
||||
return raw.decode(encoding)
|
||||
except (UnicodeDecodeError, LookupError):
|
||||
continue
|
||||
|
||||
# Last resort: decode with replacement characters
|
||||
logger.warning("Could not detect encoding for %s, using replacement", file_path)
|
||||
return raw.decode("utf-8", errors="replace")
|
||||
@@ -0,0 +1 @@
|
||||
"""External service integrations: LLM providers, OrgMyLife, Git, Nextcloud."""
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Git commit operations for auto-committing imported notes.
|
||||
|
||||
Stages new markdown files and stub pages, commits with message format:
|
||||
ingestion: import N notes from [source-type]
|
||||
|
||||
Supports --no-commit flag. Handles git failures gracefully.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def commit_imported_notes(files: list[Path], source_type: str = "import") -> bool:
|
||||
"""Stage and commit imported note files.
|
||||
|
||||
Args:
|
||||
files: List of file paths to stage and commit.
|
||||
source_type: Source description for commit message (e.g., 'batch-import', 'upload', 'nextcloud').
|
||||
|
||||
Returns:
|
||||
True if commit succeeded, False otherwise.
|
||||
"""
|
||||
if not files:
|
||||
logger.debug("No files to commit.")
|
||||
return False
|
||||
|
||||
try:
|
||||
# Stage files
|
||||
file_strs = [str(f) for f in files]
|
||||
subprocess.run(
|
||||
["git", "add"] + file_strs,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
|
||||
# Commit
|
||||
count = len(files)
|
||||
message = f"ingestion: import {count} notes from {source_type}"
|
||||
|
||||
result = subprocess.run(
|
||||
["git", "commit", "-m", message],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
logger.info("Git commit: %s", message)
|
||||
return True
|
||||
else:
|
||||
# Nothing to commit (files already tracked/unchanged)
|
||||
logger.debug("Git commit returned %d: %s", result.returncode, result.stderr.strip())
|
||||
return False
|
||||
|
||||
except FileNotFoundError:
|
||||
logger.warning("Git not found in PATH — skipping commit.")
|
||||
return False
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.warning("Git operation failed: %s", e.stderr.strip() if e.stderr else str(e))
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.warning("Unexpected git error: %s", e)
|
||||
return False
|
||||
@@ -0,0 +1,112 @@
|
||||
"""LiteLLM wrapper for model-agnostic LLM access.
|
||||
|
||||
Supports OpenAI, Anthropic, Google, Mistral, and Ollama via a unified interface.
|
||||
Implements retry logic with exponential backoff for transient failures.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ingestion.config import IngestionConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Retry configuration
|
||||
MAX_RETRIES = 3
|
||||
BASE_DELAY_SECONDS = 1.0
|
||||
MAX_DELAY_SECONDS = 30.0
|
||||
BACKOFF_FACTOR = 2.0
|
||||
|
||||
|
||||
class LLMError(Exception):
|
||||
"""Raised when an LLM call fails after all retries."""
|
||||
|
||||
def __init__(self, provider: str, model: str, message: str):
|
||||
self.provider = provider
|
||||
self.model = model
|
||||
super().__init__(
|
||||
f"LLM call failed [{provider}/{model}]: {message}"
|
||||
)
|
||||
|
||||
|
||||
class LLMClient:
|
||||
"""Model-agnostic LLM client using LiteLLM."""
|
||||
|
||||
def __init__(self, config: IngestionConfig):
|
||||
self.model = config.llm_model_string
|
||||
self.provider = config.llm_provider
|
||||
self.config = config
|
||||
|
||||
def complete(
|
||||
self,
|
||||
messages: list[dict[str, str]],
|
||||
response_format: type[BaseModel] | None = None,
|
||||
) -> str:
|
||||
"""Send a completion request with retry logic.
|
||||
|
||||
Args:
|
||||
messages: List of message dicts with 'role' and 'content' keys.
|
||||
response_format: Optional Pydantic model for structured output.
|
||||
|
||||
Returns:
|
||||
The assistant's response content as a string.
|
||||
|
||||
Raises:
|
||||
LLMError: If all retries are exhausted.
|
||||
"""
|
||||
import litellm
|
||||
|
||||
last_error: Exception | None = None
|
||||
|
||||
for attempt in range(1, MAX_RETRIES + 1):
|
||||
try:
|
||||
kwargs: dict[str, Any] = {
|
||||
"model": self.model,
|
||||
"messages": messages,
|
||||
}
|
||||
|
||||
if response_format is not None:
|
||||
kwargs["response_format"] = response_format
|
||||
|
||||
response = litellm.completion(**kwargs)
|
||||
content = response.choices[0].message.content
|
||||
return content or ""
|
||||
|
||||
except (
|
||||
litellm.RateLimitError,
|
||||
litellm.Timeout,
|
||||
litellm.ServiceUnavailableError,
|
||||
litellm.InternalServerError,
|
||||
) as e:
|
||||
last_error = e
|
||||
delay = min(
|
||||
BASE_DELAY_SECONDS * (BACKOFF_FACTOR ** (attempt - 1)),
|
||||
MAX_DELAY_SECONDS,
|
||||
)
|
||||
logger.warning(
|
||||
"LLM call attempt %d/%d failed (%s: %s), retrying in %.1fs",
|
||||
attempt,
|
||||
MAX_RETRIES,
|
||||
type(e).__name__,
|
||||
str(e),
|
||||
delay,
|
||||
)
|
||||
time.sleep(delay)
|
||||
|
||||
except Exception as e:
|
||||
# Non-retryable error
|
||||
raise LLMError(
|
||||
provider=self.provider,
|
||||
model=self.model,
|
||||
message=f"{type(e).__name__}: {e}",
|
||||
) from e
|
||||
|
||||
# All retries exhausted
|
||||
raise LLMError(
|
||||
provider=self.provider,
|
||||
model=self.model,
|
||||
message=f"All {MAX_RETRIES} retries exhausted. Last error: {last_error}",
|
||||
)
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Nextcloud WebDAV inbox sync.
|
||||
|
||||
Polls a Nextcloud folder via WebDAV (PROPFIND/GET) for new files,
|
||||
downloads them for processing, and moves processed files to a processed/ subfolder.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
from urllib.parse import unquote, urljoin
|
||||
|
||||
import httpx
|
||||
|
||||
from ingestion.config import IngestionConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# WebDAV XML namespace
|
||||
DAV_NS = "DAV:"
|
||||
|
||||
|
||||
class NextcloudInboxSync:
|
||||
"""Sync files from a Nextcloud WebDAV folder."""
|
||||
|
||||
def __init__(self, config: IngestionConfig):
|
||||
self.base_url = config.nextcloud_inbox_url.rstrip("/") + "/"
|
||||
self.user = config.nextcloud_inbox_user
|
||||
self.password = config.nextcloud_inbox_password
|
||||
self.local_inbox = Path(config.inbox_dir)
|
||||
self.local_inbox.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self._client = httpx.Client(
|
||||
auth=(self.user, self.password),
|
||||
timeout=30.0,
|
||||
)
|
||||
|
||||
def poll(self) -> list[Path]:
|
||||
"""Poll Nextcloud inbox for files and download them.
|
||||
|
||||
Returns:
|
||||
List of local paths to downloaded files.
|
||||
"""
|
||||
try:
|
||||
filenames = self._list_files()
|
||||
except Exception as e:
|
||||
logger.error("Failed to list Nextcloud inbox: %s", e)
|
||||
return []
|
||||
|
||||
if not filenames:
|
||||
logger.info("Nextcloud inbox is empty.")
|
||||
return []
|
||||
|
||||
downloaded: list[Path] = []
|
||||
for filename in filenames:
|
||||
local_path = self.download(filename)
|
||||
if local_path:
|
||||
downloaded.append(local_path)
|
||||
|
||||
logger.info("Downloaded %d file(s) from Nextcloud.", len(downloaded))
|
||||
return downloaded
|
||||
|
||||
def _list_files(self) -> list[str]:
|
||||
"""PROPFIND to list files in the inbox folder."""
|
||||
propfind_body = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<d:propfind xmlns:d="DAV:">
|
||||
<d:prop>
|
||||
<d:resourcetype/>
|
||||
<d:getcontentlength/>
|
||||
</d:prop>
|
||||
</d:propfind>"""
|
||||
|
||||
response = self._client.request(
|
||||
"PROPFIND",
|
||||
self.base_url,
|
||||
content=propfind_body,
|
||||
headers={
|
||||
"Content-Type": "application/xml",
|
||||
"Depth": "1",
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
# Parse WebDAV XML response
|
||||
root = ET.fromstring(response.text)
|
||||
filenames: list[str] = []
|
||||
|
||||
for resp_elem in root.findall(f"{{{DAV_NS}}}response"):
|
||||
href = resp_elem.findtext(f"{{{DAV_NS}}}href", "")
|
||||
# Skip the folder itself (collections)
|
||||
resourcetype = resp_elem.find(
|
||||
f".//{{{DAV_NS}}}resourcetype/{{{DAV_NS}}}collection"
|
||||
)
|
||||
if resourcetype is not None:
|
||||
continue
|
||||
|
||||
# Extract filename from href
|
||||
filename = unquote(href.rstrip("/").split("/")[-1])
|
||||
if filename and not filename.startswith("."):
|
||||
filenames.append(filename)
|
||||
|
||||
return filenames
|
||||
|
||||
def download(self, filename: str) -> Path | None:
|
||||
"""Download a file from Nextcloud inbox.
|
||||
|
||||
Args:
|
||||
filename: Name of the file to download.
|
||||
|
||||
Returns:
|
||||
Local path to the downloaded file, or None on failure.
|
||||
"""
|
||||
url = urljoin(self.base_url, filename)
|
||||
try:
|
||||
response = self._client.get(url)
|
||||
response.raise_for_status()
|
||||
|
||||
local_path = self.local_inbox / filename
|
||||
local_path.write_bytes(response.content)
|
||||
logger.info("Downloaded: %s", filename)
|
||||
return local_path
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Failed to download %s: %s", filename, e)
|
||||
return None
|
||||
|
||||
def mark_processed(self, filename: str) -> bool:
|
||||
"""Move a processed file to the processed/ subfolder on Nextcloud.
|
||||
|
||||
Args:
|
||||
filename: Name of the file to move.
|
||||
|
||||
Returns:
|
||||
True if move succeeded, False otherwise.
|
||||
"""
|
||||
source_url = urljoin(self.base_url, filename)
|
||||
dest_url = urljoin(self.base_url, f"processed/{filename}")
|
||||
|
||||
try:
|
||||
# Ensure processed/ folder exists (MKCOL)
|
||||
processed_url = urljoin(self.base_url, "processed/")
|
||||
self._client.request("MKCOL", processed_url)
|
||||
|
||||
# MOVE the file
|
||||
response = self._client.request(
|
||||
"MOVE",
|
||||
source_url,
|
||||
headers={"Destination": dest_url, "Overwrite": "T"},
|
||||
)
|
||||
if response.status_code in (201, 204):
|
||||
logger.info("Marked processed: %s", filename)
|
||||
return True
|
||||
else:
|
||||
logger.warning(
|
||||
"MOVE returned %d for %s", response.status_code, filename
|
||||
)
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("Failed to mark %s as processed: %s", filename, e)
|
||||
return False
|
||||
@@ -0,0 +1,74 @@
|
||||
"""OrgMyLife API client for creating tasks from extracted action items.
|
||||
|
||||
POST to /api/tasks with title, source_url, description.
|
||||
Handles unreachable API gracefully (log warning, continue).
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OrgMyLifeClient:
|
||||
"""HTTP client for OrgMyLife task creation."""
|
||||
|
||||
def __init__(self, base_url: str, api_key: str):
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.api_key = api_key
|
||||
|
||||
def create_task(
|
||||
self,
|
||||
title: str,
|
||||
source_url: str,
|
||||
description: str = "",
|
||||
priority: int = 4,
|
||||
) -> dict:
|
||||
"""Create a task in OrgMyLife.
|
||||
|
||||
Args:
|
||||
title: Task title (action item description).
|
||||
source_url: Reference back to the source note.
|
||||
description: Optional additional description.
|
||||
priority: Task priority (1-5, default 4).
|
||||
|
||||
Returns:
|
||||
Response data dict, or empty dict on failure.
|
||||
"""
|
||||
url = f"{self.base_url}/api/tasks"
|
||||
payload = {
|
||||
"title": title,
|
||||
"source_url": source_url,
|
||||
"description": description,
|
||||
"priority": priority,
|
||||
}
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
try:
|
||||
response = httpx.post(
|
||||
url,
|
||||
json=payload,
|
||||
headers=headers,
|
||||
timeout=10.0,
|
||||
)
|
||||
response.raise_for_status()
|
||||
logger.info("Created OrgMyLife task: %s", title[:50])
|
||||
return response.json()
|
||||
|
||||
except httpx.ConnectError:
|
||||
logger.warning(
|
||||
"OrgMyLife API unreachable at %s — skipping task creation", self.base_url
|
||||
)
|
||||
return {}
|
||||
except httpx.HTTPStatusError as e:
|
||||
logger.warning(
|
||||
"OrgMyLife API error %d: %s", e.response.status_code, e.response.text[:200]
|
||||
)
|
||||
return {}
|
||||
except Exception as e:
|
||||
logger.warning("OrgMyLife task creation failed: %s", e)
|
||||
return {}
|
||||
@@ -0,0 +1 @@
|
||||
"""Wiki-link generation and stub page creation."""
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Wiki-link generation from detected entities.
|
||||
|
||||
Converts entity references to SilverBullet-compatible wiki-links:
|
||||
- Person → [[people/firstname-lastname]]
|
||||
- Project → [[projects/project-slug]]
|
||||
|
||||
Only the first occurrence of each entity is linked; subsequent mentions remain plain text.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from slugify import slugify
|
||||
|
||||
from ingestion.enrichment.models import Entity
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _entity_to_wikilink(entity: Entity) -> str:
|
||||
"""Convert an entity to its wiki-link representation."""
|
||||
slug = slugify(entity.value)
|
||||
if entity.type == "person":
|
||||
return f"[[people/{slug}]]"
|
||||
elif entity.type == "project":
|
||||
return f"[[projects/{slug}]]"
|
||||
return ""
|
||||
|
||||
|
||||
def generate_wiki_links(text: str, entities: list[Entity]) -> str:
|
||||
"""Insert wiki-links into text for person and project entities.
|
||||
|
||||
Only the first occurrence of each entity's value is converted to a wiki-link.
|
||||
Dates and action_items are skipped (handled by frontmatter/renderer).
|
||||
|
||||
Args:
|
||||
text: The document body text.
|
||||
entities: List of detected entities from enrichment.
|
||||
|
||||
Returns:
|
||||
Text with wiki-links inserted at first occurrence of each entity.
|
||||
"""
|
||||
# Filter to linkable entity types
|
||||
linkable = [e for e in entities if e.type in ("person", "project")]
|
||||
|
||||
# Track which entity values we've already linked (case-insensitive)
|
||||
linked_values: set[str] = set()
|
||||
|
||||
for entity in linkable:
|
||||
lower_val = entity.value.lower()
|
||||
if lower_val in linked_values:
|
||||
continue
|
||||
|
||||
# Find first occurrence and replace it
|
||||
idx = text.find(entity.value)
|
||||
if idx == -1:
|
||||
# Try case-insensitive search
|
||||
idx = text.lower().find(lower_val)
|
||||
if idx == -1:
|
||||
continue
|
||||
# Use the actual text at that position for replacement
|
||||
original_text = text[idx : idx + len(entity.value)]
|
||||
else:
|
||||
original_text = entity.value
|
||||
|
||||
wikilink = _entity_to_wikilink(entity)
|
||||
if wikilink:
|
||||
# Replace only the first occurrence
|
||||
text = text[:idx] + wikilink + text[idx + len(original_text) :]
|
||||
linked_values.add(lower_val)
|
||||
|
||||
return text
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Stub page creation for entities that don't yet have a NoteGraph page.
|
||||
|
||||
Creates minimal markdown pages in the appropriate directory (people/ or projects/)
|
||||
so that wiki-links resolve immediately after import.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from slugify import slugify
|
||||
|
||||
from ingestion.enrichment.models import Entity
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def create_stub_pages(entities: list[Entity], notes_dir: Path) -> list[Path]:
|
||||
"""Create stub pages for entities that don't have existing pages.
|
||||
|
||||
Args:
|
||||
entities: List of entities to check/create stubs for.
|
||||
notes_dir: Root notes directory (e.g., ./notes).
|
||||
|
||||
Returns:
|
||||
List of paths to newly created stub pages.
|
||||
"""
|
||||
notes_dir = Path(notes_dir)
|
||||
created: list[Path] = []
|
||||
|
||||
linkable = [e for e in entities if e.type in ("person", "project")]
|
||||
seen_slugs: set[str] = set()
|
||||
|
||||
for entity in linkable:
|
||||
slug = slugify(entity.value)
|
||||
if slug in seen_slugs:
|
||||
continue
|
||||
seen_slugs.add(slug)
|
||||
|
||||
if entity.type == "person":
|
||||
target = notes_dir / "people" / f"{slug}.md"
|
||||
elif entity.type == "project":
|
||||
target = notes_dir / "projects" / f"{slug}.md"
|
||||
else:
|
||||
continue
|
||||
|
||||
if target.exists():
|
||||
logger.debug("Page already exists: %s", target)
|
||||
continue
|
||||
|
||||
# Create stub page
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(f"# {entity.value}\n", encoding="utf-8")
|
||||
logger.info("Created stub page: %s", target)
|
||||
created.append(target)
|
||||
|
||||
return created
|
||||
@@ -0,0 +1 @@
|
||||
"""Output rendering, folder routing, and filename generation."""
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Filename generation with collision avoidance.
|
||||
|
||||
Pattern: YYYY-MM-DD-slugified-title.md
|
||||
Slug: lowercase alphanumeric + hyphens only.
|
||||
Appends numeric suffix (-2, -3, ...) if file already exists.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from slugify import slugify
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def generate_filename(title: str, date: str | None = None) -> str:
|
||||
"""Generate a filename from title and optional date.
|
||||
|
||||
Args:
|
||||
title: The note title to slugify.
|
||||
date: Optional date string (YYYY-MM-DD). Uses today if not provided.
|
||||
|
||||
Returns:
|
||||
Filename in format YYYY-MM-DD-slugified-title.md
|
||||
"""
|
||||
if date is None:
|
||||
date = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
|
||||
# Ensure date is just the date portion (handle ISO datetime strings)
|
||||
date_part = date[:10] if len(date) >= 10 else date
|
||||
|
||||
slug = slugify(title, max_length=80)
|
||||
if not slug:
|
||||
slug = "untitled"
|
||||
|
||||
return f"{date_part}-{slug}.md"
|
||||
|
||||
|
||||
def resolve_collision(target_dir: Path, filename: str) -> Path:
|
||||
"""Resolve filename collisions by appending numeric suffix.
|
||||
|
||||
Args:
|
||||
target_dir: Directory where the file will be written.
|
||||
filename: Desired filename (e.g., 2024-03-15-meeting-notes.md).
|
||||
|
||||
Returns:
|
||||
Full path with unique filename (appends -2, -3, etc. if needed).
|
||||
"""
|
||||
target = Path(target_dir) / filename
|
||||
|
||||
if not target.exists():
|
||||
return target
|
||||
|
||||
# Split filename to insert suffix before .md
|
||||
stem = filename.rsplit(".md", 1)[0]
|
||||
suffix_num = 2
|
||||
|
||||
while True:
|
||||
new_filename = f"{stem}-{suffix_num}.md"
|
||||
new_target = Path(target_dir) / new_filename
|
||||
if not new_target.exists():
|
||||
logger.debug("Collision resolved: %s → %s", filename, new_filename)
|
||||
return new_target
|
||||
suffix_num += 1
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Markdown + YAML frontmatter rendering for processed notes.
|
||||
|
||||
Renders:
|
||||
- Frontmatter with title, date, tags, people, projects, source
|
||||
- Body with wiki-links inserted
|
||||
- Action Items section at the end
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from ingestion.enrichment.models import EnrichmentResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def render_note(
|
||||
enrichment: EnrichmentResult, body_text: str, source_file: Path
|
||||
) -> str:
|
||||
"""Render a complete markdown note with YAML frontmatter.
|
||||
|
||||
Args:
|
||||
enrichment: The enrichment result with metadata and entities.
|
||||
body_text: The document body (with wiki-links already inserted).
|
||||
source_file: Original source file path for provenance.
|
||||
|
||||
Returns:
|
||||
Complete markdown string with frontmatter, body, and action items.
|
||||
"""
|
||||
# Build frontmatter data
|
||||
frontmatter: dict = {
|
||||
"title": enrichment.title,
|
||||
}
|
||||
|
||||
# Extract date from entities or use today
|
||||
date_entities = [e for e in enrichment.entities if e.type == "date"]
|
||||
if date_entities:
|
||||
frontmatter["date"] = date_entities[0].value
|
||||
else:
|
||||
frontmatter["date"] = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
|
||||
if enrichment.tags:
|
||||
frontmatter["tags"] = enrichment.tags
|
||||
|
||||
# People and projects from entities
|
||||
people = list(
|
||||
{e.value for e in enrichment.entities if e.type == "person"}
|
||||
)
|
||||
projects = list(
|
||||
{e.value for e in enrichment.entities if e.type == "project"}
|
||||
)
|
||||
|
||||
if people:
|
||||
frontmatter["people"] = sorted(people)
|
||||
if projects:
|
||||
frontmatter["projects"] = sorted(projects)
|
||||
|
||||
frontmatter["source"] = {
|
||||
"file": source_file.name,
|
||||
"imported": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S"),
|
||||
}
|
||||
|
||||
# Render YAML frontmatter
|
||||
yaml_str = yaml.dump(
|
||||
frontmatter,
|
||||
default_flow_style=False,
|
||||
allow_unicode=True,
|
||||
sort_keys=False,
|
||||
)
|
||||
|
||||
# Build the full document
|
||||
parts = [
|
||||
"---",
|
||||
yaml_str.rstrip(),
|
||||
"---",
|
||||
"",
|
||||
body_text.rstrip(),
|
||||
]
|
||||
|
||||
# Add action items section if any
|
||||
if enrichment.action_items:
|
||||
parts.append("")
|
||||
parts.append("## Action Items")
|
||||
parts.append("")
|
||||
for item in enrichment.action_items:
|
||||
line = f"- [ ] {item.description}"
|
||||
if item.assignee:
|
||||
line += f" ({item.assignee})"
|
||||
if item.deadline:
|
||||
line += f" (deadline: {item.deadline})"
|
||||
parts.append(line)
|
||||
|
||||
parts.append("") # trailing newline
|
||||
return "\n".join(parts)
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Category-based folder routing for output files.
|
||||
|
||||
Routes notes to the appropriate NoteGraph folder based on detected category:
|
||||
- meeting → notes/meetings/
|
||||
- project → notes/projects/
|
||||
- decision → notes/decisions/
|
||||
- default → notes/inbox/
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Category to folder mapping
|
||||
CATEGORY_FOLDERS: dict[str, str] = {
|
||||
"meeting": "meetings",
|
||||
"project": "projects",
|
||||
"decision": "decisions",
|
||||
}
|
||||
|
||||
|
||||
def route_to_folder(
|
||||
category: str, notes_dir: Path, override: Path | None = None
|
||||
) -> Path:
|
||||
"""Determine the output folder for a note based on its category.
|
||||
|
||||
Args:
|
||||
category: The detected category (meeting, project, decision, inbox).
|
||||
notes_dir: Root notes directory (e.g., ./notes).
|
||||
override: Optional explicit output directory override (--output-dir).
|
||||
|
||||
Returns:
|
||||
Path to the target folder (created if it doesn't exist).
|
||||
"""
|
||||
if override is not None:
|
||||
target = Path(override)
|
||||
else:
|
||||
folder_name = CATEGORY_FOLDERS.get(category, "inbox")
|
||||
target = Path(notes_dir) / folder_name
|
||||
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
logger.debug("Routing category '%s' to folder: %s", category, target)
|
||||
return target
|
||||
@@ -0,0 +1,217 @@
|
||||
"""Main processing pipeline orchestrating discovery → extraction → enrichment → linking → output.
|
||||
|
||||
Handles:
|
||||
- Progress tracking (current file / total)
|
||||
- Per-file error handling (log + skip + continue)
|
||||
- Summary reporting (processed, success, failed counts)
|
||||
- Dry-run mode
|
||||
"""
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from ingestion.config import IngestionConfig
|
||||
from ingestion.discovery import discover_files
|
||||
from ingestion.enrichment.agent import EnrichmentAgent
|
||||
from ingestion.enrichment.models import EnrichmentResult
|
||||
from ingestion.extraction import get_extractor
|
||||
from ingestion.integrations.git import commit_imported_notes
|
||||
from ingestion.integrations.orgmylife import OrgMyLifeClient
|
||||
from ingestion.linking.linker import generate_wiki_links
|
||||
from ingestion.linking.stubs import create_stub_pages
|
||||
from ingestion.output.naming import generate_filename, resolve_collision
|
||||
from ingestion.output.renderer import render_note
|
||||
from ingestion.output.router import route_to_folder
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProcessResult:
|
||||
"""Result of processing a single file."""
|
||||
|
||||
source_file: Path
|
||||
output_file: Path | None = None
|
||||
success: bool = False
|
||||
error: str | None = None
|
||||
stub_pages: list[Path] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BatchResult:
|
||||
"""Summary of a batch processing run."""
|
||||
|
||||
total: int = 0
|
||||
success: int = 0
|
||||
failed: int = 0
|
||||
results: list[ProcessResult] = field(default_factory=list)
|
||||
|
||||
|
||||
def process_file(
|
||||
file_path: Path,
|
||||
config: IngestionConfig,
|
||||
enrichment_agent: EnrichmentAgent | None = None,
|
||||
dry_run: bool = False,
|
||||
no_commit: bool = False,
|
||||
create_tasks: bool = False,
|
||||
output_dir: Path | None = None,
|
||||
) -> ProcessResult:
|
||||
"""Process a single file through the full ingestion pipeline.
|
||||
|
||||
Steps: extract → enrich → link → render → write → (optional) git commit
|
||||
|
||||
Args:
|
||||
file_path: Path to the source file.
|
||||
config: Ingestion configuration.
|
||||
enrichment_agent: Optional pre-configured enrichment agent.
|
||||
dry_run: If True, don't write any files.
|
||||
no_commit: If True, skip git commit.
|
||||
create_tasks: If True, create OrgMyLife tasks for action items.
|
||||
output_dir: Optional override for output directory.
|
||||
|
||||
Returns:
|
||||
ProcessResult with success status and output path.
|
||||
"""
|
||||
result = ProcessResult(source_file=file_path)
|
||||
|
||||
try:
|
||||
# 1. Extract text
|
||||
extractor = get_extractor(file_path, config)
|
||||
if extractor is None:
|
||||
result.error = f"No extractor available for {file_path.suffix}"
|
||||
return result
|
||||
|
||||
extraction = extractor.extract(file_path)
|
||||
if not extraction.text.strip():
|
||||
result.error = "No text extracted from file"
|
||||
return result
|
||||
|
||||
# 2. Enrich with LLM
|
||||
if enrichment_agent is None:
|
||||
enrichment_agent = EnrichmentAgent(config)
|
||||
|
||||
enrichment = enrichment_agent.enrich(extraction.text)
|
||||
|
||||
# 3. Generate wiki-links
|
||||
linked_text = generate_wiki_links(extraction.text, enrichment.entities)
|
||||
|
||||
# 4. Render output
|
||||
rendered = render_note(enrichment, linked_text, file_path)
|
||||
|
||||
# 5. Determine output location
|
||||
notes_dir = Path(config.notes_dir)
|
||||
target_folder = route_to_folder(enrichment.category, notes_dir, output_dir)
|
||||
|
||||
# 6. Generate filename
|
||||
date_entities = [e for e in enrichment.entities if e.type == "date"]
|
||||
date_str = date_entities[0].value if date_entities else None
|
||||
filename = generate_filename(enrichment.title, date_str)
|
||||
output_path = resolve_collision(target_folder, filename)
|
||||
|
||||
if dry_run:
|
||||
logger.info("[DRY RUN] Would write: %s", output_path)
|
||||
result.output_file = output_path
|
||||
result.success = True
|
||||
return result
|
||||
|
||||
# 7. Write file
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(rendered, encoding="utf-8")
|
||||
result.output_file = output_path
|
||||
logger.info("Written: %s", output_path)
|
||||
|
||||
# 8. Create stub pages
|
||||
stubs = create_stub_pages(enrichment.entities, notes_dir)
|
||||
result.stub_pages = stubs
|
||||
|
||||
# 9. Create OrgMyLife tasks if requested
|
||||
if create_tasks and enrichment.action_items and config.orgmylife_api_key:
|
||||
_create_tasks(enrichment, output_path, config)
|
||||
|
||||
result.success = True
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Failed to process %s: %s", file_path, e)
|
||||
result.error = str(e)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def process_batch(
|
||||
paths: list[Path],
|
||||
config: IngestionConfig,
|
||||
dry_run: bool = False,
|
||||
no_commit: bool = False,
|
||||
create_tasks: bool = False,
|
||||
output_dir: Path | None = None,
|
||||
progress_callback=None,
|
||||
) -> BatchResult:
|
||||
"""Process a batch of files through the pipeline.
|
||||
|
||||
Args:
|
||||
paths: List of file paths to process.
|
||||
config: Ingestion configuration.
|
||||
dry_run: If True, don't write any files.
|
||||
no_commit: If True, skip git commit.
|
||||
create_tasks: If True, create OrgMyLife tasks.
|
||||
output_dir: Optional override for output directory.
|
||||
progress_callback: Optional callable(current, total, filename).
|
||||
|
||||
Returns:
|
||||
BatchResult with summary counts and per-file results.
|
||||
"""
|
||||
batch = BatchResult(total=len(paths))
|
||||
enrichment_agent = EnrichmentAgent(config)
|
||||
written_files: list[Path] = []
|
||||
|
||||
for i, file_path in enumerate(paths, 1):
|
||||
if progress_callback:
|
||||
progress_callback(i, len(paths), file_path.name)
|
||||
|
||||
logger.info("Processing [%d/%d]: %s", i, len(paths), file_path.name)
|
||||
|
||||
result = process_file(
|
||||
file_path=file_path,
|
||||
config=config,
|
||||
enrichment_agent=enrichment_agent,
|
||||
dry_run=dry_run,
|
||||
no_commit=True, # Commit once at end
|
||||
create_tasks=create_tasks,
|
||||
output_dir=output_dir,
|
||||
)
|
||||
|
||||
batch.results.append(result)
|
||||
if result.success:
|
||||
batch.success += 1
|
||||
if result.output_file:
|
||||
written_files.append(result.output_file)
|
||||
written_files.extend(result.stub_pages)
|
||||
else:
|
||||
batch.failed += 1
|
||||
logger.warning("Skipped %s: %s", file_path.name, result.error)
|
||||
|
||||
# Git commit all at once (unless dry-run or no-commit)
|
||||
if not dry_run and not no_commit and config.auto_commit and written_files:
|
||||
commit_imported_notes(written_files, source_type="batch-import")
|
||||
|
||||
return batch
|
||||
|
||||
|
||||
def _create_tasks(
|
||||
enrichment: EnrichmentResult, note_path: Path, config: IngestionConfig
|
||||
) -> None:
|
||||
"""Create OrgMyLife tasks for action items."""
|
||||
try:
|
||||
client = OrgMyLifeClient(
|
||||
base_url=config.orgmylife_api_url,
|
||||
api_key=config.orgmylife_api_key,
|
||||
)
|
||||
for item in enrichment.action_items:
|
||||
client.create_task(
|
||||
title=item.description,
|
||||
source_url=str(note_path),
|
||||
description=f"From note: {enrichment.title}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to create OrgMyLife tasks: %s", e)
|
||||
@@ -0,0 +1,40 @@
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "notegraph-ingestion"
|
||||
version = "0.1.0"
|
||||
description = "NoteGraph document ingestion and AI-powered auto-referencing pipeline"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
"click>=8.1",
|
||||
"watchdog>=4.0",
|
||||
"pytesseract>=0.3.10",
|
||||
"pdfplumber>=0.11",
|
||||
"python-docx>=1.1",
|
||||
"litellm>=1.40",
|
||||
"httpx>=0.27",
|
||||
"pydantic>=2.7",
|
||||
"pydantic-settings>=2.3",
|
||||
"fastapi>=0.111",
|
||||
"uvicorn>=0.30",
|
||||
"python-multipart>=0.0.9",
|
||||
"pyyaml>=6.0",
|
||||
"python-slugify>=8.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest>=8.0",
|
||||
"pytest-asyncio>=0.23",
|
||||
"hypothesis>=6.100",
|
||||
"pytest-mock>=3.12",
|
||||
"respx>=0.21",
|
||||
]
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["ingestion"]
|
||||
|
||||
[project.scripts]
|
||||
notegraph-ingest = "ingestion.cli:cli"
|
||||
@@ -0,0 +1 @@
|
||||
"""Tests for the NoteGraph ingestion pipeline."""
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Shared test fixtures for the ingestion test suite."""
|
||||
|
||||
import pytest
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from ingestion.config import IngestionConfig
|
||||
from ingestion.enrichment.models import ActionItem, EnrichmentResult, Entity
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tmp_notes_dir(tmp_path: Path) -> Path:
|
||||
"""Create a temporary notes directory structure."""
|
||||
notes = tmp_path / "notes"
|
||||
notes.mkdir()
|
||||
(notes / "inbox").mkdir()
|
||||
(notes / "meetings").mkdir()
|
||||
(notes / "projects").mkdir()
|
||||
(notes / "people").mkdir()
|
||||
(notes / "decisions").mkdir()
|
||||
return notes
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tmp_inbox(tmp_path: Path) -> Path:
|
||||
"""Create a temporary inbox directory."""
|
||||
inbox = tmp_path / "inbox"
|
||||
inbox.mkdir()
|
||||
return inbox
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_config(tmp_path: Path) -> IngestionConfig:
|
||||
"""Create a test configuration with temporary paths."""
|
||||
notes_dir = tmp_path / "notes"
|
||||
notes_dir.mkdir(exist_ok=True)
|
||||
inbox_dir = tmp_path / "inbox"
|
||||
inbox_dir.mkdir(exist_ok=True)
|
||||
return IngestionConfig(
|
||||
llm_provider="openai",
|
||||
llm_model="gpt-4o",
|
||||
openai_api_key="test-key-fake",
|
||||
notes_dir=str(notes_dir),
|
||||
inbox_dir=str(inbox_dir),
|
||||
auto_commit=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_enrichment_result() -> EnrichmentResult:
|
||||
"""Create a sample enrichment result for testing."""
|
||||
return EnrichmentResult(
|
||||
title="Meeting with Beier GmbH",
|
||||
category="meeting",
|
||||
tags=["meeting", "customer"],
|
||||
entities=[
|
||||
Entity(type="person", value="Thomas Beier", confidence=0.95),
|
||||
Entity(type="project", value="Druckluft", confidence=0.9),
|
||||
Entity(type="date", value="2024-03-15", confidence=0.99),
|
||||
],
|
||||
action_items=[
|
||||
ActionItem(description="Send updated proposal", assignee="André", deadline="2024-03-22"),
|
||||
ActionItem(description="Schedule follow-up meeting"),
|
||||
],
|
||||
summary="Discussion about the Druckluft project.",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_llm_client():
|
||||
"""Create a mock LLM client that returns a valid enrichment JSON."""
|
||||
import json
|
||||
|
||||
mock = MagicMock()
|
||||
mock.complete.return_value = json.dumps({
|
||||
"title": "Test Note",
|
||||
"category": "inbox",
|
||||
"tags": ["test"],
|
||||
"entities": [],
|
||||
"action_items": [],
|
||||
"summary": "A test note.",
|
||||
})
|
||||
return mock
|
||||
@@ -0,0 +1,806 @@
|
||||
"""Property-based tests for NoteGraph Ingestion using Hypothesis.
|
||||
|
||||
All 18 correctness properties from the design document are tested here.
|
||||
Each test uses @settings(max_examples=100) as specified.
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import string
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from hypothesis import given, settings, assume, HealthCheck
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from ingestion.config import IngestionConfig
|
||||
from ingestion.discovery import SUPPORTED_EXTENSIONS, discover_files
|
||||
from ingestion.enrichment.models import ActionItem, EnrichmentResult, Entity
|
||||
from ingestion.extraction.text import PlainTextExtractor
|
||||
from ingestion.integrations.git import commit_imported_notes
|
||||
from ingestion.integrations.orgmylife import OrgMyLifeClient
|
||||
from ingestion.linking.linker import generate_wiki_links, _entity_to_wikilink
|
||||
from ingestion.output.naming import generate_filename, resolve_collision
|
||||
from ingestion.output.renderer import render_note
|
||||
from ingestion.output.router import CATEGORY_FOLDERS, route_to_folder
|
||||
from ingestion.watcher import should_ignore
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Common settings for all property tests
|
||||
# ---------------------------------------------------------------------------
|
||||
pbt_settings = settings(
|
||||
max_examples=100,
|
||||
suppress_health_check=[HealthCheck.function_scoped_fixture],
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Strategy for supported file extensions
|
||||
supported_ext_st = st.sampled_from(sorted(SUPPORTED_EXTENSIONS))
|
||||
|
||||
# Strategy for unsupported file extensions
|
||||
unsupported_ext_st = st.text(
|
||||
alphabet=string.ascii_lowercase, min_size=2, max_size=5
|
||||
).map(lambda s: f".{s}").filter(lambda ext: ext not in SUPPORTED_EXTENSIONS)
|
||||
|
||||
# Strategy for valid filenames (no path separators, no null bytes, no reserved chars)
|
||||
filename_chars = st.characters(
|
||||
whitelist_categories=("L", "N"),
|
||||
)
|
||||
filename_st = st.text(alphabet=filename_chars, min_size=1, max_size=20).map(
|
||||
lambda s: s.strip() or "file"
|
||||
)
|
||||
|
||||
# Strategy for valid UTF-8 text content (no null bytes, non-empty)
|
||||
utf8_text_st = st.text(
|
||||
alphabet=st.characters(blacklist_categories=("Cs",), blacklist_characters="\x00"),
|
||||
min_size=1,
|
||||
max_size=2000,
|
||||
)
|
||||
|
||||
# Strategy for entity types
|
||||
entity_type_st = st.sampled_from(["person", "project", "date", "action_item"])
|
||||
|
||||
# Strategy for confidence scores
|
||||
confidence_st = st.floats(min_value=0.0, max_value=1.0, allow_nan=False)
|
||||
|
||||
# Strategy for person/project names (non-empty, printable, with at least one letter)
|
||||
name_st = st.text(
|
||||
alphabet=st.characters(whitelist_categories=("L", "N", "Zs")),
|
||||
min_size=2,
|
||||
max_size=40,
|
||||
).filter(lambda s: s.strip() and any(c.isalpha() for c in s))
|
||||
|
||||
# Strategy for valid providers
|
||||
provider_st = st.sampled_from(["openai", "anthropic", "google", "mistral", "ollama"])
|
||||
|
||||
# Strategy for providers requiring API keys
|
||||
keyed_provider_st = st.sampled_from(["openai", "anthropic", "google", "mistral"])
|
||||
|
||||
# Strategy for categories
|
||||
category_st = st.sampled_from(["meeting", "project", "decision", "inbox"])
|
||||
|
||||
# Strategy for dates in YYYY-MM-DD format
|
||||
date_st = st.dates().map(lambda d: d.strftime("%Y-%m-%d"))
|
||||
|
||||
# Strategy for note titles (must produce a non-empty slug)
|
||||
title_st = st.text(
|
||||
alphabet=st.characters(whitelist_categories=("L", "N", "Zs")),
|
||||
min_size=2,
|
||||
max_size=60,
|
||||
).filter(lambda s: s.strip() and any(c.isalnum() for c in s))
|
||||
|
||||
# Strategy for action item descriptions
|
||||
action_desc_st = st.text(
|
||||
alphabet=st.characters(whitelist_categories=("L", "N", "Zs", "P")),
|
||||
min_size=3,
|
||||
max_size=100,
|
||||
).filter(lambda s: s.strip())
|
||||
|
||||
# Strategy for source types
|
||||
source_type_st = st.text(
|
||||
alphabet=string.ascii_lowercase + "-",
|
||||
min_size=3,
|
||||
max_size=20,
|
||||
).filter(lambda s: s.strip("-") and not s.startswith("-") and not s.endswith("-"))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 1 (Task 3.2): File discovery filtering
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 1: File discovery returns only supported extensions
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(
|
||||
supported_files=st.lists(
|
||||
st.tuples(filename_st, supported_ext_st), min_size=0, max_size=10
|
||||
),
|
||||
unsupported_files=st.lists(
|
||||
st.tuples(filename_st, unsupported_ext_st), min_size=0, max_size=10
|
||||
),
|
||||
)
|
||||
def test_discovery_returns_only_supported_extensions(
|
||||
tmp_path: Path, supported_files, unsupported_files
|
||||
):
|
||||
"""For any directory tree with arbitrary extensions, discovery returns exactly
|
||||
those with supported extensions (.pdf, .jpg, .jpeg, .png, .md, .txt, .docx)."""
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
td_path = Path(td)
|
||||
|
||||
# Create files with supported extensions
|
||||
expected = set()
|
||||
for i, (name, ext) in enumerate(supported_files):
|
||||
fp = td_path / f"s{i}_{name}{ext}"
|
||||
fp.write_text("content", encoding="utf-8")
|
||||
expected.add(fp)
|
||||
|
||||
# Create files with unsupported extensions
|
||||
for i, (name, ext) in enumerate(unsupported_files):
|
||||
fp = td_path / f"u{i}_{name}{ext}"
|
||||
fp.write_text("content", encoding="utf-8")
|
||||
|
||||
result = discover_files(td_path)
|
||||
result_set = set(result)
|
||||
|
||||
# All returned files must have supported extensions
|
||||
for f in result:
|
||||
assert f.suffix.lower() in SUPPORTED_EXTENSIONS
|
||||
|
||||
# All supported files we created must be in the result
|
||||
for f in expected:
|
||||
assert f in result_set
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 2 (Task 9.2): Bulk import resilience
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 2: Failed files do not block remaining processing
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(
|
||||
fail_indices=st.lists(st.integers(min_value=0, max_value=4), unique=True, max_size=3),
|
||||
)
|
||||
def test_bulk_import_resilience(tmp_path: Path, fail_indices, mock_config):
|
||||
"""For any batch where a subset fails, remaining files are processed successfully."""
|
||||
from ingestion.pipeline import process_batch
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
td_path = Path(td)
|
||||
|
||||
# Create 5 text files
|
||||
files = []
|
||||
for i in range(5):
|
||||
f = td_path / f"note_{i}.txt"
|
||||
f.write_text(f"Content of note {i}", encoding="utf-8")
|
||||
files.append(f)
|
||||
|
||||
# Normalize fail indices to valid range
|
||||
fail_set = {i for i in fail_indices if 0 <= i < 5}
|
||||
|
||||
# Mock enrichment agent that fails for specific indices
|
||||
call_count = {"n": 0}
|
||||
|
||||
def mock_enrich(text):
|
||||
idx = call_count["n"]
|
||||
call_count["n"] += 1
|
||||
if idx in fail_set:
|
||||
raise RuntimeError(f"Simulated failure for file {idx}")
|
||||
return EnrichmentResult(
|
||||
title=f"Note {idx}",
|
||||
category="inbox",
|
||||
tags=[],
|
||||
entities=[],
|
||||
action_items=[],
|
||||
summary=None,
|
||||
)
|
||||
|
||||
mock_agent = MagicMock()
|
||||
mock_agent.enrich.side_effect = mock_enrich
|
||||
|
||||
# Set up output directory
|
||||
notes_dir = td_path / "notes"
|
||||
notes_dir.mkdir()
|
||||
(notes_dir / "inbox").mkdir()
|
||||
mock_config.notes_dir = str(notes_dir)
|
||||
|
||||
with patch("ingestion.pipeline.EnrichmentAgent", return_value=mock_agent):
|
||||
result = process_batch(
|
||||
paths=files,
|
||||
config=mock_config,
|
||||
dry_run=False,
|
||||
no_commit=True,
|
||||
)
|
||||
|
||||
expected_success = 5 - len(fail_set)
|
||||
assert result.success == expected_success
|
||||
assert result.failed == len(fail_set)
|
||||
assert result.total == 5
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 3 (Task 2.6): Text extraction identity
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 3: Text extraction preserves content for text-based formats
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(content=utf8_text_st)
|
||||
def test_text_extraction_identity(tmp_path: Path, content: str):
|
||||
"""For any valid UTF-8 string written to a .md or .txt file, extraction returns
|
||||
identical content (modulo platform newline normalization)."""
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
td_path = Path(td)
|
||||
extractor = PlainTextExtractor()
|
||||
|
||||
for ext in (".md", ".txt"):
|
||||
fp = td_path / f"test{ext}"
|
||||
fp.write_text(content, encoding="utf-8")
|
||||
result = extractor.extract(fp)
|
||||
# Python's read_text uses universal newlines (translates \r\n and \r to \n)
|
||||
# This is expected platform behavior, so we compare after normalization
|
||||
expected = content.replace("\r\n", "\n").replace("\r", "\n")
|
||||
assert result.text == expected
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 4 (Task 5.4): Entity parsing valid structure
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 4: Entity parsing extracts all entities with valid structure
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(
|
||||
entities=st.lists(
|
||||
st.fixed_dictionaries({
|
||||
"type": entity_type_st,
|
||||
"value": name_st,
|
||||
"confidence": confidence_st,
|
||||
}),
|
||||
min_size=1,
|
||||
max_size=10,
|
||||
),
|
||||
)
|
||||
def test_entity_parsing_valid_structure(entities):
|
||||
"""For any valid enrichment response JSON, each parsed entity has valid type,
|
||||
non-empty value, confidence in [0.0, 1.0]."""
|
||||
data = {
|
||||
"title": "Test",
|
||||
"category": "inbox",
|
||||
"tags": [],
|
||||
"entities": entities,
|
||||
"action_items": [],
|
||||
}
|
||||
|
||||
result = EnrichmentResult.model_validate(data)
|
||||
|
||||
for entity in result.entities:
|
||||
assert entity.type in ("person", "project", "date", "action_item")
|
||||
assert len(entity.value) > 0
|
||||
assert 0.0 <= entity.confidence <= 1.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 5 (Task 5.4): Confidence threshold filtering
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 5: Confidence threshold filtering
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(
|
||||
scores=st.lists(confidence_st, min_size=1, max_size=20),
|
||||
)
|
||||
def test_confidence_threshold_filtering(scores):
|
||||
"""For any list of entities with arbitrary confidence scores, filtered output
|
||||
contains exactly those with confidence >= 0.7."""
|
||||
entities = [
|
||||
Entity(type="person", value=f"Person {i}", confidence=score)
|
||||
for i, score in enumerate(scores)
|
||||
]
|
||||
|
||||
threshold = 0.7
|
||||
filtered = [e for e in entities if e.confidence >= threshold]
|
||||
|
||||
# Verify the filter matches what the enrichment agent does
|
||||
from ingestion.enrichment.agent import EnrichmentAgent
|
||||
|
||||
agent = MagicMock(spec=EnrichmentAgent)
|
||||
agent.confidence_threshold = threshold
|
||||
agent._filter_by_confidence = EnrichmentAgent._filter_by_confidence.__get__(agent)
|
||||
|
||||
result = agent._filter_by_confidence(entities)
|
||||
|
||||
assert len(result) == len(filtered)
|
||||
for entity in result:
|
||||
assert entity.confidence >= threshold
|
||||
|
||||
# Verify no entity below threshold is included
|
||||
for entity in entities:
|
||||
if entity.confidence < threshold:
|
||||
assert entity not in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 6 (Task 7.3): Wiki-link format per entity type
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 6: Wiki-link generation follows correct format per entity type
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(name=name_st)
|
||||
def test_wikilink_format_person(name: str):
|
||||
"""For any person name → [[people/slugified-name]]."""
|
||||
from slugify import slugify
|
||||
|
||||
entity = Entity(type="person", value=name, confidence=0.9)
|
||||
link = _entity_to_wikilink(entity)
|
||||
|
||||
expected_slug = slugify(name)
|
||||
assume(expected_slug) # skip if slugify produces empty string
|
||||
|
||||
assert link == f"[[people/{expected_slug}]]"
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(name=name_st)
|
||||
def test_wikilink_format_project(name: str):
|
||||
"""For any project name → [[projects/slugified-name]]."""
|
||||
from slugify import slugify
|
||||
|
||||
entity = Entity(type="project", value=name, confidence=0.9)
|
||||
link = _entity_to_wikilink(entity)
|
||||
|
||||
expected_slug = slugify(name)
|
||||
assume(expected_slug) # skip if slugify produces empty string
|
||||
|
||||
assert link == f"[[projects/{expected_slug}]]"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 7 (Task 7.3): Only first occurrence linked
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 7: Only first entity occurrence is linked
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(
|
||||
name=name_st,
|
||||
n_occurrences=st.integers(min_value=2, max_value=5),
|
||||
)
|
||||
def test_only_first_occurrence_linked(name: str, n_occurrences: int):
|
||||
"""For any text with N >= 2 occurrences of the same entity, only the first
|
||||
is converted to a wiki-link."""
|
||||
from slugify import slugify
|
||||
|
||||
slug = slugify(name)
|
||||
assume(slug) # skip if slugify produces empty string
|
||||
|
||||
# Build text with multiple occurrences separated by unique text
|
||||
separator = ". Then we discussed "
|
||||
text = separator.join([name] * n_occurrences)
|
||||
|
||||
entity = Entity(type="person", value=name, confidence=0.9)
|
||||
result = generate_wiki_links(text, [entity])
|
||||
|
||||
expected_link = f"[[people/{slug}]]"
|
||||
|
||||
# First occurrence should be a wiki-link
|
||||
assert expected_link in result
|
||||
|
||||
# Count wiki-links — should be exactly 1
|
||||
link_count = result.count(expected_link)
|
||||
assert link_count == 1, f"Expected 1 wiki-link, found {link_count}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 8 (Task 8.4): Frontmatter round-trip validity
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 8: Frontmatter round-trip validity
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(
|
||||
title=title_st,
|
||||
category=category_st,
|
||||
tags=st.lists(st.text(alphabet=string.ascii_lowercase, min_size=2, max_size=10), max_size=5),
|
||||
people=st.lists(name_st, min_size=0, max_size=3),
|
||||
projects=st.lists(name_st, min_size=0, max_size=3),
|
||||
)
|
||||
def test_frontmatter_roundtrip_validity(
|
||||
tmp_path: Path, title, category, tags, people, projects
|
||||
):
|
||||
"""For any valid EnrichmentResult, render to markdown then parse frontmatter →
|
||||
valid YAML with required fields."""
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
td_path = Path(td)
|
||||
|
||||
entities = []
|
||||
for p in people:
|
||||
entities.append(Entity(type="person", value=p, confidence=0.9))
|
||||
for proj in projects:
|
||||
entities.append(Entity(type="project", value=proj, confidence=0.9))
|
||||
entities.append(Entity(type="date", value="2024-06-15", confidence=0.99))
|
||||
|
||||
enrichment = EnrichmentResult(
|
||||
title=title,
|
||||
category=category,
|
||||
tags=tags,
|
||||
entities=entities,
|
||||
action_items=[],
|
||||
summary=None,
|
||||
)
|
||||
|
||||
source_file = td_path / "source.pdf"
|
||||
source_file.write_text("", encoding="utf-8")
|
||||
|
||||
rendered = render_note(enrichment, "Body text here.", source_file)
|
||||
|
||||
# Parse frontmatter
|
||||
assert rendered.startswith("---\n")
|
||||
parts = rendered.split("---\n", 2)
|
||||
assert len(parts) >= 3, "Expected YAML frontmatter delimiters"
|
||||
|
||||
frontmatter_yaml = parts[1]
|
||||
parsed = yaml.safe_load(frontmatter_yaml)
|
||||
|
||||
# Required fields
|
||||
assert "title" in parsed
|
||||
assert "date" in parsed
|
||||
assert "source" in parsed
|
||||
assert "file" in parsed["source"]
|
||||
assert "imported" in parsed["source"]
|
||||
|
||||
# Title matches
|
||||
assert parsed["title"] == title
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 9 (Task 4.2): Provider routing from configuration
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 9: Provider routing from configuration
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(provider=provider_st)
|
||||
def test_provider_routing_from_configuration(provider: str):
|
||||
"""For any valid provider name (openai, anthropic, google, mistral, ollama),
|
||||
configuring LLM_PROVIDER produces the correct model string."""
|
||||
config = IngestionConfig(
|
||||
llm_provider=provider,
|
||||
llm_model="test-model",
|
||||
openai_api_key="fake",
|
||||
anthropic_api_key="fake",
|
||||
google_api_key="fake",
|
||||
mistral_api_key="fake",
|
||||
)
|
||||
|
||||
model_string = config.llm_model_string
|
||||
|
||||
if provider == "ollama":
|
||||
assert model_string == "ollama/test-model"
|
||||
else:
|
||||
assert model_string == f"{provider}/test-model"
|
||||
|
||||
# Verify the provider prefix is correct
|
||||
assert model_string.startswith(f"{provider}/")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 10 (Task 4.3): Missing API key error
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 10: Missing API key produces descriptive error
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(provider=keyed_provider_st)
|
||||
def test_missing_api_key_error(provider: str):
|
||||
"""For any provider requiring an API key, when the key is empty,
|
||||
validate_api_key() raises SystemExit naming the missing variable."""
|
||||
config = IngestionConfig(
|
||||
llm_provider=provider,
|
||||
llm_model="test-model",
|
||||
openai_api_key="",
|
||||
anthropic_api_key="",
|
||||
google_api_key="",
|
||||
mistral_api_key="",
|
||||
)
|
||||
|
||||
env_var_map = {
|
||||
"openai": "OPENAI_API_KEY",
|
||||
"anthropic": "ANTHROPIC_API_KEY",
|
||||
"google": "GOOGLE_API_KEY",
|
||||
"mistral": "MISTRAL_API_KEY",
|
||||
}
|
||||
|
||||
with pytest.raises(SystemExit) as exc_info:
|
||||
config.validate_api_key()
|
||||
|
||||
error_msg = str(exc_info.value)
|
||||
assert env_var_map[provider] in error_msg
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 11 (Task 8.4): Action items in output
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 11: Action items appear in output markdown section
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(
|
||||
descriptions=st.lists(action_desc_st, min_size=1, max_size=5),
|
||||
)
|
||||
def test_action_items_in_output(tmp_path: Path, descriptions):
|
||||
"""For any list of action items, rendered markdown contains `## Action Items`
|
||||
section listing every item."""
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
td_path = Path(td)
|
||||
|
||||
action_items = [ActionItem(description=desc) for desc in descriptions]
|
||||
|
||||
enrichment = EnrichmentResult(
|
||||
title="Test Note",
|
||||
category="inbox",
|
||||
tags=[],
|
||||
entities=[Entity(type="date", value="2024-01-01", confidence=0.99)],
|
||||
action_items=action_items,
|
||||
summary=None,
|
||||
)
|
||||
|
||||
source_file = td_path / "source.txt"
|
||||
source_file.write_text("", encoding="utf-8")
|
||||
|
||||
rendered = render_note(enrichment, "Body text.", source_file)
|
||||
|
||||
assert "## Action Items" in rendered
|
||||
|
||||
for desc in descriptions:
|
||||
assert desc in rendered
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 12 (Task 15.2): OrgMyLife task payload
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 12: OrgMyLife task payload correctness
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(
|
||||
description=action_desc_st,
|
||||
note_path=st.text(
|
||||
alphabet=string.ascii_lowercase + "/.-_", min_size=5, max_size=50
|
||||
).filter(lambda s: s.strip() and not s.startswith("/")),
|
||||
)
|
||||
def test_orgmylife_task_payload(description: str, note_path: str):
|
||||
"""For any action item description and note path, the task creation payload
|
||||
has correct title and source_url."""
|
||||
import httpx
|
||||
|
||||
client = OrgMyLifeClient(base_url="https://api.example.com", api_key="test-key")
|
||||
|
||||
# Capture the payload sent to httpx.post
|
||||
captured_payload = {}
|
||||
|
||||
def mock_post(url, json=None, headers=None, timeout=None):
|
||||
captured_payload.update(json)
|
||||
response = MagicMock()
|
||||
response.status_code = 201
|
||||
response.raise_for_status = MagicMock()
|
||||
response.json.return_value = {"id": 1, "title": json["title"]}
|
||||
return response
|
||||
|
||||
with patch.object(httpx, "post", side_effect=mock_post):
|
||||
client.create_task(title=description, source_url=note_path)
|
||||
|
||||
assert captured_payload["title"] == description
|
||||
assert captured_payload["source_url"] == note_path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 13 (Task 8.4): Category-based folder routing
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 13: Category-based folder routing
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(category=category_st)
|
||||
def test_category_based_folder_routing(tmp_path: Path, category: str):
|
||||
"""For any category (meeting/project/decision/inbox), output path is within
|
||||
the correct folder."""
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
notes_dir = Path(td) / "notes"
|
||||
notes_dir.mkdir()
|
||||
|
||||
result = route_to_folder(category, notes_dir)
|
||||
|
||||
expected_folder = CATEGORY_FOLDERS.get(category, "inbox")
|
||||
assert result == notes_dir / expected_folder
|
||||
assert result.exists()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 14 (Task 8.4): Filename date-slug pattern
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 14: Output filename follows date-slug pattern
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(date=date_st, title=title_st)
|
||||
def test_filename_date_slug_pattern(date: str, title: str):
|
||||
"""For any date and title, filename matches YYYY-MM-DD-slugified-title.md."""
|
||||
filename = generate_filename(title, date)
|
||||
|
||||
# Must end with .md
|
||||
assert filename.endswith(".md")
|
||||
|
||||
# Must start with the date
|
||||
assert filename.startswith(date)
|
||||
|
||||
# Pattern: YYYY-MM-DD-slug.md (slug is lowercase alphanumeric + hyphens)
|
||||
pattern = r"^\d{4}-\d{2}-\d{2}-[a-z0-9]([a-z0-9-]*[a-z0-9])?\.md$"
|
||||
assert re.match(pattern, filename), f"Filename '{filename}' doesn't match pattern"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 15 (Task 8.4): Filename collision avoidance
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 15: Filename collision avoidance
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(
|
||||
title=title_st,
|
||||
n_existing=st.integers(min_value=1, max_value=5),
|
||||
)
|
||||
def test_filename_collision_avoidance(tmp_path: Path, title: str, n_existing: int):
|
||||
"""When a file with the same name exists, a numeric suffix is appended."""
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
td_path = Path(td)
|
||||
|
||||
filename = generate_filename(title, "2024-01-15")
|
||||
|
||||
# Create the original file
|
||||
(td_path / filename).write_text("existing", encoding="utf-8")
|
||||
|
||||
# Create collision files with suffixes
|
||||
stem = filename.rsplit(".md", 1)[0]
|
||||
for i in range(2, n_existing + 1):
|
||||
(td_path / f"{stem}-{i}.md").write_text("existing", encoding="utf-8")
|
||||
|
||||
# Resolve collision
|
||||
resolved = resolve_collision(td_path, filename)
|
||||
|
||||
# The resolved path must not exist yet
|
||||
assert not resolved.exists()
|
||||
|
||||
# The resolved path must have a numeric suffix
|
||||
expected_suffix = f"-{n_existing + 1}.md"
|
||||
assert str(resolved).endswith(expected_suffix)
|
||||
|
||||
# The resolved path must be in the same directory
|
||||
assert resolved.parent == td_path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 16 (Task 14.2): Temporary file filtering
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 16: Temporary file filtering
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(
|
||||
filename=st.text(
|
||||
alphabet=st.characters(
|
||||
whitelist_categories=("L", "N", "P"),
|
||||
blacklist_characters="/\\:\x00",
|
||||
),
|
||||
min_size=1,
|
||||
max_size=50,
|
||||
),
|
||||
)
|
||||
def test_temporary_file_filtering(filename: str):
|
||||
"""For any filename, should_ignore returns true iff name starts with '.'
|
||||
or ends with '.tmp'."""
|
||||
result = should_ignore(filename)
|
||||
expected = filename.startswith(".") or filename.endswith(".tmp")
|
||||
assert result == expected, f"should_ignore('{filename}') = {result}, expected {expected}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 17 (Task 9.2): Dry-run no file writes
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 17: Dry-run produces no file writes
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(
|
||||
n_files=st.integers(min_value=1, max_value=5),
|
||||
)
|
||||
def test_dry_run_no_file_writes(tmp_path: Path, n_files: int, mock_config):
|
||||
"""For any input with dry_run=True, zero files are created."""
|
||||
from ingestion.pipeline import process_batch
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
td_path = Path(td)
|
||||
|
||||
# Create input files
|
||||
files = []
|
||||
for i in range(n_files):
|
||||
f = td_path / f"input_{i}.txt"
|
||||
f.write_text(f"Content {i}", encoding="utf-8")
|
||||
files.append(f)
|
||||
|
||||
# Set up output directory
|
||||
notes_dir = td_path / "output_notes"
|
||||
notes_dir.mkdir()
|
||||
(notes_dir / "inbox").mkdir()
|
||||
mock_config.notes_dir = str(notes_dir)
|
||||
|
||||
# Mock the enrichment agent
|
||||
def mock_enrich(text):
|
||||
return EnrichmentResult(
|
||||
title="Test",
|
||||
category="inbox",
|
||||
tags=[],
|
||||
entities=[],
|
||||
action_items=[],
|
||||
summary=None,
|
||||
)
|
||||
|
||||
mock_agent = MagicMock()
|
||||
mock_agent.enrich.side_effect = mock_enrich
|
||||
|
||||
with patch("ingestion.pipeline.EnrichmentAgent", return_value=mock_agent):
|
||||
result = process_batch(
|
||||
paths=files,
|
||||
config=mock_config,
|
||||
dry_run=True,
|
||||
no_commit=True,
|
||||
)
|
||||
|
||||
# Count files in the output notes directory (should be 0 new markdown files)
|
||||
output_files = list(notes_dir.rglob("*.md"))
|
||||
assert len(output_files) == 0, f"Expected 0 files written, found {len(output_files)}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 18 (Task 16.2): Git commit message format
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature: notegraph-ingestion, Property 18: Git commit message format
|
||||
|
||||
|
||||
@pbt_settings
|
||||
@given(
|
||||
count=st.integers(min_value=1, max_value=1000),
|
||||
source_type=source_type_st,
|
||||
)
|
||||
def test_git_commit_message_format(count: int, source_type: str):
|
||||
"""For any count N and source type, message matches
|
||||
'ingestion: import N notes from [source-type]'."""
|
||||
files = [Path(f"/fake/note_{i}.md") for i in range(count)]
|
||||
captured_messages = []
|
||||
|
||||
def mock_run(cmd, **kwargs):
|
||||
if cmd[0] == "git" and cmd[1] == "commit":
|
||||
# Extract -m argument
|
||||
msg_idx = cmd.index("-m") + 1
|
||||
captured_messages.append(cmd[msg_idx])
|
||||
result = MagicMock()
|
||||
result.returncode = 0
|
||||
result.stdout = ""
|
||||
result.stderr = ""
|
||||
return result
|
||||
|
||||
with patch("subprocess.run", side_effect=mock_run):
|
||||
commit_imported_notes(files, source_type=source_type)
|
||||
|
||||
assert len(captured_messages) == 1
|
||||
expected = f"ingestion: import {count} notes from {source_type}"
|
||||
assert captured_messages[0] == expected
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Drop-folder watcher using watchdog for continuous file ingestion.
|
||||
|
||||
Monitors the inbox folder for new files and triggers the processing pipeline.
|
||||
Moves processed files to archive/ and failed files to failed/ subdirectory.
|
||||
Ignores temporary files (names starting with '.' or ending with '.tmp').
|
||||
"""
|
||||
|
||||
import logging
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from watchdog.events import FileCreatedEvent, FileSystemEventHandler
|
||||
from watchdog.observers import Observer
|
||||
|
||||
from ingestion.config import IngestionConfig
|
||||
from ingestion.discovery import SUPPORTED_EXTENSIONS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def should_ignore(filename: str) -> bool:
|
||||
"""Check if a file should be ignored by the watcher.
|
||||
|
||||
Ignores files starting with '.' or ending with '.tmp'.
|
||||
"""
|
||||
return filename.startswith(".") or filename.endswith(".tmp")
|
||||
|
||||
|
||||
class InboxHandler(FileSystemEventHandler):
|
||||
"""Handle new files in the inbox directory."""
|
||||
|
||||
def __init__(self, config: IngestionConfig):
|
||||
self.config = config
|
||||
self.inbox_dir = Path(config.inbox_dir)
|
||||
self.archive_dir = self.inbox_dir / "archive"
|
||||
self.failed_dir = self.inbox_dir / "failed"
|
||||
self.archive_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.failed_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def on_created(self, event: FileCreatedEvent):
|
||||
"""Triggered when a new file is created in the inbox."""
|
||||
if event.is_directory:
|
||||
return
|
||||
|
||||
file_path = Path(event.src_path)
|
||||
self._process_file(file_path)
|
||||
|
||||
def _process_file(self, file_path: Path) -> None:
|
||||
"""Process a single file and move to archive or failed."""
|
||||
if should_ignore(file_path.name):
|
||||
logger.debug("Ignoring temporary file: %s", file_path.name)
|
||||
return
|
||||
|
||||
if file_path.suffix.lower() not in SUPPORTED_EXTENSIONS:
|
||||
logger.debug("Ignoring unsupported file: %s", file_path.name)
|
||||
return
|
||||
|
||||
# Small delay to ensure file is fully written
|
||||
time.sleep(0.5)
|
||||
|
||||
logger.info("Processing new file: %s", file_path.name)
|
||||
|
||||
try:
|
||||
from ingestion.pipeline import process_file
|
||||
|
||||
result = process_file(file_path=file_path, config=self.config)
|
||||
|
||||
if result.success:
|
||||
# Move to archive
|
||||
dest = self.archive_dir / file_path.name
|
||||
shutil.move(str(file_path), str(dest))
|
||||
logger.info("Archived: %s → %s", file_path.name, dest)
|
||||
else:
|
||||
# Move to failed
|
||||
dest = self.failed_dir / file_path.name
|
||||
shutil.move(str(file_path), str(dest))
|
||||
logger.warning(
|
||||
"Failed: %s → %s (error: %s)",
|
||||
file_path.name,
|
||||
dest,
|
||||
result.error,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Unhandled error processing %s: %s", file_path.name, e)
|
||||
dest = self.failed_dir / file_path.name
|
||||
try:
|
||||
shutil.move(str(file_path), str(dest))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def start_watcher(config: IngestionConfig) -> None:
|
||||
"""Start the drop-folder watcher.
|
||||
|
||||
Processes existing files first, then watches for new ones.
|
||||
Blocks until interrupted (Ctrl+C).
|
||||
"""
|
||||
inbox_dir = Path(config.inbox_dir)
|
||||
inbox_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
handler = InboxHandler(config)
|
||||
|
||||
# Process existing files on startup
|
||||
existing = sorted(inbox_dir.iterdir())
|
||||
for f in existing:
|
||||
if f.is_file() and not should_ignore(f.name):
|
||||
handler._process_file(f)
|
||||
|
||||
# Start watching
|
||||
observer = Observer()
|
||||
observer.schedule(handler, str(inbox_dir), recursive=False)
|
||||
observer.start()
|
||||
|
||||
logger.info("Watcher started on: %s", inbox_dir)
|
||||
|
||||
try:
|
||||
while True:
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
observer.stop()
|
||||
|
||||
observer.join()
|
||||
@@ -0,0 +1 @@
|
||||
"""FastAPI web upload endpoint for mobile file ingestion."""
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,233 @@
|
||||
"""FastAPI web upload endpoint for mobile file ingestion.
|
||||
|
||||
Auth: Token-based via URL query param (?token=secret) or header (X-Upload-Token).
|
||||
Processing: File is saved immediately, enrichment runs in background.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import secrets
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI, File, HTTPException, Query, Request, UploadFile
|
||||
from fastapi.responses import HTMLResponse
|
||||
|
||||
from ingestion.config import IngestionConfig
|
||||
from ingestion.web.browse import router as browse_router
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
app = FastAPI(title="NoteGraph", version="0.1.0")
|
||||
app.include_router(browse_router)
|
||||
|
||||
|
||||
@app.get("/")
|
||||
async def root(request: Request):
|
||||
"""Redirect root to browse page."""
|
||||
from fastapi.responses import RedirectResponse
|
||||
token = request.query_params.get("token", "")
|
||||
return RedirectResponse(url=f"/browse?token={token}" if token else "/browse")
|
||||
|
||||
|
||||
def _get_config() -> IngestionConfig:
|
||||
return IngestionConfig()
|
||||
|
||||
|
||||
def _verify_token(request: Request, token: str = Query(default=None)):
|
||||
"""Verify upload token from query param or X-Upload-Token header."""
|
||||
config = _get_config()
|
||||
# Token is the password part of SB_USER (after the colon)
|
||||
sb_user = config.sb_user.strip()
|
||||
parts = sb_user.split(":", 1)
|
||||
expected_token = parts[1].strip() if len(parts) == 2 else sb_user
|
||||
|
||||
# Check query param
|
||||
if token and secrets.compare_digest(token.encode(), expected_token.encode()):
|
||||
return True
|
||||
|
||||
# Check header
|
||||
header_token = request.headers.get("X-Upload-Token", "")
|
||||
if header_token and secrets.compare_digest(header_token.encode(), expected_token.encode()):
|
||||
return True
|
||||
|
||||
raise HTTPException(status_code=401, detail="Invalid or missing token. Use ?token=YOUR_PASSWORD")
|
||||
|
||||
|
||||
def _process_in_background(file_path: Path, config: IngestionConfig):
|
||||
"""Run the enrichment pipeline in a background thread."""
|
||||
try:
|
||||
from ingestion.pipeline import process_file
|
||||
result = process_file(file_path=file_path, config=config)
|
||||
if result.success:
|
||||
logger.info("Background processing complete: %s → %s", file_path.name, result.output_file)
|
||||
else:
|
||||
logger.warning("Background processing failed for %s: %s", file_path.name, result.error)
|
||||
except Exception as e:
|
||||
logger.error("Background processing error for %s: %s", file_path.name, e)
|
||||
|
||||
|
||||
UPLOAD_HTML = """<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>NoteGraph Upload</title>
|
||||
<style>
|
||||
* { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
body {
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
||||
background: #1a1a2e;
|
||||
color: #eee;
|
||||
min-height: 100vh;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 1rem;
|
||||
}
|
||||
.container {
|
||||
background: #16213e;
|
||||
border-radius: 12px;
|
||||
padding: 2rem;
|
||||
width: 100%;
|
||||
max-width: 480px;
|
||||
box-shadow: 0 8px 32px rgba(0,0,0,0.3);
|
||||
}
|
||||
h1 { text-align: center; margin-bottom: 1.5rem; font-size: 1.5rem; color: #4fc3f7; }
|
||||
.drop-zone {
|
||||
border: 2px dashed #4fc3f7;
|
||||
border-radius: 8px;
|
||||
padding: 2rem 1rem;
|
||||
text-align: center;
|
||||
cursor: pointer;
|
||||
transition: all 0.3s;
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
.drop-zone:hover, .drop-zone.dragover {
|
||||
background: rgba(79, 195, 247, 0.1);
|
||||
border-color: #81d4fa;
|
||||
}
|
||||
.drop-zone p { margin-bottom: 0.5rem; font-size: 1.1rem; }
|
||||
.drop-zone small { color: #999; }
|
||||
input[type="file"] { display: none; }
|
||||
.btn {
|
||||
display: block; width: 100%; padding: 0.9rem;
|
||||
background: #4fc3f7; color: #1a1a2e; border: none;
|
||||
border-radius: 8px; font-size: 1rem; font-weight: 600;
|
||||
cursor: pointer; transition: background 0.3s;
|
||||
}
|
||||
.btn:hover { background: #81d4fa; }
|
||||
.btn:disabled { background: #555; cursor: not-allowed; }
|
||||
.status { margin-top: 1rem; padding: 0.8rem; border-radius: 6px; text-align: center; display: none; }
|
||||
.status.success { display: block; background: #1b5e20; color: #a5d6a7; }
|
||||
.status.error { display: block; background: #b71c1c; color: #ef9a9a; }
|
||||
.status.loading { display: block; background: #0d47a1; color: #90caf9; }
|
||||
.filename { margin-top: 0.5rem; font-size: 0.9rem; color: #aaa; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<h1>📄 NoteGraph Upload</h1>
|
||||
<form id="uploadForm" enctype="multipart/form-data">
|
||||
<div class="drop-zone" id="dropZone">
|
||||
<p>Drop file here or tap to select</p>
|
||||
<small>PDF, JPG, PNG, DOCX, TXT, MD</small>
|
||||
</div>
|
||||
<input type="file" id="fileInput" name="file" accept=".pdf,.jpg,.jpeg,.png,.docx,.txt,.md">
|
||||
<div class="filename" id="fileName"></div>
|
||||
<button type="submit" class="btn" id="submitBtn" disabled>Upload & Process</button>
|
||||
</form>
|
||||
<div class="status" id="status"></div>
|
||||
</div>
|
||||
<script>
|
||||
const dropZone = document.getElementById('dropZone');
|
||||
const fileInput = document.getElementById('fileInput');
|
||||
const form = document.getElementById('uploadForm');
|
||||
const submitBtn = document.getElementById('submitBtn');
|
||||
const statusEl = document.getElementById('status');
|
||||
const fileNameEl = document.getElementById('fileName');
|
||||
|
||||
// Get token from URL
|
||||
const urlParams = new URLSearchParams(window.location.search);
|
||||
const token = urlParams.get('token') || '';
|
||||
|
||||
dropZone.addEventListener('click', () => fileInput.click());
|
||||
dropZone.addEventListener('dragover', (e) => { e.preventDefault(); dropZone.classList.add('dragover'); });
|
||||
dropZone.addEventListener('dragleave', () => dropZone.classList.remove('dragover'));
|
||||
dropZone.addEventListener('drop', (e) => {
|
||||
e.preventDefault(); dropZone.classList.remove('dragover');
|
||||
if (e.dataTransfer.files.length) { fileInput.files = e.dataTransfer.files; updateFileName(); }
|
||||
});
|
||||
fileInput.addEventListener('change', updateFileName);
|
||||
|
||||
function updateFileName() {
|
||||
if (fileInput.files.length) { fileNameEl.textContent = fileInput.files[0].name; submitBtn.disabled = false; }
|
||||
}
|
||||
|
||||
form.addEventListener('submit', async (e) => {
|
||||
e.preventDefault();
|
||||
if (!fileInput.files.length) return;
|
||||
submitBtn.disabled = true;
|
||||
statusEl.className = 'status loading';
|
||||
statusEl.textContent = 'Uploading...';
|
||||
statusEl.style.display = 'block';
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append('file', fileInput.files[0]);
|
||||
|
||||
try {
|
||||
const resp = await fetch('/upload?token=' + encodeURIComponent(token), { method: 'POST', body: formData });
|
||||
const data = await resp.json();
|
||||
if (resp.ok) {
|
||||
statusEl.className = 'status success';
|
||||
statusEl.textContent = data.message || 'Upload successful!';
|
||||
} else {
|
||||
statusEl.className = 'status error';
|
||||
statusEl.textContent = data.detail || 'Upload failed';
|
||||
}
|
||||
} catch (err) {
|
||||
statusEl.className = 'status error';
|
||||
statusEl.textContent = 'Network error: ' + err.message;
|
||||
}
|
||||
submitBtn.disabled = false;
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
|
||||
@app.get("/", response_class=HTMLResponse)
|
||||
@app.get("/upload", response_class=HTMLResponse)
|
||||
async def upload_form(request: Request, token: str = Query(default=None)):
|
||||
"""Serve the upload form. Token required."""
|
||||
_verify_token(request, token)
|
||||
return UPLOAD_HTML
|
||||
|
||||
|
||||
@app.post("/upload")
|
||||
async def upload_file(
|
||||
request: Request,
|
||||
file: UploadFile = File(...),
|
||||
token: str = Query(default=None),
|
||||
):
|
||||
"""Accept file upload, save immediately, process in background."""
|
||||
_verify_token(request, token)
|
||||
|
||||
config = _get_config()
|
||||
inbox = Path(config.inbox_dir)
|
||||
inbox.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Save uploaded file
|
||||
target = inbox / file.filename
|
||||
content = await file.read()
|
||||
target.write_bytes(content)
|
||||
logger.info("Uploaded file saved: %s (%d bytes)", target, len(content))
|
||||
|
||||
# Process in background (don't block the response)
|
||||
thread = threading.Thread(
|
||||
target=_process_in_background,
|
||||
args=(target, config),
|
||||
daemon=True,
|
||||
)
|
||||
thread.start()
|
||||
|
||||
return {"message": f"File received: {file.filename}. Processing in background — check SilverBullet in ~30s."}
|
||||
@@ -0,0 +1,5 @@
|
||||
# ⚖️ Decisions
|
||||
|
||||
Recorded decisions with context and rationale.
|
||||
|
||||
<!-- Add entries like: [[decisions/2026-05-13-decision-title]] -->
|
||||
@@ -0,0 +1,15 @@
|
||||
---
|
||||
title: "Intro — Jury-Sitzung Startup-Bewertung (2026)"
|
||||
date: 2026-05-15
|
||||
tags: [intro, jury, unikat, presentation, UNIKAT, AI-Source]
|
||||
people: [André Knie]
|
||||
projects: [UNIKAT Ideenwettbewerb]
|
||||
---
|
||||
|
||||
# Intro — Jury-Sitzung Startup-Bewertung (2026)
|
||||
|
||||
**Dauer:** ca. 30 Sekunden
|
||||
|
||||
---
|
||||
|
||||
Hi, ich bin André. Als promovierter Physiker bringe ich der Deutschen Bahn Digitalisierung bei und habe selbst ein KI-Unternehmen für den Mittelstand gegründet. Ich schaue aus drei Perspektiven auf die Teams: der Wissenschaftler, der fragt ob es wirklich funktioniert — der Konzernmensch, der weiß wie schwer Veränderung ist — und der Gründer, der weiß wie es sich anfühlt, wenn man selbst dran glaubt und sonst erstmal keiner. Ich freue mich auf den Tag mit Euch.
|
||||
@@ -0,0 +1,44 @@
|
||||
---
|
||||
title: "reMarkable → NoteGraph Sync Ideas"
|
||||
date: 2026-05-18
|
||||
tags: [AI-Source, remarkable, notegraph, automation, backlog]
|
||||
---
|
||||
|
||||
# reMarkable → NoteGraph Sync Ideas
|
||||
|
||||
## Problem
|
||||
reMarkable doesn't support automatic background sync to external services. The built-in Google Drive integration requires manual "Send to" per document.
|
||||
|
||||
## Options Explored
|
||||
|
||||
### 1. rmapi (archived)
|
||||
- `juruen/rmapi` — archived Jul 2024, old API endpoints are 404
|
||||
- `grahamgreen/rmapi` — fork, but same deprecated API
|
||||
- `rschroll/rmcl` — Python lib, broken on Python 3.14 (uses `asks` which is unmaintained)
|
||||
- **Verdict:** Old reMarkable Cloud API (v1/v2) is dead. Not viable.
|
||||
|
||||
### 2. Google Drive Poller
|
||||
- reMarkable can "Send to Google Drive" (manual per-document)
|
||||
- Build a service on VPS that polls a Google Drive folder via API
|
||||
- New PDFs → download → ingestion pipeline (OCR + LLM) → NoteGraph
|
||||
- **Requires:** Google Cloud service account or OAuth, manual "Send to Drive" step on tablet
|
||||
- **Effort:** Medium (Google API auth setup + poller service)
|
||||
|
||||
### 3. Email Bridge
|
||||
- reMarkable can "Send by email" to any address
|
||||
- Set up a mail receiver on VPS (e.g., Postfix + script)
|
||||
- Attachments saved to ingestion inbox → auto-processed
|
||||
- **Requires:** Mail server setup, DNS MX record
|
||||
- **Effort:** Medium-High
|
||||
|
||||
### 4. Manual Upload via Browse UI
|
||||
- Export from reMarkable desktop app → local folder
|
||||
- Upload via `notes.andreknie.de/upload?token=...`
|
||||
- Already works today, just not automated
|
||||
- **Effort:** Zero (already done)
|
||||
|
||||
## Decision
|
||||
Park for now. Use manual upload. Revisit when reMarkable opens their API or a community tool catches up.
|
||||
|
||||
## Auth Token (saved)
|
||||
Device registered successfully on 2026-05-18. Token stored in `remarkable_token.json` (local only, not committed). Can be reused if API becomes available again.
|
||||
@@ -0,0 +1,6 @@
|
||||
# 📥 Quick Capture
|
||||
|
||||
Drop quick thoughts here. Sort them later.
|
||||
|
||||
---
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
# 🧠 NoteGraph
|
||||
|
||||
Welcome to your personal knowledge base.
|
||||
|
||||
## Quick Links
|
||||
|
||||
- [[inbox/capture]] — Quick capture page
|
||||
- [[projects/index]] — Project notes
|
||||
- [[people/index]] — People & contacts
|
||||
- [[meetings/index]] — Meeting notes
|
||||
- [[decisions/index]] — Recorded decisions
|
||||
|
||||
## Recent
|
||||
|
||||
<!-- This section auto-updates via SilverBullet queries -->
|
||||
|
||||
## Tags
|
||||
|
||||
Use tags to organize: `#project/orgmylife`, `#person/name`, `#decision`, `#followup`
|
||||
|
||||
## Linking to OrgMyLife Tasks
|
||||
|
||||
Reference tasks with: `#task/123` (links to OrgMyLife task #123)
|
||||
@@ -0,0 +1,7 @@
|
||||
# 📅 Meetings
|
||||
|
||||
Meeting notes organized by date.
|
||||
|
||||
## Template
|
||||
|
||||
Use [[templates/meeting]] for new meeting notes.
|
||||
@@ -0,0 +1,5 @@
|
||||
# 👥 People
|
||||
|
||||
Index of people and their context.
|
||||
|
||||
<!-- Add entries like: [[people/firstname-lastname]] -->
|
||||
@@ -0,0 +1,12 @@
|
||||
# 📁 Projects
|
||||
|
||||
## Active Projects
|
||||
|
||||
- [[projects/OrgMyLife]] — Personal task & coordination system
|
||||
- [[projects/AI-Orchestrator]] — Autonomous coding agent orchestration
|
||||
- [[projects/NoteGraph]] — This knowledge system
|
||||
- [[projects/Confluence-Bot]] — Confluence automation toolkit
|
||||
|
||||
## Archived
|
||||
|
||||
(none yet)
|
||||
@@ -0,0 +1,22 @@
|
||||
# Meeting: [Title]
|
||||
|
||||
**Date:** YYYY-MM-DD
|
||||
**Participants:**
|
||||
**Project:**
|
||||
**Tags:** #meeting
|
||||
|
||||
## Agenda
|
||||
|
||||
1.
|
||||
|
||||
## Notes
|
||||
|
||||
-
|
||||
|
||||
## Decisions
|
||||
|
||||
-
|
||||
|
||||
## Action Items
|
||||
|
||||
- [ ]
|
||||
@@ -0,0 +1,8 @@
|
||||
# [Title]
|
||||
|
||||
**Date:** YYYY-MM-DD
|
||||
**Tags:**
|
||||
**Related:**
|
||||
|
||||
---
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
.git
|
||||
.env
|
||||
__pycache__
|
||||
*.pyc
|
||||
*.pyo
|
||||
.pytest_cache
|
||||
.mypy_cache
|
||||
*.db
|
||||
alembic/versions/__pycache__
|
||||
tests/__pycache__
|
||||
@@ -0,0 +1,34 @@
|
||||
DATABASE_URL=sqlite:///./orgmylife.db
|
||||
|
||||
# For Docker/production use PostgreSQL:
|
||||
# DATABASE_URL=postgresql://orgmylife:changeme@db:5432/orgmylife
|
||||
|
||||
# PostgreSQL password (used by docker-compose)
|
||||
POSTGRES_PASSWORD=changeme
|
||||
|
||||
# API Secret for orchestrator endpoints (leave empty for local dev)
|
||||
API_SECRET=
|
||||
|
||||
# App login credentials (for the web UI)
|
||||
APP_USERNAME=andre
|
||||
APP_PASSWORD=your_login_password
|
||||
|
||||
# Nextcloud Configuration
|
||||
NEXTCLOUD_URL=https://your-nextcloud-instance.com
|
||||
NEXTCLOUD_USERNAME=your_app_username
|
||||
NEXTCLOUD_PASSWORD=your_app_password
|
||||
|
||||
# dHive Email / IMAP Configuration
|
||||
IMAP_SERVER=imap.your-email-server.com
|
||||
IMAP_PORT=993
|
||||
EMAIL_USERNAME=your@dhive-email.com
|
||||
EMAIL_PASSWORD=your_email_password
|
||||
EMAIL_WEBMAIL_URL=https://your-webmail.com
|
||||
|
||||
# Gmail Configuration (use App Password, not regular password)
|
||||
# Generate at: https://myaccount.google.com/apppasswords
|
||||
GMAIL_USERNAME=your@gmail.com
|
||||
GMAIL_PASSWORD=your_app_password
|
||||
|
||||
# NoteGraph Configuration (Send to Notes feature)
|
||||
NOTEGRAPH_TOKEN=your_notegraph_token
|
||||
@@ -0,0 +1,6 @@
|
||||
.env
|
||||
venv/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.db
|
||||
@@ -0,0 +1,77 @@
|
||||
stages:
|
||||
- sync
|
||||
- deploy
|
||||
|
||||
variables:
|
||||
DEPLOY_HOST: "217.160.174.2"
|
||||
DEPLOY_USER: "root"
|
||||
DEPLOY_PATH: "/opt/OrgMyLife"
|
||||
|
||||
.ssh_setup: &ssh_setup
|
||||
before_script:
|
||||
- apt-get update -qq && apt-get install -y openssh-client rsync curl
|
||||
- install -d -m 700 ~/.ssh
|
||||
- echo "$VPS_SSH_KEY" > ~/.ssh/deploy_key
|
||||
- chmod 600 ~/.ssh/deploy_key
|
||||
- ssh-keyscan -H "${DEPLOY_HOST}" > ~/.ssh/known_hosts
|
||||
- chmod 600 ~/.ssh/known_hosts
|
||||
|
||||
deploy-vps:
|
||||
stage: deploy
|
||||
image: node:20
|
||||
<<: *ssh_setup
|
||||
rules:
|
||||
- if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main"'
|
||||
changes:
|
||||
- "shared/OrgMyLife/**/*"
|
||||
script:
|
||||
- ssh -i ~/.ssh/deploy_key -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes "${DEPLOY_USER}@${DEPLOY_HOST}" "install -d -m 755 ${DEPLOY_PATH}"
|
||||
- |
|
||||
cat <<EOF > .env
|
||||
POSTGRES_PASSWORD=$ENV_POSTGRES_PASSWORD
|
||||
API_SECRET=$ENV_API_SECRET
|
||||
APP_USERNAME=$ENV_APP_USERNAME
|
||||
APP_PASSWORD=$ENV_APP_PASSWORD
|
||||
NEXTCLOUD_URL=$ENV_NEXTCLOUD_URL
|
||||
NEXTCLOUD_USERNAME=$ENV_NEXTCLOUD_USERNAME
|
||||
NEXTCLOUD_PASSWORD=$ENV_NEXTCLOUD_PASSWORD
|
||||
IMAP_SERVER=$ENV_IMAP_SERVER
|
||||
IMAP_PORT=993
|
||||
EMAIL_USERNAME=$ENV_EMAIL_USERNAME
|
||||
EMAIL_PASSWORD=$ENV_EMAIL_PASSWORD
|
||||
EMAIL_WEBMAIL_URL=$ENV_EMAIL_WEBMAIL_URL
|
||||
GMAIL_USERNAME=$ENV_GMAIL_USERNAME
|
||||
GMAIL_PASSWORD=$ENV_GMAIL_PASSWORD
|
||||
EOF
|
||||
- rsync -az --delete -e "ssh -i ~/.ssh/deploy_key -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes" shared/OrgMyLife/ "${DEPLOY_USER}@${DEPLOY_HOST}:${DEPLOY_PATH}/"
|
||||
- rsync -az -e "ssh -i ~/.ssh/deploy_key -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes" .env "${DEPLOY_USER}@${DEPLOY_HOST}:${DEPLOY_PATH}/.env"
|
||||
- ssh -i ~/.ssh/deploy_key -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes "${DEPLOY_USER}@${DEPLOY_HOST}" "cd ${DEPLOY_PATH} && docker compose up -d --build"
|
||||
|
||||
collect-tasks:
|
||||
stage: sync
|
||||
image: alpine:latest
|
||||
rules:
|
||||
- if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TARGET == "collect-tasks"'
|
||||
script:
|
||||
- apk add --no-cache curl jq
|
||||
- |
|
||||
curl -s -u "${ORGMYLIFE_USER}:${ORGMYLIFE_PASS}" \
|
||||
-H "Authorization: Bearer ${ORGMYLIFE_API_SECRET}" \
|
||||
"https://api.andreknie.de/api/orchestrator/tasks" > /tmp/tasks.json
|
||||
- echo "Tasks fetched. Since GitLab handles issues differently, issue creation logic needs to be adapted for GitLab API."
|
||||
|
||||
backup-dismissed:
|
||||
stage: sync
|
||||
image: alpine:latest
|
||||
rules:
|
||||
- if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TARGET == "backup-dismissed"'
|
||||
script:
|
||||
- apk add --no-cache curl git
|
||||
- |
|
||||
curl -s -u "${ORGMYLIFE_USER}:${ORGMYLIFE_PASS}" \
|
||||
-H "Authorization: Bearer ${ORGMYLIFE_API_SECRET}" \
|
||||
"https://api.andreknie.de/api/tasks/dismissed" > shared/OrgMyLife/dismissed_tasks.json
|
||||
- git config user.name "gitlab-ci[bot]"
|
||||
- git config user.email "gitlab-ci[bot]@gitlab.dhive.io"
|
||||
- git add shared/OrgMyLife/dismissed_tasks.json
|
||||
- git diff --cached --quiet || (git commit -m "chore: backup dismissed task IDs [skip ci]" && git push "https://oauth2:${PROJECT_ACCESS_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git" HEAD:$CI_COMMIT_BRANCH)
|
||||
@@ -0,0 +1,48 @@
|
||||
---
|
||||
inclusion: auto
|
||||
---
|
||||
|
||||
# OrgMyLife Standards
|
||||
|
||||
## Quality Gates (before every commit)
|
||||
|
||||
1. `gitleaks detect --source . --no-git` — no secrets in code
|
||||
2. `ruff check .` — linting passes (if configured)
|
||||
3. `pytest` — all tests pass
|
||||
4. No hardcoded secrets in source
|
||||
|
||||
## Conventional Commits
|
||||
|
||||
```
|
||||
feat(sync): add Gmail label filtering
|
||||
fix(api): handle empty task list gracefully
|
||||
test(digest): add test for overdue calculation
|
||||
docs(spec): update component descriptions
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
- pytest for unit and integration tests
|
||||
- httpx TestClient for API endpoint tests
|
||||
- Given-When-Then structure
|
||||
- 80% coverage target for new code
|
||||
|
||||
## Autonomous Mode
|
||||
|
||||
- Never push directly to main — use feature branches
|
||||
- Create PRs on GitHub targeting main
|
||||
- If stuck after 3 attempts: stop and document
|
||||
- Report: `PROGRESS: X%` / `HELP: description`
|
||||
|
||||
## Architecture
|
||||
|
||||
- FastAPI + SQLAlchemy + PostgreSQL
|
||||
- Alembic for migrations
|
||||
- MCP server for Kiro integration
|
||||
- Sync services: Nextcloud (CalDAV/ToDo), Gmail (IMAP), Backlog.md
|
||||
|
||||
## Deployment
|
||||
|
||||
- Docker on VPS (https://api.andreknie.de)
|
||||
- `docker compose up -d --build` for deploy
|
||||
- .env file on VPS for secrets (not in repo)
|
||||
@@ -0,0 +1,122 @@
|
||||
# AGENTS.md
|
||||
|
||||
---
|
||||
|
||||
## Coding rules
|
||||
|
||||
1. Read `SPEC.md` first and treat it as source of truth ("the spec"). If `SPEC.md` does not exist, **stop and ask**.
|
||||
2. Follow the user prompt exactly; do not omit explicitly requested steps.
|
||||
3. **Avoid over-engineering.** Only make changes that are directly requested or clearly necessary. Keep solutions simple and focused:
|
||||
- Scope: Don't add features, refactor code, or make "improvements" beyond what was asked. A bug fix doesn't need surrounding code cleaned up. A simple feature doesn't need extra configurability.
|
||||
- Documentation: Don't add docstrings, comments, or type annotations to code you didn't change. Only add comments where the logic isn't self-evident.
|
||||
- Defensive coding: Don't add error handling, fallbacks, or validation for scenarios that can't happen. Trust internal code and framework guarantees. Only validate at system boundaries (user input, external APIs).
|
||||
- Abstractions: Don't create helpers, utilities, or abstractions for one-time operations. Don't design for hypothetical future requirements. The right amount of complexity is the minimum needed for the current task.
|
||||
- Custom coding: Don't reinvent the wheel, use the functions the framework provides you with. If you think that using framework is not possible, consult the relevant online or MCP docs to make sure. If you still cannot find a solution within the framework, **stop and ask**.
|
||||
4. Do not hard-code values or create solutions that only work for specific test inputs; implement the actual logic that solves the problem generally.
|
||||
5. The resulting code must be robust, maintainable, and extendable.
|
||||
6. The resulting code must not contradict the spec.
|
||||
7. **Before writing code**, post a short and concise "Plan + spec mapping" summary and **ask for approval to proceed**. Do not implement until you receive the go-ahead.
|
||||
8. Documentation Rules:
|
||||
- filenames are always capitalized.
|
||||
- describes the **current state only**. Do not include change/history phrasing.
|
||||
- must always be coherent end-to-end never have stale or conflicting info.
|
||||
9. Always use repo virtual environment (unless there is none).
|
||||
|
||||
---
|
||||
|
||||
## Documentation MCP policy
|
||||
|
||||
- We run ONE docs MCP server: `docs` (multi-library).
|
||||
|
||||
### Library selection
|
||||
- For any docs query call (everything except `list_libraries`), you MUST set `library`.
|
||||
- If the prompt or repo context involves a specific product/project unambiguously, use that as `library` (e.g. `library="haystack"`).
|
||||
- If more than one library could plausibly match, call `list_libraries` and select the best match (do not invent/guess library ids).
|
||||
- If the task spans multiple products, run separate docs calls per library and label results by library.
|
||||
|
||||
### Version selection
|
||||
- If the prompt or repo context involves a version or range (e.g. `2.25`, `2.x`), call `find_version(library=..., targetVersion=...)` and record the returned `version`.
|
||||
- Use that resolved `version` for all subsequent calls for that library.
|
||||
- If `find_version` cannot resolve unambiguously, **stop and ask**.
|
||||
- If no version is mentioned, omit `version` (defaults to latest indexed for that library).
|
||||
|
||||
- We also run the `github` MCP server. Consult it always for code that is in public github instead of searching at the disk or cloning the repos locally.
|
||||
|
||||
---
|
||||
## Base branch rule
|
||||
|
||||
**BASE_BRANCH = the branch that new work branches are created from and PRs target.**
|
||||
|
||||
Default:
|
||||
- If the user did not explicitly specify a base/target branch, set:
|
||||
- `BASE_BRANCH=$(git branch --show-current)` (from the user's main checkout at session start)
|
||||
|
||||
Override:
|
||||
- If the user explicitly specifies a base/target branch in the prompt, use that as `BASE_BRANCH`.
|
||||
|
||||
Remote base ref:
|
||||
- If `origin/$BASE_BRANCH` exists, use that as `BASE_REF`, else use `$BASE_BRANCH`:
|
||||
- `git show-ref --verify --quiet "refs/remotes/origin/$BASE_BRANCH" && BASE_REF="origin/$BASE_BRANCH" || BASE_REF="$BASE_BRANCH"`
|
||||
|
||||
Notes:
|
||||
- This supports long-lived feature integration branches and stacked PRs.
|
||||
- The base branch may itself be another PR branch (for stacking); set BASE_BRANCH accordingly.
|
||||
|
||||
---
|
||||
|
||||
## Before work
|
||||
1. Determine base branch (unless user specified it explicitly):
|
||||
- `BASE_BRANCH=$(git branch --show-current)`
|
||||
2. Fetch latest refs:
|
||||
- `git fetch origin --prune`
|
||||
3. Update base branch (fast-forward only) if it has an `origin/` tracking ref:
|
||||
- `git switch "$BASE_BRANCH"`
|
||||
- `if git show-ref --verify --quiet "refs/remotes/origin/$BASE_BRANCH"; then git pull --ff-only origin "$BASE_BRANCH"; fi`
|
||||
- If the pull fails for any reason, stop and ask.
|
||||
4. Read `SPEC.md` + `README.md` once per session. Re-read only if changed/unsure:
|
||||
- `git log -1 --oneline -- SPEC.md README.md`
|
||||
5. If deviating from `SPEC.md`: **stop and ask**.
|
||||
6. Create a new local work branch (once per session) **from BASE_BRANCH**. **ALWAYS push to the branch you created.**
|
||||
- `git switch -c "agent/<topic>-YYYYMMDD-<shortid>"`
|
||||
|
||||
## Tests
|
||||
- Run tests only if the prompt requests or `SPEC.md` requires them; run all requested tests.
|
||||
- If any cannot be run: **stop and ask**.
|
||||
|
||||
## After work
|
||||
1. Make sure you did not miss requested tests.
|
||||
2. If you are addressing a **remote github issue** with
|
||||
- "to-do" checklist: for **each** item, make sure your implementation is **fully complete** and:
|
||||
- if yes, check it off
|
||||
- if not, finish implementation and recheck.
|
||||
- "acceptance" checklist: for **each** item, make the acceptance criterion is **completely fulfilled** and:
|
||||
- if yes, check it off
|
||||
- if not, implement missing code and recheck.
|
||||
3. Update all relevant documentation appropriately.
|
||||
4. **Before making a commit**: Stop and ask user to `/review` → `Review uncommited changes`.
|
||||
5. Commit **only after** user confirms code review passed.
|
||||
6. Open a PR or use already opened PR
|
||||
- **use your created branch**.
|
||||
- **Target `BASE_BRANCH`**.
|
||||
- PR text requirements:
|
||||
- commands in backticks
|
||||
- real newlines (ANSI-C quoting/heredoc)
|
||||
- if addressing a **remote github issue**, reference it so that github can close automatically after merging.
|
||||
7. **Only after** PR is merged:
|
||||
- switch back to base branch and sync it
|
||||
- `git switch "$BASE_BRANCH"`
|
||||
- `git fetch origin --prune`
|
||||
- `if git show-ref --verify --quiet "refs/remotes/origin/$BASE_BRANCH"; then git reset --hard "origin/$BASE_BRANCH"; fi`
|
||||
- If the reset fails, **stop and ask**.
|
||||
- delete the local merged work branch
|
||||
|
||||
---
|
||||
|
||||
## Required completion checklist (final response)
|
||||
- [ ] Base/target branch used correctly (BASE_BRANCH)
|
||||
- [ ] **all** Coding rules followed
|
||||
- [ ] All requested tests run (commands + results), or blocked → asked
|
||||
- [ ] Documentation updated appropriately: **list updated files**
|
||||
- [ ] PR opened (summary + command log) targeting BASE_BRANCH
|
||||
- [ ] After merge (if applicable): base branch synced; merged branch deleted
|
||||
---
|
||||
@@ -0,0 +1,125 @@
|
||||
# AI Tasks (auto-generated)
|
||||
|
||||
_Last updated: 2026-05-21T04:53:17.650Z_
|
||||
|
||||
6 open task(s):
|
||||
|
||||
---
|
||||
|
||||
## #23: [AI] OML-460: Silverbullet frontend
|
||||
|
||||
- **State:** open
|
||||
- **Created:** 2026-05-15T19:59:08Z
|
||||
- **URL:** https://github.com/DoctoDre/OrgMyLife/issues/23
|
||||
|
||||
### Description
|
||||
|
||||
**Task ID:** OML-460
|
||||
**Priority:** 2
|
||||
**State:** open
|
||||
**Due:** 2026-05-17T00:00:00+00:00
|
||||
## Description
|
||||
Ich brauche eine Suchfunktion, eine schnelle neue Notiz-Funktion, eine übersichtliche Darstellung der Notizen mit zeitlicher suche
|
||||
---
|
||||
_Auto-created from OrgMyLife task board._
|
||||
|
||||
---
|
||||
|
||||
## #22: [AI] OML-459: Call list entries bearbeitbar machen
|
||||
|
||||
- **State:** open
|
||||
- **Created:** 2026-05-15T18:01:57Z
|
||||
- **URL:** https://github.com/DoctoDre/OrgMyLife/issues/22
|
||||
|
||||
### Description
|
||||
|
||||
**Task ID:** OML-459
|
||||
**Priority:** 4
|
||||
**State:** open
|
||||
**Due:** 2026-05-31T00:00:00+00:00
|
||||
## Description
|
||||
Calls nachträglich mit Informationen anreichern wäre sehr praktisch.
|
||||
---
|
||||
_Auto-created from OrgMyLife task board._
|
||||
|
||||
---
|
||||
|
||||
## #21: [AI] OML-458: Tasks als Basis für notes ermöglichen
|
||||
|
||||
- **State:** open
|
||||
- **Created:** 2026-05-15T18:01:57Z
|
||||
- **URL:** https://github.com/DoctoDre/OrgMyLife/issues/21
|
||||
|
||||
### Description
|
||||
|
||||
**Task ID:** OML-458
|
||||
**Priority:** 3
|
||||
**State:** open
|
||||
**Due:** 2026-05-18T00:00:00+00:00
|
||||
## Description
|
||||
Neuer knopf im Taskboard an notes senden. Möglichkeit finden Duplikate zu vermeiden.
|
||||
---
|
||||
_Auto-created from OrgMyLife task board._
|
||||
|
||||
---
|
||||
|
||||
## #20: [AI] OML-457: Übertrage das aktuelle lernen aus den Agenten automatisch in notegraph
|
||||
|
||||
- **State:** open
|
||||
- **Created:** 2026-05-15T18:01:56Z
|
||||
- **URL:** https://github.com/DoctoDre/OrgMyLife/issues/20
|
||||
|
||||
### Description
|
||||
|
||||
**Task ID:** OML-457
|
||||
**Priority:** 3
|
||||
**State:** open
|
||||
**Due:** 2026-05-18T00:00:00+00:00
|
||||
## Description
|
||||
Überlege, wie alle Agenten dazu enabled werden gewonnenes wissen in die notegraph zu bekommen. Die wesentlichen Specs und alles an knowledge sollte darin enthalten sein und beim Aufräumen auch dort abgelegt werden.
|
||||
---
|
||||
_Auto-created from OrgMyLife task board._
|
||||
|
||||
---
|
||||
|
||||
## #19: [AI] OML-394: Jury-Auswertungstabelle und Bewertungsbogen
|
||||
|
||||
- **State:** open
|
||||
- **Created:** 2026-05-14T09:14:38Z
|
||||
- **URL:** https://github.com/DoctoDre/OrgMyLife/issues/19
|
||||
|
||||
### Description
|
||||
|
||||
**Task ID:** OML-394
|
||||
**Priority:** 4
|
||||
**State:** open
|
||||
## Description
|
||||
Lieber André,
|
||||
|
||||
wie besprochen Bewertungsbogen und Auswertungstabelle für die Jurysitzung.
|
||||
|
||||
Die Jurynamen und Top11 sind bereits eingetragen.
|
||||
|
||||
Melde dich jederzeit, wenn du noch etwas brauchst. Wäre cool, wenn du eine digitale Lösung schaffst. Sonst sind wir in diesem Jahr nochmal per Stift unt
|
||||
---
|
||||
_Auto-created from OrgMyLife task board._
|
||||
|
||||
---
|
||||
|
||||
## #18: [AI] OML-423: Supabase von kiq project entfernen
|
||||
|
||||
- **State:** open
|
||||
- **Created:** 2026-05-14T09:14:38Z
|
||||
- **URL:** https://github.com/DoctoDre/OrgMyLife/issues/18
|
||||
|
||||
### Description
|
||||
|
||||
**Task ID:** OML-423
|
||||
**Priority:** 2
|
||||
**State:** open
|
||||
**Due:** 2026-05-17T00:00:00+00:00
|
||||
## Description
|
||||
_No description provided._
|
||||
---
|
||||
_Auto-created from OrgMyLife task board._
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
# ARCHITECTURE.md
|
||||
|
||||
## System Architecture
|
||||
|
||||
**Product:** OrgMyLife
|
||||
**Goal:** A self-sufficient personal coordination system aggregating life domains (calendars, emails, notes, tasks) with a tidy UI, prioritizing local control over external dependencies.
|
||||
|
||||
## Technology Stack
|
||||
- **Backend:** Python (e.g., FastAPI) - Chosen for self-sufficiency and ease of future AI/LLM integration.
|
||||
- **Database:** PostgreSQL (self-hosted/local).
|
||||
- **Frontend:** Standard web technologies (HTML/CSS/JS) - Focus on a tidy, clean, functional structure before implementing fancy designs.
|
||||
|
||||
## Integration Strategy
|
||||
Data aggregation will be implemented in the following order:
|
||||
1. **Nextcloud:** Initial proof of concept for calendar and notes aggregation.
|
||||
2. **Gmail:** Email aggregation and initial task identification.
|
||||
3. **Outlook:** Handled last due to specialized security and enterprise protection requirements.
|
||||
@@ -0,0 +1,27 @@
|
||||
| ToDo | Details | Ansprechpartner | Quelle | Deadline | Priorität | Umfang |
|
||||
| :--- | :--- | :--- | :--- | :--- | :--- | :--- |
|
||||
| Bewerbung OFK2 DB Regio | | | signal | 05.05.26 | hoch | 1h |
|
||||
| Angebot RWK | Thomas Fabich | Zettel/Gespräch | | 03.05.26 | hoch | |
|
||||
| CV Rotari | Josef | Email | | 03.05.26 | hoch | |
|
||||
| Rechnung Uni Kassel | Manu | | | 03.05.26 | hoch | |
|
||||
| Rechnung Schlachthof | | | | 03.05.26 | Mittel | |
|
||||
| Termine nächste Wochen klären | | | | 03.05.26 | hoch | |
|
||||
| Folien Digitalisierungsforum | | | | 03.05.26 | sehr hoch | |
|
||||
| Projekte UKS Anlegen | | | | zeitnah | | |
|
||||
| BCIS Anfragen Sales | | | | naechste Woche | | |
|
||||
| Mit Andrea weiteres Vorgehen klären | Andrea Schlachthof | | | naechste Woche | | |
|
||||
| Steuerberater bezahlen | | | | asap | | |
|
||||
| Podcast aufnehmen | Lasse | | | 03.05.26 | terminiert | |
|
||||
| KIQ Pitchdeck aufbauen | | | | asap | | |
|
||||
| KIQ Termin mit Deck 4 machen | | | | asap | | |
|
||||
| dHive Homepage | | | | asap | | |
|
||||
| Termin mit Alex über Zukunft machen | | | | Mai 26 | | |
|
||||
| Dhive Zukunft durchdenken | | | | asap | | |
|
||||
| Hautscreening Termin machen | | | | 2026 | | |
|
||||
| Augenarzt Termin machen | | | | Juni/Juli | | |
|
||||
| Vitamin D testen lassen | | | | Juni/Juli | | |
|
||||
| Orga Setup klären, am besten Haupteingangskanal Email | eigene Email adresse? | | | asap | | |
|
||||
| Zahnreinigung vereinbaren | | | | 2026 | | |
|
||||
| KanBan Board für Thanasis aufbauen | | | | Mai 26 | | |
|
||||
| Habil Plan machen | | | | Sommer 26 | | |
|
||||
| Buch Plan machen (KI, New Work (#Einfachbahn), Science Fiction) | | | | Sommer 26 | | |
|
||||
@@ -0,0 +1,16 @@
|
||||
# Component: OrgMyLife
|
||||
|
||||
## Description
|
||||
Personal task, call, and digest coordination system (FastAPI + PostgreSQL)
|
||||
|
||||
## Metadata
|
||||
- **Deployment Target:** vps-docker
|
||||
- **Upstream URL:** https://github.com/DoctoDre/OrgMyLife
|
||||
- **Status:** active
|
||||
|
||||
## Interconnections
|
||||
- (to be documented)
|
||||
|
||||
## Notes
|
||||
- Part of shared context in the Orchestrator monorepo
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
# COMPONENTS.md
|
||||
|
||||
## Project Components
|
||||
*This document tracks all specific components used in the project.*
|
||||
|
||||
> **Rule:** Components are only added to this list once they have been actively included and utilized in the codebase.
|
||||
|
||||
### Backend Core
|
||||
- **FastAPI**: Main web framework for the API backend.
|
||||
- **Uvicorn**: ASGI web server implementation.
|
||||
- **SQLAlchemy**: SQL toolkit and Object-Relational Mapper (ORM).
|
||||
- **Alembic**: Database migration tool for SQLAlchemy.
|
||||
- **Psycopg2**: PostgreSQL database adapter for Python.
|
||||
- **caldav**: Library for CalDAV client functionality (Nextcloud sync).
|
||||
- **python-multipart**: Form data parsing for login page.
|
||||
- **python-dotenv**: Environment variable loading from .env files.
|
||||
|
||||
### Infrastructure
|
||||
- **Docker**: Containerization for app and database.
|
||||
- **PostgreSQL 16**: Primary database (Docker container with persistent volume).
|
||||
- **Caddy**: Reverse proxy with automatic HTTPS (Let's Encrypt).
|
||||
|
||||
### Frontend
|
||||
- **Vanilla HTML/CSS/JS**: No framework, clean and lightweight.
|
||||
@@ -0,0 +1,180 @@
|
||||
# Deployment Guide — OrgMyLife on a VPS
|
||||
|
||||
## Architecture (Multi-Machine)
|
||||
|
||||
```
|
||||
┌─────────────────┐ ┌──────────────────────────────┐
|
||||
│ Your Laptop │ │ VPS (e.g., IONOS) │
|
||||
│ ───────────── │ │ ────────────────── │
|
||||
│ AI-Orchestrator│────▶│ OrgMyLife (Docker) │
|
||||
│ (polls API) │ │ PostgreSQL (Docker) │
|
||||
│ │ │ https://api.knie.email │
|
||||
└─────────────────┘ └──────────────────────────────┘
|
||||
│ │
|
||||
│ │ syncs with
|
||||
▼ ▼
|
||||
┌─────────────────┐ ┌──────────────────────────────┐
|
||||
│ Other machines │ │ Nextcloud (existing) │
|
||||
│ (API clients) │ │ Calendar + ToDo persistence │
|
||||
└─────────────────┘ └──────────────────────────────┘
|
||||
```
|
||||
|
||||
## Option 1: IONOS VPS (Recommended — 3-5€/month)
|
||||
|
||||
### 1. Get a VPS
|
||||
|
||||
- Go to IONOS Cloud or VPS section
|
||||
- Pick the smallest Linux VPS (1 vCPU, 1GB RAM is enough)
|
||||
- Choose Ubuntu 22.04 or Debian 12
|
||||
- Note the IP address
|
||||
|
||||
### 2. Point your domain
|
||||
|
||||
Since you own `andreknie.de`, add a DNS record:
|
||||
- `A` record: `api.andreknie.de` → your VPS IP (`217.160.174.2`)
|
||||
|
||||
### 3. SSH into the VPS and install Docker
|
||||
|
||||
```bash
|
||||
ssh root@YOUR_VPS_IP
|
||||
|
||||
# Install Docker
|
||||
curl -fsSL https://get.docker.com | sh
|
||||
|
||||
# Install docker-compose
|
||||
apt install docker-compose-plugin
|
||||
```
|
||||
|
||||
### 4. Deploy OrgMyLife
|
||||
|
||||
```bash
|
||||
# Clone your repo
|
||||
git clone https://github.com/DoctoDre/OrgMyLife.git
|
||||
cd OrgMyLife
|
||||
|
||||
# Create .env with your real credentials
|
||||
cp .env.example .env
|
||||
nano .env
|
||||
# Fill in:
|
||||
# POSTGRES_PASSWORD=<strong random password>
|
||||
# API_SECRET=<strong random token for orchestrator auth>
|
||||
# NEXTCLOUD_URL, NEXTCLOUD_USERNAME, NEXTCLOUD_PASSWORD
|
||||
# IMAP_SERVER, EMAIL_USERNAME, EMAIL_PASSWORD
|
||||
|
||||
# Start everything
|
||||
docker compose up -d
|
||||
|
||||
# Check it's running
|
||||
docker compose logs -f app
|
||||
```
|
||||
|
||||
### 5. Add HTTPS with Caddy (reverse proxy)
|
||||
|
||||
```bash
|
||||
# Install Caddy
|
||||
apt install -y debian-keyring debian-archive-keyring apt-transport-https
|
||||
curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg
|
||||
curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | tee /etc/apt/sources.list.d/caddy-stable.list
|
||||
apt update && apt install caddy
|
||||
```
|
||||
|
||||
Create `/etc/caddy/Caddyfile`:
|
||||
```
|
||||
api.andreknie.de {
|
||||
reverse_proxy localhost:8000
|
||||
}
|
||||
```
|
||||
|
||||
```bash
|
||||
systemctl restart caddy
|
||||
```
|
||||
|
||||
Caddy automatically gets a Let's Encrypt certificate. Your API is now at `https://api.andreknie.de`.
|
||||
|
||||
### 6. Configure AI-Orchestrator on your machines
|
||||
|
||||
Update `WORKFLOW.md` on any machine:
|
||||
```yaml
|
||||
tracker:
|
||||
kind: orgmylife
|
||||
endpoint: https://api.andreknie.de
|
||||
api_key: $ORGMYLIFE_API_SECRET
|
||||
```
|
||||
|
||||
Set the env var:
|
||||
```bash
|
||||
export ORGMYLIFE_API_SECRET=<same token you put in .env on the VPS>
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Option 2: Run Locally with Docker (No Server Needed)
|
||||
|
||||
If you just want multi-machine via your home network or Tailscale:
|
||||
|
||||
```bash
|
||||
cd OrgMyLife
|
||||
|
||||
# Add to your .env:
|
||||
# POSTGRES_PASSWORD=localdev
|
||||
# API_SECRET= (empty = no auth, fine for local)
|
||||
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
Access from other machines on your network at `http://YOUR_IP:8000`.
|
||||
|
||||
For access outside your network without a VPS, use **Tailscale** (free):
|
||||
- Install Tailscale on all your machines
|
||||
- Access OrgMyLife at `http://your-tailscale-ip:8000`
|
||||
|
||||
---
|
||||
|
||||
## Option 3: IONOS Managed Domain Only (No VPS)
|
||||
|
||||
If you really don't want a VPS yet, you can use a free tier service:
|
||||
|
||||
- **Railway.app** — free tier, deploy from GitHub, auto-HTTPS
|
||||
- **Render.com** — free tier with PostgreSQL
|
||||
- **Fly.io** — free tier, deploy Docker containers
|
||||
|
||||
These all support Docker and can connect to your `knie.email` domain via CNAME records.
|
||||
|
||||
---
|
||||
|
||||
## Nextcloud as Persistence Layer
|
||||
|
||||
Regardless of where OrgMyLife runs, Nextcloud stays the durable backend:
|
||||
|
||||
1. OrgMyLife syncs tasks FROM Nextcloud (import)
|
||||
2. OrgMyLife syncs changes BACK to Nextcloud (two-way)
|
||||
3. If OrgMyLife goes down, your tasks are still in Nextcloud
|
||||
4. If you rebuild OrgMyLife, just re-sync from Nextcloud
|
||||
|
||||
This means Nextcloud is your backup and your mobile access (Nextcloud Tasks app on phone).
|
||||
|
||||
---
|
||||
|
||||
## Periodic Sync (Cron)
|
||||
|
||||
Add automatic syncing so OrgMyLife stays fresh:
|
||||
|
||||
```bash
|
||||
# Add to crontab on the VPS (or as a Docker healthcheck)
|
||||
# Sync every 5 minutes
|
||||
*/5 * * * * curl -s -X POST http://localhost:8000/api/sync/nextcloud-todo
|
||||
*/5 * * * * curl -s -X POST http://localhost:8000/api/sync/email
|
||||
```
|
||||
|
||||
Or add a background scheduler inside the app (future enhancement).
|
||||
|
||||
---
|
||||
|
||||
## Security Checklist
|
||||
|
||||
- [ ] Strong `POSTGRES_PASSWORD` (use `openssl rand -hex 32`)
|
||||
- [ ] Strong `API_SECRET` (use `openssl rand -hex 32`)
|
||||
- [ ] HTTPS enabled (Caddy handles this automatically)
|
||||
- [ ] Firewall: only ports 80, 443, 22 open on VPS
|
||||
- [ ] Don't expose PostgreSQL port (5432) to the internet
|
||||
- [ ] Keep `.env` out of git (already in .gitignore)
|
||||
@@ -0,0 +1,3 @@
|
||||
# Done Issues
|
||||
|
||||
_Empty — no issues to process._
|
||||
@@ -0,0 +1,31 @@
|
||||
FROM python:3.11-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install system dependencies for psycopg2 and lxml
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
gcc \
|
||||
libpq-dev \
|
||||
libxml2-dev \
|
||||
libxslt-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install Python dependencies
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# Copy application code
|
||||
COPY . .
|
||||
|
||||
# Create data directory for SQLite (if used)
|
||||
RUN mkdir -p /data
|
||||
|
||||
# Expose the API port
|
||||
EXPOSE 8000
|
||||
|
||||
# Health check
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s \
|
||||
CMD python -c "import httpx; r = httpx.get('http://localhost:8000/api/health'); assert r.status_code == 200"
|
||||
|
||||
# Run with uvicorn
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
@@ -0,0 +1,31 @@
|
||||
# GitLab Migration Plan & Status
|
||||
|
||||
## What has been done
|
||||
- The scheduled GitHub workflows have been disabled (`Collect AI Tasks`, `Sync Issues to File`, `Backup Dismissed Tasks`) because of GitHub billing limits that were causing pipeline failures and costing money.
|
||||
- This repository is being prepared for migration to GitLab.
|
||||
|
||||
## What needs to be done
|
||||
In order to get the CI/CD and automation processes running again on GitLab, the following tasks must be completed:
|
||||
|
||||
1. **Import the repository to GitLab**:
|
||||
- Migrate the source code.
|
||||
- Migrate GitHub Issues, labels, and Projects.
|
||||
|
||||
2. **Translate GitHub Actions to GitLab CI/CD (`.gitlab-ci.yml`)**:
|
||||
- `Backup Dismissed Tasks`: Implement as a GitLab Scheduled Pipeline running every 6 hours.
|
||||
- `Mark Issues for Review`: Migrate to a standard GitLab CI pipeline triggered by pushes changing `DONE_ISSUES.md`.
|
||||
- `Collect AI Tasks`: Implement as a GitLab Scheduled Pipeline running every 30 minutes.
|
||||
- `Update Task State on Issue Events`: Create a GitLab webhook responding to issue events, or use GitLab CI pipelines with issue event triggers.
|
||||
- `Deploy to VPS`: Migrate to a GitLab CI pipeline triggered on branch pushes to `main`.
|
||||
- `Sync Issues to File`: Migrate to a GitLab Scheduled Pipeline (every 30 minutes) and/or use GitLab Webhooks for issue event triggers.
|
||||
- `Sync OrgMyLife Plan to Projects Board`: Migrate to a GitLab CI pipeline triggered by pushes to `PLAN.md`.
|
||||
|
||||
3. **Configure CI/CD Variables**:
|
||||
- Transfer any repository secrets or environment variables (such as deployment keys, SSH credentials, API tokens) into GitLab CI/CD Settings under Variables.
|
||||
|
||||
4. **Runner Configuration**:
|
||||
- Ensure the GitLab Runners have the necessary capabilities (e.g., Node.js environments) by utilizing standard Docker images in the `.gitlab-ci.yml` (e.g., `image: node:20`).
|
||||
|
||||
5. **Optimize Pipeline Compute Efficiency**:
|
||||
- Analyze and optimize the pipeline execution to drastically reduce compute time.
|
||||
- **Context:** The previous runner consumed ~2000 minutes of compute time in just 10 days, which is highly inefficient for a small application. Strategies like reducing cron frequencies, utilizing caching, and conditionally running jobs should be implemented.
|
||||
@@ -0,0 +1,51 @@
|
||||
# PLAN.md
|
||||
|
||||
## Current Phase: Live & Operational
|
||||
|
||||
### Completed ✅
|
||||
- [x] Python backend (FastAPI) + PostgreSQL (Docker)
|
||||
- [x] Frontend (HTML/CSS/JS) with Dashboard, Tasks, Digest, Retro, Projects tabs
|
||||
- [x] Nextcloud Calendar sync (CalDAV, Berlin timezone)
|
||||
- [x] Nextcloud ToDo sync (two-way, own calendars only, respects local status)
|
||||
- [x] dHive Email/IMAP sync (IONOS, archive to Archiv/2026, delete to Papierkorb)
|
||||
- [x] Gmail sync (App Password, archive/delete both directions)
|
||||
- [x] Auto-sync every 5 minutes (background scheduler)
|
||||
- [x] Auto-dismiss emails archived/deleted externally
|
||||
- [x] Session-based login (7-day, DB-persistent, survives deploys)
|
||||
- [x] Task sorting (priority, due date, created, manual)
|
||||
- [x] Task filtering (by origin, by agent_ready)
|
||||
- [x] Delete (✕ dismiss/archive), Done (✓ complete/archive), Purge (🗑 trash/remove)
|
||||
- [x] Move to top/bottom arrows
|
||||
- [x] Due-date editing in UI
|
||||
- [x] Email sender shown below title
|
||||
- [x] Origin labels with clickable links to source app
|
||||
- [x] Prioritization Engine (rule-based suggestions on dashboard)
|
||||
- [x] Daily Digest page (overdue, due today, top priorities, schedule)
|
||||
- [x] Weekly Retrospective page (completed vs created, backlog health)
|
||||
- [x] Projects Kanban board (6 columns, multi-repo sync)
|
||||
- [x] AI-Orchestrator with OrgMyLife tracker adapter
|
||||
- [x] GitHub Actions auto-deploy (push → VPS)
|
||||
- [x] AI task pipeline (🤖 → GitHub Issue → implement → review → close)
|
||||
- [x] Dismissed tasks backup (survives VPS resets)
|
||||
- [x] Multi-repo project sync (PLAN.md → Projects board)
|
||||
- [x] Confluence Bot standalone repo
|
||||
- [x] Mobile-responsive layout
|
||||
- [x] Input validation and security fixes
|
||||
- [x] Implement Call List feature (📞 tab, phone number extraction, daily digest integration)
|
||||
|
||||
### Short-Term Goals (Next Session)
|
||||
- [ ] Fix Git Action sync error (race condition on DONE_ISSUES.md)
|
||||
- [ ] Verify Gmail archive works end-to-end after latest fix
|
||||
- [ ] Add DB GitLab (project-audit) sync via GitLab CI
|
||||
|
||||
### Medium-Term Goals
|
||||
- [ ] Add conflict resolution for two-way sync
|
||||
- [ ] Add test coverage (sync services, API endpoints)
|
||||
- [ ] Implement task labels/tags system
|
||||
- [ ] Add Prometheus metrics / structured JSON logging
|
||||
|
||||
### Long-Term Goals
|
||||
- [ ] Integrate Outlook (enterprise security constraints)
|
||||
- [ ] Implement Knowledge Graph / Notes Organization
|
||||
- [ ] Develop Availability Sharing for external booking
|
||||
- [ ] Support multi-user roles (family, colleagues)
|
||||
@@ -0,0 +1,24 @@
|
||||
# OrgMyLife
|
||||
|
||||
Personal task, call, and digest coordination system.
|
||||
|
||||
## Stack
|
||||
|
||||
- Backend: FastAPI + PostgreSQL
|
||||
- Deployment: VPS Docker (port 8000)
|
||||
- MCP: Local stdio server for Kiro integration
|
||||
|
||||
## API
|
||||
|
||||
- Endpoint: https://api.andreknie.de
|
||||
- Tasks: GET /api/tasks, POST /api/tasks
|
||||
- Projects: GET/PUT /api/projects, POST /api/projects/create
|
||||
- Agent filter: ?filter_agent=true
|
||||
|
||||
## Development
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
uvicorn main:app --reload
|
||||
```
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
# RULES.md
|
||||
|
||||
## Foundational Rules
|
||||
1. **No documents outside the folder:** All project-related documents and files must reside strictly within this repository folder.
|
||||
2. **Keep documentation concise:** Documentation should be brief, clear, and to the point.
|
||||
3. **Single source of truth:** Use single, dedicated documents for specific types of information to avoid fragmentation and redundancy.
|
||||
- Use `ARCHITECTURE.md` for all architectural information.
|
||||
- Use `PLAN.md` for ongoing steps, which must be revised after each PR.
|
||||
- Use `COMPONENTS.md` to track all specific components used in the project.
|
||||
4. **Document Components After Inclusion:** Only add components to `COMPONENTS.md` once you have actually included them in the code.
|
||||
@@ -0,0 +1,355 @@
|
||||
# SPEC.md
|
||||
|
||||
## 1. Executive Summary
|
||||
|
||||
**Product Name:** OrgMyLife
|
||||
**Stakeholder Role:** Developer and primary user
|
||||
**Product Type:** New product
|
||||
|
||||
**Goal (one sentence):**
|
||||
Create a personal coordination system that aggregates life domains, proposes prioritized actions, and reduces mental load while increasing quality of life.
|
||||
|
||||
**Business Context:**
|
||||
The stakeholder manages multiple professional roles (DB, dHive), personal life, and commitments across fragmented tools (calendars, emails, notes, task systems), leading to coordination overhead and missed prioritization.
|
||||
|
||||
**Primary Target Users:**
|
||||
|
||||
* Initially: single primary user (self)
|
||||
* Later: family members and dHive colleagues
|
||||
|
||||
**Core Functional Scope Summary:**
|
||||
The system consolidates calendars, emails, notes, and task sources, identifies actionable items, proposes prioritization, and interacts with the user through daily digests and weekly retrospectives. It supports user-confirmed execution of actions such as scheduling, task creation, and restructuring of information.
|
||||
|
||||
**In Scope:**
|
||||
|
||||
* Aggregation of multiple calendars, email systems, and note sources
|
||||
* Central task identification and management
|
||||
* Suggestion-driven prioritization
|
||||
* Daily and weekly guided interactions
|
||||
* User-confirmed execution of planning actions
|
||||
* Visibility into key planning data (tasks, deadlines, conflicts, priorities)
|
||||
|
||||
**Out of Scope:**
|
||||
|
||||
* Fully autonomous decision-making without user confirmation
|
||||
* Technical implementation details
|
||||
* Final definition of tool-specific integration rules (to be defined later)
|
||||
|
||||
**Main Open Questions:**
|
||||
|
||||
* Role-specific differences for future users (family, colleagues)
|
||||
* Exact definition of task lifecycle and statuses
|
||||
* Level of automation for time tracking analysis
|
||||
* Boundaries of knowledge graph usage and structure
|
||||
* Handling of sensitive/private calendar data across contexts
|
||||
|
||||
**Main Acceptance Indicators:**
|
||||
|
||||
* Reduced perceived mental load
|
||||
* Increased feeling of control
|
||||
* Regular engagement with daily and weekly flows
|
||||
* Fewer forgotten tasks and missed priorities
|
||||
* Measurable increase in time for sport and personal life
|
||||
|
||||
---
|
||||
|
||||
## 2. Goal
|
||||
|
||||
**One-sentence goal:**
|
||||
Reduce coordination overhead by centralizing, structuring, and actively guiding personal and professional planning.
|
||||
|
||||
**Detailed goal:**
|
||||
OrgMyLife aims to unify fragmented information sources (calendars, emails, notes, task systems) into a single coordination layer that not only aggregates data but actively assists in prioritization and execution. The system should act as a proactive assistant that continuously surfaces relevant actions, identifies conflicts, and guides the user through structured planning and reflection routines.
|
||||
|
||||
**Business value:**
|
||||
|
||||
* Increased productivity through better prioritization
|
||||
* Reduced manual coordination effort
|
||||
* Improved work-life balance
|
||||
* Better utilization of time and energy
|
||||
|
||||
**Intended outcome:**
|
||||
Users experience clarity, control, and reduced cognitive load while maintaining or increasing output quality and personal well-being.
|
||||
|
||||
---
|
||||
|
||||
## 3. Stakeholder Context
|
||||
|
||||
* **Respondent role:** Developer and primary user
|
||||
* **Perspective:** Individual productivity and life management
|
||||
* **Context:** Multiple jobs (DB, dHive), family life, sports, volunteering
|
||||
* **Scope:** This specification reflects a single stakeholder interview
|
||||
|
||||
---
|
||||
|
||||
## 4. Target Users
|
||||
|
||||
### Primary User
|
||||
|
||||
* Individual managing multiple life domains
|
||||
* Needs centralized visibility and guidance
|
||||
* Experiences fragmentation and overload
|
||||
|
||||
### Future Users
|
||||
|
||||
* Family members
|
||||
* dHive colleagues
|
||||
|
||||
### Needs
|
||||
|
||||
* Clear prioritization
|
||||
* Reliable overview of commitments
|
||||
* Reduced manual coordination
|
||||
* Guided planning routines
|
||||
|
||||
### Pain Points
|
||||
|
||||
* Fragmented tools and data sources
|
||||
* Missing synchronization
|
||||
* Forgotten tasks
|
||||
* Lack of centralized task management
|
||||
* No structured prioritization process
|
||||
* Insufficient planning for personal time (e.g., sports)
|
||||
|
||||
### Differences
|
||||
|
||||
* Primary user: full control and configuration
|
||||
* Future users: likely limited visibility and permissions (to be defined)
|
||||
|
||||
---
|
||||
|
||||
## 5. Current Situation / Current Process
|
||||
|
||||
### Current Setup
|
||||
|
||||
* Multiple calendars: Outlook (DB), Google (private/family), Nextcloud (dHive)
|
||||
* Multiple email systems: Outlook 365, Gmail, Nextcloud, GMX
|
||||
* Notes across various systems (including Confluence, Jira, loose notes)
|
||||
* Mattermost used without structured task management
|
||||
* Separate time tracking application
|
||||
|
||||
### Workarounds
|
||||
|
||||
* Manual tracking and coordination
|
||||
* Ad-hoc prioritization
|
||||
* Informal note-taking
|
||||
|
||||
### Main Gaps
|
||||
|
||||
* Incomplete or broken synchronization
|
||||
* Lack of central task list
|
||||
* Missing buffer times
|
||||
* No systematic prioritization
|
||||
* No analysis of time tracking data
|
||||
* High manual effort
|
||||
|
||||
### Failure Points
|
||||
|
||||
* Tasks forgotten
|
||||
* Deadlines missed or recognized too late
|
||||
* Conflicting or invisible calendar entries
|
||||
* Under-planning of personal priorities (e.g., sports)
|
||||
|
||||
---
|
||||
|
||||
## 6. Functional Scope
|
||||
|
||||
### Core Capabilities
|
||||
|
||||
1. **Aggregation Layer**
|
||||
|
||||
* Combine multiple calendars, email systems, and note sources
|
||||
* Provide unified visibility
|
||||
|
||||
2. **Task Identification**
|
||||
|
||||
* Extract tasks from emails and notes
|
||||
* Consolidate into a central task system
|
||||
|
||||
3. **Prioritization Support**
|
||||
|
||||
* Suggest prioritization based on:
|
||||
|
||||
* urgency
|
||||
* importance hierarchy (family > DB/dHive critical > customer potential > personal interest)
|
||||
* energy level
|
||||
|
||||
4. **Planning Assistance**
|
||||
|
||||
* Suggest scheduling (e.g., sport, tasks, buffer times)
|
||||
* Detect conflicts and gaps
|
||||
|
||||
5. **Interactive Guidance**
|
||||
|
||||
* Daily digest:
|
||||
|
||||
* overview of the day
|
||||
* new important items
|
||||
* Weekly retrospective:
|
||||
|
||||
* structured reflection
|
||||
* prioritization review
|
||||
|
||||
6. **Action Execution (User-confirmed)**
|
||||
|
||||
* Create/modify calendar events
|
||||
* Create and manage tasks
|
||||
* Block time slots
|
||||
* Send invitations
|
||||
* Organize notes and maintain knowledge graph
|
||||
|
||||
7. **Availability Sharing**
|
||||
|
||||
* Provide selected free time slots for external booking
|
||||
|
||||
8. **Transparency Layer**
|
||||
|
||||
* Show:
|
||||
|
||||
* full calendar overview
|
||||
* open tasks
|
||||
* top 5 priorities
|
||||
* deadlines
|
||||
* conflicts
|
||||
* origin of suggestions
|
||||
|
||||
---
|
||||
|
||||
## 7. Business Rules
|
||||
|
||||
* The system must not act autonomously without user confirmation
|
||||
* All suggested actions require explicit approval
|
||||
* Exceptions: explicitly commanded actions may be executed directly
|
||||
* All rules must be visible and modifiable by the user
|
||||
* The system should actively ask clarifying questions when ambiguity exists
|
||||
* Prioritization follows defined hierarchy but allows subjective override
|
||||
* The system must explain its suggestions (traceability of origin)
|
||||
|
||||
---
|
||||
|
||||
## 8. Business Objects / Functional Data Objects
|
||||
|
||||
* **Task**
|
||||
|
||||
* Represents actionable work item
|
||||
* Has priority, status, origin
|
||||
|
||||
* **Calendar Event**
|
||||
|
||||
* Represents scheduled commitment
|
||||
* May originate from different systems
|
||||
|
||||
* **Time Slot**
|
||||
|
||||
* Represents available or blocked time
|
||||
|
||||
* **Suggestion**
|
||||
|
||||
* System-generated proposal (e.g., reschedule, prioritize)
|
||||
* Includes explanation/source
|
||||
|
||||
* **Reminder**
|
||||
|
||||
* Unscheduled but relevant item
|
||||
|
||||
* **Note / Knowledge Object**
|
||||
|
||||
* Structured or unstructured information
|
||||
* Connected via knowledge graph
|
||||
|
||||
* **Priority Item (Top 5)**
|
||||
|
||||
* High-level focus topics
|
||||
|
||||
---
|
||||
|
||||
## 9. In Scope
|
||||
|
||||
* Multi-source aggregation
|
||||
* Central task management
|
||||
* Suggestion-based prioritization
|
||||
* Daily and weekly interaction formats
|
||||
* User-controlled execution of actions
|
||||
* Conflict detection and visibility
|
||||
* Knowledge organization support
|
||||
|
||||
---
|
||||
|
||||
## 10. Out of Scope
|
||||
|
||||
* Fully autonomous system behavior
|
||||
* Technical integration details
|
||||
* Final definition of all external system rules
|
||||
* Detailed UI design specifications
|
||||
|
||||
---
|
||||
|
||||
## 11. Requirements
|
||||
|
||||
1. The system must aggregate multiple calendars into a unified view.
|
||||
2. The system must aggregate inputs from multiple email systems.
|
||||
3. The system must identify potential tasks from incoming information.
|
||||
4. The system must provide a central task list.
|
||||
5. The system must suggest prioritization of tasks based on defined criteria.
|
||||
6. The system must allow user override of all prioritization decisions.
|
||||
7. The system must provide a daily digest of relevant items.
|
||||
8. The system must conduct a weekly retrospective in an interactive format.
|
||||
9. The system must propose scheduling actions (e.g., tasks, sports, buffers).
|
||||
10. The system must detect and highlight conflicts across calendars.
|
||||
11. The system must display key information (tasks, deadlines, priorities, conflicts).
|
||||
12. The system must explain the origin of suggestions.
|
||||
13. The system must require user confirmation before executing actions.
|
||||
14. The system must allow execution of user-approved actions (e.g., scheduling, task creation).
|
||||
15. The system must support availability sharing for external booking.
|
||||
16. The system must organize notes and maintain a knowledge structure.
|
||||
17. The system must ask clarifying questions when data is ambiguous.
|
||||
18. The system must allow rules to be viewed and modified.
|
||||
|
||||
---
|
||||
|
||||
## 12. Open Questions / Items to Clarify
|
||||
|
||||
* How should task statuses and lifecycle be defined?
|
||||
* What level of automation is expected for time tracking analysis?
|
||||
* How should different user roles (family, colleagues) interact with the system?
|
||||
* What data privacy boundaries exist between contexts (work vs private)?
|
||||
* How detailed should the knowledge graph be?
|
||||
* How should duplicate or conflicting tasks be resolved?
|
||||
|
||||
---
|
||||
|
||||
## 13. Risks and Ambiguities
|
||||
|
||||
* Ambiguity in prioritization logic (subjective vs rule-based)
|
||||
* Undefined role model for future multi-user usage
|
||||
* Potential overload from too many suggestions or questions
|
||||
* Integration limitations affecting completeness of data
|
||||
* Risk of user distrust if suggestions are not transparent or accurate
|
||||
|
||||
---
|
||||
|
||||
## 14. Acceptance Perspective
|
||||
|
||||
The product is successful if:
|
||||
|
||||
* The user feels in control of their commitments
|
||||
* The system is used daily and weekly as intended
|
||||
* Fewer tasks are forgotten or delayed
|
||||
* Planning effort is significantly reduced
|
||||
* The user gains measurable time for personal priorities (e.g., sports, family)
|
||||
* The backlog no longer creates mental stress
|
||||
|
||||
---
|
||||
|
||||
## 15. Glossary
|
||||
|
||||
| Original Term | English Explanation | Notes |
|
||||
| ----------------- | ---------------------------- | ------------------------------------------ |
|
||||
| Priorisierung | Prioritization | Based on subjective and rule-based factors |
|
||||
| Bauchgefühl | Gut feeling | Subjective decision influence |
|
||||
| Knowledge Graph | Structured knowledge network | Used for notes and relationships |
|
||||
| Retro | Retrospective | Weekly reflection format |
|
||||
| Mental Load | Cognitive burden | Key problem to reduce |
|
||||
| Backlogjonglage | Managing backlog overload | Informal term for task overload |
|
||||
| Zeiterfassungsapp | Time tracking application | Currently underutilized |
|
||||
| Mattermost | Team communication tool | Used without structured task management |
|
||||
@@ -0,0 +1,22 @@
|
||||
# STARTFRESH.md
|
||||
|
||||
## General Approach to New Projects
|
||||
|
||||
This document serves as a template and set of guiding principles for starting any new project, capturing the preferred methodology and structural rules.
|
||||
|
||||
### 1. Design & Architecture Philosophy
|
||||
- **Self-Sufficiency:** Minimize external dependencies. Prefer tech stacks that offer local control and self-hosting capabilities (e.g., Python backend, PostgreSQL database).
|
||||
- **Tidy First, Fancy Later:** Focus initially on clean structures, functional layouts, and standard technologies. Complex visuals and design polish should only be added once the foundational logic is proven.
|
||||
|
||||
### 2. Repository Rules
|
||||
- **Containment:** No documents outside the folder. All project-related documents, notes, and files must reside strictly within the repository.
|
||||
- **Conciseness:** Keep documentation brief, clear, and to the point.
|
||||
- **Single Source of Truth:** Use single, dedicated documents for specific types of information to avoid fragmentation.
|
||||
|
||||
### 3. Required Foundational Documents
|
||||
Every new project should initialize with the following document structure:
|
||||
|
||||
* **`RULES.md`**: To declare foundational repository and workflow rules.
|
||||
* **`ARCHITECTURE.md`**: To collect all architectural decisions, technology stack choices, and integration strategies.
|
||||
* **`PLAN.md`**: To track ongoing steps, categorized by short, medium, and long-term goals. *Must be revised after each PR.*
|
||||
* **`COMPONENTS.md`**: To track specific components used in the project. *Rule: Only add components to this document once they have been actively included and utilized in the codebase.*
|
||||
@@ -0,0 +1,149 @@
|
||||
# A generic, single database configuration.
|
||||
|
||||
[alembic]
|
||||
# path to migration scripts.
|
||||
# this is typically a path given in POSIX (e.g. forward slashes)
|
||||
# format, relative to the token %(here)s which refers to the location of this
|
||||
# ini file
|
||||
script_location = %(here)s/alembic
|
||||
|
||||
# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
|
||||
# Uncomment the line below if you want the files to be prepended with date and time
|
||||
# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
|
||||
# for all available tokens
|
||||
# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
|
||||
# Or organize into date-based subdirectories (requires recursive_version_locations = true)
|
||||
# file_template = %%(year)d/%%(month).2d/%%(day).2d_%%(hour).2d%%(minute).2d_%%(second).2d_%%(rev)s_%%(slug)s
|
||||
|
||||
# sys.path path, will be prepended to sys.path if present.
|
||||
# defaults to the current working directory. for multiple paths, the path separator
|
||||
# is defined by "path_separator" below.
|
||||
prepend_sys_path = .
|
||||
|
||||
|
||||
# timezone to use when rendering the date within the migration file
|
||||
# as well as the filename.
|
||||
# If specified, requires the tzdata library which can be installed by adding
|
||||
# `alembic[tz]` to the pip requirements.
|
||||
# string value is passed to ZoneInfo()
|
||||
# leave blank for localtime
|
||||
# timezone =
|
||||
|
||||
# max length of characters to apply to the "slug" field
|
||||
# truncate_slug_length = 40
|
||||
|
||||
# set to 'true' to run the environment during
|
||||
# the 'revision' command, regardless of autogenerate
|
||||
# revision_environment = false
|
||||
|
||||
# set to 'true' to allow .pyc and .pyo files without
|
||||
# a source .py file to be detected as revisions in the
|
||||
# versions/ directory
|
||||
# sourceless = false
|
||||
|
||||
# version location specification; This defaults
|
||||
# to <script_location>/versions. When using multiple version
|
||||
# directories, initial revisions must be specified with --version-path.
|
||||
# The path separator used here should be the separator specified by "path_separator"
|
||||
# below.
|
||||
# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions
|
||||
|
||||
# path_separator; This indicates what character is used to split lists of file
|
||||
# paths, including version_locations and prepend_sys_path within configparser
|
||||
# files such as alembic.ini.
|
||||
# The default rendered in new alembic.ini files is "os", which uses os.pathsep
|
||||
# to provide os-dependent path splitting.
|
||||
#
|
||||
# Note that in order to support legacy alembic.ini files, this default does NOT
|
||||
# take place if path_separator is not present in alembic.ini. If this
|
||||
# option is omitted entirely, fallback logic is as follows:
|
||||
#
|
||||
# 1. Parsing of the version_locations option falls back to using the legacy
|
||||
# "version_path_separator" key, which if absent then falls back to the legacy
|
||||
# behavior of splitting on spaces and/or commas.
|
||||
# 2. Parsing of the prepend_sys_path option falls back to the legacy
|
||||
# behavior of splitting on spaces, commas, or colons.
|
||||
#
|
||||
# Valid values for path_separator are:
|
||||
#
|
||||
# path_separator = :
|
||||
# path_separator = ;
|
||||
# path_separator = space
|
||||
# path_separator = newline
|
||||
#
|
||||
# Use os.pathsep. Default configuration used for new projects.
|
||||
path_separator = os
|
||||
|
||||
# set to 'true' to search source files recursively
|
||||
# in each "version_locations" directory
|
||||
# new in Alembic version 1.10
|
||||
# recursive_version_locations = false
|
||||
|
||||
# the output encoding used when revision files
|
||||
# are written from script.py.mako
|
||||
# output_encoding = utf-8
|
||||
|
||||
# database URL. This is consumed by the user-maintained env.py script only.
|
||||
# other means of configuring database URLs may be customized within the env.py
|
||||
# file.
|
||||
sqlalchemy.url = driver://user:pass@localhost/dbname
|
||||
|
||||
|
||||
[post_write_hooks]
|
||||
# post_write_hooks defines scripts or Python functions that are run
|
||||
# on newly generated revision scripts. See the documentation for further
|
||||
# detail and examples
|
||||
|
||||
# format using "black" - use the console_scripts runner, against the "black" entrypoint
|
||||
# hooks = black
|
||||
# black.type = console_scripts
|
||||
# black.entrypoint = black
|
||||
# black.options = -l 79 REVISION_SCRIPT_FILENAME
|
||||
|
||||
# lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module
|
||||
# hooks = ruff
|
||||
# ruff.type = module
|
||||
# ruff.module = ruff
|
||||
# ruff.options = check --fix REVISION_SCRIPT_FILENAME
|
||||
|
||||
# Alternatively, use the exec runner to execute a binary found on your PATH
|
||||
# hooks = ruff
|
||||
# ruff.type = exec
|
||||
# ruff.executable = ruff
|
||||
# ruff.options = check --fix REVISION_SCRIPT_FILENAME
|
||||
|
||||
# Logging configuration. This is also consumed by the user-maintained
|
||||
# env.py script only.
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARNING
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARNING
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
@@ -0,0 +1 @@
|
||||
Generic single-database configuration.
|
||||
@@ -0,0 +1,83 @@
|
||||
from logging.config import fileConfig
|
||||
|
||||
from sqlalchemy import engine_from_config
|
||||
from sqlalchemy import pool
|
||||
|
||||
from alembic import context
|
||||
|
||||
# this is the Alembic Config object, which provides
|
||||
# access to the values within the .ini file in use.
|
||||
config = context.config
|
||||
|
||||
# Interpret the config file for Python logging.
|
||||
# This line sets up loggers basically.
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
# add your model's MetaData object here
|
||||
# for 'autogenerate' support
|
||||
import os
|
||||
import sys
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
|
||||
from app.models import Base
|
||||
from app.db.session import DATABASE_URL
|
||||
target_metadata = Base.metadata
|
||||
|
||||
config.set_main_option("sqlalchemy.url", DATABASE_URL)
|
||||
|
||||
# other values from the config, defined by the needs of env.py,
|
||||
# can be acquired:
|
||||
# my_important_option = config.get_main_option("my_important_option")
|
||||
# ... etc.
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
"""Run migrations in 'offline' mode.
|
||||
|
||||
This configures the context with just a URL
|
||||
and not an Engine, though an Engine is acceptable
|
||||
here as well. By skipping the Engine creation
|
||||
we don't even need a DBAPI to be available.
|
||||
|
||||
Calls to context.execute() here emit the given string to the
|
||||
script output.
|
||||
|
||||
"""
|
||||
url = config.get_main_option("sqlalchemy.url")
|
||||
context.configure(
|
||||
url=url,
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
"""Run migrations in 'online' mode.
|
||||
|
||||
In this scenario we need to create an Engine
|
||||
and associate a connection with the context.
|
||||
|
||||
"""
|
||||
connectable = engine_from_config(
|
||||
config.get_section(config.config_ini_section, {}),
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
)
|
||||
|
||||
with connectable.connect() as connection:
|
||||
context.configure(
|
||||
connection=connection, target_metadata=target_metadata
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
@@ -0,0 +1,28 @@
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)}
|
||||
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
|
||||
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
${downgrades if downgrades else "pass"}
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Add agent_ready and labels columns to tasks
|
||||
|
||||
Revision ID: 003
|
||||
Revises:
|
||||
Create Date: 2026-05-04
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
# revision identifiers
|
||||
revision = '003'
|
||||
down_revision = None
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column('tasks', sa.Column('agent_ready', sa.Boolean(), nullable=True, server_default='false'))
|
||||
op.add_column('tasks', sa.Column('labels', sa.String(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column('tasks', 'labels')
|
||||
op.drop_column('tasks', 'agent_ready')
|
||||
@@ -0,0 +1,20 @@
|
||||
"""Add source_url column to tasks
|
||||
|
||||
Revision ID: 004
|
||||
Create Date: 2026-05-06
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = '004'
|
||||
down_revision = '003'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column('tasks', sa.Column('source_url', sa.String(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column('tasks', 'source_url')
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Add call_items table
|
||||
|
||||
Revision ID: 005
|
||||
Create Date: 2026-05-10
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
# revision identifiers
|
||||
revision = '005'
|
||||
down_revision = '004'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table('call_items',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('person', sa.String(200), nullable=False),
|
||||
sa.Column('reason', sa.String(500), nullable=False),
|
||||
sa.Column('phone_number', sa.String(30), nullable=True),
|
||||
sa.Column('original_task_id', sa.Integer(), nullable=True),
|
||||
sa.Column('priority', sa.Integer(), server_default='3'),
|
||||
sa.Column('status', sa.String(), server_default="'pending'"),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)')),
|
||||
sa.ForeignKeyConstraint(['original_task_id'], ['tasks.id']),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index(op.f('ix_call_items_id'), 'call_items', ['id'], unique=False)
|
||||
op.create_index(op.f('ix_call_items_status'), 'call_items', ['status'], unique=False)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index(op.f('ix_call_items_status'), table_name='call_items')
|
||||
op.drop_index(op.f('ix_call_items_id'), table_name='call_items')
|
||||
op.drop_table('call_items')
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Add hidden_until column to tasks table
|
||||
|
||||
Revision ID: 006
|
||||
Create Date: 2026-05-13
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
# revision identifiers
|
||||
revision = '006'
|
||||
down_revision = '005'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column('tasks', sa.Column('hidden_until', sa.DateTime(timezone=True), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column('tasks', 'hidden_until')
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Add conflict resolution columns to tasks table
|
||||
|
||||
Revision ID: 007
|
||||
Create Date: 2026-05-20
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
# revision identifiers
|
||||
revision = '007'
|
||||
down_revision = '006'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column('tasks', sa.Column('last_synced_at', sa.DateTime(timezone=True), nullable=True))
|
||||
op.add_column('tasks', sa.Column('has_conflict', sa.Boolean(), server_default='0', nullable=True))
|
||||
op.add_column('tasks', sa.Column('conflict_data', sa.Text(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column('tasks', 'conflict_data')
|
||||
op.drop_column('tasks', 'has_conflict')
|
||||
op.drop_column('tasks', 'last_synced_at')
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
"""add_position_to_tasks
|
||||
|
||||
Revision ID: 0ce1353d9433
|
||||
Revises: 9d39809cb00a
|
||||
Create Date: 2026-05-03 11:45:56.407583
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = '0ce1353d9433'
|
||||
down_revision: Union[str, Sequence[str], None] = '9d39809cb00a'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.add_column('tasks', sa.Column('position', sa.Integer(), nullable=True))
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_column('tasks', 'position')
|
||||
# ### end Alembic commands ###
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
"""add_source_id_to_tasks
|
||||
|
||||
Revision ID: 28a023783522
|
||||
Revises: 0ce1353d9433
|
||||
Create Date: 2026-05-03 21:42:57.678532
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision: str = '28a023783522'
|
||||
down_revision: Union[str, Sequence[str], None] = '0ce1353d9433'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column('tasks', sa.Column('source_id', sa.String(), nullable=True))
|
||||
op.create_index(op.f('ix_tasks_source_id'), 'tasks', ['source_id'], unique=True)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index(op.f('ix_tasks_source_id'), table_name='tasks')
|
||||
op.drop_column('tasks', 'source_id')
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Initial DB
|
||||
|
||||
Revision ID: 9d39809cb00a
|
||||
Revises:
|
||||
Create Date: 2026-04-29 23:49:47.777582
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = '9d39809cb00a'
|
||||
down_revision: Union[str, Sequence[str], None] = None
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table('events',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('title', sa.String(), nullable=False),
|
||||
sa.Column('start_time', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('end_time', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('source', sa.String(), nullable=False),
|
||||
sa.Column('is_conflict', sa.Boolean(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index(op.f('ix_events_id'), 'events', ['id'], unique=False)
|
||||
op.create_index(op.f('ix_events_title'), 'events', ['title'], unique=False)
|
||||
op.create_table('suggestions',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('content', sa.String(), nullable=False),
|
||||
sa.Column('explanation', sa.Text(), nullable=True),
|
||||
sa.Column('type', sa.String(), nullable=False),
|
||||
sa.Column('status', sa.String(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index(op.f('ix_suggestions_id'), 'suggestions', ['id'], unique=False)
|
||||
op.create_table('tasks',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('title', sa.String(), nullable=False),
|
||||
sa.Column('description', sa.Text(), nullable=True),
|
||||
sa.Column('priority', sa.Integer(), nullable=True),
|
||||
sa.Column('status', sa.String(), nullable=True),
|
||||
sa.Column('origin', sa.String(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=True),
|
||||
sa.Column('due_date', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index(op.f('ix_tasks_id'), 'tasks', ['id'], unique=False)
|
||||
op.create_index(op.f('ix_tasks_title'), 'tasks', ['title'], unique=False)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_index(op.f('ix_tasks_title'), table_name='tasks')
|
||||
op.drop_index(op.f('ix_tasks_id'), table_name='tasks')
|
||||
op.drop_table('tasks')
|
||||
op.drop_index(op.f('ix_suggestions_id'), table_name='suggestions')
|
||||
op.drop_table('suggestions')
|
||||
op.drop_index(op.f('ix_events_title'), table_name='events')
|
||||
op.drop_index(op.f('ix_events_id'), table_name='events')
|
||||
op.drop_table('events')
|
||||
# ### end Alembic commands ###
|
||||
@@ -0,0 +1,25 @@
|
||||
import os
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker, declarative_base
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///./orgmylife.db")
|
||||
|
||||
connect_args = {}
|
||||
if DATABASE_URL.startswith("sqlite"):
|
||||
connect_args["check_same_thread"] = False
|
||||
|
||||
engine = create_engine(DATABASE_URL, connect_args=connect_args)
|
||||
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||
|
||||
Base = declarative_base()
|
||||
|
||||
# Dependency for FastAPI endpoints
|
||||
def get_db():
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,81 @@
|
||||
from sqlalchemy import Column, Integer, String, Boolean, DateTime, ForeignKey, Text
|
||||
from sqlalchemy.sql import func
|
||||
from app.db.session import Base
|
||||
|
||||
class Task(Base):
|
||||
__tablename__ = "tasks"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
title = Column(String, index=True, nullable=False)
|
||||
description = Column(Text, nullable=True)
|
||||
priority = Column(Integer, default=0) # e.g. 1=High, 2=Medium, 3=Low
|
||||
position = Column(Integer, default=0) # order within priority
|
||||
status = Column(String, default="open") # open, in_progress, review, completed, dismissed
|
||||
origin = Column(String, nullable=True) # e.g. "gmail", "nextcloud_todo:Personal", "manual"
|
||||
source_id = Column(String, nullable=True, index=True) # CalDAV UID or IMAP Message-ID
|
||||
source_url = Column(String, nullable=True) # Link to original item (email, task board, etc.)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
due_date = Column(DateTime(timezone=True), nullable=True)
|
||||
hidden_until = Column(DateTime(timezone=True), nullable=True)
|
||||
agent_ready = Column(Boolean, default=False) # Flag: orchestrator can pick this up
|
||||
labels = Column(String, nullable=True) # Comma-separated labels for filtering
|
||||
last_synced_at = Column(DateTime(timezone=True), nullable=True) # When last synced with remote
|
||||
has_conflict = Column(Boolean, default=False) # True if conflict detected
|
||||
conflict_data = Column(Text, nullable=True) # JSON: remote version of conflicting fields
|
||||
|
||||
class Event(Base):
|
||||
__tablename__ = "events"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
title = Column(String, index=True, nullable=False)
|
||||
start_time = Column(DateTime(timezone=True), nullable=False)
|
||||
end_time = Column(DateTime(timezone=True), nullable=False)
|
||||
source = Column(String, nullable=False) # e.g. "google", "nextcloud"
|
||||
is_conflict = Column(Boolean, default=False)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
class Suggestion(Base):
|
||||
__tablename__ = "suggestions"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
content = Column(String, nullable=False)
|
||||
explanation = Column(Text, nullable=True)
|
||||
type = Column(String, nullable=False) # e.g. "schedule_task", "resolve_conflict"
|
||||
status = Column(String, default="pending") # pending, accepted, rejected
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class UserSession(Base):
|
||||
__tablename__ = "user_sessions"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
session_id = Column(String, unique=True, index=True, nullable=False)
|
||||
username = Column(String, nullable=False)
|
||||
expires_at = Column(DateTime, nullable=False)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class ProjectTask(Base):
|
||||
__tablename__ = "project_tasks"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
title = Column(String, nullable=False)
|
||||
description = Column(Text, nullable=True)
|
||||
repo = Column(String, nullable=True) # e.g. "OrgMyLife", "AI-Orchestrator"
|
||||
status = Column(String, default="backlog") # backlog, ready, in_progress, blocked, review, done
|
||||
priority = Column(Integer, default=3)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class CallItem(Base):
|
||||
__tablename__ = "call_items"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
person = Column(String(200), nullable=False)
|
||||
reason = Column(String(500), nullable=False)
|
||||
phone_number = Column(String(30), nullable=True)
|
||||
original_task_id = Column(Integer, ForeignKey("tasks.id"), nullable=True)
|
||||
priority = Column(Integer, default=3) # 1=Very High, 2=High, 3=Medium, 4=Low
|
||||
status = Column(String, default="pending") # "pending" or "done"
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
@@ -0,0 +1,79 @@
|
||||
import os
|
||||
from app.db.session import SessionLocal
|
||||
from app.models import Task
|
||||
|
||||
def sync_backlog():
|
||||
filepath = "BACKLOG.md"
|
||||
if not os.path.exists(filepath):
|
||||
return False
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
with open(filepath, "r", encoding="utf-8") as f:
|
||||
lines = f.readlines()
|
||||
|
||||
table_lines = [l.strip() for l in lines if l.strip().startswith("|")]
|
||||
|
||||
if len(table_lines) > 2:
|
||||
data_lines = table_lines[2:] # Skip header and separator
|
||||
|
||||
for line in data_lines:
|
||||
cols = [c.strip() for c in line.split("|")]
|
||||
if len(cols) >= 8:
|
||||
todo = cols[1]
|
||||
details = cols[2]
|
||||
ansprechpartner = cols[3]
|
||||
quelle = cols[4]
|
||||
deadline = cols[5]
|
||||
prioritaet = cols[6].lower()
|
||||
umfang = cols[7]
|
||||
|
||||
if not todo:
|
||||
continue
|
||||
|
||||
priority_val = 4
|
||||
if "sehr hoch" in prioritaet:
|
||||
priority_val = 1
|
||||
elif "hoch" in prioritaet:
|
||||
priority_val = 2
|
||||
elif "mittel" in prioritaet:
|
||||
priority_val = 3
|
||||
|
||||
# Build a structured description
|
||||
description_parts = []
|
||||
if details: description_parts.append(f"Details: {details}")
|
||||
if ansprechpartner: description_parts.append(f"Ansprechpartner: {ansprechpartner}")
|
||||
if deadline: description_parts.append(f"Deadline: {deadline}")
|
||||
if umfang: description_parts.append(f"Umfang: {umfang}")
|
||||
|
||||
full_desc = "\n".join(description_parts)
|
||||
|
||||
existing_task = db.query(Task).filter(Task.title == todo).first()
|
||||
if existing_task:
|
||||
existing_task.description = full_desc
|
||||
existing_task.origin = quelle
|
||||
existing_task.priority = priority_val
|
||||
else:
|
||||
new_task = Task(
|
||||
title=todo,
|
||||
description=full_desc,
|
||||
origin=quelle,
|
||||
priority=priority_val,
|
||||
status="open"
|
||||
)
|
||||
db.add(new_task)
|
||||
|
||||
db.commit()
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"Error syncing backlog: {e}")
|
||||
db.rollback()
|
||||
return False
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
return False
|
||||
|
||||
if __name__ == "__main__":
|
||||
success = sync_backlog()
|
||||
print(f"Sync successful: {success}")
|
||||
@@ -0,0 +1,199 @@
|
||||
"""Daily Digest and Weekly Retrospective generation."""
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from zoneinfo import ZoneInfo
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy import func
|
||||
from app.models import Task, Event, CallItem
|
||||
|
||||
BERLIN_TZ = ZoneInfo("Europe/Berlin")
|
||||
|
||||
|
||||
def generate_daily_digest(db: Session) -> dict:
|
||||
"""Generate a structured daily digest.
|
||||
|
||||
Returns a dict with sections: greeting, today_events, top_tasks,
|
||||
overdue, due_today, new_since_yesterday, stats.
|
||||
"""
|
||||
now = datetime.now(BERLIN_TZ)
|
||||
today_start = datetime.combine(now.date(), datetime.min.time()).replace(tzinfo=BERLIN_TZ)
|
||||
today_end = datetime.combine(now.date(), datetime.max.time()).replace(tzinfo=BERLIN_TZ)
|
||||
yesterday = today_start - timedelta(days=1)
|
||||
|
||||
# Greeting based on time of day
|
||||
hour = now.hour
|
||||
if hour < 12:
|
||||
greeting = "Good morning"
|
||||
elif hour < 17:
|
||||
greeting = "Good afternoon"
|
||||
else:
|
||||
greeting = "Good evening"
|
||||
|
||||
# Today's events
|
||||
todays_events = db.query(Event).filter(
|
||||
Event.start_time >= today_start,
|
||||
Event.start_time <= today_end
|
||||
).order_by(Event.start_time).all()
|
||||
|
||||
# Top 5 priority tasks
|
||||
top_tasks = db.query(Task).filter(
|
||||
Task.status == "open"
|
||||
).order_by(Task.priority, Task.position).limit(5).all()
|
||||
|
||||
# Overdue tasks
|
||||
overdue = db.query(Task).filter(
|
||||
Task.status == "open",
|
||||
Task.due_date != None,
|
||||
Task.due_date < today_start
|
||||
).order_by(Task.due_date).all()
|
||||
|
||||
# Due today
|
||||
due_today = db.query(Task).filter(
|
||||
Task.status == "open",
|
||||
Task.due_date != None,
|
||||
Task.due_date >= today_start,
|
||||
Task.due_date <= today_end
|
||||
).all()
|
||||
|
||||
# New tasks since yesterday
|
||||
new_tasks = db.query(Task).filter(
|
||||
Task.status == "open",
|
||||
Task.created_at >= yesterday
|
||||
).count()
|
||||
|
||||
# Total open
|
||||
total_open = db.query(Task).filter(Task.status == "open").count()
|
||||
|
||||
# Pending calls
|
||||
pending_calls = db.query(CallItem).filter(
|
||||
CallItem.status == "pending"
|
||||
).all()
|
||||
|
||||
priority_labels = {1: "Very High", 2: "High", 3: "Medium", 4: "Low"}
|
||||
|
||||
result = {
|
||||
"greeting": greeting,
|
||||
"date": now.strftime("%A, %B %d, %Y"),
|
||||
"total_open": total_open,
|
||||
"events_today": [
|
||||
{
|
||||
"title": e.title,
|
||||
"time": f"{e.start_time.astimezone(BERLIN_TZ).strftime('%H:%M')} - {e.end_time.astimezone(BERLIN_TZ).strftime('%H:%M')}"
|
||||
}
|
||||
for e in todays_events
|
||||
],
|
||||
"top_tasks": [
|
||||
{"id": t.id, "title": t.title, "priority": priority_labels.get(t.priority, "Low")}
|
||||
for t in top_tasks
|
||||
],
|
||||
"overdue": [
|
||||
{
|
||||
"id": t.id,
|
||||
"title": t.title,
|
||||
"days_overdue": (now.date() - t.due_date.date()).days
|
||||
}
|
||||
for t in overdue
|
||||
],
|
||||
"due_today": [
|
||||
{"id": t.id, "title": t.title}
|
||||
for t in due_today
|
||||
],
|
||||
"new_since_yesterday": new_tasks,
|
||||
"stats": {
|
||||
"events_count": len(todays_events),
|
||||
"overdue_count": len(overdue),
|
||||
"due_today_count": len(due_today),
|
||||
}
|
||||
}
|
||||
|
||||
# Only include pending_calls section if there are pending calls
|
||||
if pending_calls:
|
||||
result["pending_calls"] = [
|
||||
{"person": c.person, "reason": c.reason}
|
||||
for c in pending_calls
|
||||
]
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def generate_weekly_retro(db: Session) -> dict:
|
||||
"""Generate a weekly retrospective summary.
|
||||
|
||||
Looks at the past 7 days: what was completed, what slipped,
|
||||
what's still open, and workload trends.
|
||||
"""
|
||||
now = datetime.now(BERLIN_TZ)
|
||||
week_ago = now - timedelta(days=7)
|
||||
today_start = datetime.combine(now.date(), datetime.min.time()).replace(tzinfo=BERLIN_TZ)
|
||||
|
||||
# Tasks completed this week (check for status change isn't tracked,
|
||||
# so we look at tasks with status=completed that exist)
|
||||
completed_this_week = db.query(Task).filter(
|
||||
Task.status == "completed"
|
||||
).count()
|
||||
|
||||
# Tasks still open
|
||||
total_open = db.query(Task).filter(Task.status == "open").count()
|
||||
|
||||
# Tasks in review
|
||||
in_review = db.query(Task).filter(Task.status == "review").count()
|
||||
|
||||
# Overdue tasks
|
||||
overdue = db.query(Task).filter(
|
||||
Task.status == "open",
|
||||
Task.due_date != None,
|
||||
Task.due_date < today_start
|
||||
).order_by(Task.due_date).all()
|
||||
|
||||
# Tasks created this week
|
||||
created_this_week = db.query(Task).filter(
|
||||
Task.created_at >= week_ago,
|
||||
Task.status.notin_(["dismissed"])
|
||||
).count()
|
||||
|
||||
# Tasks by priority
|
||||
priority_breakdown = {}
|
||||
for prio in [1, 2, 3, 4]:
|
||||
count = db.query(Task).filter(
|
||||
Task.status == "open", Task.priority == prio
|
||||
).count()
|
||||
priority_breakdown[prio] = count
|
||||
|
||||
# Tasks by origin
|
||||
origin_breakdown = db.query(
|
||||
Task.origin, func.count(Task.id)
|
||||
).filter(Task.status == "open").group_by(Task.origin).all()
|
||||
|
||||
# Events this week
|
||||
events_this_week = db.query(Event).filter(
|
||||
Event.start_time >= week_ago,
|
||||
Event.start_time <= now
|
||||
).count()
|
||||
|
||||
priority_labels = {1: "Very High", 2: "High", 3: "Medium", 4: "Low"}
|
||||
|
||||
return {
|
||||
"period": f"{week_ago.strftime('%b %d')} - {now.strftime('%b %d, %Y')}",
|
||||
"completed": completed_this_week,
|
||||
"created": created_this_week,
|
||||
"total_open": total_open,
|
||||
"in_review": in_review,
|
||||
"overdue": [
|
||||
{
|
||||
"id": t.id,
|
||||
"title": t.title,
|
||||
"days_overdue": (now.date() - t.due_date.date()).days
|
||||
}
|
||||
for t in overdue
|
||||
],
|
||||
"priority_breakdown": {
|
||||
priority_labels[k]: v for k, v in priority_breakdown.items()
|
||||
},
|
||||
"origin_breakdown": {
|
||||
(origin or "unknown"): count for origin, count in origin_breakdown
|
||||
},
|
||||
"events_attended": events_this_week,
|
||||
"net_change": created_this_week - completed_this_week,
|
||||
"health": "growing" if created_this_week > completed_this_week + 5 else
|
||||
"shrinking" if completed_this_week > created_this_week + 3 else "stable",
|
||||
}
|
||||
@@ -0,0 +1,339 @@
|
||||
import os
|
||||
import imaplib
|
||||
import email as email_lib
|
||||
import traceback
|
||||
from email.header import decode_header
|
||||
from app.db.session import SessionLocal
|
||||
from app.models import Task
|
||||
|
||||
# IMAP label constants
|
||||
LABEL_TODO = "todo"
|
||||
LABEL_NO_TODO = "no_todo"
|
||||
LABEL_DONE = "done"
|
||||
|
||||
|
||||
def _get_imap_connection():
|
||||
server = os.getenv("IMAP_SERVER")
|
||||
port = int(os.getenv("IMAP_PORT", "993"))
|
||||
username = os.getenv("EMAIL_USERNAME")
|
||||
password = os.getenv("EMAIL_PASSWORD")
|
||||
|
||||
if not all([server, username, password]):
|
||||
raise ValueError("IMAP credentials missing. Set IMAP_SERVER, IMAP_PORT, EMAIL_USERNAME, EMAIL_PASSWORD.")
|
||||
|
||||
conn = imaplib.IMAP4_SSL(server, port)
|
||||
conn.login(username, password)
|
||||
return conn
|
||||
|
||||
|
||||
def _decode_header_value(value) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
parts = decode_header(value)
|
||||
decoded = []
|
||||
for part, charset in parts:
|
||||
if isinstance(part, bytes):
|
||||
decoded.append(part.decode(charset or "utf-8", errors="replace"))
|
||||
else:
|
||||
decoded.append(part)
|
||||
return " ".join(decoded)
|
||||
|
||||
|
||||
def _get_body_snippet(msg, max_chars: int = 300) -> str:
|
||||
if msg.is_multipart():
|
||||
for part in msg.walk():
|
||||
if part.get_content_type() == "text/plain" and part.get("Content-Disposition") is None:
|
||||
payload = part.get_payload(decode=True)
|
||||
if payload:
|
||||
return payload.decode(part.get_content_charset() or "utf-8", errors="replace")[:max_chars]
|
||||
else:
|
||||
payload = msg.get_payload(decode=True)
|
||||
if payload:
|
||||
return payload.decode(msg.get_content_charset() or "utf-8", errors="replace")[:max_chars]
|
||||
return ""
|
||||
|
||||
|
||||
def _apply_label(conn, uid_bytes: bytes, label: str):
|
||||
try:
|
||||
conn.uid("STORE", uid_bytes, "+FLAGS", f"({label})")
|
||||
except Exception as e:
|
||||
print(f"Warning: could not apply label '{label}': {e}")
|
||||
|
||||
|
||||
def _remove_label(conn, uid_bytes: bytes, label: str):
|
||||
try:
|
||||
conn.uid("STORE", uid_bytes, "-FLAGS", f"({label})")
|
||||
except Exception as e:
|
||||
print(f"Warning: could not remove label '{label}': {e}")
|
||||
|
||||
|
||||
def _archive_email(conn, uid_bytes: bytes):
|
||||
"""Move email to archive folder. Uses IONOS year-based Archiv folders."""
|
||||
from datetime import datetime
|
||||
current_year = str(datetime.now().year)
|
||||
archive_folders = [
|
||||
f"Archiv/{current_year}", # IONOS: "Archiv/2026"
|
||||
"Archiv",
|
||||
"Archive",
|
||||
"INBOX.Archive",
|
||||
]
|
||||
for folder in archive_folders:
|
||||
try:
|
||||
result = conn.uid("COPY", uid_bytes, folder)
|
||||
if result[0] == "OK":
|
||||
conn.uid("STORE", uid_bytes, "+FLAGS", r"(\Deleted)")
|
||||
conn.expunge()
|
||||
print(f"Email archived to: {folder}")
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
print(f"Could not archive email. Tried: {archive_folders}")
|
||||
return False
|
||||
|
||||
|
||||
def _delete_email(conn, uid_bytes: bytes):
|
||||
"""Permanently delete email (move to Papierkorb/Trash)."""
|
||||
trash_folders = [
|
||||
"Papierkorb", # IONOS German
|
||||
"Trash",
|
||||
"INBOX.Trash",
|
||||
"Deleted Items",
|
||||
]
|
||||
for folder in trash_folders:
|
||||
try:
|
||||
result = conn.uid("COPY", uid_bytes, folder)
|
||||
if result[0] == "OK":
|
||||
conn.uid("STORE", uid_bytes, "+FLAGS", r"(\Deleted)")
|
||||
conn.expunge()
|
||||
print(f"Email moved to trash: {folder}")
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
# Fallback: just mark as deleted in INBOX
|
||||
try:
|
||||
conn.uid("STORE", uid_bytes, "+FLAGS", r"(\Deleted)")
|
||||
conn.expunge()
|
||||
print("Email deleted from INBOX (no trash folder found)")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"Could not delete email: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def sync_emails():
|
||||
"""Fetch untracked INBOX emails and create tasks."""
|
||||
try:
|
||||
conn = _get_imap_connection()
|
||||
except ValueError as e:
|
||||
print(f"IMAP not configured: {e}")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"IMAP connection failed: {e}")
|
||||
traceback.print_exc()
|
||||
return False
|
||||
|
||||
try:
|
||||
conn.select("INBOX")
|
||||
|
||||
# Try keyword-based search, fall back to UNSEEN
|
||||
try:
|
||||
status, data = conn.uid(
|
||||
"SEARCH", None,
|
||||
f"UNKEYWORD {LABEL_TODO} UNKEYWORD {LABEL_NO_TODO} UNKEYWORD {LABEL_DONE}"
|
||||
)
|
||||
except Exception:
|
||||
print("UNKEYWORD not supported, falling back to UNSEEN")
|
||||
status, data = conn.uid("SEARCH", None, "UNSEEN")
|
||||
|
||||
if status != "OK" or not data[0]:
|
||||
return True # Nothing to process
|
||||
|
||||
uid_list = data[0].split()
|
||||
db = SessionLocal()
|
||||
|
||||
for uid_bytes in uid_list:
|
||||
try:
|
||||
_, msg_data = conn.uid("FETCH", uid_bytes, "(RFC822)")
|
||||
if not msg_data or not msg_data[0]:
|
||||
continue
|
||||
raw = msg_data[0][1]
|
||||
msg = email_lib.message_from_bytes(raw)
|
||||
|
||||
message_id = msg.get("Message-ID", "").strip()
|
||||
if not message_id:
|
||||
continue
|
||||
|
||||
title = _decode_header_value(msg.get("Subject", "(No Subject)"))
|
||||
body_snippet = _get_body_snippet(msg)
|
||||
|
||||
# Parse email send date
|
||||
from email.utils import parsedate_to_datetime
|
||||
email_date = None
|
||||
try:
|
||||
date_str = msg.get("Date")
|
||||
if date_str:
|
||||
email_date = parsedate_to_datetime(date_str)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
existing = db.query(Task).filter(Task.source_id == message_id).first()
|
||||
if existing:
|
||||
_apply_label(conn, uid_bytes, LABEL_TODO)
|
||||
continue
|
||||
|
||||
max_pos = db.query(Task).filter(Task.status == "open").count()
|
||||
db.add(Task(
|
||||
title=title[:200],
|
||||
description=body_snippet[:2000],
|
||||
priority=4,
|
||||
position=max_pos,
|
||||
status="open",
|
||||
origin="dhive_email",
|
||||
source_id=message_id,
|
||||
source_url=os.getenv("EMAIL_WEBMAIL_URL", ""),
|
||||
created_at=email_date,
|
||||
))
|
||||
_apply_label(conn, uid_bytes, LABEL_TODO)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error processing email UID {uid_bytes}: {e}")
|
||||
|
||||
# Cleanup: dismiss tasks whose emails are no longer in INBOX
|
||||
open_email_tasks = db.query(Task).filter(
|
||||
Task.origin == "dhive_email",
|
||||
Task.status == "open",
|
||||
Task.source_id != None
|
||||
).all()
|
||||
|
||||
dismissed_count = 0
|
||||
for task in open_email_tasks:
|
||||
try:
|
||||
# Search by Message-ID header
|
||||
status, data = conn.uid("SEARCH", "CHARSET", "UTF-8", f'HEADER Message-ID "{task.source_id}"')
|
||||
if status == "OK" and data[0]:
|
||||
# Email still in INBOX — keep task open
|
||||
continue
|
||||
# Also try without quotes (some servers are picky)
|
||||
status2, data2 = conn.uid("SEARCH", None, f'HEADER Message-ID {task.source_id}')
|
||||
if status2 == "OK" and data2[0]:
|
||||
continue
|
||||
# Email not found in INBOX — dismiss
|
||||
task.status = "dismissed"
|
||||
dismissed_count += 1
|
||||
except Exception as e:
|
||||
print(f" Cleanup check failed for {task.source_id[:30]}: {e}")
|
||||
|
||||
if dismissed_count:
|
||||
print(f" Auto-dismissed {dismissed_count} tasks (emails no longer in INBOX)")
|
||||
|
||||
db.commit()
|
||||
db.close()
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"Email sync error: {e}")
|
||||
traceback.print_exc()
|
||||
return False
|
||||
finally:
|
||||
try:
|
||||
conn.logout()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def mark_email_done_and_archive(source_id: str) -> bool:
|
||||
"""Find an email by Message-ID, mark done, and archive it."""
|
||||
try:
|
||||
conn = _get_imap_connection()
|
||||
except Exception as e:
|
||||
print(f"IMAP connection failed: {e}")
|
||||
return False
|
||||
|
||||
try:
|
||||
conn.select("INBOX")
|
||||
|
||||
# Try multiple search approaches
|
||||
uid_bytes = None
|
||||
|
||||
# Approach 1: HEADER search with quotes
|
||||
status, data = conn.uid("SEARCH", "CHARSET", "UTF-8", f'HEADER Message-ID "{source_id}"')
|
||||
if status == "OK" and data[0]:
|
||||
uid_bytes = data[0].split()[0]
|
||||
|
||||
# Approach 2: Without CHARSET
|
||||
if not uid_bytes:
|
||||
status, data = conn.uid("SEARCH", None, f'HEADER Message-ID {source_id}')
|
||||
if status == "OK" and data[0]:
|
||||
uid_bytes = data[0].split()[0]
|
||||
|
||||
# Approach 3: Search ALL and match manually (slow but reliable)
|
||||
if not uid_bytes:
|
||||
print(f" Header search failed, trying full scan for: {source_id[:40]}...")
|
||||
status, data = conn.uid("SEARCH", None, "ALL")
|
||||
if status == "OK" and data[0]:
|
||||
for uid in data[0].split()[-100:]: # Check last 100 emails
|
||||
try:
|
||||
_, msg_data = conn.uid("FETCH", uid, "(BODY.PEEK[HEADER.FIELDS (Message-ID)])")
|
||||
if msg_data and msg_data[0] and source_id.encode() in msg_data[0][1]:
|
||||
uid_bytes = uid
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if not uid_bytes:
|
||||
print(f" Email not found in INBOX: {source_id[:50]}")
|
||||
return False
|
||||
|
||||
print(f" Found email UID: {uid_bytes}")
|
||||
_remove_label(conn, uid_bytes, LABEL_TODO)
|
||||
_apply_label(conn, uid_bytes, LABEL_DONE)
|
||||
result = _archive_email(conn, uid_bytes)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error archiving email {source_id[:50]}: {e}")
|
||||
traceback.print_exc()
|
||||
return False
|
||||
finally:
|
||||
try:
|
||||
conn.logout()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def delete_email_from_server(source_id: str) -> bool:
|
||||
"""Find an email by Message-ID and permanently delete it (move to Trash)."""
|
||||
try:
|
||||
conn = _get_imap_connection()
|
||||
except Exception as e:
|
||||
print(f"IMAP connection failed: {e}")
|
||||
return False
|
||||
|
||||
try:
|
||||
conn.select("INBOX")
|
||||
|
||||
# Try multiple search approaches
|
||||
uid_bytes = None
|
||||
status, data = conn.uid("SEARCH", "CHARSET", "UTF-8", f'HEADER Message-ID "{source_id}"')
|
||||
if status == "OK" and data[0]:
|
||||
uid_bytes = data[0].split()[0]
|
||||
if not uid_bytes:
|
||||
status, data = conn.uid("SEARCH", None, f'HEADER Message-ID {source_id}')
|
||||
if status == "OK" and data[0]:
|
||||
uid_bytes = data[0].split()[0]
|
||||
|
||||
if not uid_bytes:
|
||||
print(f" Email not found for delete: {source_id[:50]}")
|
||||
return False
|
||||
|
||||
_delete_email(conn, uid_bytes)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error deleting email {source_id[:50]}: {e}")
|
||||
return False
|
||||
finally:
|
||||
try:
|
||||
conn.logout()
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,290 @@
|
||||
"""Gmail IMAP sync — second email source for task ingestion.
|
||||
|
||||
Requirements:
|
||||
- Enable IMAP in Gmail settings (Settings → See all settings → Forwarding and POP/IMAP)
|
||||
- Generate an App Password at https://myaccount.google.com/apppasswords
|
||||
(requires 2FA to be enabled on your Google account)
|
||||
- Set GMAIL_USERNAME and GMAIL_PASSWORD env vars
|
||||
"""
|
||||
|
||||
import os
|
||||
import imaplib
|
||||
import email as email_lib
|
||||
from email.header import decode_header
|
||||
import traceback
|
||||
from app.db.session import SessionLocal
|
||||
from app.models import Task
|
||||
|
||||
|
||||
def _get_gmail_connection():
|
||||
server = os.getenv("GMAIL_IMAP_SERVER", "imap.gmail.com")
|
||||
port = int(os.getenv("GMAIL_IMAP_PORT", "993"))
|
||||
username = os.getenv("GMAIL_USERNAME", "")
|
||||
password = os.getenv("GMAIL_PASSWORD", "")
|
||||
|
||||
if not username or not password:
|
||||
return None # Not configured, skip silently
|
||||
|
||||
conn = imaplib.IMAP4_SSL(server, port)
|
||||
conn.login(username, password)
|
||||
return conn
|
||||
|
||||
|
||||
def _decode_header_value(value) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
parts = decode_header(value)
|
||||
decoded = []
|
||||
for part, charset in parts:
|
||||
if isinstance(part, bytes):
|
||||
decoded.append(part.decode(charset or "utf-8", errors="replace"))
|
||||
else:
|
||||
decoded.append(part)
|
||||
return " ".join(decoded)
|
||||
|
||||
|
||||
def _get_body_snippet(msg, max_chars: int = 300) -> str:
|
||||
if msg.is_multipart():
|
||||
for part in msg.walk():
|
||||
if part.get_content_type() == "text/plain" and part.get("Content-Disposition") is None:
|
||||
payload = part.get_payload(decode=True)
|
||||
if payload:
|
||||
return payload.decode(part.get_content_charset() or "utf-8", errors="replace")[:max_chars]
|
||||
else:
|
||||
payload = msg.get_payload(decode=True)
|
||||
if payload:
|
||||
return payload.decode(msg.get_content_charset() or "utf-8", errors="replace")[:max_chars]
|
||||
return ""
|
||||
|
||||
|
||||
def sync_gmail():
|
||||
"""Fetch unread emails from Gmail inbox and create tasks.
|
||||
|
||||
Returns True on success (or if not configured), False on error.
|
||||
"""
|
||||
try:
|
||||
conn = _get_gmail_connection()
|
||||
except imaplib.IMAP4.error as e:
|
||||
print(f"Gmail IMAP auth failed: {e}")
|
||||
print("Make sure you're using an App Password (not your regular Google password).")
|
||||
print("Generate one at: https://myaccount.google.com/apppasswords")
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f"Gmail IMAP connection failed: {e}")
|
||||
traceback.print_exc()
|
||||
return False
|
||||
|
||||
if conn is None:
|
||||
# Not configured — skip silently
|
||||
return True
|
||||
|
||||
try:
|
||||
conn.select("INBOX")
|
||||
|
||||
# Search for unread emails
|
||||
status, data = conn.uid("SEARCH", None, "UNSEEN")
|
||||
if status != "OK" or not data[0]:
|
||||
conn.logout()
|
||||
return True # Nothing to process
|
||||
|
||||
uid_list = data[0].split()
|
||||
db = SessionLocal()
|
||||
|
||||
synced_count = 0
|
||||
for uid_bytes in uid_list:
|
||||
try:
|
||||
_, msg_data = conn.uid("FETCH", uid_bytes, "(RFC822)")
|
||||
if not msg_data or not msg_data[0]:
|
||||
continue
|
||||
raw = msg_data[0][1]
|
||||
msg = email_lib.message_from_bytes(raw)
|
||||
|
||||
message_id = msg.get("Message-ID", "").strip()
|
||||
if not message_id:
|
||||
continue
|
||||
|
||||
title = _decode_header_value(msg.get("Subject", "(No Subject)"))
|
||||
sender = _decode_header_value(msg.get("From", ""))
|
||||
body_snippet = _get_body_snippet(msg)
|
||||
|
||||
# Parse email send date
|
||||
from email.utils import parsedate_to_datetime
|
||||
email_date = None
|
||||
try:
|
||||
date_str = msg.get("Date")
|
||||
if date_str:
|
||||
email_date = parsedate_to_datetime(date_str)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Skip if already tracked (including dismissed)
|
||||
existing = db.query(Task).filter(Task.source_id == message_id).first()
|
||||
if existing:
|
||||
continue
|
||||
|
||||
# Limit title length
|
||||
title = title[:200] if title else "(No Subject)"
|
||||
description = f"From: {sender}\n\n{body_snippet}" if sender else body_snippet
|
||||
|
||||
max_pos = db.query(Task).filter(Task.status == "open").count()
|
||||
db.add(Task(
|
||||
title=title,
|
||||
description=description[:2000], # Limit description length
|
||||
priority=4,
|
||||
position=max_pos,
|
||||
status="open",
|
||||
origin="gmail",
|
||||
source_id=message_id,
|
||||
source_url="https://mail.google.com/mail/",
|
||||
created_at=email_date,
|
||||
))
|
||||
synced_count += 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error processing Gmail UID {uid_bytes}: {e}")
|
||||
|
||||
db.commit()
|
||||
db.close()
|
||||
print(f"Gmail sync: {synced_count} new tasks from {len(uid_list)} unread emails")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Gmail sync error: {e}")
|
||||
traceback.print_exc()
|
||||
return False
|
||||
finally:
|
||||
try:
|
||||
conn.logout()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def gmail_archive_email(source_id: str) -> bool:
|
||||
"""Archive a Gmail email (remove from Inbox, keep in All Mail)."""
|
||||
try:
|
||||
conn = _get_gmail_connection()
|
||||
except Exception as e:
|
||||
print(f"Gmail connection failed: {e}")
|
||||
return False
|
||||
if conn is None:
|
||||
return False
|
||||
|
||||
try:
|
||||
conn.select("INBOX")
|
||||
|
||||
# Find the email
|
||||
uid_bytes = _find_gmail_email(conn, source_id)
|
||||
if not uid_bytes:
|
||||
print(f" Gmail: email not found: {source_id[:40]}")
|
||||
return False
|
||||
|
||||
# Gmail archive = move to All Mail (remove INBOX label)
|
||||
# In IMAP terms: COPY to [Gmail]/All Mail, then delete from INBOX
|
||||
try:
|
||||
conn.uid("COPY", uid_bytes, "[Gmail]/All Mail")
|
||||
except Exception:
|
||||
pass # Might already be in All Mail
|
||||
conn.uid("STORE", uid_bytes, "+FLAGS", r"(\Deleted)")
|
||||
conn.expunge()
|
||||
print(f" Gmail: archived email")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"Gmail archive error: {e}")
|
||||
return False
|
||||
finally:
|
||||
try:
|
||||
conn.logout()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def gmail_delete_email(source_id: str) -> bool:
|
||||
"""Delete a Gmail email (move to Trash)."""
|
||||
try:
|
||||
conn = _get_gmail_connection()
|
||||
except Exception as e:
|
||||
print(f"Gmail connection failed: {e}")
|
||||
return False
|
||||
if conn is None:
|
||||
return False
|
||||
|
||||
try:
|
||||
conn.select("INBOX")
|
||||
|
||||
uid_bytes = _find_gmail_email(conn, source_id)
|
||||
if not uid_bytes:
|
||||
print(f" Gmail: email not found for delete: {source_id[:40]}")
|
||||
return False
|
||||
|
||||
# Move to Gmail Trash
|
||||
conn.uid("COPY", uid_bytes, "[Gmail]/Trash")
|
||||
conn.uid("STORE", uid_bytes, "+FLAGS", r"(\Deleted)")
|
||||
conn.expunge()
|
||||
print(f" Gmail: deleted email")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"Gmail delete error: {e}")
|
||||
return False
|
||||
finally:
|
||||
try:
|
||||
conn.logout()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _find_gmail_email(conn, source_id: str):
|
||||
"""Find a Gmail email by Message-ID. Returns UID bytes or None."""
|
||||
# Try HEADER search
|
||||
status, data = conn.uid("SEARCH", None, f'HEADER Message-ID "{source_id}"')
|
||||
if status == "OK" and data[0]:
|
||||
return data[0].split()[0]
|
||||
# Try without quotes
|
||||
status, data = conn.uid("SEARCH", None, f'HEADER Message-ID {source_id}')
|
||||
if status == "OK" and data[0]:
|
||||
return data[0].split()[0]
|
||||
return None
|
||||
|
||||
|
||||
def gmail_cleanup_dismissed(db_session=None):
|
||||
"""Check if Gmail tasks are still in INBOX, dismiss if not."""
|
||||
try:
|
||||
conn = _get_gmail_connection()
|
||||
except Exception:
|
||||
return
|
||||
if conn is None:
|
||||
return
|
||||
|
||||
try:
|
||||
conn.select("INBOX")
|
||||
db = db_session or SessionLocal()
|
||||
|
||||
open_gmail_tasks = db.query(Task).filter(
|
||||
Task.origin == "gmail",
|
||||
Task.status == "open",
|
||||
Task.source_id != None
|
||||
).all()
|
||||
|
||||
dismissed = 0
|
||||
for task in open_gmail_tasks:
|
||||
uid = _find_gmail_email(conn, task.source_id)
|
||||
if not uid:
|
||||
task.status = "dismissed"
|
||||
dismissed += 1
|
||||
|
||||
if dismissed:
|
||||
db.commit()
|
||||
print(f" Gmail: auto-dismissed {dismissed} tasks")
|
||||
|
||||
if not db_session:
|
||||
db.close()
|
||||
|
||||
except Exception as e:
|
||||
print(f"Gmail cleanup error: {e}")
|
||||
finally:
|
||||
try:
|
||||
conn.logout()
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,325 @@
|
||||
import os
|
||||
import caldav
|
||||
import json
|
||||
import traceback
|
||||
from datetime import datetime
|
||||
from zoneinfo import ZoneInfo
|
||||
from app.db.session import SessionLocal
|
||||
from app.models import Event, Task
|
||||
|
||||
# Berlin timezone for display
|
||||
BERLIN_TZ = ZoneInfo("Europe/Berlin")
|
||||
UTC_TZ = ZoneInfo("UTC")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _get_caldav_principal():
|
||||
url = os.getenv("NEXTCLOUD_URL")
|
||||
username = os.getenv("NEXTCLOUD_USERNAME")
|
||||
password = os.getenv("NEXTCLOUD_PASSWORD")
|
||||
|
||||
if not all([url, username, password]):
|
||||
raise ValueError("Nextcloud credentials missing. Set NEXTCLOUD_URL, NEXTCLOUD_USERNAME, NEXTCLOUD_PASSWORD.")
|
||||
|
||||
caldav_url = url if url.endswith("/remote.php/dav/") else f"{url.rstrip('/')}/remote.php/dav/"
|
||||
client = caldav.DAVClient(url=caldav_url, username=username, password=password)
|
||||
return client.principal()
|
||||
|
||||
|
||||
def _to_berlin(dt):
|
||||
"""Convert a datetime to Berlin timezone. Handles naive and aware datetimes."""
|
||||
if dt is None:
|
||||
return None
|
||||
if not isinstance(dt, datetime):
|
||||
# date-only: treat as midnight Berlin time
|
||||
return datetime.combine(dt, datetime.min.time()).replace(tzinfo=BERLIN_TZ)
|
||||
if dt.tzinfo is None:
|
||||
# Naive datetime — assume UTC
|
||||
dt = dt.replace(tzinfo=UTC_TZ)
|
||||
return dt.astimezone(BERLIN_TZ)
|
||||
|
||||
|
||||
def _is_own_calendar(cal_url_str: str) -> bool:
|
||||
"""Check if a calendar belongs to the authenticated user (not shared by others)."""
|
||||
return "_shared_by_" not in cal_url_str
|
||||
|
||||
|
||||
def _get_nextcloud_base_url() -> str:
|
||||
"""Get the base Nextcloud URL for building links."""
|
||||
url = os.getenv("NEXTCLOUD_URL", "")
|
||||
return url.rstrip("/")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Calendar events
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def sync_calendar_events():
|
||||
try:
|
||||
principal = _get_caldav_principal()
|
||||
calendars = principal.calendars()
|
||||
except Exception as e:
|
||||
print(f"Failed to connect to Nextcloud CalDAV: {e}")
|
||||
return False
|
||||
|
||||
db = SessionLocal()
|
||||
|
||||
for calendar in calendars:
|
||||
cal_url_str = str(calendar.url)
|
||||
if not _is_own_calendar(cal_url_str):
|
||||
print(f"Skipping shared calendar: {calendar.get_display_name()}")
|
||||
continue
|
||||
|
||||
try:
|
||||
events = calendar.events()
|
||||
for event in events:
|
||||
ical = event.icalendar_component
|
||||
if ical.name == "VEVENT":
|
||||
title = str(ical.get("SUMMARY", "Untitled Event"))
|
||||
dtstart = ical.get("DTSTART")
|
||||
dtend = ical.get("DTEND")
|
||||
|
||||
if not dtstart or not dtend:
|
||||
continue
|
||||
|
||||
start_time = _to_berlin(dtstart.dt)
|
||||
end_time = _to_berlin(dtend.dt)
|
||||
|
||||
existing = db.query(Event).filter(Event.title == title, Event.source == "nextcloud").first()
|
||||
if not existing:
|
||||
db.add(Event(title=title, start_time=start_time, end_time=end_time, source="nextcloud"))
|
||||
except Exception as e:
|
||||
print(f"Error reading calendar {calendar}: {e}")
|
||||
|
||||
db.commit()
|
||||
db.close()
|
||||
return True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ToDo sync (two-way, only own tasks)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _parse_vtodo_datetime(val):
|
||||
"""Parse a VTODO datetime and convert to Berlin timezone."""
|
||||
if val is None:
|
||||
return None
|
||||
return _to_berlin(val.dt)
|
||||
|
||||
|
||||
def _has_local_changes(task, remote_title: str, remote_description: str, remote_status: str) -> bool:
|
||||
"""Check if the local task differs from what was last synced (i.e., user edited locally)."""
|
||||
# If never synced, treat as no local changes (first sync)
|
||||
if task.last_synced_at is None:
|
||||
return False
|
||||
# If the task was modified after last sync, local has changes
|
||||
# We compare local fields against remote to detect divergence
|
||||
local_title = task.title or ""
|
||||
local_desc = task.description or ""
|
||||
local_status = task.status or "open"
|
||||
# If local differs from remote, and we have a last_synced_at, local was changed
|
||||
if local_title != remote_title:
|
||||
return True
|
||||
if local_desc != (remote_description or ""):
|
||||
return True
|
||||
if local_status != remote_status:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _has_remote_changes(task, remote_title: str, remote_description: str, remote_status: str) -> bool:
|
||||
"""Check if remote data differs from what's currently stored locally.
|
||||
|
||||
This detects whether the remote side changed since our last sync.
|
||||
If the remote values differ from what we have locally, remote changed.
|
||||
"""
|
||||
local_title = task.title or ""
|
||||
local_desc = task.description or ""
|
||||
local_status = task.status or "open"
|
||||
if local_title != remote_title:
|
||||
return True
|
||||
if local_desc != (remote_description or ""):
|
||||
return True
|
||||
if local_status != remote_status:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def sync_todo_tasks():
|
||||
"""Import VTODOs from own Nextcloud todo lists into local Task table.
|
||||
|
||||
Only imports from calendars owned by the authenticated user (skips shared).
|
||||
Detects conflicts when both local and remote changed since last sync.
|
||||
"""
|
||||
try:
|
||||
principal = _get_caldav_principal()
|
||||
calendars = principal.calendars()
|
||||
except Exception as e:
|
||||
print(f"Failed to connect to Nextcloud CalDAV: {e}")
|
||||
return False
|
||||
|
||||
db = SessionLocal()
|
||||
nc_username = os.getenv("NEXTCLOUD_USERNAME", "")
|
||||
base_url = _get_nextcloud_base_url()
|
||||
|
||||
for calendar in calendars:
|
||||
cal_url_str = str(calendar.url)
|
||||
cal_name = calendar.get_display_name() or "Tasks"
|
||||
|
||||
# Skip shared calendars (these belong to coworkers)
|
||||
if not _is_own_calendar(cal_url_str):
|
||||
print(f"Skipping shared todo list: {cal_name}")
|
||||
continue
|
||||
|
||||
try:
|
||||
todos = calendar.todos(include_completed=True)
|
||||
except Exception:
|
||||
continue # calendar may not support VTODOs
|
||||
|
||||
for todo in todos:
|
||||
try:
|
||||
ical = todo.icalendar_component
|
||||
if ical.name != "VTODO":
|
||||
continue
|
||||
|
||||
uid = str(ical.get("UID", ""))
|
||||
if not uid:
|
||||
continue
|
||||
|
||||
title = str(ical.get("SUMMARY", "Untitled Task"))
|
||||
description = str(ical.get("DESCRIPTION", "") or "")
|
||||
due_date = _parse_vtodo_datetime(ical.get("DUE"))
|
||||
vtodo_status = str(ical.get("STATUS", "NEEDS-ACTION")).upper()
|
||||
status = "completed" if vtodo_status == "COMPLETED" else "open"
|
||||
|
||||
# Build origin label with calendar name
|
||||
origin_label = f"nextcloud_todo:{cal_name}"
|
||||
|
||||
# Build link to Nextcloud tasks app
|
||||
source_url = f"{base_url}/apps/tasks/#/calendars/{cal_name}" if base_url else None
|
||||
|
||||
existing = db.query(Task).filter(Task.source_id == uid).first()
|
||||
if existing:
|
||||
# Don't touch dismissed or completed tasks (user already handled them)
|
||||
if existing.status in ("dismissed", "completed"):
|
||||
continue
|
||||
|
||||
# --- Conflict detection ---
|
||||
# Check if local was modified since last sync
|
||||
local_changed = _has_local_changes(existing, title, description, status)
|
||||
remote_changed = _has_remote_changes(existing, title, description, status)
|
||||
|
||||
if local_changed and remote_changed:
|
||||
# Both sides changed → conflict
|
||||
existing.has_conflict = True
|
||||
existing.conflict_data = json.dumps({
|
||||
"title": title,
|
||||
"status": status,
|
||||
"description": description,
|
||||
})
|
||||
continue
|
||||
|
||||
if not remote_changed:
|
||||
# Only local changed (or nothing changed) → push local to remote if needed
|
||||
if local_changed and existing.source_id:
|
||||
push_updates = {}
|
||||
if existing.title != title:
|
||||
push_updates["title"] = existing.title
|
||||
if existing.description != (description or ""):
|
||||
push_updates["description"] = existing.description
|
||||
if existing.status != status:
|
||||
push_updates["status"] = existing.status
|
||||
if push_updates:
|
||||
update_remote_todo(existing.source_id, push_updates)
|
||||
existing.last_synced_at = datetime.now(tz=UTC_TZ)
|
||||
continue
|
||||
|
||||
# Only remote changed → update local normally
|
||||
# Don't revert in_progress or review tasks back to open
|
||||
if existing.status in ("in_progress", "review") and status == "open":
|
||||
existing.title = title
|
||||
existing.description = description
|
||||
existing.due_date = due_date
|
||||
existing.origin = origin_label
|
||||
existing.source_url = source_url
|
||||
existing.last_synced_at = datetime.now(tz=UTC_TZ)
|
||||
continue
|
||||
|
||||
existing.title = title
|
||||
existing.description = description
|
||||
existing.due_date = due_date
|
||||
existing.status = status
|
||||
existing.origin = origin_label
|
||||
existing.source_url = source_url
|
||||
existing.last_synced_at = datetime.now(tz=UTC_TZ)
|
||||
else:
|
||||
max_pos = db.query(Task).count()
|
||||
db.add(Task(
|
||||
title=title,
|
||||
description=description,
|
||||
priority=4,
|
||||
position=max_pos,
|
||||
status=status,
|
||||
origin=origin_label,
|
||||
source_id=uid,
|
||||
source_url=source_url,
|
||||
due_date=due_date,
|
||||
last_synced_at=datetime.now(tz=UTC_TZ),
|
||||
))
|
||||
except Exception as e:
|
||||
print(f"Error processing VTODO: {e}")
|
||||
|
||||
db.commit()
|
||||
db.close()
|
||||
return True
|
||||
|
||||
|
||||
def update_remote_todo(source_id: str, updates: dict):
|
||||
"""Push local task changes back to the Nextcloud server.
|
||||
|
||||
updates: dict with any of: title, description, status
|
||||
"""
|
||||
try:
|
||||
principal = _get_caldav_principal()
|
||||
calendars = principal.calendars()
|
||||
except Exception as e:
|
||||
print(f"Failed to connect to Nextcloud CalDAV for push: {e}")
|
||||
return False
|
||||
|
||||
for calendar in calendars:
|
||||
cal_url_str = str(calendar.url)
|
||||
# Only push to own calendars
|
||||
if not _is_own_calendar(cal_url_str):
|
||||
continue
|
||||
|
||||
try:
|
||||
todos = calendar.todos(include_completed=True)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
for todo in todos:
|
||||
try:
|
||||
ical = todo.icalendar_component
|
||||
if ical.name != "VTODO":
|
||||
continue
|
||||
if str(ical.get("UID", "")) != source_id:
|
||||
continue
|
||||
|
||||
# Apply updates
|
||||
if "title" in updates:
|
||||
ical["SUMMARY"] = updates["title"]
|
||||
if "description" in updates:
|
||||
ical["DESCRIPTION"] = updates["description"]
|
||||
if "status" in updates:
|
||||
ical["STATUS"] = "COMPLETED" if updates["status"] == "completed" else "NEEDS-ACTION"
|
||||
|
||||
todo.save()
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"Error updating remote VTODO {source_id}: {e}")
|
||||
|
||||
print(f"Remote VTODO with UID {source_id} not found.")
|
||||
return False
|
||||
@@ -0,0 +1,207 @@
|
||||
"""Phone number extraction and normalization for German telephone formats.
|
||||
|
||||
Supports:
|
||||
- +49 international format (e.g., +49 30 12345678, +49-171-1234567)
|
||||
- 0xxx domestic format (e.g., 030 12345678, 0171-1234567)
|
||||
- (0xxx) parenthesized area code format (e.g., (030) 12345678, (0171) 1234567)
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
# Pattern 1: +49 international format
|
||||
# +49 followed by 9 to 12 digits with optional spaces or hyphens
|
||||
_INTERNATIONAL_PATTERN = re.compile(
|
||||
r"\+49[\s\-]?(\d[\d\s\-]{8,14}\d)"
|
||||
)
|
||||
|
||||
# Pattern 2: 0xxx domestic format
|
||||
# 0 followed by area code (2-5 digits) then subscriber (4-8 digits), with optional separators
|
||||
_DOMESTIC_PATTERN = re.compile(
|
||||
r"(?<!\()0(\d{2,5})[\s\-/]?(\d[\d\s\-]{3,9}\d)"
|
||||
)
|
||||
|
||||
# Pattern 3: (0xxx) parenthesized area code format
|
||||
# (0xxx) followed by subscriber digits with optional separators
|
||||
_PARENTHESIZED_PATTERN = re.compile(
|
||||
r"\(0(\d{2,5})\)[\s\-]?(\d[\d\s\-]{3,9}\d)"
|
||||
)
|
||||
|
||||
|
||||
def extract_phone_number(text: str) -> str | None:
|
||||
"""Extract the first German phone number from text.
|
||||
|
||||
Supports formats:
|
||||
- +49 xxx xxxx xxxx (international)
|
||||
- 0xxx xxxxxxx (domestic with area code)
|
||||
- (0xxx) xxxxxxx (domestic with parenthesized area code)
|
||||
|
||||
Returns the raw matched string or None if no match found.
|
||||
"""
|
||||
if not text:
|
||||
return None
|
||||
|
||||
# Collect all matches with their start positions to find the first one
|
||||
matches: list[tuple[int, str]] = []
|
||||
|
||||
for m in _INTERNATIONAL_PATTERN.finditer(text):
|
||||
matches.append((m.start(), m.group(0)))
|
||||
|
||||
for m in _PARENTHESIZED_PATTERN.finditer(text):
|
||||
matches.append((m.start(), m.group(0)))
|
||||
|
||||
for m in _DOMESTIC_PATTERN.finditer(text):
|
||||
# Avoid matching numbers already captured by parenthesized pattern
|
||||
start = m.start()
|
||||
raw = m.group(0)
|
||||
# Check this isn't inside parentheses
|
||||
if start > 0 and text[start - 1] == "(":
|
||||
continue
|
||||
matches.append((start, raw))
|
||||
|
||||
if not matches:
|
||||
return None
|
||||
|
||||
# Return the first match by position
|
||||
matches.sort(key=lambda x: x[0])
|
||||
return matches[0][1]
|
||||
|
||||
|
||||
def normalize_phone_number(raw: str) -> str:
|
||||
"""Normalize a phone number by removing separators and standardizing format.
|
||||
|
||||
Strips spaces, hyphens, slashes, and parentheses.
|
||||
Converts domestic format (leading 0) to international (+49) format.
|
||||
|
||||
Returns the normalized number string (e.g., "+4930123456").
|
||||
"""
|
||||
if not raw:
|
||||
return raw
|
||||
|
||||
# Remove all separators: spaces, hyphens, slashes, parentheses
|
||||
cleaned = re.sub(r"[\s\-/\(\)]", "", raw)
|
||||
|
||||
# Convert domestic (0xxx) to international (+49xxx)
|
||||
if cleaned.startswith("0") and not cleaned.startswith("+"):
|
||||
cleaned = "+49" + cleaned[1:]
|
||||
|
||||
return cleaned
|
||||
|
||||
|
||||
def parse_phone_number(text: str) -> dict | None:
|
||||
"""Parse a phone number string into components.
|
||||
|
||||
Returns a dict with keys: country_code, area_code, subscriber
|
||||
or None if the text cannot be parsed as a German phone number.
|
||||
"""
|
||||
if not text:
|
||||
return None
|
||||
|
||||
# First try to extract a phone number from the text
|
||||
raw = extract_phone_number(text)
|
||||
if raw is None:
|
||||
# Try parsing the text directly as a phone number
|
||||
normalized = normalize_phone_number(text)
|
||||
if not normalized.startswith("+49"):
|
||||
return None
|
||||
digits_after_country = normalized[3:]
|
||||
if not digits_after_country or len(digits_after_country) < 5:
|
||||
return None
|
||||
# Use heuristic: area codes are 2-5 digits, rest is subscriber
|
||||
area_code, subscriber = _split_area_subscriber(digits_after_country)
|
||||
return {
|
||||
"country_code": "+49",
|
||||
"area_code": area_code,
|
||||
"subscriber": subscriber,
|
||||
}
|
||||
|
||||
# Parse the extracted raw number
|
||||
# Try international pattern
|
||||
m = _INTERNATIONAL_PATTERN.match(raw)
|
||||
if m:
|
||||
digits = re.sub(r"[\s\-]", "", m.group(1))
|
||||
area_code, subscriber = _split_area_subscriber(digits)
|
||||
return {
|
||||
"country_code": "+49",
|
||||
"area_code": area_code,
|
||||
"subscriber": subscriber,
|
||||
}
|
||||
|
||||
# Try parenthesized pattern
|
||||
m = _PARENTHESIZED_PATTERN.match(raw)
|
||||
if m:
|
||||
area_code = m.group(1)
|
||||
subscriber = re.sub(r"[\s\-]", "", m.group(2))
|
||||
return {
|
||||
"country_code": "+49",
|
||||
"area_code": area_code,
|
||||
"subscriber": subscriber,
|
||||
}
|
||||
|
||||
# Try domestic pattern
|
||||
m = _DOMESTIC_PATTERN.match(raw)
|
||||
if m:
|
||||
area_code = m.group(1)
|
||||
subscriber = re.sub(r"[\s\-]", "", m.group(2))
|
||||
return {
|
||||
"country_code": "+49",
|
||||
"area_code": area_code,
|
||||
"subscriber": subscriber,
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _split_area_subscriber(digits: str) -> tuple[str, str]:
|
||||
"""Split a digit string into area code and subscriber number.
|
||||
|
||||
Uses known German area code lengths:
|
||||
- 2 digits: major cities (30=Berlin, 40=Hamburg, etc.)
|
||||
- 3 digits: large cities (211=Düsseldorf, 221=Köln, etc.)
|
||||
- 4 digits: medium cities and mobile prefixes (171, 172, etc.)
|
||||
- 5 digits: smaller areas
|
||||
|
||||
Heuristic: if total digits <= 10, use shorter area code.
|
||||
"""
|
||||
# Known 2-digit area codes (major cities)
|
||||
two_digit_codes = {"30", "40", "69", "89"}
|
||||
|
||||
# Known 3-digit mobile prefixes
|
||||
three_digit_mobile = {
|
||||
"151", "152", "153", "155", "156", "157", "159",
|
||||
"160", "162", "163", "170", "171", "172", "173",
|
||||
"174", "175", "176", "177", "178", "179",
|
||||
}
|
||||
|
||||
# Check 2-digit area codes
|
||||
if len(digits) >= 4 and digits[:2] in two_digit_codes:
|
||||
return digits[:2], digits[2:]
|
||||
|
||||
# Check 3-digit codes (mobile and city)
|
||||
if len(digits) >= 5 and digits[:3] in three_digit_mobile:
|
||||
return digits[:3], digits[3:]
|
||||
|
||||
# Check common 3-digit city codes
|
||||
three_digit_city = {
|
||||
"211", "212", "221", "228", "231", "241", "251",
|
||||
"261", "271", "281", "291",
|
||||
"311", "321", "331", "341", "351", "361", "371", "381", "391",
|
||||
"421", "431", "441", "451", "461", "471", "481",
|
||||
"511", "521", "531", "541", "551", "561", "571", "581", "591",
|
||||
"611", "621", "631", "641", "651", "661", "671", "681",
|
||||
"711", "721", "731", "741", "751", "761", "771",
|
||||
"811", "821", "831", "841", "851", "861", "871",
|
||||
"911", "921", "931", "941", "951", "961", "971", "981", "991",
|
||||
}
|
||||
if len(digits) >= 5 and digits[:3] in three_digit_city:
|
||||
return digits[:3], digits[3:]
|
||||
|
||||
# Default heuristic based on total length
|
||||
if len(digits) <= 8:
|
||||
# Short number: assume 3-digit area code
|
||||
return digits[:3], digits[3:]
|
||||
elif len(digits) <= 10:
|
||||
# Medium: assume 4-digit area code
|
||||
return digits[:4], digits[4:]
|
||||
else:
|
||||
# Long: assume 4-digit area code
|
||||
return digits[:4], digits[4:]
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Prioritization Engine — generates actionable suggestions based on task state."""
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from zoneinfo import ZoneInfo
|
||||
from sqlalchemy.orm import Session
|
||||
from app.models import Task
|
||||
|
||||
BERLIN_TZ = ZoneInfo("Europe/Berlin")
|
||||
|
||||
|
||||
def generate_suggestions(db: Session) -> list[dict]:
|
||||
"""Analyze open tasks and generate prioritization suggestions.
|
||||
|
||||
Returns a list of suggestion dicts with: id, content, type, severity.
|
||||
"""
|
||||
now = datetime.now(BERLIN_TZ)
|
||||
today = now.date()
|
||||
suggestions = []
|
||||
suggestion_id = 1
|
||||
|
||||
open_tasks = db.query(Task).filter(Task.status == "open").all()
|
||||
|
||||
if not open_tasks:
|
||||
return []
|
||||
|
||||
# --- 1. Overdue tasks ---
|
||||
overdue_tasks = [
|
||||
t for t in open_tasks
|
||||
if t.due_date and t.due_date.date() < today
|
||||
]
|
||||
for t in overdue_tasks[:3]: # Limit to top 3
|
||||
days_overdue = (today - t.due_date.date()).days
|
||||
suggestions.append({
|
||||
"id": suggestion_id,
|
||||
"content": f"⚠️ \"{t.title}\" is {days_overdue} day{'s' if days_overdue != 1 else ''} overdue. Reschedule or complete it.",
|
||||
"type": "overdue",
|
||||
"severity": "high",
|
||||
"task_id": t.id,
|
||||
})
|
||||
suggestion_id += 1
|
||||
|
||||
# --- 2. Due tomorrow ---
|
||||
tomorrow = today + timedelta(days=1)
|
||||
due_tomorrow = [
|
||||
t for t in open_tasks
|
||||
if t.due_date and t.due_date.date() == tomorrow
|
||||
]
|
||||
for t in due_tomorrow:
|
||||
suggestions.append({
|
||||
"id": suggestion_id,
|
||||
"content": f"📅 \"{t.title}\" is due tomorrow.",
|
||||
"type": "upcoming",
|
||||
"severity": "medium",
|
||||
"task_id": t.id,
|
||||
})
|
||||
suggestion_id += 1
|
||||
|
||||
# --- 3. Due this week ---
|
||||
week_end = today + timedelta(days=7)
|
||||
due_this_week = [
|
||||
t for t in open_tasks
|
||||
if t.due_date and today < t.due_date.date() <= week_end
|
||||
and t.due_date.date() != tomorrow # Already covered above
|
||||
]
|
||||
if len(due_this_week) > 3:
|
||||
suggestions.append({
|
||||
"id": suggestion_id,
|
||||
"content": f"📋 You have {len(due_this_week)} tasks due this week. Plan your time.",
|
||||
"type": "workload",
|
||||
"severity": "medium",
|
||||
})
|
||||
suggestion_id += 1
|
||||
|
||||
# --- 4. High-priority tasks without due dates ---
|
||||
high_no_date = [
|
||||
t for t in open_tasks
|
||||
if t.priority and t.priority <= 2 and not t.due_date
|
||||
]
|
||||
for t in high_no_date[:2]: # Limit to top 2
|
||||
suggestions.append({
|
||||
"id": suggestion_id,
|
||||
"content": f"🎯 \"{t.title}\" is high priority but has no deadline. Set a due date.",
|
||||
"type": "missing_date",
|
||||
"severity": "medium",
|
||||
"task_id": t.id,
|
||||
})
|
||||
suggestion_id += 1
|
||||
|
||||
# --- 5. Tasks idle for 7+ days ---
|
||||
idle_threshold = now - timedelta(days=7)
|
||||
idle_tasks = [
|
||||
t for t in open_tasks
|
||||
if t.created_at and t.created_at < idle_threshold
|
||||
and (not t.due_date) # Only flag idle tasks without deadlines
|
||||
and t.priority and t.priority >= 3 # Low/medium priority
|
||||
]
|
||||
if len(idle_tasks) > 5:
|
||||
suggestions.append({
|
||||
"id": suggestion_id,
|
||||
"content": f"🧹 {len(idle_tasks)} low-priority tasks have been sitting for over a week. Review and clean up.",
|
||||
"type": "stale",
|
||||
"severity": "low",
|
||||
})
|
||||
suggestion_id += 1
|
||||
elif idle_tasks:
|
||||
oldest = max(idle_tasks, key=lambda t: (now - t.created_at).days if t.created_at else 0)
|
||||
if oldest.created_at:
|
||||
days_idle = (now - oldest.created_at).days
|
||||
suggestions.append({
|
||||
"id": suggestion_id,
|
||||
"content": f"🧹 \"{oldest.title}\" has been open for {days_idle} days. Still relevant?",
|
||||
"type": "stale",
|
||||
"severity": "low",
|
||||
"task_id": oldest.id,
|
||||
})
|
||||
suggestion_id += 1
|
||||
|
||||
# --- 6. Too many high-priority tasks ---
|
||||
high_priority_count = len([t for t in open_tasks if t.priority and t.priority <= 2])
|
||||
if high_priority_count > 5:
|
||||
suggestions.append({
|
||||
"id": suggestion_id,
|
||||
"content": f"🔥 You have {high_priority_count} high-priority tasks. If everything is urgent, nothing is. Consider demoting some.",
|
||||
"type": "overload",
|
||||
"severity": "medium",
|
||||
})
|
||||
suggestion_id += 1
|
||||
|
||||
# --- 7. No tasks due soon (might need planning) ---
|
||||
tasks_with_dates = [t for t in open_tasks if t.due_date]
|
||||
if len(open_tasks) > 10 and len(tasks_with_dates) < 3:
|
||||
suggestions.append({
|
||||
"id": suggestion_id,
|
||||
"content": "📆 Most of your tasks have no due dates. Consider scheduling your top priorities.",
|
||||
"type": "planning",
|
||||
"severity": "low",
|
||||
})
|
||||
suggestion_id += 1
|
||||
|
||||
return suggestions
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,52 @@
|
||||
version: "3.8"
|
||||
|
||||
services:
|
||||
db:
|
||||
image: postgres:16-alpine
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_DB: orgmylife
|
||||
POSTGRES_USER: orgmylife
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-changeme}
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
ports:
|
||||
- "5432:5432"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U orgmylife"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
|
||||
app:
|
||||
build: .
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
- "8000:8000"
|
||||
environment:
|
||||
DATABASE_URL: postgresql://orgmylife:${POSTGRES_PASSWORD:-changeme}@db:5432/orgmylife
|
||||
NEXTCLOUD_URL: ${NEXTCLOUD_URL}
|
||||
NEXTCLOUD_USERNAME: ${NEXTCLOUD_USERNAME}
|
||||
NEXTCLOUD_PASSWORD: ${NEXTCLOUD_PASSWORD}
|
||||
IMAP_SERVER: ${IMAP_SERVER}
|
||||
IMAP_PORT: ${IMAP_PORT:-993}
|
||||
EMAIL_USERNAME: ${EMAIL_USERNAME}
|
||||
EMAIL_PASSWORD: ${EMAIL_PASSWORD}
|
||||
EMAIL_WEBMAIL_URL: ${EMAIL_WEBMAIL_URL:-}
|
||||
GMAIL_USERNAME: ${GMAIL_USERNAME:-}
|
||||
GMAIL_PASSWORD: ${GMAIL_PASSWORD:-}
|
||||
# API authentication for remote access
|
||||
API_SECRET: ${API_SECRET:-}
|
||||
APP_USERNAME: ${APP_USERNAME:-admin}
|
||||
APP_PASSWORD: ${APP_PASSWORD:-changeme}
|
||||
volumes:
|
||||
- ./BACKLOG.md:/app/BACKLOG.md:ro
|
||||
command: >
|
||||
sh -c "python -c 'from app.db.session import engine, Base; from app.models import *; Base.metadata.create_all(bind=engine)' &&
|
||||
uvicorn app.main:app --host 0.0.0.0 --port 8000"
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user