Migrate all repos into monorepo context folders
Bahn: aisupport, Analyse-O2C-C2S, awesome-bahn-mcp-servers, beam-mcp,
Confluence_Bot, db-planet-mcp-server, O2C-Harness, project-audit,
Projekt-KIQ-HP, teamlandkarte-mcp
Dhive: Jury-Voting
Privat: CV, NoteGraph (NOTE: NoteGraph needs complete redo after consolidation)
Shared: AI-Orchestrator, OrgMyLife, power_skills_and_more
Shared/references: symphony (read-only)
Bahn repos remain available as independent remotes - this monorepo
pulls them in via subtree, the originals are untouched.
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""External service integrations: LLM providers, OrgMyLife, Git, Nextcloud."""
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Git commit operations for auto-committing imported notes.
|
||||
|
||||
Stages new markdown files and stub pages, commits with message format:
|
||||
ingestion: import N notes from [source-type]
|
||||
|
||||
Supports --no-commit flag. Handles git failures gracefully.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def commit_imported_notes(files: list[Path], source_type: str = "import") -> bool:
|
||||
"""Stage and commit imported note files.
|
||||
|
||||
Args:
|
||||
files: List of file paths to stage and commit.
|
||||
source_type: Source description for commit message (e.g., 'batch-import', 'upload', 'nextcloud').
|
||||
|
||||
Returns:
|
||||
True if commit succeeded, False otherwise.
|
||||
"""
|
||||
if not files:
|
||||
logger.debug("No files to commit.")
|
||||
return False
|
||||
|
||||
try:
|
||||
# Stage files
|
||||
file_strs = [str(f) for f in files]
|
||||
subprocess.run(
|
||||
["git", "add"] + file_strs,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
|
||||
# Commit
|
||||
count = len(files)
|
||||
message = f"ingestion: import {count} notes from {source_type}"
|
||||
|
||||
result = subprocess.run(
|
||||
["git", "commit", "-m", message],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
logger.info("Git commit: %s", message)
|
||||
return True
|
||||
else:
|
||||
# Nothing to commit (files already tracked/unchanged)
|
||||
logger.debug("Git commit returned %d: %s", result.returncode, result.stderr.strip())
|
||||
return False
|
||||
|
||||
except FileNotFoundError:
|
||||
logger.warning("Git not found in PATH — skipping commit.")
|
||||
return False
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.warning("Git operation failed: %s", e.stderr.strip() if e.stderr else str(e))
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.warning("Unexpected git error: %s", e)
|
||||
return False
|
||||
@@ -0,0 +1,112 @@
|
||||
"""LiteLLM wrapper for model-agnostic LLM access.
|
||||
|
||||
Supports OpenAI, Anthropic, Google, Mistral, and Ollama via a unified interface.
|
||||
Implements retry logic with exponential backoff for transient failures.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ingestion.config import IngestionConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Retry configuration
|
||||
MAX_RETRIES = 3
|
||||
BASE_DELAY_SECONDS = 1.0
|
||||
MAX_DELAY_SECONDS = 30.0
|
||||
BACKOFF_FACTOR = 2.0
|
||||
|
||||
|
||||
class LLMError(Exception):
|
||||
"""Raised when an LLM call fails after all retries."""
|
||||
|
||||
def __init__(self, provider: str, model: str, message: str):
|
||||
self.provider = provider
|
||||
self.model = model
|
||||
super().__init__(
|
||||
f"LLM call failed [{provider}/{model}]: {message}"
|
||||
)
|
||||
|
||||
|
||||
class LLMClient:
|
||||
"""Model-agnostic LLM client using LiteLLM."""
|
||||
|
||||
def __init__(self, config: IngestionConfig):
|
||||
self.model = config.llm_model_string
|
||||
self.provider = config.llm_provider
|
||||
self.config = config
|
||||
|
||||
def complete(
|
||||
self,
|
||||
messages: list[dict[str, str]],
|
||||
response_format: type[BaseModel] | None = None,
|
||||
) -> str:
|
||||
"""Send a completion request with retry logic.
|
||||
|
||||
Args:
|
||||
messages: List of message dicts with 'role' and 'content' keys.
|
||||
response_format: Optional Pydantic model for structured output.
|
||||
|
||||
Returns:
|
||||
The assistant's response content as a string.
|
||||
|
||||
Raises:
|
||||
LLMError: If all retries are exhausted.
|
||||
"""
|
||||
import litellm
|
||||
|
||||
last_error: Exception | None = None
|
||||
|
||||
for attempt in range(1, MAX_RETRIES + 1):
|
||||
try:
|
||||
kwargs: dict[str, Any] = {
|
||||
"model": self.model,
|
||||
"messages": messages,
|
||||
}
|
||||
|
||||
if response_format is not None:
|
||||
kwargs["response_format"] = response_format
|
||||
|
||||
response = litellm.completion(**kwargs)
|
||||
content = response.choices[0].message.content
|
||||
return content or ""
|
||||
|
||||
except (
|
||||
litellm.RateLimitError,
|
||||
litellm.Timeout,
|
||||
litellm.ServiceUnavailableError,
|
||||
litellm.InternalServerError,
|
||||
) as e:
|
||||
last_error = e
|
||||
delay = min(
|
||||
BASE_DELAY_SECONDS * (BACKOFF_FACTOR ** (attempt - 1)),
|
||||
MAX_DELAY_SECONDS,
|
||||
)
|
||||
logger.warning(
|
||||
"LLM call attempt %d/%d failed (%s: %s), retrying in %.1fs",
|
||||
attempt,
|
||||
MAX_RETRIES,
|
||||
type(e).__name__,
|
||||
str(e),
|
||||
delay,
|
||||
)
|
||||
time.sleep(delay)
|
||||
|
||||
except Exception as e:
|
||||
# Non-retryable error
|
||||
raise LLMError(
|
||||
provider=self.provider,
|
||||
model=self.model,
|
||||
message=f"{type(e).__name__}: {e}",
|
||||
) from e
|
||||
|
||||
# All retries exhausted
|
||||
raise LLMError(
|
||||
provider=self.provider,
|
||||
model=self.model,
|
||||
message=f"All {MAX_RETRIES} retries exhausted. Last error: {last_error}",
|
||||
)
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Nextcloud WebDAV inbox sync.
|
||||
|
||||
Polls a Nextcloud folder via WebDAV (PROPFIND/GET) for new files,
|
||||
downloads them for processing, and moves processed files to a processed/ subfolder.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
from urllib.parse import unquote, urljoin
|
||||
|
||||
import httpx
|
||||
|
||||
from ingestion.config import IngestionConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# WebDAV XML namespace
|
||||
DAV_NS = "DAV:"
|
||||
|
||||
|
||||
class NextcloudInboxSync:
|
||||
"""Sync files from a Nextcloud WebDAV folder."""
|
||||
|
||||
def __init__(self, config: IngestionConfig):
|
||||
self.base_url = config.nextcloud_inbox_url.rstrip("/") + "/"
|
||||
self.user = config.nextcloud_inbox_user
|
||||
self.password = config.nextcloud_inbox_password
|
||||
self.local_inbox = Path(config.inbox_dir)
|
||||
self.local_inbox.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self._client = httpx.Client(
|
||||
auth=(self.user, self.password),
|
||||
timeout=30.0,
|
||||
)
|
||||
|
||||
def poll(self) -> list[Path]:
|
||||
"""Poll Nextcloud inbox for files and download them.
|
||||
|
||||
Returns:
|
||||
List of local paths to downloaded files.
|
||||
"""
|
||||
try:
|
||||
filenames = self._list_files()
|
||||
except Exception as e:
|
||||
logger.error("Failed to list Nextcloud inbox: %s", e)
|
||||
return []
|
||||
|
||||
if not filenames:
|
||||
logger.info("Nextcloud inbox is empty.")
|
||||
return []
|
||||
|
||||
downloaded: list[Path] = []
|
||||
for filename in filenames:
|
||||
local_path = self.download(filename)
|
||||
if local_path:
|
||||
downloaded.append(local_path)
|
||||
|
||||
logger.info("Downloaded %d file(s) from Nextcloud.", len(downloaded))
|
||||
return downloaded
|
||||
|
||||
def _list_files(self) -> list[str]:
|
||||
"""PROPFIND to list files in the inbox folder."""
|
||||
propfind_body = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<d:propfind xmlns:d="DAV:">
|
||||
<d:prop>
|
||||
<d:resourcetype/>
|
||||
<d:getcontentlength/>
|
||||
</d:prop>
|
||||
</d:propfind>"""
|
||||
|
||||
response = self._client.request(
|
||||
"PROPFIND",
|
||||
self.base_url,
|
||||
content=propfind_body,
|
||||
headers={
|
||||
"Content-Type": "application/xml",
|
||||
"Depth": "1",
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
# Parse WebDAV XML response
|
||||
root = ET.fromstring(response.text)
|
||||
filenames: list[str] = []
|
||||
|
||||
for resp_elem in root.findall(f"{{{DAV_NS}}}response"):
|
||||
href = resp_elem.findtext(f"{{{DAV_NS}}}href", "")
|
||||
# Skip the folder itself (collections)
|
||||
resourcetype = resp_elem.find(
|
||||
f".//{{{DAV_NS}}}resourcetype/{{{DAV_NS}}}collection"
|
||||
)
|
||||
if resourcetype is not None:
|
||||
continue
|
||||
|
||||
# Extract filename from href
|
||||
filename = unquote(href.rstrip("/").split("/")[-1])
|
||||
if filename and not filename.startswith("."):
|
||||
filenames.append(filename)
|
||||
|
||||
return filenames
|
||||
|
||||
def download(self, filename: str) -> Path | None:
|
||||
"""Download a file from Nextcloud inbox.
|
||||
|
||||
Args:
|
||||
filename: Name of the file to download.
|
||||
|
||||
Returns:
|
||||
Local path to the downloaded file, or None on failure.
|
||||
"""
|
||||
url = urljoin(self.base_url, filename)
|
||||
try:
|
||||
response = self._client.get(url)
|
||||
response.raise_for_status()
|
||||
|
||||
local_path = self.local_inbox / filename
|
||||
local_path.write_bytes(response.content)
|
||||
logger.info("Downloaded: %s", filename)
|
||||
return local_path
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Failed to download %s: %s", filename, e)
|
||||
return None
|
||||
|
||||
def mark_processed(self, filename: str) -> bool:
|
||||
"""Move a processed file to the processed/ subfolder on Nextcloud.
|
||||
|
||||
Args:
|
||||
filename: Name of the file to move.
|
||||
|
||||
Returns:
|
||||
True if move succeeded, False otherwise.
|
||||
"""
|
||||
source_url = urljoin(self.base_url, filename)
|
||||
dest_url = urljoin(self.base_url, f"processed/{filename}")
|
||||
|
||||
try:
|
||||
# Ensure processed/ folder exists (MKCOL)
|
||||
processed_url = urljoin(self.base_url, "processed/")
|
||||
self._client.request("MKCOL", processed_url)
|
||||
|
||||
# MOVE the file
|
||||
response = self._client.request(
|
||||
"MOVE",
|
||||
source_url,
|
||||
headers={"Destination": dest_url, "Overwrite": "T"},
|
||||
)
|
||||
if response.status_code in (201, 204):
|
||||
logger.info("Marked processed: %s", filename)
|
||||
return True
|
||||
else:
|
||||
logger.warning(
|
||||
"MOVE returned %d for %s", response.status_code, filename
|
||||
)
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("Failed to mark %s as processed: %s", filename, e)
|
||||
return False
|
||||
@@ -0,0 +1,74 @@
|
||||
"""OrgMyLife API client for creating tasks from extracted action items.
|
||||
|
||||
POST to /api/tasks with title, source_url, description.
|
||||
Handles unreachable API gracefully (log warning, continue).
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OrgMyLifeClient:
|
||||
"""HTTP client for OrgMyLife task creation."""
|
||||
|
||||
def __init__(self, base_url: str, api_key: str):
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.api_key = api_key
|
||||
|
||||
def create_task(
|
||||
self,
|
||||
title: str,
|
||||
source_url: str,
|
||||
description: str = "",
|
||||
priority: int = 4,
|
||||
) -> dict:
|
||||
"""Create a task in OrgMyLife.
|
||||
|
||||
Args:
|
||||
title: Task title (action item description).
|
||||
source_url: Reference back to the source note.
|
||||
description: Optional additional description.
|
||||
priority: Task priority (1-5, default 4).
|
||||
|
||||
Returns:
|
||||
Response data dict, or empty dict on failure.
|
||||
"""
|
||||
url = f"{self.base_url}/api/tasks"
|
||||
payload = {
|
||||
"title": title,
|
||||
"source_url": source_url,
|
||||
"description": description,
|
||||
"priority": priority,
|
||||
}
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
try:
|
||||
response = httpx.post(
|
||||
url,
|
||||
json=payload,
|
||||
headers=headers,
|
||||
timeout=10.0,
|
||||
)
|
||||
response.raise_for_status()
|
||||
logger.info("Created OrgMyLife task: %s", title[:50])
|
||||
return response.json()
|
||||
|
||||
except httpx.ConnectError:
|
||||
logger.warning(
|
||||
"OrgMyLife API unreachable at %s — skipping task creation", self.base_url
|
||||
)
|
||||
return {}
|
||||
except httpx.HTTPStatusError as e:
|
||||
logger.warning(
|
||||
"OrgMyLife API error %d: %s", e.response.status_code, e.response.text[:200]
|
||||
)
|
||||
return {}
|
||||
except Exception as e:
|
||||
logger.warning("OrgMyLife task creation failed: %s", e)
|
||||
return {}
|
||||
Reference in New Issue
Block a user