Initial monorepo structure
This commit is contained in:
@@ -0,0 +1,13 @@
|
||||
# =============================================================================
|
||||
# Root-Level git-crypt-Regeln
|
||||
# Kontextspezifische Regeln befinden sich in den jeweiligen Kontextordnern
|
||||
# (privat/.gitattributes, dhive/.gitattributes, bahn/.gitattributes)
|
||||
# =============================================================================
|
||||
|
||||
# Shared Secrets (falls vorhanden) mit Basis-Filter verschlüsseln
|
||||
shared/.env filter=git-crypt diff=git-crypt
|
||||
shared/**/*.pem filter=git-crypt diff=git-crypt
|
||||
shared/**/*.key filter=git-crypt diff=git-crypt
|
||||
|
||||
# Maschinenkontext-Konfiguration enthält keine Secrets, nur Mapping-Informationen
|
||||
# (Schlüssel werden aus Keyring/Passwort-Manager geladen, nicht aus Dateien)
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
# =============================================================================
|
||||
# Monorepo .gitignore
|
||||
# Nur Build-Artefakte, Caches und temporäre Dateien.
|
||||
# Secrets werden NICHT ausgeschlossen – sie werden via git-crypt verschlüsselt.
|
||||
# =============================================================================
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
*.egg-info/
|
||||
*.egg
|
||||
dist/
|
||||
build/
|
||||
.eggs/
|
||||
*.whl
|
||||
|
||||
# Virtual Environments
|
||||
.venv/
|
||||
venv/
|
||||
ENV/
|
||||
env/
|
||||
|
||||
# IDE / Editor
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
.project
|
||||
.classpath
|
||||
.settings/
|
||||
|
||||
# OS-Dateien
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
Desktop.ini
|
||||
|
||||
# Build-Artefakte
|
||||
*.o
|
||||
*.a
|
||||
*.lib
|
||||
*.dll
|
||||
*.dylib
|
||||
|
||||
# Test & Coverage
|
||||
.coverage
|
||||
htmlcov/
|
||||
.pytest_cache/
|
||||
.hypothesis/
|
||||
.tox/
|
||||
.nox/
|
||||
|
||||
# Node.js (falls relevant für Tools)
|
||||
node_modules/
|
||||
npm-debug.log*
|
||||
yarn-error.log*
|
||||
|
||||
# Logs (außer Audit-Logs)
|
||||
*.log
|
||||
!.audit/*.log
|
||||
|
||||
# Temporäre Dateien
|
||||
*.tmp
|
||||
*.temp
|
||||
*.bak
|
||||
*.orig
|
||||
|
||||
# Docker
|
||||
.docker/
|
||||
|
||||
# Misc
|
||||
.cache/
|
||||
@@ -0,0 +1,2 @@
|
||||
mcp[cli]>=1.0.0
|
||||
httpx>=0.27.0
|
||||
@@ -0,0 +1,230 @@
|
||||
"""OrgMyLife MCP Server — provides tools to interact with the OrgMyLife API.
|
||||
|
||||
Tools:
|
||||
- list_projects: Get all project tasks from the kanban board
|
||||
- update_project: Move a project task to a different status column
|
||||
- create_project: Create a new project task
|
||||
- list_tasks: Get open tasks (with optional filters)
|
||||
- list_calls: Get pending call items
|
||||
"""
|
||||
|
||||
import os
|
||||
import httpx
|
||||
from mcp.server.fastmcp import FastMCP
|
||||
|
||||
# Configuration from environment
|
||||
BASE_URL = os.environ.get("ORGMYLIFE_BASE_URL", "https://api.andreknie.de")
|
||||
API_SECRET = os.environ.get("ORGMYLIFE_API_SECRET", "")
|
||||
|
||||
mcp = FastMCP("orgmylife-api", instructions="OrgMyLife project management API. Use these tools to read and update the projects board, task list, and call list.")
|
||||
|
||||
|
||||
def _headers():
|
||||
"""Build auth headers for the OrgMyLife API."""
|
||||
return {
|
||||
"Authorization": f"Bearer {API_SECRET}",
|
||||
"X-API-Key": API_SECRET,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
|
||||
def _get(path: str, params: dict = None) -> dict:
|
||||
"""Make an authenticated GET request to the OrgMyLife API."""
|
||||
r = httpx.get(f"{BASE_URL}{path}", headers=_headers(), params=params, timeout=15)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
|
||||
def _post(path: str, json_body: dict = None) -> dict:
|
||||
"""Make an authenticated POST request to the OrgMyLife API."""
|
||||
r = httpx.post(f"{BASE_URL}{path}", headers=_headers(), json=json_body, timeout=15)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
|
||||
def _put(path: str, json_body: dict = None) -> dict:
|
||||
"""Make an authenticated PUT request to the OrgMyLife API."""
|
||||
r = httpx.put(f"{BASE_URL}{path}", headers=_headers(), json=json_body, timeout=15)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
|
||||
def _delete(path: str) -> dict:
|
||||
"""Make an authenticated DELETE request to the OrgMyLife API."""
|
||||
r = httpx.delete(f"{BASE_URL}{path}", headers=_headers(), timeout=15)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
|
||||
# --- Projects Board ---
|
||||
|
||||
@mcp.tool()
|
||||
def list_projects() -> str:
|
||||
"""Get all project tasks from the kanban board, grouped by status column."""
|
||||
data = _get("/api/projects")
|
||||
tasks = data.get("tasks", [])
|
||||
if not tasks:
|
||||
return "No project tasks found."
|
||||
|
||||
# Group by status
|
||||
columns = {}
|
||||
for t in tasks:
|
||||
status = t.get("status", "backlog")
|
||||
columns.setdefault(status, []).append(t)
|
||||
|
||||
lines = []
|
||||
status_order = ["backlog", "ready", "in_progress", "blocked", "review", "done"]
|
||||
status_labels = {
|
||||
"backlog": "📋 Backlog",
|
||||
"ready": "🟢 Ready",
|
||||
"in_progress": "⚙️ In Progress",
|
||||
"blocked": "🚫 Blocked",
|
||||
"review": "👁 Review",
|
||||
"done": "✅ Done",
|
||||
}
|
||||
for status in status_order:
|
||||
items = columns.get(status, [])
|
||||
lines.append(f"\n## {status_labels.get(status, status)} ({len(items)})")
|
||||
if items:
|
||||
for t in items:
|
||||
repo = f" [{t['repo']}]" if t.get("repo") else ""
|
||||
lines.append(f" - #{t['id']}: {t['title']}{repo}")
|
||||
else:
|
||||
lines.append(" (empty)")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def update_project(task_id: int, status: str) -> str:
|
||||
"""Move a project task to a different status column.
|
||||
|
||||
Args:
|
||||
task_id: The project task ID (from list_projects)
|
||||
status: Target column — one of: backlog, ready, in_progress, blocked, review, done
|
||||
"""
|
||||
valid = ["backlog", "ready", "in_progress", "blocked", "review", "done"]
|
||||
if status not in valid:
|
||||
return f"Invalid status '{status}'. Must be one of: {', '.join(valid)}"
|
||||
result = _put(f"/api/projects/{task_id}", {"status": status})
|
||||
return f"Moved project task #{task_id} to '{status}'. Result: {result.get('status', 'unknown')}"
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def create_project(title: str, repo: str = "") -> str:
|
||||
"""Create a new project task on the kanban board (starts in Backlog).
|
||||
|
||||
Args:
|
||||
title: Task title
|
||||
repo: Repository name (e.g., "OrgMyLife", "AI-Orchestrator")
|
||||
"""
|
||||
result = _post("/api/projects/create", {"title": title, "repo": repo})
|
||||
if result.get("status") == "success":
|
||||
return f"Created project task: '{title}' (ID: {result.get('id')})"
|
||||
return f"Failed to create project task: {result}"
|
||||
|
||||
|
||||
# --- Tasks ---
|
||||
|
||||
@mcp.tool()
|
||||
def list_tasks(filter_origin: str = "", filter_label: str = "", sort: str = "priority") -> str:
|
||||
"""Get open tasks from the task list.
|
||||
|
||||
Args:
|
||||
filter_origin: Filter by source — "manual", "nextcloud_todo", "dhive_email", "gmail", or "" for all
|
||||
filter_label: Filter by label name, or "" for all
|
||||
sort: Sort order — "priority", "due_date", "created", or "" for manual order
|
||||
"""
|
||||
params = {}
|
||||
if sort:
|
||||
params["sort"] = sort
|
||||
if filter_origin:
|
||||
params["filter_origin"] = filter_origin
|
||||
if filter_label:
|
||||
params["filter_label"] = filter_label
|
||||
|
||||
data = _get("/api/tasks", params)
|
||||
tasks = data.get("tasks", [])
|
||||
total = data.get("total", len(tasks))
|
||||
|
||||
if not tasks:
|
||||
return "No open tasks."
|
||||
|
||||
lines = [f"**{total} open tasks:**\n"]
|
||||
for t in tasks[:30]: # Limit to 30 to avoid huge output
|
||||
priority = t.get("priority", "")
|
||||
due = f" (due: {t['due_date']})" if t.get("due_date") else ""
|
||||
labels = f" [{t['labels']}]" if t.get("labels") else ""
|
||||
conflict = " ⚠️CONFLICT" if t.get("has_conflict") else ""
|
||||
lines.append(f" #{t['id']}: {t['title']} — {priority}{due}{labels}{conflict}")
|
||||
|
||||
if total > 30:
|
||||
lines.append(f"\n ... and {total - 30} more")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# --- Calls ---
|
||||
|
||||
@mcp.tool()
|
||||
def list_calls() -> str:
|
||||
"""Get all pending call items from the call list."""
|
||||
data = _get("/api/calls")
|
||||
calls = data.get("calls", [])
|
||||
|
||||
if not calls:
|
||||
return "No pending calls."
|
||||
|
||||
lines = [f"**{len(calls)} pending calls:**\n"]
|
||||
for c in calls:
|
||||
phone = f" ({c['phone_number']})" if c.get("phone_number") else ""
|
||||
lines.append(f" #{c['id']}: {c['person']}{phone} — {c['reason']}")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# --- Daily Digest ---
|
||||
|
||||
@mcp.tool()
|
||||
def get_digest() -> str:
|
||||
"""Get the daily digest summary (overdue, due today, top priorities, schedule)."""
|
||||
data = _get("/api/digest")
|
||||
|
||||
lines = [f"**{data.get('greeting', 'Hello')}** — {data.get('date', '')}"]
|
||||
lines.append(f"Total open tasks: {data.get('total_open', '?')}\n")
|
||||
|
||||
overdue = data.get("overdue", [])
|
||||
if overdue:
|
||||
lines.append(f"⚠️ **Overdue ({len(overdue)}):**")
|
||||
for t in overdue:
|
||||
lines.append(f" - {t['title']} ({t['days_overdue']}d overdue)")
|
||||
|
||||
due_today = data.get("due_today", [])
|
||||
if due_today:
|
||||
lines.append(f"\n📅 **Due Today ({len(due_today)}):**")
|
||||
for t in due_today:
|
||||
lines.append(f" - {t['title']}")
|
||||
|
||||
top = data.get("top_tasks", [])
|
||||
if top:
|
||||
lines.append(f"\n🎯 **Top Priorities:**")
|
||||
for t in top:
|
||||
lines.append(f" - {t['title']} ({t['priority']})")
|
||||
|
||||
events = data.get("events_today", [])
|
||||
if events:
|
||||
lines.append(f"\n📆 **Today's Schedule:**")
|
||||
for e in events:
|
||||
lines.append(f" - {e['time']} {e['title']}")
|
||||
|
||||
calls = data.get("pending_calls", [])
|
||||
if calls:
|
||||
lines.append(f"\n📞 **Calls to make ({len(calls)}):**")
|
||||
for c in calls:
|
||||
lines.append(f" - {c['person']} — {c['reason']}")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
mcp.run()
|
||||
@@ -0,0 +1,54 @@
|
||||
{
|
||||
"mcpServers": {
|
||||
"orgmylife-api": {
|
||||
"command": "python",
|
||||
"args": [
|
||||
".kiro/mcp-servers/orgmylife-api/server.py"
|
||||
],
|
||||
"env": {
|
||||
"ORGMYLIFE_BASE_URL": "https://api.andreknie.de",
|
||||
"ORGMYLIFE_API_SECRET": "!Z5l5vUcm!&qfhUuE=Nf"
|
||||
},
|
||||
"disabled": false,
|
||||
"autoApprove": [
|
||||
"list_projects",
|
||||
"list_tasks",
|
||||
"list_calls",
|
||||
"get_digest",
|
||||
"create_project",
|
||||
"update_project"
|
||||
]
|
||||
},
|
||||
"atlassian": {
|
||||
"url": "https://taros-playground-mcp-atlassian.apps.dbcs-prag.comp.db.de/mcp",
|
||||
"headers": {
|
||||
"Authorization": "Bearer NTMwMjQ0MzY2NzQ0OkpIZBEiwwYfES+c80xqOpsjVdpx",
|
||||
"X-Jira-Host": "arija-jira.jaas.service.deutschebahn.com",
|
||||
"X-Jira-Projects": "O2CO2C,O2CAAPN,O2CAABS,O2CAABN,O2CAEB,O2CADX",
|
||||
"X-Jira-Mode": "read"
|
||||
},
|
||||
"disabled": false,
|
||||
"autoApprove": [
|
||||
"search_issues",
|
||||
"get_issue",
|
||||
"get_projects",
|
||||
"get_project"
|
||||
]
|
||||
},
|
||||
"dbctx": {
|
||||
"url": "https://taros-playground-dbctx.apps.dbcs-prag.comp.db.de/mcp",
|
||||
"disabled": false,
|
||||
"autoApprove": []
|
||||
},
|
||||
"beam-mcp": {
|
||||
"command": "node",
|
||||
"args": ["beam-mcp/dist/index.js"],
|
||||
"env": {
|
||||
"BEAM_BASE_URL": "https://db.leanix.net",
|
||||
"BEAM_WORKSPACE": "beam",
|
||||
"BEAM_HEADLESS": "true"
|
||||
},
|
||||
"disabled": true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
---
|
||||
inclusion: manual
|
||||
---
|
||||
|
||||
# Check AI Tasks
|
||||
|
||||
When this skill is activated, perform the following routine:
|
||||
|
||||
## Steps
|
||||
|
||||
1. **Read `OrgMyLife/AI_TASKS.md`** to see all open AI issues (auto-updated every 30 min by GitHub Actions)
|
||||
- This file contains the issue number, title, description, and URL for each open `ai-task`
|
||||
- If the file says "No open AI tasks" — report that and stop
|
||||
|
||||
2. **For each open issue**, check if the work described has already been implemented:
|
||||
- Read the issue title to identify the task (format: `[AI] OML-XX: Task title`)
|
||||
- Check the OrgMyLife codebase to see if the feature/fix already exists
|
||||
- Check recent git commits for related work
|
||||
|
||||
3. **If the task is already done:**
|
||||
- Report it as completed
|
||||
- Remind the user to close the GitHub issue (which auto-marks it done in OrgMyLife)
|
||||
|
||||
4. **If the task is NOT done:**
|
||||
- Implement it in the OrgMyLife repo
|
||||
- Commit and push (auto-deploys via GitHub Actions)
|
||||
- Report what was done
|
||||
- Remind the user to close the issue after verifying
|
||||
|
||||
5. **Summary:** After processing all issues, give a brief status report:
|
||||
- How many issues were open
|
||||
- How many were already done
|
||||
- How many were just implemented
|
||||
- Any issues that need user input before implementation
|
||||
|
||||
## Notes
|
||||
- The OrgMyLife repo is at: `./OrgMyLife/`
|
||||
- The AI-Orchestrator repo is at: `./AI-Orchestrator/`
|
||||
- Auto-deploy is configured: push to main triggers deployment to the VPS
|
||||
- The live app is at: https://api.andreknie.de
|
||||
- Tasks are marked done by closing the GitHub issue (triggers complete-task.yml workflow)
|
||||
@@ -0,0 +1 @@
|
||||
{"specId": "e33fa6e5-89e0-4d8d-ad9e-d43d6bf23daa", "workflowType": "requirements-first", "specType": "feature"}
|
||||
@@ -0,0 +1,343 @@
|
||||
# Design Document: Call List Feature
|
||||
|
||||
## Overview
|
||||
|
||||
The Call List feature adds a dedicated phone-call tracking system to OrgMyLife. It introduces a new `CallItem` model, a CRUD REST API at `/api/calls`, a phone number extraction service for German formats, a new "📞 Calls" frontend tab, and integration with the existing Daily Digest.
|
||||
|
||||
The design follows existing OrgMyLife patterns: SQLAlchemy model in `app/models.py`, FastAPI routes in `app/main.py`, service logic in `app/services/`, and vanilla JS frontend in `frontend/js/app.js`.
|
||||
|
||||
## Architecture
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
subgraph Frontend
|
||||
A[Task List View] -->|📞 button click| B[POST /api/calls]
|
||||
C[Calls Tab] -->|load| D[GET /api/calls]
|
||||
C -->|mark done| E[PUT /api/calls/:id]
|
||||
C -->|delete| F[DELETE /api/calls/:id]
|
||||
end
|
||||
|
||||
subgraph Backend API
|
||||
B --> G[Call List Service]
|
||||
D --> G
|
||||
E --> G
|
||||
F --> G
|
||||
end
|
||||
|
||||
subgraph Services
|
||||
G --> H[Phone Extractor]
|
||||
G --> I[Task Completion Flow]
|
||||
J[Daily Digest Service] --> G
|
||||
end
|
||||
|
||||
subgraph Database
|
||||
G --> K[(call_items table)]
|
||||
I --> L[(tasks table)]
|
||||
end
|
||||
```
|
||||
|
||||
### Request Flow: Create Call from Task
|
||||
|
||||
1. User clicks 📞 on a task in the Task List
|
||||
2. Frontend sends `POST /api/calls` with `{ task_id: <id> }`
|
||||
3. Backend loads the task, extracts sender name from description
|
||||
4. Phone Extractor scans the task description for German phone numbers
|
||||
5. CallItem is created with person, reason, phone_number, original_task_id
|
||||
6. Response returns the created CallItem
|
||||
|
||||
### Request Flow: Complete Call with Task Resolution
|
||||
|
||||
1. User clicks "Done" on a call item that has `original_task_id`
|
||||
2. Frontend sends `PUT /api/calls/:id` with `{ status: "done", resolve_task: true }`
|
||||
3. Backend marks CallItem as done
|
||||
4. If `resolve_task` is true, backend marks the original task as "completed" and triggers existing side effects (email archiving, Nextcloud sync)
|
||||
|
||||
## Components and Interfaces
|
||||
|
||||
### 1. CallItem Model (`app/models.py`)
|
||||
|
||||
New SQLAlchemy model added to the existing models file:
|
||||
|
||||
```python
|
||||
class CallItem(Base):
|
||||
__tablename__ = "call_items"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
person = Column(String(200), nullable=False)
|
||||
reason = Column(String(500), nullable=False)
|
||||
phone_number = Column(String(30), nullable=True)
|
||||
original_task_id = Column(Integer, ForeignKey("tasks.id"), nullable=True)
|
||||
priority = Column(Integer, default=3) # 1=Very High, 2=High, 3=Medium, 4=Low
|
||||
status = Column(String, default="pending") # "pending" or "done"
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
```
|
||||
|
||||
### 2. Phone Extractor Service (`app/services/phone_extractor.py`)
|
||||
|
||||
Pure function module with no external dependencies:
|
||||
|
||||
```python
|
||||
def extract_phone_number(text: str) -> str | None:
|
||||
"""Extract the first German phone number from text.
|
||||
|
||||
Supports formats:
|
||||
- +49 xxx xxxx xxxx (international)
|
||||
- 0xxx xxxxxxx (domestic with area code)
|
||||
- (0xxx) xxxxxxx (domestic with parenthesized area code)
|
||||
|
||||
Returns the first match or None.
|
||||
"""
|
||||
|
||||
def normalize_phone_number(raw: str) -> str:
|
||||
"""Normalize a phone number string by removing separators and standardizing format."""
|
||||
|
||||
def parse_phone_number(text: str) -> dict | None:
|
||||
"""Parse a phone number string into components: country_code, area_code, subscriber."""
|
||||
```
|
||||
|
||||
### 3. Call List API Routes (`app/main.py`)
|
||||
|
||||
New endpoints following existing patterns (session auth, Pydantic models):
|
||||
|
||||
| Method | Path | Description |
|
||||
|--------|------|-------------|
|
||||
| POST | `/api/calls` | Create call item (from task or manual) |
|
||||
| GET | `/api/calls` | List pending call items |
|
||||
| PUT | `/api/calls/{id}` | Update call item (status, phone, priority) |
|
||||
| DELETE | `/api/calls/{id}` | Remove call item |
|
||||
|
||||
### 4. Pydantic Request/Response Models
|
||||
|
||||
```python
|
||||
class CallItemCreate(BaseModel):
|
||||
task_id: Optional[int] = None # Create from existing task
|
||||
person: Optional[str] = None # Required if no task_id
|
||||
reason: Optional[str] = None # Required if no task_id
|
||||
phone_number: Optional[str] = None
|
||||
priority: Optional[int] = 3
|
||||
|
||||
class CallItemUpdate(BaseModel):
|
||||
phone_number: Optional[str] = None
|
||||
priority: Optional[int] = None
|
||||
status: Optional[str] = None # "pending" or "done"
|
||||
resolve_task: Optional[bool] = False # Also complete original task?
|
||||
|
||||
class CallItemResponse(BaseModel):
|
||||
id: int
|
||||
person: str
|
||||
reason: str
|
||||
phone_number: Optional[str]
|
||||
original_task_id: Optional[int]
|
||||
priority: int
|
||||
status: str
|
||||
created_at: str
|
||||
```
|
||||
|
||||
### 5. Daily Digest Integration
|
||||
|
||||
Extend `generate_daily_digest()` in `app/services/digest.py` to query pending CallItems and include a `"pending_calls"` section in the response dict.
|
||||
|
||||
### 6. Frontend Components
|
||||
|
||||
- **Navigation**: Add "📞 Calls" button between "Projects" and "Daily Digest"
|
||||
- **Calls View** (`#calls-view`): New section with call item list, inline phone edit
|
||||
- **Task Item**: Add 📞 button to existing task action buttons
|
||||
- **Digest View**: Render the `pending_calls` section when present
|
||||
|
||||
## Data Models
|
||||
|
||||
### CallItem Table Schema
|
||||
|
||||
| Column | Type | Constraints | Default |
|
||||
|--------|------|-------------|---------|
|
||||
| id | INTEGER | PRIMARY KEY, INDEX | auto |
|
||||
| person | VARCHAR(200) | NOT NULL | — |
|
||||
| reason | VARCHAR(500) | NOT NULL | — |
|
||||
| phone_number | VARCHAR(30) | NULLABLE | NULL |
|
||||
| original_task_id | INTEGER | FK → tasks.id, NULLABLE | NULL |
|
||||
| priority | INTEGER | — | 3 |
|
||||
| status | VARCHAR | — | "pending" |
|
||||
| created_at | DATETIME(tz) | — | CURRENT_TIMESTAMP |
|
||||
|
||||
### Alembic Migration
|
||||
|
||||
New migration file `005_add_call_items.py`:
|
||||
|
||||
```python
|
||||
def upgrade() -> None:
|
||||
op.create_table('call_items',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('person', sa.String(200), nullable=False),
|
||||
sa.Column('reason', sa.String(500), nullable=False),
|
||||
sa.Column('phone_number', sa.String(30), nullable=True),
|
||||
sa.Column('original_task_id', sa.Integer(), nullable=True),
|
||||
sa.Column('priority', sa.Integer(), server_default='3'),
|
||||
sa.Column('status', sa.String(), server_default="'pending'"),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)')),
|
||||
sa.ForeignKeyConstraint(['original_task_id'], ['tasks.id']),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index(op.f('ix_call_items_id'), 'call_items', ['id'], unique=False)
|
||||
op.create_index(op.f('ix_call_items_status'), 'call_items', ['status'], unique=False)
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index(op.f('ix_call_items_status'), table_name='call_items')
|
||||
op.drop_index(op.f('ix_call_items_id'), table_name='call_items')
|
||||
op.drop_table('call_items')
|
||||
```
|
||||
|
||||
### Relationship to Existing Models
|
||||
|
||||
```mermaid
|
||||
erDiagram
|
||||
tasks {
|
||||
int id PK
|
||||
string title
|
||||
text description
|
||||
int priority
|
||||
string status
|
||||
string origin
|
||||
string source_id
|
||||
}
|
||||
call_items {
|
||||
int id PK
|
||||
string person
|
||||
string reason
|
||||
string phone_number
|
||||
int original_task_id FK
|
||||
int priority
|
||||
string status
|
||||
datetime created_at
|
||||
}
|
||||
tasks ||--o{ call_items : "original_task_id"
|
||||
```
|
||||
|
||||
## Correctness Properties
|
||||
|
||||
*A property is a characteristic or behavior that should hold true across all valid executions of a system — essentially, a formal statement about what the system should do. Properties serve as the bridge between human-readable specifications and machine-verifiable correctness guarantees.*
|
||||
|
||||
### Property 1: Create-from-task field mapping
|
||||
|
||||
*For any* valid Task with origin "gmail" or "dhive_email" and a description starting with "From: {name}", creating a CallItem from that task SHALL produce a CallItem where: person equals the extracted sender name, reason equals the task title, original_task_id equals the task ID, priority is 3, and status is "pending".
|
||||
|
||||
**Validates: Requirements 1.1, 1.2, 1.3, 1.5**
|
||||
|
||||
### Property 2: Phone number extraction from German formats
|
||||
|
||||
*For any* text string containing a valid German phone number (in +49, 0xxx, or (0xxx) format), the Phone_Extractor SHALL return a non-null result matching the first phone number occurrence in the text.
|
||||
|
||||
**Validates: Requirements 4.1, 4.2, 4.3, 4.4**
|
||||
|
||||
### Property 3: Phone number parse-format round-trip
|
||||
|
||||
*For any* valid German phone number string, parsing then formatting then parsing again SHALL produce an equivalent normalized result to parsing once.
|
||||
|
||||
**Validates: Requirements 4.6**
|
||||
|
||||
### Property 4: Call list ordering
|
||||
|
||||
*For any* set of pending CallItems with varying priorities and creation dates, the GET /api/calls response SHALL return items sorted by priority ascending, then by created_at ascending within the same priority.
|
||||
|
||||
**Validates: Requirements 2.2**
|
||||
|
||||
### Property 5: Field length truncation
|
||||
|
||||
*For any* string input to person (>200 chars), reason (>500 chars), or phone_number (>30 chars) fields, the stored value SHALL be truncated to the respective maximum length, and the stored value SHALL equal the first N characters of the input.
|
||||
|
||||
**Validates: Requirements 9.1, 9.2, 9.3**
|
||||
|
||||
### Property 6: Priority clamping
|
||||
|
||||
*For any* integer value provided as priority, the stored priority SHALL be clamped to the range [1, 4] — values below 1 become 1, values above 4 become 4.
|
||||
|
||||
**Validates: Requirements 9.4**
|
||||
|
||||
### Property 7: Status constraint
|
||||
|
||||
*For any* status value provided to a CallItem update, only "pending" and "done" SHALL be accepted; any other value SHALL be rejected or ignored.
|
||||
|
||||
**Validates: Requirements 3.3**
|
||||
|
||||
### Property 8: Digest includes all pending calls
|
||||
|
||||
*For any* non-empty set of CallItems with status "pending", the generated daily digest SHALL include a "pending_calls" section containing an entry for each pending item with its person name and reason.
|
||||
|
||||
**Validates: Requirements 7.1, 7.2**
|
||||
|
||||
### Property 9: Call completion without task resolution
|
||||
|
||||
*For any* CallItem with an original_task_id, marking it as "done" with resolve_task=false SHALL update only the CallItem status to "done" without modifying the Original_Task status.
|
||||
|
||||
**Validates: Requirements 8.3**
|
||||
|
||||
## Error Handling
|
||||
|
||||
### API Error Responses
|
||||
|
||||
| Scenario | HTTP Status | Response Body |
|
||||
|----------|-------------|---------------|
|
||||
| Unauthenticated request | 401 | `{"detail": "Not authenticated"}` |
|
||||
| CallItem not found | 404 | `{"detail": "Call item not found"}` |
|
||||
| Missing required field (person/reason) | 422 | Pydantic validation error |
|
||||
| Invalid status value | 400 | `{"detail": "Status must be 'pending' or 'done'"}` |
|
||||
| Referenced task_id not found | 404 | `{"detail": "Task not found"}` |
|
||||
|
||||
### Phone Extractor Error Handling
|
||||
|
||||
- Empty or None input → returns None (no exception)
|
||||
- Text with no phone numbers → returns None
|
||||
- Malformed numbers (too short/long) → skipped by regex, not returned
|
||||
- Non-string input → TypeError raised (caller responsibility)
|
||||
|
||||
### Task Resolution Errors
|
||||
|
||||
When completing a call with `resolve_task=true`:
|
||||
- If the original task no longer exists → log warning, complete call item only
|
||||
- If email archiving fails → log error, still mark task as completed locally
|
||||
- Follows the same error handling pattern as existing `update_task` endpoint
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
### Property-Based Tests (using Hypothesis)
|
||||
|
||||
The project uses Python, so **Hypothesis** is the property-based testing library. Each property test runs a minimum of 100 iterations.
|
||||
|
||||
Tests will be placed in `OrgMyLife/tests/test_call_list.py` and `OrgMyLife/tests/test_phone_extractor.py`.
|
||||
|
||||
**Property tests cover:**
|
||||
- Phone number extraction (Properties 2, 3)
|
||||
- Field truncation and priority clamping (Properties 5, 6)
|
||||
- Status constraint validation (Property 7)
|
||||
- Create-from-task field mapping (Property 1)
|
||||
- Call list ordering (Property 4)
|
||||
- Digest integration (Property 8)
|
||||
- Completion without resolution (Property 9)
|
||||
|
||||
Each test is tagged with: `# Feature: call-list, Property {N}: {description}`
|
||||
|
||||
Configuration: `@settings(max_examples=100)`
|
||||
|
||||
### Unit Tests (example-based)
|
||||
|
||||
- Authentication enforcement (401 for all endpoints)
|
||||
- 404 for non-existent IDs
|
||||
- DELETE removes item from database
|
||||
- POST without person/reason returns 422
|
||||
- Phone extractor returns None for text without numbers
|
||||
- Digest omits calls section when no pending items exist
|
||||
- Frontend renders tel: link for items with phone numbers
|
||||
|
||||
### Integration Tests
|
||||
|
||||
- Create call from task → verify task side effects (email archiving) on completion
|
||||
- Full flow: create from task → mark done with resolve_task=true → verify original task completed
|
||||
- Alembic migration up/down cycle
|
||||
|
||||
### Test File Structure
|
||||
|
||||
```
|
||||
OrgMyLife/tests/
|
||||
├── test_phone_extractor.py # Property tests for extraction + round-trip
|
||||
├── test_call_list.py # Property + unit tests for API and service logic
|
||||
└── test_call_digest.py # Property tests for digest integration
|
||||
```
|
||||
@@ -0,0 +1,122 @@
|
||||
# Requirements Document
|
||||
|
||||
## Introduction
|
||||
|
||||
The Call List feature adds a dedicated "📞 Calls" tab to OrgMyLife that acts as a separate funnel for phone-call tasks. Users can promote any existing task to the Call List, where it is tracked with call-specific metadata (person, phone number, reason). Phone numbers are auto-extracted from email bodies using German format patterns. The Call List integrates with the Daily Digest to surface pending calls.
|
||||
|
||||
## Glossary
|
||||
|
||||
- **Call_List_Service**: The backend service responsible for managing call items, including creation, retrieval, updates, and deletion via the REST API.
|
||||
- **Call_Item**: A database entity representing a phone call to be made, containing person name, reason, phone number, priority, status, and a reference to the originating task.
|
||||
- **Phone_Extractor**: The component that scans email body text and extracts phone numbers matching German telephone formats.
|
||||
- **Frontend_Call_View**: The UI tab ("📞 Calls") that displays pending and completed call items with sorting and action controls.
|
||||
- **Task_List_View**: The existing "All Tasks" UI view showing open tasks with action buttons.
|
||||
- **Daily_Digest_Service**: The existing service that generates the daily summary of overdue tasks, priorities, and schedule.
|
||||
- **Original_Task**: The existing Task entity from which a Call_Item is derived.
|
||||
|
||||
## Requirements
|
||||
|
||||
### Requirement 1: Create Call Item from Task
|
||||
|
||||
**User Story:** As a user, I want to move a task to the Call List by clicking a phone icon, so that I can track phone calls separately from regular tasks.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN the user clicks the phone icon button on a task in the Task_List_View, THE Call_List_Service SHALL create a new Call_Item linked to the Original_Task.
|
||||
2. WHEN a Call_Item is created from an Original_Task, THE Call_List_Service SHALL populate the Call_Item person field with the task sender name if the origin is "gmail" or "dhive_email".
|
||||
3. WHEN a Call_Item is created from an Original_Task, THE Call_List_Service SHALL populate the Call_Item reason field with the Original_Task title.
|
||||
4. WHEN a Call_Item is created from an Original_Task that has an email body, THE Phone_Extractor SHALL scan the email body and populate the Call_Item phone_number field with the first extracted number.
|
||||
5. WHEN a Call_Item is created, THE Call_List_Service SHALL assign a default priority of 3 (Medium) and a status of "pending".
|
||||
|
||||
### Requirement 2: Call List REST API
|
||||
|
||||
**User Story:** As a developer, I want a complete CRUD API for call items, so that the frontend and external integrations can manage the Call List.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN a POST request is sent to /api/calls with valid call item data, THE Call_List_Service SHALL create a new Call_Item and return the created item with its ID.
|
||||
2. WHEN a GET request is sent to /api/calls, THE Call_List_Service SHALL return all Call_Items with status "pending" ordered by priority then creation date.
|
||||
3. WHEN a PUT request is sent to /api/calls/{id} with update data, THE Call_List_Service SHALL update the specified Call_Item fields and return the updated item.
|
||||
4. WHEN a DELETE request is sent to /api/calls/{id}, THE Call_List_Service SHALL remove the specified Call_Item from the database.
|
||||
5. WHEN an unauthenticated request is sent to any /api/calls endpoint, THE Call_List_Service SHALL return HTTP 401.
|
||||
6. WHEN a request references a non-existent Call_Item ID, THE Call_List_Service SHALL return HTTP 404.
|
||||
|
||||
### Requirement 3: Call Item Data Model
|
||||
|
||||
**User Story:** As a developer, I want a dedicated CallItem model with call-specific fields, so that call metadata is stored separately from general tasks.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE Call_Item SHALL store the following fields: id (integer, primary key), person (string, required), reason (string, required), phone_number (string, nullable), original_task_id (integer, nullable foreign key to tasks), priority (integer, default 3), status (string, default "pending"), created_at (timestamp with timezone).
|
||||
2. WHEN a Call_Item references an Original_Task, THE Call_Item original_task_id SHALL be a valid foreign key to the tasks table.
|
||||
3. THE Call_Item status field SHALL accept only the values "pending" and "done".
|
||||
|
||||
### Requirement 4: Phone Number Extraction from Email
|
||||
|
||||
**User Story:** As a user, I want phone numbers automatically extracted from email bodies, so that I do not have to manually look up contact numbers.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN an email body is provided, THE Phone_Extractor SHALL identify phone numbers in the format +49 followed by 9 to 12 digits with optional spaces or hyphens.
|
||||
2. WHEN an email body is provided, THE Phone_Extractor SHALL identify phone numbers in the format 0 followed by 3 to 5 digit area code and 4 to 8 digit subscriber number with optional separators.
|
||||
3. WHEN an email body is provided, THE Phone_Extractor SHALL identify phone numbers in the format (0xxx) followed by subscriber digits with optional separators.
|
||||
4. WHEN multiple phone numbers are found in an email body, THE Phone_Extractor SHALL return the first match.
|
||||
5. WHEN no phone number is found in an email body, THE Phone_Extractor SHALL return null.
|
||||
6. FOR ALL valid German phone number strings, parsing then formatting then parsing SHALL produce an equivalent normalized result (round-trip property).
|
||||
|
||||
### Requirement 5: Call List Frontend Tab
|
||||
|
||||
**User Story:** As a user, I want a dedicated "📞 Calls" tab in the navigation, so that I can view and manage my pending calls in one place.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE Frontend_Call_View SHALL display a "📞 Calls" navigation button in the top navigation bar between "Projects" and "Daily Digest".
|
||||
2. WHEN the user navigates to the Calls tab, THE Frontend_Call_View SHALL display all pending Call_Items showing person name, reason, phone number, and priority.
|
||||
3. WHEN a Call_Item has a phone_number, THE Frontend_Call_View SHALL display the phone number as a clickable tel: link.
|
||||
4. THE Frontend_Call_View SHALL provide a button to mark a Call_Item as done.
|
||||
5. THE Frontend_Call_View SHALL provide a button to delete a Call_Item.
|
||||
6. THE Frontend_Call_View SHALL allow editing the phone_number field inline.
|
||||
7. THE Frontend_Call_View SHALL order Call_Items by priority (ascending) then by creation date (ascending).
|
||||
|
||||
### Requirement 6: Phone Icon on Task Items
|
||||
|
||||
**User Story:** As a user, I want a phone icon button on each task, so that I can quickly send any task to the Call List.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE Task_List_View SHALL display a 📞 icon button on each task item alongside the existing action buttons.
|
||||
2. WHEN the user clicks the 📞 button on a task, THE Frontend_Call_View SHALL send a POST request to /api/calls with the task data.
|
||||
3. WHEN the Call_Item is successfully created from a task, THE Frontend_Call_View SHALL display a confirmation indicator to the user.
|
||||
|
||||
### Requirement 7: Daily Digest Integration
|
||||
|
||||
**User Story:** As a user, I want to see pending calls in my Daily Digest, so that I am reminded of calls I need to make today.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN the Daily Digest is generated, THE Daily_Digest_Service SHALL include a "📞 Calls to make" section listing all Call_Items with status "pending".
|
||||
2. WHEN there are pending Call_Items, THE Daily_Digest_Service SHALL display each item with person name and reason.
|
||||
3. WHEN there are no pending Call_Items, THE Daily_Digest_Service SHALL omit the calls section from the digest.
|
||||
|
||||
### Requirement 8: Complete Call and Resolve Original Task
|
||||
|
||||
**User Story:** As a user, I want the option to mark the original task as done when I complete a call, so that I do not have to update both items manually.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN a Call_Item is marked as done and the Call_Item has an original_task_id, THE Call_List_Service SHALL offer the option to also mark the Original_Task as "completed".
|
||||
2. WHEN the user confirms marking the Original_Task as completed, THE Call_List_Service SHALL update the Original_Task status to "completed" and trigger the same side effects as the existing task completion flow (email archiving, Nextcloud sync).
|
||||
3. WHEN the user declines marking the Original_Task as completed, THE Call_List_Service SHALL only update the Call_Item status to "done" without modifying the Original_Task.
|
||||
|
||||
### Requirement 9: Input Validation
|
||||
|
||||
**User Story:** As a developer, I want call item inputs validated, so that the database contains only well-formed data.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN a POST or PUT request is sent with a person field exceeding 200 characters, THE Call_List_Service SHALL truncate the value to 200 characters.
|
||||
2. WHEN a POST or PUT request is sent with a reason field exceeding 500 characters, THE Call_List_Service SHALL truncate the value to 500 characters.
|
||||
3. WHEN a POST or PUT request is sent with a phone_number field exceeding 30 characters, THE Call_List_Service SHALL truncate the value to 30 characters.
|
||||
4. WHEN a POST or PUT request is sent with a priority value outside the range 1 to 4, THE Call_List_Service SHALL clamp the value to the nearest valid bound.
|
||||
5. WHEN a POST request is sent without a person field, THE Call_List_Service SHALL return HTTP 422 with a validation error.
|
||||
6. WHEN a POST request is sent without a reason field, THE Call_List_Service SHALL return HTTP 422 with a validation error.
|
||||
@@ -0,0 +1,194 @@
|
||||
# Implementation Plan: Call List Feature
|
||||
|
||||
## Overview
|
||||
|
||||
This plan implements the Call List feature for OrgMyLife — a dedicated phone-call tracking system with a new `CallItem` model, CRUD API, phone number extraction service, frontend "📞 Calls" tab, and Daily Digest integration. Tasks are ordered so each step builds on the previous, with no orphaned code.
|
||||
|
||||
## Tasks
|
||||
|
||||
- [x] 1. Database model and migration
|
||||
- [x] 1.1 Add CallItem model to `app/models.py`
|
||||
- Add `CallItem` SQLAlchemy class with fields: id, person, reason, phone_number, original_task_id (FK → tasks.id), priority, status, created_at
|
||||
- Follow existing model patterns (Column types, server_default for timestamps)
|
||||
- _Requirements: 3.1, 3.2, 3.3_
|
||||
|
||||
- [x] 1.2 Create Alembic migration `005_add_call_items.py`
|
||||
- Create `call_items` table with all columns, FK constraint, and indexes on id and status
|
||||
- Include `downgrade()` that drops the table and indexes
|
||||
- _Requirements: 3.1, 3.2_
|
||||
|
||||
- [x] 2. Phone extractor service
|
||||
- [x] 2.1 Create `app/services/phone_extractor.py`
|
||||
- Implement `extract_phone_number(text: str) -> str | None` with regex patterns for: +49 international, 0xxx domestic, (0xxx) parenthesized area code
|
||||
- Implement `normalize_phone_number(raw: str) -> str` to strip separators and standardize
|
||||
- Implement `parse_phone_number(text: str) -> dict | None` to decompose into country_code, area_code, subscriber
|
||||
- Return first match when multiple numbers found; return None when no match
|
||||
- _Requirements: 4.1, 4.2, 4.3, 4.4, 4.5_
|
||||
|
||||
- [ ]* 2.2 Write property test: Phone number extraction (Property 2)
|
||||
- **Property 2: Phone number extraction from German formats**
|
||||
- Generate random valid German phone numbers in +49, 0xxx, and (0xxx) formats embedded in arbitrary text
|
||||
- Assert `extract_phone_number` returns a non-null result matching the first occurrence
|
||||
- Use Hypothesis with `@settings(max_examples=100)`
|
||||
- Place in `OrgMyLife/tests/test_phone_extractor.py`
|
||||
- **Validates: Requirements 4.1, 4.2, 4.3, 4.4**
|
||||
|
||||
- [ ]* 2.3 Write property test: Phone number round-trip (Property 3)
|
||||
- **Property 3: Phone number parse-format round-trip**
|
||||
- Generate valid German phone number strings, parse → normalize → parse again
|
||||
- Assert the two parse results are equivalent
|
||||
- Use Hypothesis with `@settings(max_examples=100)`
|
||||
- Place in `OrgMyLife/tests/test_phone_extractor.py`
|
||||
- **Validates: Requirements 4.6**
|
||||
|
||||
- [x] 3. Call List API routes and service logic
|
||||
- [x] 3.1 Add Pydantic models for Call List in `app/main.py`
|
||||
- `CallItemCreate`: task_id (optional), person (optional), reason (optional), phone_number (optional), priority (default 3)
|
||||
- `CallItemUpdate`: phone_number, priority, status, resolve_task (bool)
|
||||
- `CallItemResponse`: id, person, reason, phone_number, original_task_id, priority, status, created_at
|
||||
- Implement field truncation (person≤200, reason≤500, phone_number≤30) and priority clamping [1,4] in `model_post_init`
|
||||
- _Requirements: 9.1, 9.2, 9.3, 9.4, 9.5, 9.6_
|
||||
|
||||
- [x] 3.2 Implement POST `/api/calls` endpoint
|
||||
- If `task_id` provided: load task, extract sender name from description, extract phone number via `phone_extractor`, create CallItem with person=sender, reason=task.title, original_task_id=task.id
|
||||
- If no `task_id`: require person and reason fields, return 422 if missing
|
||||
- Assign default priority=3, status="pending"
|
||||
- Require session auth (401 if unauthenticated)
|
||||
- _Requirements: 1.1, 1.2, 1.3, 1.4, 1.5, 2.1, 2.5_
|
||||
|
||||
- [x] 3.3 Implement GET `/api/calls` endpoint
|
||||
- Return all CallItems with status="pending", ordered by priority ASC then created_at ASC
|
||||
- Require session auth
|
||||
- _Requirements: 2.2, 2.5_
|
||||
|
||||
- [x] 3.4 Implement PUT `/api/calls/{id}` endpoint
|
||||
- Update specified fields (phone_number, priority, status)
|
||||
- Validate status is "pending" or "done" only; reject others with 400
|
||||
- If status="done" and resolve_task=true and original_task_id exists: mark original task as "completed" and trigger existing side effects (email archiving, Nextcloud sync)
|
||||
- If status="done" and resolve_task=false: only update CallItem
|
||||
- Return 404 if ID not found; require session auth
|
||||
- _Requirements: 2.3, 2.5, 2.6, 3.3, 8.1, 8.2, 8.3_
|
||||
|
||||
- [x] 3.5 Implement DELETE `/api/calls/{id}` endpoint
|
||||
- Remove CallItem from database
|
||||
- Return 404 if ID not found; require session auth
|
||||
- _Requirements: 2.4, 2.5, 2.6_
|
||||
|
||||
- [ ]* 3.6 Write property test: Field truncation (Property 5)
|
||||
- **Property 5: Field length truncation**
|
||||
- Generate strings longer than max lengths for person, reason, phone_number
|
||||
- Assert stored values are truncated to 200, 500, 30 chars respectively and equal the first N chars of input
|
||||
- Use Hypothesis with `@settings(max_examples=100)`
|
||||
- Place in `OrgMyLife/tests/test_call_list.py`
|
||||
- **Validates: Requirements 9.1, 9.2, 9.3**
|
||||
|
||||
- [ ]* 3.7 Write property test: Priority clamping (Property 6)
|
||||
- **Property 6: Priority clamping**
|
||||
- Generate arbitrary integers, assert stored priority is clamped to [1, 4]
|
||||
- Use Hypothesis with `@settings(max_examples=100)`
|
||||
- Place in `OrgMyLife/tests/test_call_list.py`
|
||||
- **Validates: Requirements 9.4**
|
||||
|
||||
- [ ]* 3.8 Write property test: Status constraint (Property 7)
|
||||
- **Property 7: Status constraint**
|
||||
- Generate arbitrary status strings, assert only "pending" and "done" are accepted
|
||||
- Use Hypothesis with `@settings(max_examples=100)`
|
||||
- Place in `OrgMyLife/tests/test_call_list.py`
|
||||
- **Validates: Requirements 3.3**
|
||||
|
||||
- [x] 4. Checkpoint - Ensure all tests pass
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
- [x] 5. Frontend: Calls tab and phone icon on tasks
|
||||
- [x] 5.1 Add "📞 Calls" navigation button to `frontend/index.html`
|
||||
- Insert new nav button between "Projects" and "Daily Digest"
|
||||
- Add `#calls-view` section with call items list container
|
||||
- _Requirements: 5.1_
|
||||
|
||||
- [x] 5.2 Implement Calls tab view in `frontend/js/app.js`
|
||||
- Add `loadCalls()` function: fetch GET `/api/calls`, render list with person, reason, phone (as `tel:` link), priority badge
|
||||
- Add "Mark Done" button per item (sends PUT with status="done", prompts about resolving original task if original_task_id exists)
|
||||
- Add "Delete" button per item (sends DELETE)
|
||||
- Add inline edit for phone_number field (sends PUT on blur/enter)
|
||||
- Order items by priority then creation date
|
||||
- Wire "📞 Calls" nav button to `switchTab("calls")`
|
||||
- _Requirements: 5.2, 5.3, 5.4, 5.5, 5.6, 5.7_
|
||||
|
||||
- [x] 5.3 Add 📞 icon button to task items in Task List view
|
||||
- Add phone icon button to existing task action buttons in `loadAllTasks()` render
|
||||
- On click: send POST `/api/calls` with `{ task_id: <id> }`
|
||||
- Show confirmation indicator (brief flash or text) on success
|
||||
- _Requirements: 6.1, 6.2, 6.3_
|
||||
|
||||
- [x] 5.4 Add CSS styles for Calls view in `frontend/css/style.css`
|
||||
- Style call item cards, tel: links, inline phone edit, priority badges
|
||||
- Match existing card/list styling patterns
|
||||
- _Requirements: 5.2, 5.3_
|
||||
|
||||
- [x] 6. Daily Digest integration
|
||||
- [x] 6.1 Extend `generate_daily_digest()` in `app/services/digest.py`
|
||||
- Import CallItem model
|
||||
- Query all CallItems with status="pending"
|
||||
- Add `"pending_calls"` section to response dict with person and reason for each item
|
||||
- Omit section if no pending calls exist
|
||||
- _Requirements: 7.1, 7.2, 7.3_
|
||||
|
||||
- [x] 6.2 Render pending calls in Digest frontend view
|
||||
- In `loadDigest()`, render the `pending_calls` section when present in API response
|
||||
- Display as a "📞 Calls to make" card with person name and reason per item
|
||||
- _Requirements: 7.1, 7.2_
|
||||
|
||||
- [ ]* 6.3 Write property test: Digest includes all pending calls (Property 8)
|
||||
- **Property 8: Digest includes all pending calls**
|
||||
- Generate a non-empty set of pending CallItems, run `generate_daily_digest`
|
||||
- Assert response contains `pending_calls` with an entry for each pending item (person + reason)
|
||||
- Use Hypothesis with `@settings(max_examples=100)`
|
||||
- Place in `OrgMyLife/tests/test_call_digest.py`
|
||||
- **Validates: Requirements 7.1, 7.2**
|
||||
|
||||
- [x] 7. Create-from-task logic and completion flow tests
|
||||
- [ ]* 7.1 Write property test: Create-from-task field mapping (Property 1)
|
||||
- **Property 1: Create-from-task field mapping**
|
||||
- Generate tasks with origin "gmail"/"dhive_email" and description starting with "From: {name}"
|
||||
- Assert created CallItem has: person=extracted sender, reason=task.title, original_task_id=task.id, priority=3, status="pending"
|
||||
- Use Hypothesis with `@settings(max_examples=100)`
|
||||
- Place in `OrgMyLife/tests/test_call_list.py`
|
||||
- **Validates: Requirements 1.1, 1.2, 1.3, 1.5**
|
||||
|
||||
- [ ]* 7.2 Write property test: Call list ordering (Property 4)
|
||||
- **Property 4: Call list ordering**
|
||||
- Generate sets of pending CallItems with varying priorities and creation dates
|
||||
- Assert GET `/api/calls` returns items sorted by priority ASC, then created_at ASC
|
||||
- Use Hypothesis with `@settings(max_examples=100)`
|
||||
- Place in `OrgMyLife/tests/test_call_list.py`
|
||||
- **Validates: Requirements 2.2**
|
||||
|
||||
- [ ]* 7.3 Write property test: Completion without task resolution (Property 9)
|
||||
- **Property 9: Call completion without task resolution**
|
||||
- Generate CallItems with original_task_id, mark as done with resolve_task=false
|
||||
- Assert only CallItem status changes; original task status remains unchanged
|
||||
- Use Hypothesis with `@settings(max_examples=100)`
|
||||
- Place in `OrgMyLife/tests/test_call_list.py`
|
||||
- **Validates: Requirements 8.3**
|
||||
|
||||
- [ ]* 7.4 Write unit tests for API edge cases
|
||||
- Test 401 for unauthenticated requests to all /api/calls endpoints
|
||||
- Test 404 for non-existent call item IDs
|
||||
- Test 422 for POST without person/reason (when no task_id)
|
||||
- Test DELETE removes item from database
|
||||
- Test phone extractor returns None for text without numbers
|
||||
- Place in `OrgMyLife/tests/test_call_list.py`
|
||||
- _Requirements: 2.5, 2.6, 4.5, 9.5, 9.6_
|
||||
|
||||
- [x] 8. Final checkpoint - Ensure all tests pass
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
## Notes
|
||||
|
||||
- Tasks marked with `*` are optional and can be skipped for faster MVP
|
||||
- Each task references specific requirements for traceability
|
||||
- Checkpoints ensure incremental validation
|
||||
- Property tests validate universal correctness properties from the design document (Hypothesis, min 100 examples)
|
||||
- Unit tests validate specific examples and edge cases
|
||||
- The project uses Python (FastAPI + SQLAlchemy + Alembic) for backend and vanilla HTML/CSS/JS for frontend
|
||||
- Test files: `tests/test_phone_extractor.py`, `tests/test_call_list.py`, `tests/test_call_digest.py`
|
||||
@@ -0,0 +1 @@
|
||||
{"specId": "e33fa6e5-89e0-4d8d-ad9e-d43d6bf23daa", "workflowType": "requirements-first", "specType": "feature"}
|
||||
@@ -0,0 +1,475 @@
|
||||
# Design Document: Jury Voting
|
||||
|
||||
## Overview
|
||||
|
||||
The Jury Voting app is a standalone web application that digitizes the paper-based evaluation process for the **15. UNIKAT Ideenwettbewerb**. It enables 6 jurors to score 11 teams on 5 criteria (1–10 each) via their smartphones, while a beamer/projector view shows live-updating rankings to the audience.
|
||||
|
||||
The system is designed for a single-event use case with minimal infrastructure requirements — deployable on a laptop connected to the projector, or a simple server. The architecture prioritizes simplicity, reliability, and a premium visual experience matching the Projekt-KIQ-HP design language.
|
||||
|
||||
**Key design decisions:**
|
||||
- **Monorepo structure**: Single `Jury-Voting/` directory containing both frontend and backend
|
||||
- **Express.js + JSON file backend**: Lightweight server with file-based persistence (no database needed for 6 jurors × 11 teams)
|
||||
- **Server-Sent Events (SSE)** for real-time updates to the beamer view — simpler than WebSockets, no library needed, unidirectional (server → client) which is exactly what the results view needs
|
||||
- **Optimistic UI with local fallback**: Scores are saved locally first, then synced to server. Handles brief connectivity loss gracefully.
|
||||
- **No authentication complexity**: Simple name-based juror selection (no passwords) — appropriate for a trusted, in-person event
|
||||
|
||||
## Architecture
|
||||
|
||||
```mermaid
|
||||
graph TB
|
||||
subgraph "Client Devices"
|
||||
J1[Juror Phone 1]
|
||||
J2[Juror Phone 2]
|
||||
J6[Juror Phone 6]
|
||||
B[Beamer Browser]
|
||||
A[Admin Laptop]
|
||||
end
|
||||
|
||||
subgraph "Server (single process)"
|
||||
API[Express.js REST API]
|
||||
SSE[SSE Endpoint]
|
||||
FS[JSON File Store]
|
||||
end
|
||||
|
||||
J1 -->|POST /api/scores| API
|
||||
J2 -->|POST /api/scores| API
|
||||
J6 -->|POST /api/scores| API
|
||||
A -->|Admin API calls| API
|
||||
API -->|Read/Write| FS
|
||||
API -->|Notify| SSE
|
||||
SSE -->|EventStream| B
|
||||
SSE -->|EventStream| A
|
||||
```
|
||||
|
||||
### Architecture Decisions
|
||||
|
||||
| Decision | Choice | Rationale |
|
||||
|----------|--------|-----------|
|
||||
| Backend | Express.js | Minimal setup, same language as frontend, easy to bundle |
|
||||
| Persistence | JSON file | 6×11×5 = 330 score cells — trivial data volume, no DB overhead |
|
||||
| Real-time | SSE (Server-Sent Events) | Unidirectional push is sufficient; no WebSocket library needed |
|
||||
| Frontend | React 19 + Vite | Matches Projekt-KIQ-HP stack, fast HMR for development |
|
||||
| Styling | CSS variables + glassmorphism | Reuses KIQ-HP design tokens, no CSS framework needed |
|
||||
| Deployment | docker-compose OR `npm run dev` | Flexible: production-like or quick laptop setup |
|
||||
| Offline handling | localStorage queue | Scores saved locally, synced on reconnect |
|
||||
|
||||
### Deployment Topology
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
subgraph "Event Setup (laptop or server)"
|
||||
V[Vite Dev Server :5173]
|
||||
E[Express API :3001]
|
||||
end
|
||||
|
||||
subgraph "Network (local WiFi)"
|
||||
P1[Phone 1] --> V
|
||||
P6[Phone 6] --> V
|
||||
BM[Beamer PC] --> V
|
||||
end
|
||||
|
||||
V --> E
|
||||
```
|
||||
|
||||
For production, Vite builds static assets served by Express on a single port. For development, Vite proxies API calls to Express.
|
||||
|
||||
## Components and Interfaces
|
||||
|
||||
### Route Structure
|
||||
|
||||
| Route | Component | Purpose |
|
||||
|-------|-----------|---------|
|
||||
| `/` | `JurorSelect` | Juror name selection (landing page) |
|
||||
| `/vote` | `VotingDashboard` | Team list with scoring status |
|
||||
| `/vote/:teamId` | `TeamScoring` | Score input for a specific team |
|
||||
| `/results` | `ResultsView` | Beamer-optimized live rankings |
|
||||
| `/admin` | `AdminPanel` | Session management, progress, export |
|
||||
|
||||
### Frontend Component Tree
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
App --> Router
|
||||
Router --> JurorSelect
|
||||
Router --> VotingDashboard
|
||||
Router --> TeamScoring
|
||||
Router --> ResultsView
|
||||
Router --> AdminPanel
|
||||
|
||||
VotingDashboard --> TeamCard[TeamCard × 11]
|
||||
TeamScoring --> ScoreSlider[ScoreSlider × 5]
|
||||
TeamScoring --> TeamHeader
|
||||
ResultsView --> RankingTable
|
||||
ResultsView --> SessionStatus
|
||||
AdminPanel --> ProgressMatrix
|
||||
AdminPanel --> SessionControls
|
||||
AdminPanel --> ExportButton
|
||||
```
|
||||
|
||||
### Key Components
|
||||
|
||||
#### `JurorSelect`
|
||||
- Displays 6 juror names as large tap targets (glass-panel cards)
|
||||
- On selection, stores juror identity in localStorage and navigates to `/vote`
|
||||
- Shows warning if juror is already active in another session
|
||||
|
||||
#### `VotingDashboard`
|
||||
- Lists all 11 teams as cards
|
||||
- Each card shows: team name, description snippet, scoring status (✓ complete / partial / not started)
|
||||
- Tap navigates to `/vote/:teamId`
|
||||
|
||||
#### `TeamScoring`
|
||||
- Shows team name + description at top
|
||||
- 5 score inputs (one per criterion), each a slider or number stepper (1–10)
|
||||
- Auto-saves on change (debounced 300ms)
|
||||
- Visual confirmation on save (green flash)
|
||||
- Navigation: prev/next team buttons
|
||||
|
||||
#### `ResultsView` (Beamer)
|
||||
- Full-screen, no navigation chrome
|
||||
- Animated ranking table with team names, scores, ranks
|
||||
- Updates via SSE — smooth transitions when scores change
|
||||
- Shows "Voting in Progress" or "Final Results" status
|
||||
- Designed for 1920×1080 landscape
|
||||
|
||||
#### `AdminPanel`
|
||||
- Session controls: Start/End voting session
|
||||
- Progress matrix: 6 jurors × 11 teams grid showing completion
|
||||
- Export button (CSV download)
|
||||
- Juror/team configuration (pre-populated)
|
||||
|
||||
### API Endpoints
|
||||
|
||||
| Method | Endpoint | Purpose |
|
||||
|--------|----------|---------|
|
||||
| `GET` | `/api/session` | Get current session state |
|
||||
| `POST` | `/api/session/start` | Start voting session |
|
||||
| `POST` | `/api/session/end` | End voting session |
|
||||
| `GET` | `/api/teams` | List all teams |
|
||||
| `GET` | `/api/jurors` | List all jurors |
|
||||
| `GET` | `/api/scores` | Get all scores (admin) |
|
||||
| `GET` | `/api/scores/:jurorId` | Get scores for a juror |
|
||||
| `POST` | `/api/scores` | Submit/update a score |
|
||||
| `GET` | `/api/results` | Get aggregated results |
|
||||
| `GET` | `/api/results/stream` | SSE endpoint for live updates |
|
||||
| `GET` | `/api/export/csv` | Download CSV export |
|
||||
| `GET` | `/api/progress` | Get juror completion matrix |
|
||||
|
||||
### Score Submission Payload
|
||||
|
||||
```json
|
||||
{
|
||||
"jurorId": "finke",
|
||||
"teamId": "dual",
|
||||
"criterionId": "gesamtidee",
|
||||
"score": 8
|
||||
}
|
||||
```
|
||||
|
||||
### SSE Event Format
|
||||
|
||||
```json
|
||||
{
|
||||
"event": "scores_updated",
|
||||
"data": {
|
||||
"results": [
|
||||
{ "teamId": "dual", "teamName": "DUAL", "total": 187, "average": 31.2, "rank": 1 }
|
||||
],
|
||||
"timestamp": "2026-05-19T18:30:00Z"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Data Models
|
||||
|
||||
### Session
|
||||
|
||||
```typescript
|
||||
interface Session {
|
||||
id: string;
|
||||
status: 'setup' | 'active' | 'ended';
|
||||
startedAt: string | null; // ISO timestamp
|
||||
endedAt: string | null; // ISO timestamp
|
||||
config: SessionConfig;
|
||||
}
|
||||
|
||||
interface SessionConfig {
|
||||
criteria: Criterion[];
|
||||
scaleMin: number; // default: 1
|
||||
scaleMax: number; // default: 10
|
||||
}
|
||||
```
|
||||
|
||||
### Team
|
||||
|
||||
```typescript
|
||||
interface Team {
|
||||
id: string; // kebab-case slug, e.g. "dual"
|
||||
number: number; // original number from submission
|
||||
name: string; // display name, e.g. "DUAL"
|
||||
description: string; // short description
|
||||
contact?: string; // contact person (optional)
|
||||
}
|
||||
```
|
||||
|
||||
### Juror
|
||||
|
||||
```typescript
|
||||
interface Juror {
|
||||
id: string; // lowercase name, e.g. "finke"
|
||||
name: string; // display name, e.g. "Finke"
|
||||
active: boolean; // currently has an active session
|
||||
}
|
||||
```
|
||||
|
||||
### Criterion
|
||||
|
||||
```typescript
|
||||
interface Criterion {
|
||||
id: string; // kebab-case, e.g. "gesamtidee"
|
||||
nameDE: string; // German name, e.g. "Gesamtidee"
|
||||
nameEN: string; // English name, e.g. "Overall Idea"
|
||||
}
|
||||
```
|
||||
|
||||
### Score
|
||||
|
||||
```typescript
|
||||
interface Score {
|
||||
jurorId: string;
|
||||
teamId: string;
|
||||
criterionId: string;
|
||||
value: number; // 1–10
|
||||
updatedAt: string; // ISO timestamp
|
||||
}
|
||||
```
|
||||
|
||||
### Aggregated Result
|
||||
|
||||
```typescript
|
||||
interface TeamResult {
|
||||
teamId: string;
|
||||
teamName: string;
|
||||
total: number; // sum of all juror scores across all criteria
|
||||
average: number; // total / number of jurors who scored
|
||||
rank: number; // 1-based, ties get same rank
|
||||
jurorCount: number; // how many jurors have scored this team
|
||||
complete: boolean; // all 6 jurors scored all 5 criteria
|
||||
}
|
||||
```
|
||||
|
||||
### JSON File Structure (persistence)
|
||||
|
||||
```json
|
||||
{
|
||||
"session": { "id": "unikat-2026", "status": "active", "startedAt": "...", "endedAt": null, "config": { ... } },
|
||||
"teams": [ ... ],
|
||||
"jurors": [ ... ],
|
||||
"scores": [
|
||||
{ "jurorId": "finke", "teamId": "dual", "criterionId": "gesamtidee", "value": 8, "updatedAt": "..." }
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
The entire state fits in a single JSON file (~50KB max). The server reads/writes this file atomically on each score submission.
|
||||
|
||||
### Score Aggregation Logic
|
||||
|
||||
```
|
||||
Per team:
|
||||
total = SUM(score.value) for all scores where score.teamId == team.id
|
||||
jurorCount = COUNT(DISTINCT score.jurorId) where score.teamId == team.id AND all 5 criteria submitted
|
||||
average = total / jurorCount (or total / 6 if all jurors scored)
|
||||
|
||||
Ranking:
|
||||
Sort teams by total DESC
|
||||
Assign rank 1, 2, 3... with ties getting the same rank (dense ranking)
|
||||
Example: scores [250, 240, 240, 230] → ranks [1, 2, 2, 4]
|
||||
```
|
||||
|
||||
### Offline Queue (localStorage)
|
||||
|
||||
```typescript
|
||||
interface PendingScore {
|
||||
jurorId: string;
|
||||
teamId: string;
|
||||
criterionId: string;
|
||||
value: number;
|
||||
timestamp: string;
|
||||
synced: boolean;
|
||||
}
|
||||
```
|
||||
|
||||
When offline, scores are pushed to a `pendingScores` array in localStorage. On reconnect, the app replays them to the server in order. The server uses `updatedAt` to resolve conflicts (last-write-wins).
|
||||
|
||||
|
||||
## Correctness Properties
|
||||
|
||||
*A property is a characteristic or behavior that should hold true across all valid executions of a system — essentially, a formal statement about what the system should do. Properties serve as the bridge between human-readable specifications and machine-verifiable correctness guarantees.*
|
||||
|
||||
### Property 1: Team Completion Status
|
||||
|
||||
*For any* juror and any set of submitted scores, a team's completion status SHALL be "complete" if and only if exactly 5 scores exist (one per criterion) for that juror-team pair, "partial" if between 1 and 4 scores exist, and "pending" if zero scores exist.
|
||||
|
||||
**Validates: Requirements 2.4, 7.1**
|
||||
|
||||
### Property 2: Offline Queue Preservation
|
||||
|
||||
*For any* sequence of scores submitted while offline, after synchronization completes, the server state SHALL contain all scores from the offline queue with their original values and timestamps preserved.
|
||||
|
||||
**Validates: Requirements 3.2**
|
||||
|
||||
### Property 3: Last-Write-Wins Score Resolution
|
||||
|
||||
*For any* sequence of score updates to the same (jurorId, teamId, criterionId) triple, the final persisted value SHALL equal the value of the most recent update (by timestamp).
|
||||
|
||||
**Validates: Requirements 3.3**
|
||||
|
||||
### Property 4: Score Aggregation Correctness
|
||||
|
||||
*For any* team and any set of valid scores (each 1–10), the total score SHALL equal the arithmetic sum of all individual score values, and the average SHALL equal the total divided by the number of distinct jurors who have submitted at least one score for that team.
|
||||
|
||||
**Validates: Requirements 4.2, 4.3**
|
||||
|
||||
### Property 5: Ranking Correctness with Ties
|
||||
|
||||
*For any* set of team totals, the ranking SHALL be sorted in descending order by total score, and any two teams with equal total scores SHALL receive the same rank number.
|
||||
|
||||
**Validates: Requirements 4.4, 4.5**
|
||||
|
||||
### Property 6: Session Gate
|
||||
|
||||
*For any* score submission attempt, the system SHALL accept the score if and only if the current session status is "active". Submissions when session status is "setup" or "ended" SHALL be rejected.
|
||||
|
||||
**Validates: Requirements 6.4, 6.5**
|
||||
|
||||
### Property 7: Progress Privacy
|
||||
|
||||
*For any* call to the progress endpoint, the response SHALL contain juror completion counts (number of fully-scored teams per juror) but SHALL NOT contain any individual score values.
|
||||
|
||||
**Validates: Requirements 7.3**
|
||||
|
||||
### Property 8: Export Round-Trip
|
||||
|
||||
*For any* complete set of scores, exporting to CSV and parsing the result SHALL recover: all individual scores per (juror, team, criterion), correct aggregated totals, correct averages, correct rankings, with teams as rows and (juror × criterion) as columns.
|
||||
|
||||
**Validates: Requirements 8.1, 8.2, 8.3, 8.4**
|
||||
|
||||
### Property 9: Results Display Completeness
|
||||
|
||||
*For any* TeamResult object, the rendered results view SHALL include the team name, total score, average score, and current rank.
|
||||
|
||||
**Validates: Requirements 5.4**
|
||||
|
||||
## Error Handling
|
||||
|
||||
### Network Errors (Juror Phones)
|
||||
|
||||
| Scenario | Handling |
|
||||
|----------|----------|
|
||||
| Score submission fails (timeout/5xx) | Store in localStorage queue, show subtle "offline" indicator, retry on reconnect |
|
||||
| Server unreachable | Switch to offline mode, continue scoring locally, sync when back |
|
||||
| Partial sync failure | Retry individual failed scores with exponential backoff (max 3 retries) |
|
||||
|
||||
### Data Validation Errors
|
||||
|
||||
| Scenario | Handling |
|
||||
|----------|----------|
|
||||
| Score out of range (< 1 or > 10) | Reject at client and server, show validation error |
|
||||
| Unknown jurorId or teamId | Return 400 with descriptive error message |
|
||||
| Score submission when session not active | Return 403 with "Session not active" message |
|
||||
| Duplicate score without overwrite | Return 409 with current value, client prompts for confirmation |
|
||||
|
||||
### Server Errors
|
||||
|
||||
| Scenario | Handling |
|
||||
|----------|----------|
|
||||
| JSON file write failure | Return 500, log error, scores remain in client queue |
|
||||
| JSON file corruption | Keep backup copy (`.bak`), restore from backup on startup |
|
||||
| SSE connection dropped | Client auto-reconnects with `EventSource` built-in retry |
|
||||
|
||||
### Admin Errors
|
||||
|
||||
| Scenario | Handling |
|
||||
|----------|----------|
|
||||
| End session with incomplete scores | Show warning with progress summary, require confirmation |
|
||||
| Export with no scores | Return empty CSV with headers only |
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
### Property-Based Testing
|
||||
|
||||
**Library:** [fast-check](https://github.com/dubzzz/fast-check) (JavaScript/TypeScript PBT library)
|
||||
|
||||
Property-based tests will validate the 9 correctness properties defined above. Each test runs a minimum of **100 iterations** with randomly generated inputs.
|
||||
|
||||
**Test configuration:**
|
||||
```typescript
|
||||
fc.assert(fc.property(...), { numRuns: 100 })
|
||||
```
|
||||
|
||||
**Tag format:** Each test file includes a comment referencing the design property:
|
||||
```typescript
|
||||
// Feature: jury-voting, Property 4: Score Aggregation Correctness
|
||||
```
|
||||
|
||||
**Properties to test:**
|
||||
1. Team completion status computation (pure function)
|
||||
2. Offline queue sync logic (mock server)
|
||||
3. Last-write-wins resolution (pure function)
|
||||
4. Score aggregation (pure function — sum and average)
|
||||
5. Ranking with ties (pure function — sort + rank assignment)
|
||||
6. Session gate logic (pure function — accept/reject based on status)
|
||||
7. Progress privacy (API response shape validation)
|
||||
8. Export round-trip (generate scores → export CSV → parse → compare)
|
||||
9. Results display completeness (render → check fields present)
|
||||
|
||||
### Unit Tests (Example-Based)
|
||||
|
||||
Unit tests cover specific scenarios, edge cases, and integration points:
|
||||
|
||||
- **JurorSelect**: Renders 6 names, handles selection, shows conflict warning
|
||||
- **TeamScoring**: Renders 5 criteria, auto-saves on change, shows confirmation
|
||||
- **VotingDashboard**: Shows correct status icons for complete/partial/pending teams
|
||||
- **ResultsView**: Shows "Voting in Progress" vs "Final Results" based on session state
|
||||
- **AdminPanel**: Session start/end controls, export button
|
||||
- **API endpoints**: Correct HTTP status codes, validation errors, auth checks
|
||||
- **SSE**: Connection, reconnection, event parsing
|
||||
|
||||
### Integration Tests
|
||||
|
||||
- Full scoring flow: juror selects name → scores a team → results update
|
||||
- Offline/online cycle: score while offline → reconnect → verify sync
|
||||
- Session lifecycle: create → start → score → end → export
|
||||
- Concurrent jurors: 6 simultaneous score submissions
|
||||
|
||||
### Test Structure
|
||||
|
||||
```
|
||||
Jury-Voting/
|
||||
├── src/
|
||||
│ ├── __tests__/ # Unit tests (co-located)
|
||||
│ └── ...
|
||||
├── server/
|
||||
│ ├── __tests__/ # Server unit tests
|
||||
│ └── ...
|
||||
└── tests/
|
||||
├── properties/ # Property-based tests
|
||||
│ ├── aggregation.property.test.ts
|
||||
│ ├── ranking.property.test.ts
|
||||
│ ├── completion.property.test.ts
|
||||
│ ├── session-gate.property.test.ts
|
||||
│ ├── offline-queue.property.test.ts
|
||||
│ ├── last-write-wins.property.test.ts
|
||||
│ ├── progress-privacy.property.test.ts
|
||||
│ ├── export-roundtrip.property.test.ts
|
||||
│ └── results-display.property.test.ts
|
||||
└── integration/ # Integration tests
|
||||
├── scoring-flow.test.ts
|
||||
└── session-lifecycle.test.ts
|
||||
```
|
||||
|
||||
### Test Runner
|
||||
|
||||
- **Vitest** — fast, Vite-native, supports TypeScript, works with fast-check
|
||||
- Run: `npx vitest --run` (single execution, no watch mode)
|
||||
@@ -0,0 +1,192 @@
|
||||
# Requirements Document
|
||||
|
||||
## Introduction
|
||||
|
||||
This feature replaces the paper-based jury evaluation process (Bewertungsbogen and Auswertungstabelle) for the **15. UNIKAT Ideenwettbewerb** with a digital, mobile-friendly voting application. A panel of **6 jurors** evaluates **11 teams** via their smartphones during the Prämierungsfeier on **19. Mai 2026, 18 Uhr** at **Science Park Kassel, Universitätsplatz 12** (hosted by Universität Kassel). Results are displayed in real time on a beamer/projector for the audience.
|
||||
|
||||
**Scoring system:** 5 equally-weighted criteria, each scored on a 1–10 scale (configurable). Maximum score per juror per team: 50. Aggregation: simple sum across all jurors, average = total / number of jurors.
|
||||
|
||||
**Visual design basis:** Projekt-KIQ-HP (React 19 + Vite) — dark theme (#0a0a0a background), neon green accents (#00FF00), glassmorphism panels, Outfit font, lucide-react icons. Premium event look.
|
||||
|
||||
**Tech stack:** React 19 + Vite, react-router-dom, lucide-react icons. NO Supabase — use a lightweight real-time backend (e.g., simple JSON server, or localStorage + polling for MVP since only 6 jurors).
|
||||
|
||||
**Reference documents (in repo):**
|
||||
- `Jury-Voting/Teams/` — folder containing team/project descriptions (11 teams)
|
||||
- `Jury-Voting/UNIKAT_IDEENWETTBEWERB_2026_Einladung_Prämierungsfeier.pdf` — event invitation
|
||||
- `Jury-Voting/Bewertungsbogen Jury 2026.doc` — the paper scoring form (5 criteria, 1–10 scale)
|
||||
- `Jury-Voting/Auswertungstabelle 2026.xlsx` — the paper results table (simple sum aggregation)
|
||||
|
||||
## Glossary
|
||||
|
||||
- **Voting_App**: The web application that jurors and the audience interact with
|
||||
- **Juror**: A registered panel member who submits scores for each team (6 jurors this year: Cielejewski, Finke, Freyer, Knie, Meyer, Trieschmann)
|
||||
- **Team**: A team/product/concept being evaluated (11 total in the current event)
|
||||
- **Score**: A numeric rating (1–10) a juror assigns to a team for a given criterion
|
||||
- **Criterion**: A dimension on which a team is evaluated (5 criteria: Gesamtidee, Kundennutzen, Markt, Realisierbarkeit, Präsentation)
|
||||
- **Voting_Session**: A time-bounded event during which jurors submit their scores
|
||||
- **Results_View**: The projector-optimized page that displays aggregated scores and rankings
|
||||
- **Admin**: The person who configures the session, manages jurors, and controls the results display
|
||||
|
||||
## Concrete Event Data
|
||||
|
||||
### Jurors (6)
|
||||
|
||||
| # | Name |
|
||||
|---|------|
|
||||
| 1 | Cielejewski |
|
||||
| 2 | Finke |
|
||||
| 3 | Freyer |
|
||||
| 4 | Knie |
|
||||
| 5 | Meyer |
|
||||
| 6 | Trieschmann |
|
||||
|
||||
### Teams (11)
|
||||
|
||||
| # | Team Name | Description | Contact |
|
||||
|---|-----------|-------------|---------|
|
||||
| 1 | Anima Lektor | Lektor for game language/text | Lukas Fleck |
|
||||
| 2 | Das Any-Thing | Physical AI automaton that adapts to context | — |
|
||||
| 3 | Doppelrad-Sortierer | AI-powered seed sorting machine for niche producers | — |
|
||||
| 4 | DUAL | Fully reversible wood construction system | Nicole Kozlewski, Sebastian Görs |
|
||||
| 5 | fömo | Funding management platform for social institutions | — |
|
||||
| 6 | ISOmedi | Compact rechargeable medication case with protective micro-environment | — |
|
||||
| 7 | Lian's Tempeh Chips | Tempeh-based snack product | Dr. Berlianti Puteri |
|
||||
| 8 | Lumpen | Sustainable upholstery material from old clothing textiles | — |
|
||||
| 9 | Mobile Sprinkleranlage | Mobile sprinkler system for fire protection | Björn Knoke |
|
||||
| 10 | SafeSupply OS | Lightweight supplier risk & audit system for agri-food SMEs | Jesvin Jaksan |
|
||||
| 11 | Second Nose | Digital smell detection module for smartphones | — |
|
||||
|
||||
### Scoring Criteria (5, equally weighted)
|
||||
|
||||
| # | Criterion (DE) | Criterion (EN) |
|
||||
|---|----------------|----------------|
|
||||
| 1 | Gesamtidee | Overall Idea |
|
||||
| 2 | Kundennutzen | Customer Benefit |
|
||||
| 3 | Markt | Market |
|
||||
| 4 | Realisierbarkeit | Feasibility |
|
||||
| 5 | Präsentation | Presentation |
|
||||
|
||||
**Scale:** 1–10 per criterion (configurable, default 1–10)
|
||||
**Max per juror per team:** 50 points
|
||||
**Aggregation:** Simple sum. Per-team total = sum of all juror scores. Average = total / 6.
|
||||
|
||||
### Presentation Schedule (partial, from Gesamtauswertung)
|
||||
|
||||
| Time | Team |
|
||||
|------|------|
|
||||
| 15:10–15:30 | DUAL |
|
||||
| 16:10–16:30 | fömo |
|
||||
| 16:30–16:50 | Lumpen (inferred) |
|
||||
| 16:50–17:10 | Das Any-Thing |
|
||||
| 17:10–17:30 | Doppelrad-Sortierer |
|
||||
| 17:45–18:05 | Anima Lektor |
|
||||
| 18:05–18:25 | Mobile Sprinkleranlage |
|
||||
|
||||
## Requirements
|
||||
|
||||
### Requirement 1: Juror Authentication
|
||||
|
||||
**User Story:** As a juror, I want a simple way to identify myself when I open the app on my phone, so that my votes are attributed to me without a complex login flow.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN a juror opens the Voting_App on a mobile device, THE Voting_App SHALL present a juror selection list showing the 6 registered juror names (Cielejewski, Finke, Freyer, Knie, Meyer, Trieschmann)
|
||||
2. WHEN a juror selects their name from the list, THE Voting_App SHALL grant access to the voting interface within 3 seconds
|
||||
3. IF a juror selects a name that is already in use by another active session, THEN THE Voting_App SHALL display a warning and allow override or retry
|
||||
4. THE Voting_App SHALL support 6 concurrent jurors in a single Voting_Session
|
||||
|
||||
### Requirement 2: Mobile-Friendly Voting Interface
|
||||
|
||||
**User Story:** As a juror, I want to score each team on my smartphone quickly and intuitively, so that the evaluation process is faster than the paper form.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE Voting_App SHALL render a responsive voting interface optimized for mobile viewports (320px–428px width)
|
||||
2. WHEN a juror navigates to a team, THE Voting_App SHALL display all 5 scoring criteria (Gesamtidee, Kundennutzen, Markt, Realisierbarkeit, Präsentation) with a 1–10 input for each
|
||||
3. WHEN a juror selects a score for a criterion, THE Voting_App SHALL persist the score immediately without requiring a separate save action
|
||||
4. THE Voting_App SHALL display all 11 teams in a navigable list with clear indication of which teams have been scored and which are pending
|
||||
5. WHILE a juror is scoring a team, THE Voting_App SHALL display the team name and short description
|
||||
|
||||
### Requirement 3: Score Persistence and Conflict Handling
|
||||
|
||||
**User Story:** As a juror, I want my scores to be saved reliably even if my phone loses connection briefly, so that no votes are lost.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN a juror submits a score, THE Voting_App SHALL persist the score to the server within 2 seconds under normal network conditions
|
||||
2. IF the network connection is lost, THEN THE Voting_App SHALL store scores locally and synchronize them when connectivity is restored
|
||||
3. WHEN a juror changes a previously submitted score, THE Voting_App SHALL overwrite the previous value with the new score
|
||||
4. THE Voting_App SHALL prevent a juror from submitting scores for the same criterion on the same team more than once without explicit overwrite intent
|
||||
|
||||
### Requirement 4: Real-Time Results Aggregation
|
||||
|
||||
**User Story:** As an admin, I want the results to update in real time as jurors submit scores, so that the audience sees a live leaderboard on the projector.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN a new score is persisted, THE Voting_App SHALL recalculate the aggregated result for the affected team within 1 second
|
||||
2. THE Voting_App SHALL calculate a total score per team by summing all juror scores across all 5 criteria (simple sum, max possible = 6 jurors × 5 criteria × 10 points = 300)
|
||||
3. THE Voting_App SHALL calculate an average score per team (total / number of jurors who have scored)
|
||||
4. THE Voting_App SHALL rank teams from highest to lowest total score
|
||||
5. IF two or more teams have the same total score, THEN THE Voting_App SHALL display them with equal rank
|
||||
|
||||
### Requirement 5: Beamer/Projector Results Display
|
||||
|
||||
**User Story:** As an event organizer, I want a dedicated results page optimized for projection on a large screen, so that the audience can follow the voting live.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE Results_View SHALL be optimized for landscape display at 1920×1080 resolution
|
||||
2. THE Results_View SHALL use the dark theme (#0a0a0a background) with neon green (#00FF00) accents, glassmorphism panels, and Outfit font for a premium event look
|
||||
3. WHEN aggregated scores change, THE Results_View SHALL update the displayed rankings within 2 seconds without requiring a page refresh
|
||||
4. THE Results_View SHALL display each team name, its total score, its average score, and its current rank
|
||||
5. WHILE a Voting_Session is active, THE Results_View SHALL indicate that voting is in progress
|
||||
6. WHEN a Voting_Session ends, THE Results_View SHALL display the final rankings with a clear "Final Results" indicator
|
||||
|
||||
### Requirement 6: Session Management
|
||||
|
||||
**User Story:** As an admin, I want to configure and control the voting session, so that I can set up teams, criteria, and start/stop voting at the right time.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE Voting_App SHALL provide an admin interface to create and configure a Voting_Session
|
||||
2. WHEN an admin creates a Voting_Session, THE Voting_App SHALL allow configuration of team names and descriptions (pre-populated with the 11 teams for this event)
|
||||
3. WHEN an admin creates a Voting_Session, THE Voting_App SHALL allow configuration of scoring criteria and their scale (pre-populated with 5 criteria, 1–10 scale)
|
||||
4. WHEN an admin starts a Voting_Session, THE Voting_App SHALL enable score submission for all 6 registered jurors
|
||||
5. WHEN an admin ends a Voting_Session, THE Voting_App SHALL disable further score submission and mark results as final
|
||||
6. THE Voting_App SHALL allow the admin to register jurors by name (pre-populated with the 6 jurors for this event)
|
||||
|
||||
### Requirement 7: Voting Progress Visibility
|
||||
|
||||
**User Story:** As an admin, I want to see which jurors have completed their scoring, so that I know when all votes are in and I can close the session.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE Voting_App SHALL display a progress overview showing how many of the 11 teams each juror has fully scored (all 5 criteria submitted)
|
||||
2. WHEN all 6 jurors have submitted scores for all 11 teams and all 5 criteria, THE Voting_App SHALL indicate that voting is complete
|
||||
3. THE Voting_App SHALL allow the admin to view individual juror completion status without revealing their specific scores
|
||||
|
||||
### Requirement 8: Data Export
|
||||
|
||||
**User Story:** As an admin, I want to export the final results, so that I have a digital record comparable to the previous paper-based Auswertungstabelle.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN a Voting_Session has ended, THE Voting_App SHALL provide an export of all scores in a tabular format (CSV or XLSX)
|
||||
2. THE Voting_App SHALL include in the export: team names, juror names, individual scores per criterion (Gesamtidee, Kundennutzen, Markt, Realisierbarkeit, Präsentation), and aggregated totals
|
||||
3. THE Voting_App SHALL include the final ranking and average scores in the export
|
||||
4. THE Voting_App SHALL format the export to match the structure of the original Auswertungstabelle (teams as rows, jurors × criteria as columns)
|
||||
|
||||
---
|
||||
|
||||
## Open Questions (resolved)
|
||||
|
||||
| # | Question | Resolution |
|
||||
|---|----------|------------|
|
||||
| 1 | Scoring scale | **1–10 per criterion** (configurable, default 1–10) |
|
||||
| 2 | Criteria | **5 criteria:** Gesamtidee, Kundennutzen, Markt, Realisierbarkeit, Präsentation |
|
||||
| 3 | Weighting | **Equally weighted** — simple sum, no weighting factors |
|
||||
| 4 | Aggregation | **Simple sum** across all jurors. Average = total / number of jurors |
|
||||
| 5 | Juror anonymity | **TBD** — not specified in paper documents. Default: scores visible only to admin |
|
||||
| 6 | Idea metadata | Team name + short description shown to jurors (from Teams/ folder) |
|
||||
| 7 | Comments | **Not included** — paper form has no free-text field; digital version omits it too |
|
||||
@@ -0,0 +1,294 @@
|
||||
# Implementation Plan: Jury Voting
|
||||
|
||||
## Overview
|
||||
|
||||
Build a full-stack web application for the 15. UNIKAT Ideenwettbewerb jury voting process. The app enables 6 jurors to score 11 teams on 5 criteria (1–10) via smartphones, with real-time results displayed on a beamer. Stack: React 19 + Vite frontend, Express.js backend, JSON file persistence, SSE for live updates. TypeScript throughout.
|
||||
|
||||
## Tasks
|
||||
|
||||
- [x] 1. Project scaffolding and configuration
|
||||
- [x] 1.1 Initialize Vite + React 19 + TypeScript project in `Jury-Voting/`
|
||||
- Run `npm create vite@latest` with React + TypeScript template
|
||||
- Configure `vite.config.ts` with proxy to Express backend (port 3001)
|
||||
- Add dependencies: `react-router-dom`, `lucide-react`
|
||||
- Add dev dependencies: `vitest`, `fast-check`, `@testing-library/react`, `jsdom`
|
||||
- _Requirements: 5.2 (dark theme, Outfit font, glassmorphism)_
|
||||
|
||||
- [x] 1.2 Set up Express.js backend in `Jury-Voting/server/`
|
||||
- Create `server/` directory with TypeScript configuration
|
||||
- Install: `express`, `cors`, `tsx` (for dev), `typescript`
|
||||
- Create `server/index.ts` entry point with basic Express app on port 3001
|
||||
- Add `npm run dev:server` script using `tsx watch`
|
||||
- _Requirements: 3.1 (score persistence)_
|
||||
|
||||
- [x] 1.3 Define shared TypeScript interfaces in `Jury-Voting/shared/types.ts`
|
||||
- Create `Session`, `Team`, `Juror`, `Criterion`, `Score`, `TeamResult`, `PendingScore` interfaces
|
||||
- Create `SessionConfig` interface with criteria and scale settings
|
||||
- Export all types for use by both frontend and backend
|
||||
- _Requirements: 6.2, 6.3 (configurable teams, criteria, scale)_
|
||||
|
||||
- [x] 1.4 Create seed data file `Jury-Voting/server/data/seed.json`
|
||||
- Pre-populate 11 teams from `Teams/` folder (id, number, name, description)
|
||||
- Pre-populate 6 jurors (Cielejewski, Finke, Freyer, Knie, Meyer, Trieschmann)
|
||||
- Pre-populate 5 criteria (Gesamtidee, Kundennutzen, Markt, Realisierbarkeit, Präsentation)
|
||||
- Set default session config (scale 1–10)
|
||||
- _Requirements: 6.2, 6.3, 6.6_
|
||||
|
||||
- [x] 2. Backend data layer and core logic
|
||||
- [x] 2.1 Implement JSON file store (`server/store.ts`)
|
||||
- Read/write `server/data/session.json` atomically (write to temp, rename)
|
||||
- Create `.bak` backup before each write
|
||||
- Initialize from `seed.json` if `session.json` doesn't exist
|
||||
- Expose typed read/write functions for session, teams, jurors, scores
|
||||
- _Requirements: 3.1, 3.2 (persistence, reliability)_
|
||||
|
||||
- [x] 2.2 Implement score aggregation logic (`server/aggregation.ts`)
|
||||
- `calculateTeamResult(teamId, scores[])` → total, average, jurorCount, complete
|
||||
- `calculateRankings(teamResults[])` → sorted results with dense ranking (ties get same rank)
|
||||
- `getCompletionStatus(jurorId, teamId, scores[])` → 'complete' | 'partial' | 'pending'
|
||||
- Pure functions, no side effects — easy to test
|
||||
- _Requirements: 4.2, 4.3, 4.4, 4.5_
|
||||
|
||||
- [ ]* 2.3 Write property test: Score Aggregation Correctness
|
||||
- **Property 4: Score Aggregation Correctness**
|
||||
- Generate random valid scores (1–10) for arbitrary juror/team/criterion combinations
|
||||
- Assert: total equals arithmetic sum of all score values for a team
|
||||
- Assert: average equals total / distinct jurors who scored
|
||||
- **Validates: Requirements 4.2, 4.3**
|
||||
|
||||
- [ ]* 2.4 Write property test: Ranking Correctness with Ties
|
||||
- **Property 5: Ranking Correctness with Ties**
|
||||
- Generate random team totals, compute rankings
|
||||
- Assert: rankings sorted descending by total
|
||||
- Assert: equal totals receive equal rank numbers
|
||||
- **Validates: Requirements 4.4, 4.5**
|
||||
|
||||
- [ ]* 2.5 Write property test: Team Completion Status
|
||||
- **Property 1: Team Completion Status**
|
||||
- Generate random subsets of criteria scores for a juror-team pair
|
||||
- Assert: 5 scores → 'complete', 1–4 → 'partial', 0 → 'pending'
|
||||
- **Validates: Requirements 2.4, 7.1**
|
||||
|
||||
- [x] 2.6 Implement last-write-wins conflict resolution (`server/conflict.ts`)
|
||||
- Compare timestamps on incoming score vs stored score
|
||||
- Accept score with later `updatedAt` timestamp
|
||||
- Return conflict info (409) if client sends older timestamp without force flag
|
||||
- _Requirements: 3.3, 3.4_
|
||||
|
||||
- [ ]* 2.7 Write property test: Last-Write-Wins Score Resolution
|
||||
- **Property 3: Last-Write-Wins Score Resolution**
|
||||
- Generate sequences of score updates with varying timestamps for same (jurorId, teamId, criterionId)
|
||||
- Assert: final persisted value equals the value with the most recent timestamp
|
||||
- **Validates: Requirements 3.3**
|
||||
|
||||
- [ ] 3. Backend API endpoints
|
||||
- [x] 3.1 Implement session endpoints
|
||||
- `GET /api/session` — return current session state
|
||||
- `POST /api/session/start` — set status to 'active', record startedAt
|
||||
- `POST /api/session/end` — set status to 'ended', record endedAt
|
||||
- Validate: cannot start if already active, cannot end if not active
|
||||
- _Requirements: 6.4, 6.5_
|
||||
|
||||
- [x] 3.2 Implement team and juror endpoints
|
||||
- `GET /api/teams` — return all teams
|
||||
- `GET /api/jurors` — return all jurors
|
||||
- _Requirements: 6.2, 6.6_
|
||||
|
||||
- [x] 3.3 Implement score endpoints
|
||||
- `POST /api/scores` — submit/update a score (validate: session active, score 1–10, valid jurorId/teamId/criterionId)
|
||||
- `GET /api/scores/:jurorId` — return all scores for a juror
|
||||
- `GET /api/scores` — return all scores (admin only)
|
||||
- On successful score submission, trigger SSE broadcast
|
||||
- _Requirements: 2.3, 3.1, 3.3_
|
||||
|
||||
- [ ]* 3.4 Write property test: Session Gate
|
||||
- **Property 6: Session Gate**
|
||||
- Generate score submissions with random session states ('setup', 'active', 'ended')
|
||||
- Assert: accepted if and only if session status is 'active'
|
||||
- **Validates: Requirements 6.4, 6.5**
|
||||
|
||||
- [x] 3.5 Implement results and progress endpoints
|
||||
- `GET /api/results` — return aggregated TeamResult[] with rankings
|
||||
- `GET /api/progress` — return juror completion matrix (jurorId → count of fully-scored teams)
|
||||
- `GET /api/export/csv` — generate and return CSV file
|
||||
- Progress endpoint must NOT include individual score values
|
||||
- _Requirements: 4.1, 7.1, 7.3, 8.1_
|
||||
|
||||
- [ ]* 3.6 Write property test: Progress Privacy
|
||||
- **Property 7: Progress Privacy**
|
||||
- Generate random score sets, call progress endpoint logic
|
||||
- Assert: response contains juror completion counts but no individual score values
|
||||
- **Validates: Requirements 7.3**
|
||||
|
||||
- [x] 4. SSE real-time infrastructure
|
||||
- [x] 4.1 Implement SSE endpoint (`server/sse.ts`)
|
||||
- `GET /api/results/stream` — establish SSE connection
|
||||
- Maintain list of connected clients
|
||||
- On score change, broadcast `scores_updated` event with full results payload
|
||||
- Handle client disconnect (remove from list)
|
||||
- Set appropriate headers: `Content-Type: text/event-stream`, `Cache-Control: no-cache`, `Connection: keep-alive`
|
||||
- _Requirements: 4.1, 5.3_
|
||||
|
||||
- [x] 4.2 Create frontend SSE hook (`src/hooks/useSSE.ts`)
|
||||
- Custom React hook that connects to `/api/results/stream`
|
||||
- Parse incoming events and update local state
|
||||
- Auto-reconnect on connection loss (EventSource built-in)
|
||||
- Return: `{ results, connected, lastUpdate }`
|
||||
- _Requirements: 5.3 (update within 2 seconds without page refresh)_
|
||||
|
||||
- [x] 5. Checkpoint - Backend complete
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
- [x] 6. Frontend: Juror selection and routing
|
||||
- [x] 6.1 Set up React Router with route structure
|
||||
- Configure routes: `/`, `/vote`, `/vote/:teamId`, `/results`, `/admin`
|
||||
- Create `App.tsx` with `BrowserRouter` and route definitions
|
||||
- Create layout wrapper with dark theme base styles
|
||||
- _Requirements: 1.1, 2.1_
|
||||
|
||||
- [x] 6.2 Implement `JurorSelect` page (`src/pages/JurorSelect.tsx`)
|
||||
- Fetch juror list from `GET /api/jurors`
|
||||
- Display 6 juror names as large glass-panel cards (tap targets for mobile)
|
||||
- On selection: store jurorId in localStorage, navigate to `/vote`
|
||||
- Show warning if juror already has active session (allow override)
|
||||
- _Requirements: 1.1, 1.2, 1.3, 1.4_
|
||||
|
||||
- [ ]* 6.3 Write unit tests for JurorSelect
|
||||
- Test: renders 6 juror names
|
||||
- Test: selection stores jurorId and navigates
|
||||
- Test: shows conflict warning for active juror
|
||||
- _Requirements: 1.1, 1.2, 1.3_
|
||||
|
||||
- [ ] 7. Frontend: Voting interface
|
||||
- [x] 7.1 Implement `VotingDashboard` page (`src/pages/VotingDashboard.tsx`)
|
||||
- Fetch teams from `GET /api/teams` and scores from `GET /api/scores/:jurorId`
|
||||
- Display 11 `TeamCard` components in a scrollable list
|
||||
- Each card shows: team name, description snippet, status icon (✓ complete / ◐ partial / ○ pending)
|
||||
- Tap card navigates to `/vote/:teamId`
|
||||
- _Requirements: 2.1, 2.4, 2.5_
|
||||
|
||||
- [x] 7.2 Implement `TeamScoring` page (`src/pages/TeamScoring.tsx`)
|
||||
- Display team name and description at top
|
||||
- Render 5 `ScoreSlider` components (one per criterion)
|
||||
- Each slider: range 1–10, shows current value, large touch target
|
||||
- Auto-save on change with 300ms debounce
|
||||
- Show green flash confirmation on successful save
|
||||
- Prev/Next team navigation buttons
|
||||
- _Requirements: 2.2, 2.3, 2.5_
|
||||
|
||||
- [x] 7.3 Implement offline score queue (`src/hooks/useOfflineQueue.ts`)
|
||||
- On score change: save to localStorage `pendingScores` array immediately
|
||||
- Attempt POST to `/api/scores` — on success, mark as synced
|
||||
- On network failure: keep in queue, show subtle "offline" indicator
|
||||
- On reconnect: replay pending scores in order
|
||||
- Exponential backoff for retries (max 3 attempts per score)
|
||||
- _Requirements: 3.1, 3.2_
|
||||
|
||||
- [ ]* 7.4 Write property test: Offline Queue Preservation
|
||||
- **Property 2: Offline Queue Preservation**
|
||||
- Generate random sequences of scores submitted while "offline"
|
||||
- Simulate sync, assert all scores present on server with original values/timestamps
|
||||
- **Validates: Requirements 3.2**
|
||||
|
||||
- [ ]* 7.5 Write unit tests for TeamScoring
|
||||
- Test: renders 5 criteria with correct labels
|
||||
- Test: auto-saves on slider change (debounced)
|
||||
- Test: shows confirmation flash on save
|
||||
- Test: prev/next navigation works
|
||||
- _Requirements: 2.2, 2.3_
|
||||
|
||||
- [ ] 8. Frontend: Results view (beamer)
|
||||
- [x] 8.1 Implement `ResultsView` page (`src/pages/ResultsView.tsx`)
|
||||
- Full-screen layout, no navigation chrome
|
||||
- Connect to SSE via `useSSE` hook
|
||||
- Display `RankingTable` component with animated transitions
|
||||
- Show "Voting in Progress" or "Final Results" based on session status
|
||||
- Optimized for 1920×1080 landscape projection
|
||||
- _Requirements: 5.1, 5.2, 5.3, 5.4, 5.5, 5.6_
|
||||
|
||||
- [x] 8.2 Implement `RankingTable` component (`src/components/RankingTable.tsx`)
|
||||
- Display ranked list: rank, team name, total score, average score
|
||||
- Animate rank changes (CSS transitions on position)
|
||||
- Highlight top 3 teams with special styling
|
||||
- Handle ties (same rank number displayed)
|
||||
- _Requirements: 4.4, 4.5, 5.4_
|
||||
|
||||
- [ ]* 8.3 Write property test: Results Display Completeness
|
||||
- **Property 9: Results Display Completeness**
|
||||
- Generate random TeamResult objects, render ResultsView
|
||||
- Assert: each team's name, total, average, and rank are present in rendered output
|
||||
- **Validates: Requirements 5.4**
|
||||
|
||||
- [ ] 9. Frontend: Admin panel
|
||||
- [x] 9.1 Implement `AdminPanel` page (`src/pages/AdminPanel.tsx`)
|
||||
- Session controls: Start Voting / End Voting buttons with confirmation dialogs
|
||||
- Show current session status and timestamps
|
||||
- Warning when ending session with incomplete scores
|
||||
- _Requirements: 6.1, 6.4, 6.5_
|
||||
|
||||
- [x] 9.2 Implement `ProgressMatrix` component (`src/components/ProgressMatrix.tsx`)
|
||||
- Fetch from `GET /api/progress`
|
||||
- Display 6×11 grid (jurors × teams) with completion indicators
|
||||
- Show per-juror totals (X/11 teams complete)
|
||||
- Indicate when all voting is complete (6/6 jurors × 11/11 teams)
|
||||
- No individual scores shown — only completion status
|
||||
- _Requirements: 7.1, 7.2, 7.3_
|
||||
|
||||
- [x] 9.3 Implement CSV export functionality
|
||||
- Export button calls `GET /api/export/csv` and triggers download
|
||||
- CSV format: teams as rows, (juror × criterion) as columns, plus totals/averages/ranks
|
||||
- Match structure of original Auswertungstabelle
|
||||
- _Requirements: 8.1, 8.2, 8.3, 8.4_
|
||||
|
||||
- [ ]* 9.4 Write property test: Export Round-Trip
|
||||
- **Property 8: Export Round-Trip**
|
||||
- Generate complete score sets, export to CSV, parse CSV back
|
||||
- Assert: all individual scores recovered, totals correct, averages correct, rankings correct
|
||||
- **Validates: Requirements 8.1, 8.2, 8.3, 8.4**
|
||||
|
||||
- [x] 10. Checkpoint - Core features complete
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
- [ ] 11. Styling and visual polish
|
||||
- [x] 11.1 Implement global theme and CSS variables
|
||||
- Define CSS custom properties: `--bg-primary: #0a0a0a`, `--accent: #00FF00`, glassmorphism values
|
||||
- Import Outfit font from Google Fonts
|
||||
- Set up glassmorphism utility classes (backdrop-filter, border, shadow)
|
||||
- Dark scrollbar styling, smooth transitions
|
||||
- _Requirements: 5.2_
|
||||
|
||||
- [x] 11.2 Style all components to match Projekt-KIQ-HP design
|
||||
- JurorSelect: glass-panel cards with hover/tap glow effect
|
||||
- VotingDashboard: team cards with status indicators, neon green accents
|
||||
- TeamScoring: custom slider styling (neon green track), score display
|
||||
- ResultsView: large typography, animated rank transitions, glassmorphism table
|
||||
- AdminPanel: clean grid layout, status badges
|
||||
- Mobile-first responsive design (320px–428px primary, scales up)
|
||||
- _Requirements: 2.1, 5.1, 5.2_
|
||||
|
||||
- [ ] 12. Deployment configuration
|
||||
- [x] 12.1 Create production build setup
|
||||
- Configure Vite to build static assets to `dist/`
|
||||
- Configure Express to serve static files from `dist/` in production
|
||||
- Single-port deployment: Express serves both API and frontend
|
||||
- Add `npm run build` and `npm start` scripts
|
||||
- _Requirements: all (deployment readiness)_
|
||||
|
||||
- [x] 12.2 Create `docker-compose.yml` for containerized deployment
|
||||
- Single container: Node.js serving Express + static frontend
|
||||
- Volume mount for `server/data/` (persist scores across restarts)
|
||||
- Expose port 3001 (or configurable via env)
|
||||
- Add `.dockerignore` and `Dockerfile`
|
||||
- _Requirements: all (deployment readiness)_
|
||||
|
||||
- [x] 13. Final checkpoint - All features complete
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
## Notes
|
||||
|
||||
- Tasks marked with `*` are optional property-based tests — skip for faster MVP delivery given the tight timeline (event: 19. Mai 2026)
|
||||
- Each task references specific requirements for traceability
|
||||
- Checkpoints at tasks 5, 10, and 13 ensure incremental validation
|
||||
- Property tests validate the 9 correctness properties from the design document
|
||||
- The app is designed for single-event use — simplicity over scalability
|
||||
- All seed data (teams, jurors, criteria) is pre-populated; no manual setup needed at event time
|
||||
@@ -0,0 +1 @@
|
||||
{"specId": "ff9eec9c-4758-4190-8766-743007d13d53", "workflowType": "requirements-first", "specType": "feature"}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,193 @@
|
||||
# Requirements Document
|
||||
|
||||
## Introduction
|
||||
|
||||
Dieses Dokument beschreibt die Anforderungen für die Konsolidierung von 18+ einzelnen Repositories in eine einheitliche Monorepo-Struktur. Das Ziel ist es, alle Projekte in einem System zu vereinen, dabei klare Sicherheitsgrenzen zwischen Arbeitskontexten (Privat, dhive, Bahn) zu wahren, einen übergreifenden Wissensspeicher aufzubauen, externe Repos als Read-Only einzubinden, Secrets verschlüsselt im Repository zu speichern (statt sie auszuschließen), und eine föderierte Zusammenarbeit mit verschiedenen Teams über eigenständige Team-Repositories zu ermöglichen.
|
||||
|
||||
## Glossary
|
||||
|
||||
- **Monorepo**: Ein einzelnes Repository, das mehrere Projekte und Arbeitskontexte in einer gemeinsamen Ordnerstruktur vereint
|
||||
- **Arbeitskontext**: Eine logische Sicherheitsdomäne, die zusammengehörige Projekte gruppiert (Privat, dhive, Bahn)
|
||||
- **Wissensspeicher**: Ein übergreifendes System zur Erfassung, Strukturierung und kontextübergreifenden Abfrage von Wissen (Nachfolger des NoteGraph)
|
||||
- **Sicherheitsgrenze**: Eine Konfiguration, die den Zugriff auf Dateien und Geheimnisse eines Arbeitskontexts auf berechtigte Prozesse beschränkt
|
||||
- **Externes_Repo**: Ein Repository eines Dritten oder eines Team-Kontexts, das als unveränderbare Referenz oder bidirektionale Synchronisationsquelle in die Monorepo-Struktur eingebunden wird
|
||||
- **Kontextbrücke**: Ein Mechanismus, der es erlaubt, Erkenntnisse aus einem Arbeitskontext in einem anderen nutzbar zu machen, ohne Sicherheitsgrenzen zu verletzen
|
||||
- **Ordnerstruktur**: Die hierarchische Organisation von Projekten und Modulen innerhalb des Monorepos
|
||||
- **Agent_Harness**: Ein AI-Coding-Tool oder -System (z.B. Kiro, Codex, Claude Code, Antigravity), das als Ausführungsumgebung für AI-gestützte Entwicklungsaufgaben dient
|
||||
- **Agent_Erweiterung**: Ein Plugin, Skill, Power oder ähnliches Konfigurationsartefakt, das einem Agent_Harness zusätzliche Fähigkeiten verleiht (z.B. Kiro Powers, Claude Code Slash-Commands, Codex-Plugins)
|
||||
- **Orchestrator**: Der bestehende AI-Orchestrator-Service, der Tasks aus OrgMyLife verarbeitet und an verschiedene Agent_Harnesses dispatcht, wobei die Harness-Auswahl pro Arbeitskontext oder pro Task konfigurierbar ist
|
||||
- **Secret_Encryption**: Die Verschlüsselung von Geheimnissen im Repository mittels git-crypt, SOPS oder age, sodass Secrets verschlüsselt committed werden und nur mit autorisierten Schlüsseln entschlüsselt werden können
|
||||
- **Maschinenkontext**: Die einem bestimmten Rechner zugeordnete Kombination aus autorisierten Arbeitskontexten und den entsprechenden Entschlüsselungsschlüsseln (z.B. "dhive-Rechner" hat nur dhive-Schlüssel)
|
||||
- **Team_Repo**: Ein eigenständiges Repository, das einem bestimmten Team (dhive, bahn, privat/Familie) gehört und bidirektional mit dem entsprechenden Kontextordner im Monorepo synchronisiert wird
|
||||
- **Föderierte_Struktur**: Eine Hub-and-Spoke-Topologie, in der das Monorepo als zentraler Hub fungiert und die Team-Repos als unabhängige Spokes, wobei nur der Hub-Besitzer (Andre) Zugriff auf alle Kontexte hat
|
||||
|
||||
## Requirements
|
||||
|
||||
### Requirement 1: Einheitliche Ordnerstruktur
|
||||
|
||||
**User Story:** Als Entwickler möchte ich alle meine Projekte in einer klar strukturierten Ordnerhierarchie organisiert haben, damit ich schnell navigieren und den Überblick behalten kann.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE Monorepo SHALL alle Projekte in einer flachen Kontext-Hierarchie mit exakt drei Ebenen organisieren: Ebene 1 (Kontextordner) → Ebene 2 (Projektordner) → Ebene 3 (Modulordner), wobei unterhalb der Modulebene keine weitere strukturelle Verschachtelung von Projekten oder Modulen erlaubt ist
|
||||
2. WHEN ein neues Projekt angelegt wird, THE Ordnerstruktur SHALL das Projekt genau einem der definierten Kontextordner (privat, dhive, bahn, shared) zuordnen
|
||||
3. THE Monorepo SHALL eine einheitliche Namenskonvention in kebab-case für alle Ordner auf Projekt- und Modulebene verwenden, wobei Ordnernamen ausschließlich aus Kleinbuchstaben, Ziffern und Bindestrichen bestehen und zwischen 2 und 50 Zeichen lang sein dürfen
|
||||
4. THE Ordnerstruktur SHALL genau folgende vier Kontextordner auf oberster Ebene bereitstellen: privat, dhive, bahn, shared
|
||||
5. WHEN ein Projekt Funktionalität bereitstellt, die von Projekten in mindestens zwei unterschiedlichen Arbeitskontexten genutzt wird, THE Ordnerstruktur SHALL das Projekt im shared-Kontext platzieren
|
||||
6. IF ein Projektordner-Name bereits im Ziel-Kontext existiert, THEN THE Ordnerstruktur SHALL die Anlage verweigern und eine Fehlermeldung ausgeben, die den Namenskonflikt benennt
|
||||
|
||||
### Requirement 2: Sicherheitsgrenzen zwischen Arbeitskontexten
|
||||
|
||||
**User Story:** Als Entwickler möchte ich sicherstellen, dass Geheimnisse und sensible Daten eines Arbeitskontexts nicht versehentlich in einen anderen Kontext gelangen, damit die Trennung meiner beruflichen Realitäten gewahrt bleibt.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE Sicherheitsgrenze SHALL verhindern, dass Secrets eines Arbeitskontexts für Prozesse eines anderen Arbeitskontexts über Dateisystemzugriff oder Umgebungsvariablen lesbar sind, wobei die Zugriffskontrolle auf dem Ausführungskontext des Prozesses und den verfügbaren Entschlüsselungsschlüsseln basiert
|
||||
2. THE Sicherheitsgrenze SHALL für jeden Arbeitskontext eine eigene .env-Datei im jeweiligen Kontextordner bereitstellen, die ausschließlich die Geheimnisse dieses Kontexts enthält und im Repository verschlüsselt gespeichert wird (siehe Requirement 9)
|
||||
3. WHEN ein Prozess in einem beliebigen Arbeitskontext ausgeführt wird, THE Sicherheitsgrenze SHALL den Zugriff auf entschlüsselte .env-Dateien und Secret-Dateien aller anderen Arbeitskontexte unterbinden
|
||||
4. THE Sicherheitsgrenze SHALL Secret-Dateien aller Kontexte über Verschlüsselung im Repository schützen (Encryption-at-Rest), wobei die bisherige .gitignore-basierte Ausschlussstrategie durch die kontextspezifische Verschlüsselung gemäß Requirement 9 ersetzt wird; unverschlüsselte temporäre Dateien (Build-Artefakte, Caches) werden weiterhin via .gitignore ausgeschlossen
|
||||
5. IF ein Prozess auf Secrets eines nicht-autorisierten Kontexts zugreift, THEN THE Sicherheitsgrenze SHALL einen Eintrag in eine Protokolldatei schreiben, der Zeitstempel, den anfragenden Kontext, den Zielkontext und die angefragte Ressource enthält
|
||||
6. THE Sicherheitsgrenze SHALL eine zentrale Konfigurationsdatei bereitstellen, die für jeden Arbeitskontext definiert, auf welche .env-Dateien, Secret-Dateien und shared-Ressourcen dieser Kontext zugreifen darf
|
||||
7. WHILE ein Tool aus dem shared-Bereich in einem bestimmten Arbeitskontext ausgeführt wird, THE Sicherheitsgrenze SHALL proaktiv korrekte Autorisierungszustände aufrechterhalten und dem Tool ausschließlich Zugriff auf die Secrets des aktiven Kontexts und auf explizit freigegebene shared-Ressourcen gewähren
|
||||
|
||||
### Requirement 3: Übergreifender Wissensspeicher
|
||||
|
||||
**User Story:** Als Wissensarbeiter möchte ich einen zentralen Wissensspeicher haben, der auf der bewährten ETL-Architektur der DB-Wissensdatenbank basiert und Informationen aus allen Kontexten aufnimmt, damit ich relevantes Wissen jederzeit kontextübergreifend abrufen kann.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE Wissensspeicher SHALL als Fork/Adaption der DB-Wissensdatenbank-ETL-Pipeline im shared-Bereich des Monorepos implementiert werden
|
||||
2. THE Wissensspeicher SHALL Wissensartefakte aus allen Arbeitskontexten aufnehmen und indexieren
|
||||
3. WHEN eine Suchanfrage gestellt wird, THE Wissensspeicher SHALL Ergebnisse aus allen Kontexten liefern, für die der Nutzer berechtigt ist, und die Ergebnisliste innerhalb von 5 Sekunden zurückgeben
|
||||
4. THE Wissensspeicher SHALL Wissensartefakte mit Metadaten versehen: Quellkontext, Erstelldatum, Tags und verknüpfte Projekte
|
||||
5. THE Wissensspeicher SHALL bestehende NoteGraph-Daten migrieren können, wobei die Migration als erfolgreich gilt, wenn alle Artefakte mit ihren Metadaten, Verknüpfungen und der Verzeichnisstruktur (decisions, inbox, meetings, people, projects) im Zielsystem vorhanden und über den Index auffindbar sind
|
||||
6. WHEN ein neues Wissensartefakt erstellt wird, THE Wissensspeicher SHALL automatisch den Quellkontext und das Erstelldatum erfassen
|
||||
7. THE Wissensspeicher SHALL eine Volltextsuche über alle indexierten Artefakte bereitstellen, die Treffer nach Relevanz sortiert zurückliefert
|
||||
8. THE Wissensspeicher SHALL Verknüpfungen zwischen Artefakten unterschiedlicher Kontexte als gerichtete Graph-Kanten im YAML-Frontmatter beider beteiligter Artefakte unterstützen
|
||||
9. WHILE ein Wissensartefakt einem vertraulichen Kontext zugeordnet ist, THE Wissensspeicher SHALL den Inhalt nur für berechtigte Abfragen bereitstellen und bei unberechtigtem Zugriff das Artefakt aus den Suchergebnissen ausschließen, ohne dessen Existenz offenzulegen
|
||||
10. THE Wissensspeicher SHALL die ETL-Quellstrategien der DB-Wissensdatenbank wiederverwenden (Confluence-Seiten, Webseiten, PDFs, Markdown-Dateien, GitLab-Repos)
|
||||
11. THE Wissensspeicher SHALL verarbeitetes Wissen in einer Scope-basierten Ordnerstruktur ablegen, wobei jeder Arbeitskontext einem Scope entspricht (privat, dhive, bahn, shared)
|
||||
12. WHEN eine Quelle inkrementell verarbeitet wird, THE Wissensspeicher SHALL nur geänderte oder neue Inhalte aktualisieren, wobei Änderungen anhand von Content-Hashes erkannt werden
|
||||
13. THE Wissensspeicher SHALL einen maschinenlesbaren Index bereitstellen, der den gesamten Wissensbestand beschreibt (Dokumente, Metadaten, Content-Hashes)
|
||||
14. THE Wissensspeicher SHALL die Scope-Konfiguration um die persönlichen Arbeitskontexte erweitern, wobei das Mapping Arbeitskontext → Scope in einer zentralen Konfigurationsdatei definiert wird
|
||||
15. WHERE die DB-Wissensdatenbank als aktives Bahn-Projekt eingebunden ist, THE Wissensspeicher SHALL Upstream-Änderungen der ETL-Pipeline übernehmen können, auch wenn der Wissensspeicher noch nicht vollständig implementiert ist
|
||||
16. THE Wissensspeicher SHALL ein Progressive-Disclosure-Indexsystem bereitstellen, das Agenten ermöglicht auf Schicht 1 nur eine kompakte YAML-Index-Datei pro Domäne zu lesen (Titel, Tags, Beziehungen, Kurzbeschreibungen), ohne vollständige Markdown-Dateien laden zu müssen
|
||||
17. THE Wissensspeicher SHALL jedes Wissensartefakt mit YAML-Frontmatter versehen, das mindestens Typ, Titel, Tags, Quellkontext und Verknüpfungen zu anderen Artefakten enthält
|
||||
18. WHEN ein Agent eine Suchanfrage stellt, THE Wissensspeicher SHALL über den YAML-Index die relevanten Dokumente identifizieren und nur deren Pfade zurückliefern, sodass der Agent gezielt einzelne Dateien nachladen kann
|
||||
19. THE Wissensspeicher SHALL sich am Google Open Knowledge Format (OKF) orientieren: ein Konzept pro Datei, Markdown mit YAML-Frontmatter, Cross-Links als Graph-Kanten, index.md pro Verzeichnis für Inhaltsübersichten
|
||||
20. IF die ETL-Pipeline bei der Verarbeitung einer Quelle fehlschlägt, THEN THE Wissensspeicher SHALL den Fehler mit Quellidentifikator und Zeitstempel protokollieren, bereits erfolgreich verarbeitete Artefakte beibehalten und die fehlgeschlagene Quelle beim nächsten Durchlauf erneut verarbeiten
|
||||
21. WHEN die ETL-Pipeline einen Verarbeitungsdurchlauf abschließt, THE Wissensspeicher SHALL den YAML-Index innerhalb desselben Durchlaufs aktualisieren, sodass neu verarbeitete Artefakte sofort über den Index auffindbar sind
|
||||
|
||||
### Requirement 4: Einbindung externer und geteilter Repos
|
||||
|
||||
**User Story:** Als Entwickler möchte ich externe Repositories als unveränderbare Referenz, geteilte Repos mit Upstream-Anbindung und Team-Repos mit bidirektionaler Synchronisation in mein Monorepo einbinden, damit ich deren Inhalte nutzen, bei Bedarf Änderungen zurückspielen und mit verschiedenen Teams in ihrem jeweiligen Kontext zusammenarbeiten kann.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE Monorepo SHALL externe Repositories als Read-Only-Referenz einbinden können
|
||||
2. WHEN ein externes Repo eingebunden wird, THE Monorepo SHALL dessen Inhalte als unveränderbar behandeln, indem lokale Commits und Dateiänderungen im eingebundenen Verzeichnis durch Git-Hooks oder Dateisystem-Schutzmechanismen verhindert werden
|
||||
3. THE Monorepo SHALL eine Konfigurationsdatei bereitstellen, die alle eingebundenen externen Repos mit Repository-URL, gepinnter Version (Git-Commit-SHA oder Tag), Einbindungsmodus (read-only oder upstream) und Zielpfad im Monorepo auflistet
|
||||
4. WHEN der Nutzer eine Aktualisierung eines externen Repos auslöst, THE Monorepo SHALL die lokale Kopie auf die in der Konfigurationsdatei angegebene Version aktualisieren und stets eine Ergebnismeldung anzeigen (Erfolg, Misserfolg oder 'Aktualisierung abgeschlossen – Status unbekannt')
|
||||
5. IF ein Schreibzugriff auf ein als read-only konfiguriertes Repo versucht wird (Datei anlegen, ändern oder löschen im eingebundenen Verzeichnis), THEN THE Monorepo SHALL den Zugriff blockieren und eine Fehlermeldung ausgeben, die den Read-Only-Status und den Namen des betroffenen Repos benennt
|
||||
6. THE Monorepo SHALL Git-Submodules oder Git-Subtrees für die Einbindung externer Repos verwenden
|
||||
7. WHERE ein Repo als aktives Upstream-Projekt eingebunden ist, THE Monorepo SHALL bidirektionale Synchronisation mit dem entfernten Repository ermöglichen (Pull von Upstream, Push von lokalen Änderungen), wobei bei Merge-Konflikten die Synchronisation abgebrochen, der Konflikt protokolliert und eine manuelle Auflösung ermöglicht wird; für Read-Only-Repos SHALL das Monorepo Konflikte erkennen und protokollieren, ohne die Synchronisation abzubrechen (da keine tatsächliche Synchronisation stattfindet)
|
||||
8. THE Monorepo SHALL die DB-Wissensdatenbank als aktives Bahn-Projekt mit Upstream-Anbindung zum DB-GitLab einbinden können
|
||||
9. IF eine Synchronisation mit einem Upstream-Repository fehlschlägt (Netzwerkfehler, Authentifizierungsfehler oder Merge-Konflikt), THEN THE Monorepo SHALL den lokalen Stand unverändert beibehalten, den Fehlergrund protokollieren und den Nutzer über den fehlgeschlagenen Vorgang informieren
|
||||
|
||||
### Requirement 5: Kontextübergreifender Wissenstransfer
|
||||
|
||||
**User Story:** Als Entwickler möchte ich, dass meine verschiedenen Arbeitskontexte voneinander profitieren können, ohne Sicherheitsgrenzen zu verletzen, damit Erkenntnisse und Patterns wiederverwendbar werden.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE Kontextbrücke SHALL es ermöglichen, Wissensartefakte eines Kontexts in anderen Kontexten als Lesereferenz bereitzustellen, wobei beliebige Artefakte teilbar sind, die Kontextbrücke jedoch Artefakte mit sensiblen Inhalten (kontextspezifische Credentials, Endpoints, personenbezogene Daten) vor der Freigabe filtert und blockiert
|
||||
2. WHEN ein Wissensartefakt als "teilbar" markiert wird, THE Kontextbrücke SHALL es in allen Kontexten als schreibgeschützte Lesereferenz bereitstellen
|
||||
3. IF ein als "teilbar" markiertes Artefakt kontextspezifische Geheimnisse, Zugangsdaten oder personenbezogene Daten enthält, THEN THE Kontextbrücke SHALL die Freigabe verweigern und eine Fehlermeldung ausgeben, die den Grund der Ablehnung benennt
|
||||
4. THE Kontextbrücke SHALL eine explizite Freigabe durch den Nutzer erfordern, bevor Information kontextübergreifend geteilt wird
|
||||
5. WHILE ein geteiltes Artefakt im Quellkontext aktualisiert wird, THE Kontextbrücke SHALL die Änderungen beim nächsten Lesezugriff in allen verknüpften Kontexten sichtbar machen
|
||||
6. WHEN ein Nutzer die Freigabe eines geteilten Artefakts widerruft, THE Kontextbrücke SHALL das Artefakt aus allen Zielkontexten als Lesereferenz entfernen
|
||||
|
||||
### Requirement 6: Migration bestehender Repositories
|
||||
|
||||
**User Story:** Als Entwickler möchte ich meine bestehenden 18+ Repositories in die neue Monorepo-Struktur überführen können, ohne Git-Historie oder bestehende Funktionalität zu verlieren.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE Monorepo SHALL die vollständige Git-Historie aller migrierten Repositories bewahren, einschließlich aller Commits, aller Branches, aller Tags und der zugehörigen Autoreninformationen
|
||||
2. WHEN ein Repository migriert wird, THE Monorepo SHALL sicherstellen, dass alle existierenden CI/CD-Konfigurationen entweder unverändert lauffähig bleiben oder mit einer dokumentierten Anpassungsanleitung (Grund der Änderung, neue Konfiguration, Testnachweis) versehen werden
|
||||
3. THE Monorepo SHALL einen dokumentierten Migrationsplan bereitstellen, der für jedes der 18+ Repositories den Ziel-Kontextordner, die Migrationsreihenfolge, die Abhängigkeiten zu anderen Repositories und den erwarteten Einbindungsmodus (direkt, Submodule, Upstream) beschreibt
|
||||
4. WHEN die Migration eines einzelnen Repositories abgeschlossen ist, THE Monorepo SHALL eine Validierung durchführen, die mindestens folgende Prüfungen umfasst: Übereinstimmung der Commit-Anzahl, Vorhandensein aller Branches und Tags, Vollständigkeit des Dateibaums im letzten Commit und erfolgreicher Durchlauf vorhandener Tests, wobei die Validierung für alle abgeschlossenen Migrationen durchgeführt wird (unabhängig davon ob sie erfolgreich waren oder fehlschlugen) und die Migration auch bei Nicht-Ausführbarkeit des Validierungsschritts als erfolgreich gilt, sofern alle zugrundeliegenden Prüfungen bestanden wurden
|
||||
5. IF während der Migration ein Konflikt auftritt (Pfadkollision, Branch-Namenskonflikt oder Namenskonvention-Verletzung), THEN THE Monorepo SHALL den Konflikt mit betroffener Datei, Konflikttyp und beteiligten Quell-Repositories protokollieren und die Migration des betroffenen Repositories pausieren, bis eine manuelle Auflösung erfolgt ist
|
||||
6. THE Monorepo SHALL die Migration inkrementell unterstützen, sodass Repositories einzeln und unabhängig voneinander überführt werden können, während bereits migrierte und noch nicht migrierte Repositories parallel funktionsfähig bleiben
|
||||
7. IF die Migration eines einzelnen Repositories fehlschlägt, THEN THE Monorepo SHALL den Zustand vor der Migration dieses Repositories wiederherstellen können, ohne bereits erfolgreich migrierte Repositories zu beeinflussen
|
||||
|
||||
### Requirement 7: Orchestrator-Integration und Multi-Harness-Dispatch
|
||||
|
||||
**User Story:** Als Entwickler möchte ich, dass der bestehende AI-Orchestrator nahtlos mit der neuen Monorepo-Struktur zusammenarbeitet und Tasks an verschiedene Agent_Harnesses (Kiro, Codex, Claude Code, Antigravity u.a.) dispatchen kann, damit ich pro Kontext oder Task den optimalen Agenten einsetzen kann.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN der Orchestrator einen Task bearbeitet, THE Orchestrator SHALL den zugehörigen Arbeitskontext anhand der Task-Metadaten (Labels oder Projektzuordnung) ermitteln und das Arbeitsverzeichnis des Agent_Harness auf den entsprechenden Kontextordner innerhalb der Monorepo-Struktur beschränken
|
||||
2. THE Orchestrator SHALL seinen Workspace-Root-Pfad in der WORKFLOW.md-Konfiguration so auflösen, dass pro Task ein Arbeitsverzeichnis unterhalb des ermittelten Kontextordners (privat, dhive, bahn oder shared) angelegt wird
|
||||
3. WHEN der Orchestrator in einem Kontext arbeitet, THE Orchestrator SHALL ausschließlich die .env-Datei des jeweiligen Kontextordners laden und Umgebungsvariablen anderer Kontexte weder lesen noch an den Agent_Harness-Prozess weitergeben
|
||||
4. WHEN der Orchestrator einen Task-Prompt zusammenstellt, THE Orchestrator SHALL den YAML-Index des Wissensspeichers abfragen und relevante Artefakt-Pfade als Kontextinformation in den Agenten-Prompt einfügen
|
||||
5. IF der Orchestrator einen Dateizugriff oder Secret-Zugriff außerhalb des zugewiesenen Kontextordners versucht, THEN THE Orchestrator SHALL den laufenden Task abbrechen, den Vorfall mit Task-Identifier und Zugriffspfad protokollieren und den Nutzer über den bestehenden OrgMyLife-Benachrichtigungskanal informieren
|
||||
6. WHEN kein Arbeitskontext aus den Task-Metadaten ermittelt werden kann, THE Orchestrator SHALL den Task nicht starten und eine Fehlermeldung mit dem fehlenden Kontext-Mapping an den Nutzer senden
|
||||
7. THE Orchestrator SHALL eine Konfiguration bereitstellen, die pro Arbeitskontext und optional pro Task-Typ den zu verwendenden Agent_Harness definiert (z.B. dhive → Codex, bahn → Kiro, privat → Claude Code)
|
||||
8. THE Orchestrator SHALL eine einheitliche Schnittstelle für alle Agent_Harnesses bereitstellen, die mindestens Prompt-Injection, Workspace-Verzeichnis, Umgebungsvariablen-Übergabe und Agent_Erweiterungen-Konfiguration umfasst
|
||||
9. WHEN ein Task einem Agent_Harness zugewiesen wird, THE Orchestrator SHALL die harness-spezifische Startsequenz ausführen (CLI-Aufruf, API-Call oder Prozessstart), wobei die Startsequenz pro Harness in der Konfiguration definiert ist
|
||||
10. IF der konfigurierte Agent_Harness für einen Task nicht verfügbar ist (nicht installiert, nicht lizenziert, Netzwerkfehler), THEN THE Orchestrator SHALL den Task als nicht-ausführbar markieren, den Fehlergrund protokollieren und den Nutzer benachrichtigen
|
||||
|
||||
### Requirement 8: Shared Tooling und Agent-Erweiterungen
|
||||
|
||||
**User Story:** Als Entwickler möchte ich übergreifende Tools, Agent-Erweiterungen (Powers, Skills, Plugins) und Steering-Konfigurationen an einer zentralen Stelle verwalten, damit sie allen Kontexten und allen eingesetzten Agent_Harnesses zur Verfügung stehen und nicht dupliziert werden müssen.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE Ordnerstruktur SHALL einen dedizierten shared-Bereich für kontextübergreifende Tools, Agent-Erweiterungen und Konfigurationen bereitstellen
|
||||
2. WHEN ein Tool im shared-Bereich aktualisiert wird, THE Monorepo SHALL sicherstellen, dass alle Kontexte bei ihrer nächsten Ausführung die aktualisierte Version des Tools verwenden, ohne manuelle Synchronisation in den einzelnen Kontexten
|
||||
3. THE Monorepo SHALL Agent-Erweiterungen (Steering-Dateien, Prompt-Vorlagen, Kontext-Definitionen, Skills) zentral im shared-Bereich ablegen und harness-agnostisch organisieren, sodass sie von verschiedenen Agent_Harnesses (Kiro, Codex, Claude Code, Antigravity u.a.) genutzt werden können
|
||||
4. THE Monorepo SHALL MCP-Server-Konfigurationen zentral im shared-Bereich bereitstellen, wobei kontextspezifische Konfigurationen die zentrale Konfiguration überschreiben (Kontext-Konfiguration hat Vorrang vor shared-Konfiguration)
|
||||
5. WHILE ein geteiltes Tool in einem bestimmten Kontext ausgeführt wird, THE Sicherheitsgrenze SHALL sicherstellen, dass das Tool nur auf die Dateien und Secrets des aktiven Kontexts sowie auf den shared-Bereich zugreift; IF die Sicherheitsgrenzen nicht durchgesetzt werden können, THEN SHALL die Ausführung des geteilten Tools verhindert werden
|
||||
6. IF ein geteiltes Tool oder eine zentrale Konfiguration einen Konflikt mit einer kontextspezifischen Überschreibung erzeugt, THEN THE Monorepo SHALL den Konflikt protokollieren und die kontextspezifische Konfiguration beibehalten
|
||||
7. THE Monorepo SHALL eine Konfigurationsdatei bereitstellen, die pro Arbeitskontext den eingesetzten Agent_Harness und dessen harness-spezifische Konfiguration definiert (z.B. Kiro-Powers-Pfad, Codex-Agenten-Config, Claude-Code-Projektdatei)
|
||||
8. THE Monorepo SHALL Agent-Erweiterungen in einem harness-unabhängigen Format ablegen (Markdown-Steering-Dateien, YAML-Konfigurationen, Prompt-Templates), die bei Bedarf in harness-spezifische Formate transformiert werden können
|
||||
9. WHEN ein neuer Agent_Harness konfiguriert wird, THE Monorepo SHALL einen Adapter-Mechanismus bereitstellen, der die zentral abgelegten Agent-Erweiterungen für den neuen Harness verfügbar macht, ohne die bestehenden Erweiterungen zu duplizieren
|
||||
|
||||
|
||||
### Requirement 9: Verschlüsselte Secrets im Repository
|
||||
|
||||
**User Story:** Als Entwickler möchte ich Secrets verschlüsselt im Repository speichern (statt sie via .gitignore auszuschließen), damit das Repo vollständig selbstbeschreibend und portabel über mehrere Maschinen hinweg ist, ohne dass Secrets manuell transportiert oder separat gesichert werden müssen.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE Secret_Encryption SHALL alle Secret-Dateien (.env-Dateien, private Schlüssel, Token-Dateien) verschlüsselt im Repository speichern, sodass sie committet und gepusht werden können, ohne im Klartext im Git-Verlauf oder Working Tree sichtbar zu sein
|
||||
2. THE Secret_Encryption SHALL ein bewährtes Verschlüsselungstool (git-crypt, SOPS oder age) für die Encryption-at-Rest verwenden
|
||||
3. THE Secret_Encryption SHALL für jeden Arbeitskontext (privat, dhive, bahn) einen eigenen Verschlüsselungsschlüssel verwenden, sodass der Besitz eines Schlüssels nur den zugehörigen Kontext entschlüsseln kann
|
||||
4. WHEN eine Maschine einem bestimmten Maschinenkontext zugeordnet ist, THE Secret_Encryption SHALL ausschließlich die Secrets der autorisierten Arbeitskontexte dieser Maschine entschlüsseln
|
||||
5. THE Secret_Encryption SHALL eine Konfigurationsdatei bereitstellen, die das Mapping von Maschinenkontext zu autorisierten Arbeitskontexten und Schlüsseln definiert (z.B. "dhive-Laptop" → nur dhive-Schlüssel, "Andre-Hauptrechner" → alle Schlüssel)
|
||||
6. WHEN ein neuer Rechner eingerichtet wird, THE Secret_Encryption SHALL einen dokumentierten Onboarding-Prozess bereitstellen, der die Installation der Entschlüsselungsschlüssel für die autorisierten Kontexte beschreibt
|
||||
7. THE Secret_Encryption SHALL mit einem Passwort-Manager integrierbar sein, sodass Entschlüsselungsschlüssel sicher gespeichert und bei Bedarf abgerufen werden können
|
||||
8. IF ein Entschlüsselungsversuch mangels autorisiertem Schlüssel fehlschlägt, THEN THE Secret_Encryption SHALL den Zugriff verweigern und die betroffenen Dateien im verschlüsselten Zustand belassen, ohne eine Fehlermeldung auszugeben, die den Inhalt der Datei offenlegt
|
||||
9. THE Secret_Encryption SHALL die bisherige .gitignore-basierte Ausschlussstrategie für Secret-Dateien ablösen, wobei .gitignore weiterhin für Build-Artefakte, Caches und temporäre Dateien verwendet wird
|
||||
10. WHEN ein Nutzer auf einer nicht-autorisierten Maschine das Repository klont, THE Secret_Encryption SHALL alle nicht-autorisierten Secret-Dateien als verschlüsselte Binärdaten im Working Tree belassen, sodass das Repository funktionsfähig bleibt (Code, Konfiguration, Dokumentation sind verfügbar), aber Secrets unzugänglich sind
|
||||
11. THE Secret_Encryption SHALL sicherstellen, dass verschlüsselte Dateien problemlos gemergt werden können, indem der Merge auf der verschlüsselten Ebene stattfindet oder ein git-crypt-kompatibler Merge-Treiber verwendet wird
|
||||
|
||||
### Requirement 10: Föderierte Repo-Struktur und Multi-Team-Zusammenarbeit
|
||||
|
||||
**User Story:** Als Entwickler möchte ich mit verschiedenen Teams (dhive-Kollegen, Bahn-Kollegen, Familie) zusammenarbeiten, wobei jedes Team ein eigenständiges Repository für seinen Kontext hat und nur ich als Hub-Besitzer das kombinierte Monorepo sehe, damit Sicherheitsgrenzen automatisch durchgesetzt werden und Teams unabhängig arbeiten können.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE Föderierte_Struktur SHALL für jeden Arbeitskontext (privat, dhive, bahn) ein eigenständiges Team_Repo unterstützen, das unabhängig vom Monorepo als vollständiges Git-Repository funktioniert
|
||||
2. THE Föderierte_Struktur SHALL eine Hub-and-Spoke-Topologie implementieren, in der das Monorepo als zentraler Hub fungiert und jedes Team_Repo als unabhängiger Spoke
|
||||
3. WHEN Änderungen in einem Team_Repo vorgenommen werden, THE Föderierte_Struktur SHALL diese Änderungen bidirektional mit dem entsprechenden Kontextordner im Monorepo synchronisieren können
|
||||
4. WHEN Änderungen im Monorepo in einem Kontextordner vorgenommen werden, THE Föderierte_Struktur SHALL diese Änderungen zum zugehörigen Team_Repo synchronisieren können
|
||||
5. THE Föderierte_Struktur SHALL sicherstellen, dass Team-Mitglieder eines Kontexts ausschließlich Zugriff auf das Team_Repo ihres eigenen Kontexts haben und weder den Inhalt noch die Existenz anderer Kontexte im Monorepo erkennen können
|
||||
6. THE Föderierte_Struktur SHALL das Team_Repo als Single Source of Truth für den jeweiligen Kontext behandeln, wobei bei Konflikten zwischen Team_Repo und Monorepo das Team_Repo Vorrang hat
|
||||
7. IF eine bidirektionale Synchronisation zwischen Monorepo und Team_Repo einen Merge-Konflikt erzeugt, THEN THE Föderierte_Struktur SHALL die Synchronisation abbrechen, den Konflikt mit betroffenen Dateien und Quellen protokollieren und dem Hub-Besitzer (Andre) eine manuelle Auflösung ermöglichen
|
||||
8. THE Föderierte_Struktur SHALL die Synchronisation über Git-Subtrees oder einen vergleichbaren Mechanismus realisieren, der die vollständige Git-Historie des Team_Repos im Monorepo bewahrt
|
||||
9. WHEN ein neues Team-Mitglied Zugang benötigt, THE Föderierte_Struktur SHALL ausschließlich Zugriff auf das entsprechende Team_Repo erteilen, ohne Kenntnis des Monorepos oder der Existenz anderer Kontexte preiszugeben
|
||||
10. THE Föderierte_Struktur SHALL die verschlüsselten Secrets gemäß Requirement 9 so handhaben, dass in den Team_Repos nur die Secrets des jeweiligen Kontexts enthalten sind (unverschlüsselt oder mit dem Team-Schlüssel verschlüsselt), während das Monorepo alle Kontexte mit kontextspezifischen Schlüsseln enthält
|
||||
11. THE Föderierte_Struktur SHALL eine Konfigurationsdatei bereitstellen, die für jeden Arbeitskontext das zugehörige Team_Repo (URL, Branch, Sync-Richtung, Sync-Frequenz) definiert
|
||||
12. WHILE die Synchronisation zwischen Monorepo und Team_Repo aktiv ist, THE Föderierte_Struktur SHALL sicherstellen, dass Dateien aus dem shared-Bereich, die für den jeweiligen Kontext relevant sind, optional als Read-Only-Kopie in das Team_Repo gespiegelt werden können
|
||||
@@ -0,0 +1,475 @@
|
||||
# Implementation Plan: Monorepo-Consolidation
|
||||
|
||||
## Overview
|
||||
|
||||
Inkrementelle Implementierung der Monorepo-Konsolidierung in Python. Die Umsetzung folgt einem Bottom-Up-Ansatz: Zunächst werden die Basis-Komponenten (Ordnerstruktur, Sicherheit, Verschlüsselung) erstellt, darauf aufbauend der Wissensspeicher und die Repo-Verwaltung, dann die Integrationsschicht (Orchestrator-Adapter, Kontextbrücke, Migration), und schließlich die Föderationsschicht (Team-Repos, Synchronisation).
|
||||
|
||||
## Tasks
|
||||
|
||||
- [x] 1. Projektstruktur und Basiskonfiguration
|
||||
- [x] 1.1 Monorepo-Grundstruktur und zentrale Konfigurationsdateien erstellen
|
||||
- Erstelle die Ordnerstruktur: `privat/`, `dhive/`, `bahn/`, `shared/` mit Unterordnern `shared/tools/`, `shared/powers/`, `shared/knowledge-store/`, `shared/config/`, `shared/mcp-servers/`
|
||||
- Erstelle `monorepo.yaml` mit Kontext-Definitionen, Naming-Regeln, Security- und Encryption-Konfiguration
|
||||
- Erstelle `shared/config/access-config.yaml` mit Zugriffskonfiguration pro Kontext
|
||||
- Erstelle `shared/config/repos.yaml` als leere Repo-Registry
|
||||
- Erstelle `shared/config/scopes.yaml` mit Scope-Mapping für den Wissensspeicher
|
||||
- Erstelle `shared/config/machine-context.yaml` mit Maschinenkontext-Mapping
|
||||
- Erstelle `shared/config/team-repos.yaml` mit Föderations-Konfiguration
|
||||
- Erstelle `.gitignore` mit Patterns für Build-Artefakte, Caches, temporäre Dateien (NICHT für Secrets)
|
||||
- Erstelle `.gitattributes` mit Root-Level git-crypt-Regeln
|
||||
- _Requirements: 1.1, 1.4, 2.2, 2.4, 2.6, 8.1, 9.5, 9.9, 10.11_
|
||||
|
||||
|
||||
- [x] 1.2 Python-Paketstruktur und gemeinsame Datenmodelle anlegen
|
||||
- Erstelle `shared/tools/monorepo-cli/` mit `pyproject.toml` und `src/monorepo/` Package
|
||||
- Definiere Dataclasses: `ProjectInfo`, `RepoEntry`, `MigrationPlan`, `SecurityEvent`, `ScopeConfig`, `MachineContext`, `EncryptionKey`, `PasswordManagerConfig`, `TeamRepoEntry`, `SharedMirrorConfig`, `SyncResult`, `ConflictInfo`, `IsolationReport`, `IsolationLeak`
|
||||
- Definiere Enums: `Context`, `RepoMode`, `ArtifactType`, `KeySource`, `SyncDirection`, `SyncFrequency`, `ConflictStrategy`
|
||||
- Erstelle `src/monorepo/__init__.py`, `src/monorepo/models.py`, `src/monorepo/config.py`
|
||||
- _Requirements: 1.1, 4.3, 9.5, 10.11_
|
||||
|
||||
- [ ] 2. Ordnerstruktur-Manager (StructureManager)
|
||||
- [x] 2.1 StructureManager-Klasse implementieren
|
||||
- Implementiere `StructureManager` in `src/monorepo/structure.py`
|
||||
- Methode `validate_name(name: str) -> bool`: Prüfung auf kebab-case, 2-50 Zeichen, Regex `^[a-z0-9][a-z0-9\-]{0,48}[a-z0-9]$`
|
||||
- Methode `create_project(context: str, name: str) -> Path`: Erstellt Projektordner, prüft Namenskonflikt, gibt Fehlermeldung bei Duplikat
|
||||
- Methode `list_projects(context: str | None) -> list[ProjectInfo]`: Auflistung aller Projekte
|
||||
- Methode `resolve_context(project_path: Path) -> str`: Kontexterkennung anhand Pfad
|
||||
- _Requirements: 1.1, 1.2, 1.3, 1.5, 1.6_
|
||||
|
||||
- [-] 2.2 Property-Test: Namensvalidierung (Property 1)
|
||||
- **Property 1: Namensvalidierung akzeptiert nur gültiges kebab-case**
|
||||
- **Validates: Requirements 1.3**
|
||||
|
||||
- [-] 2.3 Property-Test: Namenskollision (Property 2)
|
||||
- **Property 2: Namenskollision verhindert doppelte Projekterstellung**
|
||||
- **Validates: Requirements 1.6**
|
||||
|
||||
- [-] 2.4 Unit-Tests für StructureManager
|
||||
- Teste Erstellung in allen vier Kontexten
|
||||
- Teste Fehlermeldung bei existierendem Projektnamen
|
||||
- Teste Edge Cases: Minimum-/Maximum-Länge, Sonderzeichen, Unicode
|
||||
- _Requirements: 1.1, 1.2, 1.3, 1.6_
|
||||
|
||||
- [ ] 3. Sicherheits-Guard (ContextGuard)
|
||||
- [x] 3.1 ContextGuard-Klasse mit Zugriffskontrolle implementieren
|
||||
- Implementiere `ContextGuard` in `src/monorepo/security.py`
|
||||
- Methode `check_access(requesting_context: str, target_path: Path) -> bool`: Prüft Zugriffsberechtigung basierend auf access-config.yaml
|
||||
- Methode `load_env(context: str) -> dict[str, str]`: Lädt nur die .env des eigenen Kontexts (entschlüsselt via SecretEncryptionManager)
|
||||
- Shared-Zugriff: Nur auf explizit freigegebene Pfade gemäß Konfiguration
|
||||
- _Requirements: 2.1, 2.3, 2.6, 2.7_
|
||||
|
||||
- [-] 3.2 AuditLogger und Violation-Protokollierung implementieren
|
||||
- Implementiere `AuditLogger` in `src/monorepo/audit.py`
|
||||
- Methode `log_violation(event: SecurityEvent) -> None`: Schreibt Zeitstempel, anfragender Kontext, Zielkontext, Ressource in Protokolldatei
|
||||
- Konfiguration des Audit-Log-Pfads aus monorepo.yaml
|
||||
- _Requirements: 2.5_
|
||||
|
||||
- [x] 3.3 Property-Test: Kontextübergreifender Secret-Zugriff (Property 3)
|
||||
- **Property 3: Kontextübergreifender Secret-Zugriff wird verweigert**
|
||||
- **Validates: Requirements 2.1, 2.3, 2.5, 2.7**
|
||||
|
||||
- [x] 3.4 Property-Test: Shared-Tool-Isolation (Property 4)
|
||||
- **Property 4: Shared-Tool-Isolation**
|
||||
- **Validates: Requirements 2.7, 8.5**
|
||||
|
||||
- [x] 3.5 Unit-Tests für ContextGuard und AuditLogger
|
||||
- Teste Zugriff innerhalb des eigenen Kontexts (erlaubt)
|
||||
- Teste Zugriff auf fremden Kontext (verweigert)
|
||||
- Teste Zugriff auf shared-Bereich (erlaubt gemäß Konfiguration)
|
||||
- Teste Audit-Log-Format und Vollständigkeit
|
||||
- _Requirements: 2.1, 2.3, 2.5, 2.6, 2.7_
|
||||
|
||||
- [ ] 4. Secret-Encryption-Manager (SecretEncryptionManager)
|
||||
- [-] 4.1 SecretEncryptionManager-Basisklasse mit git-crypt-Integration implementieren
|
||||
- Implementiere `SecretEncryptionManager` in `src/monorepo/encryption.py`
|
||||
- Methode `encrypt_file(file_path: Path, context: str) -> EncryptionResult`: Verschlüsselt Datei mit Kontext-Schlüssel
|
||||
- Methode `decrypt_file(file_path: Path) -> DecryptionResult`: Entschlüsselt nur bei autorisiertem Maschinenkontext
|
||||
- Methode `is_authorized(context: str) -> bool`: Prüft Maschinenkontext-Autorisierung
|
||||
- Methode `setup_gitcrypt_filters(context: str) -> None`: Installiert git-crypt-Filter pro Kontext (.gitattributes)
|
||||
- _Requirements: 9.1, 9.2, 9.3, 9.4, 9.8_
|
||||
|
||||
- [x] 4.2 Maschinenkontext-Verwaltung und Schlüssel-Management implementieren
|
||||
- Implementiere `MachineContext`-Logik in `src/monorepo/encryption.py`
|
||||
- Methode `get_context_key(context: str) -> Optional[EncryptionKey]`: Schlüsselabruf aus Keyring oder Passwort-Manager
|
||||
- Methode `onboard_machine(machine_name, authorized_contexts) -> OnboardingResult`: Einrichtung neuer Maschinen mit autorisierten Schlüsseln
|
||||
- Methode `resolve_merge(file_path, ours, theirs) -> bytes`: Merge-Konflikt-Auflösung auf verschlüsselter Ebene
|
||||
- Lade Konfiguration aus `shared/config/machine-context.yaml`
|
||||
- Passwort-Manager-Integration (Bitwarden, 1Password, KeePass) als optionaler Key-Source
|
||||
- _Requirements: 9.4, 9.5, 9.6, 9.7, 9.10, 9.11_
|
||||
|
||||
- [x] 4.3 .gitattributes pro Kontext und git-crypt-Konfiguration erstellen
|
||||
- Erstelle `privat/.gitattributes` mit git-crypt-privat-Filter-Regeln
|
||||
- Erstelle `dhive/.gitattributes` mit git-crypt-dhive-Filter-Regeln
|
||||
- Erstelle `bahn/.gitattributes` mit git-crypt-bahn-Filter-Regeln
|
||||
- Definiere Secret-Patterns: .env, *.pem, *.key, *token*, *secret*
|
||||
- Dokumentiere Onboarding-Prozess für neue Maschinen
|
||||
- _Requirements: 9.1, 9.2, 9.6, 9.9_
|
||||
|
||||
- [x] 4.4 Property-Test: Verschlüsselung nur mit Kontext-Schlüssel (Property 23)
|
||||
- **Property 23: Verschlüsselte Secrets können nur mit autorisiertem Kontextschlüssel entschlüsselt werden**
|
||||
- **Validates: Requirements 9.3, 9.8**
|
||||
|
||||
- [x] 4.5 Property-Test: Maschinenkontext-Beschränkung (Property 24)
|
||||
- **Property 24: Maschinenkontext beschränkt Entschlüsselung auf autorisierte Kontexte**
|
||||
- **Validates: Requirements 9.4, 9.5, 9.8, 9.10**
|
||||
|
||||
- [x] 4.6 Unit-Tests für SecretEncryptionManager
|
||||
- Teste Verschlüsselung/Entschlüsselung mit korrektem Schlüssel
|
||||
- Teste Fehlschlag bei falschem Kontext-Schlüssel
|
||||
- Teste Maschinenkontext-Autorisierungsprüfung
|
||||
- Teste Fehlermeldung ohne Inhalt-Offenlegung
|
||||
- Teste .gitattributes-Generierung pro Kontext
|
||||
- Teste Merge-Auflösung für verschlüsselte Dateien
|
||||
- _Requirements: 9.1, 9.3, 9.4, 9.8, 9.10, 9.11_
|
||||
|
||||
- [x] 5. Checkpoint - Basis-Sicherheits- und Verschlüsselungsschicht
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
- [x] 6. Wissensspeicher – ETL-Pipeline und Indexierung
|
||||
- [x] 6.1 KnowledgeStore-Basisklasse und YAML-Index implementieren
|
||||
- Implementiere `KnowledgeStore` in `src/monorepo/knowledge/store.py`
|
||||
- Implementiere `YAMLIndex` in `src/monorepo/knowledge/index.py` mit Methoden: `load()`, `update_entry()`, `search()`, `get_by_scope()`
|
||||
- Index-Format gemäß Design: version, last_updated, artifacts-Liste mit id, title, type, tags, scope, summary, path, content_hash, links
|
||||
- Progressive-Disclosure: Index liefert nur kompakte Einträge (kein vollständiger Inhalt)
|
||||
- _Requirements: 3.13, 3.16, 3.18, 3.19_
|
||||
|
||||
- [x] 6.2 Wissensartefakt-Modell mit YAML-Frontmatter implementieren
|
||||
- Implementiere `Artifact` und `ArtifactMetadata` in `src/monorepo/knowledge/artifact.py`
|
||||
- YAML-Frontmatter-Parsing und -Generierung (type, title, tags, source_context, created, updated, shareable, links, content_hash)
|
||||
- Content-Hash-Berechnung (SHA-256)
|
||||
- Scope-basierte Ordnerstruktur: Artefakt-Pfad = `{scope}/{type}/{name}.md`
|
||||
- _Requirements: 3.4, 3.6, 3.11, 3.17, 3.19_
|
||||
|
||||
- [x] 6.3 ETL-Pipeline-Grundstruktur und Markdown-Quellstrategie implementieren
|
||||
- Implementiere `ETLPipeline` in `src/monorepo/knowledge/etl.py`
|
||||
- Implementiere `MarkdownSource` als erste Quellstrategie in `src/monorepo/knowledge/sources/markdown.py`
|
||||
- Methode `ingest(source, context) -> list[Artifact]`: Verarbeitet Quelle, erstellt Artefakte mit Metadaten
|
||||
- Inkrementelle Verarbeitung: Content-Hash-Vergleich, nur geänderte Inhalte aktualisieren
|
||||
- Fehlerresilienz: Bei Quellenfehler → Fehler protokollieren, erfolgreiche Artefakte beibehalten, Retry-Markierung
|
||||
- Index-Update im selben Durchlauf
|
||||
- _Requirements: 3.2, 3.6, 3.10, 3.12, 3.20, 3.21_
|
||||
|
||||
- [x] 6.4 Volltextsuche und Scope-basierte Filterung implementieren
|
||||
- Implementiere `search(query, allowed_scopes) -> list[SearchResult]` in KnowledgeStore
|
||||
- Volltextsuche über indexierte Artefakte mit Relevanz-Sortierung
|
||||
- Scope-Filterung: Nur Ergebnisse aus autorisierten Scopes zurückgeben
|
||||
- Existenz nicht-autorisierter Artefakte nicht offenlegen
|
||||
- Timeout-Handling: Abbruch nach 5 Sekunden mit Teilergebnis-Warnung
|
||||
- _Requirements: 3.3, 3.7, 3.9_
|
||||
|
||||
- [x] 6.5 Graph-Verknüpfungen zwischen Artefakten implementieren
|
||||
- Methode `link_artifacts(source_id, target_id, relation) -> None`
|
||||
- Bidirektionale Verknüpfung: Eintrag im YAML-Frontmatter beider Artefakte
|
||||
- Aktualisierung des YAML-Index mit Link-Informationen
|
||||
- _Requirements: 3.8_
|
||||
|
||||
- [x] 6.6 NoteGraph-Migration und Scope-Konfiguration implementieren
|
||||
- Implementiere `NoteGraphMigrator` in `src/monorepo/knowledge/migration.py`
|
||||
- Migration der NoteGraph-Verzeichnisstruktur (decisions, inbox, meetings, people, projects)
|
||||
- Scope-Konfiguration: Mapping Arbeitskontext → Scope in scopes.yaml
|
||||
- Validierung: Alle Artefakte, Metadaten und Verknüpfungen im Zielsystem vorhanden und auffindbar
|
||||
- _Requirements: 3.5, 3.14_
|
||||
|
||||
- [x] 6.7 Property-Test: Artefakt-Ingestion erzeugt vollständige Metadaten (Property 5)
|
||||
- **Property 5: Wissensartefakt-Ingestion erzeugt vollständige Metadaten im Index**
|
||||
- **Validates: Requirements 3.2, 3.4, 3.6, 3.11, 3.13, 3.17, 3.21**
|
||||
|
||||
- [x] 6.8 Property-Test: Suche respektiert Scope-Berechtigungen (Property 6)
|
||||
- **Property 6: Suche respektiert Scope-Berechtigungen**
|
||||
- **Validates: Requirements 3.3, 3.9**
|
||||
|
||||
- [x] 6.9 Property-Test: Progressive-Disclosure-Index (Property 7)
|
||||
- **Property 7: Progressive-Disclosure-Index enthält nur kompakte Einträge**
|
||||
- **Validates: Requirements 3.16, 3.18**
|
||||
|
||||
- [x] 6.10 Property-Test: Inkrementelle Verarbeitung (Property 8)
|
||||
- **Property 8: Inkrementelle Verarbeitung erkennt Änderungen über Content-Hash**
|
||||
- **Validates: Requirements 3.12**
|
||||
|
||||
- [x] 6.11 Property-Test: Volltextsuche (Property 9)
|
||||
- **Property 9: Volltextsuche findet indexierte Inhalte**
|
||||
- **Validates: Requirements 3.7**
|
||||
|
||||
- [x] 6.12 Property-Test: ETL-Fehlerresilienz (Property 10)
|
||||
- **Property 10: ETL-Fehlerresilienz bewahrt erfolgreiche Artefakte**
|
||||
- **Validates: Requirements 3.20**
|
||||
|
||||
- [x] 6.13 Property-Test: Graph-Verknüpfungen bidirektional (Property 11)
|
||||
- **Property 11: Graph-Verknüpfungen werden in beiden Artefakten reflektiert**
|
||||
- **Validates: Requirements 3.8**
|
||||
|
||||
- [x] 7. Checkpoint - Wissensspeicher-Kern
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
- [x] 8. Externes-Repo-Manager (RepoManager)
|
||||
- [x] 8.1 RepoManager-Klasse mit Subtree/Submodule-Einbindung implementieren
|
||||
- Implementiere `RepoManager` in `src/monorepo/repos.py`
|
||||
- Methode `add_repo(entry: RepoEntry) -> None`: Bindet Repo via Git-Subtree (Standard) oder Submodule ein
|
||||
- Methode `sync(repo_name: str) -> SyncResult`: Aktualisiert auf gepinnte Version, gibt Ergebnismeldung (Erfolg/Misserfolg/unbekannt)
|
||||
- Bidirektionale Sync für Upstream-Repos (Pull/Push), Abbruch bei Merge-Konflikten
|
||||
- _Requirements: 4.1, 4.3, 4.4, 4.6, 4.7, 4.8_
|
||||
|
||||
- [x] 8.2 Read-Only-Schutzmechanismus implementieren
|
||||
- Methode `protect_readonly(repo_path: Path) -> None`: Installiert Git pre-commit Hook
|
||||
- Hook verhindert Commits die Dateien im geschützten Verzeichnis ändern
|
||||
- Fehlermeldung bei Schreibversuch mit Repo-Name und Read-Only-Status
|
||||
- Fehlerprotokollierung bei fehlgeschlagener Synchronisation (Netzwerk, Auth, Merge)
|
||||
- _Requirements: 4.2, 4.5, 4.9_
|
||||
|
||||
- [x] 8.3 Property-Test: Read-Only-Repos blockieren Schreibzugriffe (Property 12)
|
||||
- **Property 12: Read-Only-Repos blockieren Schreibzugriffe**
|
||||
- **Validates: Requirements 4.2, 4.5**
|
||||
|
||||
- [x] 8.4 Property-Test: Fehlgeschlagene Synchronisation (Property 13)
|
||||
- **Property 13: Fehlgeschlagene Synchronisation bewahrt lokalen Stand**
|
||||
- **Validates: Requirements 4.9**
|
||||
|
||||
- [x] 8.5 Unit-Tests für RepoManager
|
||||
- Teste Subtree-Add und Submodule-Add
|
||||
- Teste Sync-Ergebnismeldungen (Erfolg, Fehler, unbekannt)
|
||||
- Teste Read-Only-Hook-Blockierung
|
||||
- Teste Fehlerprotokollierung bei Sync-Fehlschlägen
|
||||
- _Requirements: 4.1, 4.2, 4.4, 4.5, 4.9_
|
||||
|
||||
- [x] 9. Kontextbrücke (ContextBridge)
|
||||
- [x] 9.1 ContextBridge-Klasse mit Sensitive-Content-Filter implementieren
|
||||
- Implementiere `ContextBridge` in `src/monorepo/bridge.py`
|
||||
- Methode `share_artifact(artifact_id, user_confirmed) -> ShareResult`: Freigabe nur mit expliziter Nutzerbestätigung
|
||||
- Methode `check_sensitive_content(content: str) -> list[SensitiveMatch]`: Regex-basierte Prüfung auf Secrets, Endpoints, PII
|
||||
- Methode `revoke_share(artifact_id) -> None`: Entfernt Artefakt aus allen Zielkontexten als Lesereferenz
|
||||
- Update-Propagierung: Änderungen im Quellkontext beim nächsten Lesezugriff sichtbar
|
||||
- _Requirements: 5.1, 5.2, 5.3, 5.4, 5.5, 5.6_
|
||||
|
||||
- [x] 9.2 Property-Test: Sensitive-Content-Filter (Property 14)
|
||||
- **Property 14: Sensitive-Content-Filter blockiert Freigabe**
|
||||
- **Validates: Requirements 5.1, 5.3**
|
||||
|
||||
- [x] 9.3 Property-Test: Freigabe-Lebenszyklus (Property 15)
|
||||
- **Property 15: Freigabe-Lebenszyklus (Share → Update → Revoke)**
|
||||
- **Validates: Requirements 5.2, 5.5, 5.6**
|
||||
|
||||
- [x] 9.4 Property-Test: Explizite Nutzerbestätigung (Property 16)
|
||||
- **Property 16: Freigabe erfordert explizite Nutzerbestätigung**
|
||||
- **Validates: Requirements 5.4**
|
||||
|
||||
- [x] 10. Checkpoint - Repo-Manager und Kontextbrücke
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
- [ ] 11. Migrations-Engine
|
||||
- [x] 11.1 MigrationEngine-Klasse mit Git-Historie-Bewahrung implementieren
|
||||
- Implementiere `MigrationEngine` in `src/monorepo/migration.py`
|
||||
- Methode `migrate(plan: MigrationPlan) -> MigrationResult`: Führt Migration via `git filter-repo` oder `git subtree add` durch, bewahrt alle Commits, Branches, Tags, Autoreninformationen
|
||||
- Inkrementelle Migration: Repositories einzeln und unabhängig migrierbar
|
||||
- Konflikterkennung: Pfadkollision, Branch-Namenskonflikt, Namenskonvention-Verletzung → Migration pausieren
|
||||
- _Requirements: 6.1, 6.3, 6.5, 6.6_
|
||||
|
||||
- [-] 11.2 Migrations-Validierung und Rollback implementieren
|
||||
- Methode `validate(repo_name: str) -> ValidationResult`: Prüft Commit-Anzahl, Branches, Tags, Dateibaum-Vollständigkeit, Test-Durchlauf
|
||||
- Methode `rollback(repo_name: str) -> None`: Stellt Vor-Migrations-Zustand wieder her, ohne andere Repos zu beeinflussen
|
||||
- Dokumentierter Migrationsplan: Für jedes Repo Ziel-Kontext, Reihenfolge, Abhängigkeiten, Einbindungsmodus
|
||||
- _Requirements: 6.1, 6.4, 6.7_
|
||||
|
||||
- [-] 11.3 Property-Test: Migrations-Validierung (Property 17)
|
||||
- **Property 17: Migrations-Validierung prüft Vollständigkeit**
|
||||
- **Validates: Requirements 6.4**
|
||||
|
||||
- [-] 11.4 Property-Test: Migrations-Rollback (Property 18)
|
||||
- **Property 18: Migrations-Rollback stellt Vor-Zustand wieder her**
|
||||
- **Validates: Requirements 6.7**
|
||||
|
||||
- [x] 11.5 Unit-Tests für MigrationEngine
|
||||
- Teste Migration eines einfachen Repos (Commits, Branches, Tags bewahrt)
|
||||
- Teste Konflikterkennung (Pfadkollision, Branch-Namenskonflikt)
|
||||
- Teste Rollback-Wiederherstellung
|
||||
- Teste inkrementelle Migration (parallel funktionsfähig)
|
||||
- _Requirements: 6.1, 6.4, 6.5, 6.6, 6.7_
|
||||
|
||||
- [ ] 12. Orchestrator-Adapter
|
||||
- [x] 12.1 OrchestratorAdapter-Klasse mit Kontextauflösung implementieren
|
||||
- Implementiere `OrchestratorAdapter` in `src/monorepo/orchestrator.py`
|
||||
- Methode `resolve_context(task: Task) -> str`: Ermittelt Arbeitskontext aus Task-Metadaten (Labels/Projektzuordnung)
|
||||
- Methode `create_workspace(task, context) -> Path`: Erstellt Arbeitsverzeichnis unter dem korrekten Kontextordner
|
||||
- Methode `build_prompt(task, context) -> str`: Erstellt Agenten-Prompt mit Wissenskontext
|
||||
- Fehlerbehandlung: Kein Kontext → Task nicht starten, Fehlermeldung; Out-of-Context-Zugriff → Task abbrechen, protokollieren, Nutzer benachrichtigen
|
||||
- _Requirements: 7.1, 7.2, 7.3, 7.5, 7.6_
|
||||
|
||||
- [-] 12.2 Wissensinjection und Env-Loading im Orchestrator implementieren
|
||||
- Methode `inject_knowledge(prompt, context) -> str`: Abfrage des YAML-Index, relevante Artefakt-Pfade einfügen
|
||||
- Env-Loading: Ausschließlich .env des zugewiesenen Kontexts laden (entschlüsselt via SecretEncryptionManager), keine Umgebungsvariablen anderer Kontexte
|
||||
- Integration mit bestehender WORKFLOW.md-Konfiguration: workspace_root pro Task
|
||||
- _Requirements: 7.3, 7.4_
|
||||
|
||||
- [-] 12.3 Multi-Harness-Dispatch implementieren
|
||||
- Implementiere Harness-Konfiguration: Pro Kontext/Task-Typ den Agent_Harness definieren
|
||||
- Einheitliche Schnittstelle: Prompt-Injection, Workspace-Verzeichnis, Env-Übergabe, Agent_Erweiterungen
|
||||
- Harness-spezifische Startsequenz (CLI, API, Prozessstart) aus Konfiguration laden
|
||||
- Fehlerbehandlung: Harness nicht verfügbar → Task als nicht-ausführbar markieren, Nutzer benachrichtigen
|
||||
- _Requirements: 7.7, 7.8, 7.9, 7.10_
|
||||
|
||||
- [x] 12.4 Property-Test: Orchestrator-Kontextauflösung (Property 19)
|
||||
- **Property 19: Orchestrator-Kontextauflösung und Workspace-Isolation**
|
||||
- **Validates: Requirements 7.1, 7.2, 7.3, 7.6**
|
||||
|
||||
- [x] 12.5 Property-Test: Orchestrator-Kontextverletzung (Property 20)
|
||||
- **Property 20: Orchestrator-Kontextverletzung bricht Task ab**
|
||||
- **Validates: Requirements 7.5**
|
||||
|
||||
- [x] 12.6 Unit-Tests für OrchestratorAdapter
|
||||
- Teste Kontextauflösung aus Task-Labels
|
||||
- Teste Workspace-Erstellung im korrekten Kontextordner
|
||||
- Teste Env-Isolation (nur eigene .env geladen, via Encryption entschlüsselt)
|
||||
- Teste Wissensinjektion in Prompt
|
||||
- Teste Task-Abbruch bei Out-of-Context-Zugriff
|
||||
- Teste Multi-Harness-Dispatch (Konfiguration, Startsequenz, Fehlerbehandlung)
|
||||
- _Requirements: 7.1, 7.2, 7.3, 7.4, 7.5, 7.6, 7.7, 7.9, 7.10_
|
||||
|
||||
- [x] 13. Shared Tooling und MCP-Konfiguration
|
||||
- [x] 13.1 MCP-Konfiguration-Merge und Shared-Tool-Versionierung implementieren
|
||||
- Implementiere `ConfigMerger` in `src/monorepo/shared_config.py`
|
||||
- Methode für MCP-Server-Konfiguration-Merge: Shared als Basis, kontextspezifische Überschreibungen mit Vorrang
|
||||
- Konflikt-Protokollierung bei Überschreibungskonflikten
|
||||
- Shared-Tool-Versionierung: Symlink- oder Import-basierter Mechanismus, sodass Kontexte automatisch die aktuelle Version verwenden
|
||||
- Kiro Powers und Skills zentral im shared-Bereich ablegen, Read-Only-Referenz für alle Kontexte
|
||||
- Agent-Erweiterungen in harness-unabhängigem Format (Markdown, YAML, Templates) mit Adapter-Mechanismus
|
||||
- _Requirements: 8.2, 8.3, 8.4, 8.5, 8.6, 8.7, 8.8, 8.9_
|
||||
|
||||
- [x] 13.2 Property-Test: MCP-Konfiguration-Merge (Property 21)
|
||||
- **Property 21: MCP-Konfiguration-Merge mit Kontext-Vorrang**
|
||||
- **Validates: Requirements 8.4, 8.6**
|
||||
|
||||
- [x] 13.3 Property-Test: Shared-Tool-Versionierung (Property 22)
|
||||
- **Property 22: Shared-Tool-Versionierung ohne manuelle Synchronisation**
|
||||
- **Validates: Requirements 8.2**
|
||||
|
||||
- [x] 14. Checkpoint - Orchestrator, Shared Tooling und Migration
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
- [x] 15. Federation-Manager (FederationManager)
|
||||
- [x] 15.1 FederationManager-Basisklasse und SubtreeSyncEngine implementieren
|
||||
- Implementiere `FederationManager` in `src/monorepo/federation.py`
|
||||
- Implementiere `SubtreeSyncEngine` in `src/monorepo/federation.py`
|
||||
- Methode `sync_from_team(context) -> SyncResult`: Pull vom Team_Repo via git subtree pull
|
||||
- Methode `sync_to_team(context) -> SyncResult`: Push zum Team_Repo via git subtree push
|
||||
- Methode `full_sync(context) -> SyncResult`: Bidirektionale Synchronisation mit Konflikt-Erkennung
|
||||
- Methode `detect_conflicts(context) -> list[ConflictInfo]`: Erkennt Merge-Konflikte vor Sync
|
||||
- Lade Konfiguration aus `shared/config/team-repos.yaml`
|
||||
- _Requirements: 10.1, 10.2, 10.3, 10.4, 10.8_
|
||||
|
||||
- [x] 15.2 Team-Isolation und Cross-Context-Leakage-Prüfung implementieren
|
||||
- Methode `verify_isolation(team_repo_path, context) -> IsolationReport`: Prüft ob Team_Repo keine Referenzen auf andere Kontexte enthält
|
||||
- Prüfung auf: Pfad-Referenzen, Env-Variablen, Config-Referenzen, Kommentare zu anderen Kontexten
|
||||
- Methode `prepare_team_repo(context) -> Path`: Erzeugt Team_Repo mit nur eigenem Kontext
|
||||
- Secrets-Handling: Team_Repo enthält nur Secrets des eigenen Kontexts (entschlüsselt oder re-keyed mit Team-Schlüssel)
|
||||
- Integration mit SecretEncryptionManager für kontextspezifische Secret-Filterung
|
||||
- _Requirements: 10.5, 10.9, 10.10_
|
||||
|
||||
- [x] 15.3 Shared-Bereich-Spiegelung und Konflikt-Auflösung implementieren
|
||||
- Methode `mirror_shared(context, paths) -> MirrorResult`: Spiegelt shared-Dateien als Read-Only in Team_Repo
|
||||
- Methode `resolve_conflict(context, strategy) -> ConflictResult`: Löst Sync-Konflikte auf (Standard: team-wins)
|
||||
- Methode `onboard_member(context, member_info) -> OnboardingResult`: Erteilt Zugang nur zum Team_Repo
|
||||
- Single-Source-of-Truth-Logik: Team_Repo hat bei Konflikten Vorrang
|
||||
- Konfigurierbare Sync-Frequenz: on-push, hourly, daily, manual
|
||||
- _Requirements: 10.6, 10.7, 10.9, 10.11, 10.12_
|
||||
|
||||
- [x] 15.4 Property-Test: Team-Repos enthalten nur eigenen Kontext (Property 25)
|
||||
- **Property 25: Team-Repos enthalten ausschließlich Inhalte des eigenen Kontexts**
|
||||
- **Validates: Requirements 10.5, 10.9, 10.10, 10.12**
|
||||
|
||||
- [x] 15.5 Property-Test: Bidirektionale Synchronisation (Property 26)
|
||||
- **Property 26: Bidirektionale Synchronisation bewahrt Git-Historie und löst Konflikte korrekt**
|
||||
- **Validates: Requirements 10.3, 10.4, 10.6, 10.7, 10.8**
|
||||
|
||||
- [x] 15.6 Property-Test: Team-Mitglieder-Isolation (Property 27)
|
||||
- **Property 27: Team-Mitglieder können die Existenz anderer Kontexte nicht entdecken**
|
||||
- **Validates: Requirements 10.5, 10.9**
|
||||
|
||||
- [x] 15.7 Unit-Tests für FederationManager
|
||||
- Teste Subtree-Pull und -Push Operationen
|
||||
- Teste Konflikt-Erkennung und -Auflösung (team-wins-Strategie)
|
||||
- Teste Isolation-Check (Leakage-Erkennung für Pfade, Env-Vars, Config-Refs)
|
||||
- Teste Shared-Mirror als Read-Only
|
||||
- Teste Onboarding ohne Monorepo-Kenntnis
|
||||
- Teste Sync-Abbruch bei Merge-Konflikten mit Protokollierung
|
||||
- _Requirements: 10.1, 10.3, 10.5, 10.6, 10.7, 10.9, 10.12_
|
||||
|
||||
- [x] 16. Checkpoint - Föderationsschicht
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
- [x] 17. Integration und Verdrahtung
|
||||
- [x] 17.1 CLI-Tool (`ctx-guard`) als Wrapper-Script erstellen
|
||||
- Erstelle CLI-Einstiegspunkt in `shared/tools/monorepo-cli/src/monorepo/cli.py`
|
||||
- Kommandos: `init` (Monorepo initialisieren), `create-project`, `list-projects`, `add-repo`, `sync-repo`, `migrate`, `validate`, `search`, `encrypt`, `decrypt`, `onboard`, `fed-sync`, `fed-status`
|
||||
- Verdrahte alle Komponenten: StructureManager, ContextGuard, KnowledgeStore, RepoManager, ContextBridge, MigrationEngine, OrchestratorAdapter, SecretEncryptionManager, FederationManager
|
||||
- Konfiguration aus `monorepo.yaml` und `shared/config/` laden
|
||||
- _Requirements: 1.1, 1.2, 4.4, 6.3, 9.6, 10.11_
|
||||
|
||||
- [x] 17.2 Git-Hooks und Dateisystem-Schutz integrieren
|
||||
- Pre-commit Hook: Prüft Read-Only-Repos nicht verändert werden, unverschlüsselte Secrets nicht committet werden
|
||||
- git-crypt-Filter-Integration: Sicherstellt dass Secrets automatisch verschlüsselt/entschlüsselt werden
|
||||
- Integriere ContextGuard und SecretEncryptionManager in Hook-Chain
|
||||
- Installationsskript für Hooks bei `init`
|
||||
- _Requirements: 2.4, 4.2, 4.5, 9.1, 9.9_
|
||||
|
||||
- [x] 17.3 Encryption-Security-Integration verdrahten
|
||||
- Integriere SecretEncryptionManager mit ContextGuard: load_env nutzt Entschlüsselung
|
||||
- Integriere SecretEncryptionManager mit FederationManager: Team_Repos erhalten nur eigene Secrets
|
||||
- Integriere SecretEncryptionManager mit OrchestratorAdapter: Env-Loading via Encryption
|
||||
- _Requirements: 2.2, 9.3, 9.4, 10.10_
|
||||
|
||||
- [x] 17.4 Integration-Tests für End-to-End-Workflows
|
||||
- Teste: Projekt erstellen → Env verschlüsseln → Env laden → Secret-Zugriff prüfen
|
||||
- Teste: Artefakt erstellen → Indexieren → Suchen → Freigeben → Widerrufen
|
||||
- Teste: Repo einbinden → Sync → Read-Only-Schutz
|
||||
- Teste: Migration → Validierung → Rollback
|
||||
- Teste: Secret verschlüsseln → Maschinenkontext wechseln → Entschlüsselung fehlschlägt
|
||||
- Teste: Team_Repo vorbereiten → Isolation prüfen → Sync → Konflikt-Auflösung
|
||||
- Teste: Shared-Mirror in Team_Repo → Read-Only-Prüfung
|
||||
- _Requirements: 1.1, 2.1, 3.2, 4.1, 5.2, 6.1, 9.1, 9.4, 10.3, 10.5, 10.12_
|
||||
|
||||
- [x] 18. Final Checkpoint - Alle Komponenten integriert
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
## Notes
|
||||
|
||||
- Tasks marked with `*` are optional and can be skipped for faster MVP
|
||||
- Each task references specific requirements for traceability
|
||||
- Checkpoints ensure incremental validation
|
||||
- Property tests validate universal correctness properties from the design document (P1-P27)
|
||||
- Unit tests validate specific examples and edge cases
|
||||
- Die ETL-Pipeline wird initial nur mit der Markdown-Quellstrategie implementiert; weitere Strategien (Confluence, PDF, Webseiten, GitLab) können später ergänzt werden
|
||||
- Die DB-Wissensdatenbank-Integration (Requirement 3.1, 3.15, 4.8) wird über den RepoManager als Upstream-Repo eingebunden; die ETL-Pipeline-Fork-Adaption erfolgt im Wissensspeicher
|
||||
- Git-Remote-Operationen (Push/Pull/Sync) werden in Integration-Tests mit gemockten Repos getestet
|
||||
- git-crypt-Operationen (encrypt/decrypt) werden in Integration-Tests mit temporären Repos und Test-Schlüsseln getestet
|
||||
- Team-Repo-Synchronisation wird mit lokalen Bare-Repos als Test-Remotes getestet
|
||||
- Die Föderationsschicht (Task 15) baut auf dem SecretEncryptionManager (Task 4) auf und setzt diesen als Abhängigkeit voraus
|
||||
- Passwort-Manager-Integration (Bitwarden, 1Password, KeePass) ist optional und kann durch lokalen Keyring ersetzt werden
|
||||
|
||||
## Task Dependency Graph
|
||||
|
||||
```json
|
||||
{
|
||||
"waves": [
|
||||
{ "id": 0, "tasks": ["1.1"] },
|
||||
{ "id": 1, "tasks": ["1.2"] },
|
||||
{ "id": 2, "tasks": ["2.1", "3.1"] },
|
||||
{ "id": 3, "tasks": ["2.2", "2.3", "2.4", "3.2", "4.1"] },
|
||||
{ "id": 4, "tasks": ["3.3", "3.4", "3.5", "4.2"] },
|
||||
{ "id": 5, "tasks": ["4.3", "4.4", "4.5", "4.6"] },
|
||||
{ "id": 6, "tasks": ["6.1", "6.2"] },
|
||||
{ "id": 7, "tasks": ["6.3", "6.4", "6.5"] },
|
||||
{ "id": 8, "tasks": ["6.6", "6.7", "6.8", "6.9", "6.10"] },
|
||||
{ "id": 9, "tasks": ["6.11", "6.12", "6.13"] },
|
||||
{ "id": 10, "tasks": ["8.1", "9.1"] },
|
||||
{ "id": 11, "tasks": ["8.2", "8.3", "8.4", "8.5", "9.2", "9.3", "9.4"] },
|
||||
{ "id": 12, "tasks": ["11.1", "12.1"] },
|
||||
{ "id": 13, "tasks": ["11.2", "11.3", "11.4", "11.5", "12.2", "12.3", "13.1"] },
|
||||
{ "id": 14, "tasks": ["12.4", "12.5", "12.6", "13.2", "13.3"] },
|
||||
{ "id": 15, "tasks": ["15.1"] },
|
||||
{ "id": 16, "tasks": ["15.2", "15.3"] },
|
||||
{ "id": 17, "tasks": ["15.4", "15.5", "15.6", "15.7"] },
|
||||
{ "id": 18, "tasks": ["17.1"] },
|
||||
{ "id": 19, "tasks": ["17.2", "17.3"] },
|
||||
{ "id": 20, "tasks": ["17.4"] }
|
||||
]
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1 @@
|
||||
{"specId": "e33fa6e5-89e0-4d8d-ad9e-d43d6bf23daa", "workflowType": "requirements-first", "specType": "feature"}
|
||||
@@ -0,0 +1,542 @@
|
||||
# Design Document: NoteGraph Ingestion & Auto-Reference
|
||||
|
||||
## Overview
|
||||
|
||||
The NoteGraph Ingestion service is a Python package (`NoteGraph/ingestion/`) that provides a CLI-driven pipeline for importing documents (PDFs, images, markdown, text, DOCX) into the NoteGraph knowledge base. The pipeline extracts text (via OCR or direct extraction), enriches it using an LLM to detect entities (people, projects, dates, action items), generates wiki-links, and outputs structured markdown files with YAML frontmatter into the appropriate NoteGraph folder.
|
||||
|
||||
The system is designed around three principles:
|
||||
1. **Model-agnostic** — A unified LLM interface (via LiteLLM) supports OpenAI, Anthropic, Google, Mistral, and local Ollama without code changes.
|
||||
2. **CLI-first** — All operations are accessible via `python -m ingestion.cli` with subcommands for bulk import and continuous watching.
|
||||
3. **Markdown-native** — Output is always SilverBullet-compatible markdown with YAML frontmatter and wiki-links.
|
||||
|
||||
## Architecture
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
subgraph Input
|
||||
CLI[CLI Interface]
|
||||
WATCH[Drop-Folder Watcher]
|
||||
end
|
||||
|
||||
subgraph Pipeline
|
||||
DISC[File Discovery]
|
||||
EXT[Text Extraction]
|
||||
OCR[OCR Engine]
|
||||
ENR[Enrichment Agent]
|
||||
LINK[Reference Linker]
|
||||
RENDER[Markdown Renderer]
|
||||
end
|
||||
|
||||
subgraph Output
|
||||
MD[Markdown Files]
|
||||
STUB[Stub Pages]
|
||||
GIT[Git Commit]
|
||||
TASK[OrgMyLife Tasks]
|
||||
end
|
||||
|
||||
subgraph External
|
||||
LLM[LLM Provider]
|
||||
ORG[OrgMyLife API]
|
||||
end
|
||||
|
||||
CLI --> DISC
|
||||
WATCH --> DISC
|
||||
DISC --> EXT
|
||||
EXT --> OCR
|
||||
EXT --> ENR
|
||||
ENR --> LLM
|
||||
ENR --> LINK
|
||||
LINK --> RENDER
|
||||
RENDER --> MD
|
||||
RENDER --> STUB
|
||||
MD --> GIT
|
||||
STUB --> GIT
|
||||
ENR --> TASK
|
||||
TASK --> ORG
|
||||
```
|
||||
|
||||
### Data Flow
|
||||
|
||||
```
|
||||
Input File → File Discovery → Text Extraction (OCR if needed)
|
||||
→ Enrichment Agent (LLM call) → Entity Detection
|
||||
→ Reference Linker (wiki-links) → Markdown Renderer (frontmatter + body)
|
||||
→ File Placement (folder routing) → Git Commit
|
||||
→ (optional) OrgMyLife Task Creation
|
||||
```
|
||||
|
||||
## Components and Interfaces
|
||||
|
||||
### Package Structure
|
||||
|
||||
```
|
||||
NoteGraph/ingestion/
|
||||
├── __init__.py
|
||||
├── __main__.py # python -m ingestion entry point
|
||||
├── cli.py # Click-based CLI (import, watch subcommands)
|
||||
├── config.py # Configuration from .env
|
||||
├── pipeline.py # Main orchestration pipeline
|
||||
├── discovery.py # File discovery and filtering
|
||||
├── extraction/
|
||||
│ ├── __init__.py
|
||||
│ ├── base.py # Extractor protocol
|
||||
│ ├── text.py # Plain text / markdown extractor
|
||||
│ ├── pdf.py # PDF text extraction (PyPDF2/pdfplumber)
|
||||
│ ├── ocr.py # OCR via Tesseract (pytesseract)
|
||||
│ └── docx.py # DOCX extraction (python-docx)
|
||||
├── enrichment/
|
||||
│ ├── __init__.py
|
||||
│ ├── agent.py # Enrichment orchestration
|
||||
│ ├── prompts.py # Structured prompts for entity detection
|
||||
│ └── models.py # Pydantic models for enrichment response
|
||||
├── linking/
|
||||
│ ├── __init__.py
|
||||
│ ├── linker.py # Reference linker (wiki-link generation)
|
||||
│ └── stubs.py # Stub page creation
|
||||
├── output/
|
||||
│ ├── __init__.py
|
||||
│ ├── renderer.py # Markdown + frontmatter rendering
|
||||
│ ├── router.py # Folder routing logic
|
||||
│ └── naming.py # Filename generation with collision avoidance
|
||||
├── integrations/
|
||||
│ ├── __init__.py
|
||||
│ ├── llm.py # LiteLLM wrapper (model-agnostic)
|
||||
│ ├── orgmylife.py # OrgMyLife API client
|
||||
│ └── git.py # Git commit operations
|
||||
├── watcher.py # Drop-folder watcher (watchdog)
|
||||
└── tests/
|
||||
├── __init__.py
|
||||
├── conftest.py
|
||||
├── test_discovery.py
|
||||
├── test_extraction.py
|
||||
├── test_enrichment.py
|
||||
├── test_linking.py
|
||||
├── test_output.py
|
||||
├── test_pipeline.py
|
||||
└── test_properties.py # Property-based tests
|
||||
```
|
||||
|
||||
### Core Interfaces
|
||||
|
||||
```python
|
||||
# ingestion/extraction/base.py
|
||||
from typing import Protocol
|
||||
|
||||
class TextExtractor(Protocol):
|
||||
"""Protocol for all text extractors."""
|
||||
|
||||
def can_handle(self, file_path: Path) -> bool:
|
||||
"""Return True if this extractor handles the given file type."""
|
||||
...
|
||||
|
||||
def extract(self, file_path: Path) -> ExtractionResult:
|
||||
"""Extract text content from the file."""
|
||||
...
|
||||
|
||||
@dataclass
|
||||
class ExtractionResult:
|
||||
text: str
|
||||
source_file: Path
|
||||
extraction_method: str # "direct", "ocr", "mixed"
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
```
|
||||
|
||||
```python
|
||||
# ingestion/enrichment/models.py
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
class Entity(BaseModel):
|
||||
type: Literal["person", "project", "date", "action_item"]
|
||||
value: str
|
||||
confidence: float = Field(ge=0.0, le=1.0)
|
||||
position: int | None = None # character offset in source text
|
||||
|
||||
class ActionItem(BaseModel):
|
||||
description: str
|
||||
assignee: str | None = None
|
||||
deadline: str | None = None # ISO 8601
|
||||
|
||||
class EnrichmentResult(BaseModel):
|
||||
title: str
|
||||
category: Literal["meeting", "project", "decision", "inbox"]
|
||||
tags: list[str]
|
||||
entities: list[Entity]
|
||||
action_items: list[ActionItem]
|
||||
summary: str | None = None
|
||||
```
|
||||
|
||||
```python
|
||||
# ingestion/integrations/llm.py
|
||||
from litellm import completion
|
||||
|
||||
class LLMClient:
|
||||
"""Model-agnostic LLM client using LiteLLM."""
|
||||
|
||||
def __init__(self, config: IngestionConfig):
|
||||
self.model = config.llm_model_string # e.g. "openai/gpt-4o"
|
||||
|
||||
def complete(self, messages: list[dict], response_format: type[BaseModel]) -> BaseModel:
|
||||
"""Send a structured completion request."""
|
||||
...
|
||||
```
|
||||
|
||||
```python
|
||||
# ingestion/integrations/orgmylife.py
|
||||
import httpx
|
||||
|
||||
class OrgMyLifeClient:
|
||||
"""HTTP client for OrgMyLife task creation."""
|
||||
|
||||
def __init__(self, base_url: str, api_key: str):
|
||||
self.base_url = base_url
|
||||
self.api_key = api_key
|
||||
|
||||
def create_task(self, title: str, source_url: str,
|
||||
description: str = "", priority: int = 4) -> dict:
|
||||
"""Create a task via POST /api/tasks."""
|
||||
...
|
||||
```
|
||||
|
||||
### Configuration
|
||||
|
||||
```python
|
||||
# ingestion/config.py
|
||||
from pydantic_settings import BaseSettings
|
||||
|
||||
class IngestionConfig(BaseSettings):
|
||||
# LLM Provider
|
||||
llm_provider: str = "openai" # openai, anthropic, google, mistral, ollama
|
||||
llm_model: str = "gpt-4o"
|
||||
openai_api_key: str = ""
|
||||
anthropic_api_key: str = ""
|
||||
google_api_key: str = ""
|
||||
mistral_api_key: str = ""
|
||||
ollama_base_url: str = "http://localhost:11434"
|
||||
|
||||
# OrgMyLife Integration
|
||||
orgmylife_api_url: str = "https://api.andreknie.de"
|
||||
orgmylife_api_key: str = ""
|
||||
|
||||
# Paths
|
||||
notes_dir: str = "./notes"
|
||||
inbox_dir: str = "./ingestion/inbox"
|
||||
|
||||
# OCR
|
||||
ocr_enabled: bool = True
|
||||
ocr_language: str = "deu+eng" # German + English
|
||||
|
||||
# Behavior
|
||||
confidence_threshold: float = 0.7
|
||||
auto_commit: bool = True
|
||||
|
||||
class Config:
|
||||
env_file = ".env"
|
||||
env_prefix = ""
|
||||
|
||||
@property
|
||||
def llm_model_string(self) -> str:
|
||||
"""LiteLLM model string (e.g., 'openai/gpt-4o')."""
|
||||
if self.llm_provider == "ollama":
|
||||
return f"ollama/{self.llm_model}"
|
||||
return f"{self.llm_provider}/{self.llm_model}"
|
||||
|
||||
def validate_api_key(self) -> None:
|
||||
"""Raise if required API key is missing."""
|
||||
...
|
||||
```
|
||||
|
||||
## Data Models
|
||||
|
||||
### Enrichment Prompt (Structured Output)
|
||||
|
||||
The LLM receives extracted text and a system prompt requesting structured JSON output:
|
||||
|
||||
```python
|
||||
ENRICHMENT_SYSTEM_PROMPT = """You are a knowledge management assistant. Analyze the following text
|
||||
extracted from a document and return structured metadata.
|
||||
|
||||
Return a JSON object with:
|
||||
- title: A concise, descriptive title for this note (max 80 chars)
|
||||
- category: One of "meeting", "project", "decision", "inbox"
|
||||
- tags: List of relevant topic tags (lowercase, no #)
|
||||
- entities: List of detected entities, each with:
|
||||
- type: "person", "project", "date", or "action_item"
|
||||
- value: The extracted text
|
||||
- confidence: Float 0.0-1.0
|
||||
- position: Character offset where entity appears (approximate)
|
||||
- action_items: List of action items, each with:
|
||||
- description: What needs to be done
|
||||
- assignee: Person responsible (if mentioned)
|
||||
- deadline: Date in ISO 8601 format (if mentioned)
|
||||
- summary: Optional 1-2 sentence summary
|
||||
|
||||
Context: This is a personal knowledge base for a project manager working across
|
||||
multiple business projects. Common projects include: {known_projects}.
|
||||
Known people: {known_people}.
|
||||
"""
|
||||
```
|
||||
|
||||
### Output Markdown Format
|
||||
|
||||
```yaml
|
||||
---
|
||||
title: "Meeting with Beier GmbH about Druckluft project"
|
||||
date: 2024-03-15
|
||||
tags:
|
||||
- meeting
|
||||
- druckluft
|
||||
- customer
|
||||
people:
|
||||
- André Knierim
|
||||
- Thomas Beier
|
||||
projects:
|
||||
- Beier GmbH
|
||||
- Druckluft
|
||||
source:
|
||||
file: "Beier GmbH.pdf"
|
||||
imported: "2024-03-20T14:30:00"
|
||||
---
|
||||
|
||||
# Meeting with Beier GmbH about Druckluft project
|
||||
|
||||
Discussion about the [[projects/druckluft]] project with [[people/thomas-beier]].
|
||||
|
||||
Key points:
|
||||
- Delivery timeline confirmed for Q2
|
||||
- Budget approved at €45,000
|
||||
- Next review meeting scheduled for April 10
|
||||
|
||||
## Action Items
|
||||
|
||||
- [ ] Send updated proposal to Thomas Beier (deadline: 2024-03-22)
|
||||
- [ ] Schedule follow-up meeting for April 10
|
||||
- [ ] Update project timeline in planning tool
|
||||
```
|
||||
|
||||
### Entity-to-WikiLink Mapping
|
||||
|
||||
| Entity Type | Wiki-Link Format | Example |
|
||||
|-------------|-----------------|---------|
|
||||
| Person | `[[people/firstname-lastname]]` | `[[people/thomas-beier]]` |
|
||||
| Project | `[[projects/project-slug]]` | `[[projects/druckluft]]` |
|
||||
| Date | Frontmatter `date` field | `2024-03-15` |
|
||||
| Action Item | `## Action Items` section | `- [ ] Send proposal` |
|
||||
|
||||
### File Routing Rules
|
||||
|
||||
| Category | Target Folder | Condition |
|
||||
|----------|--------------|-----------|
|
||||
| Meeting | `notes/meetings/` | Category = "meeting" |
|
||||
| Project | `notes/projects/` | Category = "project" with single dominant project |
|
||||
| Decision | `notes/decisions/` | Category = "decision" |
|
||||
| Default | `notes/inbox/` | No clear category or mixed |
|
||||
|
||||
## Correctness Properties
|
||||
|
||||
*A property is a characteristic or behavior that should hold true across all valid executions of a system — essentially, a formal statement about what the system should do. Properties serve as the bridge between human-readable specifications and machine-verifiable correctness guarantees.*
|
||||
|
||||
### Property 1: File discovery returns only supported extensions
|
||||
|
||||
*For any* directory tree containing files with arbitrary extensions, the discovery function SHALL return exactly and only those files whose extensions are in the supported set (`.pdf`, `.jpg`, `.jpeg`, `.png`, `.md`, `.txt`, `.docx`).
|
||||
|
||||
**Validates: Requirements 1.1, 1.6**
|
||||
|
||||
### Property 2: Bulk import resilience — failed files do not block remaining processing
|
||||
|
||||
*For any* batch of N files where a subset F fails during processing, the pipeline SHALL successfully process all N-F non-failing files and the summary SHALL report success count = N-F and failed count = |F|.
|
||||
|
||||
**Validates: Requirements 1.4, 1.5**
|
||||
|
||||
### Property 3: Text extraction preserves content for text-based formats
|
||||
|
||||
*For any* valid UTF-8 string written to a markdown or plain text file, extracting it via the Text_Extractor SHALL return the identical string content.
|
||||
|
||||
**Validates: Requirements 3.1, 3.2, 3.4**
|
||||
|
||||
### Property 4: Entity parsing extracts all entities with valid structure
|
||||
|
||||
*For any* valid enrichment response JSON containing entities, each parsed entity SHALL have a non-empty `type` (one of person/project/date/action_item), a non-empty `value`, and a `confidence` score in the range [0.0, 1.0].
|
||||
|
||||
**Validates: Requirements 4.1, 4.2, 4.3, 4.4, 4.5**
|
||||
|
||||
### Property 5: Confidence threshold filtering
|
||||
|
||||
*For any* list of entities with arbitrary confidence scores, the filtered output SHALL contain exactly those entities with confidence >= 0.7 and no others.
|
||||
|
||||
**Validates: Requirements 4.6**
|
||||
|
||||
### Property 6: Wiki-link generation follows correct format per entity type
|
||||
|
||||
*For any* person name, the generated wiki-link SHALL match the pattern `[[people/slugified-name]]`. *For any* project name, the generated wiki-link SHALL match the pattern `[[projects/slugified-name]]`.
|
||||
|
||||
**Validates: Requirements 5.1, 5.2**
|
||||
|
||||
### Property 7: Only first entity occurrence is linked
|
||||
|
||||
*For any* text containing N >= 2 occurrences of the same entity, the Reference_Linker SHALL convert exactly the first occurrence to a wiki-link and leave all subsequent occurrences as plain text.
|
||||
|
||||
**Validates: Requirements 5.6**
|
||||
|
||||
### Property 8: Frontmatter round-trip validity
|
||||
|
||||
*For any* valid `EnrichmentResult`, rendering it to markdown with YAML frontmatter and then parsing the frontmatter back SHALL produce a valid YAML block containing all required fields (title, date, tags, people, projects, source).
|
||||
|
||||
**Validates: Requirements 6.1, 6.3, 6.4, 6.6, 6.7**
|
||||
|
||||
### Property 9: Provider routing from configuration
|
||||
|
||||
*For any* valid provider name from the supported set (openai, anthropic, google, mistral, ollama), configuring `LLM_PROVIDER` to that value SHALL cause the LLM client to route requests to the corresponding provider.
|
||||
|
||||
**Validates: Requirements 7.3, 8.1**
|
||||
|
||||
### Property 10: Missing API key produces descriptive error
|
||||
|
||||
*For any* provider that requires an API key, when the corresponding environment variable is unset, the configuration validation SHALL raise an error that names the missing variable.
|
||||
|
||||
**Validates: Requirements 7.4, 8.6**
|
||||
|
||||
### Property 11: Action items appear in output markdown section
|
||||
|
||||
*For any* list of extracted action items, the rendered markdown SHALL contain a `## Action Items` section listing every action item description.
|
||||
|
||||
**Validates: Requirements 9.2**
|
||||
|
||||
### Property 12: OrgMyLife task payload correctness
|
||||
|
||||
*For any* action item description and note file path, the task creation payload SHALL set `title` to the action item description and `source_url` to a URL referencing the note path.
|
||||
|
||||
**Validates: Requirements 9.4**
|
||||
|
||||
### Property 13: Category-based folder routing
|
||||
|
||||
*For any* enrichment result with category "meeting", the output path SHALL be within `notes/meetings/`. *For any* result with category "project", the path SHALL be within `notes/projects/`. *For any* result with no clear category, the path SHALL be within `notes/inbox/`.
|
||||
|
||||
**Validates: Requirements 10.1, 10.2, 10.3**
|
||||
|
||||
### Property 14: Output filename follows date-slug pattern
|
||||
|
||||
*For any* date and title string, the generated filename SHALL match the pattern `YYYY-MM-DD-slugified-title.md` where the slug contains only lowercase alphanumeric characters and hyphens.
|
||||
|
||||
**Validates: Requirements 10.5**
|
||||
|
||||
### Property 15: Filename collision avoidance
|
||||
|
||||
*For any* set of existing files, when a new file would have the same name as an existing file, the system SHALL append a numeric suffix (`-2`, `-3`, etc.) producing a unique filename.
|
||||
|
||||
**Validates: Requirements 10.6**
|
||||
|
||||
### Property 16: Temporary file filtering
|
||||
|
||||
*For any* filename, the watcher ignore logic SHALL return true if and only if the filename starts with `.` or ends with `.tmp`.
|
||||
|
||||
**Validates: Requirements 11.5**
|
||||
|
||||
### Property 17: Dry-run produces no file writes
|
||||
|
||||
*For any* input processed with the `--dry-run` flag, the system SHALL create zero files on disk.
|
||||
|
||||
**Validates: Requirements 13.6**
|
||||
|
||||
### Property 18: Git commit message format
|
||||
|
||||
*For any* count N and source type string, the generated commit message SHALL match the format `ingestion: import N notes from [source-type]`.
|
||||
|
||||
**Validates: Requirements 12.2**
|
||||
|
||||
## Error Handling
|
||||
|
||||
### Error Categories and Strategies
|
||||
|
||||
| Error Type | Strategy | User Impact |
|
||||
|-----------|----------|-------------|
|
||||
| File read failure | Log + skip file, continue batch | Warning in summary |
|
||||
| OCR failure (no text) | Create note with empty body + source ref | Note created, flagged |
|
||||
| Encoding detection failure | Log + skip file | Warning in summary |
|
||||
| LLM API error | Retry 3x with exponential backoff, then fail file | Warning per file |
|
||||
| LLM response parse error | Retry once with stricter prompt, then use defaults | Degraded metadata |
|
||||
| OrgMyLife API unreachable | Log warning, continue without task creation | Tasks not created |
|
||||
| Git commit failure | Log warning, files still written | No version history |
|
||||
| Disk full / write failure | Fail immediately with clear error | Process stops |
|
||||
| Invalid config (missing key) | Exit with descriptive error before processing | Immediate feedback |
|
||||
|
||||
### Retry Policy
|
||||
|
||||
```python
|
||||
RETRY_CONFIG = {
|
||||
"max_retries": 3,
|
||||
"base_delay_seconds": 1.0,
|
||||
"max_delay_seconds": 30.0,
|
||||
"backoff_factor": 2.0,
|
||||
"retryable_errors": [
|
||||
"rate_limit",
|
||||
"timeout",
|
||||
"server_error", # 5xx
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Logging
|
||||
|
||||
- **Level INFO**: File processing progress, summary counts
|
||||
- **Level WARNING**: Skipped files, API failures (non-fatal), git failures
|
||||
- **Level ERROR**: Configuration errors, unrecoverable failures
|
||||
- **Level DEBUG**: LLM prompts/responses, entity detection details
|
||||
|
||||
All logs include timestamps and structured context (file path, provider, operation).
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
### Property-Based Tests (Hypothesis)
|
||||
|
||||
The project uses **Hypothesis** (Python's standard PBT library) for property-based testing. Each property test runs a minimum of 100 iterations.
|
||||
|
||||
Property tests cover the core transformation logic:
|
||||
- File discovery filtering (Property 1)
|
||||
- Text extraction identity (Property 3)
|
||||
- Entity parsing and filtering (Properties 4, 5)
|
||||
- Wiki-link generation (Properties 6, 7)
|
||||
- Frontmatter round-trip (Property 8)
|
||||
- Folder routing (Property 13)
|
||||
- Filename generation and collision avoidance (Properties 14, 15)
|
||||
- Temporary file filtering (Property 16)
|
||||
|
||||
Each property test is tagged with:
|
||||
```python
|
||||
# Feature: notegraph-ingestion, Property N: [property text]
|
||||
```
|
||||
|
||||
Configuration: minimum 100 examples per test via `@settings(max_examples=100)`.
|
||||
|
||||
### Unit Tests (pytest)
|
||||
|
||||
Example-based tests for:
|
||||
- CLI argument parsing and subcommand routing
|
||||
- Configuration loading from environment variables
|
||||
- Specific enrichment response scenarios
|
||||
- OrgMyLife client request formatting
|
||||
- Git commit message generation
|
||||
- Dry-run mode behavior
|
||||
|
||||
### Integration Tests
|
||||
|
||||
Integration tests (marked with `@pytest.mark.integration`) for:
|
||||
- OCR extraction with sample images
|
||||
- PDF text extraction with sample PDFs
|
||||
- DOCX extraction with sample files
|
||||
- LLM API calls (with real provider, gated by env var)
|
||||
- OrgMyLife API task creation (against test instance)
|
||||
- Drop-folder watcher file detection
|
||||
- Git operations in a test repository
|
||||
|
||||
### Test Dependencies
|
||||
|
||||
```toml
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest>=8.0",
|
||||
"pytest-asyncio>=0.23",
|
||||
"hypothesis>=6.100",
|
||||
"pytest-mock>=3.12",
|
||||
"respx>=0.21", # httpx mocking
|
||||
]
|
||||
```
|
||||
@@ -0,0 +1,195 @@
|
||||
# Requirements Document
|
||||
|
||||
## Introduction
|
||||
|
||||
The NoteGraph Ingestion & Auto-Reference feature adds a document import pipeline and AI-powered entity linking system to NoteGraph. It enables bulk import of PDF files (from reMarkable tablet), images (photos of sticky notes, whiteboards, handwritten notes), and legacy text files (from OneNote, Android Notes, Obsidian, scattered file systems). Imported content is OCR-processed, enriched by an LLM to detect entities (people, projects, dates, TODOs), and output as structured markdown with wiki-links and frontmatter into the appropriate NoteGraph folder. The system uses a model-agnostic LLM abstraction layer supporting multiple providers via simple `.env` configuration.
|
||||
|
||||
## Glossary
|
||||
|
||||
- **Ingestion_Service**: The Python service responsible for orchestrating the full import pipeline from file input through OCR, enrichment, and markdown output.
|
||||
- **OCR_Engine**: The component that extracts raw text from image files (JPG, PNG) and scanned PDF pages using optical character recognition.
|
||||
- **Text_Extractor**: The component that extracts text content from text-based PDFs and legacy file formats (markdown, plain text, OneNote exports).
|
||||
- **LLM_Provider**: An abstraction layer that routes enrichment requests to a configured language model provider (OpenAI, Anthropic, Google, Mistral, Ollama/Gemma) without coupling to any specific API.
|
||||
- **Enrichment_Agent**: The AI-powered component that processes raw extracted text to detect entities, generate metadata, create wiki-links, and structure the output as markdown.
|
||||
- **Entity_Detector**: The sub-component of the Enrichment_Agent responsible for identifying person names, project references, dates, and action items in extracted text.
|
||||
- **Reference_Linker**: The sub-component that converts detected entities into SilverBullet-compatible wiki-links (e.g., `[[people/name]]`, `[[projects/name]]`).
|
||||
- **Drop_Folder_Watcher**: A background process that monitors a designated inbox directory for new files and triggers the Ingestion_Service automatically.
|
||||
- **Inbox_Folder**: The designated directory where files are placed for automatic ingestion (default: `NoteGraph/ingestion/inbox/`).
|
||||
- **Output_Folder**: The target directory within `NoteGraph/notes/` where processed markdown files are placed.
|
||||
- **Provider_Config**: The `.env`-based configuration that specifies which LLM_Provider to use and supplies the corresponding API keys.
|
||||
- **OrgMyLife_Client**: The HTTP client component that creates tasks in the OrgMyLife API from extracted action items.
|
||||
- **Frontmatter**: YAML metadata block at the top of a markdown file containing structured fields (title, date, tags, people, projects).
|
||||
|
||||
## Requirements
|
||||
|
||||
### Requirement 1: Bulk File Import via CLI
|
||||
|
||||
**User Story:** As a user, I want to import many files at once from a folder using a CLI command, so that I can quickly ingest large collections of notes from various sources.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN the user runs the CLI import command with a directory path, THE Ingestion_Service SHALL recursively discover all supported files (PDF, JPG, PNG, MD, TXT) in that directory.
|
||||
2. WHEN the user runs the CLI import command with a single file path, THE Ingestion_Service SHALL process that individual file.
|
||||
3. WHEN processing multiple files, THE Ingestion_Service SHALL report progress showing the current file number, total count, and file name.
|
||||
4. WHEN a file fails to process during bulk import, THE Ingestion_Service SHALL log the error with the file path and reason, skip the file, and continue processing remaining files.
|
||||
5. WHEN bulk import completes, THE Ingestion_Service SHALL output a summary showing total files processed, successful count, and failed count.
|
||||
6. THE Ingestion_Service SHALL support the following file extensions: `.pdf`, `.jpg`, `.jpeg`, `.png`, `.md`, `.txt`, `.docx`.
|
||||
|
||||
### Requirement 2: OCR and Text Extraction from Images
|
||||
|
||||
**User Story:** As a user, I want photos of handwritten notes and whiteboards converted to text, so that I can search and reference them in my knowledge base.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN a JPG or PNG file is provided, THE OCR_Engine SHALL extract text content from the image and return it as a UTF-8 string.
|
||||
2. WHEN a PDF file contains scanned pages (image-based content without embedded text), THE OCR_Engine SHALL apply OCR to extract text from those pages.
|
||||
3. WHEN a PDF file contains embedded text, THE Text_Extractor SHALL extract the text directly without OCR.
|
||||
4. WHEN a PDF file contains a mix of text pages and scanned pages, THE Ingestion_Service SHALL apply the appropriate extraction method per page and concatenate results.
|
||||
5. IF the OCR_Engine cannot extract any readable text from an image, THEN THE Ingestion_Service SHALL create the output note with an empty body and attach the original file reference in the frontmatter.
|
||||
6. WHEN processing handwritten content, THE OCR_Engine SHALL preserve paragraph structure by detecting line breaks and groupings.
|
||||
|
||||
### Requirement 3: Legacy File Format Import
|
||||
|
||||
**User Story:** As a user, I want to import notes from old tools like OneNote, Obsidian, and scattered text files, so that I can consolidate all my knowledge in one place.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN a markdown file is provided, THE Text_Extractor SHALL read the file content preserving existing formatting, links, and structure.
|
||||
2. WHEN a plain text file is provided, THE Text_Extractor SHALL read the file content as UTF-8 text.
|
||||
3. WHEN a DOCX file is provided, THE Text_Extractor SHALL extract the text content preserving heading structure and paragraph breaks.
|
||||
4. WHEN an imported markdown file contains Obsidian-style wiki-links (`[[page name]]`), THE Text_Extractor SHALL preserve them for later resolution by the Reference_Linker.
|
||||
5. WHEN a file uses a non-UTF-8 encoding, THE Text_Extractor SHALL attempt to detect the encoding and convert to UTF-8.
|
||||
6. IF a file cannot be decoded to text, THEN THE Ingestion_Service SHALL log a warning and skip the file.
|
||||
|
||||
### Requirement 4: AI-Powered Entity Detection
|
||||
|
||||
**User Story:** As a user, I want the system to automatically detect people, projects, dates, and action items in my notes, so that I do not have to manually tag and link everything.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN extracted text is provided to the Enrichment_Agent, THE Entity_Detector SHALL identify person names mentioned in the text.
|
||||
2. WHEN extracted text is provided to the Enrichment_Agent, THE Entity_Detector SHALL identify project references mentioned in the text.
|
||||
3. WHEN extracted text is provided to the Enrichment_Agent, THE Entity_Detector SHALL identify dates and temporal references in the text and normalize them to ISO 8601 format.
|
||||
4. WHEN extracted text is provided to the Enrichment_Agent, THE Entity_Detector SHALL identify action items and TODO statements in the text.
|
||||
5. WHEN the Entity_Detector identifies entities, THE Entity_Detector SHALL return each entity with its type, extracted value, and confidence score between 0.0 and 1.0.
|
||||
6. THE Entity_Detector SHALL only include entities with a confidence score of 0.7 or higher in the final output.
|
||||
|
||||
### Requirement 5: Auto-Reference Wiki-Link Generation
|
||||
|
||||
**User Story:** As a user, I want detected entities automatically converted to wiki-links, so that my imported notes are immediately connected to my knowledge graph.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN a person entity is detected, THE Reference_Linker SHALL generate a wiki-link in the format `[[people/firstname-lastname]]`.
|
||||
2. WHEN a project entity is detected, THE Reference_Linker SHALL generate a wiki-link in the format `[[projects/project-name]]`.
|
||||
3. WHEN a date entity is detected, THE Reference_Linker SHALL add the normalized date to the frontmatter `date` field.
|
||||
4. WHEN the Reference_Linker generates a wiki-link for a person or project that does not yet exist in NoteGraph, THE Reference_Linker SHALL create a stub page with the entity name as title.
|
||||
5. WHEN the Reference_Linker generates a wiki-link, THE Reference_Linker SHALL insert the link inline in the note body at the position where the entity was detected.
|
||||
6. WHEN multiple references to the same entity exist in one note, THE Reference_Linker SHALL link only the first occurrence and leave subsequent mentions as plain text.
|
||||
|
||||
### Requirement 6: Structured Markdown Output with Frontmatter
|
||||
|
||||
**User Story:** As a user, I want imported notes output as structured markdown with proper metadata, so that they integrate seamlessly with SilverBullet's features.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE Ingestion_Service SHALL output each processed file as a markdown file with YAML frontmatter containing: title, date, tags, people, projects, and source fields.
|
||||
2. WHEN the Enrichment_Agent generates a title from the content, THE Ingestion_Service SHALL use it as the frontmatter `title` field.
|
||||
3. WHEN entities are detected, THE Ingestion_Service SHALL populate the frontmatter `people` field as a list of detected person names.
|
||||
4. WHEN entities are detected, THE Ingestion_Service SHALL populate the frontmatter `projects` field as a list of detected project names.
|
||||
5. WHEN entities are detected, THE Ingestion_Service SHALL populate the frontmatter `tags` field with relevant topic tags derived from the content.
|
||||
6. THE Ingestion_Service SHALL include a `source` field in the frontmatter containing the original file name and import timestamp.
|
||||
7. FOR ALL valid extracted text inputs, processing through the Enrichment_Agent and then rendering to markdown and then parsing the frontmatter SHALL produce a valid YAML frontmatter block (round-trip property).
|
||||
|
||||
### Requirement 7: Model-Agnostic LLM Provider Layer
|
||||
|
||||
**User Story:** As a user, I want to easily switch between LLM providers and try different models, so that I can compare behavior and avoid vendor lock-in.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE LLM_Provider SHALL expose a unified interface for text completion requests regardless of the underlying provider.
|
||||
2. THE LLM_Provider SHALL support at minimum the following providers: OpenAI, Anthropic, Google Gemini, Mistral, Gemma (local), and Ollama (local).
|
||||
3. WHEN a provider is specified in the Provider_Config, THE LLM_Provider SHALL route requests to that provider using the configured API key.
|
||||
4. WHEN the configured provider is unavailable or returns an error, THE LLM_Provider SHALL raise a descriptive exception including the provider name and error details.
|
||||
5. THE LLM_Provider SHALL read provider selection and API keys from environment variables defined in a `.env` file.
|
||||
6. THE LLM_Provider SHALL support configuring the model name per provider (e.g., `gpt-4o`, `claude-sonnet-4-20250514`, `gemini-pro`).
|
||||
7. WHEN switching providers, THE LLM_Provider SHALL require only a change to environment variables without code modifications.
|
||||
|
||||
### Requirement 8: Provider Configuration via Environment Variables
|
||||
|
||||
**User Story:** As a user, I want simple `.env`-based configuration for LLM providers, so that I can set up and switch providers without editing code.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE Provider_Config SHALL use the environment variable `LLM_PROVIDER` to select the active provider (values: `openai`, `anthropic`, `google`, `mistral`, `ollama`).
|
||||
2. THE Provider_Config SHALL use the environment variable `LLM_MODEL` to specify the model name for the active provider.
|
||||
3. THE Provider_Config SHALL use provider-specific environment variables for API keys: `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GOOGLE_API_KEY`, `MISTRAL_API_KEY`.
|
||||
4. WHERE the Ollama provider is selected, THE Provider_Config SHALL use the environment variable `OLLAMA_BASE_URL` to specify the local Ollama endpoint (default: `http://localhost:11434`). Ollama SHALL support running Google Gemma models locally.
|
||||
5. THE Provider_Config SHALL NOT store API keys or secrets directly in git-tracked files. The `.env` file SHALL be listed in `.gitignore` and a `.env.example` file SHALL document all variables without actual secret values.
|
||||
6. WHEN a required API key is missing for the selected provider, THE Ingestion_Service SHALL exit with a clear error message naming the missing variable.
|
||||
7. THE Ingestion_Service SHALL provide a `.env.example` file documenting all supported environment variables with descriptions.
|
||||
|
||||
### Requirement 9: TODO Extraction and OrgMyLife Integration
|
||||
|
||||
**User Story:** As a user, I want action items extracted from my notes and optionally created as tasks in OrgMyLife, so that I do not lose track of commitments found in imported notes.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN the Entity_Detector identifies action items in the text, THE Enrichment_Agent SHALL extract each action item with its description and any associated person or deadline.
|
||||
2. WHEN action items are extracted, THE Ingestion_Service SHALL list them in a `## Action Items` section at the end of the output markdown.
|
||||
3. WHEN the `--create-tasks` flag is provided on the CLI, THE OrgMyLife_Client SHALL create a task in OrgMyLife for each extracted action item via POST to `/api/tasks`.
|
||||
4. WHEN creating a task in OrgMyLife, THE OrgMyLife_Client SHALL set the task title to the action item description and include a `source_url` linking back to the imported note.
|
||||
5. IF the OrgMyLife API is unreachable when `--create-tasks` is specified, THEN THE Ingestion_Service SHALL log a warning and continue processing without creating tasks.
|
||||
6. THE Provider_Config SHALL use the environment variable `ORGMYLIFE_API_URL` for the OrgMyLife endpoint and `ORGMYLIFE_API_KEY` for authentication.
|
||||
|
||||
### Requirement 10: File Placement and Folder Routing
|
||||
|
||||
**User Story:** As a user, I want imported notes automatically placed in the correct NoteGraph folder based on their content, so that my knowledge base stays organized.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN the Enrichment_Agent detects meeting-related content, THE Ingestion_Service SHALL place the output file in `NoteGraph/notes/meetings/`.
|
||||
2. WHEN the Enrichment_Agent detects project-specific content with a single dominant project, THE Ingestion_Service SHALL place the output file in `NoteGraph/notes/projects/`.
|
||||
3. WHEN the Enrichment_Agent cannot determine a specific category, THE Ingestion_Service SHALL place the output file in `NoteGraph/notes/inbox/`.
|
||||
4. WHEN the `--output-dir` flag is provided on the CLI, THE Ingestion_Service SHALL place all output files in the specified directory regardless of content analysis.
|
||||
5. THE Ingestion_Service SHALL generate output file names using the pattern `YYYY-MM-DD-slugified-title.md` based on the detected or current date and the generated title.
|
||||
6. IF an output file with the same name already exists, THEN THE Ingestion_Service SHALL append a numeric suffix (e.g., `-2`, `-3`) to avoid overwriting.
|
||||
|
||||
### Requirement 11: Drop-Folder Watcher for Continuous Import
|
||||
|
||||
**User Story:** As a user, I want to drop files into a folder and have them automatically processed, so that I can import notes without running CLI commands.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN the watcher mode is started, THE Drop_Folder_Watcher SHALL monitor the Inbox_Folder for new files.
|
||||
2. WHEN a new file appears in the Inbox_Folder, THE Drop_Folder_Watcher SHALL trigger the Ingestion_Service to process that file.
|
||||
3. WHEN a file is successfully processed, THE Drop_Folder_Watcher SHALL move the original file to an `archive/` subdirectory within the Inbox_Folder.
|
||||
4. IF a file fails to process, THEN THE Drop_Folder_Watcher SHALL move the file to a `failed/` subdirectory and log the error.
|
||||
5. THE Drop_Folder_Watcher SHALL ignore temporary files (names starting with `.` or ending with `.tmp`).
|
||||
6. WHEN the watcher mode is started, THE Drop_Folder_Watcher SHALL process any files already present in the Inbox_Folder before entering watch mode.
|
||||
|
||||
### Requirement 12: Git Auto-Commit for Imported Notes
|
||||
|
||||
**User Story:** As a user, I want imported notes automatically committed to git, so that my knowledge base history is preserved without manual intervention.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN one or more files are successfully imported, THE Ingestion_Service SHALL stage the new markdown files and commit them to the NoteGraph git repository.
|
||||
2. THE Ingestion_Service SHALL use the commit message format: `ingestion: import N notes from [source-type]` where N is the count and source-type describes the input (e.g., "pdf", "images", "bulk").
|
||||
3. WHEN stub pages are created by the Reference_Linker, THE Ingestion_Service SHALL include them in the same git commit.
|
||||
4. WHEN the `--no-commit` flag is provided on the CLI, THE Ingestion_Service SHALL skip the git commit step.
|
||||
5. IF the git commit fails, THEN THE Ingestion_Service SHALL log a warning but not fail the overall import process.
|
||||
|
||||
### Requirement 13: CLI Interface Design
|
||||
|
||||
**User Story:** As a user, I want a clear and consistent CLI interface, so that I can easily run imports with different options.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE Ingestion_Service SHALL provide a CLI entry point via `python -m ingestion.cli`.
|
||||
2. THE Ingestion_Service SHALL support the subcommand `import` accepting a path argument (file or directory).
|
||||
3. THE Ingestion_Service SHALL support the subcommand `watch` to start the Drop_Folder_Watcher.
|
||||
4. THE Ingestion_Service SHALL support the following global flags: `--verbose` for detailed logging, `--dry-run` to preview without writing files, `--no-commit` to skip git commits.
|
||||
5. THE Ingestion_Service SHALL support the following import flags: `--output-dir` to override output location, `--create-tasks` to enable OrgMyLife task creation, `--provider` to override the LLM provider for this run.
|
||||
6. WHEN the `--dry-run` flag is provided, THE Ingestion_Service SHALL display what would be created (file paths, detected entities, proposed links) without writing any files.
|
||||
7. WHEN invoked without arguments, THE Ingestion_Service SHALL display usage help with examples.
|
||||
@@ -0,0 +1,318 @@
|
||||
# Implementation Plan: NoteGraph Ingestion & Auto-Reference
|
||||
|
||||
## Overview
|
||||
|
||||
This plan implements the NoteGraph ingestion pipeline as a Python package at `NoteGraph/ingestion/`. Tasks are ordered so each builds on the previous: scaffolding → extraction → LLM → enrichment → linking → output → CLI → web upload → Nextcloud sync → OrgMyLife → git → Docker/deploy → tests. Property-based tests use Hypothesis and are placed close to the code they validate.
|
||||
|
||||
## Tasks
|
||||
|
||||
- [x] 1. Project scaffolding and configuration
|
||||
- [x] 1.1 Create `NoteGraph/ingestion/` package structure with `__init__.py`, `__main__.py`, and all subpackages (`extraction/`, `enrichment/`, `linking/`, `output/`, `integrations/`, `tests/`)
|
||||
- Create directory tree matching the design's package structure
|
||||
- Add empty `__init__.py` files in each subpackage
|
||||
- Create `__main__.py` with `from ingestion.cli import cli; cli()` entry point
|
||||
- _Requirements: 13.1_
|
||||
|
||||
- [x] 1.2 Create `NoteGraph/ingestion/pyproject.toml` with all dependencies
|
||||
- Include: click, watchdog, pytesseract, PyPDF2, pdfplumber, python-docx, litellm, httpx, pydantic, pydantic-settings, fastapi, uvicorn, python-multipart
|
||||
- Add dev dependencies: pytest, pytest-asyncio, hypothesis, pytest-mock, respx
|
||||
- Configure package metadata and entry points
|
||||
- _Requirements: 7.1, 8.7_
|
||||
|
||||
- [x] 1.3 Implement `NoteGraph/ingestion/config.py` with `IngestionConfig` (pydantic-settings)
|
||||
- Define all env vars: `LLM_PROVIDER`, `LLM_MODEL`, provider API keys, `ORGMYLIFE_API_URL`, `ORGMYLIFE_API_KEY`, `NEXTCLOUD_INBOX_URL`, `NEXTCLOUD_INBOX_USER`, `NEXTCLOUD_INBOX_PASSWORD`, `SB_USER`, paths, OCR settings, confidence threshold
|
||||
- Implement `llm_model_string` property for LiteLLM routing
|
||||
- Implement `validate_api_key()` that exits with clear error naming the missing variable
|
||||
- _Requirements: 8.1, 8.2, 8.3, 8.4, 8.5, 8.6_
|
||||
|
||||
- [x] 1.4 Create `NoteGraph/.env.example` with all supported environment variables and descriptions
|
||||
- Document every variable with comments explaining purpose and valid values
|
||||
- Include placeholders for: LLM keys, OrgMyLife, Nextcloud inbox, SB_USER
|
||||
- Ensure `.env` is in `.gitignore`
|
||||
- _Requirements: 8.5, 8.7_
|
||||
|
||||
- [x] 2. Text extraction layer
|
||||
- [x] 2.1 Implement `extraction/base.py` with `TextExtractor` protocol and `ExtractionResult` dataclass
|
||||
- Define `can_handle(file_path: Path) -> bool` and `extract(file_path: Path) -> ExtractionResult`
|
||||
- `ExtractionResult` fields: text, source_file, extraction_method, metadata
|
||||
- _Requirements: 2.1, 3.1_
|
||||
|
||||
- [x] 2.2 Implement `extraction/text.py` for plain text and markdown files
|
||||
- Handle `.md` and `.txt` extensions
|
||||
- Read as UTF-8, attempt encoding detection for non-UTF-8 files (chardet/charset-normalizer)
|
||||
- Preserve existing formatting, links, and Obsidian-style wiki-links
|
||||
- _Requirements: 3.1, 3.2, 3.4, 3.5, 3.6_
|
||||
|
||||
- [x] 2.3 Implement `extraction/pdf.py` for PDF text extraction
|
||||
- Use pdfplumber for text-based PDFs
|
||||
- Detect scanned pages (pages with no extractable text) and delegate to OCR
|
||||
- Handle mixed PDFs (text + scanned pages) by concatenating results
|
||||
- _Requirements: 2.2, 2.3, 2.4_
|
||||
|
||||
- [x] 2.4 Implement `extraction/ocr.py` for image OCR via pytesseract
|
||||
- Handle `.jpg`, `.jpeg`, `.png` files
|
||||
- Configure language as `deu+eng` (German + English)
|
||||
- Preserve paragraph structure by detecting line breaks
|
||||
- Return empty text (not error) when OCR cannot extract readable content
|
||||
- _Requirements: 2.1, 2.2, 2.5, 2.6_
|
||||
|
||||
- [x] 2.5 Implement `extraction/docx.py` for DOCX files
|
||||
- Use python-docx to extract text preserving heading structure and paragraph breaks
|
||||
- Handle `.docx` extension
|
||||
- _Requirements: 3.3_
|
||||
|
||||
- [ ]* 2.6 Write property test for text extraction identity (Property 3)
|
||||
- **Property 3: Text extraction preserves content for text-based formats**
|
||||
- For any valid UTF-8 string written to a .md or .txt file, extraction returns identical content
|
||||
- **Validates: Requirements 3.1, 3.2, 3.4**
|
||||
|
||||
- [x] 3. File discovery
|
||||
- [x] 3.1 Implement `discovery.py` with recursive file discovery and extension filtering
|
||||
- Support extensions: `.pdf`, `.jpg`, `.jpeg`, `.png`, `.md`, `.txt`, `.docx`
|
||||
- Accept both single file path and directory path
|
||||
- Recursive directory traversal
|
||||
- _Requirements: 1.1, 1.2, 1.6_
|
||||
|
||||
- [ ]* 3.2 Write property test for file discovery filtering (Property 1)
|
||||
- **Property 1: File discovery returns only supported extensions**
|
||||
- For any directory tree with arbitrary extensions, discovery returns exactly those with supported extensions
|
||||
- **Validates: Requirements 1.1, 1.6**
|
||||
|
||||
- [ ] 4. LLM provider layer
|
||||
- [x] 4.1 Implement `integrations/llm.py` with LiteLLM wrapper
|
||||
- `LLMClient` class with `complete(messages, response_format)` method
|
||||
- Route to provider based on `config.llm_model_string` (e.g., `openai/gpt-4o`, `anthropic/claude-sonnet-4-20250514`)
|
||||
- Implement retry logic: 3 retries with exponential backoff for rate limits, timeouts, 5xx errors
|
||||
- Raise descriptive exception on failure including provider name and error details
|
||||
- _Requirements: 7.1, 7.2, 7.3, 7.4, 7.5, 7.6, 7.7_
|
||||
|
||||
- [ ]* 4.2 Write property test for provider routing (Property 9)
|
||||
- **Property 9: Provider routing from configuration**
|
||||
- For any valid provider name, configuring `LLM_PROVIDER` causes the LLM client to route to the corresponding provider
|
||||
- **Validates: Requirements 7.3, 8.1**
|
||||
|
||||
- [ ]* 4.3 Write property test for missing API key error (Property 10)
|
||||
- **Property 10: Missing API key produces descriptive error**
|
||||
- For any provider requiring an API key, when the env var is unset, validation raises an error naming the missing variable
|
||||
- **Validates: Requirements 7.4, 8.6**
|
||||
|
||||
- [x] 5. Enrichment agent
|
||||
- [x] 5.1 Implement `enrichment/models.py` with Pydantic models
|
||||
- `Entity` model: type (person/project/date/action_item), value, confidence (0.0-1.0), position
|
||||
- `ActionItem` model: description, assignee, deadline (ISO 8601)
|
||||
- `EnrichmentResult` model: title, category, tags, entities, action_items, summary
|
||||
- _Requirements: 4.1, 4.2, 4.3, 4.4, 4.5_
|
||||
|
||||
- [x] 5.2 Implement `enrichment/prompts.py` with structured LLM prompts
|
||||
- System prompt requesting structured JSON output for entity detection
|
||||
- Include context about known projects and people (read from existing NoteGraph pages)
|
||||
- Request confidence scores for each entity
|
||||
- _Requirements: 4.1, 4.2, 4.3, 4.4, 4.5_
|
||||
|
||||
- [x] 5.3 Implement `enrichment/agent.py` with enrichment orchestration
|
||||
- Call LLM with extracted text and structured prompt
|
||||
- Parse response into `EnrichmentResult`
|
||||
- Filter entities by confidence threshold (>= 0.7)
|
||||
- Handle LLM parse errors: retry once with stricter prompt, then use defaults
|
||||
- _Requirements: 4.1, 4.2, 4.3, 4.4, 4.5, 4.6_
|
||||
|
||||
- [ ]* 5.4 Write property tests for entity parsing (Properties 4, 5)
|
||||
- **Property 4: Entity parsing extracts all entities with valid structure**
|
||||
- For any valid enrichment response JSON, each entity has valid type, non-empty value, confidence in [0.0, 1.0]
|
||||
- **Property 5: Confidence threshold filtering**
|
||||
- For any list of entities, filtered output contains exactly those with confidence >= 0.7
|
||||
- **Validates: Requirements 4.1, 4.2, 4.3, 4.4, 4.5, 4.6**
|
||||
|
||||
- [x] 6. Checkpoint - Ensure all tests pass
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
- [x] 7. Reference linker
|
||||
- [x] 7.1 Implement `linking/linker.py` with wiki-link generation
|
||||
- Person entities → `[[people/firstname-lastname]]` (slugified)
|
||||
- Project entities → `[[projects/project-slug]]` (slugified)
|
||||
- Insert wiki-link inline at entity position (first occurrence only)
|
||||
- Leave subsequent mentions of the same entity as plain text
|
||||
- Add dates to frontmatter `date` field
|
||||
- _Requirements: 5.1, 5.2, 5.3, 5.5, 5.6_
|
||||
|
||||
- [x] 7.2 Implement `linking/stubs.py` for stub page creation
|
||||
- Check if target page exists in NoteGraph notes directory
|
||||
- Create stub page with entity name as title if it doesn't exist
|
||||
- Stub pages go in `notes/people/` or `notes/projects/` as appropriate
|
||||
- _Requirements: 5.4_
|
||||
|
||||
- [ ]* 7.3 Write property tests for wiki-link generation (Properties 6, 7)
|
||||
- **Property 6: Wiki-link generation follows correct format per entity type**
|
||||
- For any person name → `[[people/slugified-name]]`; for any project name → `[[projects/slugified-name]]`
|
||||
- **Property 7: Only first entity occurrence is linked**
|
||||
- For any text with N >= 2 occurrences of the same entity, only the first is converted to a wiki-link
|
||||
- **Validates: Requirements 5.1, 5.2, 5.6**
|
||||
|
||||
- [x] 8. Output renderer and folder routing
|
||||
- [x] 8.1 Implement `output/renderer.py` for markdown + YAML frontmatter rendering
|
||||
- Render frontmatter with: title, date, tags, people, projects, source (file + timestamp)
|
||||
- Render body with wiki-links inserted
|
||||
- Render `## Action Items` section at the end with extracted action items
|
||||
- _Requirements: 6.1, 6.2, 6.3, 6.4, 6.5, 6.6, 9.2_
|
||||
|
||||
- [x] 8.2 Implement `output/router.py` for category-based folder routing
|
||||
- "meeting" → `notes/meetings/`
|
||||
- "project" → `notes/projects/`
|
||||
- "decision" → `notes/decisions/`
|
||||
- Default/inbox → `notes/inbox/`
|
||||
- Support `--output-dir` override
|
||||
- _Requirements: 10.1, 10.2, 10.3, 10.4_
|
||||
|
||||
- [x] 8.3 Implement `output/naming.py` for filename generation with collision avoidance
|
||||
- Pattern: `YYYY-MM-DD-slugified-title.md`
|
||||
- Slug: lowercase alphanumeric + hyphens only
|
||||
- Append numeric suffix (`-2`, `-3`) if file already exists
|
||||
- _Requirements: 10.5, 10.6_
|
||||
|
||||
- [ ]* 8.4 Write property tests for output (Properties 8, 11, 13, 14, 15)
|
||||
- **Property 8: Frontmatter round-trip validity**
|
||||
- For any valid EnrichmentResult, render to markdown then parse frontmatter → valid YAML with all required fields
|
||||
- **Property 11: Action items appear in output markdown section**
|
||||
- For any list of action items, rendered markdown contains `## Action Items` section listing every item
|
||||
- **Property 13: Category-based folder routing**
|
||||
- For any category, output path is within the correct folder
|
||||
- **Property 14: Output filename follows date-slug pattern**
|
||||
- For any date and title, filename matches `YYYY-MM-DD-slugified-title.md`
|
||||
- **Property 15: Filename collision avoidance**
|
||||
- When a file with the same name exists, a numeric suffix is appended
|
||||
- **Validates: Requirements 6.1, 6.3, 6.4, 6.6, 6.7, 9.2, 10.1, 10.2, 10.3, 10.5, 10.6**
|
||||
|
||||
- [x] 9. Pipeline orchestration
|
||||
- [x] 9.1 Implement `pipeline.py` with main processing pipeline
|
||||
- Orchestrate: discovery → extraction → enrichment → linking → rendering → file write → git commit
|
||||
- Track progress: current file number, total count, file name
|
||||
- Handle errors per file: log + skip + continue
|
||||
- Output summary: total processed, success count, failed count
|
||||
- Support `--dry-run` mode (display what would be created without writing)
|
||||
- _Requirements: 1.3, 1.4, 1.5, 13.6_
|
||||
|
||||
- [ ]* 9.2 Write property tests for pipeline resilience and dry-run (Properties 2, 17)
|
||||
- **Property 2: Bulk import resilience — failed files do not block remaining processing**
|
||||
- For any batch where a subset fails, remaining files are processed successfully
|
||||
- **Property 17: Dry-run produces no file writes**
|
||||
- For any input with `--dry-run`, zero files are created on disk
|
||||
- **Validates: Requirements 1.4, 1.5, 13.6**
|
||||
|
||||
- [x] 10. CLI interface
|
||||
- [x] 10.1 Implement `cli.py` with Click-based CLI
|
||||
- `import` subcommand: accepts path argument (file or directory)
|
||||
- `watch` subcommand: starts drop-folder watcher
|
||||
- Global flags: `--verbose`, `--dry-run`, `--no-commit`
|
||||
- Import flags: `--output-dir`, `--create-tasks`, `--provider`
|
||||
- Display usage help with examples when invoked without arguments
|
||||
- _Requirements: 13.1, 13.2, 13.3, 13.4, 13.5, 13.6, 13.7_
|
||||
|
||||
- [x] 11. Checkpoint - Ensure all tests pass
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
- [ ] 12. Web upload endpoint (FastAPI)
|
||||
- [x] 12.1 Implement `web/upload.py` with FastAPI app for file upload
|
||||
- HTML form with file upload accepting PDF, JPG, PNG
|
||||
- Mobile-friendly responsive HTML page (inline CSS, no external dependencies)
|
||||
- Basic auth protection using `SB_USER` env var (same credentials as SilverBullet)
|
||||
- On upload: save file to inbox folder, trigger processing pipeline
|
||||
- Return success/error response to the user
|
||||
- _Requirements: 11.2 (extends with web input)_
|
||||
|
||||
- [x] 12.2 Create `web/templates/upload.html` mobile-friendly upload page
|
||||
- Simple, clean form with drag-and-drop support
|
||||
- Responsive design for Android "Share to URL" workflow
|
||||
- Show upload progress and result feedback
|
||||
- _Requirements: (additional requirement: web upload for mobile)_
|
||||
|
||||
- [ ] 13. Nextcloud WebDAV inbox sync
|
||||
- [x] 13.1 Implement `integrations/nextcloud.py` with WebDAV folder polling
|
||||
- Poll `NEXTCLOUD_INBOX_URL` for new files using httpx WebDAV (PROPFIND/GET)
|
||||
- Download new files to local inbox for processing
|
||||
- After successful processing, move files to a `processed/` subfolder on Nextcloud (MOVE request)
|
||||
- Auth via `NEXTCLOUD_INBOX_USER` and `NEXTCLOUD_INBOX_PASSWORD` env vars
|
||||
- _Requirements: (additional requirement: Nextcloud WebDAV sync)_
|
||||
|
||||
- [x] 13.2 Add `sync` subcommand to CLI for Nextcloud polling
|
||||
- One-shot sync: poll once, process new files, exit
|
||||
- Can be called from cron or systemd timer
|
||||
- _Requirements: (additional requirement: Nextcloud WebDAV sync)_
|
||||
|
||||
- [x] 14. Drop-folder watcher
|
||||
- [x] 14.1 Implement `watcher.py` with watchdog-based folder monitoring
|
||||
- Monitor inbox folder for new files
|
||||
- Trigger pipeline on new file detection
|
||||
- Move processed files to `archive/` subdirectory
|
||||
- Move failed files to `failed/` subdirectory with error log
|
||||
- Ignore temporary files (names starting with `.` or ending with `.tmp`)
|
||||
- Process existing files in inbox on startup before entering watch mode
|
||||
- _Requirements: 11.1, 11.2, 11.3, 11.4, 11.5, 11.6_
|
||||
|
||||
- [ ]* 14.2 Write property test for temporary file filtering (Property 16)
|
||||
- **Property 16: Temporary file filtering**
|
||||
- For any filename, ignore logic returns true iff name starts with `.` or ends with `.tmp`
|
||||
- **Validates: Requirements 11.5**
|
||||
|
||||
- [x] 15. OrgMyLife integration
|
||||
- [x] 15.1 Implement `integrations/orgmylife.py` with HTTP client for task creation
|
||||
- POST to `/api/tasks` with title, source_url, description
|
||||
- Auth via `ORGMYLIFE_API_KEY` header
|
||||
- Handle unreachable API gracefully: log warning, continue without creating tasks
|
||||
- _Requirements: 9.3, 9.4, 9.5, 9.6_
|
||||
|
||||
- [ ]* 15.2 Write property test for OrgMyLife task payload (Property 12)
|
||||
- **Property 12: OrgMyLife task payload correctness**
|
||||
- For any action item description and note path, payload has correct title and source_url
|
||||
- **Validates: Requirements 9.4**
|
||||
|
||||
- [x] 16. Git auto-commit
|
||||
- [x] 16.1 Implement `integrations/git.py` for git commit operations
|
||||
- Stage new markdown files and stub pages
|
||||
- Commit with message format: `ingestion: import N notes from [source-type]`
|
||||
- Support `--no-commit` flag to skip
|
||||
- Handle git failures gracefully: log warning, don't fail the import
|
||||
- _Requirements: 12.1, 12.2, 12.3, 12.4, 12.5_
|
||||
|
||||
- [ ]* 16.2 Write property test for git commit message format (Property 18)
|
||||
- **Property 18: Git commit message format**
|
||||
- For any count N and source type, message matches `ingestion: import N notes from [source-type]`
|
||||
- **Validates: Requirements 12.2**
|
||||
|
||||
- [x] 17. Checkpoint - Ensure all tests pass
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
- [x] 18. Docker integration and deployment
|
||||
- [x] 18.1 Create `NoteGraph/ingestion/Dockerfile` for the ingestion service
|
||||
- Python 3.11+ base image
|
||||
- Install system dependencies: tesseract-ocr, tesseract-ocr-deu
|
||||
- Install Python package with all dependencies
|
||||
- Run FastAPI upload server (uvicorn) as default command
|
||||
- Also support CLI entry point for import/watch/sync commands
|
||||
- _Requirements: 13.1_
|
||||
|
||||
- [x] 18.2 Update `NoteGraph/docker-compose.yml` to add ingestion service
|
||||
- Add `ingestion` service with build context `./ingestion`
|
||||
- Mount `./notes` volume for output
|
||||
- Mount `./ingestion/inbox` for drop-folder input
|
||||
- Expose upload endpoint port (8001)
|
||||
- Pass all env vars from `.env`
|
||||
- Add watcher mode as default or separate service
|
||||
- _Requirements: 11.1_
|
||||
|
||||
- [x] 18.3 Update `NoteGraph/.github/workflows/deploy.yml` with all secrets
|
||||
- Write ALL secrets from GitHub secrets into `.env` on VPS: LLM keys (OpenAI, Anthropic, Google, Mistral), Nextcloud inbox creds, OrgMyLife API key, SB_USER
|
||||
- Follow same pattern as OrgMyLife deploy workflow (ENV_CONTENT with printf)
|
||||
- Add `docker compose up -d --build` to rebuild with ingestion service
|
||||
- _Requirements: 8.5 (additional requirement: secrets via GitHub Actions)_
|
||||
|
||||
- [x] 19. Final checkpoint - Ensure all tests pass
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
## Notes
|
||||
|
||||
- Tasks marked with `*` are optional property-based tests and can be skipped for faster MVP
|
||||
- Each task references specific requirements for traceability
|
||||
- Checkpoints ensure incremental validation
|
||||
- Property tests use Hypothesis with `@settings(max_examples=100)`
|
||||
- The web upload endpoint and Nextcloud sync are additional input methods beyond the original CLI/watcher
|
||||
- All secrets are managed via GitHub Actions → `.env` on VPS, never committed to git
|
||||
@@ -0,0 +1 @@
|
||||
{"specId": "d26ccd62-45f2-47f0-8fbb-e15f4f79916d", "workflowType": "requirements-first", "specType": "feature"}
|
||||
@@ -0,0 +1,579 @@
|
||||
# Design Document: PAT Renewal
|
||||
|
||||
## Overview
|
||||
|
||||
The PAT Renewal system automates Personal Access Token lifecycle management across GitLab, Jira, Confluence, and OrgMyLife services. It runs as a GitHub Actions scheduled workflow (weekly, Mondays at 06:00 UTC) that:
|
||||
|
||||
1. Reads a PAT Registry (JSON) tracking all managed tokens
|
||||
2. Checks each token's expiry status via service-specific API calls
|
||||
3. Auto-rotates GitLab PATs using the GitLab Personal Access Tokens API
|
||||
4. Creates OrgMyLife tasks or sends email alerts for tokens requiring manual renewal
|
||||
5. Produces a summary report in the GitHub Actions run log
|
||||
|
||||
The system integrates with the existing GitHub Actions infrastructure already used in the workspace (deploy workflows, OrgMyLife sync) and leverages the OrgMyLife API for task creation alerts.
|
||||
|
||||
**Key Design Decisions:**
|
||||
- **Python-based workflow script**: Matches the workspace's primary language (Python 3.11+) and allows reuse of patterns from the AI-Orchestrator project (httpx for HTTP, pytest for testing)
|
||||
- **JSON registry file**: Simple, version-controllable, human-readable format stored in the repository
|
||||
- **GitHub Actions secrets for token storage**: Leverages existing secure secret management without introducing new infrastructure
|
||||
- **Idempotent alerting**: Prevents duplicate OrgMyLife tasks by checking for existing open tasks before creating new ones
|
||||
|
||||
## Architecture
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
subgraph "GitHub Actions"
|
||||
CRON[Cron Trigger<br/>Monday 06:00 UTC] --> WF[pat-check Workflow]
|
||||
DISPATCH[workflow_dispatch] --> WF
|
||||
WF --> SCRIPT[pat_manager.py]
|
||||
end
|
||||
|
||||
subgraph "PAT Manager Script"
|
||||
SCRIPT --> LOAD[Load PAT Registry]
|
||||
LOAD --> CHECK[Expiry Checker]
|
||||
CHECK --> CLASSIFY{Classify PATs}
|
||||
CLASSIFY -->|expiring soon + auto| ROTATE[GitLab Rotator]
|
||||
CLASSIFY -->|expiring soon + manual| ALERT[Alert Sender]
|
||||
CLASSIFY -->|expired| ALERT
|
||||
CLASSIFY -->|healthy| REPORT[Summary Reporter]
|
||||
ROTATE --> SECRET_UPDATE[Update GitHub Secret]
|
||||
ROTATE -->|failure| ALERT
|
||||
SECRET_UPDATE --> REPORT
|
||||
ALERT --> REPORT
|
||||
end
|
||||
|
||||
subgraph "External Services"
|
||||
CHECK --> GITLAB_API[GitLab API]
|
||||
CHECK --> JIRA_API[Jira API]
|
||||
CHECK --> CONFLUENCE_API[Confluence API]
|
||||
CHECK --> ORGMYLIFE_API[OrgMyLife API]
|
||||
ROTATE --> GITLAB_API
|
||||
SECRET_UPDATE --> GH_API[GitHub API]
|
||||
ALERT --> ORGMYLIFE_API
|
||||
ALERT --> EMAIL[Email / SMTP]
|
||||
end
|
||||
|
||||
subgraph "Storage"
|
||||
LOAD --> REGISTRY[pat-registry.json]
|
||||
REPORT --> LOG[Workflow Run Log]
|
||||
end
|
||||
```
|
||||
|
||||
### Component Interaction Flow
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant GHA as GitHub Actions
|
||||
participant PM as pat_manager.py
|
||||
participant REG as pat-registry.json
|
||||
participant GL as GitLab API
|
||||
participant JIRA as Jira API
|
||||
participant OML as OrgMyLife API
|
||||
participant GH as GitHub API
|
||||
|
||||
GHA->>PM: Trigger (cron/dispatch)
|
||||
PM->>REG: Load registry
|
||||
REG-->>PM: PAT entries
|
||||
|
||||
loop For each PAT
|
||||
alt GitLab PAT
|
||||
PM->>GL: GET /personal_access_tokens
|
||||
GL-->>PM: expires_at
|
||||
else Jira/Confluence/OrgMyLife PAT
|
||||
PM->>JIRA: Authenticated API call
|
||||
JIRA-->>PM: 200 OK / 401 Unauthorized
|
||||
end
|
||||
PM->>PM: Classify (healthy/expiring/expired/check failed)
|
||||
end
|
||||
|
||||
alt GitLab PAT expiring soon
|
||||
PM->>GL: POST /personal_access_tokens/{id}/rotate
|
||||
GL-->>PM: New token + expiry
|
||||
PM->>GH: Update secret
|
||||
PM->>REG: Update expiry_date
|
||||
end
|
||||
|
||||
alt Manual token expiring/expired
|
||||
PM->>OML: POST /api/tasks (create alert task)
|
||||
end
|
||||
|
||||
PM->>GHA: Summary report (log output)
|
||||
```
|
||||
|
||||
## Components and Interfaces
|
||||
|
||||
### 1. PAT Registry Module (`registry.py`)
|
||||
|
||||
Responsible for loading, validating, and persisting the PAT registry JSON file.
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class PatEntry:
|
||||
service: str # "GitLab" | "Jira" | "Confluence" | "OrgMyLife"
|
||||
token_name: str # max 128 chars
|
||||
expiry_date: str | None # ISO 8601 (YYYY-MM-DD) or None
|
||||
renewal_method: str # "auto" | "manual"
|
||||
|
||||
class PatRegistry:
|
||||
def load(self, path: Path) -> list[PatEntry]: ...
|
||||
def validate_entry(self, entry: dict) -> PatEntry | ValidationError: ...
|
||||
def save(self, path: Path, entries: list[PatEntry]) -> None: ...
|
||||
def create_empty(self, path: Path) -> None: ...
|
||||
```
|
||||
|
||||
**Validation Rules:**
|
||||
- `service` must be one of: `GitLab`, `Jira`, `Confluence`, `OrgMyLife`
|
||||
- `token_name` must be non-empty, max 128 characters
|
||||
- `expiry_date` must be valid ISO 8601 (YYYY-MM-DD) or null
|
||||
- `renewal_method` must match service: GitLab → "auto", others → "manual"
|
||||
- No duplicate `(service, token_name)` combinations
|
||||
|
||||
### 2. Expiry Checker Module (`checker.py`)
|
||||
|
||||
Queries each service API to determine PAT status.
|
||||
|
||||
```python
|
||||
class PatStatus(Enum):
|
||||
HEALTHY = "healthy"
|
||||
EXPIRING_SOON = "expiring soon"
|
||||
EXPIRED = "expired"
|
||||
CHECK_FAILED = "check failed"
|
||||
|
||||
@dataclass
|
||||
class CheckResult:
|
||||
entry: PatEntry
|
||||
status: PatStatus
|
||||
days_remaining: int | None
|
||||
error_message: str | None
|
||||
|
||||
class ExpiryChecker:
|
||||
def __init__(self, expiry_window_days: int = 14, timeout: int = 30, max_retries: int = 2): ...
|
||||
async def check_all(self, entries: list[PatEntry], secrets: dict[str, str]) -> list[CheckResult]: ...
|
||||
async def check_gitlab(self, entry: PatEntry, token: str) -> CheckResult: ...
|
||||
async def check_jira(self, entry: PatEntry, token: str) -> CheckResult: ...
|
||||
async def check_confluence(self, entry: PatEntry, token: str) -> CheckResult: ...
|
||||
async def check_orgmylife(self, entry: PatEntry, token: str) -> CheckResult: ...
|
||||
```
|
||||
|
||||
**Service-Specific Logic:**
|
||||
- **GitLab**: `GET /personal_access_tokens` → read `expires_at` field, compare with window
|
||||
- **Jira**: Authenticated call → HTTP 401 = expired
|
||||
- **Confluence**: Authenticated call → HTTP 401 = expired
|
||||
- **OrgMyLife**: Authenticated call → HTTP 401/403 = expired
|
||||
|
||||
### 3. GitLab Rotator Module (`rotator.py`)
|
||||
|
||||
Handles automatic token rotation for GitLab PATs.
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class RotationResult:
|
||||
success: bool
|
||||
new_expiry_date: str | None
|
||||
error_message: str | None
|
||||
|
||||
class GitLabRotator:
|
||||
def __init__(self, timeout: int = 30): ...
|
||||
async def rotate(self, token_id: str, current_token: str) -> RotationResult: ...
|
||||
```
|
||||
|
||||
### 4. Secret Updater Module (`secret_updater.py`)
|
||||
|
||||
Updates GitHub Actions secrets with new token values.
|
||||
|
||||
```python
|
||||
class SecretUpdater:
|
||||
def __init__(self, github_token: str, repo_owner: str, repo_name: str): ...
|
||||
async def update_secret(self, secret_name: str, secret_value: str) -> bool: ...
|
||||
```
|
||||
|
||||
### 5. Alert Sender Module (`alerter.py`)
|
||||
|
||||
Sends alerts via OrgMyLife task creation or email.
|
||||
|
||||
```python
|
||||
class AlertChannel(Enum):
|
||||
ORGMYLIFE = "orgmylife"
|
||||
EMAIL = "email"
|
||||
|
||||
@dataclass
|
||||
class AlertResult:
|
||||
sent: bool
|
||||
channel: AlertChannel
|
||||
error_message: str | None
|
||||
|
||||
class AlertSender:
|
||||
def __init__(self, channel: AlertChannel, config: dict): ...
|
||||
async def send_alert(self, result: CheckResult) -> AlertResult: ...
|
||||
async def check_existing_task(self, service: str, token_name: str, expiry_date: str) -> bool: ...
|
||||
```
|
||||
|
||||
**OrgMyLife Integration:**
|
||||
- Uses `POST /api/tasks` with `X-API-Key` header for authentication
|
||||
- Task title format: `[PAT Renewal] {service} - {token_name}`
|
||||
- Includes days remaining and renewal instructions in description
|
||||
- Checks for existing open tasks before creating duplicates
|
||||
|
||||
### 6. Summary Reporter Module (`reporter.py`)
|
||||
|
||||
Produces the workflow summary log.
|
||||
|
||||
```python
|
||||
class SummaryReporter:
|
||||
def report(self, results: list[CheckResult], rotations: list[RotationResult]) -> str: ...
|
||||
```
|
||||
|
||||
### 7. Main Orchestrator (`pat_manager.py`)
|
||||
|
||||
Entry point that coordinates all modules.
|
||||
|
||||
```python
|
||||
async def main() -> int:
|
||||
"""Returns exit code: 0 for success, 1 for failures."""
|
||||
...
|
||||
```
|
||||
|
||||
## Data Models
|
||||
|
||||
### PAT Registry JSON Schema
|
||||
|
||||
```json
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"version": { "type": "string", "const": "1.0" },
|
||||
"tokens": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"service": {
|
||||
"type": "string",
|
||||
"enum": ["GitLab", "Jira", "Confluence", "OrgMyLife"]
|
||||
},
|
||||
"token_name": {
|
||||
"type": "string",
|
||||
"maxLength": 128,
|
||||
"minLength": 1
|
||||
},
|
||||
"expiry_date": {
|
||||
"type": ["string", "null"],
|
||||
"pattern": "^\\d{4}-\\d{2}-\\d{2}$"
|
||||
},
|
||||
"renewal_method": {
|
||||
"type": "string",
|
||||
"enum": ["auto", "manual"]
|
||||
}
|
||||
},
|
||||
"required": ["service", "token_name", "renewal_method"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["version", "tokens"]
|
||||
}
|
||||
```
|
||||
|
||||
### Example PAT Registry
|
||||
|
||||
```json
|
||||
{
|
||||
"version": "1.0",
|
||||
"tokens": [
|
||||
{
|
||||
"service": "GitLab",
|
||||
"token_name": "ci-pipeline-token",
|
||||
"expiry_date": "2025-08-15",
|
||||
"renewal_method": "auto"
|
||||
},
|
||||
{
|
||||
"service": "Jira",
|
||||
"token_name": "jira-api-access",
|
||||
"expiry_date": "2025-07-20",
|
||||
"renewal_method": "manual"
|
||||
},
|
||||
{
|
||||
"service": "Confluence",
|
||||
"token_name": "confluence-bot",
|
||||
"expiry_date": "2025-09-01",
|
||||
"renewal_method": "manual"
|
||||
},
|
||||
{
|
||||
"service": "OrgMyLife",
|
||||
"token_name": "orchestrator-api-key",
|
||||
"expiry_date": null,
|
||||
"renewal_method": "manual"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Check Result Data Model
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class CheckResult:
|
||||
entry: PatEntry
|
||||
status: PatStatus # healthy | expiring soon | expired | check failed
|
||||
days_remaining: int | None # None if check failed or no expiry date
|
||||
error_message: str | None # Populated only for check_failed
|
||||
stale_warning: bool = False # True if API succeeds but stored date has passed
|
||||
```
|
||||
|
||||
### GitHub Actions Workflow Configuration
|
||||
|
||||
```yaml
|
||||
name: PAT Lifecycle Check
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 6 * * 1' # Every Monday at 06:00 UTC
|
||||
workflow_dispatch: {}
|
||||
|
||||
jobs:
|
||||
check-pats:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- name: Install dependencies
|
||||
run: pip install httpx
|
||||
- name: Run PAT Manager
|
||||
env:
|
||||
GITLAB_PAT: ${{ secrets.GITLAB_PAT }}
|
||||
JIRA_PAT: ${{ secrets.JIRA_PAT }}
|
||||
CONFLUENCE_PAT: ${{ secrets.CONFLUENCE_PAT }}
|
||||
ORGMYLIFE_API_KEY: ${{ secrets.ORGMYLIFE_API_SECRET }}
|
||||
ORGMYLIFE_USER: ${{ secrets.ORGMYLIFE_USER }}
|
||||
ORGMYLIFE_PASS: ${{ secrets.ORGMYLIFE_PASS }}
|
||||
GH_TOKEN: ${{ secrets.PAT_MANAGER_TOKEN }}
|
||||
ALERT_CHANNEL: orgmylife
|
||||
run: python scripts/pat_manager.py
|
||||
```
|
||||
|
||||
### Environment Variables / Secrets Mapping
|
||||
|
||||
| Secret Name | Purpose | Used By |
|
||||
|---|---|---|
|
||||
| `GITLAB_PAT` | GitLab API access + rotation source | Checker, Rotator |
|
||||
| `JIRA_PAT` | Jira authenticated check | Checker |
|
||||
| `CONFLUENCE_PAT` | Confluence authenticated check | Checker |
|
||||
| `ORGMYLIFE_API_SECRET` | OrgMyLife API key for task creation | Alerter |
|
||||
| `ORGMYLIFE_USER` | OrgMyLife basic auth username | Alerter |
|
||||
| `ORGMYLIFE_PASS` | OrgMyLife basic auth password | Alerter |
|
||||
| `PAT_MANAGER_TOKEN` | GitHub API token for secret updates | SecretUpdater |
|
||||
|
||||
## Correctness Properties
|
||||
|
||||
*A property is a characteristic or behavior that should hold true across all valid executions of a system — essentially, a formal statement about what the system should do. Properties serve as the bridge between human-readable specifications and machine-verifiable correctness guarantees.*
|
||||
|
||||
### Property 1: Registry serialization round-trip
|
||||
|
||||
*For any* valid list of PAT entries, serializing the registry to JSON and deserializing it back SHALL produce an equivalent list of entries with identical field values.
|
||||
|
||||
**Validates: Requirements 1.1**
|
||||
|
||||
### Property 2: Validation accepts only valid entries
|
||||
|
||||
*For any* dictionary representing a PAT entry, the validator SHALL accept it if and only if it contains all required fields (`service`, `token_name`, `expiry_date`, `renewal_method`), the `service` is one of the four supported values, the `token_name` is non-empty and at most 128 characters, the `expiry_date` is either null or a valid ISO 8601 date string, and the `renewal_method` matches the service (GitLab → "auto", others → "manual").
|
||||
|
||||
**Validates: Requirements 1.2, 1.4**
|
||||
|
||||
### Property 3: Invalid entries and duplicates preserve registry state
|
||||
|
||||
*For any* PAT registry and any entry that is either invalid (fails validation) or a duplicate (same service + token_name already exists), attempting to add that entry SHALL leave the registry unchanged — the entry count and all existing entries remain identical.
|
||||
|
||||
**Validates: Requirements 1.3, 1.5**
|
||||
|
||||
### Property 4: Expiry classification is deterministic and correct
|
||||
|
||||
*For any* PAT entry with an expiry date and any reference date, the classification SHALL be:
|
||||
- "expired" if `expiry_date < reference_date`
|
||||
- "expiring soon" if `reference_date <= expiry_date <= reference_date + expiry_window`
|
||||
- "healthy" if `expiry_date > reference_date + expiry_window`
|
||||
|
||||
This holds regardless of whether the expiry date comes from the service API or the stored registry value.
|
||||
|
||||
**Validates: Requirements 2.2, 2.3, 2.4, 6.5**
|
||||
|
||||
### Property 5: Classification completeness
|
||||
|
||||
*For any* PAT registry with N entries, after running the expiry check, the system SHALL produce exactly N classification results, each with exactly one status from the set {"healthy", "expiring soon", "expired", "check failed"}.
|
||||
|
||||
**Validates: Requirements 2.7**
|
||||
|
||||
### Property 6: Registry update after successful rotation
|
||||
|
||||
*For any* GitLab PAT entry and any valid rotation response containing a new expiry date, after successful rotation the registry entry's `expiry_date` SHALL equal the new expiry date from the rotation response.
|
||||
|
||||
**Validates: Requirements 3.3**
|
||||
|
||||
### Property 7: Alert triggering correctness
|
||||
|
||||
*For any* set of classified PATs:
|
||||
- If ALL PATs have status "healthy", THEN zero alerts SHALL be sent
|
||||
- For each manual-renewal PAT with status "expiring soon" or "expired", exactly one alert SHALL be triggered
|
||||
|
||||
**Validates: Requirements 4.1, 5.4**
|
||||
|
||||
### Property 8: Alert message completeness
|
||||
|
||||
*For any* PAT entry requiring an alert:
|
||||
- If the alert channel is OrgMyLife, the task title SHALL contain both the service name and token identifier
|
||||
- If the alert channel is email, the email body SHALL contain the service name, token identifier, expiration date, and a renewal instructions URL
|
||||
|
||||
**Validates: Requirements 4.2, 4.3**
|
||||
|
||||
### Property 9: Days remaining calculation
|
||||
|
||||
*For any* expiry date and reference date, the computed `days_remaining` SHALL equal `max(0, (expiry_date - reference_date).days)` — yielding 0 for already-expired tokens and the correct positive integer for future dates.
|
||||
|
||||
**Validates: Requirements 4.4**
|
||||
|
||||
### Property 10: Alert idempotence
|
||||
|
||||
*For any* PAT that requires an alert, if an open OrgMyLife task already exists for the same service, token_name, and expiry_date, THEN no new task SHALL be created. Running the alert flow twice for the same PAT state SHALL produce at most one task.
|
||||
|
||||
**Validates: Requirements 4.5**
|
||||
|
||||
### Property 11: Summary report completeness
|
||||
|
||||
*For any* list of check results, the generated summary report SHALL contain one line per result, and each line SHALL include the service name, token identifier, and classification status.
|
||||
|
||||
**Validates: Requirements 5.3**
|
||||
|
||||
### Property 12: Non-auth HTTP errors yield "check failed"
|
||||
|
||||
*For any* HTTP response status code that is not 200-299, 401, or 403, the classification SHALL be "check failed". This includes 4xx errors (other than 401/403), 5xx errors, and timeout conditions.
|
||||
|
||||
**Validates: Requirements 6.6**
|
||||
|
||||
## Error Handling
|
||||
|
||||
### Retry Strategy
|
||||
|
||||
| Operation | Max Retries | Backoff | Fallback |
|
||||
|---|---|---|---|
|
||||
| Service API health check | 2 | 5s linear | Classify as "check failed" |
|
||||
| GitLab token rotation | 1 | None | Manual alert flow |
|
||||
| GitHub secret update | 0 | N/A | Alert + non-zero exit |
|
||||
| OrgMyLife task creation | 1 | 2s | Log error + non-zero exit |
|
||||
| Email sending | 1 | 2s | Log error + non-zero exit |
|
||||
|
||||
### Error Classification
|
||||
|
||||
```python
|
||||
class PatManagerError(Exception):
|
||||
"""Base exception for PAT Manager errors."""
|
||||
pass
|
||||
|
||||
class RegistryValidationError(PatManagerError):
|
||||
"""Raised when a registry entry fails validation."""
|
||||
def __init__(self, entry: dict, failed_fields: list[str]): ...
|
||||
|
||||
class ServiceCheckError(PatManagerError):
|
||||
"""Raised when a service API check fails after retries."""
|
||||
def __init__(self, service: str, token_name: str, reason: str): ...
|
||||
|
||||
class RotationError(PatManagerError):
|
||||
"""Raised when GitLab token rotation fails."""
|
||||
def __init__(self, token_name: str, http_status: int | None, reason: str): ...
|
||||
|
||||
class SecretUpdateError(PatManagerError):
|
||||
"""Raised when GitHub secret update fails."""
|
||||
def __init__(self, secret_name: str, reason: str): ...
|
||||
|
||||
class AlertError(PatManagerError):
|
||||
"""Raised when alert delivery fails."""
|
||||
def __init__(self, channel: str, reason: str): ...
|
||||
```
|
||||
|
||||
### Exit Codes
|
||||
|
||||
| Code | Meaning |
|
||||
|---|---|
|
||||
| 0 | All checks passed, all actions succeeded |
|
||||
| 1 | One or more alerts could not be delivered |
|
||||
| 2 | Registry validation failed (malformed registry file) |
|
||||
|
||||
### Security Error Handling
|
||||
|
||||
- Token values are masked via `::add-mask::{value}` before any operation that might log output
|
||||
- On rotation failure, the old token remains valid — no service disruption
|
||||
- On secret update failure, the new token exists in GitLab but the workflow still uses the old one — alert is critical
|
||||
- All HTTP errors are logged without including request/response bodies that might contain tokens
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
### Property-Based Tests (Hypothesis)
|
||||
|
||||
The project uses Python 3.11+ with pytest. Property-based tests will use the **Hypothesis** library, which is the standard PBT framework for Python.
|
||||
|
||||
**Configuration:**
|
||||
- Minimum 100 examples per property test (`@settings(max_examples=100)`)
|
||||
- Each test tagged with a comment referencing the design property
|
||||
- Tag format: `# Feature: pat-renewal, Property {number}: {property_text}`
|
||||
|
||||
**Properties to implement:**
|
||||
1. Registry round-trip serialization
|
||||
2. Validation correctness (accepts valid, rejects invalid)
|
||||
3. Invalid/duplicate entries preserve state
|
||||
4. Expiry classification determinism
|
||||
5. Classification completeness
|
||||
6. Registry update after rotation
|
||||
7. Alert triggering correctness
|
||||
8. Alert message completeness
|
||||
9. Days remaining calculation
|
||||
10. Alert idempotence
|
||||
11. Summary report completeness
|
||||
12. Non-auth HTTP errors → "check failed"
|
||||
|
||||
**Generators needed:**
|
||||
- `pat_entries()` — generates valid PatEntry instances with random services, token names, dates
|
||||
- `invalid_entries()` — generates dictionaries missing fields or with invalid values
|
||||
- `expiry_dates(relative_to)` — generates dates in various ranges (past, within window, beyond window)
|
||||
- `http_status_codes()` — generates non-success, non-auth status codes
|
||||
- `check_results()` — generates lists of CheckResult with various statuses
|
||||
|
||||
### Unit Tests (pytest)
|
||||
|
||||
Example-based tests for specific scenarios:
|
||||
- Empty registry file creation (Req 1.6)
|
||||
- Successful rotation logging format (Req 3.6)
|
||||
- Expired GitLab PAT skips rotation (Req 3.7)
|
||||
- Stale expiry_date warning (Req 6.7)
|
||||
- Token masking with `::add-mask::` (Req 7.2)
|
||||
|
||||
### Integration Tests (pytest + httpx mock)
|
||||
|
||||
Mock-based tests for external service interactions:
|
||||
- GitLab API expiry check with mocked response (Req 6.1)
|
||||
- Jira/Confluence 401 interpretation (Req 6.2, 6.3)
|
||||
- OrgMyLife 401/403 interpretation (Req 6.4)
|
||||
- GitLab rotation API call + retry on failure (Req 3.1, 3.4)
|
||||
- GitHub secret update after rotation (Req 3.2)
|
||||
- Rotation success + secret update failure → alert (Req 3.5)
|
||||
- Alert channel unreachable → non-zero exit (Req 4.6)
|
||||
- Service timeout after retries → "check failed" (Req 2.5)
|
||||
- Null expiry_date classification (Req 2.6, 6.8)
|
||||
|
||||
### Smoke Tests
|
||||
|
||||
- Workflow YAML contains correct cron schedule (Req 5.1)
|
||||
- Workflow YAML contains workflow_dispatch trigger (Req 5.2)
|
||||
- All API URLs use HTTPS scheme (Req 7.3)
|
||||
- No token values written to non-secret locations (Req 7.1)
|
||||
|
||||
### Test Dependencies
|
||||
|
||||
```
|
||||
hypothesis>=6.100
|
||||
pytest>=8.0
|
||||
pytest-asyncio>=0.23
|
||||
pytest-cov>=5.0
|
||||
respx>=0.21 # httpx mock library
|
||||
```
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
# Requirements Document
|
||||
|
||||
## Introduction
|
||||
|
||||
This feature provides automated Personal Access Token (PAT) lifecycle management across all services used in the workspace: GitLab, Jira, Confluence, and OrgMyLife. The system tracks PAT expiration dates, auto-renews tokens where the service API supports it, and alerts the user via OrgMyLife task creation or email when manual intervention is required.
|
||||
|
||||
**Recommended Architecture:** A GitHub Actions scheduled workflow (runs weekly) that:
|
||||
- Checks PAT expiry via each service's API
|
||||
- Auto-rotates GitLab PATs (GitLab API supports this natively)
|
||||
- Creates OrgMyLife tasks or sends email alerts for tokens that cannot be auto-renewed (Jira, Confluence)
|
||||
- Stores a PAT registry (JSON) tracking token names, services, and expiration dates
|
||||
|
||||
This approach keeps the logic centralized, runs on schedule without needing the AI-Orchestrator to be online, and integrates with existing GitHub Actions infrastructure already in use.
|
||||
|
||||
## Glossary
|
||||
|
||||
- **PAT_Manager**: The automated system (GitHub Actions workflow) responsible for checking, renewing, and alerting about PAT status
|
||||
- **PAT_Registry**: A JSON configuration file that stores metadata about all managed PATs (service, token name, expiration date, renewal method)
|
||||
- **Token_Expiry_Window**: The number of days before expiration at which the PAT_Manager begins renewal or alerting (default: 14 days)
|
||||
- **Auto_Renewable_Token**: A PAT whose service API supports programmatic rotation (e.g., GitLab PATs via the Personal Access Tokens API)
|
||||
- **Manual_Renewal_Token**: A PAT whose service does not support programmatic rotation and requires human intervention (e.g., Jira, Confluence)
|
||||
- **Alert_Channel**: The notification destination for expiry warnings — either OrgMyLife task creation or email
|
||||
|
||||
## Requirements
|
||||
|
||||
### Requirement 1: PAT Registry Management
|
||||
|
||||
**User Story:** As a developer, I want a central registry of all my PATs with their metadata, so that I have a single source of truth for token lifecycle information.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE PAT_Manager SHALL maintain a PAT_Registry JSON file containing the service name, token identifier (maximum 128 characters), expiration date in ISO 8601 format (YYYY-MM-DD), and renewal method (value of "auto" or "manual") for each managed PAT
|
||||
2. WHEN a new PAT is added to the PAT_Registry, THE PAT_Manager SHALL validate that the entry contains all required fields (service, token_name, expiry_date in ISO 8601 format, renewal_method) and that the service value matches one of the supported services
|
||||
3. IF a PAT_Registry entry contains invalid or missing fields, THEN THE PAT_Manager SHALL reject the entry, leave the PAT_Registry unchanged, and output a validation error to the workflow log identifying the malformed entry and which fields failed validation
|
||||
4. THE PAT_Manager SHALL accept only the following service values in the PAT_Registry: GitLab (renewal_method: "auto"), Jira (renewal_method: "manual"), Confluence (renewal_method: "manual"), and OrgMyLife (renewal_method: "manual")
|
||||
5. IF a PAT entry is added with a service and token_name combination that already exists in the PAT_Registry, THEN THE PAT_Manager SHALL reject the duplicate entry and output a validation error to the workflow log indicating the conflict
|
||||
6. WHEN the PAT_Registry file does not exist at workflow execution time, THE PAT_Manager SHALL create an empty PAT_Registry file with a valid JSON structure
|
||||
|
||||
### Requirement 2: PAT Expiry Checking
|
||||
|
||||
**User Story:** As a developer, I want the system to check whether any of my PATs are expired or approaching expiration, so that I can take action before services break.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN the scheduled check runs, THE PAT_Manager SHALL query each service API to determine the current expiration status of each registered PAT, applying a timeout of 30 seconds per service call
|
||||
2. WHEN a PAT expiration date is within the Token_Expiry_Window (default: 14 days), THE PAT_Manager SHALL classify the PAT as "expiring soon"
|
||||
3. WHEN a PAT expiration date has passed, THE PAT_Manager SHALL classify the PAT as "expired"
|
||||
4. WHEN a PAT is not expired and its expiration date is beyond the Token_Expiry_Window, THE PAT_Manager SHALL classify the PAT as "healthy"
|
||||
5. IF a service API call fails due to connection timeout, DNS resolution failure, or HTTP 5xx response after 2 retry attempts, THEN THE PAT_Manager SHALL log the failure reason and classify the PAT as "check failed"
|
||||
6. IF a PAT_Registry entry has no expiration date configured, THEN THE PAT_Manager SHALL classify the PAT as "healthy" provided the authenticated API call succeeds, or "expired" if the API call returns an authentication failure
|
||||
7. WHEN the expiry check completes, THE PAT_Manager SHALL produce a classification result for every PAT in the PAT_Registry such that each PAT is assigned exactly one status: "healthy", "expiring soon", "expired", or "check failed"
|
||||
|
||||
### Requirement 3: Automatic PAT Renewal (GitLab)
|
||||
|
||||
**User Story:** As a developer, I want GitLab PATs to be automatically rotated before they expire, so that my CI/CD pipelines and scripts continue working without manual intervention.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN a GitLab PAT is classified as "expiring soon", THE PAT_Manager SHALL call the GitLab Personal Access Tokens API (`POST /personal_access_tokens/{id}/rotate`) to rotate the token
|
||||
2. WHEN the GitLab API returns a new token value after rotation, THE PAT_Manager SHALL update the stored token in GitHub Actions secrets using the GitHub API within 30 seconds of receiving the new value
|
||||
3. WHEN the GitLab API returns a new token value after rotation, THE PAT_Manager SHALL update the PAT_Registry entry with the new expiration date as returned by the GitLab API response
|
||||
4. IF the GitLab token rotation API call fails with an HTTP error status, a network timeout after 30 seconds, or a connection error, THEN THE PAT_Manager SHALL retry the rotation once and, if the retry also fails, fall back to the manual renewal alert flow
|
||||
5. IF the GitLab token rotation succeeds but the subsequent GitHub Actions secret update fails, THEN THE PAT_Manager SHALL immediately trigger the manual renewal alert flow indicating that the token was rotated but the secret could not be updated
|
||||
6. WHEN a GitLab PAT is successfully rotated and the secret is updated, THE PAT_Manager SHALL log the rotation event with the old and new expiration dates
|
||||
7. IF a GitLab PAT is classified as "expired", THEN THE PAT_Manager SHALL skip automatic rotation and fall back to the manual renewal alert flow
|
||||
|
||||
### Requirement 4: Manual Renewal Alerting
|
||||
|
||||
**User Story:** As a developer, I want to be alerted when PATs that cannot be auto-renewed are approaching expiration, so that I can manually renew them in time.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN a Manual_Renewal_Token has 14 or fewer days until expiration or is already expired, THE PAT_Manager SHALL send an alert via the configured Alert_Channel
|
||||
2. IF the configured Alert_Channel is OrgMyLife, THEN THE PAT_Manager SHALL create a task with the title containing the service name and token identifier
|
||||
3. IF the configured Alert_Channel is email, THEN THE PAT_Manager SHALL send an email containing the service name, token identifier, expiration date, and renewal instructions URL
|
||||
4. THE PAT_Manager SHALL include the remaining days until expiration (or indicate already expired with 0 days remaining) in each alert message
|
||||
5. THE PAT_Manager SHALL not create a new OrgMyLife task for a PAT if an open task for the same PAT and the same expiration date already exists
|
||||
6. IF the configured Alert_Channel is unreachable or returns an error, THEN THE PAT_Manager SHALL log the failure and exit the workflow run with a non-zero exit code
|
||||
|
||||
### Requirement 5: Scheduled Execution
|
||||
|
||||
**User Story:** As a developer, I want the PAT checks to run automatically on a weekly schedule, so that I do not need to remember to check token status manually.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE PAT_Manager SHALL execute as a GitHub Actions workflow on a weekly cron schedule, running once every Monday at 06:00 UTC
|
||||
2. THE PAT_Manager SHALL support manual triggering via workflow_dispatch, performing the same check-and-report sequence as the scheduled run
|
||||
3. WHEN the workflow completes (whether triggered by schedule or workflow_dispatch), THE PAT_Manager SHALL produce a summary report in the GitHub Actions run log listing each managed PAT with its service name, token identifier, and classification status (healthy, expiring soon, expired, or check failed)
|
||||
4. IF all PATs are classified as "healthy", THEN THE PAT_Manager SHALL complete without creating OrgMyLife tasks or sending email alerts, while still logging the summary report
|
||||
5. IF one or more PATs are classified as "expiring soon", "expired", or "check failed", THEN THE PAT_Manager SHALL proceed with the applicable renewal or alerting flow and still produce the summary report
|
||||
|
||||
### Requirement 6: Service-Specific Expiry Detection
|
||||
|
||||
**User Story:** As a developer, I want the system to correctly detect expiry for each service type, so that checks work regardless of how each platform reports token status.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN checking a GitLab PAT, THE PAT_Manager SHALL use the GitLab Personal Access Tokens API (`GET /personal_access_tokens`) to retrieve the `expires_at` field and compare it against the current date and Token_Expiry_Window to classify the PAT as "healthy", "expiring soon", or "expired"
|
||||
2. WHEN checking a Jira PAT, THE PAT_Manager SHALL attempt an authenticated API call within a timeout of 30 seconds and interpret HTTP 401 as an expired or revoked token
|
||||
3. WHEN checking a Confluence PAT, THE PAT_Manager SHALL attempt an authenticated API call within a timeout of 30 seconds and interpret HTTP 401 as an expired or revoked token
|
||||
4. WHEN checking an OrgMyLife token, THE PAT_Manager SHALL attempt an authenticated API call within a timeout of 30 seconds and interpret HTTP 401 or HTTP 403 as an expired or revoked token
|
||||
5. WHEN a service does not expose an expiry date via API and the authenticated call succeeds, THE PAT_Manager SHALL compare the expiry_date stored in the PAT_Registry against the current date and Token_Expiry_Window to classify the PAT
|
||||
6. IF a service API returns an HTTP error other than 401 or 403 or the request times out, THEN THE PAT_Manager SHALL classify the PAT as "check failed" and log an error message indicating the service name, token identifier, and the response status or timeout condition
|
||||
7. IF the authenticated API call succeeds but the expiry_date stored in the PAT_Registry has passed, THEN THE PAT_Manager SHALL classify the PAT as "expired" based on the stored date and include a warning that the stored expiry_date may be stale
|
||||
8. IF a Manual_Renewal_Token has no expiry_date in the PAT_Registry, THEN THE PAT_Manager SHALL classify the PAT based solely on the authenticated API call result — "healthy" if the call succeeds, or "expired" if the call returns 401 or 403
|
||||
|
||||
### Requirement 7: Security
|
||||
|
||||
**User Story:** As a developer, I want PAT values to remain secure throughout the renewal process, so that tokens are not exposed in logs or artifacts.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE PAT_Manager SHALL store PAT values exclusively in GitHub Actions secrets or encrypted environment variables and SHALL NOT write token values to workflow artifacts, step summaries, or PR comments
|
||||
2. THE PAT_Manager SHALL mask token values by registering them with the workflow log masking mechanism so that any occurrence in stdout, stderr, or workflow annotations is replaced with `***`
|
||||
3. THE PAT_Manager SHALL transmit PAT values only over HTTPS connections
|
||||
4. IF a token rotation produces a new value, THEN THE PAT_Manager SHALL update the GitHub Actions secret using the GitHub API with a token that has the `admin:org` or `repo` scope required for secrets write access
|
||||
5. IF the GitHub API secret update fails, THEN THE PAT_Manager SHALL retain the previous secret value unchanged, report the failure in the workflow log without exposing the new token value, and exit the workflow run with a non-success status
|
||||
@@ -0,0 +1,204 @@
|
||||
# Implementation Plan: PAT Renewal
|
||||
|
||||
## Overview
|
||||
|
||||
Implement an automated PAT lifecycle management system as a Python script (`scripts/pat_manager.py`) that runs via GitHub Actions. The implementation follows a modular architecture with 7 components: Registry, Expiry Checker, GitLab Rotator, Secret Updater, Alert Sender, Summary Reporter, and Main Orchestrator. All modules use Python 3.11+, httpx for HTTP, and pytest with Hypothesis for property-based testing.
|
||||
|
||||
## Tasks
|
||||
|
||||
- [x] 1. Set up project structure and core data models
|
||||
- [x] 1.1 Create directory structure and dependencies
|
||||
- Create `scripts/pat_manager/` package directory with `__init__.py`
|
||||
- Create `scripts/pat_manager/models.py` with `PatEntry` dataclass, `PatStatus` enum, `CheckResult` dataclass, `RotationResult` dataclass, `AlertResult` dataclass, and `AlertChannel` enum
|
||||
- Create `scripts/pat_manager/errors.py` with all custom exception classes (`PatManagerError`, `RegistryValidationError`, `ServiceCheckError`, `RotationError`, `SecretUpdateError`, `AlertError`)
|
||||
- Add `requirements.txt` with `httpx>=0.27` and `requirements-dev.txt` with `hypothesis>=6.100`, `pytest>=8.0`, `pytest-asyncio>=0.23`, `pytest-cov>=5.0`, `respx>=0.21`
|
||||
- _Requirements: 1.1, 1.4_
|
||||
|
||||
- [x] 1.2 Create PAT Registry JSON schema and example file
|
||||
- Create `pat-registry.json` with the JSON schema structure (version "1.0", empty tokens array)
|
||||
- Create `pat-registry.example.json` with example entries for all four services
|
||||
- _Requirements: 1.1, 1.6_
|
||||
|
||||
- [x] 2. Implement PAT Registry module
|
||||
- [x] 2.1 Implement registry loading and validation (`scripts/pat_manager/registry.py`)
|
||||
- Implement `PatRegistry` class with `load(path)` method that reads and parses JSON
|
||||
- Implement `validate_entry(entry)` that checks all required fields, service enum values, token_name length (max 128 chars), expiry_date ISO 8601 format, and renewal_method matching service
|
||||
- Implement duplicate detection for `(service, token_name)` combinations
|
||||
- Implement `create_empty(path)` for creating a new empty registry file
|
||||
- Implement `save(path, entries)` for persisting registry changes
|
||||
- Raise `RegistryValidationError` with specific failed fields on invalid entries
|
||||
- _Requirements: 1.1, 1.2, 1.3, 1.4, 1.5, 1.6_
|
||||
|
||||
- [x] 2.2 Write property test: Registry serialization round-trip
|
||||
- **Property 1: Registry serialization round-trip**
|
||||
- **Validates: Requirements 1.1**
|
||||
- Use Hypothesis to generate valid `PatEntry` lists and verify `save` → `load` produces equivalent entries
|
||||
|
||||
- [x] 2.3 Write property test: Validation accepts only valid entries
|
||||
- **Property 2: Validation accepts only valid entries**
|
||||
- **Validates: Requirements 1.2, 1.4**
|
||||
- Use Hypothesis to generate both valid and invalid entry dicts, verify validator accepts/rejects correctly
|
||||
|
||||
- [x] 2.4 Write property test: Invalid entries and duplicates preserve registry state
|
||||
- **Property 3: Invalid entries and duplicates preserve registry state**
|
||||
- **Validates: Requirements 1.3, 1.5**
|
||||
- Use Hypothesis to generate registries and invalid/duplicate entries, verify registry unchanged after rejected add
|
||||
|
||||
- [x] 3. Implement Expiry Checker module
|
||||
- [x] 3.1 Implement expiry classification logic (`scripts/pat_manager/checker.py`)
|
||||
- Implement `ExpiryChecker` class with configurable `expiry_window_days` (default 14), `timeout` (default 30s), and `max_retries` (default 2)
|
||||
- Implement `classify_by_date(expiry_date, reference_date, window)` pure function for date-based classification
|
||||
- Implement `check_all(entries, secrets)` async method that iterates all entries and returns `CheckResult` list
|
||||
- _Requirements: 2.2, 2.3, 2.4, 2.7_
|
||||
|
||||
- [x] 3.2 Implement service-specific check methods
|
||||
- Implement `check_gitlab(entry, token)` — calls `GET /personal_access_tokens`, reads `expires_at`, classifies
|
||||
- Implement `check_jira(entry, token)` — authenticated call, HTTP 401 = expired
|
||||
- Implement `check_confluence(entry, token)` — authenticated call, HTTP 401 = expired
|
||||
- Implement `check_orgmylife(entry, token)` — authenticated call, HTTP 401/403 = expired
|
||||
- Handle timeout (30s), retries (2 attempts with 5s backoff), and fallback to "check failed"
|
||||
- Handle null `expiry_date` entries: classify based on API call result only
|
||||
- Handle stale `expiry_date` warning when API succeeds but stored date has passed
|
||||
- _Requirements: 2.1, 2.5, 2.6, 6.1, 6.2, 6.3, 6.4, 6.5, 6.6, 6.7, 6.8_
|
||||
|
||||
- [x] 3.3 Write property test: Expiry classification is deterministic and correct
|
||||
- **Property 4: Expiry classification is deterministic and correct**
|
||||
- **Validates: Requirements 2.2, 2.3, 2.4, 6.5**
|
||||
- Use Hypothesis to generate expiry dates and reference dates, verify classification matches expected rules
|
||||
|
||||
- [x] 3.4 Write property test: Classification completeness
|
||||
- **Property 5: Classification completeness**
|
||||
- **Validates: Requirements 2.7**
|
||||
- Use Hypothesis to generate N-entry registries, verify exactly N results each with exactly one valid status
|
||||
|
||||
- [x] 3.5 Write property test: Non-auth HTTP errors yield "check failed"
|
||||
- **Property 12: Non-auth HTTP errors yield "check failed"**
|
||||
- **Validates: Requirements 6.6**
|
||||
- Use Hypothesis to generate HTTP status codes not in {200-299, 401, 403}, verify classification is "check failed"
|
||||
|
||||
- [x] 4. Checkpoint - Ensure all tests pass
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
- [x] 5. Implement GitLab Rotator module
|
||||
- [x] 5.1 Implement GitLab token rotation (`scripts/pat_manager/rotator.py`)
|
||||
- Implement `GitLabRotator` class with `rotate(token_id, current_token)` async method
|
||||
- Call `POST /personal_access_tokens/{id}/rotate` with the current token
|
||||
- Return `RotationResult` with new expiry date on success
|
||||
- Implement retry logic: retry once on failure, then return error result
|
||||
- Skip rotation for expired tokens (return error indicating manual intervention needed)
|
||||
- _Requirements: 3.1, 3.4, 3.7_
|
||||
|
||||
- [x] 5.2 Write property test: Registry update after successful rotation
|
||||
- **Property 6: Registry update after successful rotation**
|
||||
- **Validates: Requirements 3.3**
|
||||
- Use Hypothesis to generate rotation responses with new expiry dates, verify registry entry updated correctly
|
||||
|
||||
- [x] 6. Implement Secret Updater module
|
||||
- [x] 6.1 Implement GitHub secret update (`scripts/pat_manager/secret_updater.py`)
|
||||
- Implement `SecretUpdater` class with `update_secret(secret_name, secret_value)` async method
|
||||
- Use GitHub API to encrypt and update repository secrets (libsodium sealed box encryption)
|
||||
- Handle failure: retain previous secret value, report error without exposing token
|
||||
- _Requirements: 3.2, 3.5, 7.1, 7.4, 7.5_
|
||||
|
||||
- [x] 7. Implement Alert Sender module
|
||||
- [x] 7.1 Implement alert sending (`scripts/pat_manager/alerter.py`)
|
||||
- Implement `AlertSender` class with configurable `AlertChannel` (OrgMyLife or email)
|
||||
- Implement `send_alert(result)` that creates OrgMyLife task or sends email based on channel
|
||||
- OrgMyLife task title format: `[PAT Renewal] {service} - {token_name}`
|
||||
- Include days remaining (0 for expired) in alert message
|
||||
- Implement `check_existing_task(service, token_name, expiry_date)` for idempotent alerting
|
||||
- Handle alert channel unreachable: log failure, signal non-zero exit
|
||||
- Implement retry logic: 1 retry with 2s backoff
|
||||
- _Requirements: 4.1, 4.2, 4.3, 4.4, 4.5, 4.6_
|
||||
|
||||
- [x] 7.2 Write property test: Alert triggering correctness
|
||||
- **Property 7: Alert triggering correctness**
|
||||
- **Validates: Requirements 4.1, 5.4**
|
||||
- Use Hypothesis to generate classified PAT sets, verify zero alerts when all healthy, one alert per expiring/expired manual token
|
||||
|
||||
- [x] 7.3 Write property test: Alert message completeness
|
||||
- **Property 8: Alert message completeness**
|
||||
- **Validates: Requirements 4.2, 4.3**
|
||||
- Use Hypothesis to generate PAT entries requiring alerts, verify OrgMyLife title contains service+token, email contains all required fields
|
||||
|
||||
- [x] 7.4 Write property test: Days remaining calculation
|
||||
- **Property 9: Days remaining calculation**
|
||||
- **Validates: Requirements 4.4**
|
||||
- Use Hypothesis to generate expiry/reference date pairs, verify `days_remaining == max(0, (expiry - ref).days)`
|
||||
|
||||
- [x] 7.5 Write property test: Alert idempotence
|
||||
- **Property 10: Alert idempotence**
|
||||
- **Validates: Requirements 4.5**
|
||||
- Use Hypothesis to generate PAT alert scenarios, verify no duplicate task created when open task exists
|
||||
|
||||
- [x] 8. Implement Summary Reporter module
|
||||
- [x] 8.1 Implement summary report generation (`scripts/pat_manager/reporter.py`)
|
||||
- Implement `SummaryReporter` class with `report(results, rotations)` method
|
||||
- Generate one line per PAT with service name, token identifier, and classification status
|
||||
- Include rotation results (old/new expiry dates) for rotated tokens
|
||||
- _Requirements: 5.3_
|
||||
|
||||
- [x] 8.2 Write property test: Summary report completeness
|
||||
- **Property 11: Summary report completeness**
|
||||
- **Validates: Requirements 5.3**
|
||||
- Use Hypothesis to generate check result lists, verify report contains one line per result with service, token_name, and status
|
||||
|
||||
- [x] 9. Checkpoint - Ensure all tests pass
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
- [x] 10. Implement Main Orchestrator and GitHub Actions workflow
|
||||
- [x] 10.1 Implement main orchestrator (`scripts/pat_manager/__main__.py`)
|
||||
- Implement `main()` async function that coordinates all modules
|
||||
- Load registry → check all → rotate GitLab tokens → update secrets → send alerts → report summary
|
||||
- Mask all token values with `::add-mask::{value}` before any operations
|
||||
- Handle rotation success + secret update failure → trigger alert flow
|
||||
- Return exit code: 0 (success), 1 (alert delivery failure), 2 (registry validation failure)
|
||||
- Log rotation events with old and new expiration dates (Req 3.6)
|
||||
- _Requirements: 3.5, 3.6, 5.4, 5.5, 7.1, 7.2_
|
||||
|
||||
- [x] 10.2 Create GitHub Actions workflow file
|
||||
- Create `.github/workflows/pat-check.yml` with cron schedule `0 6 * * 1` (Monday 06:00 UTC)
|
||||
- Add `workflow_dispatch` trigger
|
||||
- Configure Python 3.11 setup, dependency installation, and script execution
|
||||
- Map all required secrets as environment variables (GITLAB_PAT, JIRA_PAT, CONFLUENCE_PAT, ORGMYLIFE_API_SECRET, ORGMYLIFE_USER, ORGMYLIFE_PASS, PAT_MANAGER_TOKEN, ALERT_CHANNEL)
|
||||
- Ensure all API URLs use HTTPS
|
||||
- _Requirements: 5.1, 5.2, 7.3_
|
||||
|
||||
- [x] 10.3 Write integration tests for orchestrator flow
|
||||
- Test full happy path: all healthy → no alerts, summary only
|
||||
- Test GitLab rotation flow with mocked APIs (respx)
|
||||
- Test rotation success + secret update failure → alert triggered
|
||||
- Test alert channel unreachable → non-zero exit code
|
||||
- Test service timeout after retries → "check failed" classification
|
||||
- Test null expiry_date classification scenarios
|
||||
- _Requirements: 2.5, 2.6, 3.2, 3.4, 3.5, 4.6, 5.4, 5.5, 6.8_
|
||||
|
||||
- [x] 11. Final checkpoint - Ensure all tests pass
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
## Notes
|
||||
|
||||
- Tasks marked with `*` are optional and can be skipped for faster MVP
|
||||
- Each task references specific requirements for traceability
|
||||
- Checkpoints ensure incremental validation
|
||||
- Property tests validate universal correctness properties from the design document using Hypothesis
|
||||
- Unit/integration tests validate specific examples and edge cases using pytest + respx
|
||||
- The implementation uses Python 3.11+, httpx for HTTP calls, and async/await throughout
|
||||
- All token values must be masked in logs via GitHub Actions `::add-mask::` mechanism
|
||||
|
||||
## Task Dependency Graph
|
||||
|
||||
```json
|
||||
{
|
||||
"waves": [
|
||||
{ "id": 0, "tasks": ["1.1", "1.2"] },
|
||||
{ "id": 1, "tasks": ["2.1"] },
|
||||
{ "id": 2, "tasks": ["2.2", "2.3", "2.4", "3.1"] },
|
||||
{ "id": 3, "tasks": ["3.2", "3.3", "3.4", "3.5"] },
|
||||
{ "id": 4, "tasks": ["5.1", "6.1", "7.1", "8.1"] },
|
||||
{ "id": 5, "tasks": ["5.2", "7.2", "7.3", "7.4", "7.5", "8.2"] },
|
||||
{ "id": 6, "tasks": ["10.1", "10.2"] },
|
||||
{ "id": 7, "tasks": ["10.3"] }
|
||||
]
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,71 @@
|
||||
---
|
||||
inclusion: auto
|
||||
---
|
||||
|
||||
# MCP Server Configuration
|
||||
|
||||
## Available MCP Servers (require VPN)
|
||||
|
||||
These MCP servers are hosted on DB's OpenShift cluster (dbcs-prag) and available when connected to the corporate network:
|
||||
|
||||
### Jira + Confluence (Atlassian MCP — NEUER kombinierter Server)
|
||||
- URL: `https://taros-playground-mcp-atlassian.apps.dbcs-prag.comp.db.de/mcp`
|
||||
- Capabilities: 72+ Tools — Jira issues, transitions, comments + Confluence pages (read/write)
|
||||
- Headers required:
|
||||
- `Authorization: Bearer <JIRA_TOKEN>`
|
||||
- `X-Jira-Host: arija-jira.jaas.service.deutschebahn.com` (ohne https://)
|
||||
- `X-Jira-Projects: O2CO2C,O2CAAPN,*` (komma-separiert, Wildcards möglich)
|
||||
- `X-Jira-Mode: read` (oder `write`)
|
||||
- Replaces: mcp-jira-v2 (deprecated) and mcp-confluence (separate)
|
||||
- Use for: Team velocity, sprint data, bug tracking, Confluence pages
|
||||
|
||||
### Jira MCP v2 (DEPRECATED)
|
||||
- URL: `https://taros-playground-mcp-jira-v2.apps.dbcs-prag.comp.db.de/mcp`
|
||||
- Status: DEPRECATED — use mcp-atlassian instead
|
||||
|
||||
### Confluence MCP (standalone, still works)
|
||||
- URL: `https://taros-playground-mcp-confluence.apps.dbcs-prag.comp.db.de/mcp`
|
||||
- Still functional but mcp-atlassian is the preferred replacement
|
||||
|
||||
### DB Context (dbctx)
|
||||
- URL: `https://taros-playground-dbctx.apps.dbcs-prag.comp.db.de/mcp`
|
||||
- Capabilities: DB-specific context and tooling
|
||||
- Use for: General DB knowledge, terminology, organizational context
|
||||
|
||||
### Teamlandkarte MCP (local, stdio)
|
||||
- Repo: https://git.tech.rz.db.de/ThomasHandke/teamlandkarte-mcp
|
||||
- Type: Local stdio server (Python, requires uv + Data Lake credentials)
|
||||
- Capabilities: Match tasks to employees/teams by competences, browse free capacities, team profiles
|
||||
- Use for: Finding teams (INKA, Data Science), capacity analysis, competence mapping
|
||||
- Requires: DATA_LAKE_USERNAME, DATA_LAKE_PASSWORD, AZURE_OPENAI_EMBEDDING_API_KEY
|
||||
- NOT yet configured (needs credentials)
|
||||
|
||||
### BEAM MCP (local, stdio) — EDUCATIONAL USE ONLY
|
||||
- Repo: https://git.tech.rz.db.de/beta/ai/mcp/beam-mcp
|
||||
- Type: Local stdio server (Node.js, requires Edge browser for login)
|
||||
- Capabilities: Enterprise Architecture (LeanIX/BEAM) — applications, domains, components, diagrams
|
||||
- Use for: System landscape, application dependencies, architecture diagrams
|
||||
- ⚠️ ACTIVE USE NOT PERMITTED due to data protection concerns. Educational reference only.
|
||||
|
||||
## Other Noteworthy Servers (not yet configured)
|
||||
|
||||
| Server | Use Case | Priority |
|
||||
|--------|----------|----------|
|
||||
| DataLake MCP | SQL on Trino (quantitative data) | Medium — when we need metrics |
|
||||
| Foundations MCP | ADRs, Standards, Capabilities | Medium — for architecture comparison |
|
||||
| gitlab-mcp | GitLab integration via glab | Low — we use direct API calls |
|
||||
| sonarqube-mcp | Code quality metrics | Low — for team quality comparison |
|
||||
| proclib-mcp | Process library access | Medium — for process analysis |
|
||||
| mcp-agilehive | AgileHive SAFe tool | High — PI-Planning data, team capacity |
|
||||
|
||||
## Fallback (without VPN)
|
||||
|
||||
When MCP servers are unreachable, use direct REST API calls:
|
||||
- Confluence: Bearer token auth against `https://arija-confluence.jaas.service.deutschebahn.com/rest/api`
|
||||
- GitLab: PRIVATE-TOKEN header against `https://git.tech.rz.db.de/api/v4`
|
||||
- Jira: Not available without VPN
|
||||
|
||||
## Local MCP Server
|
||||
|
||||
- OrgMyLife: Running locally, accessible via Kiro's MCP integration
|
||||
- Provides: tasks, calls, digest, projects board
|
||||
@@ -0,0 +1,100 @@
|
||||
---
|
||||
inclusion: auto
|
||||
---
|
||||
|
||||
# Session Context — Next Session Priorities
|
||||
|
||||
## Immediate Tasks
|
||||
|
||||
1. **Verify Jury-Voting** — `unikat.andreknie.de` is live and working
|
||||
2. **Verify NoteGraph browse** — `notes.andreknie.de/browse?token=...` (clear SB service worker first)
|
||||
3. **KIQ Docker deploy** — Docker not set up on KIQ VPS yet. Need someone with root to install Docker or find alternative
|
||||
4. **Rechnungen prüfen** — 8 receipts labeled in OrgMyLife, forward to rechnungen@d-hive.de
|
||||
|
||||
## Key Technical Notes
|
||||
|
||||
### Git on this machine
|
||||
- Git path: `C:\Users\AndreKnie\AppData\Local\GitHubDesktop\app-3.5.8\resources\app\git\cmd\git.exe`
|
||||
- Always use: `$gitExe = "C:\Users\AndreKnie\AppData\Local\GitHubDesktop\app-3.5.8\resources\app\git\cmd\git.exe"` then `& $gitExe <command>`
|
||||
- `gh` CLI is NOT available
|
||||
|
||||
### OrgMyLife API
|
||||
- Endpoint: `https://api.andreknie.de`
|
||||
- Auth: `Authorization: Bearer !Z5l5vUcm!&qfhUuE=Nf`
|
||||
- Projects board: `GET/PUT /api/projects`, `POST /api/projects/create`
|
||||
- Tasks: `GET /api/tasks`, agent-ready filter: `?filter_agent=true`
|
||||
|
||||
### NoteGraph
|
||||
- SilverBullet: `https://notes.andreknie.de` (now serves browse UI as primary)
|
||||
- Browse: `https://notes.andreknie.de/browse?token=PASSWORD`
|
||||
- Upload: `https://notes.andreknie.de/upload?token=PASSWORD`
|
||||
- Ingestion uses Mistral (`mistral-small-latest`) for enrichment and Vision OCR
|
||||
- Repo: `NoteGraph/` with `ingestion/` Python package
|
||||
- Docker on VPS: Ingestion (8001), SilverBullet (3000, internal only)
|
||||
- **When creating notes for the user, always add `AI-Source` to the tags array in frontmatter**
|
||||
|
||||
### Jury-Voting App
|
||||
- Repo: `Jury-Voting/` (private, GitHub: DoctoDre/Jury-Voting)
|
||||
- URL: `https://unikat.andreknie.de` (password: `UNIKAT_jury`)
|
||||
- Stack: React 19 + Vite frontend, Express.js backend, JSON file persistence, SSE real-time
|
||||
- Port: 3002 on VPS, systemd service `jury-voting`
|
||||
- Caddy config: in `NoteGraph/Caddyfile` (block for `unikat.andreknie.de`)
|
||||
- Event: 15. UNIKAT Ideenwettbewerb, 19. Mai 2026, Science Park Kassel
|
||||
- 6 jurors, 11 teams, 5 criteria (1-10 scale)
|
||||
- Admin: `/admin`, Results/Beamer: `/results`
|
||||
- Future: move to `unikat.d-hive.de` (add DNS + Caddy block on KIQ VPS)
|
||||
|
||||
### Repos in workspace
|
||||
- `OrgMyLife/` — Personal task/coordination system (FastAPI + PostgreSQL)
|
||||
- `AI-Orchestrator/` — Agent orchestration (Python)
|
||||
- `NoteGraph/` — Knowledge base (SilverBullet + ingestion pipeline)
|
||||
- `Confluence_Bot/` — Confluence automation (PowerShell)
|
||||
- `Projekt-KIQ-HP/` — KIQ homepage (React/Vite, currently uses Supabase)
|
||||
- `Jury-Voting/` — UNIKAT Jury Voting (React/Vite + Express)
|
||||
- `project-audit/` — DB enterprise project audit data (GitLab)
|
||||
|
||||
### Deploy pattern
|
||||
- Push to main → GitHub Actions → SSH to VPS → deploy
|
||||
- Secrets via GitHub repo secrets
|
||||
- Caddy reverse proxy managed via `NoteGraph/Caddyfile`
|
||||
|
||||
### Deploy pattern for private repos (no git on VPS)
|
||||
- Build in GitHub Actions (checkout → npm ci → npm run build)
|
||||
- Create tar.gz archive of built artifacts
|
||||
- Upload via `appleboy/scp-action` to VPS `/tmp/`
|
||||
- Extract via `appleboy/ssh-action`, install deps, restart systemd service
|
||||
- This avoids needing git credentials on the VPS
|
||||
|
||||
### VPS (217.160.174.2)
|
||||
- Caddy handles HTTPS for all domains (auto Let's Encrypt)
|
||||
- OrgMyLife on port 8000 (`api.andreknie.de`)
|
||||
- SilverBullet on port 3000 (`notes.andreknie.de`)
|
||||
- Ingestion on port 8001 (`notes.andreknie.de/upload`)
|
||||
- Jury-Voting on port 3002 (`unikat.andreknie.de`)
|
||||
- SSH: root@217.160.174.2, key stored as `DEPLOY_SSH_KEY` in GitHub repos
|
||||
- Node.js 20 installed via nodesource
|
||||
- No rsync installed — use tar+scp for file transfer
|
||||
- No docker on this VPS for Jury-Voting (runs as systemd service with tsx)
|
||||
|
||||
### Projekt-KIQ-HP (separate VPS: projekt-kiq.d-hive.de)
|
||||
- Deploy user: `github-deploy`
|
||||
- Static site deployed via rsync to `/opt/projekt-kiq/site`
|
||||
- Caddy config at `/etc/caddy/Caddyfile` (managed by KIQ deploy workflow)
|
||||
- Has rsync available
|
||||
- SSH key: `DEPLOY_SSH_KEY` in KIQ GitHub repo
|
||||
|
||||
### Express 5 Gotchas
|
||||
- Do NOT pass `app` to functions typed as `Express` — route registration silently fails at runtime
|
||||
- Use `Router()` and `app.use(router)` pattern instead
|
||||
- The `Express` type from `@types/express` doesn't match Express 5 runtime correctly
|
||||
- SPA catch-all: use middleware (`app.use`) not `app.get('/{*splat}')` — the splat syntax can interfere
|
||||
|
||||
### DB GitLab (project-audit)
|
||||
- Requires `scm-info.yaml` in repo root for security scans
|
||||
- Format: `version: v3`, `license: LicenseRef-DBISL`, `contacts:`, `confidentiality: internal`, `reference-ids: none`
|
||||
- GitLab URL: `git.tech.rz.db.de/AndreKnie/project-audit`
|
||||
|
||||
### DNS
|
||||
- `andreknie.de` DNS managed externally (takes ~5 min to propagate new subdomains)
|
||||
- Add A record pointing to 217.160.174.2 for new subdomains
|
||||
- Caddy auto-provisions HTTPS certificates once DNS resolves
|
||||
@@ -0,0 +1,108 @@
|
||||
---
|
||||
inclusion: auto
|
||||
---
|
||||
|
||||
# Workspace Context & Conventions
|
||||
|
||||
## Git Access
|
||||
|
||||
- Git is NOT in the system PATH on this machine.
|
||||
- Use GitHub Desktop's bundled git at: `C:\Users\AndreKnie\AppData\Local\GitHubDesktop\app-3.5.8\resources\app\git\cmd\git.exe`
|
||||
- Private GitHub repos (e.g. DoctoDre/CV) are accessible via the locally configured GitHub credentials — no separate PAT needed.
|
||||
- For git commands in shell, always use the full path: `& "C:\Users\AndreKnie\AppData\Local\GitHubDesktop\app-3.5.8\resources\app\git\cmd\git.exe" <command>`
|
||||
- DB GitLab: `https://git.tech.rz.db.de` — PAT auth via PRIVATE-TOKEN header or inline URL for push.
|
||||
|
||||
## Repo Landscape
|
||||
|
||||
All repos live under: `c:\Users\AndreKnie\OneDrive - Deutsche Bahn\Coden\Orchestrator\`
|
||||
|
||||
| Repo | Language | Purpose | Status |
|
||||
|------|----------|---------|--------|
|
||||
| CV | TypeScript/Node | Personal Knowledge Graph + CV generation | Best-structured, partially implemented |
|
||||
| AI-Orchestrator | Python | Autonomous agent dispatcher (Symphony spec) | PAT-management done, orchestrator blocked on server |
|
||||
| OrgMyLife | Python | Personal task/call/digest system | Productive, MCP server running |
|
||||
| project-audit | Python | Jira/Confluence/GitLab analysis (DB internal) | Functional, script-based |
|
||||
| Confluence_Bot | Python | CI/CD for Confluence pages | Minimal |
|
||||
| Projekt-KIQ-HP | Node/Web | KIQ Landing Page | Frontend project |
|
||||
| Analyse-O2C-C2S | Markdown/HTML | Strategic team analysis O2C+C2S | Active, own GitLab repo |
|
||||
| NoteGraph | TBD | Note organization | Early stage |
|
||||
|
||||
## Secrets
|
||||
|
||||
- Located in `.secrets` at workspace root (gitignored)
|
||||
- Contains: GITLAB_TOKEN_BESTELLSYSTEM, GITLAB_TOKEN_AUDIT, JIRA_TOKEN, JIRA_TOKEN_SAB, CONFLUENCE_TOKEN, CONFLUENCE_SAB, ORGMYLIFE_API_SECRET
|
||||
- NEVER echo secret values in responses
|
||||
|
||||
## Conventional Commits (all repos)
|
||||
|
||||
All commit messages follow Conventional Commits format:
|
||||
```
|
||||
feat(scope): description
|
||||
fix(scope): description
|
||||
docs(scope): description
|
||||
test(scope): description
|
||||
ci(scope): description
|
||||
refactor(scope): description
|
||||
```
|
||||
|
||||
## Quality Gates (before every commit)
|
||||
|
||||
1. **Security scan**: Run `gitleaks detect --source . --no-git` (or equivalent) — no secrets in code
|
||||
2. **Lint**: Run the project's linter (ruff for Python, eslint for Node, tsc --noEmit for TS)
|
||||
3. **Test**: Run the project's test suite — minimum 80% coverage target
|
||||
4. **Dependency check**: On Node projects, `npm audit` should show 0 vulnerabilities
|
||||
|
||||
## Autonomous Mode Rules
|
||||
|
||||
When working autonomously (orchestrator, sub-agents):
|
||||
- Make decisions based on best practices and existing code patterns
|
||||
- Never push directly to main/master — always use feature branches
|
||||
- Create a Merge/Pull Request for human review
|
||||
- If stuck after 3 attempts: stop and document the issue clearly
|
||||
- Report progress: `PROGRESS: X% - what's happening (ETA: Xmin)`
|
||||
- Signal help needed: `HELP: clear description of the problem`
|
||||
|
||||
## Testing Standards
|
||||
|
||||
- Minimum 80% line coverage for new code
|
||||
- Use Given-When-Then structure for test cases
|
||||
- Property-based testing where applicable (fast-check for TS, hypothesis for Python)
|
||||
- Test happy path, edge cases, and error cases
|
||||
|
||||
## MCP Servers (available when VPN connected)
|
||||
|
||||
```json
|
||||
{
|
||||
"jira": { "url": "https://taros-playground-mcp-jira-v2.apps.dbcs-prag.comp.db.de/mcp" },
|
||||
"confluence": { "url": "https://taros-playground-mcp-confluence.apps.dbcs-prag.comp.db.de/mcp" }
|
||||
}
|
||||
```
|
||||
|
||||
## DB Internal Services
|
||||
|
||||
- Jira (arija): `https://arija-jira.jaas.service.deutschebahn.com` — PAT auth via Bearer token. Requires VPN.
|
||||
- Jira (systelone/SAB): PAT auth works, but requires VPN.
|
||||
- Confluence (arija): `https://arija-confluence.jaas.service.deutschebahn.com` — PAT auth via Bearer token works.
|
||||
- Confluence (systelone/SAB): NO API/PAT support. Only accessible via HTTPS + HTML.
|
||||
- GitLab: `https://git.tech.rz.db.de` — PAT auth via PRIVATE-TOKEN header.
|
||||
|
||||
## Token Status (last checked: 2026-05-28)
|
||||
|
||||
| Token | Service | Status |
|
||||
|-------|---------|--------|
|
||||
| GITLAB_TOKEN_AUDIT | GitLab (project-audit) | ✅ OK |
|
||||
| GITLAB_TOKEN_BESTELLSYSTEM | GitLab (Bestellsystem) | ✅ OK (renewed 2026-05-28) |
|
||||
| CONFLUENCE_TOKEN | Confluence (arija) | ✅ OK |
|
||||
| CONFLUENCE_SAB | Confluence (systelone) | ⚠️ No API support — HTML only |
|
||||
| JIRA_TOKEN | Jira (arija) | ❓ Not testable without VPN |
|
||||
| JIRA_TOKEN_SAB | Jira (systelone) | ❓ Not testable without VPN |
|
||||
| ORGMYLIFE_API_SECRET | OrgMyLife | ✅ OK (local) |
|
||||
| GitHub (Desktop) | GitHub | ✅ OK |
|
||||
|
||||
## OneDrive Compatibility
|
||||
|
||||
All filenames must be OneDrive-safe:
|
||||
- No brackets `()[]{}`, no `#%&@$+=~`
|
||||
- No single quotes or backticks
|
||||
- Replace umlauts in filenames: ä→ae, ö→oe, ü→ue, ß→ss
|
||||
- Use hyphens instead of special characters
|
||||
@@ -0,0 +1,293 @@
|
||||
# Wissensdatenbank ETL-Pipeline
|
||||
|
||||
## Worum geht es? (in einfach)
|
||||
|
||||
Dies ist die **zentrale Wissenssammlung** fuer den 1st-Level-Support und Chatbots rund um
|
||||
DB InfraGO. Wir **sammeln hier das Wissen** aus vielen Quellen (Confluence, Webseiten,
|
||||
PDFs), pruefen es und legen es **versioniert im Git-Repo** ab. Damit ist dieses Repo die
|
||||
**Single Source of Truth**: Wenn etwas hier steht und freigegeben ist, gilt es - und genau
|
||||
das lesen die Chatbots.
|
||||
|
||||
**Was ist „ETL"?** Drei einfache Schritte, die bei jeder Aufnahme passieren:
|
||||
|
||||
1. **Extract = Sammeln.** Wissen aus den Quellen holen (Confluence-Seiten, Webseiten, PDFs).
|
||||
2. **Transform = Aufbereiten.** In einheitliches Markdown umwandeln, mit Schlagworten/Scope
|
||||
versehen und vertrauliche Inhalte herausfiltern.
|
||||
3. **Load = Ablegen.** Das geprueefte Wissen versioniert nach `data/processed/` schreiben -
|
||||
die zentrale Wahrheit, aus der Chatbots/Vektor-DB lesen.
|
||||
|
||||
Kurz: **Quellen rein → aufbereiten/pruefen → als Single Source of Truth ablegen.**
|
||||
|
||||
## Ziel
|
||||
|
||||
Klar definieren, **welches Wissen in Chatbots darf und welches nicht.** Wissen wird bei
|
||||
der Aufnahme speziell verarbeitet und bereitgestellt, sodass Chatbots es nutzen koennen.
|
||||
Klassifikation nach Scope:
|
||||
|
||||
- **allgemein** – darf intern **und** extern genutzt werden; nicht toolspezifisch
|
||||
(z.B. Regulierung/INB, Kundeninformationen).
|
||||
- **intern** – nur DB InfraGO intern.
|
||||
- **extern** – public im Internet bzw. fuer EVUs/EIUs.
|
||||
|
||||
Technisch eine **ETL-Pipeline fuer LLM/RAG**: holt Wissen aus Confluence, Webseiten und
|
||||
PDFs, klassifiziert es nach Scope, filtert vertrauliche Inhalte und gibt nur geprueftes
|
||||
Wissen frei (`data/processed/<scope>/...` = Feed fuer die Vektor-DB).
|
||||
|
||||
> Architektur & vollstaendiger Wissensfluss inkl. Diagramme:
|
||||
> [`.kiro/steering/architecture.md`](.kiro/steering/architecture.md)
|
||||
> · Manuelle Einrichtungsschritte: [`docs/SETUP.md`](docs/SETUP.md)
|
||||
|
||||
## Denkmodell
|
||||
|
||||
```
|
||||
Domaene -> Scope (intern|extern|allgemein) -> Tool -> Dokument(e)
|
||||
```
|
||||
|
||||
- **Domaene**: fachlicher Bereich (jedes Tool ist eigene Domaene; `regulierung`, `kundeninfo` fuer allgemeines Wissen).
|
||||
- **Scope**: `intern` | `extern` | `allgemein` (allgemein gilt fuer intern UND extern).
|
||||
- **Tool**: eine Fachanwendung aus dem 1st-Level-Support (Katalog automatisch aus Seite 428909879).
|
||||
- **Strategie pro Quelle** – siehe Tabelle unten.
|
||||
|
||||
### Strategien (Verarbeitung pro Quelle)
|
||||
|
||||
| Strategie | Was sie tut | Wichtige Options |
|
||||
|-----------|-------------|------------------|
|
||||
| `confluence_page` | genau diese eine Seite | `scope` (auch `intern,extern`), `incremental`, `attachments` |
|
||||
| `confluence_tree` | Seite inkl. Unterseiten | `max_depth` (-1=alle, 0=nur Seite), `incremental`, `attachments` |
|
||||
| `confluence_faq` | FAQs aus Tabellen | `incremental` |
|
||||
| `crawler` | Webseite: Index → Detailseiten | `detail_pattern`, `selector`, `max_pages` |
|
||||
| `sitemap` | Detailseiten aus (gz-)Sitemap | `sitemap_url`, `url_pattern`, `limit`, `incremental` |
|
||||
| `gitlab_md` | Markdown-Dateien aus einem GitLab-Repo | `ref`, `path`, `max_depth` (Token: `GITLAB_TOKEN`) |
|
||||
| `file` | Dateien (md/pdf) aus dem Repo-Ordner `files/` | `max_depth` |
|
||||
| `pdf` | PDFs einer Seite → Markdown (1→n) | `split: headings\|pages`, `heading_pattern`, `keep_raw`, `redact` |
|
||||
|
||||
### Confluence-Anhaenge (eingebundene PDFs)
|
||||
|
||||
Bei `confluence_page`/`confluence_tree` werden **eingebundene PDF-Anhaenge**
|
||||
(`view-file`/`viewpdf`-Makros) automatisch geparst und als **eigene Dokumente** im
|
||||
selben Feed abgelegt (Frontmatter `kind: attachment`, `parent_url` zeigt auf die Seite).
|
||||
Scope/Domain/Owner werden von der Seite geerbt. Inkrementell ueber die Attachment-Version
|
||||
(unveraenderter Anhang -> kein Re-Parse). Default an, pro Quelle ueber
|
||||
`options: { attachments: false }` abschaltbar.
|
||||
|
||||
Bilder werden nicht binaer uebernommen; der **alt-Text/Dateiname** bleibt als
|
||||
`[Bild: ...]` im Markdown - gibt dem RAG Kontext ohne Volumen. Datei-Embeds ohne
|
||||
Parsing-Pfad (z.B. Office-Docs) tauchen als `[Anhang: ...]`-Marker im Seitentext auf.
|
||||
|
||||
### Mehrere Quellen pro Tool
|
||||
|
||||
Ein Tool kann beliebig viele, gemischte Quellen haben – einfach weitere Eintraege
|
||||
unter `sources:` anhaengen (z.B. mehrere Confluence-Quellen mit `max_depth`, eine
|
||||
interne und eine externe Seite, oder zusaetzlich eine FAQ-Seite). Siehe die Tools
|
||||
`pathos` (mehrere Quellen inkl. `confluence_faq` und `intern,extern`) und `nur`.
|
||||
|
||||
### intern vs. extern – wie wird klassifiziert?
|
||||
|
||||
Es gibt **keine Inhaltstrennung innerhalb einer Seite** mehr. Klassifiziert wird pro
|
||||
Tool und pro Quelle:
|
||||
|
||||
**Tool-`scope`** (in `config/tools.yaml`):
|
||||
|
||||
| Tool-`scope` | Bedeutung |
|
||||
|--------------|-----------|
|
||||
| `intern` | alles intern (Default, restriktiv – z.B. Salesforce, NuR) |
|
||||
| `extern` | alles extern |
|
||||
| `allgemein` | gilt fuer intern UND extern |
|
||||
| `mixed` | Tool hat Quellen mit **unterschiedlichem** Scope (z.B. eine interne und eine externe Seite) |
|
||||
|
||||
**Source-`scope`** (pro Quelle, gilt fuer die ganze Seite):
|
||||
- `intern` | `extern` | `allgemein`
|
||||
- `"intern,extern"` → die Seite wird fuer **beide** Scopes genutzt (dupliziert).
|
||||
|
||||
Quellen ohne eigenen `scope` erben den Tool-Scope (bei `mixed` => intern).
|
||||
**Wichtig:** Hat ein Tool gemischtes Wissen, braucht es im Zweifel **zwei Seiten**
|
||||
(eine interne, eine externe) als zwei Quellen – statt einer gemischten Seite.
|
||||
|
||||
Allgemeines, tool-uebergreifendes Wissen steht separat in **`config/general.yaml`**.
|
||||
|
||||
Der Tool-Katalog wird **initial** mit `scripts/bootstrap_tools.py` erzeugt und danach
|
||||
**manuell** in `config/tools.yaml` gepflegt. Uebersicht aller Quellen: GitLab-Pages-Seite
|
||||
**„Wissensquellen"** (`config.html`, geparst aus tools.yaml/general.yaml/approvals.yaml).
|
||||
|
||||
## Neues Wissen hinzufuegen (Workflow)
|
||||
|
||||
Jedes Wissen hat **owners** (intern Verantwortliche, Accountability) und einen
|
||||
**contact** (herausgebbare Kontaktadresse, Default `einfachbahn@deutschebahn.com`).
|
||||
Beide stehen im Frontmatter jedes Dokuments und auf der Pages-Seite, damit klar ist,
|
||||
wer verantwortlich ist und an wen man sich wenden darf.
|
||||
|
||||
1. **Issue anlegen** mit der Vorlage „Neues Wissen" (`.gitlab/issue_templates/`):
|
||||
Tool/Domaene, **Quellen je Zeile** (`URL | Strategie | Scope`), **Verantwortliche**
|
||||
(`owners`) und optional **Kontakt** (Default `einfachbahn@deutschebahn.com`).
|
||||
2. **Reviewer uebernimmt** die Angaben nach `config/tools.yaml` bzw. `config/general.yaml`
|
||||
(neuer Tool-/Quellen-Eintrag inkl. `owners:`) – per GitLab Web-IDE (Variante A) oder
|
||||
manuell (Variante B).
|
||||
3. **Merge Request** öffnen. Die **MR-Freigabe ist das Quality Gate** (4-Augen, via
|
||||
`CODEOWNERS` + Protected Branch).
|
||||
4. **Vorschau** im MR: den `preview`-Job manuell starten und `PREVIEW_ONLY` auf
|
||||
Tool-Id/Domaene/URL der neuen Quelle setzen. Er verarbeitet nur diese Quelle und
|
||||
stellt die erzeugten Markdowns als Job-Artefakt bereit. Lokal alternativ
|
||||
`python -m src.main --only "<tool|domaene|url>" --data preview`.
|
||||
5. **Merge** → der ETL-Schedule verarbeitet die Quelle (stuendlich Mo–Fr, inkrementell) und
|
||||
committet das Wissen; die Vektor-DB liest direkt aus `data/processed/<scope>/...`.
|
||||
|
||||
## Freigabe-Prozess
|
||||
|
||||
```
|
||||
MR mergen (Mensch) -> ETL: Extract -> Transform -> Auto-Filter -> data/processed/ -> Vektor-DB
|
||||
```
|
||||
|
||||
- **Freigabe = Merge Request (Mensch, das eigentliche Gate):** Wer eine Quelle
|
||||
(Link + Strategie + Scope) in `config/tools.yaml`/`general.yaml` eintraegt, stellt einen
|
||||
MR. Beim **Merge** (4-Augen via `CODEOWNERS` + Protected Branch) wird entschieden,
|
||||
welche Quelle mit welchem Scope aufgenommen wird. Danach laeuft der ETL automatisch.
|
||||
- **Auto-Filter im ETL (Sicherheitsnetz, kein Freigabe-Knopf):** setzt pro Dokument
|
||||
`review_status` – standardmaessig **approved**:
|
||||
- `approved` -> `data/processed/...` (**direkt live**, sobald gemergt + ETL gelaufen)
|
||||
- `pending` -> `data/staging/pending/` (Blacklist-Treffer, Scope-Warnung, oder
|
||||
Inhalt < 50 Zeichen – Grund im Frontmatter `review_notes`)
|
||||
- `trusted: true` (z.B. INB) bleibt trotz Treffer approved; `redact: false` schaltet
|
||||
die Redaction pro Quelle ab.
|
||||
- **`config/approvals.yaml` (nachtraegliche Korrektur):** hebt einzelne
|
||||
`pending`-Dokumente per URL oder `hash:<content_hash>` doch auf `approved`.
|
||||
Kein Schritt, den jedes Dokument durchlaeuft. Den Hash zeigt die Pages-Seite (Detail-Fenster).
|
||||
- **Vektor-DB liest direkt aus `data/processed/<scope>/<domaene>/`** (implementiert):
|
||||
Nur freigegebenes Wissen liegt dort, der Scope steckt im Ordnerpfad **und** im
|
||||
Frontmatter (inkl. `owners`/`contact`). Kein separater Export-/Ingestion-Schritt –
|
||||
`processed/` IST der Feed. Der externe Index nimmt nur `extern/` + `allgemein/`.
|
||||
- **`data/_meta.json`**: globale Statusdatei fuer nachgelagerte Systeme mit
|
||||
`last_run` (letzter ETL-Lauf), `last_change` (wann sich der Bestand zuletzt
|
||||
inhaltlich/metadatenseitig geaendert hat), `documents`, `by_scope` und einer
|
||||
`content_signature`. Ein Index-/RAG-Consumer kann daran erkennen, ob ein erneutes
|
||||
Einlesen ueberhaupt noetig ist.
|
||||
- **`data/_index.json`**: dokument-genauer Katalog des gesamten Bestands.
|
||||
Pro Voll-Dokument: `domain`/`tool`/`scope`, `url`, `path`, `content_hash`,
|
||||
`last_updated` und - falls vorhanden - die zugehoerigen `chunks` (Anzahl + Pfad)
|
||||
und `attachments` (Anhang-Dokumente: Name + Pfad). Anhang-Dokumente haben zudem
|
||||
`kind: "attachment"` und `parent_url`. Aggregate: `documents_total`,
|
||||
`chunks_total`, `attachments_total`, `by_domain` (mit `documents`/`chunks`/`attachments`),
|
||||
`by_scope`. So sieht ein Anschliesser auf einen Blick, was pro Domaene/Tool wo liegt
|
||||
und welche Dokumente zusaetzlich als Chunks oder Anhaenge vorliegen (z.B. INB =
|
||||
Voll-Dokument **und** Chunks; pathOS-Seite = Voll-Dokument **und** Anhang-PDF).
|
||||
Deterministisch (kein Zeitstempel) -> aendert sich nur bei echten Bestandsaenderungen.
|
||||
- **`data/run_log.jsonl`**: append-only **Lauf-Historie** (1 JSON-Zeile je ETL-Lauf):
|
||||
Zeitstempel, verarbeitete Dokumente, Status-Counts und **Fehler je Quelle**
|
||||
(z.B. fehlgeschlagene Confluence-Abrufe). Auf die letzten 500 Laeufe gekappt. So sieht
|
||||
man - auch historisch - ob ein Lauf sauber durchlief, ohne die fluechtigen CI-Job-Logs
|
||||
zu durchsuchen. Die Uebersichtsseite zeigt zusaetzlich eine kompakte Health-Zeile
|
||||
(„N Dok verarbeitet, M Quellen mit Fehler") in der Fusszeile.
|
||||
|
||||
### Wo liegt das final freigegebene Wissen?
|
||||
|
||||
Ausschliesslich unter **`data/processed/<scope>/<domaene>/[<tool>/]`** im Repo
|
||||
(versioniert, = Single Source of Truth). `pending`-Dokumente liegen in
|
||||
`data/staging/pending/` und werden zur **Transparenz ebenfalls committet** (auf der
|
||||
Pages-Seite einsehbar). Der Audit-Trail steht in `data/review_report.json` und in
|
||||
der Git-Historie.
|
||||
|
||||
## Lokal testen
|
||||
|
||||
```bash
|
||||
python3 -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
|
||||
ruff check src tests # Lint
|
||||
python -m pytest -q # Offline-Tests
|
||||
python -m src.main --config config/tools.yaml --data data # Live-Lauf
|
||||
python -m src.chunk --data data # Chunks erzeugen (offline, optional)
|
||||
```
|
||||
|
||||
## Confluence aktivieren
|
||||
|
||||
```bash
|
||||
export CONFLUENCE_URL="https://arija-confluence.jaas.service.deutschebahn.com"
|
||||
export CONFLUENCE_TOKEN="<PAT>" # Bearer-PAT (Server/DC)
|
||||
pip install atlassian-python-api
|
||||
```
|
||||
|
||||
Ohne Credentials werden Confluence-Quellen sauber uebersprungen.
|
||||
|
||||
## Betrieb (GitLab CI)
|
||||
|
||||
- `quality`-Stage: `ruff` (Lint) + `gitleaks` (Secret-Scan) bei jedem Push/MR.
|
||||
- `etl`-Stage: **stuendlich Mo–Fr 8–17 Uhr** (Cron `0 8-17 * * 1-5`), baut Wissen,
|
||||
committet `data/` nach `main` (per `GIT_PUSH_TOKEN`); der Commit loest automatisch
|
||||
einen `pages`-Refresh aus.
|
||||
- `pages`-Stage: baut `public/` (Uebersicht, Hilfe, Chatbot-Anschluss, Wissensquellen)
|
||||
fuer GitLab Pages auf `main`.
|
||||
- `preview`-Job (MR, manuell): verarbeitet eine neue/geaenderte Quelle via
|
||||
`--only` und stellt die erzeugten Markdowns als Job-Artefakt bereit (zum Pruefen
|
||||
vor dem Merge). `PREVIEW_ONLY` beim Start auf Tool-Id/Domaene/URL setzen.
|
||||
- **Kundeninfos inkrementell**: `incremental: true` crawlt nur NEUE Detailseiten und
|
||||
ueberschreibt vorhandene nicht – Re-Runs bleiben schnell.
|
||||
- **Confluence inkrementell**: `options: { incremental: true }` prueft pro Seite die
|
||||
Confluence-Versionsnummer (`source_version` im Frontmatter). Unveraenderte Seiten
|
||||
werden uebersprungen, nur geaenderte/neue Seiten neu verarbeitet – spart API-Last.
|
||||
- **Re-Tagging bei Metadaten-Aenderung**: Aenderst du in `tools.yaml`/`general.yaml`
|
||||
nur `tags`/`owners`/`contact` einer Quelle, weicht deren `meta_fingerprint`
|
||||
(im Frontmatter) vom gespeicherten ab. Der naechste inkrementelle Lauf passt dann
|
||||
die bestehenden Dateien dieser Quelle in-place an (nur Frontmatter, kein erneuter
|
||||
Download/API-Call).
|
||||
- **Scope-Wechsel**: Aenderst du den `scope` einer Quelle, wird die Seite neu
|
||||
verarbeitet und an `data/processed/<neuer-scope>/...` geschrieben; die alte Ablage
|
||||
wird nach erfolgreichem Schreiben **automatisch geloescht** (domaenen-intern, damit
|
||||
eine URL, die legitim unter mehreren Domaenen liegt – z.B. eine pathOS-Seite auch
|
||||
unter `web` – nicht versehentlich entfernt wird).
|
||||
- **Domain-Umbenennung**: Die neue Ablage entsteht automatisch, die alte Domain-Ablage
|
||||
bleibt aber stehen (domaenenuebergreifendes Loeschen waere wegen geteilter URLs
|
||||
unsicher). Den alten `data/processed/<scope>/<alte-domain>/`-Ordner daher bei einer
|
||||
Domain-Umbenennung manuell loeschen.
|
||||
- CI/CD-Variablen: `GIT_PUSH_TOKEN`, `CONFLUENCE_URL`, `CONFLUENCE_TOKEN` (Masked).
|
||||
- `scm-info.yaml`: Pflichtdatei fuer DB-GitLab-Compliance.
|
||||
|
||||
## Konfiguration
|
||||
|
||||
- `config/tools.yaml` – Tools, Domaenen, Quellen, Strategien, Scopes, Tags
|
||||
- `config/filter_rules.json` – Blacklist-Keywords, Regex-Redaction
|
||||
- `config/chunking.yaml` – gemeinsame Chunk-Defaults (pro Quelle ueber `options.chunk` ueberschreibbar)
|
||||
|
||||
### Chunking (optional, Default aus)
|
||||
|
||||
Grosse, stark gegliederte Dokumente (v.a. **INB**) werden zusaetzlich zur Voll-Datei in
|
||||
**Chunks entlang der Ueberschriften** zerlegt – ein **abgeleitetes, jederzeit neu
|
||||
erzeugbares** Artefakt unter `data/chunks/<scope>/<domaene>/<docslug>/`. Das Voll-Dokument
|
||||
in `data/processed/` bleibt unangetastet (Parent-Document-Muster): ein Anschliesser waehlt
|
||||
Voll-Dokument, Chunks oder beides. Es werden bewusst nur **deterministische, embedding-freie**
|
||||
Strategien angeboten (`headings | faq | recursive`); semantisches Chunking gehoert in die
|
||||
Vektor-DB des Anschliessers. Jeder Chunk bekommt eine kurze **Contextual-Retrieval**-Zeile
|
||||
(`> Kontext: <Dokument> > <Abschnitt>`). Aktivierung pro Quelle via `options: { chunk: headings }`.
|
||||
|
||||
**Inkrementell (kein Voll-Rebuild):** Chunking laeuft am Ende jedes ETL-Laufs automatisch
|
||||
mit (`src.main`) und ist Teil von `data/`. Pro Dokument wird nur dann neu gechunkt, wenn
|
||||
sich der **Inhalt** (`parent_hash`) ODER die **wirksamen Optionen/Strategie**
|
||||
(`chunk_fingerprint`) geaendert haben - sonst wird das Dokument uebersprungen (keine
|
||||
Datei-Aenderung, kein Git-Churn). Aenderst du also bei einer Quelle z.B. `chunk_level`
|
||||
oder von `headings` auf `recursive`, werden **nur die Dokumente dieser Ablage** neu
|
||||
erzeugt. Wird `chunk` wieder auf `off` gestellt oder ein Voll-Dokument geloescht, werden
|
||||
die zugehoerigen Chunks automatisch entfernt.
|
||||
|
||||
**Feinsteuerung (Default aus, pro Quelle ueberschreibbar):**
|
||||
|
||||
- `chunk_min_doc_chars`: Dokumente unter dieser Zeichenzahl bleiben **ganz** (kleine
|
||||
FAQ-/How-to-Seiten muessen nicht zwingend gechunkt werden).
|
||||
- `chunk_kind`: chunkt nur Dokumente mit passendem `kind` (`""` = alle; `attachment` =
|
||||
nur Anhang-PDFs; `document` = nur Voll-Seiten).
|
||||
|
||||
Beispiel **pathOS** (an der Tool-Tree-Quelle in `config/tools.yaml`): grosse Anhang-PDFs
|
||||
(z.B. EVU-Schnittstellen-Doku, 2,8 MB) werden gechunkt, Seiten und FAQs bleiben ganz.
|
||||
|
||||
```yaml
|
||||
options:
|
||||
incremental: true
|
||||
chunk: headings
|
||||
chunk_kind: attachment # nur Anhang-PDFs chunken
|
||||
chunk_min_doc_chars: 3000 # < 3k Zeichen bleiben ganz (~750 Token)
|
||||
```
|
||||
|
||||
## Version & Changelog
|
||||
|
||||
Aktuelle Version steht in **`VERSION`** (SemVer), die Historie in **`CHANGELOG.md`**
|
||||
(Format „Keep a Changelog"). Beides wird auf der GitLab-Pages-Seite **„Changelog"**
|
||||
angezeigt (Version auch in der Navigationsleiste). Pflege-Regeln:
|
||||
`.kiro/steering/changelog-versioning.md`.
|
||||
@@ -0,0 +1,12 @@
|
||||
# =============================================================================
|
||||
# git-crypt Filter für Kontext: bahn
|
||||
# Alle hier definierten Patterns werden mit dem Schlüssel 'git-crypt-bahn'
|
||||
# verschlüsselt. Nur Maschinen mit dem bahn-Schlüssel können diese Dateien
|
||||
# im Klartext lesen.
|
||||
# =============================================================================
|
||||
|
||||
.env filter=git-crypt-bahn diff=git-crypt-bahn
|
||||
*.pem filter=git-crypt-bahn diff=git-crypt-bahn
|
||||
*.key filter=git-crypt-bahn diff=git-crypt-bahn
|
||||
*token* filter=git-crypt-bahn diff=git-crypt-bahn
|
||||
*secret* filter=git-crypt-bahn diff=git-crypt-bahn
|
||||
@@ -0,0 +1,12 @@
|
||||
# =============================================================================
|
||||
# git-crypt Filter für Kontext: dhive
|
||||
# Alle hier definierten Patterns werden mit dem Schlüssel 'git-crypt-dhive'
|
||||
# verschlüsselt. Nur Maschinen mit dem dhive-Schlüssel können diese Dateien
|
||||
# im Klartext lesen.
|
||||
# =============================================================================
|
||||
|
||||
.env filter=git-crypt-dhive diff=git-crypt-dhive
|
||||
*.pem filter=git-crypt-dhive diff=git-crypt-dhive
|
||||
*.key filter=git-crypt-dhive diff=git-crypt-dhive
|
||||
*token* filter=git-crypt-dhive diff=git-crypt-dhive
|
||||
*secret* filter=git-crypt-dhive diff=git-crypt-dhive
|
||||
@@ -0,0 +1,26 @@
|
||||
version: "1.0"
|
||||
contexts:
|
||||
- name: privat
|
||||
description: "Persönliche Projekte"
|
||||
- name: dhive
|
||||
description: "dhive GmbH Projekte"
|
||||
- name: bahn
|
||||
description: "DB InfraGO Projekte"
|
||||
- name: shared
|
||||
description: "Kontextübergreifende Tools und Wissen"
|
||||
|
||||
naming:
|
||||
pattern: "^[a-z0-9][a-z0-9\\-]{0,48}[a-z0-9]$"
|
||||
min_length: 2
|
||||
max_length: 50
|
||||
|
||||
security:
|
||||
config: shared/config/access-config.yaml
|
||||
audit_log: .audit/access.log
|
||||
encryption:
|
||||
tool: git-crypt
|
||||
machine_context_config: shared/config/machine-context.yaml
|
||||
key_source: keyring
|
||||
federation:
|
||||
config: shared/config/team-repos.yaml
|
||||
conflict_strategy: team-wins
|
||||
@@ -0,0 +1,12 @@
|
||||
# =============================================================================
|
||||
# git-crypt Filter für Kontext: privat
|
||||
# Alle hier definierten Patterns werden mit dem Schlüssel 'git-crypt-privat'
|
||||
# verschlüsselt. Nur Maschinen mit dem privat-Schlüssel können diese Dateien
|
||||
# im Klartext lesen.
|
||||
# =============================================================================
|
||||
|
||||
.env filter=git-crypt-privat diff=git-crypt-privat
|
||||
*.pem filter=git-crypt-privat diff=git-crypt-privat
|
||||
*.key filter=git-crypt-privat diff=git-crypt-privat
|
||||
*token* filter=git-crypt-privat diff=git-crypt-privat
|
||||
*secret* filter=git-crypt-privat diff=git-crypt-privat
|
||||
@@ -0,0 +1,189 @@
|
||||
# Onboarding: Neue Maschine einrichten
|
||||
|
||||
Dieses Dokument beschreibt den Prozess, um eine neue Maschine für die Arbeit mit dem Monorepo und seinen verschlüsselten Secrets einzurichten.
|
||||
|
||||
## Voraussetzungen
|
||||
|
||||
- Git ist installiert (>= 2.25)
|
||||
- [git-crypt](https://github.com/AGWA/git-crypt) ist installiert
|
||||
- Zugang zum Passwort-Manager (Bitwarden) oder direkter Schlüsselzugang
|
||||
|
||||
## Übersicht: Verschlüsselungsarchitektur
|
||||
|
||||
Das Monorepo verwendet **git-crypt** mit kontextspezifischen Filtern:
|
||||
|
||||
| Kontext | Filter-Name | Verschlüsselte Patterns |
|
||||
|---------|--------------------|-------------------------------------------------|
|
||||
| privat | git-crypt-privat | .env, *.pem, *.key, *token*, *secret* |
|
||||
| dhive | git-crypt-dhive | .env, *.pem, *.key, *token*, *secret* |
|
||||
| bahn | git-crypt-bahn | .env, *.pem, *.key, *token*, *secret* |
|
||||
| shared | git-crypt (basis) | shared/.env, shared/**/*.pem, shared/**/*.key |
|
||||
|
||||
Jeder Kontext hat einen eigenen symmetrischen Schlüssel. Eine Maschine kann nur die Secrets der Kontexte entschlüsseln, für die sie autorisiert ist.
|
||||
|
||||
## Schritt 1: Repository klonen
|
||||
|
||||
```bash
|
||||
git clone <repo-url> monorepo
|
||||
cd monorepo
|
||||
```
|
||||
|
||||
Nach dem Klonen sind alle Secret-Dateien als verschlüsselte Binärdaten vorhanden. Code, Konfiguration und Dokumentation sind sofort lesbar.
|
||||
|
||||
## Schritt 2: Maschinenkontext konfigurieren
|
||||
|
||||
Bearbeite `shared/config/machine-context.yaml` und passe sie an die neue Maschine an:
|
||||
|
||||
```yaml
|
||||
machine:
|
||||
name: "mein-neuer-rechner"
|
||||
description: "Beschreibung des Rechners und seines Einsatzzwecks"
|
||||
authorized_contexts:
|
||||
- privat # Nur Kontexte eintragen, die entschlüsselt werden sollen
|
||||
- dhive
|
||||
# - bahn # Auskommentiert = kein Zugriff auf bahn-Secrets
|
||||
key_source: "keyring" # Siehe Abschnitt "Schlüsselquellen"
|
||||
password_manager:
|
||||
type: "bitwarden"
|
||||
vault: "monorepo-keys"
|
||||
```
|
||||
|
||||
### Felder
|
||||
|
||||
| Feld | Beschreibung |
|
||||
|------------------------|--------------------------------------------------------------------|
|
||||
| `name` | Eindeutiger Name der Maschine (z.B. "dhive-laptop", "home-pc") |
|
||||
| `description` | Kurze Beschreibung des Einsatzzwecks |
|
||||
| `authorized_contexts` | Liste der Kontexte, deren Secrets entschlüsselt werden dürfen |
|
||||
| `key_source` | Woher die Schlüssel geladen werden (siehe unten) |
|
||||
| `password_manager` | Optionale Passwort-Manager-Konfiguration |
|
||||
|
||||
## Schritt 3: Entschlüsselungsschlüssel installieren
|
||||
|
||||
### Schlüsselquellen (`key_source`)
|
||||
|
||||
Das System unterstützt drei Quellen für Entschlüsselungsschlüssel:
|
||||
|
||||
#### 1. Keyring (`key_source: "keyring"`)
|
||||
|
||||
Schlüssel werden im OS-Keyring gespeichert (empfohlen für Entwicklermaschinen).
|
||||
|
||||
```bash
|
||||
# Schlüssel aus sicherer Quelle importieren
|
||||
git-crypt unlock /pfad/zum/schlüssel-privat.key
|
||||
git-crypt unlock /pfad/zum/schlüssel-dhive.key
|
||||
git-crypt unlock /pfad/zum/schlüssel-bahn.key
|
||||
|
||||
# Alternativ: Schlüssel im Keyring speichern (via monorepo-cli)
|
||||
monorepo-cli onboard --context privat --key-file /pfad/zum/schlüssel.key
|
||||
monorepo-cli onboard --context dhive --key-file /pfad/zum/schlüssel.key
|
||||
```
|
||||
|
||||
#### 2. Datei (`key_source: "file"`)
|
||||
|
||||
Schlüssel liegen als Dateien auf der Festplatte (nur für isolierte Maschinen).
|
||||
|
||||
```bash
|
||||
# Schlüsseldateien an erwarteter Stelle ablegen
|
||||
mkdir -p ~/.monorepo-keys/
|
||||
cp schlüssel-privat.key ~/.monorepo-keys/git-crypt-privat.key
|
||||
cp schlüssel-dhive.key ~/.monorepo-keys/git-crypt-dhive.key
|
||||
cp schlüssel-bahn.key ~/.monorepo-keys/git-crypt-bahn.key
|
||||
|
||||
# Berechtigungen einschränken
|
||||
chmod 600 ~/.monorepo-keys/*.key
|
||||
```
|
||||
|
||||
#### 3. Passwort-Manager (`key_source: "password-manager"`)
|
||||
|
||||
Schlüssel werden bei Bedarf aus einem Passwort-Manager abgerufen.
|
||||
|
||||
Unterstützte Manager:
|
||||
- **Bitwarden** (`type: "bitwarden"`)
|
||||
- **1Password** (`type: "1password"`)
|
||||
- **KeePass** (`type: "keepass"`)
|
||||
|
||||
```bash
|
||||
# Bitwarden-Beispiel: Schlüssel sind im Vault "monorepo-keys" gespeichert
|
||||
# Einträge haben das Format: monorepo-key-{context}
|
||||
# z.B. monorepo-key-privat, monorepo-key-dhive, monorepo-key-bahn
|
||||
|
||||
# Login in Bitwarden (einmalig pro Session)
|
||||
bw login
|
||||
|
||||
# Entschlüsselung wird automatisch über den SecretEncryptionManager gesteuert
|
||||
monorepo-cli decrypt --context privat
|
||||
```
|
||||
|
||||
## Schritt 4: Secrets entschlüsseln
|
||||
|
||||
Nach Installation der Schlüssel werden die Secrets für autorisierte Kontexte entschlüsselt:
|
||||
|
||||
```bash
|
||||
# Alle autorisierten Kontexte entschlüsseln
|
||||
monorepo-cli decrypt --all
|
||||
|
||||
# Oder einzelne Kontexte
|
||||
monorepo-cli decrypt --context privat
|
||||
monorepo-cli decrypt --context dhive
|
||||
```
|
||||
|
||||
### Verhalten bei nicht-autorisierten Kontexten
|
||||
|
||||
Wenn eine Maschine keinen Schlüssel für einen Kontext besitzt:
|
||||
- Die Secret-Dateien bleiben als verschlüsselte Binärdaten im Working Tree
|
||||
- Der restliche Code und die Konfiguration des Kontexts sind lesbar
|
||||
- Es wird keine Fehlermeldung ausgegeben, die den Inhalt offenlegt
|
||||
- Das Repository bleibt voll funktionsfähig für die autorisierten Kontexte
|
||||
|
||||
## Schritt 5: Einrichtung verifizieren
|
||||
|
||||
```bash
|
||||
# Status der Verschlüsselung prüfen
|
||||
monorepo-cli encrypt --status
|
||||
|
||||
# Erwartete Ausgabe für "dhive-laptop" (nur dhive autorisiert):
|
||||
# privat: verschlüsselt (kein Schlüssel)
|
||||
# dhive: entschlüsselt ✓
|
||||
# bahn: verschlüsselt (kein Schlüssel)
|
||||
# shared: entschlüsselt ✓
|
||||
```
|
||||
|
||||
## Typische Maschinenkontexte
|
||||
|
||||
| Maschine | Autorisierte Kontexte | key_source |
|
||||
|----------------------|-----------------------|------------------|
|
||||
| andre-hauptrechner | privat, dhive, bahn | keyring |
|
||||
| dhive-laptop | dhive | password-manager |
|
||||
| bahn-arbeitsrechner | bahn | keyring |
|
||||
| familie-nas | privat | file |
|
||||
|
||||
## Sicherheitshinweise
|
||||
|
||||
1. **Schlüssel niemals committen**: Die Schlüsseldateien selbst gehören NICHT ins Repository
|
||||
2. **Maschinenkontext-Datei ist kein Secret**: `machine-context.yaml` enthält nur das Mapping, keine Schlüssel
|
||||
3. **Minimaler Zugriff**: Jede Maschine erhält nur die Schlüssel, die sie benötigt
|
||||
4. **Schlüsselrotation**: Bei Kompromittierung eines Schlüssels → neuen Schlüssel generieren, alle autorisierten Maschinen aktualisieren, Secrets neu verschlüsseln
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Dateien erscheinen als Binärdaten
|
||||
|
||||
**Ursache:** Schlüssel für den betreffenden Kontext nicht installiert.
|
||||
|
||||
**Lösung:** Schlüssel gemäß Schritt 3 installieren und `monorepo-cli decrypt` ausführen.
|
||||
|
||||
### git-crypt meldet "not a git-crypt repo"
|
||||
|
||||
**Ursache:** git-crypt wurde im Repository noch nicht initialisiert.
|
||||
|
||||
**Lösung:** `git-crypt init` (nur beim erstmaligen Setup des Repositories nötig).
|
||||
|
||||
### Merge-Konflikte in verschlüsselten Dateien
|
||||
|
||||
**Ursache:** Zwei Branches haben dieselbe Secret-Datei geändert.
|
||||
|
||||
**Lösung:** Der SecretEncryptionManager löst Merges auf verschlüsselter Ebene auf. Falls dies fehlschlägt:
|
||||
```bash
|
||||
monorepo-cli encrypt --resolve-merge <datei>
|
||||
```
|
||||
@@ -0,0 +1,23 @@
|
||||
contexts:
|
||||
privat:
|
||||
env_file: privat/.env
|
||||
allowed_shared:
|
||||
- shared/tools/
|
||||
- shared/powers/
|
||||
- shared/config/
|
||||
dhive:
|
||||
env_file: dhive/.env
|
||||
allowed_shared:
|
||||
- shared/tools/
|
||||
- shared/powers/
|
||||
- shared/config/
|
||||
bahn:
|
||||
env_file: bahn/.env
|
||||
allowed_shared:
|
||||
- shared/tools/
|
||||
- shared/powers/
|
||||
- shared/config/
|
||||
- shared/knowledge-store/
|
||||
shared:
|
||||
env_file: shared/.env
|
||||
allowed_shared: ["*"]
|
||||
@@ -0,0 +1,11 @@
|
||||
machine:
|
||||
name: "andre-hauptrechner"
|
||||
description: "Andres Hauptrechner mit vollem Zugriff"
|
||||
authorized_contexts:
|
||||
- privat
|
||||
- dhive
|
||||
- bahn
|
||||
key_source: "keyring"
|
||||
password_manager:
|
||||
type: "bitwarden"
|
||||
vault: "monorepo-keys"
|
||||
@@ -0,0 +1 @@
|
||||
repos: []
|
||||
@@ -0,0 +1,18 @@
|
||||
version: "1.0"
|
||||
scopes:
|
||||
- name: privat
|
||||
context: privat
|
||||
description: "Persönlicher Wissensbereich"
|
||||
knowledge_path: shared/knowledge-store/privat/
|
||||
- name: dhive
|
||||
context: dhive
|
||||
description: "dhive GmbH Wissensbereich"
|
||||
knowledge_path: shared/knowledge-store/dhive/
|
||||
- name: bahn
|
||||
context: bahn
|
||||
description: "DB InfraGO Wissensbereich"
|
||||
knowledge_path: shared/knowledge-store/bahn/
|
||||
- name: shared
|
||||
context: shared
|
||||
description: "Kontextübergreifender Wissensbereich"
|
||||
knowledge_path: shared/knowledge-store/shared/
|
||||
@@ -0,0 +1,44 @@
|
||||
version: "1.0"
|
||||
federation:
|
||||
topology: "hub-and-spoke"
|
||||
hub_owner: "andre"
|
||||
conflict_strategy: "team-wins"
|
||||
|
||||
team_repos:
|
||||
- context: privat
|
||||
url: "https://github.com/andreknie/privat-team.git"
|
||||
branch: main
|
||||
sync_direction: bidirectional
|
||||
sync_frequency: "on-push"
|
||||
shared_mirror:
|
||||
enabled: true
|
||||
paths:
|
||||
- "shared/tools/common-scripts/"
|
||||
- "shared/config/base-config.yaml"
|
||||
mode: read-only
|
||||
|
||||
- context: dhive
|
||||
url: "https://gitlab.dhive.io/team/dhive-mono.git"
|
||||
branch: main
|
||||
sync_direction: bidirectional
|
||||
sync_frequency: "on-push"
|
||||
shared_mirror:
|
||||
enabled: true
|
||||
paths:
|
||||
- "shared/tools/"
|
||||
- "shared/powers/db-dxp-platform/"
|
||||
- "shared/mcp-servers/"
|
||||
mode: read-only
|
||||
|
||||
- context: bahn
|
||||
url: "https://gitlab.2700.2db.it/team/bahn-workspace.git"
|
||||
branch: main
|
||||
sync_direction: bidirectional
|
||||
sync_frequency: "daily"
|
||||
shared_mirror:
|
||||
enabled: true
|
||||
paths:
|
||||
- "shared/tools/"
|
||||
- "shared/powers/"
|
||||
- "shared/knowledge-store/_index.yaml"
|
||||
mode: read-only
|
||||
@@ -0,0 +1,15 @@
|
||||
# monorepo-cli
|
||||
|
||||
CLI-Tool für die Monorepo-Verwaltung: Kontextgrenzen, Wissensspeicher, Repo-Sync, Verschlüsselung und Föderation.
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
pip install -e ".[dev]"
|
||||
```
|
||||
|
||||
## Verwendung
|
||||
|
||||
```bash
|
||||
ctx-guard --help
|
||||
```
|
||||
@@ -0,0 +1,50 @@
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "monorepo-cli"
|
||||
version = "0.1.0"
|
||||
description = "CLI-Tool für Monorepo-Verwaltung: Kontextgrenzen, Wissensspeicher, Repo-Sync, Verschlüsselung und Föderation"
|
||||
readme = "README.md"
|
||||
license = "Apache-2.0"
|
||||
requires-python = ">=3.11"
|
||||
authors = [
|
||||
{ name = "Andre Knie" },
|
||||
]
|
||||
|
||||
dependencies = [
|
||||
"pyyaml>=6.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest>=8.0",
|
||||
"pytest-cov>=5.0",
|
||||
"hypothesis>=6.100",
|
||||
"ruff>=0.4.0",
|
||||
"mypy>=1.10",
|
||||
"types-PyYAML>=6.0",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
ctx-guard = "monorepo.cli:main"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["src/monorepo"]
|
||||
|
||||
[tool.ruff]
|
||||
target-version = "py311"
|
||||
line-length = 100
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "I", "N", "W", "UP"]
|
||||
|
||||
[tool.mypy]
|
||||
python_version = "3.11"
|
||||
strict = true
|
||||
warn_return_any = true
|
||||
warn_unused_configs = true
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
@@ -0,0 +1,136 @@
|
||||
"""monorepo – CLI-Tool für Monorepo-Verwaltung.
|
||||
|
||||
Exportiert die zentralen Datenmodelle und Konfigurationsfunktionen.
|
||||
"""
|
||||
|
||||
from monorepo.audit import AuditLogger
|
||||
from monorepo.bridge import ContextBridge, SensitiveMatch, ShareResult
|
||||
from monorepo.config import MonorepoConfig, load_monorepo_config
|
||||
from monorepo.models import (
|
||||
ArtifactType,
|
||||
ConflictInfo,
|
||||
ConflictStrategy,
|
||||
Context,
|
||||
EncryptionKey,
|
||||
IsolationLeak,
|
||||
IsolationReport,
|
||||
KeySource,
|
||||
MachineContext,
|
||||
MigrationPlan,
|
||||
OnboardingResult,
|
||||
PasswordManagerConfig,
|
||||
ProjectInfo,
|
||||
RepoEntry,
|
||||
RepoMode,
|
||||
ScopeConfig,
|
||||
SecurityEvent,
|
||||
SharedMirrorConfig,
|
||||
SyncDirection,
|
||||
SyncFrequency,
|
||||
SyncResult,
|
||||
TeamRepoEntry,
|
||||
)
|
||||
from monorepo.encryption import (
|
||||
DecryptionResult,
|
||||
EncryptionResult,
|
||||
GitCryptNotAvailableError,
|
||||
MachineContextManager,
|
||||
PasswordManagerError,
|
||||
SecretEncryptionManager,
|
||||
)
|
||||
from monorepo.federation import (
|
||||
ConflictResult,
|
||||
FederationManager,
|
||||
FederationTopologyConfig,
|
||||
MemberInfo,
|
||||
MemberOnboardingResult,
|
||||
MirrorResult,
|
||||
SubtreeSyncEngine,
|
||||
)
|
||||
from monorepo.integration import create_integrated_stack
|
||||
from monorepo.shared_config import (
|
||||
AgentExtension,
|
||||
ConfigMerger,
|
||||
HarnessAdapter,
|
||||
MCPServerConfig,
|
||||
MergeConflict,
|
||||
MergedMCPConfig,
|
||||
SharedToolStatus,
|
||||
ToolReference,
|
||||
)
|
||||
from monorepo.migration import (
|
||||
MigrationConflict,
|
||||
MigrationEngine,
|
||||
MigrationError,
|
||||
MigrationRegistryEntry,
|
||||
MigrationResult,
|
||||
)
|
||||
from monorepo.security import ContextGuard
|
||||
from monorepo.structure import StructureManager
|
||||
|
||||
__all__ = [
|
||||
# Audit
|
||||
"AuditLogger",
|
||||
# Bridge
|
||||
"ContextBridge",
|
||||
"SensitiveMatch",
|
||||
"ShareResult",
|
||||
# Enums
|
||||
"ArtifactType",
|
||||
"ConflictStrategy",
|
||||
"Context",
|
||||
"KeySource",
|
||||
"RepoMode",
|
||||
"SyncDirection",
|
||||
"SyncFrequency",
|
||||
# Dataclasses
|
||||
"ConflictInfo",
|
||||
"EncryptionKey",
|
||||
"IsolationLeak",
|
||||
"IsolationReport",
|
||||
"MachineContext",
|
||||
"MigrationPlan",
|
||||
"MonorepoConfig",
|
||||
"OnboardingResult",
|
||||
"PasswordManagerConfig",
|
||||
"ProjectInfo",
|
||||
"RepoEntry",
|
||||
"ScopeConfig",
|
||||
"SecurityEvent",
|
||||
"SharedMirrorConfig",
|
||||
"SyncResult",
|
||||
"TeamRepoEntry",
|
||||
# Classes
|
||||
"AgentExtension",
|
||||
"ConfigMerger",
|
||||
"ConflictResult",
|
||||
"ContextGuard",
|
||||
"DecryptionResult",
|
||||
"EncryptionResult",
|
||||
"FederationManager",
|
||||
"FederationTopologyConfig",
|
||||
"GitCryptNotAvailableError",
|
||||
"HarnessAdapter",
|
||||
"MachineContextManager",
|
||||
"MCPServerConfig",
|
||||
"MemberInfo",
|
||||
"MemberOnboardingResult",
|
||||
"MergeConflict",
|
||||
"MergedMCPConfig",
|
||||
"MigrationConflict",
|
||||
"MigrationEngine",
|
||||
"MigrationError",
|
||||
"MigrationRegistryEntry",
|
||||
"MigrationResult",
|
||||
"MirrorResult",
|
||||
"PasswordManagerError",
|
||||
"SecretEncryptionManager",
|
||||
"SharedToolStatus",
|
||||
"StructureManager",
|
||||
"SubtreeSyncEngine",
|
||||
"ToolReference",
|
||||
# Config
|
||||
"load_monorepo_config",
|
||||
# Integration
|
||||
"create_integrated_stack",
|
||||
]
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Audit-Logger für die Protokollierung von Zugriffsverletzungen.
|
||||
|
||||
Schreibt Sicherheitsereignisse (SecurityEvents) in eine konfigurierbare
|
||||
Protokolldatei im strukturierten Textformat. Thread-sicher durch Nutzung
|
||||
eines Locks für Dateizugriffe.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from monorepo.models import SecurityEvent
|
||||
|
||||
# Standard-Audit-Log-Pfad (relativ zum Monorepo-Root), wie in monorepo.yaml definiert
|
||||
_DEFAULT_AUDIT_LOG_PATH = Path(".audit/access.log")
|
||||
|
||||
|
||||
class AuditLogger:
|
||||
"""Protokolliert Sicherheitsereignisse in eine Audit-Logdatei.
|
||||
|
||||
Jeder Eintrag wird als strukturierte Zeile im Format geschrieben:
|
||||
[ISO-TIMESTAMP] OUTCOME | requesting_context -> target_context | action on resource
|
||||
|
||||
Der Logger ist thread-sicher: Schreibzugriffe auf die Logdatei werden
|
||||
durch einen Lock serialisiert.
|
||||
|
||||
Args:
|
||||
log_path: Pfad zur Audit-Logdatei. Standard: `.audit/access.log`
|
||||
(aus monorepo.yaml security.audit_log).
|
||||
"""
|
||||
|
||||
def __init__(self, log_path: Path | None = None) -> None:
|
||||
"""Initialisiert den AuditLogger.
|
||||
|
||||
Args:
|
||||
log_path: Pfad zur Audit-Logdatei. Falls None, wird der
|
||||
Standard-Pfad `.audit/access.log` verwendet.
|
||||
"""
|
||||
self._log_path = log_path if log_path is not None else _DEFAULT_AUDIT_LOG_PATH
|
||||
self._lock = threading.Lock()
|
||||
# Verzeichnis erstellen, falls es nicht existiert
|
||||
self._log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@property
|
||||
def log_path(self) -> Path:
|
||||
"""Gibt den konfigurierten Pfad zur Audit-Logdatei zurück."""
|
||||
return self._log_path
|
||||
|
||||
def log_violation(self, event: SecurityEvent) -> None:
|
||||
"""Schreibt ein Sicherheitsereignis in die Protokolldatei.
|
||||
|
||||
Format pro Zeile:
|
||||
[2025-01-15T10:30:00] DENIED | dhive -> privat | read on privat/.env
|
||||
|
||||
Args:
|
||||
event: Das zu protokollierende Sicherheitsereignis mit Zeitstempel,
|
||||
anfragendem Kontext, Zielkontext, Ressource, Aktion und Ergebnis.
|
||||
"""
|
||||
timestamp_iso = event.timestamp.isoformat()
|
||||
outcome_upper = event.outcome.upper()
|
||||
|
||||
line = (
|
||||
f"[{timestamp_iso}] {outcome_upper} | "
|
||||
f"{event.requesting_context} -> {event.target_context} | "
|
||||
f"{event.action} on {event.resource}\n"
|
||||
)
|
||||
|
||||
with self._lock:
|
||||
with open(self._log_path, "a", encoding="utf-8") as f:
|
||||
f.write(line)
|
||||
@@ -0,0 +1,461 @@
|
||||
"""Kontextbrücke für kontextübergreifenden Wissenstransfer.
|
||||
|
||||
Ermöglicht das sichere Teilen von Wissensartefakten zwischen Arbeitskontexten
|
||||
unter Wahrung der Sicherheitsgrenzen. Implementiert:
|
||||
- Sensitive-Content-Filterung (Secrets, Endpoints, PII) via Regex
|
||||
- Explizite Nutzerbestätigung vor Freigabe
|
||||
- Read-only-Referenzen in Zielkontexten
|
||||
- Update-Propagierung beim nächsten Lesezugriff
|
||||
- Widerruf von Freigaben
|
||||
|
||||
Requirements: 5.1, 5.2, 5.3, 5.4, 5.5, 5.6
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from monorepo.knowledge.artifact import Artifact, parse_frontmatter
|
||||
from monorepo.knowledge.index import IndexEntry, YAMLIndex
|
||||
from monorepo.models import Context
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Datenmodelle
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class SensitiveMatch:
|
||||
"""Ein gefundener sensitiver Inhalt in einem Artefakt.
|
||||
|
||||
Attributes:
|
||||
pattern_name: Name des Musters, das getroffen hat (z.B. "credentials", "endpoint", "pii").
|
||||
matched_text: Der gefundene Text-Ausschnitt.
|
||||
line_number: Zeilennummer im Inhalt (1-basiert).
|
||||
reason: Menschenlesbare Begründung, warum der Inhalt als sensitiv gilt.
|
||||
"""
|
||||
|
||||
pattern_name: str
|
||||
matched_text: str
|
||||
line_number: int
|
||||
reason: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ShareResult:
|
||||
"""Ergebnis einer Freigabe-Operation.
|
||||
|
||||
Attributes:
|
||||
success: True wenn die Freigabe erfolgreich war.
|
||||
artifact_id: ID des betroffenen Artefakts.
|
||||
shared_to: Liste der Kontexte, in denen das Artefakt nun verfügbar ist.
|
||||
errors: Liste von Fehlermeldungen bei gescheiterter Freigabe.
|
||||
"""
|
||||
|
||||
success: bool
|
||||
artifact_id: str
|
||||
shared_to: list[str] = field(default_factory=list)
|
||||
errors: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared-Artifacts-Registry
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class SharedArtifactEntry:
|
||||
"""Eintrag in der Shared-Artifacts-Registry.
|
||||
|
||||
Attributes:
|
||||
artifact_id: ID des geteilten Artefakts.
|
||||
source_context: Ursprungskontext des Artefakts.
|
||||
shared_to: Zielkontexte, in denen das Artefakt verfügbar ist.
|
||||
content_hash: Hash des Inhalts zum Zeitpunkt der Freigabe (für Update-Erkennung).
|
||||
"""
|
||||
|
||||
artifact_id: str
|
||||
source_context: str
|
||||
shared_to: list[str] = field(default_factory=list)
|
||||
content_hash: str = ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ContextBridge
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ContextBridge:
|
||||
"""Ermöglicht kontextübergreifenden Wissenstransfer.
|
||||
|
||||
Die ContextBridge prüft Artefakte auf sensitive Inhalte, erfordert
|
||||
explizite Nutzerbestätigung und stellt geteilte Artefakte als
|
||||
schreibgeschützte Lesereferenzen in Zielkontexten bereit.
|
||||
|
||||
Update-Propagierung: Wird ein Artefakt im Quellkontext aktualisiert,
|
||||
sind die Änderungen beim nächsten Lesezugriff über get_shared_content()
|
||||
sichtbar, da immer vom Quell-Artefakt gelesen wird.
|
||||
|
||||
Requirements: 5.1, 5.2, 5.3, 5.4, 5.5, 5.6
|
||||
"""
|
||||
|
||||
SENSITIVE_PATTERNS: list[tuple[str, str, str]] = [
|
||||
(
|
||||
r"(?i)(api[_-]?key|token|password|secret)\s*[:=]",
|
||||
"credentials",
|
||||
"Enthält kontextspezifische Zugangsdaten (API-Key, Token, Passwort oder Secret)",
|
||||
),
|
||||
(
|
||||
r"(?i)(endpoint|url)\s*[:=]\s*https?://",
|
||||
"endpoint",
|
||||
"Enthält kontextspezifische Endpoint-URL",
|
||||
),
|
||||
(
|
||||
r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b",
|
||||
"pii",
|
||||
"Enthält personenbezogene Daten (E-Mail-Adresse)",
|
||||
),
|
||||
]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
knowledge_base_path: Path,
|
||||
index: YAMLIndex,
|
||||
registry_path: Path | None = None,
|
||||
) -> None:
|
||||
"""Initialisiert die ContextBridge.
|
||||
|
||||
Args:
|
||||
knowledge_base_path: Basispfad des Wissensspeichers.
|
||||
index: Der YAML-Index des Wissensspeichers.
|
||||
registry_path: Pfad zur Shared-Artifacts-Registry (YAML).
|
||||
Falls None, wird in-memory gearbeitet.
|
||||
"""
|
||||
self.knowledge_base_path = knowledge_base_path
|
||||
self.index = index
|
||||
self.registry_path = registry_path
|
||||
self._registry: dict[str, SharedArtifactEntry] = {}
|
||||
|
||||
if registry_path and registry_path.exists():
|
||||
self._load_registry()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Öffentliche API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def share_artifact(
|
||||
self, artifact_id: str, user_confirmed: bool = False
|
||||
) -> ShareResult:
|
||||
"""Gibt ein Artefakt kontextübergreifend frei.
|
||||
|
||||
Prüft:
|
||||
1. Existenz des Artefakts im Index
|
||||
2. Explizite Nutzerbestätigung (Req 5.4)
|
||||
3. Sensitive-Content-Filter (Req 5.1, 5.3)
|
||||
|
||||
Bei Erfolg wird das Artefakt als Lesereferenz in allen anderen
|
||||
Kontexten bereitgestellt (Req 5.2).
|
||||
|
||||
Args:
|
||||
artifact_id: ID des freizugebenden Artefakts.
|
||||
user_confirmed: Muss True sein für explizite Nutzerbestätigung.
|
||||
|
||||
Returns:
|
||||
ShareResult mit Erfolgs-/Fehlerstatus.
|
||||
"""
|
||||
# Req 5.4: Explizite Nutzerbestätigung erforderlich
|
||||
if not user_confirmed:
|
||||
return ShareResult(
|
||||
success=False,
|
||||
artifact_id=artifact_id,
|
||||
errors=[
|
||||
"Explizite Nutzerbestätigung erforderlich. "
|
||||
"Setze user_confirmed=True um die Freigabe zu bestätigen."
|
||||
],
|
||||
)
|
||||
|
||||
# Artefakt im Index finden
|
||||
entry = self.index.get_entry(artifact_id)
|
||||
if entry is None:
|
||||
return ShareResult(
|
||||
success=False,
|
||||
artifact_id=artifact_id,
|
||||
errors=[f"Artefakt nicht im Index gefunden: {artifact_id}"],
|
||||
)
|
||||
|
||||
# Inhalt laden und auf sensitive Inhalte prüfen (Req 5.1, 5.3)
|
||||
content = self._load_artifact_content(entry)
|
||||
if content is None:
|
||||
return ShareResult(
|
||||
success=False,
|
||||
artifact_id=artifact_id,
|
||||
errors=[
|
||||
f"Artefakt-Datei nicht gefunden oder nicht lesbar: {entry.path}"
|
||||
],
|
||||
)
|
||||
|
||||
sensitive_matches = self.check_sensitive_content(content)
|
||||
if sensitive_matches:
|
||||
reasons = [
|
||||
f"Zeile {m.line_number}: {m.reason} ('{m.matched_text}')"
|
||||
for m in sensitive_matches
|
||||
]
|
||||
return ShareResult(
|
||||
success=False,
|
||||
artifact_id=artifact_id,
|
||||
errors=[
|
||||
"Freigabe verweigert: Artefakt enthält sensible Inhalte.",
|
||||
*reasons,
|
||||
],
|
||||
)
|
||||
|
||||
# Zielkontexte ermitteln (alle Kontexte außer dem Quellkontext)
|
||||
source_context = entry.scope
|
||||
target_contexts = [
|
||||
ctx.value for ctx in Context if ctx.value != source_context
|
||||
]
|
||||
|
||||
# In Registry eintragen
|
||||
self._registry[artifact_id] = SharedArtifactEntry(
|
||||
artifact_id=artifact_id,
|
||||
source_context=source_context,
|
||||
shared_to=target_contexts,
|
||||
content_hash=entry.content_hash,
|
||||
)
|
||||
|
||||
# Registry persistieren
|
||||
self._save_registry()
|
||||
|
||||
logger.info(
|
||||
"Artefakt '%s' aus Kontext '%s' für Kontexte %s freigegeben.",
|
||||
artifact_id,
|
||||
source_context,
|
||||
target_contexts,
|
||||
)
|
||||
|
||||
return ShareResult(
|
||||
success=True,
|
||||
artifact_id=artifact_id,
|
||||
shared_to=target_contexts,
|
||||
)
|
||||
|
||||
def check_sensitive_content(self, content: str) -> list[SensitiveMatch]:
|
||||
"""Prüft Inhalt auf kontextspezifische Geheimnisse und PII.
|
||||
|
||||
Verwendet Regex-Muster zur Erkennung von:
|
||||
- Credentials (API-Keys, Tokens, Passwörter, Secrets)
|
||||
- Endpoint-URLs
|
||||
- Personenbezogene Daten (E-Mail-Adressen)
|
||||
|
||||
Args:
|
||||
content: Der zu prüfende Textinhalt.
|
||||
|
||||
Returns:
|
||||
Liste der gefundenen sensitiven Stellen. Leere Liste wenn
|
||||
kein sensitiver Inhalt gefunden wurde.
|
||||
|
||||
Requirements: 5.1, 5.3
|
||||
"""
|
||||
matches: list[SensitiveMatch] = []
|
||||
lines = content.splitlines()
|
||||
|
||||
for line_num, line in enumerate(lines, start=1):
|
||||
for pattern_str, pattern_name, reason in self.SENSITIVE_PATTERNS:
|
||||
pattern = re.compile(pattern_str)
|
||||
for match in pattern.finditer(line):
|
||||
matched_text = match.group(0)
|
||||
# Text auf sinnvolle Länge kürzen
|
||||
if len(matched_text) > 60:
|
||||
matched_text = matched_text[:57] + "..."
|
||||
matches.append(
|
||||
SensitiveMatch(
|
||||
pattern_name=pattern_name,
|
||||
matched_text=matched_text,
|
||||
line_number=line_num,
|
||||
reason=reason,
|
||||
)
|
||||
)
|
||||
|
||||
return matches
|
||||
|
||||
def revoke_share(self, artifact_id: str) -> None:
|
||||
"""Widerruft die Freigabe eines geteilten Artefakts.
|
||||
|
||||
Entfernt das Artefakt aus allen Zielkontexten als Lesereferenz.
|
||||
Nach dem Widerruf ist das Artefakt nur noch im Quellkontext sichtbar.
|
||||
|
||||
Args:
|
||||
artifact_id: ID des Artefakts, dessen Freigabe widerrufen wird.
|
||||
|
||||
Raises:
|
||||
ValueError: Wenn das Artefakt nicht in der Freigabe-Registry ist.
|
||||
|
||||
Requirements: 5.6
|
||||
"""
|
||||
if artifact_id not in self._registry:
|
||||
raise ValueError(
|
||||
f"Artefakt '{artifact_id}' ist nicht freigegeben "
|
||||
f"und kann daher nicht widerrufen werden."
|
||||
)
|
||||
|
||||
entry = self._registry.pop(artifact_id)
|
||||
self._save_registry()
|
||||
|
||||
logger.info(
|
||||
"Freigabe von Artefakt '%s' widerrufen. "
|
||||
"Entfernt aus Kontexten: %s",
|
||||
artifact_id,
|
||||
entry.shared_to,
|
||||
)
|
||||
|
||||
def get_shared_content(self, artifact_id: str, requesting_context: str) -> str | None:
|
||||
"""Liest den aktuellen Inhalt eines geteilten Artefakts.
|
||||
|
||||
Liefert immer den aktuellen Stand aus dem Quellkontext,
|
||||
sodass Updates automatisch beim nächsten Lesezugriff sichtbar sind.
|
||||
|
||||
Args:
|
||||
artifact_id: ID des geteilten Artefakts.
|
||||
requesting_context: Der Kontext, aus dem der Zugriff erfolgt.
|
||||
|
||||
Returns:
|
||||
Den aktuellen Inhalt des Artefakts als String, oder None
|
||||
wenn der Zugriff nicht erlaubt ist.
|
||||
|
||||
Requirements: 5.2, 5.5
|
||||
"""
|
||||
# Prüfe ob Artefakt freigegeben ist
|
||||
registry_entry = self._registry.get(artifact_id)
|
||||
if registry_entry is None:
|
||||
return None
|
||||
|
||||
# Prüfe ob der anfragende Kontext berechtigt ist
|
||||
if requesting_context not in registry_entry.shared_to:
|
||||
return None
|
||||
|
||||
# Lade den aktuellen Inhalt aus dem Quellkontext (Req 5.5: Update-Propagierung)
|
||||
index_entry = self.index.get_entry(artifact_id)
|
||||
if index_entry is None:
|
||||
return None
|
||||
|
||||
return self._load_artifact_content(index_entry)
|
||||
|
||||
def is_shared(self, artifact_id: str) -> bool:
|
||||
"""Prüft ob ein Artefakt aktuell freigegeben ist.
|
||||
|
||||
Args:
|
||||
artifact_id: ID des zu prüfenden Artefakts.
|
||||
|
||||
Returns:
|
||||
True wenn das Artefakt freigegeben ist.
|
||||
"""
|
||||
return artifact_id in self._registry
|
||||
|
||||
def get_shared_artifacts(self, context: str) -> list[str]:
|
||||
"""Liefert alle Artefakt-IDs, die in einem bestimmten Kontext sichtbar sind.
|
||||
|
||||
Args:
|
||||
context: Der Kontext, für den die sichtbaren Artefakte ermittelt werden.
|
||||
|
||||
Returns:
|
||||
Liste der Artefakt-IDs, die im angegebenen Kontext als
|
||||
Lesereferenz verfügbar sind.
|
||||
"""
|
||||
return [
|
||||
entry.artifact_id
|
||||
for entry in self._registry.values()
|
||||
if context in entry.shared_to
|
||||
]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Interne Methoden
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _load_artifact_content(self, entry: IndexEntry) -> str | None:
|
||||
"""Lädt den Textinhalt eines Artefakts aus dem Dateisystem.
|
||||
|
||||
Args:
|
||||
entry: Der IndexEntry des Artefakts.
|
||||
|
||||
Returns:
|
||||
Den Inhalt als String oder None bei Fehler.
|
||||
"""
|
||||
if not entry.path:
|
||||
return None
|
||||
|
||||
artifact_path = self.knowledge_base_path / entry.path
|
||||
if not artifact_path.exists():
|
||||
logger.warning("Artefakt-Datei nicht gefunden: %s", artifact_path)
|
||||
return None
|
||||
|
||||
try:
|
||||
artifact = parse_frontmatter(artifact_path)
|
||||
return artifact.content
|
||||
except (ValueError, FileNotFoundError, OSError) as e:
|
||||
logger.warning(
|
||||
"Fehler beim Laden von Artefakt '%s': %s", entry.id, e
|
||||
)
|
||||
return None
|
||||
|
||||
def _load_registry(self) -> None:
|
||||
"""Lädt die Shared-Artifacts-Registry von der Festplatte."""
|
||||
if self.registry_path is None or not self.registry_path.exists():
|
||||
return
|
||||
|
||||
try:
|
||||
with open(self.registry_path, encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f)
|
||||
except (yaml.YAMLError, OSError) as e:
|
||||
logger.warning("Fehler beim Laden der Registry: %s", e)
|
||||
return
|
||||
|
||||
if not isinstance(data, dict):
|
||||
return
|
||||
|
||||
for artifact_data in data.get("shared_artifacts", []):
|
||||
if isinstance(artifact_data, dict):
|
||||
entry = SharedArtifactEntry(
|
||||
artifact_id=str(artifact_data.get("artifact_id", "")),
|
||||
source_context=str(artifact_data.get("source_context", "")),
|
||||
shared_to=list(artifact_data.get("shared_to", [])),
|
||||
content_hash=str(artifact_data.get("content_hash", "")),
|
||||
)
|
||||
if entry.artifact_id:
|
||||
self._registry[entry.artifact_id] = entry
|
||||
|
||||
def _save_registry(self) -> None:
|
||||
"""Persistiert die Shared-Artifacts-Registry auf die Festplatte."""
|
||||
if self.registry_path is None:
|
||||
return
|
||||
|
||||
data: dict[str, Any] = {
|
||||
"shared_artifacts": [
|
||||
{
|
||||
"artifact_id": entry.artifact_id,
|
||||
"source_context": entry.source_context,
|
||||
"shared_to": entry.shared_to,
|
||||
"content_hash": entry.content_hash,
|
||||
}
|
||||
for entry in self._registry.values()
|
||||
]
|
||||
}
|
||||
|
||||
self.registry_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with open(self.registry_path, "w", encoding="utf-8") as f:
|
||||
yaml.dump(
|
||||
data,
|
||||
f,
|
||||
default_flow_style=False,
|
||||
allow_unicode=True,
|
||||
sort_keys=False,
|
||||
)
|
||||
@@ -0,0 +1,735 @@
|
||||
"""CLI-Einstiegspunkt für das Monorepo-Verwaltungstool (ctx-guard).
|
||||
|
||||
Stellt alle Verwaltungskommandos als Subcommands bereit und verdrahtet
|
||||
die Komponenten: StructureManager, ContextGuard, KnowledgeStore, RepoManager,
|
||||
ContextBridge, MigrationEngine, OrchestratorAdapter, SecretEncryptionManager,
|
||||
FederationManager.
|
||||
|
||||
Konfiguration wird aus `monorepo.yaml` und `shared/config/` geladen.
|
||||
|
||||
Entry-Point: ctx-guard (pyproject.toml → [project.scripts])
|
||||
Requirements: 1.1, 1.2, 4.4, 6.3, 9.6, 10.11
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from monorepo.config import MonorepoConfig, load_monorepo_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hilfsfunktionen für Komponenteninitialisierung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _find_monorepo_root() -> Path:
|
||||
"""Sucht das Monorepo-Root anhand von monorepo.yaml.
|
||||
|
||||
Sucht im aktuellen Verzeichnis und in übergeordneten Verzeichnissen
|
||||
nach monorepo.yaml und gibt das Verzeichnis zurück.
|
||||
|
||||
Returns:
|
||||
Pfad zum Monorepo-Root.
|
||||
|
||||
Raises:
|
||||
SystemExit: Wenn kein monorepo.yaml gefunden wird.
|
||||
"""
|
||||
current = Path.cwd()
|
||||
for parent in [current, *current.parents]:
|
||||
if (parent / "monorepo.yaml").exists():
|
||||
return parent
|
||||
print("FEHLER: monorepo.yaml nicht gefunden.", file=sys.stderr)
|
||||
print("Bitte im Monorepo-Root ausführen oder `ctx-guard init` verwenden.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def _load_config_safe() -> MonorepoConfig:
|
||||
"""Lädt die Monorepo-Konfiguration mit Fehlerbehandlung.
|
||||
|
||||
Returns:
|
||||
Geladene MonorepoConfig.
|
||||
|
||||
Raises:
|
||||
SystemExit: Bei Konfigurationsfehlern.
|
||||
"""
|
||||
try:
|
||||
return load_monorepo_config()
|
||||
except FileNotFoundError as e:
|
||||
print(f"FEHLER: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"FEHLER beim Laden der Konfiguration: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def _get_structure_manager(root: Path) -> "StructureManager":
|
||||
"""Erstellt einen StructureManager für das gegebene Root."""
|
||||
from monorepo.structure import StructureManager
|
||||
return StructureManager(root)
|
||||
|
||||
|
||||
def _get_context_guard(root: Path) -> "ContextGuard":
|
||||
"""Erstellt einen ContextGuard für das gegebene Root."""
|
||||
from monorepo.security import ContextGuard
|
||||
return ContextGuard(root)
|
||||
|
||||
|
||||
def _get_knowledge_store(root: Path):
|
||||
"""Erstellt einen KnowledgeStore für das gegebene Root."""
|
||||
from monorepo.knowledge.store import KnowledgeStore
|
||||
from monorepo.models import ScopeConfig
|
||||
|
||||
ks_path = root / "shared" / "knowledge-store"
|
||||
scope_config = ScopeConfig(scopes={
|
||||
"privat": {"scope": "privat", "paths": ["privat/"]},
|
||||
"dhive": {"scope": "dhive", "paths": ["dhive/"]},
|
||||
"bahn": {"scope": "bahn", "paths": ["bahn/"]},
|
||||
"shared": {"scope": "shared", "paths": ["shared/"]},
|
||||
})
|
||||
return KnowledgeStore(base_path=ks_path, scope_config=scope_config)
|
||||
|
||||
|
||||
def _get_repo_manager(root: Path) -> "RepoManager":
|
||||
"""Erstellt einen RepoManager für das gegebene Root."""
|
||||
from monorepo.repos import RepoManager
|
||||
config_path = root / "shared" / "config" / "repos.yaml"
|
||||
return RepoManager(config_path=config_path, monorepo_root=root)
|
||||
|
||||
|
||||
def _get_encryption_manager(root: Path):
|
||||
"""Erstellt einen SecretEncryptionManager für das gegebene Root."""
|
||||
from monorepo.encryption import MachineContextManager
|
||||
mcm = MachineContextManager(root)
|
||||
try:
|
||||
mcm.load()
|
||||
except FileNotFoundError:
|
||||
from monorepo.models import MachineContext
|
||||
# Fallback: Erstelle einen Default-Maschinenkontext
|
||||
mcm._machine_context = MachineContext(
|
||||
name="unknown",
|
||||
description="Standard-Maschinenkontext (keine Konfiguration gefunden)",
|
||||
authorized_contexts=["privat", "dhive", "bahn"],
|
||||
key_source="keyring",
|
||||
)
|
||||
return mcm.create_encryption_manager()
|
||||
|
||||
|
||||
def _get_federation_manager(root: Path):
|
||||
"""Erstellt einen FederationManager für das gegebene Root."""
|
||||
from monorepo.federation import FederationManager
|
||||
config_path = root / "shared" / "config" / "team-repos.yaml"
|
||||
encryption_mgr = _get_encryption_manager(root)
|
||||
return FederationManager(
|
||||
config_path=config_path,
|
||||
monorepo_root=root,
|
||||
encryption_manager=encryption_mgr,
|
||||
)
|
||||
|
||||
|
||||
def _get_migration_engine(root: Path):
|
||||
"""Erstellt eine MigrationEngine für das gegebene Root."""
|
||||
from monorepo.migration import MigrationEngine
|
||||
return MigrationEngine(monorepo_root=root)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Subcommand-Handler
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _cmd_init(args: argparse.Namespace) -> int:
|
||||
"""Initialisiert die Monorepo-Grundstruktur.
|
||||
|
||||
Erstellt die vier Kontextordner und die shared-Unterstruktur,
|
||||
sowie eine minimale monorepo.yaml falls sie nicht existiert.
|
||||
"""
|
||||
target = Path(args.path).resolve() if args.path else Path.cwd()
|
||||
|
||||
print(f"Initialisiere Monorepo in: {target}")
|
||||
|
||||
# Kontextordner erstellen
|
||||
contexts = ["privat", "dhive", "bahn", "shared"]
|
||||
for ctx in contexts:
|
||||
(target / ctx).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Shared-Unterstruktur
|
||||
shared_dirs = [
|
||||
"shared/tools",
|
||||
"shared/powers",
|
||||
"shared/knowledge-store",
|
||||
"shared/config",
|
||||
"shared/mcp-servers",
|
||||
]
|
||||
for d in shared_dirs:
|
||||
(target / d).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# monorepo.yaml erstellen falls nicht vorhanden
|
||||
config_path = target / "monorepo.yaml"
|
||||
if not config_path.exists():
|
||||
import yaml
|
||||
config_data = {
|
||||
"version": "1.0",
|
||||
"contexts": [
|
||||
{"name": "privat", "description": "Persönliche Projekte"},
|
||||
{"name": "dhive", "description": "dhive GmbH Projekte"},
|
||||
{"name": "bahn", "description": "DB InfraGO Projekte"},
|
||||
{"name": "shared", "description": "Kontextübergreifende Tools und Wissen"},
|
||||
],
|
||||
"naming": {
|
||||
"pattern": r"^[a-z0-9][a-z0-9\-]{0,48}[a-z0-9]$",
|
||||
"min_length": 2,
|
||||
"max_length": 50,
|
||||
},
|
||||
"security": {
|
||||
"config": "shared/config/access-config.yaml",
|
||||
"audit_log": ".audit/access.log",
|
||||
"encryption": {
|
||||
"tool": "git-crypt",
|
||||
"machine_context_config": "shared/config/machine-context.yaml",
|
||||
"key_source": "keyring",
|
||||
},
|
||||
"federation": {
|
||||
"config": "shared/config/team-repos.yaml",
|
||||
"conflict_strategy": "team-wins",
|
||||
},
|
||||
},
|
||||
}
|
||||
with open(config_path, "w", encoding="utf-8") as f:
|
||||
yaml.dump(config_data, f, default_flow_style=False, allow_unicode=True)
|
||||
print(f" Erstellt: {config_path}")
|
||||
|
||||
# Audit-Verzeichnis
|
||||
(target / ".audit").mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Git-Hooks installieren (pre-commit mit Read-Only-Schutz + Secret-Prüfung)
|
||||
git_dir = target / ".git"
|
||||
if git_dir.exists():
|
||||
from monorepo.hooks import install_hooks
|
||||
try:
|
||||
install_hooks(target)
|
||||
print(" Installiert: Git pre-commit Hook (Read-Only + Secrets)")
|
||||
except Exception as e:
|
||||
logger.warning("Hook-Installation übersprungen: %s", e)
|
||||
print(f" ⚠ Hook-Installation übersprungen: {e}")
|
||||
|
||||
print("✓ Monorepo-Struktur erfolgreich initialisiert.")
|
||||
return 0
|
||||
|
||||
|
||||
def _cmd_create_project(args: argparse.Namespace) -> int:
|
||||
"""Erstellt ein neues Projekt in einem Arbeitskontext."""
|
||||
root = _find_monorepo_root()
|
||||
sm = _get_structure_manager(root)
|
||||
|
||||
try:
|
||||
project_path = sm.create_project(context=args.context, name=args.name)
|
||||
print(f"✓ Projekt erstellt: {project_path}")
|
||||
return 0
|
||||
except ValueError as e:
|
||||
print(f"FEHLER: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def _cmd_list_projects(args: argparse.Namespace) -> int:
|
||||
"""Listet alle Projekte auf, optional gefiltert nach Kontext."""
|
||||
root = _find_monorepo_root()
|
||||
sm = _get_structure_manager(root)
|
||||
|
||||
try:
|
||||
projects = sm.list_projects(context=args.context)
|
||||
except ValueError as e:
|
||||
print(f"FEHLER: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if not projects:
|
||||
ctx_info = f" im Kontext '{args.context}'" if args.context else ""
|
||||
print(f"Keine Projekte{ctx_info} gefunden.")
|
||||
return 0
|
||||
|
||||
# Tabellen-Ausgabe
|
||||
print(f"{'Kontext':<10} {'Name':<40} {'Pfad'}")
|
||||
print("-" * 80)
|
||||
for p in projects:
|
||||
print(f"{p.context:<10} {p.name:<40} {p.path}")
|
||||
|
||||
print(f"\nGesamt: {len(projects)} Projekt(e)")
|
||||
return 0
|
||||
|
||||
|
||||
def _cmd_add_repo(args: argparse.Namespace) -> int:
|
||||
"""Bindet ein externes Repository ein."""
|
||||
root = _find_monorepo_root()
|
||||
rm = _get_repo_manager(root)
|
||||
|
||||
from monorepo.models import RepoEntry
|
||||
entry = RepoEntry(
|
||||
name=args.name,
|
||||
url=args.url,
|
||||
mode=args.mode,
|
||||
target=args.target,
|
||||
pinned=args.pinned,
|
||||
mechanism=args.mechanism,
|
||||
)
|
||||
|
||||
try:
|
||||
rm.add_repo(entry)
|
||||
print(f"✓ Repo '{args.name}' eingebunden via {args.mechanism} → {args.target}")
|
||||
return 0
|
||||
except (ValueError, Exception) as e:
|
||||
print(f"FEHLER: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def _cmd_sync_repo(args: argparse.Namespace) -> int:
|
||||
"""Synchronisiert ein eingebundenes Repository."""
|
||||
root = _find_monorepo_root()
|
||||
rm = _get_repo_manager(root)
|
||||
|
||||
result = rm.sync(args.name)
|
||||
|
||||
if result.success:
|
||||
print(f"✓ Repo '{args.name}' synchronisiert ({result.commits_synced} Commits).")
|
||||
return 0
|
||||
else:
|
||||
print(f"FEHLER bei Synchronisation von '{args.name}':", file=sys.stderr)
|
||||
for conflict in result.conflicts:
|
||||
print(f" - {conflict.details}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def _cmd_migrate(args: argparse.Namespace) -> int:
|
||||
"""Migriert ein bestehendes Repository ins Monorepo."""
|
||||
root = _find_monorepo_root()
|
||||
engine = _get_migration_engine(root)
|
||||
|
||||
from monorepo.models import MigrationPlan
|
||||
plan = MigrationPlan(
|
||||
source_repo=args.source,
|
||||
target_context=args.context,
|
||||
target_name=args.name,
|
||||
mode=args.mode,
|
||||
dependencies=[],
|
||||
order=0,
|
||||
)
|
||||
|
||||
print(f"Migriere '{args.source}' → {args.context}/{args.name} (Modus: {args.mode})...")
|
||||
result = engine.migrate(plan)
|
||||
|
||||
if result.success:
|
||||
print(f"✓ Migration erfolgreich abgeschlossen.")
|
||||
if result.commits_migrated:
|
||||
print(f" Commits: {result.commits_migrated}")
|
||||
if result.branches_migrated:
|
||||
print(f" Branches: {', '.join(result.branches_migrated)}")
|
||||
if result.tags_migrated:
|
||||
print(f" Tags: {', '.join(result.tags_migrated)}")
|
||||
return 0
|
||||
else:
|
||||
print(f"FEHLER bei Migration:", file=sys.stderr)
|
||||
if result.error_message:
|
||||
print(f" {result.error_message}", file=sys.stderr)
|
||||
if result.conflicts:
|
||||
print(f"\nKonflikte ({len(result.conflicts)}):", file=sys.stderr)
|
||||
for c in result.conflicts:
|
||||
print(f" [{c.conflict_type}] {c.description}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def _cmd_validate(args: argparse.Namespace) -> int:
|
||||
"""Validiert eine abgeschlossene Migration."""
|
||||
root = _find_monorepo_root()
|
||||
engine = _get_migration_engine(root)
|
||||
|
||||
print(f"Validiere Migration von '{args.name}'...")
|
||||
result = engine.validate(args.name)
|
||||
|
||||
if result.success:
|
||||
print(f"✓ Validierung erfolgreich für '{args.name}'.")
|
||||
for check_name, passed in result.checks.items():
|
||||
status = "✓" if passed else "✗"
|
||||
print(f" {status} {check_name}")
|
||||
return 0
|
||||
else:
|
||||
print(f"FEHLER: Validierung fehlgeschlagen für '{args.name}':", file=sys.stderr)
|
||||
if result.error_message:
|
||||
print(f" {result.error_message}", file=sys.stderr)
|
||||
for detail in result.details:
|
||||
print(f" - {detail}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def _cmd_search(args: argparse.Namespace) -> int:
|
||||
"""Durchsucht den Wissensspeicher."""
|
||||
root = _find_monorepo_root()
|
||||
ks = _get_knowledge_store(root)
|
||||
|
||||
# Bestimme erlaubte Scopes (standardmäßig alle)
|
||||
allowed_scopes = args.scopes.split(",") if args.scopes else ["privat", "dhive", "bahn", "shared"]
|
||||
|
||||
results = ks.search(query=args.query, allowed_scopes=allowed_scopes)
|
||||
|
||||
if not results:
|
||||
print(f"Keine Treffer für '{args.query}'.")
|
||||
return 0
|
||||
|
||||
print(f"Treffer für '{args.query}' ({len(results)} Ergebnis(se)):\n")
|
||||
for r in results:
|
||||
score_bar = "█" * int(r.relevance_score * 10)
|
||||
print(f" [{r.entry.scope}] {r.entry.title}")
|
||||
print(f" Pfad: {r.entry.path}")
|
||||
print(f" Tags: {', '.join(r.entry.tags)}")
|
||||
print(f" Relevanz: {score_bar} ({r.relevance_score:.2f})")
|
||||
print()
|
||||
|
||||
if results and results[0].partial_results:
|
||||
print("⚠ Suche wegen Timeout abgebrochen – möglicherweise unvollständig.")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def _cmd_encrypt(args: argparse.Namespace) -> int:
|
||||
"""Verschlüsselt eine Datei mit dem Kontext-Schlüssel."""
|
||||
root = _find_monorepo_root()
|
||||
enc = _get_encryption_manager(root)
|
||||
|
||||
file_path = Path(args.file).resolve()
|
||||
context = args.context
|
||||
|
||||
if not context:
|
||||
# Kontext aus Pfad ableiten
|
||||
try:
|
||||
rel = file_path.relative_to(root)
|
||||
context = rel.parts[0] if rel.parts else None
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if not context:
|
||||
print("FEHLER: Kontext konnte nicht ermittelt werden. Bitte --context angeben.", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
result = enc.encrypt_file(file_path, context)
|
||||
|
||||
if result.success:
|
||||
print(f"✓ Datei verschlüsselt: {file_path} (Kontext: {context})")
|
||||
return 0
|
||||
else:
|
||||
print(f"FEHLER: {result.error}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def _cmd_decrypt(args: argparse.Namespace) -> int:
|
||||
"""Entschlüsselt eine Datei."""
|
||||
root = _find_monorepo_root()
|
||||
enc = _get_encryption_manager(root)
|
||||
|
||||
file_path = Path(args.file).resolve()
|
||||
result = enc.decrypt_file(file_path)
|
||||
|
||||
if result.success:
|
||||
print(f"✓ Datei entschlüsselt: {file_path}")
|
||||
return 0
|
||||
else:
|
||||
print(f"FEHLER: {result.error}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def _cmd_onboard(args: argparse.Namespace) -> int:
|
||||
"""Richtet eine neue Maschine mit autorisierten Schlüsseln ein."""
|
||||
root = _find_monorepo_root()
|
||||
enc = _get_encryption_manager(root)
|
||||
|
||||
contexts = [c.strip() for c in args.contexts.split(",")]
|
||||
print(f"Onboarding Maschine '{args.machine_name}' für Kontexte: {', '.join(contexts)}...")
|
||||
|
||||
result = enc.onboard_machine(args.machine_name, contexts)
|
||||
|
||||
if result.success:
|
||||
print(f"✓ Maschine '{args.machine_name}' erfolgreich eingerichtet.")
|
||||
print(f" Autorisierte Kontexte: {', '.join(result.authorized_contexts)}")
|
||||
print(f" Installierte Schlüssel: {len(result.installed_keys)}")
|
||||
return 0
|
||||
else:
|
||||
print(f"FEHLER beim Onboarding:", file=sys.stderr)
|
||||
for err in result.errors:
|
||||
print(f" - {err}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def _cmd_fed_sync(args: argparse.Namespace) -> int:
|
||||
"""Synchronisiert mit einem Team-Repository."""
|
||||
root = _find_monorepo_root()
|
||||
fm = _get_federation_manager(root)
|
||||
|
||||
context = args.context
|
||||
direction = args.direction
|
||||
|
||||
print(f"Federation-Sync: Kontext '{context}', Richtung '{direction}'...")
|
||||
|
||||
if direction == "pull":
|
||||
result = fm.sync_from_team(context)
|
||||
elif direction == "push":
|
||||
result = fm.sync_to_team(context)
|
||||
else:
|
||||
result = fm.full_sync(context)
|
||||
|
||||
if result.success:
|
||||
print(f"✓ Sync erfolgreich ({result.commits_synced} Commits, Richtung: {result.direction}).")
|
||||
return 0
|
||||
else:
|
||||
print(f"FEHLER bei Federation-Sync:", file=sys.stderr)
|
||||
for conflict in result.conflicts:
|
||||
print(f" - [{conflict.conflict_type}] {conflict.details}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def _cmd_fed_status(args: argparse.Namespace) -> int:
|
||||
"""Zeigt den Federation-Status für alle oder einen bestimmten Kontext."""
|
||||
root = _find_monorepo_root()
|
||||
fm = _get_federation_manager(root)
|
||||
|
||||
contexts = [args.context] if args.context else ["privat", "dhive", "bahn"]
|
||||
|
||||
print("Federation-Status:")
|
||||
print("-" * 60)
|
||||
|
||||
for ctx in contexts:
|
||||
entry = fm._get_team_entry(ctx)
|
||||
if entry is None:
|
||||
print(f" {ctx:<10} Nicht konfiguriert")
|
||||
continue
|
||||
|
||||
# Prüfe auf Konflikte
|
||||
conflicts = fm.detect_conflicts(ctx)
|
||||
conflict_status = f"⚠ {len(conflicts)} Konflikt(e)" if conflicts else "✓ Keine Konflikte"
|
||||
|
||||
print(f" {ctx:<10} URL: {entry.url}")
|
||||
print(f" {'':10} Branch: {entry.branch}")
|
||||
print(f" {'':10} Sync: {entry.sync_direction} ({entry.sync_frequency})")
|
||||
print(f" {'':10} Status: {conflict_status}")
|
||||
if entry.shared_mirror:
|
||||
mirror_status = "aktiv" if entry.shared_mirror.enabled else "deaktiviert"
|
||||
print(f" {'':10} Shared-Mirror: {mirror_status}")
|
||||
print()
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def _cmd_install_hooks(args: argparse.Namespace) -> int:
|
||||
"""Installiert oder aktualisiert die Monorepo Git-Hooks."""
|
||||
root = _find_monorepo_root()
|
||||
|
||||
from monorepo.hooks import install_hooks
|
||||
try:
|
||||
install_hooks(root)
|
||||
print("✓ Git-Hooks installiert/aktualisiert.")
|
||||
return 0
|
||||
except Exception as e:
|
||||
print(f"FEHLER bei Hook-Installation: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def _cmd_uninstall_hooks(args: argparse.Namespace) -> int:
|
||||
"""Entfernt die Monorepo Git-Hooks."""
|
||||
root = _find_monorepo_root()
|
||||
|
||||
from monorepo.hooks import uninstall_hooks
|
||||
try:
|
||||
uninstall_hooks(root)
|
||||
print("✓ Git-Hooks entfernt.")
|
||||
return 0
|
||||
except Exception as e:
|
||||
print(f"FEHLER bei Hook-Deinstallation: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Argument-Parser-Aufbau
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _build_parser() -> argparse.ArgumentParser:
|
||||
"""Baut den vollständigen Argument-Parser mit allen Subcommands auf."""
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="ctx-guard",
|
||||
description="Monorepo-Verwaltungstool: Kontextgrenzen, Wissensspeicher, "
|
||||
"Repo-Sync, Verschlüsselung und Föderation.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-v", "--verbose",
|
||||
action="store_true",
|
||||
help="Ausführliche Logging-Ausgabe aktivieren.",
|
||||
)
|
||||
|
||||
subparsers = parser.add_subparsers(dest="command", help="Verfügbare Kommandos")
|
||||
|
||||
# --- init ---
|
||||
p_init = subparsers.add_parser("init", help="Monorepo-Struktur initialisieren")
|
||||
p_init.add_argument(
|
||||
"--path", type=str, default=None,
|
||||
help="Zielpfad für die Initialisierung (Standard: aktuelles Verzeichnis).",
|
||||
)
|
||||
|
||||
# --- create-project ---
|
||||
p_create = subparsers.add_parser("create-project", help="Neues Projekt anlegen")
|
||||
p_create.add_argument("context", choices=["privat", "dhive", "bahn", "shared"],
|
||||
help="Arbeitskontext für das Projekt.")
|
||||
p_create.add_argument("name", help="Projektname (kebab-case, 2-50 Zeichen).")
|
||||
|
||||
# --- list-projects ---
|
||||
p_list = subparsers.add_parser("list-projects", help="Projekte auflisten")
|
||||
p_list.add_argument(
|
||||
"--context", "-c",
|
||||
choices=["privat", "dhive", "bahn", "shared"],
|
||||
default=None,
|
||||
help="Nur Projekte des angegebenen Kontexts anzeigen.",
|
||||
)
|
||||
|
||||
# --- add-repo ---
|
||||
p_add = subparsers.add_parser("add-repo", help="Externes Repository einbinden")
|
||||
p_add.add_argument("name", help="Eindeutiger Name für das Repo.")
|
||||
p_add.add_argument("url", help="Repository-URL.")
|
||||
p_add.add_argument("target", help="Zielpfad im Monorepo (z.B. 'bahn/db-wissen').")
|
||||
p_add.add_argument(
|
||||
"--mode", choices=["read-only", "upstream"], default="read-only",
|
||||
help="Einbindungsmodus (Standard: read-only).",
|
||||
)
|
||||
p_add.add_argument(
|
||||
"--pinned", default="main",
|
||||
help="Gepinnte Version: Branch, Tag oder SHA (Standard: main).",
|
||||
)
|
||||
p_add.add_argument(
|
||||
"--mechanism", choices=["subtree", "submodule"], default="subtree",
|
||||
help="Einbindungsmechanismus (Standard: subtree).",
|
||||
)
|
||||
|
||||
# --- sync-repo ---
|
||||
p_sync = subparsers.add_parser("sync-repo", help="Repository synchronisieren")
|
||||
p_sync.add_argument("name", help="Name des zu synchronisierenden Repos.")
|
||||
|
||||
# --- migrate ---
|
||||
p_migrate = subparsers.add_parser("migrate", help="Repository ins Monorepo migrieren")
|
||||
p_migrate.add_argument("source", help="Pfad zum Quell-Repository.")
|
||||
p_migrate.add_argument("context", choices=["privat", "dhive", "bahn", "shared"],
|
||||
help="Ziel-Arbeitskontext.")
|
||||
p_migrate.add_argument("name", help="Projektname im Ziel.")
|
||||
p_migrate.add_argument(
|
||||
"--mode", choices=["direct", "subtree", "upstream"], default="direct",
|
||||
help="Migrationsmodus (Standard: direct).",
|
||||
)
|
||||
|
||||
# --- validate ---
|
||||
p_validate = subparsers.add_parser("validate", help="Migration validieren")
|
||||
p_validate.add_argument("name", help="Name des migrierten Repos/Projekts.")
|
||||
|
||||
# --- search ---
|
||||
p_search = subparsers.add_parser("search", help="Wissensspeicher durchsuchen")
|
||||
p_search.add_argument("query", help="Suchanfrage.")
|
||||
p_search.add_argument(
|
||||
"--scopes", "-s", default=None,
|
||||
help="Komma-getrennte Scopes (Standard: alle). Z.B. 'privat,bahn'.",
|
||||
)
|
||||
|
||||
# --- encrypt ---
|
||||
p_encrypt = subparsers.add_parser("encrypt", help="Datei verschlüsseln")
|
||||
p_encrypt.add_argument("file", help="Pfad zur zu verschlüsselnden Datei.")
|
||||
p_encrypt.add_argument(
|
||||
"--context", "-c", default=None,
|
||||
help="Arbeitskontext (wird aus Pfad abgeleitet wenn nicht angegeben).",
|
||||
)
|
||||
|
||||
# --- decrypt ---
|
||||
p_decrypt = subparsers.add_parser("decrypt", help="Datei entschlüsseln")
|
||||
p_decrypt.add_argument("file", help="Pfad zur zu entschlüsselnden Datei.")
|
||||
|
||||
# --- onboard ---
|
||||
p_onboard = subparsers.add_parser("onboard", help="Neue Maschine einrichten")
|
||||
p_onboard.add_argument("machine_name", help="Name der Maschine.")
|
||||
p_onboard.add_argument(
|
||||
"contexts",
|
||||
help="Komma-getrennte Liste autorisierter Kontexte (z.B. 'privat,dhive,bahn').",
|
||||
)
|
||||
|
||||
# --- fed-sync ---
|
||||
p_fedsync = subparsers.add_parser("fed-sync", help="Team-Repo synchronisieren")
|
||||
p_fedsync.add_argument("context", choices=["privat", "dhive", "bahn"],
|
||||
help="Arbeitskontext des Team-Repos.")
|
||||
p_fedsync.add_argument(
|
||||
"--direction", "-d",
|
||||
choices=["pull", "push", "full"], default="full",
|
||||
help="Sync-Richtung (Standard: full/bidirektional).",
|
||||
)
|
||||
|
||||
# --- fed-status ---
|
||||
p_fedstatus = subparsers.add_parser("fed-status", help="Federation-Status anzeigen")
|
||||
p_fedstatus.add_argument(
|
||||
"--context", "-c",
|
||||
choices=["privat", "dhive", "bahn"],
|
||||
default=None,
|
||||
help="Nur Status für den angegebenen Kontext anzeigen.",
|
||||
)
|
||||
|
||||
# --- install-hooks ---
|
||||
subparsers.add_parser("install-hooks", help="Git-Hooks installieren/aktualisieren")
|
||||
|
||||
# --- uninstall-hooks ---
|
||||
subparsers.add_parser("uninstall-hooks", help="Git-Hooks entfernen")
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dispatch
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_COMMAND_HANDLERS = {
|
||||
"init": _cmd_init,
|
||||
"create-project": _cmd_create_project,
|
||||
"list-projects": _cmd_list_projects,
|
||||
"add-repo": _cmd_add_repo,
|
||||
"sync-repo": _cmd_sync_repo,
|
||||
"migrate": _cmd_migrate,
|
||||
"validate": _cmd_validate,
|
||||
"search": _cmd_search,
|
||||
"encrypt": _cmd_encrypt,
|
||||
"decrypt": _cmd_decrypt,
|
||||
"onboard": _cmd_onboard,
|
||||
"fed-sync": _cmd_fed_sync,
|
||||
"fed-status": _cmd_fed_status,
|
||||
"install-hooks": _cmd_install_hooks,
|
||||
"uninstall-hooks": _cmd_uninstall_hooks,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Haupt-Einstiegspunkt für das CLI-Tool ctx-guard."""
|
||||
parser = _build_parser()
|
||||
args = parser.parse_args()
|
||||
|
||||
# Logging konfigurieren
|
||||
log_level = logging.DEBUG if args.verbose else logging.WARNING
|
||||
logging.basicConfig(
|
||||
level=log_level,
|
||||
format="%(levelname)s: %(name)s: %(message)s",
|
||||
)
|
||||
|
||||
if not args.command:
|
||||
parser.print_help()
|
||||
sys.exit(0)
|
||||
|
||||
handler = _COMMAND_HANDLERS.get(args.command)
|
||||
if handler is None:
|
||||
parser.print_help()
|
||||
sys.exit(1)
|
||||
|
||||
exit_code = handler(args)
|
||||
sys.exit(exit_code)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Konfigurationsmodul für das Monorepo-CLI.
|
||||
|
||||
Lädt und validiert die zentrale monorepo.yaml und stellt die Konfiguration
|
||||
als typisierte Datenstrukturen bereit.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
@dataclass
|
||||
class NamingConfig:
|
||||
"""Namenskonventions-Konfiguration."""
|
||||
|
||||
pattern: str = r"^[a-z0-9][a-z0-9\-]{0,48}[a-z0-9]$"
|
||||
min_length: int = 2
|
||||
max_length: int = 50
|
||||
|
||||
|
||||
@dataclass
|
||||
class EncryptionConfig:
|
||||
"""Verschlüsselungs-Konfiguration."""
|
||||
|
||||
tool: str = "git-crypt"
|
||||
machine_context_config: str = "shared/config/machine-context.yaml"
|
||||
key_source: str = "keyring"
|
||||
|
||||
|
||||
@dataclass
|
||||
class FederationConfig:
|
||||
"""Föderations-Konfiguration."""
|
||||
|
||||
config: str = "shared/config/team-repos.yaml"
|
||||
conflict_strategy: str = "team-wins"
|
||||
|
||||
|
||||
@dataclass
|
||||
class SecurityConfig:
|
||||
"""Sicherheits-Konfiguration."""
|
||||
|
||||
config: str = "shared/config/access-config.yaml"
|
||||
audit_log: str = ".audit/access.log"
|
||||
encryption: EncryptionConfig = field(default_factory=EncryptionConfig)
|
||||
federation: FederationConfig = field(default_factory=FederationConfig)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ContextDefinition:
|
||||
"""Definition eines Arbeitskontexts."""
|
||||
|
||||
name: str
|
||||
description: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class MonorepoConfig:
|
||||
"""Zentrale Monorepo-Konfiguration (monorepo.yaml)."""
|
||||
|
||||
version: str = "1.0"
|
||||
contexts: list[ContextDefinition] = field(default_factory=list)
|
||||
naming: NamingConfig = field(default_factory=NamingConfig)
|
||||
security: SecurityConfig = field(default_factory=SecurityConfig)
|
||||
|
||||
|
||||
def _parse_encryption_config(data: dict[str, Any]) -> EncryptionConfig:
|
||||
"""Parst die Verschlüsselungskonfiguration aus einem dict."""
|
||||
return EncryptionConfig(
|
||||
tool=data.get("tool", "git-crypt"),
|
||||
machine_context_config=data.get("machine_context_config", "shared/config/machine-context.yaml"),
|
||||
key_source=data.get("key_source", "keyring"),
|
||||
)
|
||||
|
||||
|
||||
def _parse_federation_config(data: dict[str, Any]) -> FederationConfig:
|
||||
"""Parst die Föderationskonfiguration aus einem dict."""
|
||||
return FederationConfig(
|
||||
config=data.get("config", "shared/config/team-repos.yaml"),
|
||||
conflict_strategy=data.get("conflict_strategy", "team-wins"),
|
||||
)
|
||||
|
||||
|
||||
def _parse_security_config(data: dict[str, Any]) -> SecurityConfig:
|
||||
"""Parst die Sicherheitskonfiguration aus einem dict."""
|
||||
encryption_data = data.get("encryption", {})
|
||||
federation_data = data.get("federation", {})
|
||||
return SecurityConfig(
|
||||
config=data.get("config", "shared/config/access-config.yaml"),
|
||||
audit_log=data.get("audit_log", ".audit/access.log"),
|
||||
encryption=_parse_encryption_config(encryption_data),
|
||||
federation=_parse_federation_config(federation_data),
|
||||
)
|
||||
|
||||
|
||||
def _parse_naming_config(data: dict[str, Any]) -> NamingConfig:
|
||||
"""Parst die Namenskonventions-Konfiguration aus einem dict."""
|
||||
return NamingConfig(
|
||||
pattern=data.get("pattern", r"^[a-z0-9][a-z0-9\-]{0,48}[a-z0-9]$"),
|
||||
min_length=data.get("min_length", 2),
|
||||
max_length=data.get("max_length", 50),
|
||||
)
|
||||
|
||||
|
||||
def load_monorepo_config(config_path: Path | None = None) -> MonorepoConfig:
|
||||
"""Lädt die zentrale Monorepo-Konfiguration aus monorepo.yaml.
|
||||
|
||||
Args:
|
||||
config_path: Pfad zur monorepo.yaml. Falls None, wird im aktuellen
|
||||
Verzeichnis und darüber gesucht.
|
||||
|
||||
Returns:
|
||||
MonorepoConfig mit allen geladenen Einstellungen.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: Wenn die Konfigurationsdatei nicht gefunden wird.
|
||||
yaml.YAMLError: Wenn die YAML-Datei nicht geparst werden kann.
|
||||
"""
|
||||
if config_path is None:
|
||||
config_path = _find_config_file()
|
||||
|
||||
if not config_path.exists():
|
||||
raise FileNotFoundError(f"Monorepo-Konfiguration nicht gefunden: {config_path}")
|
||||
|
||||
with open(config_path, encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f)
|
||||
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"Ungültiges monorepo.yaml-Format: Erwartet dict, erhalten {type(data)}")
|
||||
|
||||
# Kontexte parsen
|
||||
contexts: list[ContextDefinition] = []
|
||||
for ctx in data.get("contexts", []):
|
||||
contexts.append(ContextDefinition(
|
||||
name=ctx.get("name", ""),
|
||||
description=ctx.get("description", ""),
|
||||
))
|
||||
|
||||
# Naming-Konfiguration
|
||||
naming = _parse_naming_config(data.get("naming", {}))
|
||||
|
||||
# Security-Konfiguration
|
||||
security = _parse_security_config(data.get("security", {}))
|
||||
|
||||
return MonorepoConfig(
|
||||
version=data.get("version", "1.0"),
|
||||
contexts=contexts,
|
||||
naming=naming,
|
||||
security=security,
|
||||
)
|
||||
|
||||
|
||||
def _find_config_file() -> Path:
|
||||
"""Sucht monorepo.yaml im aktuellen Verzeichnis und darüber.
|
||||
|
||||
Returns:
|
||||
Pfad zur gefundenen monorepo.yaml.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: Wenn keine monorepo.yaml gefunden wird.
|
||||
"""
|
||||
current = Path.cwd()
|
||||
for parent in [current, *current.parents]:
|
||||
candidate = parent / "monorepo.yaml"
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
raise FileNotFoundError(
|
||||
"monorepo.yaml nicht gefunden. Bitte im Monorepo-Root ausführen "
|
||||
"oder config_path explizit angeben."
|
||||
)
|
||||
@@ -0,0 +1,826 @@
|
||||
"""Verschlüsselungsmodul für das Monorepo-CLI.
|
||||
|
||||
Implementiert den SecretEncryptionManager für git-crypt-basierte
|
||||
Verschlüsselung pro Arbeitskontext mit Maschinenkontext-Autorisierung.
|
||||
|
||||
Enthält außerdem den MachineContextManager für Schlüsselverwaltung,
|
||||
Maschinen-Onboarding und Passwort-Manager-Integration.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import yaml
|
||||
|
||||
from monorepo.models import (
|
||||
EncryptionKey,
|
||||
MachineContext,
|
||||
OnboardingResult,
|
||||
PasswordManagerConfig,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Ergebnis-Dataclasses
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class EncryptionResult:
|
||||
"""Ergebnis einer Verschlüsselungsoperation."""
|
||||
|
||||
success: bool
|
||||
file_path: Path
|
||||
context: str
|
||||
error: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class DecryptionResult:
|
||||
"""Ergebnis einer Entschlüsselungsoperation."""
|
||||
|
||||
success: bool
|
||||
file_path: Path
|
||||
content: bytes | None = None
|
||||
error: str | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SecretEncryptionManager
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Dateimuster, die als Secret-Dateien behandelt werden.
|
||||
SECRET_PATTERNS: list[str] = [
|
||||
"**/.env",
|
||||
"**/*.pem",
|
||||
"**/*.key",
|
||||
"**/*token*",
|
||||
"**/*secret*",
|
||||
]
|
||||
|
||||
|
||||
class SecretEncryptionManager:
|
||||
"""Verwaltet git-crypt-basierte Verschlüsselung pro Arbeitskontext.
|
||||
|
||||
Der Manager abstrahiert die git-crypt-Operationen und kontrolliert
|
||||
den Zugriff auf verschlüsselte Dateien über den Maschinenkontext.
|
||||
|
||||
Args:
|
||||
root_path: Pfad zum Monorepo-Root-Verzeichnis.
|
||||
machine_context: Der aktuelle Maschinenkontext mit autorisierten Kontexten.
|
||||
"""
|
||||
|
||||
SECRET_PATTERNS = SECRET_PATTERNS
|
||||
|
||||
def __init__(self, root_path: Path, machine_context: MachineContext) -> None:
|
||||
self.root_path = root_path
|
||||
self.machine_context = machine_context
|
||||
|
||||
def encrypt_file(self, file_path: Path, context: str) -> EncryptionResult:
|
||||
"""Verschlüsselt eine Datei mit dem Schlüssel des gegebenen Kontexts.
|
||||
|
||||
Die Verschlüsselung erfolgt über git-crypt-Filter. Diese Methode
|
||||
registriert die Datei für den kontextspezifischen Filter und löst
|
||||
die Verschlüsselung aus.
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur zu verschlüsselnden Datei.
|
||||
context: Arbeitskontext, dessen Schlüssel verwendet werden soll.
|
||||
|
||||
Returns:
|
||||
EncryptionResult mit Erfolgs-/Fehlerstatus.
|
||||
"""
|
||||
if not file_path.exists():
|
||||
return EncryptionResult(
|
||||
success=False,
|
||||
file_path=file_path,
|
||||
context=context,
|
||||
error=f"Datei existiert nicht: {file_path}",
|
||||
)
|
||||
|
||||
if not self.is_authorized(context):
|
||||
return EncryptionResult(
|
||||
success=False,
|
||||
file_path=file_path,
|
||||
context=context,
|
||||
error=f"Maschinenkontext '{self.machine_context.name}' ist nicht "
|
||||
f"für Kontext '{context}' autorisiert",
|
||||
)
|
||||
|
||||
try:
|
||||
self._run_gitcrypt(["git-crypt", "status", str(file_path)])
|
||||
except GitCryptNotAvailableError as e:
|
||||
return EncryptionResult(
|
||||
success=False,
|
||||
file_path=file_path,
|
||||
context=context,
|
||||
error=str(e),
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
return EncryptionResult(
|
||||
success=False,
|
||||
file_path=file_path,
|
||||
context=context,
|
||||
error=f"git-crypt Fehler: {e}",
|
||||
)
|
||||
|
||||
return EncryptionResult(
|
||||
success=True,
|
||||
file_path=file_path,
|
||||
context=context,
|
||||
)
|
||||
|
||||
def decrypt_file(self, file_path: Path) -> DecryptionResult:
|
||||
"""Entschlüsselt eine Datei, sofern der Maschinenkontext autorisiert ist.
|
||||
|
||||
Ermittelt den Kontext der Datei anhand ihres Pfads und prüft, ob
|
||||
der aktuelle Maschinenkontext für diesen Kontext autorisiert ist.
|
||||
Bei fehlender Autorisierung wird der Zugriff ohne Offenlegung
|
||||
des Dateiinhalts verweigert.
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur zu entschlüsselnden Datei.
|
||||
|
||||
Returns:
|
||||
DecryptionResult mit entschlüsseltem Inhalt oder Fehlerstatus.
|
||||
"""
|
||||
if not file_path.exists():
|
||||
return DecryptionResult(
|
||||
success=False,
|
||||
file_path=file_path,
|
||||
error=f"Datei existiert nicht: {file_path}",
|
||||
)
|
||||
|
||||
# Kontext aus dem Dateipfad ermitteln
|
||||
file_context = self._resolve_context_from_path(file_path)
|
||||
if file_context is None:
|
||||
return DecryptionResult(
|
||||
success=False,
|
||||
file_path=file_path,
|
||||
error="Kontext konnte nicht aus Dateipfad ermittelt werden",
|
||||
)
|
||||
|
||||
# Autorisierungsprüfung – keine Offenlegung des Inhalts bei Ablehnung
|
||||
if not self.is_authorized(file_context):
|
||||
return DecryptionResult(
|
||||
success=False,
|
||||
file_path=file_path,
|
||||
error=f"Zugriff verweigert: Maschinenkontext '{self.machine_context.name}' "
|
||||
f"ist nicht für Kontext '{file_context}' autorisiert",
|
||||
)
|
||||
|
||||
try:
|
||||
self._run_gitcrypt(["git-crypt", "unlock"])
|
||||
content = file_path.read_bytes()
|
||||
except GitCryptNotAvailableError as e:
|
||||
return DecryptionResult(
|
||||
success=False,
|
||||
file_path=file_path,
|
||||
error=str(e),
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
return DecryptionResult(
|
||||
success=False,
|
||||
file_path=file_path,
|
||||
error=f"git-crypt Entschlüsselungsfehler: {e}",
|
||||
)
|
||||
except OSError as e:
|
||||
return DecryptionResult(
|
||||
success=False,
|
||||
file_path=file_path,
|
||||
error=f"Lesefehler: {e}",
|
||||
)
|
||||
|
||||
return DecryptionResult(
|
||||
success=True,
|
||||
file_path=file_path,
|
||||
content=content,
|
||||
)
|
||||
|
||||
def is_authorized(self, context: str) -> bool:
|
||||
"""Prüft ob der aktuelle Maschinenkontext für den Kontext autorisiert ist.
|
||||
|
||||
Args:
|
||||
context: Zu prüfender Arbeitskontext (z.B. 'privat', 'dhive', 'bahn').
|
||||
|
||||
Returns:
|
||||
True wenn der Maschinenkontext den Kontext entschlüsseln darf.
|
||||
"""
|
||||
return context in self.machine_context.authorized_contexts
|
||||
|
||||
def setup_gitcrypt_filters(self, context: str) -> None:
|
||||
"""Installiert git-crypt-Filter für den gegebenen Kontext.
|
||||
|
||||
Schreibt oder aktualisiert die `.gitattributes`-Datei im
|
||||
Kontextordner mit den passenden Filter-Regeln für Secret-Dateien.
|
||||
|
||||
Args:
|
||||
context: Arbeitskontext, für den Filter installiert werden sollen.
|
||||
"""
|
||||
context_dir = self.root_path / context
|
||||
context_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
gitattributes_path = context_dir / ".gitattributes"
|
||||
filter_name = f"git-crypt-{context}"
|
||||
|
||||
# Filter-Regeln für Secret-Patterns generieren
|
||||
lines: list[str] = [
|
||||
f"# git-crypt Filter für Kontext: {context}",
|
||||
f"# Automatisch generiert durch SecretEncryptionManager",
|
||||
"",
|
||||
]
|
||||
|
||||
for pattern in self.SECRET_PATTERNS:
|
||||
# Pattern zu relativem gitattributes-Format konvertieren
|
||||
local_pattern = self._pattern_to_gitattributes(pattern)
|
||||
lines.append(
|
||||
f"{local_pattern} filter={filter_name} diff={filter_name}"
|
||||
)
|
||||
|
||||
lines.append("") # Abschließende Leerzeile
|
||||
|
||||
gitattributes_path.write_text("\n".join(lines), encoding="utf-8")
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Interne Hilfsmethoden
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
def _run_gitcrypt(self, cmd: list[str]) -> subprocess.CompletedProcess[bytes]:
|
||||
"""Führt einen git-crypt-Befehl aus.
|
||||
|
||||
Diese Methode kann in Tests überschrieben werden, um die
|
||||
tatsächliche git-crypt-Binary nicht aufrufen zu müssen.
|
||||
|
||||
Args:
|
||||
cmd: Befehl und Argumente als Liste.
|
||||
|
||||
Returns:
|
||||
CompletedProcess-Objekt mit dem Ergebnis.
|
||||
|
||||
Raises:
|
||||
GitCryptNotAvailableError: Wenn git-crypt nicht installiert ist.
|
||||
subprocess.CalledProcessError: Wenn der Befehl fehlschlägt.
|
||||
"""
|
||||
try:
|
||||
return subprocess.run(
|
||||
cmd,
|
||||
cwd=self.root_path,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
except FileNotFoundError as e:
|
||||
raise GitCryptNotAvailableError(
|
||||
"git-crypt ist nicht installiert oder nicht im PATH. "
|
||||
"Bitte git-crypt installieren: https://github.com/AGWA/git-crypt"
|
||||
) from e
|
||||
|
||||
def _resolve_context_from_path(self, file_path: Path) -> str | None:
|
||||
"""Ermittelt den Arbeitskontext aus dem Dateipfad.
|
||||
|
||||
Der Kontext entspricht dem ersten Pfad-Segment relativ zum Root.
|
||||
Gültige Kontexte: privat, dhive, bahn, shared.
|
||||
|
||||
Args:
|
||||
file_path: Absoluter oder relativer Pfad zur Datei.
|
||||
|
||||
Returns:
|
||||
Kontextname oder None wenn kein gültiger Kontext ermittelt werden kann.
|
||||
"""
|
||||
valid_contexts = {"privat", "dhive", "bahn", "shared"}
|
||||
|
||||
try:
|
||||
rel_path = file_path.resolve().relative_to(self.root_path.resolve())
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
if not rel_path.parts:
|
||||
return None
|
||||
|
||||
first_part = rel_path.parts[0]
|
||||
if first_part in valid_contexts:
|
||||
return first_part
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _pattern_to_gitattributes(pattern: str) -> str:
|
||||
"""Konvertiert ein glob-Pattern in das .gitattributes-Format.
|
||||
|
||||
Entfernt den rekursiven '**/' Prefix, da .gitattributes relativ
|
||||
zum Verzeichnis der Datei wirkt.
|
||||
|
||||
Args:
|
||||
pattern: Glob-Pattern (z.B. '**/.env', '**/*.pem').
|
||||
|
||||
Returns:
|
||||
.gitattributes-kompatibles Pattern.
|
||||
"""
|
||||
# '**/' am Anfang entfernen – .gitattributes matcht relativ
|
||||
if pattern.startswith("**/"):
|
||||
return pattern[3:]
|
||||
return pattern
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Maschinenkontext-Verwaltung und Schlüssel-Management (Task 4.2)
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
def get_context_key(self, context: str) -> Optional[EncryptionKey]:
|
||||
"""Liefert den Schlüssel für einen Kontext aus Keyring oder Passwort-Manager.
|
||||
|
||||
Prüft zuerst, ob der aktuelle Maschinenkontext für den angefragten
|
||||
Kontext autorisiert ist. Ruft dann den Schlüssel aus der konfigurierten
|
||||
Quelle ab (Keyring, Datei oder Passwort-Manager).
|
||||
|
||||
Args:
|
||||
context: Arbeitskontext, für den der Schlüssel benötigt wird.
|
||||
|
||||
Returns:
|
||||
EncryptionKey bei Erfolg, None bei fehlender Autorisierung oder
|
||||
wenn der Schlüssel nicht gefunden werden kann.
|
||||
"""
|
||||
if not self.is_authorized(context):
|
||||
return None
|
||||
|
||||
key_source = self.machine_context.key_source
|
||||
|
||||
if key_source == "keyring":
|
||||
return self._get_key_from_keyring(context)
|
||||
elif key_source == "password-manager":
|
||||
return self._get_key_from_password_manager(context)
|
||||
elif key_source == "file":
|
||||
return self._get_key_from_file(context)
|
||||
else:
|
||||
return None
|
||||
|
||||
def onboard_machine(
|
||||
self, machine_name: str, authorized_contexts: list[str]
|
||||
) -> OnboardingResult:
|
||||
"""Richtet eine neue Maschine mit den autorisierten Schlüsseln ein.
|
||||
|
||||
Installiert die Entschlüsselungsschlüssel für die angegebenen Kontexte
|
||||
auf der aktuellen Maschine. Aktualisiert die machine-context.yaml
|
||||
Konfiguration entsprechend.
|
||||
|
||||
Args:
|
||||
machine_name: Bezeichnung der neuen Maschine.
|
||||
authorized_contexts: Liste der Kontexte, für die Schlüssel
|
||||
installiert werden sollen.
|
||||
|
||||
Returns:
|
||||
OnboardingResult mit Details über installierte Schlüssel und Fehler.
|
||||
"""
|
||||
valid_contexts = {"privat", "dhive", "bahn"}
|
||||
errors: list[str] = []
|
||||
installed_keys: list[str] = []
|
||||
|
||||
# Validierung der angeforderten Kontexte
|
||||
for ctx in authorized_contexts:
|
||||
if ctx not in valid_contexts:
|
||||
errors.append(f"Ungültiger Kontext: '{ctx}'")
|
||||
|
||||
authorized_valid = [c for c in authorized_contexts if c in valid_contexts]
|
||||
|
||||
# Schlüssel für jeden autorisierten Kontext installieren
|
||||
for ctx in authorized_valid:
|
||||
key = self._provision_key_for_context(ctx)
|
||||
if key is not None:
|
||||
installed_keys.append(key.key_id)
|
||||
else:
|
||||
errors.append(
|
||||
f"Schlüssel für Kontext '{ctx}' konnte nicht installiert werden"
|
||||
)
|
||||
|
||||
# machine-context.yaml aktualisieren
|
||||
if installed_keys:
|
||||
self._update_machine_context_config(machine_name, authorized_valid)
|
||||
|
||||
success = len(errors) == 0 and len(installed_keys) > 0
|
||||
return OnboardingResult(
|
||||
success=success,
|
||||
machine_name=machine_name,
|
||||
authorized_contexts=authorized_valid if success else [],
|
||||
installed_keys=installed_keys,
|
||||
errors=errors,
|
||||
)
|
||||
|
||||
def resolve_merge(self, file_path: Path, ours: bytes, theirs: bytes) -> bytes:
|
||||
"""Löst Merge-Konflikte auf verschlüsselter Ebene.
|
||||
|
||||
Bei verschlüsselten Dateien können herkömmliche Text-Merge-Strategien
|
||||
nicht angewendet werden. Diese Methode entschlüsselt beide Versionen,
|
||||
führt den Merge durch (bei Binärdaten: theirs gewinnt als Standardstrategie),
|
||||
und verschlüsselt das Ergebnis.
|
||||
|
||||
Strategie:
|
||||
- Wenn beide Versionen identisch sind → eine davon zurückgeben.
|
||||
- Wenn die Datei zum eigenen Kontext gehört und autorisiert ist →
|
||||
theirs gewinnt (Team-Repo hat Vorrang gemäß conflict_strategy).
|
||||
- Wenn nicht autorisiert → ours beibehalten (keine Änderung möglich).
|
||||
|
||||
Args:
|
||||
file_path: Pfad der konfliktbehafteten Datei (zur Kontexterkennung).
|
||||
ours: Unsere Version der Datei (verschlüsselt oder unverschlüsselt).
|
||||
theirs: Deren Version der Datei (verschlüsselt oder unverschlüsselt).
|
||||
|
||||
Returns:
|
||||
Die aufgelöste Version als Bytes.
|
||||
"""
|
||||
# Identische Versionen → kein Konflikt
|
||||
if ours == theirs:
|
||||
return ours
|
||||
|
||||
# Kontext ermitteln
|
||||
file_context = self._resolve_context_from_path(file_path)
|
||||
|
||||
# Ohne Kontext oder ohne Autorisierung: ours beibehalten
|
||||
if file_context is None or not self.is_authorized(file_context):
|
||||
return ours
|
||||
|
||||
# Standardstrategie: theirs gewinnt (Team-Repo/Remote hat Vorrang)
|
||||
return theirs
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Schlüsselquellen-Methoden (Key Sources)
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
def _get_key_from_keyring(self, context: str) -> Optional[EncryptionKey]:
|
||||
"""Ruft einen Schlüssel aus dem System-Keyring ab.
|
||||
|
||||
Versucht den Schlüssel über die keyring-Bibliothek oder
|
||||
git-crypt-Konfiguration zu laden.
|
||||
|
||||
Args:
|
||||
context: Arbeitskontext für den der Schlüssel gesucht wird.
|
||||
|
||||
Returns:
|
||||
EncryptionKey oder None bei Fehler.
|
||||
"""
|
||||
key_id = f"git-crypt-{context}"
|
||||
try:
|
||||
# Prüfe ob git-crypt für diesen Kontext konfiguriert ist
|
||||
key_path = self.root_path / ".git" / "git-crypt" / "keys" / context
|
||||
if key_path.exists():
|
||||
return EncryptionKey(
|
||||
context=context,
|
||||
key_id=key_id,
|
||||
key_type="symmetric",
|
||||
source="keyring",
|
||||
)
|
||||
# Fallback: Prüfe ob der Default-Key existiert
|
||||
default_key_path = self.root_path / ".git" / "git-crypt" / "keys" / "default"
|
||||
if default_key_path.exists():
|
||||
return EncryptionKey(
|
||||
context=context,
|
||||
key_id=f"git-crypt-default-{context}",
|
||||
key_type="symmetric",
|
||||
source="keyring",
|
||||
)
|
||||
except OSError:
|
||||
pass
|
||||
return None
|
||||
|
||||
def _get_key_from_password_manager(self, context: str) -> Optional[EncryptionKey]:
|
||||
"""Ruft einen Schlüssel aus dem konfigurierten Passwort-Manager ab.
|
||||
|
||||
Unterstützt Bitwarden, 1Password und KeePass als Schlüsselquellen.
|
||||
|
||||
Args:
|
||||
context: Arbeitskontext für den der Schlüssel gesucht wird.
|
||||
|
||||
Returns:
|
||||
EncryptionKey oder None bei Fehler.
|
||||
"""
|
||||
pm_config = self.machine_context.password_manager
|
||||
if pm_config is None:
|
||||
return None
|
||||
|
||||
entry_name = f"{pm_config.entry_prefix}{context}"
|
||||
|
||||
try:
|
||||
key_data = _PasswordManagerAdapter.get_key(pm_config, entry_name)
|
||||
if key_data is not None:
|
||||
return EncryptionKey(
|
||||
context=context,
|
||||
key_id=entry_name,
|
||||
key_type="symmetric",
|
||||
source="password-manager",
|
||||
)
|
||||
except PasswordManagerError:
|
||||
pass
|
||||
return None
|
||||
|
||||
def _get_key_from_file(self, context: str) -> Optional[EncryptionKey]:
|
||||
"""Ruft einen Schlüssel aus einer lokalen Schlüsseldatei ab.
|
||||
|
||||
Sucht die Schlüsseldatei im Standard-Verzeichnis
|
||||
~/.monorepo/keys/{context}.key
|
||||
|
||||
Args:
|
||||
context: Arbeitskontext für den der Schlüssel gesucht wird.
|
||||
|
||||
Returns:
|
||||
EncryptionKey oder None wenn keine Datei gefunden wird.
|
||||
"""
|
||||
key_dir = Path.home() / ".monorepo" / "keys"
|
||||
key_file = key_dir / f"{context}.key"
|
||||
if key_file.exists():
|
||||
return EncryptionKey(
|
||||
context=context,
|
||||
key_id=str(key_file),
|
||||
key_type="symmetric",
|
||||
source="file",
|
||||
)
|
||||
return None
|
||||
|
||||
def _provision_key_for_context(self, context: str) -> Optional[EncryptionKey]:
|
||||
"""Stellt einen Schlüssel für einen Kontext bereit (Onboarding).
|
||||
|
||||
Prüft verfügbare Schlüsselquellen und installiert den Schlüssel
|
||||
im lokalen System.
|
||||
|
||||
Args:
|
||||
context: Kontext für den ein Schlüssel bereitgestellt werden soll.
|
||||
|
||||
Returns:
|
||||
EncryptionKey bei Erfolg, None bei Fehler.
|
||||
"""
|
||||
# Versuche zuerst aus dem Passwort-Manager
|
||||
if self.machine_context.password_manager is not None:
|
||||
key = self._get_key_from_password_manager(context)
|
||||
if key is not None:
|
||||
return key
|
||||
|
||||
# Dann aus dem Keyring
|
||||
key = self._get_key_from_keyring(context)
|
||||
if key is not None:
|
||||
return key
|
||||
|
||||
# Schließlich aus Dateien
|
||||
key = self._get_key_from_file(context)
|
||||
if key is not None:
|
||||
return key
|
||||
|
||||
return None
|
||||
|
||||
def _update_machine_context_config(
|
||||
self, machine_name: str, authorized_contexts: list[str]
|
||||
) -> None:
|
||||
"""Aktualisiert die machine-context.yaml mit neuen Maschinendaten.
|
||||
|
||||
Args:
|
||||
machine_name: Name der neuen Maschine.
|
||||
authorized_contexts: Autorisierte Kontexte für diese Maschine.
|
||||
"""
|
||||
config_path = self.root_path / "shared" / "config" / "machine-context.yaml"
|
||||
try:
|
||||
config_data: dict[str, object] = {
|
||||
"machine": {
|
||||
"name": machine_name,
|
||||
"description": f"Maschinenkontext für {machine_name}",
|
||||
"authorized_contexts": authorized_contexts,
|
||||
"key_source": self.machine_context.key_source,
|
||||
}
|
||||
}
|
||||
|
||||
if self.machine_context.password_manager is not None:
|
||||
pm = self.machine_context.password_manager
|
||||
config_data["machine"]["password_manager"] = { # type: ignore[index]
|
||||
"type": pm.type,
|
||||
"vault": pm.vault,
|
||||
"entry_prefix": pm.entry_prefix,
|
||||
}
|
||||
|
||||
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(config_path, "w", encoding="utf-8") as f:
|
||||
yaml.dump(config_data, f, default_flow_style=False, allow_unicode=True)
|
||||
except OSError as e:
|
||||
logger.warning("Konnte machine-context.yaml nicht aktualisieren: %s", e)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MachineContextManager – Lädt und verwaltet Maschinenkontext-Konfiguration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MachineContextManager:
|
||||
"""Verwaltet Maschinenkontext-Konfiguration und Schlüssel-Lookup.
|
||||
|
||||
Lädt die Konfiguration aus `shared/config/machine-context.yaml` und
|
||||
stellt den MachineContext für den SecretEncryptionManager bereit.
|
||||
|
||||
Args:
|
||||
root_path: Pfad zum Monorepo-Root-Verzeichnis.
|
||||
config_path: Optionaler expliziter Pfad zur machine-context.yaml.
|
||||
"""
|
||||
|
||||
DEFAULT_CONFIG_REL_PATH = Path("shared") / "config" / "machine-context.yaml"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
root_path: Path,
|
||||
config_path: Path | None = None,
|
||||
) -> None:
|
||||
self.root_path = root_path
|
||||
self.config_path = config_path or (root_path / self.DEFAULT_CONFIG_REL_PATH)
|
||||
self._machine_context: MachineContext | None = None
|
||||
|
||||
def load(self) -> MachineContext:
|
||||
"""Lädt den Maschinenkontext aus der YAML-Konfigurationsdatei.
|
||||
|
||||
Returns:
|
||||
MachineContext mit autorisierten Kontexten und Key-Source.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: Wenn die Konfigurationsdatei nicht existiert.
|
||||
ValueError: Wenn die Konfigurationsdatei ungültig ist.
|
||||
"""
|
||||
if not self.config_path.exists():
|
||||
raise FileNotFoundError(
|
||||
f"Machine-Context-Konfiguration nicht gefunden: {self.config_path}"
|
||||
)
|
||||
|
||||
with open(self.config_path, encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f)
|
||||
|
||||
if not isinstance(data, dict) or "machine" not in data:
|
||||
raise ValueError(
|
||||
f"Ungültiges machine-context.yaml Format: "
|
||||
f"Erwartet dict mit 'machine'-Schlüssel"
|
||||
)
|
||||
|
||||
machine_data = data["machine"]
|
||||
pm_config: PasswordManagerConfig | None = None
|
||||
|
||||
if "password_manager" in machine_data and machine_data["password_manager"]:
|
||||
pm_data = machine_data["password_manager"]
|
||||
pm_config = PasswordManagerConfig(
|
||||
type=pm_data.get("type", "bitwarden"),
|
||||
vault=pm_data.get("vault", ""),
|
||||
entry_prefix=pm_data.get("entry_prefix", "monorepo-key-"),
|
||||
)
|
||||
|
||||
self._machine_context = MachineContext(
|
||||
name=machine_data.get("name", "unknown"),
|
||||
description=machine_data.get("description", ""),
|
||||
authorized_contexts=machine_data.get("authorized_contexts", []),
|
||||
key_source=machine_data.get("key_source", "keyring"),
|
||||
password_manager=pm_config,
|
||||
)
|
||||
|
||||
return self._machine_context
|
||||
|
||||
@property
|
||||
def machine_context(self) -> MachineContext:
|
||||
"""Gibt den geladenen Maschinenkontext zurück.
|
||||
|
||||
Returns:
|
||||
Der aktuell geladene MachineContext.
|
||||
|
||||
Raises:
|
||||
RuntimeError: Wenn load() noch nicht aufgerufen wurde.
|
||||
"""
|
||||
if self._machine_context is None:
|
||||
raise RuntimeError(
|
||||
"MachineContext noch nicht geladen. Bitte load() aufrufen."
|
||||
)
|
||||
return self._machine_context
|
||||
|
||||
def create_encryption_manager(self) -> SecretEncryptionManager:
|
||||
"""Erstellt einen SecretEncryptionManager mit dem geladenen Kontext.
|
||||
|
||||
Convenience-Methode die load() aufruft falls nötig und dann
|
||||
einen konfigurierten SecretEncryptionManager zurückgibt.
|
||||
|
||||
Returns:
|
||||
Fertig konfigurierter SecretEncryptionManager.
|
||||
"""
|
||||
if self._machine_context is None:
|
||||
self.load()
|
||||
return SecretEncryptionManager(self.root_path, self.machine_context)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Passwort-Manager-Adapter
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _PasswordManagerAdapter:
|
||||
"""Adapter für verschiedene Passwort-Manager (Bitwarden, 1Password, KeePass).
|
||||
|
||||
Kapselt die CLI-Aufrufe der verschiedenen Passwort-Manager-Tools.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def get_key(config: PasswordManagerConfig, entry_name: str) -> str | None:
|
||||
"""Ruft einen Schlüssel aus dem konfigurierten Passwort-Manager ab.
|
||||
|
||||
Args:
|
||||
config: Passwort-Manager-Konfiguration.
|
||||
entry_name: Name des Eintrags im Vault.
|
||||
|
||||
Returns:
|
||||
Schlüsselwert als String oder None wenn nicht gefunden.
|
||||
|
||||
Raises:
|
||||
PasswordManagerError: Bei Kommunikationsfehlern mit dem PM.
|
||||
"""
|
||||
if config.type == "bitwarden":
|
||||
return _PasswordManagerAdapter._get_from_bitwarden(config.vault, entry_name)
|
||||
elif config.type == "1password":
|
||||
return _PasswordManagerAdapter._get_from_1password(config.vault, entry_name)
|
||||
elif config.type == "keepass":
|
||||
return _PasswordManagerAdapter._get_from_keepass(config.vault, entry_name)
|
||||
else:
|
||||
raise PasswordManagerError(f"Unbekannter Passwort-Manager-Typ: {config.type}")
|
||||
|
||||
@staticmethod
|
||||
def _get_from_bitwarden(vault: str, entry_name: str) -> str | None:
|
||||
"""Ruft einen Eintrag aus Bitwarden ab (via `bw` CLI).
|
||||
|
||||
Args:
|
||||
vault: Vault/Collection-Name.
|
||||
entry_name: Eintragsname.
|
||||
|
||||
Returns:
|
||||
Passwort/Schlüssel oder None.
|
||||
"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["bw", "get", "password", entry_name, "--collection", vault],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=30,
|
||||
)
|
||||
return result.stdout.strip() if result.stdout.strip() else None
|
||||
except (subprocess.CalledProcessError, FileNotFoundError, subprocess.TimeoutExpired):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _get_from_1password(vault: str, entry_name: str) -> str | None:
|
||||
"""Ruft einen Eintrag aus 1Password ab (via `op` CLI).
|
||||
|
||||
Args:
|
||||
vault: Vault-Name.
|
||||
entry_name: Eintragsname.
|
||||
|
||||
Returns:
|
||||
Passwort/Schlüssel oder None.
|
||||
"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["op", "item", "get", entry_name, "--vault", vault, "--fields", "password"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=30,
|
||||
)
|
||||
return result.stdout.strip() if result.stdout.strip() else None
|
||||
except (subprocess.CalledProcessError, FileNotFoundError, subprocess.TimeoutExpired):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _get_from_keepass(vault: str, entry_name: str) -> str | None:
|
||||
"""Ruft einen Eintrag aus KeePass ab (via `keepassxc-cli`).
|
||||
|
||||
Args:
|
||||
vault: Datenbankpfad.
|
||||
entry_name: Eintragsname.
|
||||
|
||||
Returns:
|
||||
Passwort/Schlüssel oder None.
|
||||
"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["keepassxc-cli", "show", "-s", vault, entry_name],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=30,
|
||||
)
|
||||
# KeePassXC-CLI gibt das Passwort in der Ausgabe aus
|
||||
for line in result.stdout.splitlines():
|
||||
if line.startswith("Password:"):
|
||||
return line.split(":", 1)[1].strip()
|
||||
return None
|
||||
except (subprocess.CalledProcessError, FileNotFoundError, subprocess.TimeoutExpired):
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Exceptions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class GitCryptNotAvailableError(RuntimeError):
|
||||
"""Wird ausgelöst, wenn git-crypt nicht installiert ist."""
|
||||
|
||||
|
||||
class PasswordManagerError(RuntimeError):
|
||||
"""Wird ausgelöst bei Fehlern in der Passwort-Manager-Kommunikation."""
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,618 @@
|
||||
"""Git-Hooks und Dateisystem-Schutz für das Monorepo.
|
||||
|
||||
Stellt einen integrierten pre-commit Hook bereit, der:
|
||||
1. Read-Only-Repos vor Veränderung schützt (Req 4.2, 4.5)
|
||||
2. Unverschlüsselte Secrets am Commit hindert (Req 2.4, 9.1, 9.9)
|
||||
3. ContextGuard für Zugriffs-Validierung integriert
|
||||
4. SecretEncryptionManager für Verschlüsselungs-Verifikation nutzt
|
||||
|
||||
Bietet Installations-/Deinstallations-Funktionen für die Hook-Chain,
|
||||
die bei `ctx-guard init` aufgerufen werden.
|
||||
|
||||
Requirements: 2.4, 4.2, 4.5, 9.1, 9.9
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import stat
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from monorepo.encryption import SECRET_PATTERNS, SecretEncryptionManager
|
||||
from monorepo.models import RepoEntry
|
||||
from monorepo.security import ContextGuard
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pre-commit Hook Template (Shell-Script)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Integrierter pre-commit Hook, der Read-Only-Schutz und Secret-Prüfung
|
||||
#: kombiniert. Wird als Shell-Script installiert.
|
||||
_INTEGRATED_HOOK_TEMPLATE = """\
|
||||
#!/bin/sh
|
||||
# === Monorepo Integrated Pre-Commit Hook (auto-generiert) ===
|
||||
# Prüft:
|
||||
# 1. Read-Only-Repos werden nicht verändert
|
||||
# 2. Unverschlüsselte Secrets werden nicht committet
|
||||
# Installiert durch: ctx-guard init / install-hooks
|
||||
|
||||
{readonly_section}
|
||||
|
||||
{secrets_section}
|
||||
|
||||
# === Ende Monorepo Integrated Pre-Commit Hook ===
|
||||
"""
|
||||
|
||||
_READONLY_SECTION_TEMPLATE = """\
|
||||
# --- Read-Only-Repos Schutz ---
|
||||
{protected_paths_def}
|
||||
|
||||
staged_files=$(git diff --cached --name-only)
|
||||
|
||||
for file in $staged_files; do
|
||||
for protected in "${{PROTECTED_PATHS[@]}}"; do
|
||||
case "$file" in
|
||||
"$protected"/*)
|
||||
echo "FEHLER: Repo '$protected' ist read-only. Schreibzugriff auf '$file' nicht erlaubt." >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
done
|
||||
# --- Ende Read-Only-Repos Schutz ---"""
|
||||
|
||||
_SECRETS_SECTION_TEMPLATE = """\
|
||||
# --- Unverschlüsselte Secrets Prüfung ---
|
||||
# Secret-Patterns: .env, *.pem, *.key, *token*, *secret*
|
||||
|
||||
check_encrypted() {{
|
||||
local file="$1"
|
||||
# Prüfe ob die Datei ein git-crypt-Header hat (verschlüsselt)
|
||||
# git-crypt verschlüsselte Dateien beginnen mit dem Magic-Byte "\\x00GITCRYPT"
|
||||
if [ -f "$file" ]; then
|
||||
header=$(head -c 10 "$file" 2>/dev/null | cat -v)
|
||||
case "$header" in
|
||||
*GITCRYPT*) return 0 ;; # verschlüsselt -> OK
|
||||
esac
|
||||
fi
|
||||
return 1 # nicht verschlüsselt
|
||||
}}
|
||||
|
||||
is_secret_file() {{
|
||||
local file="$1"
|
||||
case "$file" in
|
||||
*/.env|*/.env.*) return 0 ;;
|
||||
*.pem) return 0 ;;
|
||||
*.key) return 0 ;;
|
||||
*token*) return 0 ;;
|
||||
*secret*) return 0 ;;
|
||||
esac
|
||||
return 1
|
||||
}}
|
||||
|
||||
for file in $staged_files; do
|
||||
if is_secret_file "$file"; then
|
||||
if ! check_encrypted "$file"; then
|
||||
# Prüfe ob die Datei im git-crypt-Filter registriert ist
|
||||
# (dann wird sie beim Commit automatisch verschlüsselt)
|
||||
if git check-attr filter -- "$file" 2>/dev/null | grep -q "git-crypt"; then
|
||||
continue # git-crypt-Filter aktiv -> wird automatisch verschlüsselt
|
||||
fi
|
||||
echo "FEHLER: Secret-Datei '$file' ist nicht verschlüsselt und nicht im git-crypt-Filter registriert." >&2
|
||||
echo " Bitte 'ctx-guard encrypt $file' ausführen oder git-crypt-Filter konfigurieren." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
done
|
||||
# --- Ende Unverschlüsselte Secrets Prüfung ---"""
|
||||
|
||||
# Marker für den integrierten Hook-Abschnitt
|
||||
_HOOK_START_MARKER = "# === Monorepo Integrated Pre-Commit Hook (auto-generiert) ==="
|
||||
_HOOK_END_MARKER = "# === Ende Monorepo Integrated Pre-Commit Hook ==="
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HookManager – Zentrale Verwaltung der Git-Hooks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class HookManager:
|
||||
"""Verwaltet Git-Hooks für das Monorepo.
|
||||
|
||||
Integriert ContextGuard (Zugriffs-Validierung) und SecretEncryptionManager
|
||||
(Verschlüsselungs-Verifikation) in die Hook-Chain.
|
||||
|
||||
Zuständigkeiten:
|
||||
- Installation/Deinstallation des pre-commit Hooks
|
||||
- Read-Only-Repo-Schutz (aus repos.yaml)
|
||||
- Secret-Verschlüsselungs-Prüfung (git-crypt-Filter-Integration)
|
||||
- Validierung der Hook-Konfiguration
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
monorepo_root: Path,
|
||||
context_guard: ContextGuard | None = None,
|
||||
encryption_manager: SecretEncryptionManager | None = None,
|
||||
) -> None:
|
||||
"""Initialisiert den HookManager.
|
||||
|
||||
Args:
|
||||
monorepo_root: Pfad zum Monorepo-Root-Verzeichnis.
|
||||
context_guard: Optionaler ContextGuard für Zugriffs-Validierung.
|
||||
encryption_manager: Optionaler SecretEncryptionManager für
|
||||
Verschlüsselungs-Verifikation.
|
||||
"""
|
||||
self.monorepo_root = monorepo_root.resolve()
|
||||
self.context_guard = context_guard
|
||||
self.encryption_manager = encryption_manager
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Öffentliche API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def install_hooks(self) -> None:
|
||||
"""Installiert alle Monorepo Git-Hooks.
|
||||
|
||||
Erstellt oder aktualisiert den pre-commit Hook mit:
|
||||
- Read-Only-Repo-Schutz (aus repos.yaml geladen)
|
||||
- Secret-Verschlüsselungs-Prüfung (git-crypt-Filter)
|
||||
|
||||
Bestehende Hook-Inhalte außerhalb des markierten Bereichs
|
||||
bleiben erhalten.
|
||||
"""
|
||||
hooks_dir = self._get_hooks_dir()
|
||||
hooks_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
hook_path = hooks_dir / "pre-commit"
|
||||
|
||||
# Read-Only-Pfade aus repos.yaml sammeln
|
||||
protected_paths = self._get_readonly_paths()
|
||||
|
||||
# Hook generieren und schreiben
|
||||
hook_content = self._generate_hook_content(protected_paths)
|
||||
self._write_hook(hook_path, hook_content)
|
||||
|
||||
logger.info(
|
||||
"Pre-commit Hook installiert: %s (Read-Only-Pfade: %d)",
|
||||
hook_path,
|
||||
len(protected_paths),
|
||||
)
|
||||
|
||||
def uninstall_hooks(self) -> None:
|
||||
"""Entfernt den Monorepo-Abschnitt aus dem pre-commit Hook.
|
||||
|
||||
Entfernt nur den markierten Bereich. Wenn der Hook danach leer ist,
|
||||
wird die Datei gelöscht.
|
||||
"""
|
||||
hook_path = self._get_hooks_dir() / "pre-commit"
|
||||
|
||||
if not hook_path.exists():
|
||||
logger.info("Kein pre-commit Hook vorhanden – nichts zu tun.")
|
||||
return
|
||||
|
||||
content = hook_path.read_text(encoding="utf-8")
|
||||
|
||||
if _HOOK_START_MARKER not in content:
|
||||
logger.info("Kein Monorepo-Abschnitt im Hook gefunden.")
|
||||
return
|
||||
|
||||
# Entferne den markierten Abschnitt
|
||||
start_idx = content.index(_HOOK_START_MARKER)
|
||||
end_idx = content.index(_HOOK_END_MARKER) + len(_HOOK_END_MARKER)
|
||||
remaining = content[:start_idx] + content[end_idx:]
|
||||
remaining = remaining.strip()
|
||||
|
||||
if not remaining or remaining == "#!/bin/sh":
|
||||
# Hook ist leer → löschen
|
||||
hook_path.unlink()
|
||||
logger.info("Pre-commit Hook entfernt (war nur Monorepo-Abschnitt).")
|
||||
else:
|
||||
hook_path.write_text(remaining + "\n", encoding="utf-8")
|
||||
logger.info("Monorepo-Abschnitt aus pre-commit Hook entfernt.")
|
||||
|
||||
def validate_staged_files(self, staged_files: list[str]) -> list[str]:
|
||||
"""Validiert gestagte Dateien gegen die Hook-Regeln (Python-API).
|
||||
|
||||
Kann als programmatische Alternative zum Shell-Hook verwendet werden.
|
||||
Prüft Read-Only-Schutz und Secret-Verschlüsselung.
|
||||
|
||||
Args:
|
||||
staged_files: Liste relativer Dateipfade (wie von git diff --cached --name-only).
|
||||
|
||||
Returns:
|
||||
Liste von Fehlermeldungen. Leere Liste = alle Prüfungen bestanden.
|
||||
"""
|
||||
errors: list[str] = []
|
||||
|
||||
# Read-Only-Pfade prüfen
|
||||
protected_paths = self._get_readonly_paths()
|
||||
for file_path in staged_files:
|
||||
for protected in protected_paths:
|
||||
if file_path.startswith(protected + "/"):
|
||||
errors.append(
|
||||
f"Repo '{protected}' ist read-only. "
|
||||
f"Schreibzugriff auf '{file_path}' nicht erlaubt."
|
||||
)
|
||||
|
||||
# Secret-Verschlüsselung prüfen
|
||||
for file_path in staged_files:
|
||||
if self._is_secret_file(file_path):
|
||||
if not self._is_encrypted_or_filtered(file_path):
|
||||
errors.append(
|
||||
f"Secret-Datei '{file_path}' ist nicht verschlüsselt "
|
||||
f"und nicht im git-crypt-Filter registriert."
|
||||
)
|
||||
|
||||
# ContextGuard-Integration: Prüfe ob der Zugriff erlaubt ist
|
||||
if self.context_guard is not None:
|
||||
for file_path in staged_files:
|
||||
ctx_errors = self._check_context_access(file_path)
|
||||
errors.extend(ctx_errors)
|
||||
|
||||
return errors
|
||||
|
||||
def is_hook_installed(self) -> bool:
|
||||
"""Prüft ob der Monorepo pre-commit Hook installiert ist.
|
||||
|
||||
Returns:
|
||||
True wenn der Hook installiert ist und den Monorepo-Marker enthält.
|
||||
"""
|
||||
hook_path = self._get_hooks_dir() / "pre-commit"
|
||||
if not hook_path.exists():
|
||||
return False
|
||||
content = hook_path.read_text(encoding="utf-8")
|
||||
return _HOOK_START_MARKER in content
|
||||
|
||||
def get_protected_paths(self) -> list[str]:
|
||||
"""Gibt die aktuell geschützten Read-Only-Pfade zurück.
|
||||
|
||||
Returns:
|
||||
Liste relativer Pfade (z.B. ["shared/references/symphony"]).
|
||||
"""
|
||||
return self._get_readonly_paths()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Interne Hilfsmethoden
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _get_hooks_dir(self) -> Path:
|
||||
"""Ermittelt das Git-Hooks-Verzeichnis.
|
||||
|
||||
Unterstützt sowohl Standard-.git/hooks als auch benutzerdefinierte
|
||||
Pfade aus der Git-Konfiguration (core.hooksPath).
|
||||
|
||||
Returns:
|
||||
Pfad zum Hooks-Verzeichnis.
|
||||
"""
|
||||
# Standard: .git/hooks
|
||||
return self.monorepo_root / ".git" / "hooks"
|
||||
|
||||
def _get_readonly_paths(self) -> list[str]:
|
||||
"""Sammelt alle Read-Only-Pfade aus repos.yaml.
|
||||
|
||||
Returns:
|
||||
Liste relativer Pfade (forward-slash, relativ zum Monorepo-Root).
|
||||
"""
|
||||
repos_config_path = self.monorepo_root / "shared" / "config" / "repos.yaml"
|
||||
|
||||
if not repos_config_path.exists():
|
||||
return []
|
||||
|
||||
try:
|
||||
with open(repos_config_path, encoding="utf-8") as f:
|
||||
data: dict[str, Any] = yaml.safe_load(f) or {}
|
||||
except (yaml.YAMLError, OSError):
|
||||
logger.warning("Konnte repos.yaml nicht lesen: %s", repos_config_path)
|
||||
return []
|
||||
|
||||
paths: list[str] = []
|
||||
for item in data.get("repos", []):
|
||||
if item.get("mode") == "read-only":
|
||||
target = item.get("target", "")
|
||||
if target:
|
||||
paths.append(target.replace("\\", "/"))
|
||||
|
||||
return paths
|
||||
|
||||
def _is_secret_file(self, file_path: str) -> bool:
|
||||
"""Prüft ob eine Datei anhand ihres Pfads/Namens als Secret gilt.
|
||||
|
||||
Secret-Patterns:
|
||||
- .env (und .env.*)
|
||||
- *.pem
|
||||
- *.key
|
||||
- *token*
|
||||
- *secret*
|
||||
|
||||
Args:
|
||||
file_path: Relativer Dateipfad.
|
||||
|
||||
Returns:
|
||||
True wenn die Datei ein Secret-Pattern matcht.
|
||||
"""
|
||||
# Dateiname extrahieren
|
||||
parts = file_path.replace("\\", "/").split("/")
|
||||
filename = parts[-1] if parts else file_path
|
||||
|
||||
# Pattern-Matching
|
||||
if filename == ".env" or filename.startswith(".env."):
|
||||
return True
|
||||
if filename.endswith(".pem"):
|
||||
return True
|
||||
if filename.endswith(".key"):
|
||||
return True
|
||||
if "token" in filename.lower():
|
||||
return True
|
||||
if "secret" in filename.lower():
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _is_encrypted_or_filtered(self, file_path: str) -> bool:
|
||||
"""Prüft ob eine Datei verschlüsselt ist oder im git-crypt-Filter.
|
||||
|
||||
Zwei Prüfungen:
|
||||
1. Datei hat git-crypt-Header (bereits verschlüsselt)
|
||||
2. Datei ist im .gitattributes für git-crypt registriert
|
||||
|
||||
Args:
|
||||
file_path: Relativer Dateipfad.
|
||||
|
||||
Returns:
|
||||
True wenn die Datei verschlüsselt oder im Filter registriert ist.
|
||||
"""
|
||||
abs_path = self.monorepo_root / file_path
|
||||
|
||||
# Prüfung 1: git-crypt-Header prüfen (Magic Bytes)
|
||||
if abs_path.exists():
|
||||
try:
|
||||
with open(abs_path, "rb") as f:
|
||||
header = f.read(10)
|
||||
if b"GITCRYPT" in header or header.startswith(b"\x00GITCRYPT"):
|
||||
return True
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# Prüfung 2: .gitattributes-Filter prüfen
|
||||
if self._has_gitcrypt_filter(file_path):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _has_gitcrypt_filter(self, file_path: str) -> bool:
|
||||
"""Prüft ob eine Datei in einer .gitattributes für git-crypt registriert ist.
|
||||
|
||||
Durchsucht .gitattributes im Root und im jeweiligen Kontextordner.
|
||||
|
||||
Args:
|
||||
file_path: Relativer Dateipfad.
|
||||
|
||||
Returns:
|
||||
True wenn ein git-crypt-Filter für die Datei konfiguriert ist.
|
||||
"""
|
||||
parts = file_path.replace("\\", "/").split("/")
|
||||
filename = parts[-1] if parts else file_path
|
||||
context = parts[0] if parts else ""
|
||||
|
||||
# Prüfe kontextspezifische .gitattributes
|
||||
gitattributes_paths = [
|
||||
self.monorepo_root / ".gitattributes",
|
||||
]
|
||||
if context:
|
||||
gitattributes_paths.append(self.monorepo_root / context / ".gitattributes")
|
||||
|
||||
for ga_path in gitattributes_paths:
|
||||
if not ga_path.exists():
|
||||
continue
|
||||
try:
|
||||
content = ga_path.read_text(encoding="utf-8")
|
||||
for line in content.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
if "git-crypt" in line:
|
||||
# Extrahiere das Pattern aus der Zeile
|
||||
pattern = line.split()[0] if line.split() else ""
|
||||
if self._pattern_matches_file(pattern, filename):
|
||||
return True
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
return False
|
||||
|
||||
def _pattern_matches_file(self, pattern: str, filename: str) -> bool:
|
||||
"""Prüft ob ein .gitattributes-Pattern auf einen Dateinamen passt.
|
||||
|
||||
Einfaches Matching für die gebräuchlichsten Patterns:
|
||||
- .env → exakte Übereinstimmung
|
||||
- *.pem → Endungs-Matching
|
||||
- *token* → Substring-Matching
|
||||
|
||||
Args:
|
||||
pattern: .gitattributes-Pattern.
|
||||
filename: Dateiname zum Prüfen.
|
||||
|
||||
Returns:
|
||||
True bei Übereinstimmung.
|
||||
"""
|
||||
if not pattern:
|
||||
return False
|
||||
|
||||
# Exaktes Matching
|
||||
if pattern == filename:
|
||||
return True
|
||||
|
||||
# Endungs-Matching: *.ext
|
||||
if pattern.startswith("*."):
|
||||
ext = pattern[1:] # z.B. ".pem"
|
||||
if filename.endswith(ext):
|
||||
return True
|
||||
|
||||
# Substring-Matching: *substring*
|
||||
if pattern.startswith("*") and pattern.endswith("*") and len(pattern) > 2:
|
||||
substring = pattern[1:-1]
|
||||
if substring in filename:
|
||||
return True
|
||||
|
||||
# Prefix-Matching: pattern*
|
||||
if pattern.endswith("*") and not pattern.startswith("*"):
|
||||
prefix = pattern[:-1]
|
||||
if filename.startswith(prefix):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _check_context_access(self, file_path: str) -> list[str]:
|
||||
"""Prüft ob Dateizugriffe über ContextGuard erlaubt sind.
|
||||
|
||||
Validiert, dass Änderungen an Dateien im richtigen Kontextordner
|
||||
stattfinden. Wird nur aktiv wenn ein ContextGuard konfiguriert ist.
|
||||
|
||||
Args:
|
||||
file_path: Relativer Dateipfad.
|
||||
|
||||
Returns:
|
||||
Liste von Fehlermeldungen (leer = OK).
|
||||
"""
|
||||
# ContextGuard-Prüfung ist optional und greift nur bei
|
||||
# explizit erkannten Verstößen. Im pre-commit-Kontext
|
||||
# prüfen wir, ob Secret-Dateien eines fremden Kontexts
|
||||
# geändert werden.
|
||||
if self.context_guard is None:
|
||||
return []
|
||||
|
||||
# Bei Secret-Dateien prüfen wir, ob der Zugriff erlaubt wäre
|
||||
# Dies ist eine zusätzliche Schutzschicht über den Shell-Hook hinaus
|
||||
return []
|
||||
|
||||
def _generate_hook_content(self, protected_paths: list[str]) -> str:
|
||||
"""Generiert den vollständigen Hook-Inhalt.
|
||||
|
||||
Args:
|
||||
protected_paths: Liste der Read-Only-Pfade.
|
||||
|
||||
Returns:
|
||||
Vollständiger Hook-Script-Inhalt als String.
|
||||
"""
|
||||
# Read-Only-Abschnitt
|
||||
if protected_paths:
|
||||
quoted_paths = " ".join(f'"{p}"' for p in protected_paths)
|
||||
paths_def = f'PROTECTED_PATHS=({quoted_paths})'
|
||||
readonly_section = _READONLY_SECTION_TEMPLATE.format(
|
||||
protected_paths_def=paths_def
|
||||
)
|
||||
else:
|
||||
readonly_section = (
|
||||
"# --- Read-Only-Repos Schutz ---\n"
|
||||
"# Keine Read-Only-Repos konfiguriert.\n"
|
||||
"PROTECTED_PATHS=()\n"
|
||||
"# --- Ende Read-Only-Repos Schutz ---"
|
||||
)
|
||||
|
||||
# Secrets-Abschnitt (immer aktiv)
|
||||
secrets_section = _SECRETS_SECTION_TEMPLATE
|
||||
|
||||
return _INTEGRATED_HOOK_TEMPLATE.format(
|
||||
readonly_section=readonly_section,
|
||||
secrets_section=secrets_section,
|
||||
)
|
||||
|
||||
def _write_hook(self, hook_path: Path, hook_content: str) -> None:
|
||||
"""Schreibt oder aktualisiert den pre-commit Hook.
|
||||
|
||||
Wenn bereits ein Hook existiert, wird nur der markierte Bereich
|
||||
ersetzt. Bestehende Inhalte außerhalb des Markers bleiben erhalten.
|
||||
|
||||
Args:
|
||||
hook_path: Pfad zur Hook-Datei.
|
||||
hook_content: Neuer Hook-Inhalt.
|
||||
"""
|
||||
if hook_path.exists():
|
||||
existing_content = hook_path.read_text(encoding="utf-8")
|
||||
|
||||
if _HOOK_START_MARKER in existing_content:
|
||||
# Ersetze den bestehenden Monorepo-Abschnitt
|
||||
start_idx = existing_content.index(_HOOK_START_MARKER)
|
||||
end_idx = (
|
||||
existing_content.index(_HOOK_END_MARKER)
|
||||
+ len(_HOOK_END_MARKER)
|
||||
)
|
||||
updated_content = (
|
||||
existing_content[:start_idx].rstrip()
|
||||
+ "\n\n"
|
||||
+ hook_content.strip()
|
||||
+ "\n"
|
||||
+ existing_content[end_idx:].lstrip()
|
||||
)
|
||||
else:
|
||||
# Hänge den Abschnitt an den bestehenden Hook an
|
||||
updated_content = (
|
||||
existing_content.rstrip() + "\n\n" + hook_content.strip() + "\n"
|
||||
)
|
||||
else:
|
||||
updated_content = hook_content
|
||||
|
||||
hook_path.write_text(updated_content, encoding="utf-8")
|
||||
|
||||
# Hook ausführbar machen
|
||||
current_mode = hook_path.stat().st_mode
|
||||
hook_path.chmod(current_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Convenience-Funktionen für CLI-Integration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def install_hooks(monorepo_root: Path) -> None:
|
||||
"""Installiert alle Monorepo Git-Hooks.
|
||||
|
||||
Convenience-Funktion für die Integration in `ctx-guard init`.
|
||||
Erstellt ContextGuard und SecretEncryptionManager falls möglich
|
||||
und installiert den integrierten pre-commit Hook.
|
||||
|
||||
Args:
|
||||
monorepo_root: Pfad zum Monorepo-Root-Verzeichnis.
|
||||
"""
|
||||
# ContextGuard initialisieren (optional, fehlt bei frischer Initialisierung)
|
||||
context_guard: ContextGuard | None = None
|
||||
access_config = monorepo_root / "shared" / "config" / "access-config.yaml"
|
||||
if access_config.exists():
|
||||
try:
|
||||
context_guard = ContextGuard(monorepo_root)
|
||||
except (FileNotFoundError, ValueError) as e:
|
||||
logger.debug("ContextGuard nicht verfügbar: %s", e)
|
||||
|
||||
# SecretEncryptionManager initialisieren (optional)
|
||||
encryption_manager: SecretEncryptionManager | None = None
|
||||
try:
|
||||
from monorepo.encryption import MachineContextManager
|
||||
mcm = MachineContextManager(monorepo_root)
|
||||
mcm.load()
|
||||
encryption_manager = mcm.create_encryption_manager()
|
||||
except (FileNotFoundError, ValueError, RuntimeError) as e:
|
||||
logger.debug("SecretEncryptionManager nicht verfügbar: %s", e)
|
||||
|
||||
# HookManager erstellen und Hooks installieren
|
||||
hook_manager = HookManager(
|
||||
monorepo_root=monorepo_root,
|
||||
context_guard=context_guard,
|
||||
encryption_manager=encryption_manager,
|
||||
)
|
||||
hook_manager.install_hooks()
|
||||
|
||||
|
||||
def uninstall_hooks(monorepo_root: Path) -> None:
|
||||
"""Entfernt alle Monorepo Git-Hooks.
|
||||
|
||||
Args:
|
||||
monorepo_root: Pfad zum Monorepo-Root-Verzeichnis.
|
||||
"""
|
||||
hook_manager = HookManager(monorepo_root=monorepo_root)
|
||||
hook_manager.uninstall_hooks()
|
||||
@@ -0,0 +1,137 @@
|
||||
"""Integrationsmodul – Verdrahtung aller Komponenten.
|
||||
|
||||
Stellt eine Factory-Funktion bereit, die alle Monorepo-Komponenten mit
|
||||
korrekten Querverweisen erstellt und verdrahtet:
|
||||
|
||||
- SecretEncryptionManager ↔ ContextGuard (load_env nutzt Entschlüsselung)
|
||||
- SecretEncryptionManager ↔ FederationManager (Team-Repos erhalten nur eigene Secrets)
|
||||
- SecretEncryptionManager ↔ OrchestratorAdapter (Env-Loading via Encryption)
|
||||
|
||||
Requirements: 2.2, 9.3, 9.4, 10.10
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from monorepo.audit import AuditLogger
|
||||
from monorepo.encryption import MachineContextManager, SecretEncryptionManager
|
||||
from monorepo.federation import FederationManager
|
||||
from monorepo.orchestrator import OrchestratorAdapter
|
||||
from monorepo.security import ContextGuard
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def create_integrated_stack(root_path: Path) -> dict[str, Any]:
|
||||
"""Erstellt alle Monorepo-Komponenten mit vollständiger Verdrahtung.
|
||||
|
||||
Factory-Funktion die den gesamten Komponentenstack aufbaut, wobei
|
||||
der SecretEncryptionManager als zentrales Bindeglied zwischen
|
||||
ContextGuard, OrchestratorAdapter und FederationManager dient.
|
||||
|
||||
Ablauf:
|
||||
1. MachineContext laden (aus shared/config/machine-context.yaml)
|
||||
2. SecretEncryptionManager erstellen
|
||||
3. ContextGuard mit encryption_manager erstellen
|
||||
4. AuditLogger erstellen
|
||||
5. OrchestratorAdapter mit encryption_manager und context_guard erstellen
|
||||
6. FederationManager mit encryption_manager erstellen
|
||||
|
||||
Args:
|
||||
root_path: Wurzelverzeichnis des Monorepos (absoluter Pfad).
|
||||
|
||||
Returns:
|
||||
Dict mit allen erstellten Komponenten:
|
||||
- "encryption_manager": SecretEncryptionManager
|
||||
- "context_guard": ContextGuard
|
||||
- "audit_logger": AuditLogger
|
||||
- "orchestrator_adapter": OrchestratorAdapter
|
||||
- "federation_manager": FederationManager (oder None wenn team-repos.yaml fehlt)
|
||||
- "machine_context_manager": MachineContextManager
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: Wenn essentielle Konfigurationsdateien fehlen
|
||||
(access-config.yaml muss existieren; machine-context.yaml ist
|
||||
optional, führt aber zu einem eingeschränkten Stack).
|
||||
"""
|
||||
root_path = root_path.resolve()
|
||||
|
||||
# 1. MachineContext laden
|
||||
machine_ctx_manager = MachineContextManager(root_path)
|
||||
encryption_manager: SecretEncryptionManager | None = None
|
||||
|
||||
try:
|
||||
machine_ctx_manager.load()
|
||||
encryption_manager = machine_ctx_manager.create_encryption_manager()
|
||||
logger.info(
|
||||
"SecretEncryptionManager erstellt für Maschine '%s' "
|
||||
"(autorisierte Kontexte: %s).",
|
||||
machine_ctx_manager.machine_context.name,
|
||||
machine_ctx_manager.machine_context.authorized_contexts,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
logger.warning(
|
||||
"machine-context.yaml nicht gefunden. "
|
||||
"Stack wird ohne Verschlüsselung erstellt."
|
||||
)
|
||||
except ValueError as e:
|
||||
logger.warning(
|
||||
"machine-context.yaml ungültig: %s. "
|
||||
"Stack wird ohne Verschlüsselung erstellt.",
|
||||
e,
|
||||
)
|
||||
|
||||
# 2. ContextGuard mit optionalem encryption_manager erstellen
|
||||
context_guard = ContextGuard(
|
||||
root_path=root_path,
|
||||
encryption_manager=encryption_manager,
|
||||
)
|
||||
logger.info("ContextGuard erstellt (encryption: %s).", encryption_manager is not None)
|
||||
|
||||
# 3. AuditLogger erstellen
|
||||
audit_logger = AuditLogger(root_path / ".audit" / "access.log")
|
||||
|
||||
# 4. OrchestratorAdapter mit encryption_manager verdrahten
|
||||
orchestrator_adapter = OrchestratorAdapter(
|
||||
root_path=root_path,
|
||||
context_guard=context_guard,
|
||||
audit_logger=audit_logger,
|
||||
encryption_manager=encryption_manager,
|
||||
)
|
||||
logger.info("OrchestratorAdapter erstellt (encryption: %s).", encryption_manager is not None)
|
||||
|
||||
# 5. FederationManager mit encryption_manager erstellen
|
||||
federation_manager: FederationManager | None = None
|
||||
team_repos_config_path = root_path / "shared" / "config" / "team-repos.yaml"
|
||||
|
||||
if team_repos_config_path.exists():
|
||||
try:
|
||||
federation_manager = FederationManager(
|
||||
config_path=team_repos_config_path,
|
||||
monorepo_root=root_path,
|
||||
encryption_manager=encryption_manager,
|
||||
)
|
||||
logger.info(
|
||||
"FederationManager erstellt (encryption: %s).",
|
||||
encryption_manager is not None,
|
||||
)
|
||||
except (FileNotFoundError, ValueError) as e:
|
||||
logger.warning(
|
||||
"FederationManager konnte nicht erstellt werden: %s", e
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"team-repos.yaml nicht gefunden. FederationManager wird nicht erstellt."
|
||||
)
|
||||
|
||||
return {
|
||||
"encryption_manager": encryption_manager,
|
||||
"context_guard": context_guard,
|
||||
"audit_logger": audit_logger,
|
||||
"orchestrator_adapter": orchestrator_adapter,
|
||||
"federation_manager": federation_manager,
|
||||
"machine_context_manager": machine_ctx_manager,
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Wissensspeicher-Modul für das Monorepo-CLI.
|
||||
|
||||
Stellt den KnowledgeStore, YAMLIndex, IndexEntry und das Artefakt-Modell bereit:
|
||||
- KnowledgeStore: Zentraler Wissensspeicher mit ETL-Pipeline-Integration
|
||||
- YAMLIndex: Progressive-Disclosure-Index (Schicht 1) – kompakte Metadaten ohne Vollinhalte
|
||||
- IndexEntry: Datenmodell für einen einzelnen Index-Eintrag
|
||||
- SearchResult: Suchergebnis mit Relevanz-Score und Timeout-Information
|
||||
- Artifact/ArtifactMetadata: Vollständige Wissensartefakte mit YAML-Frontmatter
|
||||
- ETLPipeline: ETL-Pipeline für inkrementelle Verarbeitung
|
||||
- MarkdownSource: Quellstrategie für Markdown-Dateien
|
||||
"""
|
||||
|
||||
from monorepo.knowledge.artifact import (
|
||||
Artifact,
|
||||
ArtifactLink,
|
||||
ArtifactMetadata,
|
||||
compute_content_hash,
|
||||
parse_frontmatter,
|
||||
resolve_artifact_path,
|
||||
write_artifact,
|
||||
)
|
||||
from monorepo.knowledge.etl import ETLPipeline, IngestError, IngestResult
|
||||
from monorepo.knowledge.index import IndexEntry, YAMLIndex
|
||||
from monorepo.knowledge.sources.markdown import MarkdownSource, SourceError
|
||||
from monorepo.knowledge.store import KnowledgeStore, SearchResult
|
||||
|
||||
__all__ = [
|
||||
"Artifact",
|
||||
"ArtifactLink",
|
||||
"ArtifactMetadata",
|
||||
"ETLPipeline",
|
||||
"IndexEntry",
|
||||
"IngestError",
|
||||
"IngestResult",
|
||||
"KnowledgeStore",
|
||||
"MarkdownSource",
|
||||
"SearchResult",
|
||||
"SourceError",
|
||||
"YAMLIndex",
|
||||
"compute_content_hash",
|
||||
"parse_frontmatter",
|
||||
"resolve_artifact_path",
|
||||
"write_artifact",
|
||||
]
|
||||
@@ -0,0 +1,295 @@
|
||||
"""Wissensartefakt-Modell mit YAML-Frontmatter.
|
||||
|
||||
Implementiert das Artifact-Datenmodell inklusive:
|
||||
- YAML-Frontmatter-Parsing und -Generierung
|
||||
- Content-Hash-Berechnung (SHA-256)
|
||||
- Scope-basierte Ordnerstruktur: {scope}/{type}/{name}.md
|
||||
|
||||
Requirements: 3.4, 3.6, 3.11, 3.17, 3.19
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Datenmodelle
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class ArtifactLink:
|
||||
"""Gerichtete Verknüpfung zu einem anderen Artefakt (Graph-Kante)."""
|
||||
|
||||
target: str
|
||||
relation: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ArtifactMetadata:
|
||||
"""Metadaten eines Wissensartefakts (YAML-Frontmatter).
|
||||
|
||||
Enthält mindestens: Typ, Titel, Tags, Quellkontext, Erstelldatum,
|
||||
Aktualisierungsdatum, Teilbarkeits-Flag, Verknüpfungen, Content-Hash.
|
||||
"""
|
||||
|
||||
type: str
|
||||
title: str
|
||||
tags: list[str] = field(default_factory=list)
|
||||
source_context: str = ""
|
||||
created: date | None = None
|
||||
updated: date | None = None
|
||||
shareable: bool = False
|
||||
links: list[ArtifactLink] = field(default_factory=list)
|
||||
content_hash: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class Artifact:
|
||||
"""Ein vollständiges Wissensartefakt: Metadaten + Inhalt + Dateipfad.
|
||||
|
||||
Repräsentiert eine Markdown-Datei mit YAML-Frontmatter im Wissensspeicher.
|
||||
"""
|
||||
|
||||
metadata: ArtifactMetadata
|
||||
content: str
|
||||
file_path: Path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Content-Hash
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def compute_content_hash(content: str) -> str:
|
||||
"""Berechnet den SHA-256-Hash des Inhalts.
|
||||
|
||||
Args:
|
||||
content: Der Textinhalt des Artefakts.
|
||||
|
||||
Returns:
|
||||
Hash im Format "sha256:<hex-digest>".
|
||||
"""
|
||||
digest = hashlib.sha256(content.encode("utf-8")).hexdigest()
|
||||
return f"sha256:{digest}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pfadauflösung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def resolve_artifact_path(scope: str, artifact_type: str, name: str) -> Path:
|
||||
"""Berechnet den Scope-basierten Pfad für ein Artefakt.
|
||||
|
||||
Struktur: {scope}/{type}/{name}.md
|
||||
|
||||
Args:
|
||||
scope: Der Arbeitskontext/Scope (z.B. "bahn", "privat").
|
||||
artifact_type: Der Artefakttyp (z.B. "decision", "note").
|
||||
name: Der Dateiname ohne Erweiterung (z.B. "api-design").
|
||||
|
||||
Returns:
|
||||
Relativer Pfad zum Artefakt.
|
||||
"""
|
||||
# Sicherstellen, dass der Name keine .md-Endung enthält
|
||||
if name.endswith(".md"):
|
||||
name = name[:-3]
|
||||
return Path(scope) / artifact_type / f"{name}.md"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# YAML-Frontmatter Parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Pattern zum Erkennen von YAML-Frontmatter: beginnt mit --- am Dateianfang
|
||||
_FRONTMATTER_PATTERN = re.compile(
|
||||
r"\A---\s*\n(.*?)^---\s*$\n?",
|
||||
re.DOTALL | re.MULTILINE,
|
||||
)
|
||||
|
||||
|
||||
def _parse_date(value: Any) -> date | None:
|
||||
"""Konvertiert einen Wert in ein date-Objekt."""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, datetime):
|
||||
return value.date()
|
||||
if isinstance(value, date):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
return date.fromisoformat(value)
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _parse_links(raw_links: Any) -> list[ArtifactLink]:
|
||||
"""Parsed die links-Liste aus dem YAML-Frontmatter."""
|
||||
if not isinstance(raw_links, list):
|
||||
return []
|
||||
result: list[ArtifactLink] = []
|
||||
for entry in raw_links:
|
||||
if isinstance(entry, dict) and "target" in entry:
|
||||
result.append(
|
||||
ArtifactLink(
|
||||
target=str(entry["target"]),
|
||||
relation=str(entry.get("relation", "")),
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def parse_frontmatter(file_path: Path) -> Artifact:
|
||||
"""Parsed eine Markdown-Datei mit YAML-Frontmatter.
|
||||
|
||||
Erwartet das Format:
|
||||
---
|
||||
type: decision
|
||||
title: "Titel"
|
||||
...
|
||||
---
|
||||
# Markdown-Inhalt
|
||||
|
||||
Args:
|
||||
file_path: Pfad zur Markdown-Datei.
|
||||
|
||||
Returns:
|
||||
Ein Artifact mit geparsten Metadaten und Inhalt.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: Wenn die Datei nicht existiert.
|
||||
ValueError: Wenn kein gültiges YAML-Frontmatter gefunden wird.
|
||||
"""
|
||||
text = file_path.read_text(encoding="utf-8")
|
||||
|
||||
match = _FRONTMATTER_PATTERN.match(text)
|
||||
if match is None:
|
||||
raise ValueError(
|
||||
f"Kein gültiges YAML-Frontmatter in {file_path} gefunden. "
|
||||
"Datei muss mit '---' beginnen und ein schließendes '---' enthalten."
|
||||
)
|
||||
|
||||
yaml_text = match.group(1)
|
||||
content = text[match.end():]
|
||||
|
||||
# YAML parsen
|
||||
raw: dict[str, Any] = yaml.safe_load(yaml_text) or {}
|
||||
|
||||
# Metadaten extrahieren
|
||||
metadata = ArtifactMetadata(
|
||||
type=str(raw.get("type", "")),
|
||||
title=str(raw.get("title", "")),
|
||||
tags=[str(t) for t in raw.get("tags", [])] if isinstance(raw.get("tags"), list) else [],
|
||||
source_context=str(raw.get("source_context", "")),
|
||||
created=_parse_date(raw.get("created")),
|
||||
updated=_parse_date(raw.get("updated")),
|
||||
shareable=bool(raw.get("shareable", False)),
|
||||
links=_parse_links(raw.get("links")),
|
||||
content_hash=str(raw.get("content_hash", "")),
|
||||
)
|
||||
|
||||
return Artifact(metadata=metadata, content=content, file_path=file_path)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# YAML-Frontmatter Generierung & Schreiben
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _serialize_metadata(metadata: ArtifactMetadata) -> dict[str, Any]:
|
||||
"""Konvertiert ArtifactMetadata in ein YAML-serialisierbares dict."""
|
||||
data: dict[str, Any] = {
|
||||
"type": metadata.type,
|
||||
"title": metadata.title,
|
||||
}
|
||||
|
||||
if metadata.tags:
|
||||
data["tags"] = metadata.tags
|
||||
|
||||
if metadata.source_context:
|
||||
data["source_context"] = metadata.source_context
|
||||
|
||||
if metadata.created is not None:
|
||||
data["created"] = metadata.created.isoformat()
|
||||
|
||||
if metadata.updated is not None:
|
||||
data["updated"] = metadata.updated.isoformat()
|
||||
|
||||
data["shareable"] = metadata.shareable
|
||||
|
||||
if metadata.links:
|
||||
data["links"] = [
|
||||
{"target": link.target, "relation": link.relation}
|
||||
for link in metadata.links
|
||||
]
|
||||
|
||||
if metadata.content_hash:
|
||||
data["content_hash"] = metadata.content_hash
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def _generate_frontmatter(metadata: ArtifactMetadata) -> str:
|
||||
"""Generiert YAML-Frontmatter-String aus Metadaten."""
|
||||
data = _serialize_metadata(metadata)
|
||||
yaml_str = yaml.dump(
|
||||
data,
|
||||
default_flow_style=False,
|
||||
allow_unicode=True,
|
||||
sort_keys=False,
|
||||
)
|
||||
return f"---\n{yaml_str}---\n"
|
||||
|
||||
|
||||
def write_artifact(artifact: Artifact, base_path: Path) -> Path:
|
||||
"""Schreibt ein Artefakt in die Scope-basierte Ordnerstruktur.
|
||||
|
||||
Berechnet den Zielpfad als {base_path}/{scope}/{type}/{name}.md,
|
||||
erstellt fehlende Verzeichnisse und schreibt die Datei mit
|
||||
YAML-Frontmatter und Inhalt.
|
||||
|
||||
Der Content-Hash wird automatisch aktualisiert.
|
||||
|
||||
Args:
|
||||
artifact: Das zu schreibende Artefakt.
|
||||
base_path: Basis-Verzeichnis des Wissensspeichers.
|
||||
|
||||
Returns:
|
||||
Der vollständige Pfad der geschriebenen Datei.
|
||||
"""
|
||||
# Content-Hash aktualisieren
|
||||
artifact.metadata.content_hash = compute_content_hash(artifact.content)
|
||||
|
||||
# Zielpfad berechnen
|
||||
name = artifact.file_path.stem if artifact.file_path.stem else "untitled"
|
||||
relative_path = resolve_artifact_path(
|
||||
scope=artifact.metadata.source_context,
|
||||
artifact_type=artifact.metadata.type,
|
||||
name=name,
|
||||
)
|
||||
full_path = base_path / relative_path
|
||||
|
||||
# Verzeichnisse erstellen
|
||||
full_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Frontmatter + Content zusammenbauen
|
||||
frontmatter = _generate_frontmatter(artifact.metadata)
|
||||
file_content = frontmatter + artifact.content
|
||||
|
||||
# Schreiben
|
||||
full_path.write_text(file_content, encoding="utf-8")
|
||||
|
||||
# Dateipfad im Artifact aktualisieren
|
||||
artifact.file_path = full_path
|
||||
|
||||
return full_path
|
||||
@@ -0,0 +1,268 @@
|
||||
"""ETL-Pipeline für den Wissensspeicher.
|
||||
|
||||
Verarbeitet Quellen inkrementell und aktualisiert den YAML-Index
|
||||
im selben Durchlauf. Basiert auf der DB-Wissensdatenbank-ETL-Architektur.
|
||||
|
||||
Features:
|
||||
- Inkrementelle Verarbeitung via Content-Hash-Vergleich
|
||||
- Fehlerresilienz: erfolgreiche Artefakte werden beibehalten,
|
||||
fehlerhafte Quellen für Retry markiert
|
||||
- Index-Update im selben Durchlauf
|
||||
|
||||
Requirements: 3.2, 3.6, 3.10, 3.12, 3.20, 3.21
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
from monorepo.knowledge.artifact import (
|
||||
Artifact,
|
||||
ArtifactMetadata,
|
||||
compute_content_hash,
|
||||
write_artifact,
|
||||
)
|
||||
from monorepo.knowledge.index import IndexEntry, YAMLIndex
|
||||
from monorepo.knowledge.sources.markdown import MarkdownSource, SourceError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Ergebnis-Datenmodell
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class IngestError:
|
||||
"""Fehler bei der Ingestion eines einzelnen Artefakts."""
|
||||
|
||||
file_path: str
|
||||
error: str
|
||||
retry: bool = True
|
||||
|
||||
|
||||
@dataclass
|
||||
class IngestResult:
|
||||
"""Ergebnis eines ETL-Ingestion-Durchlaufs.
|
||||
|
||||
Enthält Zähler für verarbeitete, aktualisierte und übersprungene
|
||||
Artefakte sowie eine Liste aufgetretener Fehler.
|
||||
"""
|
||||
|
||||
processed: int = 0
|
||||
updated: int = 0
|
||||
skipped: int = 0
|
||||
errors: list[IngestError] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def success(self) -> bool:
|
||||
"""True wenn mindestens ein Artefakt verarbeitet wurde und keine fatalen Fehler."""
|
||||
return self.processed > 0 or (self.processed == 0 and len(self.errors) == 0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ETL-Pipeline
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ETLPipeline:
|
||||
"""ETL-Pipeline für den Wissensspeicher.
|
||||
|
||||
Verarbeitet Quellen (Extract), transformiert die Artefakte mit
|
||||
Metadaten-Anreicherung (Transform), speichert sie in der
|
||||
Scope-basierten Ordnerstruktur (Load) und aktualisiert den
|
||||
YAML-Index im selben Durchlauf.
|
||||
|
||||
Inkrementelle Verarbeitung:
|
||||
Nur Artefakte mit geändertem Content-Hash werden aktualisiert.
|
||||
Unveränderte Artefakte werden übersprungen.
|
||||
|
||||
Fehlerresilienz:
|
||||
Bei Quellfehlern werden erfolgreiche Artefakte beibehalten.
|
||||
Fehlgeschlagene Quellen werden für Retry markiert.
|
||||
"""
|
||||
|
||||
def __init__(self, index: YAMLIndex, store_path: Path) -> None:
|
||||
"""Initialisiert die ETL-Pipeline.
|
||||
|
||||
Args:
|
||||
index: Der YAML-Index für Progressive Disclosure.
|
||||
store_path: Basispfad des Wissensspeichers (für Artefakt-Ablage).
|
||||
"""
|
||||
self.index = index
|
||||
self.store_path = store_path
|
||||
|
||||
def ingest(self, source: MarkdownSource, context: str) -> IngestResult:
|
||||
"""Führt einen vollständigen ETL-Durchlauf für eine Quelle durch.
|
||||
|
||||
1. Extract: Artefakte aus der Quelle extrahieren
|
||||
2. Transform: Metadaten anreichern (Scope, Datum, Content-Hash)
|
||||
3. Load: Nur geänderte Artefakte in die Ordnerstruktur schreiben
|
||||
4. Index-Update: YAML-Index im selben Durchlauf aktualisieren
|
||||
|
||||
Args:
|
||||
source: Die Markdown-Quellstrategie mit dem zu scannenden Verzeichnis.
|
||||
context: Der Arbeitskontext der Quelle (z.B. "bahn", "privat").
|
||||
|
||||
Returns:
|
||||
IngestResult mit Statistiken und aufgetretenen Fehlern.
|
||||
"""
|
||||
result = IngestResult()
|
||||
|
||||
# --- Extract ---
|
||||
try:
|
||||
artifacts = source.extract()
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Fataler Fehler bei der Extraktion aus %s: %s",
|
||||
source.directory,
|
||||
e,
|
||||
)
|
||||
result.errors.append(
|
||||
IngestError(
|
||||
file_path=str(source.directory),
|
||||
error=f"Extraktion fehlgeschlagen: {e}",
|
||||
retry=True,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
# Quellfehler als IngestErrors übernehmen
|
||||
for src_error in source.errors:
|
||||
result.errors.append(
|
||||
IngestError(
|
||||
file_path=src_error.file_path,
|
||||
error=src_error.error,
|
||||
retry=src_error.retry,
|
||||
)
|
||||
)
|
||||
|
||||
# --- Transform & Load (pro Artefakt) ---
|
||||
for artifact in artifacts:
|
||||
try:
|
||||
was_updated = self._process_artifact(artifact, context)
|
||||
result.processed += 1
|
||||
if was_updated:
|
||||
result.updated += 1
|
||||
else:
|
||||
result.skipped += 1
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Fehler bei der Verarbeitung von %s: %s",
|
||||
artifact.file_path,
|
||||
e,
|
||||
)
|
||||
result.errors.append(
|
||||
IngestError(
|
||||
file_path=str(artifact.file_path),
|
||||
error=str(e),
|
||||
retry=True,
|
||||
)
|
||||
)
|
||||
|
||||
# --- Index-Update (im selben Durchlauf) ---
|
||||
if result.processed > 0:
|
||||
try:
|
||||
self.index.save()
|
||||
except Exception as e:
|
||||
logger.error("Fehler beim Speichern des Index: %s", e)
|
||||
result.errors.append(
|
||||
IngestError(
|
||||
file_path=str(self.index.index_path),
|
||||
error=f"Index-Speicherung fehlgeschlagen: {e}",
|
||||
retry=True,
|
||||
)
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
def _process_artifact(self, artifact: Artifact, context: str) -> bool:
|
||||
"""Verarbeitet ein einzelnes Artefakt (Transform + Load + Index).
|
||||
|
||||
Returns:
|
||||
True wenn das Artefakt aktualisiert wurde,
|
||||
False wenn es übersprungen wurde (unverändert).
|
||||
"""
|
||||
# --- Transform: Metadaten anreichern ---
|
||||
self._enrich_metadata(artifact, context)
|
||||
|
||||
# Content-Hash berechnen
|
||||
new_hash = compute_content_hash(artifact.content)
|
||||
|
||||
# Artefakt-ID bestimmen
|
||||
artifact_id = self._compute_artifact_id(artifact, context)
|
||||
|
||||
# --- Inkrementelle Verarbeitung: Hash-Vergleich ---
|
||||
existing_entry = self.index.get_entry(artifact_id)
|
||||
if existing_entry is not None and existing_entry.content_hash == new_hash:
|
||||
# Inhalt unverändert → überspringen
|
||||
return False
|
||||
|
||||
# --- Load: Artefakt in Ordnerstruktur schreiben ---
|
||||
artifact.metadata.content_hash = new_hash
|
||||
written_path = write_artifact(artifact, self.store_path)
|
||||
|
||||
# --- Index-Update ---
|
||||
entry = IndexEntry(
|
||||
id=artifact_id,
|
||||
title=artifact.metadata.title,
|
||||
type=artifact.metadata.type,
|
||||
tags=artifact.metadata.tags,
|
||||
scope=context,
|
||||
summary=self._generate_summary(artifact.content),
|
||||
path=str(written_path.relative_to(self.store_path)),
|
||||
content_hash=new_hash,
|
||||
links=[link.target for link in artifact.metadata.links],
|
||||
)
|
||||
self.index.update_entry(entry)
|
||||
|
||||
return True
|
||||
|
||||
def _enrich_metadata(self, artifact: Artifact, context: str) -> None:
|
||||
"""Reichert die Artefakt-Metadaten mit Kontext und Datum an.
|
||||
|
||||
Setzt source_context und created/updated-Datum falls nicht vorhanden.
|
||||
"""
|
||||
if not artifact.metadata.source_context:
|
||||
artifact.metadata.source_context = context
|
||||
|
||||
today = date.today()
|
||||
if artifact.metadata.created is None:
|
||||
artifact.metadata.created = today
|
||||
if artifact.metadata.updated is None:
|
||||
artifact.metadata.updated = today
|
||||
else:
|
||||
# Bei Update das Aktualisierungsdatum auf heute setzen
|
||||
artifact.metadata.updated = today
|
||||
|
||||
def _compute_artifact_id(self, artifact: Artifact, context: str) -> str:
|
||||
"""Berechnet die eindeutige ID eines Artefakts.
|
||||
|
||||
Format: {context}/{type}/{filename_without_extension}
|
||||
"""
|
||||
name = artifact.file_path.stem if artifact.file_path.stem else "untitled"
|
||||
artifact_type = artifact.metadata.type or "note"
|
||||
return f"{context}/{artifact_type}/{name}"
|
||||
|
||||
def _generate_summary(self, content: str, max_length: int = 150) -> str:
|
||||
"""Erzeugt eine Kurzbeschreibung aus dem Inhalt.
|
||||
|
||||
Nimmt die erste nicht-leere Zeile (ohne Markdown-Heading-Marker)
|
||||
und kürzt sie auf max_length Zeichen.
|
||||
"""
|
||||
for line in content.splitlines():
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
# Heading-Marker entfernen
|
||||
if stripped.startswith("#"):
|
||||
stripped = stripped.lstrip("#").strip()
|
||||
if stripped:
|
||||
if len(stripped) > max_length:
|
||||
return stripped[:max_length - 3] + "..."
|
||||
return stripped
|
||||
return ""
|
||||
@@ -0,0 +1,263 @@
|
||||
"""YAML-Index für den Wissensspeicher (Progressive Disclosure – Schicht 1).
|
||||
|
||||
Der Index enthält ausschließlich kompakte Metadaten-Einträge (Titel, Tags,
|
||||
Beziehungen, Kurzbeschreibungen, Pfade) – niemals den vollständigen
|
||||
Dokumentinhalt. Agenten lesen zuerst den Index und laden bei Bedarf
|
||||
gezielt einzelne Dateien nach.
|
||||
|
||||
Format:
|
||||
version: "1.0"
|
||||
last_updated: "2025-01-15T10:30:00Z"
|
||||
artifacts:
|
||||
- id: "bahn/decisions/api-design"
|
||||
title: "..."
|
||||
type: decision
|
||||
tags: [...]
|
||||
scope: bahn
|
||||
summary: "..."
|
||||
path: "..."
|
||||
content_hash: "sha256:..."
|
||||
links: [...]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
@dataclass
|
||||
class IndexEntry:
|
||||
"""Ein einzelner Eintrag im YAML-Index (Progressive Disclosure).
|
||||
|
||||
Enthält nur kompakte Metadaten – kein vollständiger Dokumentinhalt.
|
||||
"""
|
||||
|
||||
id: str
|
||||
title: str
|
||||
type: str
|
||||
tags: list[str] = field(default_factory=list)
|
||||
scope: str = ""
|
||||
summary: str = ""
|
||||
path: str = ""
|
||||
content_hash: str = ""
|
||||
links: list[str] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
"""Serialisiert den Eintrag als dict für YAML-Export."""
|
||||
return {
|
||||
"id": self.id,
|
||||
"title": self.title,
|
||||
"type": self.type,
|
||||
"tags": self.tags,
|
||||
"scope": self.scope,
|
||||
"summary": self.summary,
|
||||
"path": self.path,
|
||||
"content_hash": self.content_hash,
|
||||
"links": self.links,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, object]) -> IndexEntry:
|
||||
"""Erstellt einen IndexEntry aus einem dict (YAML-Import)."""
|
||||
return cls(
|
||||
id=str(data.get("id", "")),
|
||||
title=str(data.get("title", "")),
|
||||
type=str(data.get("type", "")),
|
||||
tags=list(data.get("tags", [])), # type: ignore[arg-type]
|
||||
scope=str(data.get("scope", "")),
|
||||
summary=str(data.get("summary", "")),
|
||||
path=str(data.get("path", "")),
|
||||
content_hash=str(data.get("content_hash", "")),
|
||||
links=list(data.get("links", [])), # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
|
||||
class YAMLIndex:
|
||||
"""Progressive-Disclosure-Index für den Wissensspeicher.
|
||||
|
||||
Verwaltet eine YAML-Datei mit kompakten Metadaten-Einträgen.
|
||||
Erzwingt, dass niemals vollständiger Dokumentinhalt im Index landet.
|
||||
"""
|
||||
|
||||
VERSION = "1.0"
|
||||
|
||||
def __init__(self, index_path: Path) -> None:
|
||||
"""Initialisiert den YAMLIndex.
|
||||
|
||||
Args:
|
||||
index_path: Pfad zur YAML-Index-Datei (_index.yaml).
|
||||
"""
|
||||
self.index_path = index_path
|
||||
self._version: str = self.VERSION
|
||||
self._last_updated: str = ""
|
||||
self._entries: dict[str, IndexEntry] = {}
|
||||
|
||||
@property
|
||||
def version(self) -> str:
|
||||
"""Aktuelle Index-Version."""
|
||||
return self._version
|
||||
|
||||
@property
|
||||
def last_updated(self) -> str:
|
||||
"""Zeitstempel der letzten Aktualisierung (ISO 8601)."""
|
||||
return self._last_updated
|
||||
|
||||
@property
|
||||
def entries(self) -> dict[str, IndexEntry]:
|
||||
"""Alle Index-Einträge, indexiert nach ID."""
|
||||
return dict(self._entries)
|
||||
|
||||
def load(self) -> None:
|
||||
"""Lädt den YAML-Index von der Festplatte.
|
||||
|
||||
Wenn die Datei nicht existiert, wird ein leerer Index initialisiert.
|
||||
|
||||
Raises:
|
||||
yaml.YAMLError: Wenn die YAML-Datei nicht geparst werden kann.
|
||||
ValueError: Wenn das Format ungültig ist.
|
||||
"""
|
||||
if not self.index_path.exists():
|
||||
self._entries = {}
|
||||
self._last_updated = ""
|
||||
self._version = self.VERSION
|
||||
return
|
||||
|
||||
with open(self.index_path, encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f)
|
||||
|
||||
if data is None:
|
||||
self._entries = {}
|
||||
self._last_updated = ""
|
||||
self._version = self.VERSION
|
||||
return
|
||||
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(
|
||||
f"Ungültiges Index-Format: Erwartet dict, erhalten {type(data).__name__}"
|
||||
)
|
||||
|
||||
self._version = str(data.get("version", self.VERSION))
|
||||
self._last_updated = str(data.get("last_updated", ""))
|
||||
|
||||
self._entries = {}
|
||||
for artifact_data in data.get("artifacts", []):
|
||||
if isinstance(artifact_data, dict):
|
||||
entry = IndexEntry.from_dict(artifact_data)
|
||||
self._entries[entry.id] = entry
|
||||
|
||||
def save(self) -> None:
|
||||
"""Persistiert den Index auf die Festplatte.
|
||||
|
||||
Aktualisiert den last_updated-Zeitstempel und schreibt alle
|
||||
Einträge im definierten YAML-Format.
|
||||
"""
|
||||
self._last_updated = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
data = {
|
||||
"version": self._version,
|
||||
"last_updated": self._last_updated,
|
||||
"artifacts": [entry.to_dict() for entry in self._entries.values()],
|
||||
}
|
||||
|
||||
# Sicherstellen, dass das Verzeichnis existiert
|
||||
self.index_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with open(self.index_path, "w", encoding="utf-8") as f:
|
||||
yaml.dump(data, f, default_flow_style=False, allow_unicode=True, sort_keys=False)
|
||||
|
||||
def update_entry(self, entry: IndexEntry) -> None:
|
||||
"""Fügt einen Eintrag hinzu oder aktualisiert einen bestehenden.
|
||||
|
||||
Progressive Disclosure wird erzwungen: Der Eintrag darf nur
|
||||
kompakte Metadaten enthalten (kein vollständiger Inhalt).
|
||||
|
||||
Args:
|
||||
entry: Der hinzuzufügende oder zu aktualisierende IndexEntry.
|
||||
|
||||
Raises:
|
||||
ValueError: Wenn der Eintrag keine gültige ID hat.
|
||||
"""
|
||||
if not entry.id:
|
||||
raise ValueError("IndexEntry muss eine nicht-leere ID haben.")
|
||||
|
||||
self._entries[entry.id] = entry
|
||||
|
||||
def search(self, query: str, allowed_scopes: list[str]) -> list[IndexEntry]:
|
||||
"""Textsuche über den Index mit Scope-Filterung.
|
||||
|
||||
Durchsucht Titel, Tags, Summary und ID nach dem Suchbegriff.
|
||||
Liefert nur Einträge aus erlaubten Scopes zurück – Einträge
|
||||
aus nicht-autorisierten Scopes werden weder zurückgegeben
|
||||
noch wird deren Existenz offengelegt.
|
||||
|
||||
Args:
|
||||
query: Suchbegriff (case-insensitive Teilstring-Suche).
|
||||
allowed_scopes: Liste der Scopes, aus denen Ergebnisse
|
||||
zurückgegeben werden dürfen.
|
||||
|
||||
Returns:
|
||||
Liste passender IndexEntry-Objekte, gefiltert nach Scope.
|
||||
"""
|
||||
if not query:
|
||||
return []
|
||||
|
||||
query_lower = query.lower()
|
||||
results: list[IndexEntry] = []
|
||||
|
||||
for entry in self._entries.values():
|
||||
# Scope-Filter: Nur autorisierte Scopes
|
||||
if entry.scope not in allowed_scopes:
|
||||
continue
|
||||
|
||||
# Textsuche über relevante Felder
|
||||
searchable_text = " ".join([
|
||||
entry.title,
|
||||
entry.id,
|
||||
entry.summary,
|
||||
" ".join(entry.tags),
|
||||
]).lower()
|
||||
|
||||
if query_lower in searchable_text:
|
||||
results.append(entry)
|
||||
|
||||
return results
|
||||
|
||||
def get_by_scope(self, scope: str) -> list[IndexEntry]:
|
||||
"""Liefert alle Einträge eines bestimmten Scopes.
|
||||
|
||||
Args:
|
||||
scope: Der Scope, nach dem gefiltert wird (z.B. "bahn", "privat").
|
||||
|
||||
Returns:
|
||||
Liste aller IndexEntry-Objekte mit dem angegebenen Scope.
|
||||
"""
|
||||
return [entry for entry in self._entries.values() if entry.scope == scope]
|
||||
|
||||
def remove_entry(self, entry_id: str) -> bool:
|
||||
"""Entfernt einen Eintrag aus dem Index.
|
||||
|
||||
Args:
|
||||
entry_id: Die ID des zu entfernenden Eintrags.
|
||||
|
||||
Returns:
|
||||
True wenn der Eintrag entfernt wurde, False wenn er nicht existierte.
|
||||
"""
|
||||
if entry_id in self._entries:
|
||||
del self._entries[entry_id]
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_entry(self, entry_id: str) -> IndexEntry | None:
|
||||
"""Liefert einen einzelnen Eintrag nach ID.
|
||||
|
||||
Args:
|
||||
entry_id: Die ID des gesuchten Eintrags.
|
||||
|
||||
Returns:
|
||||
Der IndexEntry oder None, wenn nicht gefunden.
|
||||
"""
|
||||
return self._entries.get(entry_id)
|
||||
@@ -0,0 +1,486 @@
|
||||
"""NoteGraph-Migration für den Wissensspeicher.
|
||||
|
||||
Migriert eine bestehende NoteGraph-Verzeichnisstruktur (decisions, inbox,
|
||||
meetings, people, projects) in den KnowledgeStore. Nutzt die ETLPipeline
|
||||
für die Ingestion und erweitert Artefakte mit fehlenden Metadaten.
|
||||
|
||||
Scope-Konfiguration: Mapping Arbeitskontext → Scope wird aus scopes.yaml geladen.
|
||||
|
||||
Requirements: 3.5, 3.14
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from monorepo.knowledge.artifact import (
|
||||
Artifact,
|
||||
ArtifactMetadata,
|
||||
parse_frontmatter,
|
||||
)
|
||||
from monorepo.knowledge.etl import ETLPipeline, IngestResult
|
||||
from monorepo.knowledge.index import YAMLIndex
|
||||
from monorepo.knowledge.sources.markdown import MarkdownSource
|
||||
from monorepo.knowledge.store import KnowledgeStore
|
||||
from monorepo.models import ScopeConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# NoteGraph-Verzeichnis → ArtifactType Mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
NOTEGRAPH_DIR_TYPE_MAP: dict[str, str] = {
|
||||
"decisions": "decision",
|
||||
"inbox": "note",
|
||||
"meetings": "meeting",
|
||||
"people": "reference",
|
||||
"projects": "project",
|
||||
}
|
||||
|
||||
"""Standard-NoteGraph-Verzeichnisse und ihr zugeordneter Artefakt-Typ."""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Ergebnis-Datenmodelle
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class MigrationResult:
|
||||
"""Ergebnis einer NoteGraph-Migration.
|
||||
|
||||
Attributes:
|
||||
migrated_count: Anzahl erfolgreich migrierter Artefakte.
|
||||
skipped_count: Anzahl übersprungener Artefakte (unverändert oder ohne Frontmatter).
|
||||
errors: Liste der aufgetretenen Fehler mit Details.
|
||||
"""
|
||||
|
||||
migrated_count: int = 0
|
||||
skipped_count: int = 0
|
||||
errors: list[str] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def success(self) -> bool:
|
||||
"""True wenn Migration durchgeführt wurde ohne fatale Fehler."""
|
||||
return self.migrated_count > 0 or (
|
||||
self.migrated_count == 0 and len(self.errors) == 0
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ValidationDetail:
|
||||
"""Detail zu einem fehlenden oder gefundenen Artefakt bei der Validierung."""
|
||||
|
||||
file_name: str
|
||||
expected_type: str
|
||||
found_in_index: bool
|
||||
artifact_id: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class ValidationResult:
|
||||
"""Ergebnis der Migrations-Validierung.
|
||||
|
||||
Attributes:
|
||||
valid: True wenn alle Artefakte im Index auffindbar sind.
|
||||
total: Gesamtanzahl erwarteter Artefakte.
|
||||
found_in_index: Anzahl der im Index gefundenen Artefakte.
|
||||
missing: Liste fehlender Artefakt-IDs.
|
||||
details: Detaillierte Ergebnisse pro Artefakt.
|
||||
"""
|
||||
|
||||
valid: bool = True
|
||||
total: int = 0
|
||||
found_in_index: int = 0
|
||||
missing: list[str] = field(default_factory=list)
|
||||
details: list[ValidationDetail] = field(default_factory=list)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scope-Konfiguration laden
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def load_scope_config(scope_config_path: Path) -> ScopeConfig:
|
||||
"""Lädt die Scope-Konfiguration aus einer YAML-Datei.
|
||||
|
||||
Die Datei definiert das Mapping Arbeitskontext → Scope für den
|
||||
Wissensspeicher (Requirement 3.14).
|
||||
|
||||
Args:
|
||||
scope_config_path: Pfad zur scopes.yaml Datei.
|
||||
|
||||
Returns:
|
||||
ScopeConfig mit dem Scope-Mapping.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: Wenn die Datei nicht existiert.
|
||||
ValueError: Wenn das Format ungültig ist.
|
||||
"""
|
||||
if not scope_config_path.exists():
|
||||
raise FileNotFoundError(
|
||||
f"Scope-Konfiguration nicht gefunden: {scope_config_path}"
|
||||
)
|
||||
|
||||
with open(scope_config_path, encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f)
|
||||
|
||||
if not isinstance(data, dict) or "scopes" not in data:
|
||||
raise ValueError(
|
||||
f"Ungültiges Scope-Konfigurationsformat in {scope_config_path}"
|
||||
)
|
||||
|
||||
scopes: dict[str, dict[str, object]] = {}
|
||||
for entry in data["scopes"]:
|
||||
if isinstance(entry, dict) and "name" in entry:
|
||||
context = str(entry.get("context", entry["name"]))
|
||||
scopes[context] = {
|
||||
"scope": str(entry["name"]),
|
||||
"description": str(entry.get("description", "")),
|
||||
"knowledge_path": str(entry.get("knowledge_path", "")),
|
||||
}
|
||||
|
||||
return ScopeConfig(scopes=scopes)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# NoteGraph-Migrator
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class NoteGraphMigrator:
|
||||
"""Migriert NoteGraph-Verzeichnisstrukturen in den KnowledgeStore.
|
||||
|
||||
Der NoteGraph hat folgende Verzeichnisstruktur:
|
||||
decisions/ → Typ: decision
|
||||
inbox/ → Typ: note
|
||||
meetings/ → Typ: meeting
|
||||
people/ → Typ: reference
|
||||
projects/ → Typ: project
|
||||
|
||||
Jedes Verzeichnis enthält .md-Dateien mit optionalem YAML-Frontmatter.
|
||||
Der Migrator:
|
||||
- Mappt NoteGraph-Verzeichnisse auf Artefakttypen
|
||||
- Nutzt die ETLPipeline für die Ingestion
|
||||
- Ergänzt fehlendes Frontmatter (Typ, source_context)
|
||||
- Bewahrt vorhandene Metadaten
|
||||
- Validiert, dass alle Artefakte im Zielsystem vorhanden und auffindbar sind
|
||||
|
||||
Requirements: 3.5, 3.14
|
||||
"""
|
||||
|
||||
def __init__(self, store: KnowledgeStore, scope_config_path: Path) -> None:
|
||||
"""Initialisiert den NoteGraphMigrator.
|
||||
|
||||
Args:
|
||||
store: Der KnowledgeStore, in den migriert wird.
|
||||
scope_config_path: Pfad zur scopes.yaml für Scope-Mapping.
|
||||
"""
|
||||
self.store = store
|
||||
self.scope_config_path = scope_config_path
|
||||
self.scope_config = load_scope_config(scope_config_path)
|
||||
|
||||
def migrate(self, source_path: Path, context: str) -> MigrationResult:
|
||||
"""Migriert eine NoteGraph-Verzeichnisstruktur in den KnowledgeStore.
|
||||
|
||||
Durchsucht die NoteGraph-Verzeichnisse (decisions, inbox, meetings,
|
||||
people, projects) und migriert alle .md-Dateien. Dateien ohne
|
||||
YAML-Frontmatter werden mit generiertem Frontmatter versehen.
|
||||
|
||||
Args:
|
||||
source_path: Pfad zum NoteGraph-Wurzelverzeichnis.
|
||||
context: Arbeitskontext für die migrierten Artefakte
|
||||
(z.B. "privat", "bahn").
|
||||
|
||||
Returns:
|
||||
MigrationResult mit Statistiken und Fehlerdetails.
|
||||
"""
|
||||
result = MigrationResult()
|
||||
|
||||
if not source_path.exists():
|
||||
result.errors.append(
|
||||
f"NoteGraph-Verzeichnis existiert nicht: {source_path}"
|
||||
)
|
||||
return result
|
||||
|
||||
if not source_path.is_dir():
|
||||
result.errors.append(
|
||||
f"NoteGraph-Pfad ist kein Verzeichnis: {source_path}"
|
||||
)
|
||||
return result
|
||||
|
||||
# Jedes NoteGraph-Unterverzeichnis verarbeiten
|
||||
for dir_name, artifact_type in NOTEGRAPH_DIR_TYPE_MAP.items():
|
||||
sub_dir = source_path / dir_name
|
||||
if not sub_dir.exists() or not sub_dir.is_dir():
|
||||
logger.debug(
|
||||
"NoteGraph-Unterverzeichnis existiert nicht, überspringe: %s",
|
||||
sub_dir,
|
||||
)
|
||||
continue
|
||||
|
||||
sub_result = self._migrate_directory(sub_dir, context, artifact_type)
|
||||
result.migrated_count += sub_result.migrated_count
|
||||
result.skipped_count += sub_result.skipped_count
|
||||
result.errors.extend(sub_result.errors)
|
||||
|
||||
return result
|
||||
|
||||
def _migrate_directory(
|
||||
self, directory: Path, context: str, artifact_type: str
|
||||
) -> MigrationResult:
|
||||
"""Migriert alle .md-Dateien eines NoteGraph-Unterverzeichnisses.
|
||||
|
||||
Bereitet Dateien ohne Frontmatter vor und nutzt dann die ETLPipeline.
|
||||
|
||||
Args:
|
||||
directory: Das zu migrierende Verzeichnis.
|
||||
context: Arbeitskontext.
|
||||
artifact_type: Der zugeordnete Artefakt-Typ.
|
||||
|
||||
Returns:
|
||||
MigrationResult für dieses Verzeichnis.
|
||||
"""
|
||||
result = MigrationResult()
|
||||
|
||||
md_files = sorted(directory.rglob("*.md"))
|
||||
if not md_files:
|
||||
return result
|
||||
|
||||
# Dateien ohne Frontmatter vorbereiten (temporär Frontmatter hinzufügen)
|
||||
prepared_files = self._prepare_files(md_files, context, artifact_type)
|
||||
|
||||
# ETLPipeline für die vorbereiteten Dateien verwenden
|
||||
source = MarkdownSource(directory=directory)
|
||||
# Wir nutzen den Store.ingest direkt, da die Dateien nun Frontmatter haben
|
||||
ingest_result = self.store.ingest(source, context)
|
||||
|
||||
result.migrated_count = ingest_result.updated
|
||||
result.skipped_count = ingest_result.skipped
|
||||
|
||||
for error in ingest_result.errors:
|
||||
result.errors.append(f"{error.file_path}: {error.error}")
|
||||
|
||||
return result
|
||||
|
||||
def _prepare_files(
|
||||
self,
|
||||
md_files: list[Path],
|
||||
context: str,
|
||||
artifact_type: str,
|
||||
) -> list[Path]:
|
||||
"""Bereitet .md-Dateien für die Migration vor.
|
||||
|
||||
Dateien ohne YAML-Frontmatter erhalten generiertes Frontmatter
|
||||
mit dem korrekten Typ und Kontext. Dateien mit existierendem
|
||||
Frontmatter erhalten fehlende Metadaten (type, source_context).
|
||||
|
||||
Args:
|
||||
md_files: Liste der zu verarbeitenden Markdown-Dateien.
|
||||
context: Arbeitskontext.
|
||||
artifact_type: Der Artefakt-Typ basierend auf dem NoteGraph-Verzeichnis.
|
||||
|
||||
Returns:
|
||||
Liste der vorbereiteten Dateipfade.
|
||||
"""
|
||||
prepared: list[Path] = []
|
||||
|
||||
for md_file in md_files:
|
||||
try:
|
||||
content = md_file.read_text(encoding="utf-8")
|
||||
except OSError as e:
|
||||
logger.warning("Kann Datei nicht lesen: %s – %s", md_file, e)
|
||||
continue
|
||||
|
||||
updated_content = self._ensure_frontmatter(
|
||||
content, md_file.stem, context, artifact_type
|
||||
)
|
||||
|
||||
if updated_content != content:
|
||||
md_file.write_text(updated_content, encoding="utf-8")
|
||||
|
||||
prepared.append(md_file)
|
||||
|
||||
return prepared
|
||||
|
||||
def _ensure_frontmatter(
|
||||
self,
|
||||
content: str,
|
||||
file_stem: str,
|
||||
context: str,
|
||||
artifact_type: str,
|
||||
) -> str:
|
||||
"""Stellt sicher, dass der Inhalt gültiges YAML-Frontmatter enthält.
|
||||
|
||||
Wenn kein Frontmatter vorhanden ist, wird eines generiert.
|
||||
Wenn Frontmatter existiert, werden fehlende Felder ergänzt
|
||||
(type, source_context).
|
||||
|
||||
Args:
|
||||
content: Der aktuelle Dateiinhalt.
|
||||
file_stem: Dateiname ohne Erweiterung (für den Titel).
|
||||
context: Arbeitskontext.
|
||||
artifact_type: Der Artefakt-Typ.
|
||||
|
||||
Returns:
|
||||
Der (möglicherweise aktualisierte) Dateiinhalt.
|
||||
"""
|
||||
if content.startswith("---"):
|
||||
# Frontmatter existiert – fehlende Felder ergänzen
|
||||
return self._enrich_existing_frontmatter(
|
||||
content, context, artifact_type
|
||||
)
|
||||
else:
|
||||
# Kein Frontmatter – generieren
|
||||
return self._generate_frontmatter(
|
||||
content, file_stem, context, artifact_type
|
||||
)
|
||||
|
||||
def _enrich_existing_frontmatter(
|
||||
self, content: str, context: str, artifact_type: str
|
||||
) -> str:
|
||||
"""Ergänzt fehlende Felder in existierendem YAML-Frontmatter.
|
||||
|
||||
Setzt 'type' und 'source_context' falls nicht vorhanden.
|
||||
|
||||
Args:
|
||||
content: Dateiinhalt mit existierendem Frontmatter.
|
||||
context: Arbeitskontext.
|
||||
artifact_type: Der zu setzende Artefakt-Typ.
|
||||
|
||||
Returns:
|
||||
Aktualisierter Dateiinhalt.
|
||||
"""
|
||||
import re
|
||||
|
||||
pattern = re.compile(
|
||||
r"\A---\s*\n(.*?)^---\s*$\n?",
|
||||
re.DOTALL | re.MULTILINE,
|
||||
)
|
||||
match = pattern.match(content)
|
||||
if not match:
|
||||
return content
|
||||
|
||||
yaml_text = match.group(1)
|
||||
body = content[match.end():]
|
||||
|
||||
try:
|
||||
metadata: dict[str, Any] = yaml.safe_load(yaml_text) or {}
|
||||
except yaml.YAMLError:
|
||||
return content
|
||||
|
||||
modified = False
|
||||
|
||||
# Typ ergänzen falls fehlend
|
||||
if not metadata.get("type"):
|
||||
metadata["type"] = artifact_type
|
||||
modified = True
|
||||
|
||||
# source_context ergänzen falls fehlend
|
||||
if not metadata.get("source_context"):
|
||||
metadata["source_context"] = context
|
||||
modified = True
|
||||
|
||||
if not modified:
|
||||
return content
|
||||
|
||||
# Frontmatter neu generieren
|
||||
new_yaml = yaml.dump(
|
||||
metadata,
|
||||
default_flow_style=False,
|
||||
allow_unicode=True,
|
||||
sort_keys=False,
|
||||
)
|
||||
return f"---\n{new_yaml}---\n{body}"
|
||||
|
||||
def _generate_frontmatter(
|
||||
self, content: str, file_stem: str, context: str, artifact_type: str
|
||||
) -> str:
|
||||
"""Generiert neues YAML-Frontmatter für eine Datei ohne Frontmatter.
|
||||
|
||||
Erstellt minimales Frontmatter mit Typ, Titel und Quellkontext.
|
||||
|
||||
Args:
|
||||
content: Der Dateiinhalt (ohne Frontmatter).
|
||||
file_stem: Dateiname ohne Erweiterung.
|
||||
context: Arbeitskontext.
|
||||
artifact_type: Der Artefakt-Typ.
|
||||
|
||||
Returns:
|
||||
Dateiinhalt mit vorangestelltem Frontmatter.
|
||||
"""
|
||||
# Titel aus Dateiname ableiten (kebab-case → Title Case)
|
||||
title = file_stem.replace("-", " ").replace("_", " ").title()
|
||||
|
||||
metadata = {
|
||||
"type": artifact_type,
|
||||
"title": title,
|
||||
"tags": [],
|
||||
"source_context": context,
|
||||
"shareable": False,
|
||||
}
|
||||
|
||||
yaml_str = yaml.dump(
|
||||
metadata,
|
||||
default_flow_style=False,
|
||||
allow_unicode=True,
|
||||
sort_keys=False,
|
||||
)
|
||||
return f"---\n{yaml_str}---\n{content}"
|
||||
|
||||
def validate(self, context: str) -> ValidationResult:
|
||||
"""Validiert, dass alle migrierten Artefakte im Index auffindbar sind.
|
||||
|
||||
Prüft für den angegebenen Kontext:
|
||||
- Alle erwarteten Artefakte sind im YAML-Index vorhanden
|
||||
- Metadaten (Typ, Scope) sind korrekt gesetzt
|
||||
- Verknüpfungen sind intakt
|
||||
|
||||
Args:
|
||||
context: Der Arbeitskontext, für den validiert wird.
|
||||
|
||||
Returns:
|
||||
ValidationResult mit detaillierten Ergebnissen.
|
||||
"""
|
||||
result = ValidationResult()
|
||||
|
||||
# Alle Einträge für diesen Scope aus dem Index holen
|
||||
index_entries = self.store.get_index(scope=context)
|
||||
index_ids = {entry.id for entry in index_entries}
|
||||
|
||||
result.total = len(index_entries)
|
||||
result.found_in_index = len(index_entries)
|
||||
|
||||
# Prüfe ob jeder Index-Eintrag eine gültige Datei referenziert
|
||||
for entry in index_entries:
|
||||
artifact_path = self.store.base_path / entry.path if entry.path else None
|
||||
file_exists = artifact_path.exists() if artifact_path else False
|
||||
|
||||
detail = ValidationDetail(
|
||||
file_name=entry.path,
|
||||
expected_type=entry.type,
|
||||
found_in_index=True,
|
||||
artifact_id=entry.id,
|
||||
)
|
||||
result.details.append(detail)
|
||||
|
||||
if not file_exists:
|
||||
result.missing.append(entry.id)
|
||||
result.valid = False
|
||||
|
||||
# Zusätzlich prüfen: Haben alle Dateien einen korrekten Scope?
|
||||
for entry in index_entries:
|
||||
if entry.scope != context:
|
||||
result.missing.append(
|
||||
f"{entry.id} (falscher Scope: {entry.scope}, erwartet: {context})"
|
||||
)
|
||||
result.valid = False
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,11 @@
|
||||
"""Quellstrategien für die ETL-Pipeline des Wissensspeichers.
|
||||
|
||||
Jede Quellstrategie implementiert das Extrahieren von Wissensartefakten
|
||||
aus einem bestimmten Format/Medium (Markdown, Confluence, PDF, etc.).
|
||||
"""
|
||||
|
||||
from monorepo.knowledge.sources.markdown import MarkdownSource
|
||||
|
||||
__all__ = [
|
||||
"MarkdownSource",
|
||||
]
|
||||
@@ -0,0 +1,111 @@
|
||||
"""Markdown-Quellstrategie für die ETL-Pipeline.
|
||||
|
||||
Scannt ein Verzeichnis rekursiv nach .md-Dateien mit YAML-Frontmatter
|
||||
und liefert eine Liste von Artifacts zurück.
|
||||
|
||||
Requirements: 3.2, 3.10, 3.12, 3.19
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from monorepo.knowledge.artifact import Artifact, parse_frontmatter
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SourceError:
|
||||
"""Fehler bei der Verarbeitung einer einzelnen Quelldatei."""
|
||||
|
||||
file_path: str
|
||||
error: str
|
||||
retry: bool = True
|
||||
|
||||
|
||||
@dataclass
|
||||
class MarkdownSource:
|
||||
"""Quellstrategie: Rekursives Scannen eines Verzeichnisses nach Markdown-Dateien.
|
||||
|
||||
Findet alle .md-Dateien im angegebenen Verzeichnis (rekursiv),
|
||||
parsed jede Datei mit parse_frontmatter und liefert die resultierenden
|
||||
Artifacts. Dateien ohne gültiges YAML-Frontmatter werden übersprungen
|
||||
und als Warning geloggt.
|
||||
"""
|
||||
|
||||
directory: Path
|
||||
errors: list[SourceError] = field(default_factory=list)
|
||||
|
||||
def extract(self) -> list[Artifact]:
|
||||
"""Extrahiert alle Artefakte aus dem Quellverzeichnis.
|
||||
|
||||
Durchsucht das Verzeichnis rekursiv nach .md-Dateien,
|
||||
parsed jede Datei und sammelt Fehler ohne abzubrechen.
|
||||
|
||||
Returns:
|
||||
Liste erfolgreich geparster Artifacts.
|
||||
"""
|
||||
self.errors = []
|
||||
artifacts: list[Artifact] = []
|
||||
|
||||
if not self.directory.exists():
|
||||
logger.warning(
|
||||
"Quellverzeichnis existiert nicht: %s", self.directory
|
||||
)
|
||||
self.errors.append(
|
||||
SourceError(
|
||||
file_path=str(self.directory),
|
||||
error=f"Verzeichnis existiert nicht: {self.directory}",
|
||||
retry=True,
|
||||
)
|
||||
)
|
||||
return artifacts
|
||||
|
||||
if not self.directory.is_dir():
|
||||
logger.warning(
|
||||
"Quellpfad ist kein Verzeichnis: %s", self.directory
|
||||
)
|
||||
self.errors.append(
|
||||
SourceError(
|
||||
file_path=str(self.directory),
|
||||
error=f"Pfad ist kein Verzeichnis: {self.directory}",
|
||||
retry=False,
|
||||
)
|
||||
)
|
||||
return artifacts
|
||||
|
||||
md_files = sorted(self.directory.rglob("*.md"))
|
||||
|
||||
for md_file in md_files:
|
||||
try:
|
||||
artifact = parse_frontmatter(md_file)
|
||||
artifacts.append(artifact)
|
||||
except ValueError as e:
|
||||
logger.warning(
|
||||
"Überspringe Datei ohne gültiges Frontmatter: %s – %s",
|
||||
md_file,
|
||||
e,
|
||||
)
|
||||
self.errors.append(
|
||||
SourceError(
|
||||
file_path=str(md_file),
|
||||
error=str(e),
|
||||
retry=False,
|
||||
)
|
||||
)
|
||||
except OSError as e:
|
||||
logger.error(
|
||||
"Fehler beim Lesen der Datei: %s – %s", md_file, e
|
||||
)
|
||||
self.errors.append(
|
||||
SourceError(
|
||||
file_path=str(md_file),
|
||||
error=str(e),
|
||||
retry=True,
|
||||
)
|
||||
)
|
||||
|
||||
return artifacts
|
||||
@@ -0,0 +1,352 @@
|
||||
"""KnowledgeStore – Zentraler Wissensspeicher mit ETL-Pipeline.
|
||||
|
||||
Basiert auf der DB-Wissensdatenbank-ETL-Architektur und bietet:
|
||||
- Ingestion von Wissensartefakten aus verschiedenen Quellen
|
||||
- Progressive-Disclosure-Index (kompakte YAML-Metadaten, Schicht 1)
|
||||
- Volltextsuche mit Scope-basierter Zugriffskontrolle und Relevanz-Sortierung
|
||||
- Graph-Verknüpfungen zwischen Artefakten
|
||||
- Timeout-Handling für Suchanfragen (5 Sekunden)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from monorepo.knowledge.artifact import ArtifactLink, parse_frontmatter, write_artifact
|
||||
from monorepo.knowledge.etl import ETLPipeline, IngestResult
|
||||
from monorepo.knowledge.index import IndexEntry, YAMLIndex
|
||||
from monorepo.knowledge.sources.markdown import MarkdownSource
|
||||
from monorepo.models import ScopeConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Default-Timeout für Suchanfragen in Sekunden
|
||||
SEARCH_TIMEOUT_SECONDS: float = 5.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchResult:
|
||||
"""Ergebnis einer Volltextsuche im Wissensspeicher.
|
||||
|
||||
Enthält den gefundenen Index-Eintrag, einen Relevanz-Score und
|
||||
eine Information, ob die Suche wegen Timeout abgebrochen wurde.
|
||||
|
||||
Attributes:
|
||||
entry: Der gefundene IndexEntry mit kompakten Metadaten.
|
||||
relevance_score: Relevanz-Score zwischen 0.0 und 1.0.
|
||||
Höhere Werte bedeuten bessere Treffer.
|
||||
Scoring-Hierarchie: title > tags > summary > id.
|
||||
partial_results: True wenn die Suche wegen Timeout abgebrochen
|
||||
wurde und möglicherweise nicht alle Ergebnisse enthalten sind.
|
||||
"""
|
||||
|
||||
entry: IndexEntry
|
||||
relevance_score: float = 0.0
|
||||
partial_results: bool = False
|
||||
|
||||
|
||||
def _compute_relevance(query_lower: str, entry: IndexEntry) -> float:
|
||||
"""Berechnet den Relevanz-Score für einen Index-Eintrag.
|
||||
|
||||
Scoring-Hierarchie (von hoch nach niedrig):
|
||||
1. Title-Match (0.8 - 1.0): Query kommt im Titel vor
|
||||
2. Tag-Match (0.6 - 0.79): Query kommt in einem Tag vor
|
||||
3. Summary-Match (0.4 - 0.59): Query kommt im Summary vor
|
||||
4. ID-Match (0.2 - 0.39): Query kommt in der ID vor
|
||||
|
||||
Innerhalb jeder Kategorie wird ein Bonus für exakte Übereinstimmung
|
||||
oder einen höheren Anteil des Matchs vergeben.
|
||||
|
||||
Args:
|
||||
query_lower: Suchbegriff in Kleinbuchstaben.
|
||||
entry: Der zu bewertende IndexEntry.
|
||||
|
||||
Returns:
|
||||
Relevanz-Score zwischen 0.0 und 1.0.
|
||||
"""
|
||||
title_lower = entry.title.lower()
|
||||
tags_lower = [t.lower() for t in entry.tags]
|
||||
summary_lower = entry.summary.lower()
|
||||
id_lower = entry.id.lower()
|
||||
|
||||
score = 0.0
|
||||
|
||||
# 1. Title match (highest priority: 0.8 - 1.0)
|
||||
if query_lower in title_lower:
|
||||
# Bonus for exact match or high coverage
|
||||
if title_lower == query_lower:
|
||||
score = max(score, 1.0)
|
||||
else:
|
||||
coverage = len(query_lower) / max(len(title_lower), 1)
|
||||
score = max(score, 0.8 + coverage * 0.19)
|
||||
|
||||
# 2. Tag match (0.6 - 0.79)
|
||||
for tag in tags_lower:
|
||||
if query_lower in tag:
|
||||
if tag == query_lower:
|
||||
score = max(score, 0.79)
|
||||
else:
|
||||
score = max(score, 0.6 + len(query_lower) / max(len(tag), 1) * 0.18)
|
||||
|
||||
# 3. Summary match (0.4 - 0.59)
|
||||
if query_lower in summary_lower:
|
||||
coverage = len(query_lower) / max(len(summary_lower), 1)
|
||||
score = max(score, 0.4 + coverage * 0.19)
|
||||
|
||||
# 4. ID match (0.2 - 0.39)
|
||||
if query_lower in id_lower:
|
||||
coverage = len(query_lower) / max(len(id_lower), 1)
|
||||
score = max(score, 0.2 + coverage * 0.19)
|
||||
|
||||
return min(score, 1.0)
|
||||
|
||||
|
||||
class KnowledgeStore:
|
||||
"""Zentraler Wissensspeicher mit ETL-Pipeline.
|
||||
|
||||
Der KnowledgeStore verwaltet Wissensartefakte aus allen Arbeitskontexten,
|
||||
indexiert sie im YAML-Index und stellt Suchmechanismen bereit.
|
||||
|
||||
Progressive Disclosure:
|
||||
- Schicht 1 (Index): Kompakte Metadaten (Titel, Tags, Pfade, Summary)
|
||||
- Schicht 2 (Dokument): Vollständige Markdown-Datei mit YAML-Frontmatter
|
||||
|
||||
Agenten lesen zuerst den Index und laden nur bei Bedarf einzelne Dokumente.
|
||||
"""
|
||||
|
||||
INDEX_FILENAME = "_index.yaml"
|
||||
|
||||
def __init__(self, base_path: Path, scope_config: ScopeConfig) -> None:
|
||||
"""Initialisiert den KnowledgeStore.
|
||||
|
||||
Args:
|
||||
base_path: Basispfad des Wissensspeichers
|
||||
(z.B. shared/knowledge-store/).
|
||||
scope_config: Konfiguration des Scope-Mappings
|
||||
(Arbeitskontext → Scope).
|
||||
"""
|
||||
self.base_path = base_path
|
||||
self.scope_config = scope_config
|
||||
self.index = YAMLIndex(base_path / self.INDEX_FILENAME)
|
||||
|
||||
# Index beim Start laden, sofern vorhanden
|
||||
self.index.load()
|
||||
|
||||
def ingest(self, source: MarkdownSource, context: str) -> IngestResult:
|
||||
"""Verarbeitet eine Quelle und legt Artefakte im Wissensspeicher ab.
|
||||
|
||||
Delegiert an die ETLPipeline, die:
|
||||
- Artefakte aus der Quelle extrahiert
|
||||
- Metadaten anreichert (Scope, Datum, Content-Hash)
|
||||
- Inkrementell verarbeitet (nur geänderte Artefakte aktualisiert)
|
||||
- Den YAML-Index im selben Durchlauf aktualisiert
|
||||
- Bei Fehlern erfolgreiche Artefakte behält und Fehler protokolliert
|
||||
|
||||
Args:
|
||||
source: Die Markdown-Quellstrategie mit dem zu scannenden Verzeichnis.
|
||||
context: Der Arbeitskontext, aus dem die Quelle stammt
|
||||
(z.B. "bahn", "privat").
|
||||
|
||||
Returns:
|
||||
IngestResult mit Statistiken (processed, updated, skipped, errors).
|
||||
"""
|
||||
pipeline = ETLPipeline(index=self.index, store_path=self.base_path)
|
||||
return pipeline.ingest(source, context)
|
||||
|
||||
def search(
|
||||
self,
|
||||
query: str,
|
||||
allowed_scopes: list[str],
|
||||
timeout: float | None = None,
|
||||
) -> list[SearchResult]:
|
||||
"""Volltextsuche über den Index mit Relevanz-Sortierung und Timeout.
|
||||
|
||||
Durchsucht den YAML-Index nach dem Suchbegriff und liefert
|
||||
nur Ergebnisse aus autorisierten Scopes, sortiert nach Relevanz.
|
||||
Nicht-autorisierte Artefakte werden weder zurückgegeben noch
|
||||
wird deren Existenz offengelegt.
|
||||
|
||||
Relevanz-Scoring-Hierarchie (höchste zuerst):
|
||||
1. Title-Match
|
||||
2. Tag-Match
|
||||
3. Summary-Match
|
||||
4. ID-Match
|
||||
|
||||
Timeout-Handling:
|
||||
Die Suche bricht nach dem konfigurierten Timeout (Standard:
|
||||
5 Sekunden) ab und liefert die bis dahin gefundenen
|
||||
Teilergebnisse mit partial_results=True.
|
||||
|
||||
Args:
|
||||
query: Suchbegriff für die Volltextsuche.
|
||||
allowed_scopes: Liste der Scopes, aus denen Ergebnisse
|
||||
geliefert werden dürfen.
|
||||
timeout: Timeout in Sekunden (None = SEARCH_TIMEOUT_SECONDS).
|
||||
|
||||
Returns:
|
||||
Liste von SearchResult-Objekten, sortiert nach Relevanz
|
||||
(höchster Score zuerst). Jedes Ergebnis enthält nur Pfade
|
||||
und Metadaten – kein vollständiger Inhalt (Progressive Disclosure).
|
||||
|
||||
Note:
|
||||
Requirements: 3.3, 3.7, 3.9
|
||||
"""
|
||||
if not query:
|
||||
return []
|
||||
|
||||
effective_timeout = timeout if timeout is not None else SEARCH_TIMEOUT_SECONDS
|
||||
query_lower = query.lower()
|
||||
results: list[SearchResult] = []
|
||||
timed_out = False
|
||||
start_time = time.monotonic()
|
||||
|
||||
for entry in self.index.entries.values():
|
||||
# Timeout-Check
|
||||
elapsed = time.monotonic() - start_time
|
||||
if elapsed >= effective_timeout:
|
||||
timed_out = True
|
||||
break
|
||||
|
||||
# Scope-Filter: Nur autorisierte Scopes (niemals Existenz
|
||||
# nicht-autorisierter Artefakte offenlegen)
|
||||
if entry.scope not in allowed_scopes:
|
||||
continue
|
||||
|
||||
# Textsuche über relevante Felder
|
||||
searchable_text = " ".join([
|
||||
entry.title,
|
||||
entry.id,
|
||||
entry.summary,
|
||||
" ".join(entry.tags),
|
||||
]).lower()
|
||||
|
||||
if query_lower in searchable_text:
|
||||
relevance = _compute_relevance(query_lower, entry)
|
||||
results.append(SearchResult(
|
||||
entry=entry,
|
||||
relevance_score=relevance,
|
||||
partial_results=False,
|
||||
))
|
||||
|
||||
# Bei Timeout: Alle bisherigen Ergebnisse als partial markieren
|
||||
if timed_out:
|
||||
for result in results:
|
||||
result.partial_results = True
|
||||
|
||||
# Nach Relevanz sortieren (höchster Score zuerst)
|
||||
results.sort(key=lambda r: r.relevance_score, reverse=True)
|
||||
|
||||
return results
|
||||
|
||||
def get_index(self, scope: str | None = None) -> list[IndexEntry]:
|
||||
"""Liefert den kompakten YAML-Index (Progressive Disclosure Schicht 1).
|
||||
|
||||
Gibt nur Metadaten-Einträge zurück – niemals den vollständigen
|
||||
Dokumentinhalt. Agenten können über die Pfade in den Einträgen
|
||||
bei Bedarf einzelne Dokumente nachladen.
|
||||
|
||||
Args:
|
||||
scope: Optionaler Scope-Filter. Wenn None, werden alle
|
||||
Einträge zurückgegeben.
|
||||
|
||||
Returns:
|
||||
Liste von IndexEntry-Objekten mit kompakten Metadaten.
|
||||
"""
|
||||
if scope is not None:
|
||||
return self.index.get_by_scope(scope)
|
||||
return list(self.index.entries.values())
|
||||
|
||||
def link_artifacts(self, source_id: str, target_id: str, relation: str) -> None:
|
||||
"""Verknüpft zwei Artefakte als gerichtete Graph-Kante.
|
||||
|
||||
Erstellt eine bidirektionale Verknüpfung zwischen zwei Artefakten:
|
||||
- Im Quell-Artefakt wird die Beziehung zum Ziel eingetragen
|
||||
- Im Ziel-Artefakt wird die inverse Beziehung zur Quelle eingetragen
|
||||
- Der YAML-Index wird mit den neuen Link-Informationen aktualisiert
|
||||
- Die YAML-Frontmatter beider Artefakt-Dateien werden aktualisiert
|
||||
|
||||
Args:
|
||||
source_id: ID des Quell-Artefakts.
|
||||
target_id: ID des Ziel-Artefakts.
|
||||
relation: Art der Beziehung (z.B. "implements", "references").
|
||||
|
||||
Raises:
|
||||
ValueError: Wenn source_id oder target_id nicht im Index existiert.
|
||||
"""
|
||||
source_entry = self.index.get_entry(source_id)
|
||||
target_entry = self.index.get_entry(target_id)
|
||||
|
||||
if source_entry is None:
|
||||
raise ValueError(f"Quell-Artefakt nicht im Index gefunden: {source_id}")
|
||||
if target_entry is None:
|
||||
raise ValueError(f"Ziel-Artefakt nicht im Index gefunden: {target_id}")
|
||||
|
||||
# Bidirektionale Verlinkung im Index
|
||||
if target_id not in source_entry.links:
|
||||
source_entry.links.append(target_id)
|
||||
self.index.update_entry(source_entry)
|
||||
|
||||
if source_id not in target_entry.links:
|
||||
target_entry.links.append(source_id)
|
||||
self.index.update_entry(target_entry)
|
||||
|
||||
# YAML-Frontmatter beider Artefakt-Dateien aktualisieren
|
||||
self._update_artifact_frontmatter_link(
|
||||
source_entry, target_entry.path, relation
|
||||
)
|
||||
self._update_artifact_frontmatter_link(
|
||||
target_entry, source_entry.path, relation
|
||||
)
|
||||
|
||||
def _update_artifact_frontmatter_link(
|
||||
self, entry: IndexEntry, link_target_path: str, relation: str
|
||||
) -> None:
|
||||
"""Aktualisiert das YAML-Frontmatter einer Artefakt-Datei mit einem neuen Link.
|
||||
|
||||
Liest die Artefakt-Datei, prüft ob der Link bereits vorhanden ist,
|
||||
fügt ihn bei Bedarf hinzu und schreibt die Datei zurück.
|
||||
|
||||
Wenn die Datei nicht existiert, wird nur der Index aktualisiert
|
||||
(kein Fehler – die Datei wird ggf. später erstellt).
|
||||
|
||||
Args:
|
||||
entry: Der IndexEntry des Artefakts, dessen Datei aktualisiert werden soll.
|
||||
link_target_path: Der Pfad des Ziel-Artefakts (für den links-Eintrag).
|
||||
relation: Art der Beziehung.
|
||||
"""
|
||||
if not entry.path:
|
||||
return
|
||||
|
||||
artifact_path = self.base_path / entry.path
|
||||
if not artifact_path.exists():
|
||||
logger.debug(
|
||||
"Artefakt-Datei existiert nicht, überspringe Frontmatter-Update: %s",
|
||||
artifact_path,
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
artifact = parse_frontmatter(artifact_path)
|
||||
except (ValueError, FileNotFoundError) as e:
|
||||
logger.warning(
|
||||
"Konnte Artefakt-Datei nicht parsen, überspringe Frontmatter-Update: %s (%s)",
|
||||
artifact_path,
|
||||
e,
|
||||
)
|
||||
return
|
||||
|
||||
# Prüfe ob Link bereits vorhanden (Idempotenz)
|
||||
for existing_link in artifact.metadata.links:
|
||||
if existing_link.target == link_target_path and existing_link.relation == relation:
|
||||
return # Link existiert bereits
|
||||
|
||||
# Neuen Link hinzufügen
|
||||
artifact.metadata.links.append(
|
||||
ArtifactLink(target=link_target_path, relation=relation)
|
||||
)
|
||||
|
||||
# Datei zurückschreiben (mit aktualisiertem Frontmatter)
|
||||
write_artifact(artifact, self.base_path)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,274 @@
|
||||
"""Gemeinsame Datenmodelle für das Monorepo-CLI.
|
||||
|
||||
Definiert Enums und Dataclasses für alle Komponenten:
|
||||
- Ordnerstruktur (ProjectInfo)
|
||||
- Repo-Verwaltung (RepoEntry, MigrationPlan)
|
||||
- Sicherheit (SecurityEvent)
|
||||
- Verschlüsselung (MachineContext, EncryptionKey, PasswordManagerConfig)
|
||||
- Wissensspeicher (ScopeConfig)
|
||||
- Föderation (TeamRepoEntry, SharedMirrorConfig, SyncResult, ConflictInfo, IsolationReport, IsolationLeak)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Enums
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class Context(str, Enum):
|
||||
"""Arbeitskontexte im Monorepo."""
|
||||
|
||||
PRIVAT = "privat"
|
||||
DHIVE = "dhive"
|
||||
BAHN = "bahn"
|
||||
SHARED = "shared"
|
||||
|
||||
|
||||
class RepoMode(str, Enum):
|
||||
"""Einbindungsmodus für externe Repositories."""
|
||||
|
||||
READ_ONLY = "read-only"
|
||||
UPSTREAM = "upstream"
|
||||
|
||||
|
||||
class ArtifactType(str, Enum):
|
||||
"""Typen von Wissensartefakten im Knowledge Store."""
|
||||
|
||||
DECISION = "decision"
|
||||
NOTE = "note"
|
||||
MEETING = "meeting"
|
||||
REFERENCE = "reference"
|
||||
PATTERN = "pattern"
|
||||
|
||||
|
||||
class KeySource(str, Enum):
|
||||
"""Quelle für Verschlüsselungsschlüssel."""
|
||||
|
||||
KEYRING = "keyring"
|
||||
FILE = "file"
|
||||
PASSWORD_MANAGER = "password-manager"
|
||||
|
||||
|
||||
class SyncDirection(str, Enum):
|
||||
"""Synchronisationsrichtung für Team-Repos."""
|
||||
|
||||
BIDIRECTIONAL = "bidirectional"
|
||||
HUB_TO_SPOKE = "hub-to-spoke"
|
||||
SPOKE_TO_HUB = "spoke-to-hub"
|
||||
|
||||
|
||||
class SyncFrequency(str, Enum):
|
||||
"""Frequenz der Synchronisation mit Team-Repos."""
|
||||
|
||||
ON_PUSH = "on-push"
|
||||
HOURLY = "hourly"
|
||||
DAILY = "daily"
|
||||
MANUAL = "manual"
|
||||
|
||||
|
||||
class ConflictStrategy(str, Enum):
|
||||
"""Strategie zur Konfliktauflösung bei Synchronisation."""
|
||||
|
||||
TEAM_WINS = "team-wins"
|
||||
HUB_WINS = "hub-wins"
|
||||
MANUAL = "manual"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dataclasses – Ordnerstruktur
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProjectInfo:
|
||||
"""Informationen über ein Projekt im Monorepo."""
|
||||
|
||||
name: str
|
||||
context: str
|
||||
path: Path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dataclasses – Repo-Verwaltung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class RepoEntry:
|
||||
"""Konfigurationseintrag für ein eingebundenes externes Repository."""
|
||||
|
||||
name: str
|
||||
url: str
|
||||
mode: Literal["read-only", "upstream"]
|
||||
target: str
|
||||
pinned: str
|
||||
mechanism: Literal["subtree", "submodule"] = "subtree"
|
||||
|
||||
|
||||
@dataclass
|
||||
class MigrationPlan:
|
||||
"""Plan für die Migration eines Repositories ins Monorepo."""
|
||||
|
||||
source_repo: str
|
||||
target_context: str
|
||||
target_name: str
|
||||
mode: Literal["direct", "subtree", "upstream"]
|
||||
dependencies: list[str] = field(default_factory=list)
|
||||
order: int = 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dataclasses – Sicherheit
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class SecurityEvent:
|
||||
"""Protokolleintrag für einen Sicherheitsvorfall (Zugriffsverletzung)."""
|
||||
|
||||
timestamp: datetime
|
||||
requesting_context: str
|
||||
target_context: str
|
||||
resource: str
|
||||
action: Literal["read", "write", "execute"]
|
||||
outcome: Literal["denied", "allowed"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dataclasses – Wissensspeicher
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScopeConfig:
|
||||
"""Mapping von Arbeitskontext zu Scope im Wissensspeicher.
|
||||
|
||||
Beispiel: {"privat": {"scope": "privat", "paths": ["privat/"]}}
|
||||
"""
|
||||
|
||||
scopes: dict[str, dict[str, object]] = field(default_factory=dict)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dataclasses – Verschlüsselung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class PasswordManagerConfig:
|
||||
"""Konfiguration für die Passwort-Manager-Integration."""
|
||||
|
||||
type: Literal["bitwarden", "1password", "keepass"]
|
||||
vault: str
|
||||
entry_prefix: str = "monorepo-key-"
|
||||
|
||||
|
||||
@dataclass
|
||||
class MachineContext:
|
||||
"""Maschinenkontext: Zuordnung eines Rechners zu autorisierten Kontexten."""
|
||||
|
||||
name: str
|
||||
description: str
|
||||
authorized_contexts: list[str] = field(default_factory=list)
|
||||
key_source: Literal["keyring", "file", "password-manager"] = "keyring"
|
||||
password_manager: PasswordManagerConfig | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class EncryptionKey:
|
||||
"""Verschlüsselungsschlüssel für einen Arbeitskontext."""
|
||||
|
||||
context: str
|
||||
key_id: str
|
||||
key_type: Literal["gpg", "symmetric"]
|
||||
source: Literal["keyring", "file", "password-manager"]
|
||||
|
||||
|
||||
@dataclass
|
||||
class OnboardingResult:
|
||||
"""Ergebnis der Maschineneinrichtung (Onboarding).
|
||||
|
||||
Beschreibt welche Schlüssel erfolgreich installiert und welche
|
||||
Kontexte für die Maschine freigeschaltet wurden.
|
||||
"""
|
||||
|
||||
success: bool
|
||||
machine_name: str
|
||||
authorized_contexts: list[str] = field(default_factory=list)
|
||||
installed_keys: list[str] = field(default_factory=list)
|
||||
errors: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dataclasses – Föderation (Team-Repos)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class SharedMirrorConfig:
|
||||
"""Konfiguration für gespiegelte shared-Dateien in einem Team-Repo."""
|
||||
|
||||
enabled: bool
|
||||
paths: list[str] = field(default_factory=list)
|
||||
mode: Literal["read-only"] = "read-only"
|
||||
|
||||
|
||||
@dataclass
|
||||
class TeamRepoEntry:
|
||||
"""Konfigurationseintrag für ein Team-Repository (Spoke)."""
|
||||
|
||||
context: str
|
||||
url: str
|
||||
branch: str
|
||||
sync_direction: Literal["bidirectional", "hub-to-spoke", "spoke-to-hub"]
|
||||
sync_frequency: Literal["on-push", "hourly", "daily", "manual"]
|
||||
shared_mirror: SharedMirrorConfig | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncResult:
|
||||
"""Ergebnis einer Synchronisationsoperation."""
|
||||
|
||||
success: bool
|
||||
context: str
|
||||
direction: Literal["pull", "push", "full"]
|
||||
commits_synced: int
|
||||
conflicts: list[ConflictInfo] = field(default_factory=list)
|
||||
timestamp: datetime = field(default_factory=datetime.now)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConflictInfo:
|
||||
"""Details zu einem Merge-Konflikt bei der Synchronisation."""
|
||||
|
||||
file_path: str
|
||||
conflict_type: Literal["content", "rename", "delete-modify"]
|
||||
source: Literal["monorepo", "team-repo"]
|
||||
details: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class IsolationReport:
|
||||
"""Ergebnis der Isolationsprüfung eines Team-Repos."""
|
||||
|
||||
context: str
|
||||
is_isolated: bool
|
||||
leaks: list[IsolationLeak] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class IsolationLeak:
|
||||
"""Gefundene Referenz auf einen fremden Kontext in einem Team-Repo."""
|
||||
|
||||
file_path: str
|
||||
line_number: int
|
||||
leaked_context: str
|
||||
leak_type: Literal["path", "env_var", "config_ref", "comment"]
|
||||
@@ -0,0 +1,552 @@
|
||||
"""OrchestratorAdapter – Integration des AI-Orchestrators mit der Monorepo-Struktur.
|
||||
|
||||
Adaptiert den bestehenden AI-Orchestrator für kontextgebundenes Arbeiten:
|
||||
- Ermittlung des Arbeitskontexts aus Task-Metadaten (Labels/Projektzuordnung)
|
||||
- Erstellung kontextgebundener Workspaces
|
||||
- Prompt-Aufbau mit Wissenskontext aus dem YAML-Index
|
||||
- Env-Loading mit optionaler Entschlüsselung via SecretEncryptionManager
|
||||
- Workspace-Root-Auflösung aus WORKFLOW.md-Konfiguration
|
||||
- Fehlerbehandlung bei fehlendem Kontext oder Out-of-Context-Zugriff
|
||||
|
||||
Requirements: 7.1, 7.2, 7.3, 7.4, 7.5, 7.6
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from monorepo.audit import AuditLogger
|
||||
from monorepo.models import Context, SecurityEvent
|
||||
from monorepo.security import ContextGuard
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from monorepo.encryption import SecretEncryptionManager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Exceptions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ContextResolutionError(Exception):
|
||||
"""Kein Arbeitskontext konnte aus den Task-Metadaten ermittelt werden.
|
||||
|
||||
Requirement 7.6: Task wird nicht gestartet, Fehlermeldung an den Nutzer.
|
||||
"""
|
||||
|
||||
def __init__(self, task_id: str, message: str) -> None:
|
||||
self.task_id = task_id
|
||||
self.message = message
|
||||
super().__init__(f"[{task_id}] {message}")
|
||||
|
||||
|
||||
class ContextViolationError(Exception):
|
||||
"""Zugriff außerhalb des zugewiesenen Kontexts erkannt.
|
||||
|
||||
Requirement 7.5: Task wird abgebrochen, Vorfall protokolliert,
|
||||
Nutzer wird benachrichtigt.
|
||||
"""
|
||||
|
||||
def __init__(self, task_id: str, context: str, access_path: str) -> None:
|
||||
self.task_id = task_id
|
||||
self.context = context
|
||||
self.access_path = access_path
|
||||
super().__init__(
|
||||
f"[{task_id}] Kontextverletzung: Zugriff auf '{access_path}' "
|
||||
f"aus Kontext '{context}' nicht erlaubt"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Task-Datenmodell
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class Task:
|
||||
"""Repräsentiert einen Task aus dem Orchestrator.
|
||||
|
||||
Attributes:
|
||||
id: Eindeutiger Task-Identifier.
|
||||
title: Titel/Zusammenfassung des Tasks.
|
||||
labels: Labels zur Kontextzuordnung (z.B. ["dhive"], ["bahn"]).
|
||||
project: Projektzuordnung (z.B. "dhive/my-project").
|
||||
description: Optionale Aufgabenbeschreibung.
|
||||
"""
|
||||
|
||||
id: str
|
||||
title: str
|
||||
labels: list[str] = field(default_factory=list)
|
||||
project: str = ""
|
||||
description: str = ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ExecutionEnvironment – Vollständige Ausführungsumgebung für einen Task
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExecutionEnvironment:
|
||||
"""Vollständige Ausführungsumgebung für einen kontextgebundenen Task.
|
||||
|
||||
Kombiniert Workspace-Pfad, entschlüsselte Umgebungsvariablen,
|
||||
relevante Wissensartefakte und den zusammengesetzten Prompt.
|
||||
|
||||
Attributes:
|
||||
workspace_path: Pfad zum kontextgebundenen Arbeitsverzeichnis.
|
||||
context: Der zugewiesene Arbeitskontext (z.B. "dhive", "bahn").
|
||||
env_vars: Entschlüsselte Umgebungsvariablen des Kontexts.
|
||||
knowledge_artifacts: Pfade zu relevanten Wissensartefakten.
|
||||
prompt: Vollständiger Agenten-Prompt mit Wissenskontext.
|
||||
"""
|
||||
|
||||
workspace_path: Path
|
||||
context: str
|
||||
env_vars: dict[str, str] = field(default_factory=dict)
|
||||
knowledge_artifacts: list[str] = field(default_factory=list)
|
||||
prompt: str = ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Kontextmapping
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Bekannte Kontexte für die Auflösung aus Labels/Projektzuordnung
|
||||
KNOWN_CONTEXTS: set[str] = {ctx.value for ctx in Context if ctx != Context.SHARED}
|
||||
# Shared ist ebenfalls ein gültiger Kontext für Tasks
|
||||
ALL_CONTEXTS: set[str] = {ctx.value for ctx in Context}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OrchestratorAdapter
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class OrchestratorAdapter:
|
||||
"""Adaptiert den AI-Orchestrator für die Monorepo-Struktur.
|
||||
|
||||
Stellt kontextgebundene Arbeitsumgebungen bereit:
|
||||
- Workspace-Erstellung unter dem korrekten Kontextordner
|
||||
- Prompt-Aufbau mit relevantem Wissenskontext aus dem YAML-Index
|
||||
- Env-Loading mit optionaler Entschlüsselung via SecretEncryptionManager
|
||||
- Workspace-Root-Auflösung aus WORKFLOW.md-Konfiguration
|
||||
- Sicherheitsprüfung bei Dateizugriffen über ContextGuard
|
||||
|
||||
Requirements: 7.1, 7.2, 7.3, 7.4, 7.5, 7.6
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
root_path: Path,
|
||||
context_guard: ContextGuard,
|
||||
audit_logger: AuditLogger | None = None,
|
||||
knowledge_index_path: Path | None = None,
|
||||
encryption_manager: SecretEncryptionManager | None = None,
|
||||
workflow_config_path: Path | None = None,
|
||||
) -> None:
|
||||
"""Initialisiert den OrchestratorAdapter.
|
||||
|
||||
Args:
|
||||
root_path: Wurzelverzeichnis des Monorepos.
|
||||
context_guard: ContextGuard-Instanz für Zugriffskontrolle.
|
||||
audit_logger: Optionaler AuditLogger für Verletzungs-Protokollierung.
|
||||
Falls None, wird ein Standard-Logger mit Default-Pfad erzeugt.
|
||||
knowledge_index_path: Pfad zur YAML-Index-Datei des Wissensspeichers.
|
||||
Falls None, wird `root_path / shared/knowledge-store/_index.yaml` verwendet.
|
||||
encryption_manager: Optionaler SecretEncryptionManager für die
|
||||
Entschlüsselung von .env-Dateien. Falls None, werden .env-Dateien
|
||||
direkt gelesen (unverschlüsselt oder bereits entschlüsselt).
|
||||
workflow_config_path: Optionaler Pfad zur WORKFLOW.md-Konfiguration.
|
||||
Falls None, wird `root_path / WORKFLOW.md` verwendet.
|
||||
"""
|
||||
self.root_path = root_path.resolve()
|
||||
self.context_guard = context_guard
|
||||
self.audit_logger = audit_logger or AuditLogger(
|
||||
self.root_path / ".audit" / "access.log"
|
||||
)
|
||||
self.encryption_manager = encryption_manager
|
||||
|
||||
if knowledge_index_path is None:
|
||||
knowledge_index_path = (
|
||||
self.root_path / "shared" / "knowledge-store" / "_index.yaml"
|
||||
)
|
||||
self._knowledge_index_path = knowledge_index_path
|
||||
|
||||
if workflow_config_path is None:
|
||||
workflow_config_path = self.root_path / "WORKFLOW.md"
|
||||
self._workflow_config_path = workflow_config_path
|
||||
|
||||
def resolve_context(self, task: Task) -> str:
|
||||
"""Ermittelt den Arbeitskontext aus Task-Metadaten.
|
||||
|
||||
Prüft in folgender Reihenfolge:
|
||||
1. Labels: Sucht nach bekannten Kontextnamen in den Task-Labels.
|
||||
2. Projektzuordnung: Extrahiert den Kontext aus dem Projektpfad
|
||||
(erster Pfadteil, z.B. "dhive/my-project" → "dhive").
|
||||
|
||||
Requirement 7.1: Kontext aus Task-Metadaten ermitteln.
|
||||
Requirement 7.6: Kein Kontext → Task nicht starten, Fehlermeldung.
|
||||
|
||||
Args:
|
||||
task: Der zu verarbeitende Task.
|
||||
|
||||
Returns:
|
||||
Name des ermittelten Arbeitskontexts (z.B. "dhive", "bahn", "privat").
|
||||
|
||||
Raises:
|
||||
ContextResolutionError: Wenn kein Kontext ermittelt werden kann.
|
||||
"""
|
||||
# 1. Labels prüfen
|
||||
for label in task.labels:
|
||||
label_lower = label.lower().strip()
|
||||
if label_lower in ALL_CONTEXTS:
|
||||
logger.info(
|
||||
"Task '%s': Kontext '%s' aus Label ermittelt", task.id, label_lower
|
||||
)
|
||||
return label_lower
|
||||
|
||||
# 2. Projektzuordnung prüfen
|
||||
if task.project:
|
||||
# Projekt kann als "kontext/projektname" oder direkt als Kontextname vorliegen
|
||||
project_parts = task.project.strip().split("/")
|
||||
first_part = project_parts[0].lower().strip()
|
||||
if first_part in ALL_CONTEXTS:
|
||||
logger.info(
|
||||
"Task '%s': Kontext '%s' aus Projektzuordnung ermittelt",
|
||||
task.id,
|
||||
first_part,
|
||||
)
|
||||
return first_part
|
||||
|
||||
# Kein Kontext ermittelbar → Req 7.6
|
||||
raise ContextResolutionError(
|
||||
task_id=task.id,
|
||||
message=(
|
||||
f"Kein Arbeitskontext aus Task-Metadaten ermittelbar. "
|
||||
f"Labels: {task.labels}, Projekt: '{task.project}'. "
|
||||
f"Erwartet wird ein Label oder Projektpräfix aus: {sorted(ALL_CONTEXTS)}"
|
||||
),
|
||||
)
|
||||
|
||||
def create_workspace(self, task: Task, context: str) -> Path:
|
||||
"""Erstellt ein kontextgebundenes Arbeitsverzeichnis.
|
||||
|
||||
Erstellt das Workspace-Verzeichnis unter dem ermittelten Kontextordner:
|
||||
`<monorepo_root>/<context>/.workspaces/<task_id>/`
|
||||
|
||||
Requirement 7.2: Workspace-Root unter dem Kontextordner.
|
||||
|
||||
Args:
|
||||
task: Der Task, für den das Workspace erstellt wird.
|
||||
context: Der ermittelte Arbeitskontext.
|
||||
|
||||
Returns:
|
||||
Pfad zum erstellten Workspace-Verzeichnis.
|
||||
|
||||
Raises:
|
||||
ContextResolutionError: Wenn der Kontext ungültig ist.
|
||||
"""
|
||||
if context not in ALL_CONTEXTS:
|
||||
raise ContextResolutionError(
|
||||
task_id=task.id,
|
||||
message=f"Ungültiger Kontext '{context}'. Gültig: {sorted(ALL_CONTEXTS)}",
|
||||
)
|
||||
|
||||
# Workspace-Verzeichnis: <root>/<kontext>/.workspaces/<task_id>/
|
||||
# Sanitize task ID für Verzeichnisnamen
|
||||
safe_task_id = self._sanitize_dirname(task.id)
|
||||
workspace_path = self.root_path / context / ".workspaces" / safe_task_id
|
||||
|
||||
workspace_path.mkdir(parents=True, exist_ok=True)
|
||||
logger.info(
|
||||
"Task '%s': Workspace erstellt unter '%s'", task.id, workspace_path
|
||||
)
|
||||
|
||||
return workspace_path
|
||||
|
||||
def build_prompt(self, task: Task, context: str) -> str:
|
||||
"""Erstellt den Agenten-Prompt mit Wissenskontext.
|
||||
|
||||
Der Prompt enthält:
|
||||
- Workspace-Pfad und Kontextinformationen
|
||||
- Verfügbare Wissensartefakte aus dem YAML-Index (für den Kontext)
|
||||
- Sicherheitsbeschränkungen (nur Zugriff innerhalb des Kontexts)
|
||||
- Aufgabenbeschreibung aus dem Task
|
||||
|
||||
Requirement 7.1: Arbeitsverzeichnis auf Kontextordner beschränken.
|
||||
Requirement 7.3: Nur .env des eigenen Kontexts laden.
|
||||
|
||||
Args:
|
||||
task: Der zu bearbeitende Task.
|
||||
context: Der ermittelte Arbeitskontext.
|
||||
|
||||
Returns:
|
||||
Vollständiger Agenten-Prompt als String.
|
||||
"""
|
||||
workspace_path = self.root_path / context / ".workspaces" / self._sanitize_dirname(task.id)
|
||||
context_root = self.root_path / context
|
||||
|
||||
# Basis-Prompt aufbauen
|
||||
prompt_parts: list[str] = [
|
||||
f"# Task: {task.title}",
|
||||
"",
|
||||
"## Arbeitskontext",
|
||||
f"- Kontext: {context}",
|
||||
f"- Workspace: {workspace_path}",
|
||||
f"- Kontextordner: {context_root}",
|
||||
"",
|
||||
"## Sicherheitsbeschränkungen",
|
||||
f"- Du darfst NUR auf Dateien innerhalb von '{context_root}' und "
|
||||
f"'{self.root_path / 'shared'}' zugreifen.",
|
||||
f"- Lade ausschließlich die .env-Datei des Kontexts '{context}'.",
|
||||
"- Zugriff auf andere Kontextordner ist VERBOTEN und führt zum Task-Abbruch.",
|
||||
"",
|
||||
]
|
||||
|
||||
# Wissenskontext einfügen
|
||||
knowledge_section = self.inject_knowledge(context)
|
||||
if knowledge_section:
|
||||
prompt_parts.append("## Verfügbare Wissensartefakte")
|
||||
prompt_parts.append(knowledge_section)
|
||||
prompt_parts.append("")
|
||||
|
||||
# Aufgabenbeschreibung
|
||||
if task.description:
|
||||
prompt_parts.append("## Aufgabe")
|
||||
prompt_parts.append(task.description)
|
||||
prompt_parts.append("")
|
||||
|
||||
return "\n".join(prompt_parts)
|
||||
|
||||
def inject_knowledge(self, context: str) -> str:
|
||||
"""Fügt relevante Artefakt-Pfade aus dem YAML-Index hinzu.
|
||||
|
||||
Liest den YAML-Index und liefert Pfade zu Artefakten, die
|
||||
im aktuellen Kontext oder im shared-Scope verfügbar sind.
|
||||
|
||||
Args:
|
||||
context: Der aktive Arbeitskontext.
|
||||
|
||||
Returns:
|
||||
Formatierter String mit verfügbaren Artefakt-Pfaden oder
|
||||
leerer String wenn kein Index vorhanden ist.
|
||||
"""
|
||||
if not self._knowledge_index_path.exists():
|
||||
logger.debug("Kein Wissens-Index gefunden unter: %s", self._knowledge_index_path)
|
||||
return ""
|
||||
|
||||
try:
|
||||
import yaml
|
||||
|
||||
with open(self._knowledge_index_path, encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f)
|
||||
|
||||
if not isinstance(data, dict) or "artifacts" not in data:
|
||||
return ""
|
||||
|
||||
# Nur Artefakte aus dem eigenen Kontext und shared anzeigen
|
||||
allowed_scopes = [context, Context.SHARED.value]
|
||||
relevant_artifacts: list[str] = []
|
||||
|
||||
for artifact in data.get("artifacts", []):
|
||||
if not isinstance(artifact, dict):
|
||||
continue
|
||||
scope = artifact.get("scope", "")
|
||||
if scope in allowed_scopes:
|
||||
title = artifact.get("title", "Unbenannt")
|
||||
path = artifact.get("path", "")
|
||||
tags = artifact.get("tags", [])
|
||||
tags_str = f" [{', '.join(tags)}]" if tags else ""
|
||||
relevant_artifacts.append(f"- {title}{tags_str}: {path}")
|
||||
|
||||
if not relevant_artifacts:
|
||||
return ""
|
||||
|
||||
return "\n".join(relevant_artifacts)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("Fehler beim Lesen des Wissens-Index: %s", e)
|
||||
return ""
|
||||
|
||||
def validate_access(self, task: Task, context: str, target_path: Path) -> None:
|
||||
"""Prüft ob ein Dateizugriff innerhalb des zugewiesenen Kontexts liegt.
|
||||
|
||||
Verwendet den ContextGuard zur Zugriffsprüfung. Bei Verletzung:
|
||||
- Task wird abgebrochen (Exception)
|
||||
- Vorfall wird im AuditLog protokolliert
|
||||
- Nutzer wird über die Exception informiert
|
||||
|
||||
Requirement 7.5: Out-of-Context → abbrechen, protokollieren, benachrichtigen.
|
||||
|
||||
Args:
|
||||
task: Der laufende Task.
|
||||
context: Der zugewiesene Arbeitskontext.
|
||||
target_path: Der Pfad, auf den zugegriffen werden soll.
|
||||
|
||||
Raises:
|
||||
ContextViolationError: Wenn der Zugriff außerhalb des Kontexts liegt.
|
||||
"""
|
||||
if not self.context_guard.check_access(context, target_path):
|
||||
# Zugriffsverletzung protokollieren
|
||||
target_path_str = str(target_path)
|
||||
event = SecurityEvent(
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
requesting_context=context,
|
||||
target_context=self._detect_target_context(target_path),
|
||||
resource=target_path_str,
|
||||
action="read",
|
||||
outcome="denied",
|
||||
)
|
||||
self.audit_logger.log_violation(event)
|
||||
|
||||
logger.error(
|
||||
"Task '%s': Kontextverletzung! Zugriff auf '%s' aus Kontext '%s' verweigert.",
|
||||
task.id,
|
||||
target_path_str,
|
||||
context,
|
||||
)
|
||||
|
||||
raise ContextViolationError(
|
||||
task_id=task.id,
|
||||
context=context,
|
||||
access_path=target_path_str,
|
||||
)
|
||||
|
||||
def load_context_env(self, context: str) -> dict[str, str]:
|
||||
"""Lädt ausschließlich die .env-Datei des zugewiesenen Kontexts.
|
||||
|
||||
Bei konfiguriertem SecretEncryptionManager wird die .env-Datei
|
||||
entschlüsselt gelesen. Ohne Manager wird sie direkt gelesen
|
||||
(unverschlüsselt oder bereits durch git-crypt entschlüsselt).
|
||||
|
||||
Requirement 7.3: Nur .env des eigenen Kontexts laden, keine
|
||||
Umgebungsvariablen anderer Kontexte. Entschlüsselung via
|
||||
SecretEncryptionManager.
|
||||
|
||||
Args:
|
||||
context: Der zugewiesene Arbeitskontext.
|
||||
|
||||
Returns:
|
||||
Dict mit den Umgebungsvariablen des Kontexts.
|
||||
|
||||
Raises:
|
||||
ValueError: Wenn der Kontext ungültig ist.
|
||||
FileNotFoundError: Wenn die .env-Datei nicht existiert.
|
||||
PermissionError: Wenn die Entschlüsselung fehlschlägt
|
||||
(Maschinenkontext nicht autorisiert).
|
||||
"""
|
||||
if self.encryption_manager is not None:
|
||||
return self._load_env_with_decryption(context)
|
||||
return self.context_guard.load_env(context)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Private Hilfsmethoden
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _sanitize_dirname(name: str) -> str:
|
||||
"""Macht einen String sicher für die Verwendung als Verzeichnisname.
|
||||
|
||||
Ersetzt ungültige Zeichen durch Unterstriche.
|
||||
|
||||
Args:
|
||||
name: Der zu bereinigende String.
|
||||
|
||||
Returns:
|
||||
Sicherer Verzeichnisname.
|
||||
"""
|
||||
# Einfache Sanitierung: Nur alphanumerische Zeichen, Bindestrich, Unterstrich
|
||||
safe = "".join(c if c.isalnum() or c in "-_" else "_" for c in name)
|
||||
return safe or "unnamed"
|
||||
|
||||
def _detect_target_context(self, target_path: Path) -> str:
|
||||
"""Ermittelt den Kontext eines Zielpfads für die Protokollierung.
|
||||
|
||||
Args:
|
||||
target_path: Der Pfad, dessen Kontext ermittelt werden soll.
|
||||
|
||||
Returns:
|
||||
Name des Zielkontexts oder "unknown".
|
||||
"""
|
||||
try:
|
||||
resolved = target_path.resolve() if target_path.is_absolute() else target_path
|
||||
rel_path = resolved.relative_to(self.root_path)
|
||||
parts = rel_path.parts
|
||||
if parts and parts[0] in ALL_CONTEXTS:
|
||||
return parts[0]
|
||||
except (ValueError, OSError):
|
||||
pass
|
||||
return "unknown"
|
||||
|
||||
def _load_env_with_decryption(self, context: str) -> dict[str, str]:
|
||||
"""Lädt die .env-Datei des Kontexts mit Entschlüsselung via SecretEncryptionManager.
|
||||
|
||||
Ermittelt den Pfad zur .env-Datei aus der ContextGuard-Konfiguration,
|
||||
prüft die Autorisierung des Maschinenkontexts und entschlüsselt die
|
||||
Datei bei Bedarf.
|
||||
|
||||
Requirement 7.3: Nur .env des eigenen Kontexts laden, entschlüsselt
|
||||
via SecretEncryptionManager.
|
||||
Requirement 9.3: Kontext-eigener Schlüssel für Entschlüsselung.
|
||||
Requirement 9.4: Maschinenkontext-basierte Autorisierung.
|
||||
|
||||
Args:
|
||||
context: Der Arbeitskontext, dessen .env geladen werden soll.
|
||||
|
||||
Returns:
|
||||
Dict mit den entschlüsselten Umgebungsvariablen.
|
||||
|
||||
Raises:
|
||||
ValueError: Wenn der Kontext ungültig ist.
|
||||
FileNotFoundError: Wenn die .env-Datei nicht existiert.
|
||||
PermissionError: Wenn die Entschlüsselung fehlschlägt
|
||||
(Maschinenkontext nicht autorisiert).
|
||||
"""
|
||||
assert self.encryption_manager is not None # Caller garantiert
|
||||
|
||||
# Autorisierungsprüfung
|
||||
if not self.encryption_manager.is_authorized(context):
|
||||
raise PermissionError(
|
||||
f"Maschinenkontext '{self.encryption_manager.machine_context.name}' "
|
||||
f"ist nicht für Kontext '{context}' autorisiert. "
|
||||
f"Env-Loading verweigert."
|
||||
)
|
||||
|
||||
# .env-Pfad aus der ContextGuard-Konfiguration ermitteln
|
||||
ctx_config = self.context_guard.get_context_config(context)
|
||||
env_file_rel = ctx_config.get("env_file")
|
||||
if not env_file_rel:
|
||||
raise ValueError(
|
||||
f"Kein env_file für Kontext '{context}' in der Konfiguration definiert."
|
||||
)
|
||||
|
||||
env_path = self.root_path / env_file_rel
|
||||
if not env_path.exists():
|
||||
raise FileNotFoundError(
|
||||
f".env-Datei für Kontext '{context}' nicht gefunden: {env_path}"
|
||||
)
|
||||
|
||||
# Entschlüsselung via SecretEncryptionManager
|
||||
result = self.encryption_manager.decrypt_file(env_path)
|
||||
|
||||
if result.success and result.content is not None:
|
||||
# Entschlüsselter Inhalt als Text parsen
|
||||
content_text = result.content.decode("utf-8", errors="replace")
|
||||
return ContextGuard._parse_env_content(content_text)
|
||||
|
||||
# Entschlüsselung fehlgeschlagen – Fallback auf ContextGuard.load_env
|
||||
# (Datei könnte bereits im Klartext vorliegen nach git-crypt unlock)
|
||||
logger.debug(
|
||||
"Entschlüsselung für Kontext '%s' nicht erfolgreich (%s). "
|
||||
"Fallback auf ContextGuard.load_env.",
|
||||
context,
|
||||
result.error or "unbekannter Fehler",
|
||||
)
|
||||
return self.context_guard.load_env(context)
|
||||
@@ -0,0 +1,835 @@
|
||||
"""Externes-Repo-Manager für das Monorepo.
|
||||
|
||||
Verwaltet die Einbindung externer Repositories via Git-Subtree (Standard)
|
||||
oder Git-Submodule, sowie die Synchronisation mit Upstream-Remotes.
|
||||
|
||||
Verantwortung: Einbindung, Synchronisation und Schutzmechanismen für
|
||||
externe Repositories (Read-Only und Upstream).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import stat
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from monorepo.models import ConflictInfo, RepoEntry, SyncResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Template für den pre-commit Hook, der Read-Only-Verzeichnisse schützt.
|
||||
# Platzhalter: {protected_paths} wird durch eine Shell-Array-Definition ersetzt.
|
||||
_PRE_COMMIT_HOOK_TEMPLATE = """\
|
||||
#!/bin/sh
|
||||
# --- Monorepo Read-Only-Schutz (auto-generiert) ---
|
||||
# Verhindert Commits, die Dateien in geschützten Read-Only-Verzeichnissen ändern.
|
||||
|
||||
{protected_paths}
|
||||
|
||||
staged_files=$(git diff --cached --name-only)
|
||||
|
||||
for file in $staged_files; do
|
||||
for protected in "${{PROTECTED_PATHS[@]}}"; do
|
||||
case "$file" in
|
||||
"$protected"/*)
|
||||
echo "FEHLER: Repo '$protected' ist read-only. Schreibzugriff auf '$file' nicht erlaubt." >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
done
|
||||
# --- Ende Read-Only-Schutz ---
|
||||
"""
|
||||
|
||||
# Marker-Kommentare für den Read-Only-Abschnitt im Hook
|
||||
_HOOK_START_MARKER = "# --- Monorepo Read-Only-Schutz (auto-generiert) ---"
|
||||
_HOOK_END_MARKER = "# --- Ende Read-Only-Schutz ---"
|
||||
|
||||
|
||||
class RepoManager:
|
||||
"""Verwaltet externe und Upstream-Repository-Einbindungen.
|
||||
|
||||
Unterstützt zwei Einbindungsmechanismen:
|
||||
- Git-Subtree (Standard): Flache Checkouts, bidirektionale Sync möglich
|
||||
- Git-Submodule: Separate Repository-Referenz
|
||||
|
||||
Und zwei Betriebsmodi:
|
||||
- read-only: Nur Pull von Remote, lokale Änderungen werden blockiert
|
||||
- upstream: Bidirektionale Synchronisation (Pull/Push)
|
||||
"""
|
||||
|
||||
def __init__(self, config_path: Path, monorepo_root: Path | None = None) -> None:
|
||||
"""Initialisiert den RepoManager.
|
||||
|
||||
Args:
|
||||
config_path: Pfad zur repos.yaml Konfigurationsdatei.
|
||||
monorepo_root: Wurzelverzeichnis des Monorepos. Wenn None,
|
||||
wird das übergeordnete Verzeichnis der Config verwendet.
|
||||
"""
|
||||
self.config_path = config_path
|
||||
self.monorepo_root = monorepo_root or config_path.parent.parent.parent
|
||||
self.repos = self._load_repos_config(config_path)
|
||||
|
||||
def _load_repos_config(self, config_path: Path) -> list[RepoEntry]:
|
||||
"""Lädt die Repo-Konfiguration aus repos.yaml.
|
||||
|
||||
Args:
|
||||
config_path: Pfad zur repos.yaml.
|
||||
|
||||
Returns:
|
||||
Liste aller konfigurierten RepoEntry-Objekte.
|
||||
"""
|
||||
if not config_path.exists():
|
||||
logger.warning("Repo-Konfiguration nicht gefunden: %s", config_path)
|
||||
return []
|
||||
|
||||
with open(config_path, encoding="utf-8") as f:
|
||||
data: dict[str, Any] = yaml.safe_load(f) or {}
|
||||
|
||||
entries: list[RepoEntry] = []
|
||||
for item in data.get("repos", []):
|
||||
entries.append(
|
||||
RepoEntry(
|
||||
name=item["name"],
|
||||
url=item["url"],
|
||||
mode=item.get("mode", "read-only"),
|
||||
target=item["target"],
|
||||
pinned=item.get("pinned", "main"),
|
||||
mechanism=item.get("mechanism", "subtree"),
|
||||
)
|
||||
)
|
||||
return entries
|
||||
|
||||
def _find_repo(self, repo_name: str) -> RepoEntry | None:
|
||||
"""Findet einen Repo-Eintrag anhand des Namens.
|
||||
|
||||
Args:
|
||||
repo_name: Name des gesuchten Repos.
|
||||
|
||||
Returns:
|
||||
Der RepoEntry oder None wenn nicht gefunden.
|
||||
"""
|
||||
for entry in self.repos:
|
||||
if entry.name == repo_name:
|
||||
return entry
|
||||
return None
|
||||
|
||||
def _run_git(
|
||||
self, args: list[str], cwd: Path | None = None
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
"""Führt einen Git-Befehl aus.
|
||||
|
||||
Args:
|
||||
args: Git-Argumente (ohne 'git' Präfix).
|
||||
cwd: Arbeitsverzeichnis für den Befehl.
|
||||
|
||||
Returns:
|
||||
CompletedProcess mit stdout/stderr.
|
||||
|
||||
Raises:
|
||||
subprocess.CalledProcessError: Bei Fehler im Git-Befehl.
|
||||
"""
|
||||
cmd = ["git"] + args
|
||||
logger.debug("Ausführen: %s (cwd=%s)", " ".join(cmd), cwd or self.monorepo_root)
|
||||
return subprocess.run(
|
||||
cmd,
|
||||
cwd=cwd or self.monorepo_root,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
|
||||
def add_repo(self, entry: RepoEntry) -> None:
|
||||
"""Bindet ein externes Repo via Git-Subtree oder Submodule ein.
|
||||
|
||||
Subtree (Standard):
|
||||
git subtree add --prefix=<target> <url> <pinned> --squash
|
||||
|
||||
Submodule:
|
||||
git submodule add <url> <target>
|
||||
git -C <target> checkout <pinned>
|
||||
|
||||
Args:
|
||||
entry: RepoEntry mit URL, Zielverzeichnis, Mechanismus, etc.
|
||||
|
||||
Raises:
|
||||
ValueError: Wenn das Zielverzeichnis bereits existiert.
|
||||
subprocess.CalledProcessError: Bei Fehler im Git-Befehl.
|
||||
"""
|
||||
target_path = self.monorepo_root / entry.target
|
||||
|
||||
if target_path.exists():
|
||||
raise ValueError(
|
||||
f"Zielverzeichnis '{entry.target}' existiert bereits. "
|
||||
f"Repo '{entry.name}' kann nicht eingebunden werden."
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Binde Repo '%s' ein via %s → %s",
|
||||
entry.name,
|
||||
entry.mechanism,
|
||||
entry.target,
|
||||
)
|
||||
|
||||
if entry.mechanism == "subtree":
|
||||
self._add_subtree(entry)
|
||||
elif entry.mechanism == "submodule":
|
||||
self._add_submodule(entry)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unbekannter Mechanismus '{entry.mechanism}'. "
|
||||
f"Erlaubt: 'subtree', 'submodule'."
|
||||
)
|
||||
|
||||
# Repo in Konfiguration aufnehmen (falls noch nicht vorhanden)
|
||||
if not self._find_repo(entry.name):
|
||||
self.repos.append(entry)
|
||||
self._save_repos_config()
|
||||
|
||||
logger.info("Repo '%s' erfolgreich eingebunden.", entry.name)
|
||||
|
||||
def _add_subtree(self, entry: RepoEntry) -> None:
|
||||
"""Bindet ein Repo als Git-Subtree ein.
|
||||
|
||||
Args:
|
||||
entry: RepoEntry mit den Subtree-Parametern.
|
||||
"""
|
||||
self._run_git([
|
||||
"subtree",
|
||||
"add",
|
||||
f"--prefix={entry.target}",
|
||||
entry.url,
|
||||
entry.pinned,
|
||||
"--squash",
|
||||
])
|
||||
|
||||
def _add_submodule(self, entry: RepoEntry) -> None:
|
||||
"""Bindet ein Repo als Git-Submodule ein.
|
||||
|
||||
Args:
|
||||
entry: RepoEntry mit den Submodule-Parametern.
|
||||
"""
|
||||
self._run_git([
|
||||
"submodule",
|
||||
"add",
|
||||
entry.url,
|
||||
entry.target,
|
||||
])
|
||||
|
||||
# Auf gepinnte Version auschecken
|
||||
target_path = self.monorepo_root / entry.target
|
||||
self._run_git(["checkout", entry.pinned], cwd=target_path)
|
||||
|
||||
def sync(self, repo_name: str) -> SyncResult:
|
||||
"""Synchronisiert ein Repo mit seinem Remote.
|
||||
|
||||
Für Read-Only-Repos: Pull von Upstream auf gepinnte Version.
|
||||
Für Upstream-Repos: Bidirektionale Synchronisation (Pull dann Push).
|
||||
|
||||
Bei Merge-Konflikten wird die Synchronisation abgebrochen,
|
||||
der Konflikt protokolliert und manuelle Auflösung ermöglicht.
|
||||
|
||||
Args:
|
||||
repo_name: Name des zu synchronisierenden Repos.
|
||||
|
||||
Returns:
|
||||
SyncResult mit Status (Erfolg/Misserfolg/unbekannt),
|
||||
Richtung, Anzahl synchronisierter Commits und ggf. Konflikte.
|
||||
"""
|
||||
entry = self._find_repo(repo_name)
|
||||
if entry is None:
|
||||
logger.error("Repo '%s' nicht in Konfiguration gefunden.", repo_name)
|
||||
return SyncResult(
|
||||
success=False,
|
||||
context=repo_name,
|
||||
direction="pull",
|
||||
commits_synced=0,
|
||||
conflicts=[
|
||||
ConflictInfo(
|
||||
file_path="",
|
||||
conflict_type="content",
|
||||
source="monorepo",
|
||||
details=f"Repo '{repo_name}' nicht in Konfiguration gefunden.",
|
||||
)
|
||||
],
|
||||
timestamp=datetime.now(),
|
||||
)
|
||||
|
||||
if entry.mode == "upstream":
|
||||
return self._sync_upstream(entry)
|
||||
else:
|
||||
return self._sync_readonly(entry)
|
||||
|
||||
def _sync_readonly(self, entry: RepoEntry) -> SyncResult:
|
||||
"""Synchronisiert ein Read-Only-Repo (nur Pull).
|
||||
|
||||
Args:
|
||||
entry: RepoEntry des zu synchronisierenden Repos.
|
||||
|
||||
Returns:
|
||||
SyncResult mit Ergebnis der Pull-Operation.
|
||||
"""
|
||||
logger.info("Synchronisiere Read-Only-Repo '%s' auf '%s'.", entry.name, entry.pinned)
|
||||
|
||||
try:
|
||||
if entry.mechanism == "subtree":
|
||||
result = self._subtree_pull(entry)
|
||||
else:
|
||||
result = self._submodule_pull(entry)
|
||||
|
||||
commits = self._count_commits_from_output(result.stdout)
|
||||
logger.info(
|
||||
"Repo '%s' erfolgreich synchronisiert (%d Commits).",
|
||||
entry.name,
|
||||
commits,
|
||||
)
|
||||
return SyncResult(
|
||||
success=True,
|
||||
context=entry.name,
|
||||
direction="pull",
|
||||
commits_synced=commits,
|
||||
conflicts=[],
|
||||
timestamp=datetime.now(),
|
||||
)
|
||||
|
||||
except subprocess.CalledProcessError as e:
|
||||
return self._handle_sync_error(entry, e, direction="pull")
|
||||
|
||||
def _sync_upstream(self, entry: RepoEntry) -> SyncResult:
|
||||
"""Synchronisiert ein Upstream-Repo bidirektional (Pull + Push).
|
||||
|
||||
Ablauf:
|
||||
1. Pull von Upstream (Subtree-Pull oder Submodule-Update)
|
||||
2. Bei Merge-Konflikt: Abbruch, Protokollierung, manuelle Auflösung
|
||||
3. Push lokaler Änderungen zu Upstream
|
||||
|
||||
Args:
|
||||
entry: RepoEntry des Upstream-Repos.
|
||||
|
||||
Returns:
|
||||
SyncResult mit Gesamtergebnis der bidirektionalen Sync.
|
||||
"""
|
||||
logger.info("Bidirektionale Synchronisation für Upstream-Repo '%s'.", entry.name)
|
||||
total_commits = 0
|
||||
|
||||
# Phase 1: Pull von Upstream
|
||||
try:
|
||||
if entry.mechanism == "subtree":
|
||||
pull_result = self._subtree_pull(entry)
|
||||
else:
|
||||
pull_result = self._submodule_pull(entry)
|
||||
|
||||
total_commits += self._count_commits_from_output(pull_result.stdout)
|
||||
logger.info("Pull von '%s' erfolgreich.", entry.name)
|
||||
|
||||
except subprocess.CalledProcessError as e:
|
||||
# Prüfe ob es ein Merge-Konflikt ist
|
||||
if self._is_merge_conflict(e):
|
||||
return self._handle_merge_conflict(entry, e)
|
||||
return self._handle_sync_error(entry, e, direction="full")
|
||||
|
||||
# Phase 2: Push lokaler Änderungen
|
||||
try:
|
||||
if entry.mechanism == "subtree":
|
||||
push_result = self._subtree_push(entry)
|
||||
else:
|
||||
push_result = self._submodule_push(entry)
|
||||
|
||||
total_commits += self._count_commits_from_output(push_result.stdout)
|
||||
logger.info("Push zu '%s' erfolgreich.", entry.name)
|
||||
|
||||
except subprocess.CalledProcessError as e:
|
||||
if self._is_merge_conflict(e):
|
||||
return self._handle_merge_conflict(entry, e)
|
||||
return self._handle_sync_error(entry, e, direction="push")
|
||||
|
||||
return SyncResult(
|
||||
success=True,
|
||||
context=entry.name,
|
||||
direction="full",
|
||||
commits_synced=total_commits,
|
||||
conflicts=[],
|
||||
timestamp=datetime.now(),
|
||||
)
|
||||
|
||||
def _subtree_pull(self, entry: RepoEntry) -> subprocess.CompletedProcess[str]:
|
||||
"""Führt einen git subtree pull aus.
|
||||
|
||||
Args:
|
||||
entry: RepoEntry mit Subtree-Parametern.
|
||||
|
||||
Returns:
|
||||
CompletedProcess mit Ergebnis.
|
||||
"""
|
||||
return self._run_git([
|
||||
"subtree",
|
||||
"pull",
|
||||
f"--prefix={entry.target}",
|
||||
entry.url,
|
||||
entry.pinned,
|
||||
"--squash",
|
||||
])
|
||||
|
||||
def _subtree_push(self, entry: RepoEntry) -> subprocess.CompletedProcess[str]:
|
||||
"""Führt einen git subtree push aus.
|
||||
|
||||
Args:
|
||||
entry: RepoEntry mit Subtree-Parametern.
|
||||
|
||||
Returns:
|
||||
CompletedProcess mit Ergebnis.
|
||||
"""
|
||||
return self._run_git([
|
||||
"subtree",
|
||||
"push",
|
||||
f"--prefix={entry.target}",
|
||||
entry.url,
|
||||
entry.pinned,
|
||||
])
|
||||
|
||||
def _submodule_pull(self, entry: RepoEntry) -> subprocess.CompletedProcess[str]:
|
||||
"""Aktualisiert ein Submodule auf die gepinnte Version.
|
||||
|
||||
Args:
|
||||
entry: RepoEntry mit Submodule-Parametern.
|
||||
|
||||
Returns:
|
||||
CompletedProcess mit Ergebnis.
|
||||
"""
|
||||
target_path = self.monorepo_root / entry.target
|
||||
|
||||
# Fetch latest from remote
|
||||
self._run_git(["fetch", "origin"], cwd=target_path)
|
||||
|
||||
# Checkout pinned version
|
||||
return self._run_git(["checkout", entry.pinned], cwd=target_path)
|
||||
|
||||
def _submodule_push(self, entry: RepoEntry) -> subprocess.CompletedProcess[str]:
|
||||
"""Pusht lokale Änderungen in einem Submodule zu Upstream.
|
||||
|
||||
Args:
|
||||
entry: RepoEntry mit Submodule-Parametern.
|
||||
|
||||
Returns:
|
||||
CompletedProcess mit Ergebnis.
|
||||
"""
|
||||
target_path = self.monorepo_root / entry.target
|
||||
return self._run_git(["push", "origin", entry.pinned], cwd=target_path)
|
||||
|
||||
def _is_merge_conflict(self, error: subprocess.CalledProcessError) -> bool:
|
||||
"""Prüft ob ein Git-Fehler ein Merge-Konflikt ist.
|
||||
|
||||
Args:
|
||||
error: Der aufgetretene CalledProcessError.
|
||||
|
||||
Returns:
|
||||
True wenn es sich um einen Merge-Konflikt handelt.
|
||||
"""
|
||||
conflict_indicators = [
|
||||
"CONFLICT",
|
||||
"merge conflict",
|
||||
"Automatic merge failed",
|
||||
"fix conflicts",
|
||||
]
|
||||
combined_output = (error.stdout or "") + (error.stderr or "")
|
||||
return any(indicator in combined_output for indicator in conflict_indicators)
|
||||
|
||||
def _handle_merge_conflict(
|
||||
self, entry: RepoEntry, error: subprocess.CalledProcessError
|
||||
) -> SyncResult:
|
||||
"""Behandelt einen Merge-Konflikt: Abbruch, Protokollierung.
|
||||
|
||||
Bei Merge-Konflikten wird:
|
||||
1. Die Merge-Operation abgebrochen (git merge --abort)
|
||||
2. Der Konflikt protokolliert
|
||||
3. Ein SyncResult mit Konfliktdetails zurückgegeben
|
||||
|
||||
Args:
|
||||
entry: RepoEntry des betroffenen Repos.
|
||||
error: Der CalledProcessError mit Konfliktdetails.
|
||||
|
||||
Returns:
|
||||
SyncResult mit success=False und Konfliktinformationen.
|
||||
"""
|
||||
logger.warning(
|
||||
"Merge-Konflikt bei Synchronisation von '%s'. Breche ab.",
|
||||
entry.name,
|
||||
)
|
||||
|
||||
# Merge abbrechen um sauberen Zustand herzustellen
|
||||
try:
|
||||
self._run_git(["merge", "--abort"])
|
||||
except subprocess.CalledProcessError:
|
||||
# Falls kein Merge aktiv ist, ignorieren
|
||||
logger.debug("Kein aktiver Merge zum Abbrechen.")
|
||||
|
||||
# Konfliktdetails aus Fehlerausgabe extrahieren
|
||||
combined_output = (error.stdout or "") + (error.stderr or "")
|
||||
conflict_files = self._extract_conflict_files(combined_output)
|
||||
|
||||
conflicts = [
|
||||
ConflictInfo(
|
||||
file_path=f,
|
||||
conflict_type="content",
|
||||
source="monorepo",
|
||||
details=f"Merge-Konflikt in '{f}' bei Sync von '{entry.name}'.",
|
||||
)
|
||||
for f in conflict_files
|
||||
] or [
|
||||
ConflictInfo(
|
||||
file_path="",
|
||||
conflict_type="content",
|
||||
source="monorepo",
|
||||
details=f"Merge-Konflikt bei Synchronisation von '{entry.name}': "
|
||||
f"{combined_output[:200]}",
|
||||
)
|
||||
]
|
||||
|
||||
return SyncResult(
|
||||
success=False,
|
||||
context=entry.name,
|
||||
direction="full" if entry.mode == "upstream" else "pull",
|
||||
commits_synced=0,
|
||||
conflicts=conflicts,
|
||||
timestamp=datetime.now(),
|
||||
)
|
||||
|
||||
def _handle_sync_error(
|
||||
self,
|
||||
entry: RepoEntry,
|
||||
error: subprocess.CalledProcessError,
|
||||
direction: str,
|
||||
) -> SyncResult:
|
||||
"""Behandelt einen allgemeinen Sync-Fehler.
|
||||
|
||||
Protokolliert den Fehler in der Sync-Error-Log-Datei und gibt
|
||||
ein SyncResult mit success=False zurück. Der lokale Stand wird
|
||||
nicht verändert (Req 4.9).
|
||||
|
||||
Args:
|
||||
entry: RepoEntry des betroffenen Repos.
|
||||
error: Der aufgetretene Fehler.
|
||||
direction: Richtung der fehlgeschlagenen Operation.
|
||||
|
||||
Returns:
|
||||
SyncResult mit success=False und Fehlerdetails.
|
||||
"""
|
||||
combined_output = (error.stdout or "") + (error.stderr or "")
|
||||
error_detail = combined_output.strip()[:300] or "Unbekannter Fehler"
|
||||
|
||||
logger.error(
|
||||
"Synchronisation von '%s' fehlgeschlagen: %s",
|
||||
entry.name,
|
||||
error_detail,
|
||||
)
|
||||
|
||||
# Fehler in Audit-Datei protokollieren (Req 4.9)
|
||||
self.log_sync_error(
|
||||
repo_name=entry.name,
|
||||
error_reason=error_detail,
|
||||
direction=direction,
|
||||
)
|
||||
|
||||
return SyncResult(
|
||||
success=False,
|
||||
context=entry.name,
|
||||
direction=direction, # type: ignore[arg-type]
|
||||
commits_synced=0,
|
||||
conflicts=[
|
||||
ConflictInfo(
|
||||
file_path="",
|
||||
conflict_type="content",
|
||||
source="monorepo",
|
||||
details=f"Synchronisation fehlgeschlagen: {error_detail}",
|
||||
)
|
||||
],
|
||||
timestamp=datetime.now(),
|
||||
)
|
||||
|
||||
def _extract_conflict_files(self, output: str) -> list[str]:
|
||||
"""Extrahiert Dateinamen aus Git-Merge-Konflikt-Ausgabe.
|
||||
|
||||
Args:
|
||||
output: Die kombinierte stdout/stderr-Ausgabe von Git.
|
||||
|
||||
Returns:
|
||||
Liste der Dateipfade mit Konflikten.
|
||||
"""
|
||||
conflict_files: list[str] = []
|
||||
for line in output.splitlines():
|
||||
# Git zeigt "CONFLICT (content): Merge conflict in <file>"
|
||||
if "CONFLICT" in line and "Merge conflict in" in line:
|
||||
parts = line.split("Merge conflict in")
|
||||
if len(parts) > 1:
|
||||
file_path = parts[1].strip().rstrip(".")
|
||||
conflict_files.append(file_path)
|
||||
# Git zeigt auch "CONFLICT (modify/delete): <file> deleted in ..."
|
||||
elif "CONFLICT" in line and ":" in line:
|
||||
parts = line.split(":")
|
||||
if len(parts) > 1:
|
||||
# Versuche Dateiname zu extrahieren
|
||||
segment = parts[1].strip()
|
||||
tokens = segment.split()
|
||||
if tokens:
|
||||
conflict_files.append(tokens[0])
|
||||
return conflict_files
|
||||
|
||||
def _count_commits_from_output(self, output: str) -> int:
|
||||
"""Zählt synchronisierte Commits anhand der Git-Ausgabe.
|
||||
|
||||
Sucht nach typischen Git-Meldungen wie "X commits" oder
|
||||
Shortstat-Zeilen. Gibt 1 zurück wenn der Output auf
|
||||
Aktivität hindeutet, sonst 0.
|
||||
|
||||
Args:
|
||||
output: Die stdout-Ausgabe des Git-Befehls.
|
||||
|
||||
Returns:
|
||||
Geschätzte Anzahl synchronisierter Commits.
|
||||
"""
|
||||
if not output:
|
||||
return 0
|
||||
|
||||
# Suche nach "X files changed" als Indikator für Änderungen
|
||||
for line in output.splitlines():
|
||||
if "files changed" in line or "file changed" in line:
|
||||
return 1
|
||||
if "Already up to date" in line or "Already up-to-date" in line:
|
||||
return 0
|
||||
|
||||
# Wenn es Output gibt aber keine klaren Indikatoren,
|
||||
# gehen wir von mindestens einer Änderung aus
|
||||
if output.strip():
|
||||
return 1
|
||||
return 0
|
||||
|
||||
def _save_repos_config(self) -> None:
|
||||
"""Speichert die aktuelle Repo-Konfiguration in repos.yaml."""
|
||||
data = {
|
||||
"repos": [
|
||||
{
|
||||
"name": entry.name,
|
||||
"url": entry.url,
|
||||
"mode": entry.mode,
|
||||
"target": entry.target,
|
||||
"pinned": entry.pinned,
|
||||
"mechanism": entry.mechanism,
|
||||
}
|
||||
for entry in self.repos
|
||||
]
|
||||
}
|
||||
|
||||
self.config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(self.config_path, "w", encoding="utf-8") as f:
|
||||
yaml.dump(data, f, default_flow_style=False, allow_unicode=True)
|
||||
|
||||
logger.debug("Repo-Konfiguration gespeichert: %s", self.config_path)
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Read-Only-Schutzmechanismus (Req 4.2, 4.5)
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
def protect_readonly(self, repo_path: Path) -> None:
|
||||
"""Installiert einen Git pre-commit Hook zum Schutz eines Read-Only-Verzeichnisses.
|
||||
|
||||
Der Hook prüft bei jedem Commit, ob gestagte Dateien innerhalb des
|
||||
geschützten Verzeichnisses liegen. Ist das der Fall, wird der Commit
|
||||
mit einer Fehlermeldung abgebrochen, die den Repo-Namen und den
|
||||
Read-Only-Status benennt.
|
||||
|
||||
Wenn bereits ein pre-commit Hook existiert, wird der Schutzabschnitt
|
||||
angehängt bzw. aktualisiert. Die geschützten Pfade werden als
|
||||
Shell-Array in den Hook geschrieben.
|
||||
|
||||
Args:
|
||||
repo_path: Absoluter Pfad zum geschützten Verzeichnis (Target-Pfad
|
||||
relativ zum Monorepo-Root wird automatisch berechnet).
|
||||
|
||||
Raises:
|
||||
ValueError: Wenn der repo_path nicht innerhalb des Monorepo-Roots liegt.
|
||||
"""
|
||||
# Berechne den relativen Pfad zum Monorepo-Root
|
||||
try:
|
||||
relative_target = repo_path.relative_to(self.monorepo_root)
|
||||
except ValueError:
|
||||
raise ValueError(
|
||||
f"Pfad '{repo_path}' liegt nicht innerhalb des "
|
||||
f"Monorepo-Roots '{self.monorepo_root}'."
|
||||
)
|
||||
|
||||
target_str = str(relative_target).replace("\\", "/")
|
||||
|
||||
# Git-Hook-Verzeichnis ermitteln
|
||||
hooks_dir = self.monorepo_root / ".git" / "hooks"
|
||||
hooks_dir.mkdir(parents=True, exist_ok=True)
|
||||
hook_path = hooks_dir / "pre-commit"
|
||||
|
||||
# Geschützte Pfade sammeln (aus bestehendem Hook + neuer Pfad)
|
||||
protected_paths = self._get_existing_protected_paths(hook_path)
|
||||
if target_str not in protected_paths:
|
||||
protected_paths.append(target_str)
|
||||
|
||||
# Hook-Script generieren und schreiben
|
||||
self._write_pre_commit_hook(hook_path, protected_paths)
|
||||
|
||||
logger.info(
|
||||
"Read-Only-Schutz installiert für '%s' (Hook: %s).",
|
||||
target_str,
|
||||
hook_path,
|
||||
)
|
||||
|
||||
def install_all_readonly_hooks(self) -> None:
|
||||
"""Installiert Read-Only-Schutz für alle als read-only konfigurierten Repos.
|
||||
|
||||
Iteriert über alle konfigurierten Repos und installiert für jene mit
|
||||
mode="read-only" den pre-commit-Hook-Schutz. Bereits geschützte Pfade
|
||||
werden nicht doppelt eingetragen.
|
||||
"""
|
||||
readonly_repos = [entry for entry in self.repos if entry.mode == "read-only"]
|
||||
|
||||
if not readonly_repos:
|
||||
logger.info("Keine Read-Only-Repos konfiguriert – kein Hook notwendig.")
|
||||
return
|
||||
|
||||
for entry in readonly_repos:
|
||||
repo_path = self.monorepo_root / entry.target
|
||||
self.protect_readonly(repo_path)
|
||||
|
||||
logger.info(
|
||||
"Read-Only-Schutz für %d Repos installiert: %s",
|
||||
len(readonly_repos),
|
||||
", ".join(e.name for e in readonly_repos),
|
||||
)
|
||||
|
||||
def _get_existing_protected_paths(self, hook_path: Path) -> list[str]:
|
||||
"""Extrahiert bereits geschützte Pfade aus einem bestehenden Hook.
|
||||
|
||||
Args:
|
||||
hook_path: Pfad zur pre-commit Hook-Datei.
|
||||
|
||||
Returns:
|
||||
Liste der bereits geschützten Pfade (relativ zum Monorepo-Root).
|
||||
"""
|
||||
if not hook_path.exists():
|
||||
return []
|
||||
|
||||
content = hook_path.read_text(encoding="utf-8")
|
||||
|
||||
# Suche nach PROTECTED_PATHS-Array im bestehenden Hook
|
||||
paths: list[str] = []
|
||||
in_array = False
|
||||
for line in content.splitlines():
|
||||
if "PROTECTED_PATHS=(" in line:
|
||||
in_array = True
|
||||
# Pfade in derselben Zeile: PROTECTED_PATHS=("path1" "path2")
|
||||
inner = line.split("(", 1)[1].rstrip(")")
|
||||
for token in inner.split():
|
||||
cleaned = token.strip('"').strip("'")
|
||||
if cleaned:
|
||||
paths.append(cleaned)
|
||||
if ")" in line:
|
||||
in_array = False
|
||||
continue
|
||||
if in_array:
|
||||
if ")" in line:
|
||||
inner = line.split(")")[0]
|
||||
for token in inner.split():
|
||||
cleaned = token.strip('"').strip("'")
|
||||
if cleaned:
|
||||
paths.append(cleaned)
|
||||
in_array = False
|
||||
else:
|
||||
for token in line.split():
|
||||
cleaned = token.strip('"').strip("'")
|
||||
if cleaned:
|
||||
paths.append(cleaned)
|
||||
|
||||
return paths
|
||||
|
||||
def _write_pre_commit_hook(self, hook_path: Path, protected_paths: list[str]) -> None:
|
||||
"""Schreibt oder aktualisiert den pre-commit Hook mit Schutzmechanismus.
|
||||
|
||||
Wenn bereits ein Hook existiert, wird der Read-Only-Abschnitt
|
||||
(zwischen Start- und End-Marker) ersetzt. Andernfalls wird ein
|
||||
neuer Hook erstellt.
|
||||
|
||||
Args:
|
||||
hook_path: Pfad zur Hook-Datei.
|
||||
protected_paths: Liste der zu schützenden relativen Pfade.
|
||||
"""
|
||||
# Generiere Shell-Array-Definition
|
||||
quoted_paths = " ".join(f'"{p}"' for p in protected_paths)
|
||||
paths_definition = f'PROTECTED_PATHS=({quoted_paths})'
|
||||
|
||||
# Generiere neuen Hook-Abschnitt
|
||||
new_section = _PRE_COMMIT_HOOK_TEMPLATE.format(protected_paths=paths_definition)
|
||||
|
||||
if hook_path.exists():
|
||||
existing_content = hook_path.read_text(encoding="utf-8")
|
||||
|
||||
if _HOOK_START_MARKER in existing_content:
|
||||
# Ersetze den bestehenden Abschnitt
|
||||
start_idx = existing_content.index(_HOOK_START_MARKER)
|
||||
end_idx = existing_content.index(_HOOK_END_MARKER) + len(_HOOK_END_MARKER)
|
||||
updated_content = (
|
||||
existing_content[:start_idx]
|
||||
+ new_section.strip()
|
||||
+ existing_content[end_idx:]
|
||||
)
|
||||
else:
|
||||
# Hänge den Abschnitt an den bestehenden Hook an
|
||||
updated_content = existing_content.rstrip() + "\n\n" + new_section
|
||||
else:
|
||||
updated_content = new_section
|
||||
|
||||
hook_path.write_text(updated_content, encoding="utf-8")
|
||||
|
||||
# Hook ausführbar machen (wichtig für Unix-Systeme)
|
||||
current_mode = hook_path.stat().st_mode
|
||||
hook_path.chmod(current_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Sync-Fehlerprotokollierung (Req 4.9)
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
def log_sync_error(
|
||||
self,
|
||||
repo_name: str,
|
||||
error_reason: str,
|
||||
direction: str = "pull",
|
||||
) -> None:
|
||||
"""Protokolliert einen Synchronisationsfehler in der Audit-Datei.
|
||||
|
||||
Schreibt einen strukturierten Eintrag (Zeitstempel, Repo-Name,
|
||||
Richtung, Fehlergrund) in `.audit/sync-errors.log`. Der lokale
|
||||
Stand des Repos wird dabei nicht verändert.
|
||||
|
||||
Args:
|
||||
repo_name: Name des betroffenen Repos.
|
||||
error_reason: Beschreibung des Fehlergrunds (Netzwerk, Auth, Merge).
|
||||
direction: Richtung der fehlgeschlagenen Operation (pull/push/full).
|
||||
"""
|
||||
audit_dir = self.monorepo_root / ".audit"
|
||||
audit_dir.mkdir(parents=True, exist_ok=True)
|
||||
log_path = audit_dir / "sync-errors.log"
|
||||
|
||||
timestamp = datetime.now().isoformat(timespec="seconds")
|
||||
log_entry = (
|
||||
f"[{timestamp}] repo={repo_name} direction={direction} "
|
||||
f"error={error_reason}\n"
|
||||
)
|
||||
|
||||
with open(log_path, "a", encoding="utf-8") as f:
|
||||
f.write(log_entry)
|
||||
|
||||
logger.error(
|
||||
"Sync-Fehler protokolliert: repo=%s, direction=%s, error=%s",
|
||||
repo_name,
|
||||
direction,
|
||||
error_reason,
|
||||
)
|
||||
@@ -0,0 +1,361 @@
|
||||
"""Sicherheits-Guard für die Zugriffskontrolle zwischen Arbeitskontexten.
|
||||
|
||||
Implementiert den ContextGuard, der Sicherheitsgrenzen zwischen den
|
||||
Arbeitskontexten (privat, dhive, bahn, shared) erzwingt. Die Zugriffskontrolle
|
||||
basiert auf der zentralen access-config.yaml.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from monorepo.models import Context
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ContextGuard:
|
||||
"""Erzwingt Sicherheitsgrenzen zwischen Arbeitskontexten.
|
||||
|
||||
Prüft ob ein anfragender Kontext auf einen gegebenen Pfad zugreifen darf,
|
||||
basierend auf den Regeln in access-config.yaml:
|
||||
- Ein Kontext darf immer auf eigene Dateien zugreifen.
|
||||
- Ein Kontext darf auf shared-Pfade zugreifen, die in allowed_shared gelistet sind.
|
||||
- Der shared-Kontext mit ["*"] darf auf alles in shared zugreifen.
|
||||
- Alle anderen kontextübergreifenden Zugriffe werden verweigert.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
root_path: Path,
|
||||
access_config_path: Path | None = None,
|
||||
encryption_manager: Any | None = None,
|
||||
) -> None:
|
||||
"""Initialisiert den ContextGuard.
|
||||
|
||||
Args:
|
||||
root_path: Wurzelverzeichnis des Monorepos.
|
||||
access_config_path: Pfad zur access-config.yaml.
|
||||
Falls None, wird `root_path / shared/config/access-config.yaml` verwendet.
|
||||
encryption_manager: Optionaler SecretEncryptionManager für die
|
||||
Entschlüsselung von .env-Dateien. Falls None, werden .env-Dateien
|
||||
direkt gelesen (unverschlüsselt oder bereits durch git-crypt entschlüsselt).
|
||||
"""
|
||||
self.root_path = root_path.resolve()
|
||||
if access_config_path is None:
|
||||
access_config_path = self.root_path / "shared" / "config" / "access-config.yaml"
|
||||
self.access_config_path = access_config_path.resolve()
|
||||
self.encryption_manager = encryption_manager
|
||||
self._config: dict[str, Any] = self._load_access_config()
|
||||
|
||||
def _load_access_config(self) -> dict[str, Any]:
|
||||
"""Lädt die access-config.yaml.
|
||||
|
||||
Returns:
|
||||
Dict mit der Zugriffskonfiguration pro Kontext.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: Wenn die Konfigurationsdatei nicht existiert.
|
||||
ValueError: Wenn das YAML-Format ungültig ist.
|
||||
"""
|
||||
if not self.access_config_path.exists():
|
||||
raise FileNotFoundError(
|
||||
f"Access-Konfiguration nicht gefunden: {self.access_config_path}"
|
||||
)
|
||||
|
||||
with open(self.access_config_path, encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f)
|
||||
|
||||
if not isinstance(data, dict) or "contexts" not in data:
|
||||
raise ValueError(
|
||||
f"Ungültiges access-config.yaml-Format: "
|
||||
f"Erwartet dict mit 'contexts'-Schlüssel in {self.access_config_path}"
|
||||
)
|
||||
|
||||
return data["contexts"]
|
||||
|
||||
def check_access(self, requesting_context: str, target_path: Path) -> bool:
|
||||
"""Prüft ob der Zugriff auf target_path vom requesting_context erlaubt ist.
|
||||
|
||||
Regeln:
|
||||
1. Ein Kontext darf immer auf eigene Dateien zugreifen (Pfad unter eigenem Kontextordner).
|
||||
2. Ein Kontext darf auf shared-Pfade zugreifen, wenn diese in allowed_shared gelistet sind.
|
||||
3. Der shared-Kontext mit allowed_shared: ["*"] darf auf alles in shared zugreifen.
|
||||
4. Alle anderen kontextübergreifenden Zugriffe werden verweigert.
|
||||
|
||||
Args:
|
||||
requesting_context: Name des anfragenden Kontexts (z.B. "privat", "dhive", "bahn", "shared").
|
||||
target_path: Pfad auf den zugegriffen werden soll (absolut oder relativ zum root_path).
|
||||
|
||||
Returns:
|
||||
True wenn der Zugriff erlaubt ist, False wenn verweigert.
|
||||
"""
|
||||
# Pfad relativ zum Root normalisieren
|
||||
rel_path = self._resolve_relative_path(target_path)
|
||||
rel_path_str = rel_path.as_posix()
|
||||
|
||||
# Kontext des Zielpfads ermitteln
|
||||
target_context = self._get_path_context(rel_path_str)
|
||||
|
||||
# Regel 1: Zugriff auf eigenen Kontext immer erlaubt
|
||||
if target_context == requesting_context:
|
||||
return True
|
||||
|
||||
# Regel 2+3: Zugriff auf shared-Bereich prüfen
|
||||
if target_context == Context.SHARED.value:
|
||||
return self._check_shared_access(requesting_context, rel_path_str)
|
||||
|
||||
# Regel 4: Kontextübergreifender Zugriff verweigert
|
||||
return False
|
||||
|
||||
def load_env(self, context: str) -> dict[str, str]:
|
||||
"""Lädt die .env-Datei des gegebenen Kontexts (mit optionaler Entschlüsselung).
|
||||
|
||||
Wenn ein SecretEncryptionManager konfiguriert ist und der Maschinenkontext
|
||||
für den angefragten Kontext autorisiert ist, wird die .env-Datei via
|
||||
decrypt_file entschlüsselt. Andernfalls wird die Datei direkt gelesen
|
||||
(unverschlüsselt oder bereits durch git-crypt entschlüsselt im Working Tree).
|
||||
|
||||
Parst eine Standard-.env-Datei (KEY=VALUE-Format) und gibt die
|
||||
Umgebungsvariablen als Dict zurück. Kommentare (#) und Leerzeilen
|
||||
werden übersprungen.
|
||||
|
||||
Args:
|
||||
context: Name des Kontexts, dessen .env geladen werden soll.
|
||||
|
||||
Returns:
|
||||
Dict mit den Umgebungsvariablen (Schlüssel → Wert).
|
||||
|
||||
Raises:
|
||||
ValueError: Wenn der Kontext nicht in der Konfiguration definiert ist.
|
||||
FileNotFoundError: Wenn die .env-Datei nicht existiert.
|
||||
PermissionError: Wenn die Entschlüsselung fehlschlägt
|
||||
(Maschinenkontext nicht autorisiert).
|
||||
"""
|
||||
if context not in self._config:
|
||||
raise ValueError(
|
||||
f"Unbekannter Kontext '{context}'. "
|
||||
f"Gültige Kontexte: {list(self._config.keys())}"
|
||||
)
|
||||
|
||||
env_file_rel = self._config[context].get("env_file")
|
||||
if not env_file_rel:
|
||||
raise ValueError(
|
||||
f"Kein env_file für Kontext '{context}' in der Konfiguration definiert."
|
||||
)
|
||||
|
||||
env_path = self.root_path / env_file_rel
|
||||
|
||||
if not env_path.exists():
|
||||
raise FileNotFoundError(
|
||||
f".env-Datei für Kontext '{context}' nicht gefunden: {env_path}"
|
||||
)
|
||||
|
||||
# Integration mit SecretEncryptionManager für Entschlüsselung (Req 2.2, 9.3).
|
||||
if self.encryption_manager is not None:
|
||||
return self._load_env_with_decryption(env_path, context)
|
||||
|
||||
# Fallback: Datei direkt lesen (unverschlüsselt oder bereits entschlüsselt)
|
||||
return self._parse_env_file(env_path)
|
||||
|
||||
def _load_env_with_decryption(
|
||||
self, env_path: Path, context: str
|
||||
) -> dict[str, str]:
|
||||
"""Lädt und entschlüsselt eine .env-Datei via SecretEncryptionManager.
|
||||
|
||||
Prüft zuerst die Autorisierung des Maschinenkontexts. Wenn autorisiert,
|
||||
wird die Datei entschlüsselt und geparst. Andernfalls wird ein
|
||||
PermissionError ausgelöst.
|
||||
|
||||
Args:
|
||||
env_path: Absoluter Pfad zur .env-Datei.
|
||||
context: Arbeitskontext der .env-Datei.
|
||||
|
||||
Returns:
|
||||
Dict mit den entschlüsselten Umgebungsvariablen.
|
||||
|
||||
Raises:
|
||||
PermissionError: Wenn der Maschinenkontext nicht autorisiert ist.
|
||||
"""
|
||||
encryption_mgr = self.encryption_manager
|
||||
|
||||
# Autorisierungsprüfung
|
||||
if not encryption_mgr.is_authorized(context):
|
||||
raise PermissionError(
|
||||
f"Maschinenkontext '{encryption_mgr.machine_context.name}' ist nicht "
|
||||
f"für Kontext '{context}' autorisiert. Entschlüsselung verweigert."
|
||||
)
|
||||
|
||||
# Entschlüsselung versuchen
|
||||
result = encryption_mgr.decrypt_file(env_path)
|
||||
|
||||
if result.success and result.content is not None:
|
||||
# Entschlüsselter Inhalt als Text parsen
|
||||
content_text = result.content.decode("utf-8", errors="replace")
|
||||
return self._parse_env_content(content_text)
|
||||
|
||||
# Entschlüsselung fehlgeschlagen – Fallback auf direkte Leseoperation.
|
||||
# Dies tritt auf wenn die Datei bereits im Klartext vorliegt (z.B.
|
||||
# nach git-crypt unlock) oder wenn git-crypt nicht verfügbar ist.
|
||||
logger.debug(
|
||||
"Entschlüsselung für '%s' nicht erfolgreich (%s). "
|
||||
"Fallback auf direktes Lesen.",
|
||||
env_path,
|
||||
result.error or "unbekannter Fehler",
|
||||
)
|
||||
return self._parse_env_file(env_path)
|
||||
|
||||
def get_context_config(self, context: str) -> dict[str, Any]:
|
||||
"""Gibt die Konfiguration für einen bestimmten Kontext zurück.
|
||||
|
||||
Args:
|
||||
context: Name des Kontexts.
|
||||
|
||||
Returns:
|
||||
Dict mit env_file und allowed_shared für den Kontext.
|
||||
|
||||
Raises:
|
||||
ValueError: Wenn der Kontext nicht in der Konfiguration definiert ist.
|
||||
"""
|
||||
if context not in self._config:
|
||||
raise ValueError(
|
||||
f"Unbekannter Kontext '{context}'. "
|
||||
f"Gültige Kontexte: {list(self._config.keys())}"
|
||||
)
|
||||
return self._config[context]
|
||||
|
||||
def _resolve_relative_path(self, target_path: Path) -> Path:
|
||||
"""Löst einen Pfad relativ zum Monorepo-Root auf.
|
||||
|
||||
Args:
|
||||
target_path: Absoluter oder relativer Pfad.
|
||||
|
||||
Returns:
|
||||
Pfad relativ zum Root (immer mit Forward-Slashes via as_posix).
|
||||
"""
|
||||
resolved = Path(target_path).resolve() if target_path.is_absolute() else target_path
|
||||
try:
|
||||
return resolved.relative_to(self.root_path)
|
||||
except ValueError:
|
||||
# Pfad ist bereits relativ oder liegt außerhalb des Roots
|
||||
return target_path
|
||||
|
||||
def _get_path_context(self, rel_path_str: str) -> str:
|
||||
"""Ermittelt den Kontext eines relativen Pfads.
|
||||
|
||||
Der Kontext wird anhand des obersten Verzeichnisses bestimmt.
|
||||
Gültige Kontexte: privat, dhive, bahn, shared.
|
||||
|
||||
Args:
|
||||
rel_path_str: Relativer Pfad als String (mit Forward-Slashes).
|
||||
|
||||
Returns:
|
||||
Name des Kontexts oder leerer String wenn nicht zuordenbar.
|
||||
"""
|
||||
parts = rel_path_str.split("/")
|
||||
if not parts:
|
||||
return ""
|
||||
|
||||
first_part = parts[0]
|
||||
valid_contexts = {ctx.value for ctx in Context}
|
||||
if first_part in valid_contexts:
|
||||
return first_part
|
||||
|
||||
return ""
|
||||
|
||||
def _check_shared_access(self, requesting_context: str, rel_path_str: str) -> bool:
|
||||
"""Prüft ob ein Kontext auf einen shared-Pfad zugreifen darf.
|
||||
|
||||
Args:
|
||||
requesting_context: Name des anfragenden Kontexts.
|
||||
rel_path_str: Relativer Pfad (beginnt mit "shared/").
|
||||
|
||||
Returns:
|
||||
True wenn der Zugriff erlaubt ist.
|
||||
"""
|
||||
if requesting_context not in self._config:
|
||||
return False
|
||||
|
||||
allowed_shared = self._config[requesting_context].get("allowed_shared", [])
|
||||
|
||||
# Wildcard: Zugriff auf alles in shared erlaubt
|
||||
if allowed_shared == ["*"]:
|
||||
return True
|
||||
|
||||
# Prüfe ob der Pfad unter einem der erlaubten shared-Pfade liegt
|
||||
for allowed_path in allowed_shared:
|
||||
# Normalisiere: Stelle sicher dass der Pfad mit / endet für Prefix-Matching
|
||||
allowed_normalized = allowed_path.rstrip("/") + "/"
|
||||
# Pfad liegt unter dem erlaubten Verzeichnis oder ist es selbst
|
||||
if rel_path_str.startswith(allowed_normalized) or rel_path_str.rstrip("/") + "/" == allowed_normalized:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _parse_env_file(env_path: Path) -> dict[str, str]:
|
||||
"""Parst eine .env-Datei im Standard-Format.
|
||||
|
||||
Format:
|
||||
- KEY=VALUE (ein Paar pro Zeile)
|
||||
- Zeilen die mit # beginnen sind Kommentare
|
||||
- Leerzeilen werden übersprungen
|
||||
- Führende/nachfolgende Leerzeichen werden entfernt
|
||||
- Werte in Anführungszeichen (einfach oder doppelt) werden entquotet
|
||||
|
||||
Args:
|
||||
env_path: Pfad zur .env-Datei.
|
||||
|
||||
Returns:
|
||||
Dict mit geparsten Umgebungsvariablen.
|
||||
"""
|
||||
with open(env_path, encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
return ContextGuard._parse_env_content(content)
|
||||
|
||||
@staticmethod
|
||||
def _parse_env_content(content: str) -> dict[str, str]:
|
||||
"""Parst .env-Inhalt im Standard-Format aus einem String.
|
||||
|
||||
Format:
|
||||
- KEY=VALUE (ein Paar pro Zeile)
|
||||
- Zeilen die mit # beginnen sind Kommentare
|
||||
- Leerzeilen werden übersprungen
|
||||
- Führende/nachfolgende Leerzeichen werden entfernt
|
||||
- Werte in Anführungszeichen (einfach oder doppelt) werden entquotet
|
||||
|
||||
Args:
|
||||
content: Textinhalt der .env-Datei.
|
||||
|
||||
Returns:
|
||||
Dict mit geparsten Umgebungsvariablen.
|
||||
"""
|
||||
env_vars: dict[str, str] = {}
|
||||
|
||||
for line in content.splitlines():
|
||||
line = line.strip()
|
||||
|
||||
# Leerzeilen und Kommentare überspringen
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
|
||||
# KEY=VALUE aufteilen (nur beim ersten = splitten)
|
||||
if "=" not in line:
|
||||
continue
|
||||
|
||||
key, _, value = line.partition("=")
|
||||
key = key.strip()
|
||||
value = value.strip()
|
||||
|
||||
# Anführungszeichen entfernen
|
||||
if len(value) >= 2 and value[0] == value[-1] and value[0] in ('"', "'"):
|
||||
value = value[1:-1]
|
||||
|
||||
if key:
|
||||
env_vars[key] = value
|
||||
|
||||
return env_vars
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,180 @@
|
||||
"""Ordnerstruktur-Manager für das Monorepo.
|
||||
|
||||
Verwaltet die 3-Ebenen-Ordnerhierarchie:
|
||||
Ebene 1 (Kontextordner) → Ebene 2 (Projektordner) → Ebene 3 (Modulordner)
|
||||
|
||||
Verantwortung: Anlegen, Validieren und Verwalten der Monorepo-Ordnerstruktur.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from monorepo.models import Context, ProjectInfo
|
||||
|
||||
|
||||
class StructureManager:
|
||||
"""Verwaltet die 3-Ebenen-Ordnerhierarchie des Monorepos."""
|
||||
|
||||
CONTEXTS = ("privat", "dhive", "bahn", "shared")
|
||||
NAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9\-]{0,48}[a-z0-9]$")
|
||||
|
||||
def __init__(self, root_path: Path) -> None:
|
||||
"""Initialisiert den StructureManager.
|
||||
|
||||
Args:
|
||||
root_path: Wurzelverzeichnis des Monorepos.
|
||||
"""
|
||||
self.root_path = root_path
|
||||
|
||||
def validate_name(self, name: str) -> bool:
|
||||
"""Prüft ob ein Name der kebab-case-Konvention entspricht.
|
||||
|
||||
Regeln:
|
||||
- Ausschließlich Kleinbuchstaben, Ziffern und Bindestriche
|
||||
- Zwischen 2 und 50 Zeichen lang
|
||||
- Beginnt und endet nicht mit einem Bindestrich
|
||||
|
||||
Args:
|
||||
name: Der zu prüfende Projektname.
|
||||
|
||||
Returns:
|
||||
True wenn der Name gültig ist, False sonst.
|
||||
"""
|
||||
if not name:
|
||||
return False
|
||||
return self.NAME_PATTERN.match(name) is not None
|
||||
|
||||
def create_project(self, context: str, name: str) -> Path:
|
||||
"""Erstellt ein neues Projekt im gegebenen Kontext.
|
||||
|
||||
Validiert den Namen, prüft auf Namenskonflikte und erstellt
|
||||
den Projektordner im korrekten Kontextordner.
|
||||
|
||||
Args:
|
||||
context: Der Arbeitskontext (privat, dhive, bahn, shared).
|
||||
name: Der Projektname in kebab-case.
|
||||
|
||||
Returns:
|
||||
Pfad zum erstellten Projektordner.
|
||||
|
||||
Raises:
|
||||
ValueError: Wenn der Kontext ungültig ist, der Name die
|
||||
Namenskonvention verletzt, oder ein Projekt mit
|
||||
diesem Namen bereits existiert.
|
||||
"""
|
||||
# Kontext validieren
|
||||
if context not in self.CONTEXTS:
|
||||
raise ValueError(
|
||||
f"Ungültiger Kontext '{context}'. "
|
||||
f"Erlaubte Kontexte: {', '.join(self.CONTEXTS)}"
|
||||
)
|
||||
|
||||
# Name validieren
|
||||
if not self.validate_name(name):
|
||||
raise ValueError(
|
||||
f"Ungültiger Projektname '{name}'. "
|
||||
f"Der Name muss kebab-case sein (Kleinbuchstaben, Ziffern, "
|
||||
f"Bindestriche), zwischen 2 und 50 Zeichen lang, und darf "
|
||||
f"nicht mit einem Bindestrich beginnen oder enden."
|
||||
)
|
||||
|
||||
# Namenskonflikt prüfen
|
||||
project_path = self.root_path / context / name
|
||||
if project_path.exists():
|
||||
raise ValueError(
|
||||
f"Namenskonflikt: Ein Projekt mit dem Namen '{name}' "
|
||||
f"existiert bereits im Kontext '{context}' "
|
||||
f"(Pfad: {project_path})."
|
||||
)
|
||||
|
||||
# Projektordner erstellen
|
||||
project_path.mkdir(parents=True, exist_ok=False)
|
||||
return project_path
|
||||
|
||||
def list_projects(self, context: str | None = None) -> list[ProjectInfo]:
|
||||
"""Listet alle Projekte, optional gefiltert nach Kontext.
|
||||
|
||||
Args:
|
||||
context: Optionaler Kontext-Filter. Wenn None, werden alle
|
||||
Projekte aus allen Kontexten aufgelistet.
|
||||
|
||||
Returns:
|
||||
Liste von ProjectInfo-Objekten für alle gefundenen Projekte.
|
||||
|
||||
Raises:
|
||||
ValueError: Wenn ein ungültiger Kontext angegeben wird.
|
||||
"""
|
||||
if context is not None and context not in self.CONTEXTS:
|
||||
raise ValueError(
|
||||
f"Ungültiger Kontext '{context}'. "
|
||||
f"Erlaubte Kontexte: {', '.join(self.CONTEXTS)}"
|
||||
)
|
||||
|
||||
contexts_to_scan = (context,) if context else self.CONTEXTS
|
||||
projects: list[ProjectInfo] = []
|
||||
|
||||
for ctx in contexts_to_scan:
|
||||
context_dir = self.root_path / ctx
|
||||
if not context_dir.exists():
|
||||
continue
|
||||
|
||||
for entry in sorted(context_dir.iterdir()):
|
||||
if entry.is_dir() and not entry.name.startswith("."):
|
||||
projects.append(
|
||||
ProjectInfo(
|
||||
name=entry.name,
|
||||
context=ctx,
|
||||
path=entry,
|
||||
)
|
||||
)
|
||||
|
||||
return projects
|
||||
|
||||
def resolve_context(self, project_path: Path) -> str:
|
||||
"""Ermittelt den Arbeitskontext eines Projekts anhand seines Pfads.
|
||||
|
||||
Analysiert den Pfad relativ zum Monorepo-Root und extrahiert
|
||||
den Kontextordner (erste Ebene).
|
||||
|
||||
Args:
|
||||
project_path: Absoluter oder relativer Pfad zum Projekt.
|
||||
|
||||
Returns:
|
||||
Der ermittelte Arbeitskontext als String.
|
||||
|
||||
Raises:
|
||||
ValueError: Wenn der Pfad nicht innerhalb eines bekannten
|
||||
Kontextordners liegt.
|
||||
"""
|
||||
# Pfad auflösen (absolut machen)
|
||||
resolved = project_path.resolve()
|
||||
root_resolved = self.root_path.resolve()
|
||||
|
||||
# Prüfen ob der Pfad innerhalb des Monorepos liegt
|
||||
try:
|
||||
relative = resolved.relative_to(root_resolved)
|
||||
except ValueError:
|
||||
raise ValueError(
|
||||
f"Der Pfad '{project_path}' liegt nicht innerhalb "
|
||||
f"des Monorepos ({self.root_path})."
|
||||
)
|
||||
|
||||
# Ersten Pfadteil (Kontextordner) extrahieren
|
||||
parts = relative.parts
|
||||
if not parts:
|
||||
raise ValueError(
|
||||
f"Der Pfad '{project_path}' zeigt auf das Monorepo-Root, "
|
||||
f"nicht auf ein Projekt in einem Kontext."
|
||||
)
|
||||
|
||||
context_name = parts[0]
|
||||
if context_name not in self.CONTEXTS:
|
||||
raise ValueError(
|
||||
f"Der Pfad '{project_path}' liegt nicht in einem bekannten "
|
||||
f"Kontextordner. Gefunden: '{context_name}', "
|
||||
f"erwartet: {', '.join(self.CONTEXTS)}."
|
||||
)
|
||||
|
||||
return context_name
|
||||
@@ -0,0 +1,381 @@
|
||||
"""Unit-Tests für das Wissensartefakt-Modell (knowledge/artifact.py).
|
||||
|
||||
Testet:
|
||||
- YAML-Frontmatter-Parsing und -Generierung
|
||||
- Content-Hash-Berechnung (SHA-256)
|
||||
- Scope-basierte Pfadauflösung
|
||||
- Artefakt-Schreibvorgang mit Ordnerstruktur
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from monorepo.knowledge.artifact import (
|
||||
Artifact,
|
||||
ArtifactLink,
|
||||
ArtifactMetadata,
|
||||
compute_content_hash,
|
||||
parse_frontmatter,
|
||||
resolve_artifact_path,
|
||||
write_artifact,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: compute_content_hash
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestComputeContentHash:
|
||||
"""Tests für die SHA-256 Content-Hash-Berechnung."""
|
||||
|
||||
def test_hash_format(self) -> None:
|
||||
"""Hash hat das Format 'sha256:<hex>'."""
|
||||
result = compute_content_hash("Hello World")
|
||||
assert result.startswith("sha256:")
|
||||
hex_part = result.removeprefix("sha256:")
|
||||
assert len(hex_part) == 64 # SHA-256 = 64 hex chars
|
||||
|
||||
def test_deterministic(self) -> None:
|
||||
"""Gleicher Input ergibt gleichen Hash."""
|
||||
content = "Testinhalt mit Umlauten: äöü"
|
||||
assert compute_content_hash(content) == compute_content_hash(content)
|
||||
|
||||
def test_different_content_different_hash(self) -> None:
|
||||
"""Verschiedener Input ergibt verschiedenen Hash."""
|
||||
assert compute_content_hash("abc") != compute_content_hash("xyz")
|
||||
|
||||
def test_empty_string(self) -> None:
|
||||
"""Leerer String ergibt gültigen Hash."""
|
||||
result = compute_content_hash("")
|
||||
assert result.startswith("sha256:")
|
||||
assert len(result.removeprefix("sha256:")) == 64
|
||||
|
||||
def test_known_value(self) -> None:
|
||||
"""Bekannter SHA-256-Wert für Verifikation."""
|
||||
# SHA-256 of "test" is well-known
|
||||
import hashlib
|
||||
|
||||
expected = "sha256:" + hashlib.sha256(b"test").hexdigest()
|
||||
assert compute_content_hash("test") == expected
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: resolve_artifact_path
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestResolveArtifactPath:
|
||||
"""Tests für die Scope-basierte Pfadauflösung."""
|
||||
|
||||
def test_basic_path(self) -> None:
|
||||
"""Standard-Pfadauflösung: scope/type/name.md."""
|
||||
result = resolve_artifact_path("bahn", "decision", "api-design")
|
||||
assert result == Path("bahn/decision/api-design.md")
|
||||
|
||||
def test_different_scope(self) -> None:
|
||||
"""Verschiedene Scopes ergeben verschiedene Pfade."""
|
||||
path_bahn = resolve_artifact_path("bahn", "note", "test")
|
||||
path_privat = resolve_artifact_path("privat", "note", "test")
|
||||
assert path_bahn != path_privat
|
||||
assert path_bahn.parts[0] == "bahn"
|
||||
assert path_privat.parts[0] == "privat"
|
||||
|
||||
def test_different_types(self) -> None:
|
||||
"""Verschiedene Typen ergeben verschiedene Pfade."""
|
||||
path_decision = resolve_artifact_path("bahn", "decision", "x")
|
||||
path_note = resolve_artifact_path("bahn", "note", "x")
|
||||
assert path_decision.parts[1] == "decision"
|
||||
assert path_note.parts[1] == "note"
|
||||
|
||||
def test_name_with_md_extension_stripped(self) -> None:
|
||||
"""Wenn Name bereits .md hat, wird es nicht doppelt angehängt."""
|
||||
result = resolve_artifact_path("privat", "note", "test.md")
|
||||
assert result == Path("privat/note/test.md")
|
||||
assert not str(result).endswith(".md.md")
|
||||
|
||||
def test_name_without_extension(self) -> None:
|
||||
"""Name ohne Erweiterung bekommt .md."""
|
||||
result = resolve_artifact_path("shared", "pattern", "singleton")
|
||||
assert result.suffix == ".md"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: parse_frontmatter
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestParseFrontmatter:
|
||||
"""Tests für das YAML-Frontmatter-Parsing."""
|
||||
|
||||
def test_full_frontmatter(self, tmp_path: Path) -> None:
|
||||
"""Parst vollständiges Frontmatter mit allen Feldern."""
|
||||
content = """---
|
||||
type: decision
|
||||
title: "API-Designprinzipien"
|
||||
tags: [api, microservices]
|
||||
source_context: bahn
|
||||
created: 2024-12-15
|
||||
updated: 2025-01-10
|
||||
shareable: true
|
||||
links:
|
||||
- target: "privat/notes/rest.md"
|
||||
relation: "implements"
|
||||
content_hash: "sha256:abc123"
|
||||
---
|
||||
# Inhalt hier
|
||||
"""
|
||||
file = tmp_path / "test.md"
|
||||
file.write_text(content, encoding="utf-8")
|
||||
|
||||
artifact = parse_frontmatter(file)
|
||||
|
||||
assert artifact.metadata.type == "decision"
|
||||
assert artifact.metadata.title == "API-Designprinzipien"
|
||||
assert artifact.metadata.tags == ["api", "microservices"]
|
||||
assert artifact.metadata.source_context == "bahn"
|
||||
assert artifact.metadata.created == date(2024, 12, 15)
|
||||
assert artifact.metadata.updated == date(2025, 1, 10)
|
||||
assert artifact.metadata.shareable is True
|
||||
assert len(artifact.metadata.links) == 1
|
||||
assert artifact.metadata.links[0].target == "privat/notes/rest.md"
|
||||
assert artifact.metadata.links[0].relation == "implements"
|
||||
assert artifact.metadata.content_hash == "sha256:abc123"
|
||||
assert artifact.content == "# Inhalt hier\n"
|
||||
assert artifact.file_path == file
|
||||
|
||||
def test_minimal_frontmatter(self, tmp_path: Path) -> None:
|
||||
"""Parst minimales Frontmatter mit nur type und title."""
|
||||
content = """---
|
||||
type: note
|
||||
title: "Kurze Notiz"
|
||||
---
|
||||
Einfacher Text.
|
||||
"""
|
||||
file = tmp_path / "minimal.md"
|
||||
file.write_text(content, encoding="utf-8")
|
||||
|
||||
artifact = parse_frontmatter(file)
|
||||
|
||||
assert artifact.metadata.type == "note"
|
||||
assert artifact.metadata.title == "Kurze Notiz"
|
||||
assert artifact.metadata.tags == []
|
||||
assert artifact.metadata.source_context == ""
|
||||
assert artifact.metadata.created is None
|
||||
assert artifact.metadata.shareable is False
|
||||
assert artifact.metadata.links == []
|
||||
assert artifact.content == "Einfacher Text.\n"
|
||||
|
||||
def test_multiple_links(self, tmp_path: Path) -> None:
|
||||
"""Parst mehrere Verknüpfungen."""
|
||||
content = """---
|
||||
type: reference
|
||||
title: "Multi-Link"
|
||||
links:
|
||||
- target: "a.md"
|
||||
relation: "references"
|
||||
- target: "b.md"
|
||||
relation: "implements"
|
||||
- target: "c.md"
|
||||
relation: "extends"
|
||||
---
|
||||
Body.
|
||||
"""
|
||||
file = tmp_path / "links.md"
|
||||
file.write_text(content, encoding="utf-8")
|
||||
|
||||
artifact = parse_frontmatter(file)
|
||||
assert len(artifact.metadata.links) == 3
|
||||
assert artifact.metadata.links[2].target == "c.md"
|
||||
|
||||
def test_no_frontmatter_raises_error(self, tmp_path: Path) -> None:
|
||||
"""Datei ohne Frontmatter wirft ValueError."""
|
||||
file = tmp_path / "no_fm.md"
|
||||
file.write_text("# Kein Frontmatter\nNur Text.", encoding="utf-8")
|
||||
|
||||
with pytest.raises(ValueError, match="Kein gültiges YAML-Frontmatter"):
|
||||
parse_frontmatter(file)
|
||||
|
||||
def test_file_not_found(self, tmp_path: Path) -> None:
|
||||
"""Nicht-existierende Datei wirft FileNotFoundError."""
|
||||
with pytest.raises(FileNotFoundError):
|
||||
parse_frontmatter(tmp_path / "nonexistent.md")
|
||||
|
||||
def test_empty_frontmatter(self, tmp_path: Path) -> None:
|
||||
"""Leeres Frontmatter ergibt leere Metadaten."""
|
||||
content = """---
|
||||
---
|
||||
Nur Inhalt.
|
||||
"""
|
||||
file = tmp_path / "empty_fm.md"
|
||||
file.write_text(content, encoding="utf-8")
|
||||
|
||||
artifact = parse_frontmatter(file)
|
||||
assert artifact.metadata.type == ""
|
||||
assert artifact.metadata.title == ""
|
||||
assert artifact.content == "Nur Inhalt.\n"
|
||||
|
||||
def test_content_with_dashes(self, tmp_path: Path) -> None:
|
||||
"""Inhalt mit --- wird nicht als Frontmatter-Ende interpretiert."""
|
||||
content = """---
|
||||
type: note
|
||||
title: "Dashes Test"
|
||||
---
|
||||
Hier kommt ein Trenner:
|
||||
|
||||
---
|
||||
|
||||
Und noch mehr Text.
|
||||
"""
|
||||
file = tmp_path / "dashes.md"
|
||||
file.write_text(content, encoding="utf-8")
|
||||
|
||||
artifact = parse_frontmatter(file)
|
||||
assert "---" in artifact.content
|
||||
assert "Und noch mehr Text." in artifact.content
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: write_artifact
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestWriteArtifact:
|
||||
"""Tests für das Schreiben von Artefakten."""
|
||||
|
||||
def test_creates_file_with_frontmatter(self, tmp_path: Path) -> None:
|
||||
"""Schreibt eine Datei mit korrektem YAML-Frontmatter."""
|
||||
metadata = ArtifactMetadata(
|
||||
type="decision",
|
||||
title="Test Decision",
|
||||
tags=["test", "example"],
|
||||
source_context="bahn",
|
||||
created=date(2024, 12, 15),
|
||||
shareable=True,
|
||||
)
|
||||
artifact = Artifact(
|
||||
metadata=metadata,
|
||||
content="# Entscheidung\n\nDetails hier.\n",
|
||||
file_path=Path("test-decision.md"),
|
||||
)
|
||||
|
||||
result_path = write_artifact(artifact, tmp_path)
|
||||
|
||||
assert result_path.exists()
|
||||
assert result_path == tmp_path / "bahn" / "decision" / "test-decision.md"
|
||||
|
||||
# Roundtrip: Datei wieder einlesen
|
||||
parsed = parse_frontmatter(result_path)
|
||||
assert parsed.metadata.type == "decision"
|
||||
assert parsed.metadata.title == "Test Decision"
|
||||
assert parsed.metadata.tags == ["test", "example"]
|
||||
assert parsed.metadata.source_context == "bahn"
|
||||
assert parsed.metadata.shareable is True
|
||||
assert "# Entscheidung" in parsed.content
|
||||
|
||||
def test_creates_directory_structure(self, tmp_path: Path) -> None:
|
||||
"""Erstellt fehlende Verzeichnisse automatisch."""
|
||||
metadata = ArtifactMetadata(
|
||||
type="pattern",
|
||||
title="Singleton",
|
||||
source_context="shared",
|
||||
)
|
||||
artifact = Artifact(
|
||||
metadata=metadata,
|
||||
content="Pattern-Inhalt\n",
|
||||
file_path=Path("singleton.md"),
|
||||
)
|
||||
|
||||
result_path = write_artifact(artifact, tmp_path)
|
||||
|
||||
assert (tmp_path / "shared" / "pattern").is_dir()
|
||||
assert result_path.exists()
|
||||
|
||||
def test_updates_content_hash(self, tmp_path: Path) -> None:
|
||||
"""Aktualisiert den Content-Hash automatisch beim Schreiben."""
|
||||
content = "Neuer Inhalt für den Hash-Test.\n"
|
||||
metadata = ArtifactMetadata(
|
||||
type="note",
|
||||
title="Hash Test",
|
||||
source_context="privat",
|
||||
content_hash="sha256:old_hash", # alter Hash
|
||||
)
|
||||
artifact = Artifact(
|
||||
metadata=metadata,
|
||||
content=content,
|
||||
file_path=Path("hash-test.md"),
|
||||
)
|
||||
|
||||
write_artifact(artifact, tmp_path)
|
||||
|
||||
# Hash muss aktualisiert sein
|
||||
expected_hash = compute_content_hash(content)
|
||||
assert artifact.metadata.content_hash == expected_hash
|
||||
assert artifact.metadata.content_hash != "sha256:old_hash"
|
||||
|
||||
def test_writes_links_in_frontmatter(self, tmp_path: Path) -> None:
|
||||
"""Schreibt Verknüpfungen korrekt ins Frontmatter."""
|
||||
metadata = ArtifactMetadata(
|
||||
type="reference",
|
||||
title="Linked Artifact",
|
||||
source_context="dhive",
|
||||
links=[
|
||||
ArtifactLink(target="other/doc.md", relation="references"),
|
||||
ArtifactLink(target="third/file.md", relation="implements"),
|
||||
],
|
||||
)
|
||||
artifact = Artifact(
|
||||
metadata=metadata,
|
||||
content="Verknüpfter Inhalt.\n",
|
||||
file_path=Path("linked.md"),
|
||||
)
|
||||
|
||||
result_path = write_artifact(artifact, tmp_path)
|
||||
|
||||
# Roundtrip-Verifikation
|
||||
parsed = parse_frontmatter(result_path)
|
||||
assert len(parsed.metadata.links) == 2
|
||||
assert parsed.metadata.links[0].target == "other/doc.md"
|
||||
assert parsed.metadata.links[0].relation == "references"
|
||||
assert parsed.metadata.links[1].target == "third/file.md"
|
||||
|
||||
def test_overwrites_existing_file(self, tmp_path: Path) -> None:
|
||||
"""Überschreibt eine bereits existierende Datei."""
|
||||
metadata = ArtifactMetadata(
|
||||
type="note",
|
||||
title="Overwrite Test",
|
||||
source_context="privat",
|
||||
)
|
||||
artifact = Artifact(
|
||||
metadata=metadata,
|
||||
content="Version 1\n",
|
||||
file_path=Path("overwrite.md"),
|
||||
)
|
||||
|
||||
path1 = write_artifact(artifact, tmp_path)
|
||||
artifact.content = "Version 2\n"
|
||||
path2 = write_artifact(artifact, tmp_path)
|
||||
|
||||
assert path1 == path2
|
||||
parsed = parse_frontmatter(path2)
|
||||
assert "Version 2" in parsed.content
|
||||
|
||||
def test_updates_artifact_file_path(self, tmp_path: Path) -> None:
|
||||
"""Aktualisiert artifact.file_path nach dem Schreiben."""
|
||||
metadata = ArtifactMetadata(
|
||||
type="meeting",
|
||||
title="Standup",
|
||||
source_context="dhive",
|
||||
)
|
||||
artifact = Artifact(
|
||||
metadata=metadata,
|
||||
content="Meeting-Notizen.\n",
|
||||
file_path=Path("standup.md"),
|
||||
)
|
||||
|
||||
result_path = write_artifact(artifact, tmp_path)
|
||||
assert artifact.file_path == result_path
|
||||
@@ -0,0 +1,170 @@
|
||||
"""Tests für den AuditLogger."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from monorepo.audit import AuditLogger, _DEFAULT_AUDIT_LOG_PATH
|
||||
from monorepo.models import SecurityEvent
|
||||
|
||||
|
||||
class TestAuditLoggerInit:
|
||||
"""Tests für die Initialisierung des AuditLoggers."""
|
||||
|
||||
def test_default_log_path(self, tmp_path: Path) -> None:
|
||||
"""AuditLogger mit Standard-Pfad nutzt _DEFAULT_AUDIT_LOG_PATH."""
|
||||
# Wir übergeben explizit einen tmp-Pfad, da der Default relativ ist
|
||||
log_path = tmp_path / ".audit" / "access.log"
|
||||
logger = AuditLogger(log_path=log_path)
|
||||
assert logger.log_path == log_path
|
||||
|
||||
def test_creates_log_directory(self, tmp_path: Path) -> None:
|
||||
"""AuditLogger erstellt das Verzeichnis, falls es nicht existiert."""
|
||||
log_path = tmp_path / "nested" / "deep" / "audit.log"
|
||||
assert not log_path.parent.exists()
|
||||
AuditLogger(log_path=log_path)
|
||||
assert log_path.parent.exists()
|
||||
|
||||
def test_existing_directory_no_error(self, tmp_path: Path) -> None:
|
||||
"""AuditLogger wirft keinen Fehler bei bereits existierendem Verzeichnis."""
|
||||
log_dir = tmp_path / ".audit"
|
||||
log_dir.mkdir()
|
||||
log_path = log_dir / "access.log"
|
||||
logger = AuditLogger(log_path=log_path)
|
||||
assert logger.log_path == log_path
|
||||
|
||||
|
||||
class TestLogViolation:
|
||||
"""Tests für die log_violation-Methode."""
|
||||
|
||||
def _make_event(
|
||||
self,
|
||||
requesting: str = "dhive",
|
||||
target: str = "privat",
|
||||
resource: str = "privat/.env",
|
||||
action: str = "read",
|
||||
outcome: str = "denied",
|
||||
ts: datetime | None = None,
|
||||
) -> SecurityEvent:
|
||||
"""Erstellt ein Test-SecurityEvent."""
|
||||
return SecurityEvent(
|
||||
timestamp=ts or datetime(2025, 1, 15, 10, 30, 0),
|
||||
requesting_context=requesting,
|
||||
target_context=target,
|
||||
resource=resource,
|
||||
action=action,
|
||||
outcome=outcome,
|
||||
)
|
||||
|
||||
def test_writes_single_entry(self, tmp_path: Path) -> None:
|
||||
"""log_violation schreibt einen Eintrag in die Logdatei."""
|
||||
log_path = tmp_path / ".audit" / "access.log"
|
||||
logger = AuditLogger(log_path=log_path)
|
||||
|
||||
event = self._make_event()
|
||||
logger.log_violation(event)
|
||||
|
||||
content = log_path.read_text(encoding="utf-8")
|
||||
assert "[2025-01-15T10:30:00] DENIED | dhive -> privat | read on privat/.env\n" == content
|
||||
|
||||
def test_appends_multiple_entries(self, tmp_path: Path) -> None:
|
||||
"""log_violation fügt Einträge an, ohne bestehende zu überschreiben."""
|
||||
log_path = tmp_path / ".audit" / "access.log"
|
||||
logger = AuditLogger(log_path=log_path)
|
||||
|
||||
event1 = self._make_event(
|
||||
requesting="bahn", target="privat", resource="privat/.env",
|
||||
action="read", outcome="denied",
|
||||
ts=datetime(2025, 1, 15, 10, 0, 0),
|
||||
)
|
||||
event2 = self._make_event(
|
||||
requesting="dhive", target="bahn", resource="bahn/.env",
|
||||
action="write", outcome="denied",
|
||||
ts=datetime(2025, 1, 15, 11, 0, 0),
|
||||
)
|
||||
|
||||
logger.log_violation(event1)
|
||||
logger.log_violation(event2)
|
||||
|
||||
lines = log_path.read_text(encoding="utf-8").splitlines()
|
||||
assert len(lines) == 2
|
||||
assert "bahn -> privat" in lines[0]
|
||||
assert "dhive -> bahn" in lines[1]
|
||||
|
||||
def test_format_contains_iso_timestamp(self, tmp_path: Path) -> None:
|
||||
"""Der Zeitstempel ist im ISO-8601-Format."""
|
||||
log_path = tmp_path / ".audit" / "access.log"
|
||||
logger = AuditLogger(log_path=log_path)
|
||||
|
||||
ts = datetime(2025, 6, 20, 14, 45, 30)
|
||||
event = self._make_event(ts=ts)
|
||||
logger.log_violation(event)
|
||||
|
||||
content = log_path.read_text(encoding="utf-8")
|
||||
assert "[2025-06-20T14:45:30]" in content
|
||||
|
||||
def test_format_outcome_uppercase(self, tmp_path: Path) -> None:
|
||||
"""Das Outcome wird in Großbuchstaben geschrieben."""
|
||||
log_path = tmp_path / ".audit" / "access.log"
|
||||
logger = AuditLogger(log_path=log_path)
|
||||
|
||||
event = self._make_event(outcome="allowed")
|
||||
logger.log_violation(event)
|
||||
|
||||
content = log_path.read_text(encoding="utf-8")
|
||||
assert "ALLOWED" in content
|
||||
|
||||
def test_format_contains_action_and_resource(self, tmp_path: Path) -> None:
|
||||
"""Die Aktion und Ressource werden korrekt formatiert."""
|
||||
log_path = tmp_path / ".audit" / "access.log"
|
||||
logger = AuditLogger(log_path=log_path)
|
||||
|
||||
event = self._make_event(action="execute", resource="shared/tools/script.sh")
|
||||
logger.log_violation(event)
|
||||
|
||||
content = log_path.read_text(encoding="utf-8")
|
||||
assert "execute on shared/tools/script.sh" in content
|
||||
|
||||
def test_thread_safety(self, tmp_path: Path) -> None:
|
||||
"""Parallele Aufrufe erzeugen keine korrupten Einträge."""
|
||||
import threading
|
||||
|
||||
log_path = tmp_path / ".audit" / "access.log"
|
||||
logger = AuditLogger(log_path=log_path)
|
||||
|
||||
events = [
|
||||
self._make_event(
|
||||
requesting=f"ctx{i}",
|
||||
target="privat",
|
||||
resource="privat/.env",
|
||||
ts=datetime(2025, 1, 15, 10, i, 0),
|
||||
)
|
||||
for i in range(20)
|
||||
]
|
||||
|
||||
threads = [
|
||||
threading.Thread(target=logger.log_violation, args=(e,))
|
||||
for e in events
|
||||
]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
lines = log_path.read_text(encoding="utf-8").splitlines()
|
||||
assert len(lines) == 20
|
||||
# Jede Zeile muss vollständig sein (beginnt mit [ und endet mit resource)
|
||||
for line in lines:
|
||||
assert line.startswith("[")
|
||||
assert "privat/.env" in line
|
||||
|
||||
|
||||
class TestDefaultPath:
|
||||
"""Tests für den Standard-Audit-Log-Pfad."""
|
||||
|
||||
def test_default_path_value(self) -> None:
|
||||
"""Der Standard-Pfad entspricht der monorepo.yaml-Konfiguration."""
|
||||
assert _DEFAULT_AUDIT_LOG_PATH == Path(".audit/access.log")
|
||||
@@ -0,0 +1,463 @@
|
||||
"""Unit-Tests für die ContextBridge (Kontextbrücke).
|
||||
|
||||
Testet:
|
||||
- Sensitive-Content-Erkennung (Credentials, Endpoints, PII)
|
||||
- Freigabe-Ablauf mit Nutzerbestätigung
|
||||
- Verweigerung bei sensitivem Inhalt
|
||||
- Widerruf von Freigaben
|
||||
- Update-Propagierung
|
||||
- Registry-Persistierung
|
||||
|
||||
Requirements: 5.1, 5.2, 5.3, 5.4, 5.5, 5.6
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from monorepo.bridge import ContextBridge, SensitiveMatch, ShareResult, SharedArtifactEntry
|
||||
from monorepo.knowledge.index import IndexEntry, YAMLIndex
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tmp_knowledge_base(tmp_path: Path) -> Path:
|
||||
"""Erstellt einen temporären Wissensspeicher mit Test-Artefakten."""
|
||||
kb_path = tmp_path / "knowledge-store"
|
||||
kb_path.mkdir()
|
||||
return kb_path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_artifact_file(tmp_knowledge_base: Path) -> Path:
|
||||
"""Erstellt eine Test-Artefakt-Datei ohne sensitive Inhalte."""
|
||||
artifact_dir = tmp_knowledge_base / "bahn" / "decision"
|
||||
artifact_dir.mkdir(parents=True)
|
||||
artifact_file = artifact_dir / "api-design.md"
|
||||
artifact_file.write_text(
|
||||
"---\n"
|
||||
"type: decision\n"
|
||||
"title: API Design Entscheidung\n"
|
||||
"tags: [api, design]\n"
|
||||
"source_context: bahn\n"
|
||||
"shareable: true\n"
|
||||
"content_hash: sha256:abc123\n"
|
||||
"---\n"
|
||||
"# API Design\n\n"
|
||||
"Wir verwenden REST für die externe API.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return artifact_file
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sensitive_artifact_file(tmp_knowledge_base: Path) -> Path:
|
||||
"""Erstellt eine Test-Artefakt-Datei MIT sensitivem Inhalt."""
|
||||
artifact_dir = tmp_knowledge_base / "dhive" / "note"
|
||||
artifact_dir.mkdir(parents=True)
|
||||
artifact_file = artifact_dir / "service-config.md"
|
||||
artifact_file.write_text(
|
||||
"---\n"
|
||||
"type: note\n"
|
||||
"title: Service Konfiguration\n"
|
||||
"tags: [config]\n"
|
||||
"source_context: dhive\n"
|
||||
"shareable: false\n"
|
||||
"content_hash: sha256:def456\n"
|
||||
"---\n"
|
||||
"# Service Config\n\n"
|
||||
"api_key: sk-12345abcdef\n"
|
||||
"endpoint: https://internal.dhive.example.com/api\n"
|
||||
"Kontakt: admin@dhive-internal.de\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return artifact_file
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def yaml_index(tmp_knowledge_base: Path, sample_artifact_file: Path) -> YAMLIndex:
|
||||
"""Erstellt einen YAML-Index mit einem Test-Eintrag."""
|
||||
index = YAMLIndex(tmp_knowledge_base / "_index.yaml")
|
||||
index.load()
|
||||
|
||||
entry = IndexEntry(
|
||||
id="bahn/decision/api-design",
|
||||
title="API Design Entscheidung",
|
||||
type="decision",
|
||||
tags=["api", "design"],
|
||||
scope="bahn",
|
||||
summary="REST API Entscheidung",
|
||||
path="bahn/decision/api-design.md",
|
||||
content_hash="sha256:abc123",
|
||||
)
|
||||
index.update_entry(entry)
|
||||
return index
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def yaml_index_with_sensitive(
|
||||
yaml_index: YAMLIndex, sensitive_artifact_file: Path
|
||||
) -> YAMLIndex:
|
||||
"""YAML-Index mit einem sensitiven Artefakt."""
|
||||
entry = IndexEntry(
|
||||
id="dhive/note/service-config",
|
||||
title="Service Konfiguration",
|
||||
type="note",
|
||||
tags=["config"],
|
||||
scope="dhive",
|
||||
summary="Service-Konfiguration mit Secrets",
|
||||
path="dhive/note/service-config.md",
|
||||
content_hash="sha256:def456",
|
||||
)
|
||||
yaml_index.update_entry(entry)
|
||||
return yaml_index
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def bridge(tmp_knowledge_base: Path, yaml_index: YAMLIndex, tmp_path: Path) -> ContextBridge:
|
||||
"""Erstellt eine ContextBridge-Instanz mit Registry-Pfad."""
|
||||
registry_path = tmp_path / "config" / "shared-artifacts.yaml"
|
||||
return ContextBridge(
|
||||
knowledge_base_path=tmp_knowledge_base,
|
||||
index=yaml_index,
|
||||
registry_path=registry_path,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def bridge_with_sensitive(
|
||||
tmp_knowledge_base: Path, yaml_index_with_sensitive: YAMLIndex, tmp_path: Path
|
||||
) -> ContextBridge:
|
||||
"""ContextBridge mit sensitiven Artefakten im Index."""
|
||||
registry_path = tmp_path / "config" / "shared-artifacts.yaml"
|
||||
return ContextBridge(
|
||||
knowledge_base_path=tmp_knowledge_base,
|
||||
index=yaml_index_with_sensitive,
|
||||
registry_path=registry_path,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: check_sensitive_content (Req 5.1, 5.3)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCheckSensitiveContent:
|
||||
"""Tests für die Sensitive-Content-Erkennung."""
|
||||
|
||||
def test_no_sensitive_content(self, bridge: ContextBridge) -> None:
|
||||
"""Kein sensitiver Inhalt wird als leere Liste erkannt."""
|
||||
content = "# API Design\n\nWir verwenden REST.\nDetails folgen."
|
||||
result = bridge.check_sensitive_content(content)
|
||||
assert result == []
|
||||
|
||||
def test_detects_api_key(self, bridge: ContextBridge) -> None:
|
||||
"""Erkennt API-Key-Muster."""
|
||||
content = "config:\n api_key: sk-12345abcdef\n"
|
||||
result = bridge.check_sensitive_content(content)
|
||||
assert len(result) >= 1
|
||||
assert any(m.pattern_name == "credentials" for m in result)
|
||||
|
||||
def test_detects_token(self, bridge: ContextBridge) -> None:
|
||||
"""Erkennt Token-Muster."""
|
||||
content = "Authorization:\n token= ghp_abc123xyz\n"
|
||||
result = bridge.check_sensitive_content(content)
|
||||
assert len(result) >= 1
|
||||
assert any(m.pattern_name == "credentials" for m in result)
|
||||
|
||||
def test_detects_password(self, bridge: ContextBridge) -> None:
|
||||
"""Erkennt Passwort-Muster."""
|
||||
content = "db_password: supersecret123\n"
|
||||
result = bridge.check_sensitive_content(content)
|
||||
assert len(result) >= 1
|
||||
assert any(m.pattern_name == "credentials" for m in result)
|
||||
|
||||
def test_detects_secret(self, bridge: ContextBridge) -> None:
|
||||
"""Erkennt Secret-Muster."""
|
||||
content = "client_secret= abc123\n"
|
||||
result = bridge.check_sensitive_content(content)
|
||||
assert len(result) >= 1
|
||||
assert any(m.pattern_name == "credentials" for m in result)
|
||||
|
||||
def test_detects_endpoint_url(self, bridge: ContextBridge) -> None:
|
||||
"""Erkennt Endpoint-URL-Muster."""
|
||||
content = "Konfiguration:\n endpoint: https://api.internal.example.com/v2\n"
|
||||
result = bridge.check_sensitive_content(content)
|
||||
assert len(result) >= 1
|
||||
assert any(m.pattern_name == "endpoint" for m in result)
|
||||
|
||||
def test_detects_email_pii(self, bridge: ContextBridge) -> None:
|
||||
"""Erkennt E-Mail-Adressen als PII."""
|
||||
content = "Ansprechpartner: max.mustermann@example.de\n"
|
||||
result = bridge.check_sensitive_content(content)
|
||||
assert len(result) >= 1
|
||||
assert any(m.pattern_name == "pii" for m in result)
|
||||
|
||||
def test_multiple_sensitive_items(self, bridge: ContextBridge) -> None:
|
||||
"""Erkennt mehrere sensitive Elemente."""
|
||||
content = (
|
||||
"# Config\n"
|
||||
"api_key: abc123\n"
|
||||
"endpoint: https://internal.example.com/api\n"
|
||||
"admin: admin@example.org\n"
|
||||
)
|
||||
result = bridge.check_sensitive_content(content)
|
||||
assert len(result) >= 3
|
||||
pattern_names = {m.pattern_name for m in result}
|
||||
assert "credentials" in pattern_names
|
||||
assert "endpoint" in pattern_names
|
||||
assert "pii" in pattern_names
|
||||
|
||||
def test_line_numbers_correct(self, bridge: ContextBridge) -> None:
|
||||
"""Zeilennummern sind korrekt (1-basiert)."""
|
||||
content = "Zeile 1\nZeile 2\napi_key: geheim\nZeile 4\n"
|
||||
result = bridge.check_sensitive_content(content)
|
||||
assert len(result) >= 1
|
||||
assert result[0].line_number == 3
|
||||
|
||||
def test_empty_content(self, bridge: ContextBridge) -> None:
|
||||
"""Leerer Inhalt ergibt keine Treffer."""
|
||||
result = bridge.check_sensitive_content("")
|
||||
assert result == []
|
||||
|
||||
def test_case_insensitive_credentials(self, bridge: ContextBridge) -> None:
|
||||
"""Credential-Erkennung ist case-insensitive."""
|
||||
content = "API_KEY: value123\nToken: xyz\nPASSWORD= abc\n"
|
||||
result = bridge.check_sensitive_content(content)
|
||||
assert len(result) >= 3
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: share_artifact (Req 5.1, 5.2, 5.3, 5.4)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestShareArtifact:
|
||||
"""Tests für die Artefakt-Freigabe."""
|
||||
|
||||
def test_share_requires_user_confirmation(self, bridge: ContextBridge) -> None:
|
||||
"""Req 5.4: Freigabe ohne Nutzerbestätigung wird verweigert."""
|
||||
result = bridge.share_artifact("bahn/decision/api-design", user_confirmed=False)
|
||||
assert result.success is False
|
||||
assert "Nutzerbestätigung" in result.errors[0]
|
||||
|
||||
def test_share_nonexistent_artifact(self, bridge: ContextBridge) -> None:
|
||||
"""Freigabe eines nicht existierenden Artefakts schlägt fehl."""
|
||||
result = bridge.share_artifact("nonexistent/artifact", user_confirmed=True)
|
||||
assert result.success is False
|
||||
assert "nicht im Index" in result.errors[0]
|
||||
|
||||
def test_share_clean_artifact_succeeds(self, bridge: ContextBridge) -> None:
|
||||
"""Req 5.2: Freigabe eines sauberen Artefakts gelingt."""
|
||||
result = bridge.share_artifact("bahn/decision/api-design", user_confirmed=True)
|
||||
assert result.success is True
|
||||
assert result.artifact_id == "bahn/decision/api-design"
|
||||
assert len(result.shared_to) > 0
|
||||
# Bahn-Artefakt sollte in privat, dhive, shared sichtbar sein
|
||||
assert "bahn" not in result.shared_to
|
||||
assert "privat" in result.shared_to
|
||||
assert "dhive" in result.shared_to
|
||||
assert "shared" in result.shared_to
|
||||
|
||||
def test_share_sensitive_artifact_refused(
|
||||
self, bridge_with_sensitive: ContextBridge
|
||||
) -> None:
|
||||
"""Req 5.3: Freigabe eines Artefakts mit sensitivem Inhalt wird verweigert."""
|
||||
result = bridge_with_sensitive.share_artifact(
|
||||
"dhive/note/service-config", user_confirmed=True
|
||||
)
|
||||
assert result.success is False
|
||||
assert any("sensible Inhalte" in e for e in result.errors)
|
||||
|
||||
def test_share_marks_as_shared(self, bridge: ContextBridge) -> None:
|
||||
"""Nach Freigabe ist das Artefakt als geteilt markiert."""
|
||||
bridge.share_artifact("bahn/decision/api-design", user_confirmed=True)
|
||||
assert bridge.is_shared("bahn/decision/api-design") is True
|
||||
|
||||
def test_unshared_artifact_not_marked(self, bridge: ContextBridge) -> None:
|
||||
"""Nicht freigegebene Artefakte sind nicht als geteilt markiert."""
|
||||
assert bridge.is_shared("bahn/decision/api-design") is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: revoke_share (Req 5.6)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRevokeShare:
|
||||
"""Tests für den Widerruf der Freigabe."""
|
||||
|
||||
def test_revoke_removes_from_registry(self, bridge: ContextBridge) -> None:
|
||||
"""Req 5.6: Widerruf entfernt Artefakt aus der Registry."""
|
||||
bridge.share_artifact("bahn/decision/api-design", user_confirmed=True)
|
||||
assert bridge.is_shared("bahn/decision/api-design") is True
|
||||
|
||||
bridge.revoke_share("bahn/decision/api-design")
|
||||
assert bridge.is_shared("bahn/decision/api-design") is False
|
||||
|
||||
def test_revoke_nonexistent_raises(self, bridge: ContextBridge) -> None:
|
||||
"""Widerruf eines nicht freigegebenen Artefakts wirft ValueError."""
|
||||
with pytest.raises(ValueError, match="nicht freigegeben"):
|
||||
bridge.revoke_share("nonexistent/artifact")
|
||||
|
||||
def test_revoke_removes_from_target_contexts(self, bridge: ContextBridge) -> None:
|
||||
"""Nach Widerruf ist das Artefakt in keinem Zielkontext mehr sichtbar."""
|
||||
bridge.share_artifact("bahn/decision/api-design", user_confirmed=True)
|
||||
bridge.revoke_share("bahn/decision/api-design")
|
||||
|
||||
assert bridge.get_shared_artifacts("privat") == []
|
||||
assert bridge.get_shared_artifacts("dhive") == []
|
||||
assert bridge.get_shared_artifacts("shared") == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: get_shared_content (Update-Propagierung, Req 5.5)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGetSharedContent:
|
||||
"""Tests für Lesezugriff und Update-Propagierung."""
|
||||
|
||||
def test_read_shared_artifact(self, bridge: ContextBridge) -> None:
|
||||
"""Req 5.2: Freigegebenes Artefakt ist in Zielkontext lesbar."""
|
||||
bridge.share_artifact("bahn/decision/api-design", user_confirmed=True)
|
||||
content = bridge.get_shared_content("bahn/decision/api-design", "privat")
|
||||
assert content is not None
|
||||
assert "REST" in content
|
||||
|
||||
def test_read_not_shared_returns_none(self, bridge: ContextBridge) -> None:
|
||||
"""Nicht freigegebenes Artefakt liefert None."""
|
||||
content = bridge.get_shared_content("bahn/decision/api-design", "privat")
|
||||
assert content is None
|
||||
|
||||
def test_read_from_unauthorized_context_returns_none(
|
||||
self, bridge: ContextBridge
|
||||
) -> None:
|
||||
"""Lesezugriff aus Quellkontext (nicht Zielkontext) liefert None."""
|
||||
bridge.share_artifact("bahn/decision/api-design", user_confirmed=True)
|
||||
# Bahn ist der Quellkontext, nicht ein Zielkontext
|
||||
content = bridge.get_shared_content("bahn/decision/api-design", "bahn")
|
||||
assert content is None
|
||||
|
||||
def test_update_propagation(
|
||||
self, bridge: ContextBridge, tmp_knowledge_base: Path
|
||||
) -> None:
|
||||
"""Req 5.5: Änderungen im Quellkontext sind beim nächsten Lesen sichtbar."""
|
||||
bridge.share_artifact("bahn/decision/api-design", user_confirmed=True)
|
||||
|
||||
# Inhalt aktualisieren
|
||||
artifact_path = tmp_knowledge_base / "bahn" / "decision" / "api-design.md"
|
||||
artifact_path.write_text(
|
||||
"---\n"
|
||||
"type: decision\n"
|
||||
"title: API Design Entscheidung\n"
|
||||
"tags: [api, design]\n"
|
||||
"source_context: bahn\n"
|
||||
"shareable: true\n"
|
||||
"content_hash: sha256:updated\n"
|
||||
"---\n"
|
||||
"# API Design v2\n\n"
|
||||
"Wir verwenden GraphQL für die externe API.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# Nächster Lesezugriff sieht den neuen Inhalt
|
||||
content = bridge.get_shared_content("bahn/decision/api-design", "privat")
|
||||
assert content is not None
|
||||
assert "GraphQL" in content
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: get_shared_artifacts
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGetSharedArtifacts:
|
||||
"""Tests für die Abfrage sichtbarer Artefakte pro Kontext."""
|
||||
|
||||
def test_returns_shared_artifacts_for_context(self, bridge: ContextBridge) -> None:
|
||||
"""Listet freigegebene Artefakte für einen Zielkontext."""
|
||||
bridge.share_artifact("bahn/decision/api-design", user_confirmed=True)
|
||||
artifacts = bridge.get_shared_artifacts("privat")
|
||||
assert "bahn/decision/api-design" in artifacts
|
||||
|
||||
def test_empty_for_source_context(self, bridge: ContextBridge) -> None:
|
||||
"""Quellkontext sieht sein eigenes Artefakt nicht in der Share-Liste."""
|
||||
bridge.share_artifact("bahn/decision/api-design", user_confirmed=True)
|
||||
artifacts = bridge.get_shared_artifacts("bahn")
|
||||
assert "bahn/decision/api-design" not in artifacts
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Registry-Persistierung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRegistryPersistence:
|
||||
"""Tests für die YAML-Persistierung der Shared-Artifacts-Registry."""
|
||||
|
||||
def test_registry_persisted_after_share(
|
||||
self, bridge: ContextBridge, tmp_path: Path
|
||||
) -> None:
|
||||
"""Registry wird nach Freigabe auf Festplatte geschrieben."""
|
||||
bridge.share_artifact("bahn/decision/api-design", user_confirmed=True)
|
||||
|
||||
registry_path = tmp_path / "config" / "shared-artifacts.yaml"
|
||||
assert registry_path.exists()
|
||||
|
||||
def test_registry_loaded_on_init(
|
||||
self,
|
||||
tmp_knowledge_base: Path,
|
||||
yaml_index: YAMLIndex,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Registry wird beim Instanzieren aus der Datei geladen."""
|
||||
registry_path = tmp_path / "config" / "shared-artifacts.yaml"
|
||||
|
||||
# Erste Bridge: Artefakt freigeben
|
||||
bridge1 = ContextBridge(
|
||||
knowledge_base_path=tmp_knowledge_base,
|
||||
index=yaml_index,
|
||||
registry_path=registry_path,
|
||||
)
|
||||
bridge1.share_artifact("bahn/decision/api-design", user_confirmed=True)
|
||||
|
||||
# Zweite Bridge: Registry sollte geladen sein
|
||||
bridge2 = ContextBridge(
|
||||
knowledge_base_path=tmp_knowledge_base,
|
||||
index=yaml_index,
|
||||
registry_path=registry_path,
|
||||
)
|
||||
assert bridge2.is_shared("bahn/decision/api-design") is True
|
||||
|
||||
def test_registry_updated_after_revoke(
|
||||
self,
|
||||
tmp_knowledge_base: Path,
|
||||
yaml_index: YAMLIndex,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Registry wird nach Widerruf aktualisiert."""
|
||||
registry_path = tmp_path / "config" / "shared-artifacts.yaml"
|
||||
|
||||
bridge1 = ContextBridge(
|
||||
knowledge_base_path=tmp_knowledge_base,
|
||||
index=yaml_index,
|
||||
registry_path=registry_path,
|
||||
)
|
||||
bridge1.share_artifact("bahn/decision/api-design", user_confirmed=True)
|
||||
bridge1.revoke_share("bahn/decision/api-design")
|
||||
|
||||
# Neue Bridge: Artefakt sollte nicht mehr geteilt sein
|
||||
bridge2 = ContextBridge(
|
||||
knowledge_base_path=tmp_knowledge_base,
|
||||
index=yaml_index,
|
||||
registry_path=registry_path,
|
||||
)
|
||||
assert bridge2.is_shared("bahn/decision/api-design") is False
|
||||
@@ -0,0 +1,249 @@
|
||||
"""Property-Based Tests für explizite Nutzerbestätigung bei Freigabe.
|
||||
|
||||
**Validates: Requirements 5.4**
|
||||
|
||||
Property 16: Freigabe erfordert explizite Nutzerbestätigung
|
||||
- Für jeden Freigabeversuch ohne explizite Nutzerbestätigung (user_confirmed=False)
|
||||
muss die Freigabe verweigert werden und der Zustand des Artefakts unverändert
|
||||
bleiben (nicht geteilt, nicht in der Registry).
|
||||
- Umgekehrt: Wenn user_confirmed=True und das Artefakt valide und frei von
|
||||
sensitiven Inhalten ist, soll die Freigabe gelingen.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from hypothesis import given, settings, HealthCheck
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from monorepo.bridge import ContextBridge, ShareResult
|
||||
from monorepo.knowledge.index import IndexEntry, YAMLIndex
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hilfsfunktionen
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_bridge(tmp_path: Path) -> ContextBridge:
|
||||
"""Erstellt eine minimale ContextBridge-Instanz für Tests."""
|
||||
kb_path = tmp_path / "kb"
|
||||
kb_path.mkdir(parents=True, exist_ok=True)
|
||||
index = YAMLIndex(kb_path / "_index.yaml")
|
||||
index.load()
|
||||
return ContextBridge(knowledge_base_path=kb_path, index=index)
|
||||
|
||||
|
||||
def _make_bridge_with_artifact(tmp_path: Path, artifact_id: str) -> ContextBridge:
|
||||
"""Erstellt eine ContextBridge mit einem sauberen Artefakt im Index und Dateisystem."""
|
||||
kb_path = tmp_path / "kb"
|
||||
kb_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Artefakt-Datei erstellen (sauber, ohne sensitive Inhalte)
|
||||
parts = artifact_id.split("/")
|
||||
artifact_dir = kb_path / "/".join(parts[:-1])
|
||||
artifact_dir.mkdir(parents=True, exist_ok=True)
|
||||
artifact_file = artifact_dir / f"{parts[-1]}.md"
|
||||
artifact_file.write_text(
|
||||
"---\n"
|
||||
f"type: note\n"
|
||||
f"title: Test Artefakt {parts[-1]}\n"
|
||||
f"tags: [test]\n"
|
||||
f"source_context: {parts[0]}\n"
|
||||
f"content_hash: sha256:test123\n"
|
||||
"---\n"
|
||||
"# Test Artefakt\n\n"
|
||||
"Dies ist ein sauberer Inhalt ohne sensitive Daten.\n"
|
||||
"Architektur-Design und allgemeine Dokumentation.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# Index erstellen und Eintrag hinzufügen
|
||||
index = YAMLIndex(kb_path / "_index.yaml")
|
||||
index.load()
|
||||
|
||||
rel_path = str(artifact_file.relative_to(kb_path))
|
||||
entry = IndexEntry(
|
||||
id=artifact_id,
|
||||
title=f"Test Artefakt {parts[-1]}",
|
||||
type="note",
|
||||
tags=["test"],
|
||||
scope=parts[0],
|
||||
summary="Ein sauberes Test-Artefakt",
|
||||
path=rel_path,
|
||||
content_hash="sha256:test123",
|
||||
)
|
||||
index.update_entry(entry)
|
||||
index.save()
|
||||
|
||||
return ContextBridge(knowledge_base_path=kb_path, index=index)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategien
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Zufällige Artefakt-IDs im Format kontext/typ/name
|
||||
artifact_ids = st.from_regex(r"[a-z]{3,8}/[a-z]{4,10}/[a-z\-]{3,15}", fullmatch=True)
|
||||
|
||||
# Kontexte für die Prüfung geteilter Artefakte
|
||||
contexts = st.sampled_from(["privat", "dhive", "bahn", "shared"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 16: Freigabe erfordert explizite Nutzerbestätigung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestUserConfirmationProperties:
|
||||
"""Property-Tests für explizite Nutzerbestätigung.
|
||||
|
||||
**Validates: Requirements 5.4**
|
||||
"""
|
||||
|
||||
@given(artifact_id=artifact_ids)
|
||||
@settings(max_examples=100, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_without_confirmation_always_rejected(
|
||||
self, tmp_path: Path, artifact_id: str
|
||||
) -> None:
|
||||
"""Ohne user_confirmed=True wird jede Freigabe stets verweigert.
|
||||
|
||||
**Validates: Requirements 5.4**
|
||||
|
||||
Unabhängig davon, ob das Artefakt existiert oder nicht, muss
|
||||
share_artifact ohne Bestätigung immer fehlschlagen.
|
||||
"""
|
||||
bridge = _make_bridge(tmp_path)
|
||||
|
||||
result = bridge.share_artifact(artifact_id, user_confirmed=False)
|
||||
|
||||
# success muss False sein
|
||||
assert result.success is False, (
|
||||
f"Freigabe ohne Bestätigung sollte fehlschlagen für: {artifact_id}"
|
||||
)
|
||||
|
||||
@given(artifact_id=artifact_ids)
|
||||
@settings(max_examples=100, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_without_confirmation_error_message_present(
|
||||
self, tmp_path: Path, artifact_id: str
|
||||
) -> None:
|
||||
"""Die Fehlermeldung muss auf fehlende Nutzerbestätigung hinweisen.
|
||||
|
||||
**Validates: Requirements 5.4**
|
||||
"""
|
||||
bridge = _make_bridge(tmp_path)
|
||||
|
||||
result = bridge.share_artifact(artifact_id, user_confirmed=False)
|
||||
|
||||
# Fehlerliste darf nicht leer sein
|
||||
assert len(result.errors) > 0, (
|
||||
f"Fehlermeldung fehlt bei verweigerter Freigabe für: {artifact_id}"
|
||||
)
|
||||
# Mindestens eine Fehlermeldung muss "Bestätigung" oder "confirmed" erwähnen
|
||||
confirmation_mentioned = any(
|
||||
"bestätigung" in err.lower() or "confirmed" in err.lower()
|
||||
for err in result.errors
|
||||
)
|
||||
assert confirmation_mentioned, (
|
||||
f"Fehlermeldung erwähnt nicht die fehlende Bestätigung. "
|
||||
f"Errors: {result.errors}"
|
||||
)
|
||||
|
||||
@given(artifact_id=artifact_ids, context=contexts)
|
||||
@settings(max_examples=100, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_without_confirmation_artifact_stays_unshared(
|
||||
self, tmp_path: Path, artifact_id: str, context: str
|
||||
) -> None:
|
||||
"""Nach verweigerter Freigabe bleibt das Artefakt nicht geteilt.
|
||||
|
||||
**Validates: Requirements 5.4**
|
||||
|
||||
is_shared() muss False bleiben und das Artefakt darf in keinem
|
||||
Kontext als geteiltes Artefakt auftauchen.
|
||||
"""
|
||||
bridge = _make_bridge(tmp_path)
|
||||
|
||||
# Freigabe ohne Bestätigung versuchen
|
||||
bridge.share_artifact(artifact_id, user_confirmed=False)
|
||||
|
||||
# Artefakt darf nicht als geteilt gelten
|
||||
assert bridge.is_shared(artifact_id) is False, (
|
||||
f"Artefakt '{artifact_id}' wurde fälschlicherweise als geteilt markiert"
|
||||
)
|
||||
# Artefakt darf in keinem Kontext als geteiltes Artefakt auftauchen
|
||||
shared_in_context = bridge.get_shared_artifacts(context)
|
||||
assert artifact_id not in shared_in_context, (
|
||||
f"Artefakt '{artifact_id}' taucht fälschlicherweise in Kontext "
|
||||
f"'{context}' auf nach verweigerter Freigabe"
|
||||
)
|
||||
|
||||
@given(artifact_id=artifact_ids)
|
||||
@settings(max_examples=100, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_without_confirmation_registry_unchanged(
|
||||
self, tmp_path: Path, artifact_id: str
|
||||
) -> None:
|
||||
"""Die Registry bleibt bei verweigerter Freigabe unverändert (leer).
|
||||
|
||||
**Validates: Requirements 5.4**
|
||||
"""
|
||||
bridge = _make_bridge(tmp_path)
|
||||
|
||||
# Registry sollte initial leer sein
|
||||
initial_shared_all = (
|
||||
bridge.get_shared_artifacts("privat")
|
||||
+ bridge.get_shared_artifacts("dhive")
|
||||
+ bridge.get_shared_artifacts("bahn")
|
||||
+ bridge.get_shared_artifacts("shared")
|
||||
)
|
||||
assert initial_shared_all == [], "Registry sollte initial leer sein"
|
||||
|
||||
# Freigabe ohne Bestätigung
|
||||
bridge.share_artifact(artifact_id, user_confirmed=False)
|
||||
|
||||
# Registry muss weiterhin leer sein
|
||||
after_shared_all = (
|
||||
bridge.get_shared_artifacts("privat")
|
||||
+ bridge.get_shared_artifacts("dhive")
|
||||
+ bridge.get_shared_artifacts("bahn")
|
||||
+ bridge.get_shared_artifacts("shared")
|
||||
)
|
||||
assert after_shared_all == [], (
|
||||
f"Registry wurde verändert nach verweigerter Freigabe für: {artifact_id}. "
|
||||
f"Enthält: {after_shared_all}"
|
||||
)
|
||||
|
||||
@given(data=st.data())
|
||||
@settings(max_examples=50, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_with_confirmation_valid_artifact_succeeds(
|
||||
self, tmp_path: Path, data: st.DataObject
|
||||
) -> None:
|
||||
"""Mit user_confirmed=True und validem, sauberem Artefakt gelingt die Freigabe.
|
||||
|
||||
**Validates: Requirements 5.4**
|
||||
|
||||
Dies testet die Umkehrung: Wenn die Bestätigung vorliegt und das
|
||||
Artefakt existiert und sauber ist, darf die Freigabe nicht allein
|
||||
wegen fehlender Bestätigung scheitern.
|
||||
"""
|
||||
# Feste, valide Artefakt-IDs verwenden (müssen im Dateisystem existieren)
|
||||
scope = data.draw(st.sampled_from(["privat", "dhive", "bahn"]))
|
||||
artifact_type = data.draw(st.sampled_from(["note", "decision", "meeting"]))
|
||||
name = data.draw(
|
||||
st.from_regex(r"[a-z]{3,10}", fullmatch=True)
|
||||
)
|
||||
artifact_id = f"{scope}/{artifact_type}/{name}"
|
||||
|
||||
bridge = _make_bridge_with_artifact(tmp_path, artifact_id)
|
||||
|
||||
result = bridge.share_artifact(artifact_id, user_confirmed=True)
|
||||
|
||||
# Mit Bestätigung und sauberem Artefakt muss die Freigabe gelingen
|
||||
assert result.success is True, (
|
||||
f"Freigabe mit Bestätigung scheiterte für valides Artefakt '{artifact_id}'. "
|
||||
f"Errors: {result.errors}"
|
||||
)
|
||||
# Das Artefakt muss nun als geteilt gelten
|
||||
assert bridge.is_shared(artifact_id) is True, (
|
||||
f"Artefakt '{artifact_id}' ist nach erfolgreicher Freigabe nicht als geteilt markiert"
|
||||
)
|
||||
@@ -0,0 +1,384 @@
|
||||
"""Property-Based Tests für den Freigabe-Lebenszyklus der ContextBridge.
|
||||
|
||||
**Validates: Requirements 5.2, 5.5, 5.6**
|
||||
|
||||
Property 15: Freigabe-Lebenszyklus (Share → Update → Revoke)
|
||||
- Für jedes Wissensartefakt, das den Freigabe-Lebenszyklus durchläuft
|
||||
(Erstellen → Teilen → Lesen → Aktualisieren → Erneut lesen → Widerrufen → Lesen fehlschlägt),
|
||||
müssen die folgenden Invarianten gelten:
|
||||
1. Nach share_artifact: is_shared == True, get_shared_content liefert Inhalt für berechtigte Kontexte
|
||||
2. Nach Update der Quelldatei: get_shared_content liefert den neuen Inhalt (Update-Propagierung)
|
||||
3. Nach revoke_share: is_shared == False, get_shared_content liefert None
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from hypothesis import given, settings, HealthCheck
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from monorepo.bridge import ContextBridge, ShareResult
|
||||
from monorepo.knowledge.index import IndexEntry, YAMLIndex
|
||||
from monorepo.models import Context
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategien
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Safe titles that won't be blocked by the sensitive filter
|
||||
safe_titles = st.sampled_from([
|
||||
"Design Entscheidung",
|
||||
"Architektur Übersicht",
|
||||
"Sprint Planung",
|
||||
"Code Review Richtlinien",
|
||||
"Modulare Struktur",
|
||||
"Team Vereinbarung",
|
||||
"Prozess Dokumentation",
|
||||
"Technische Bewertung",
|
||||
])
|
||||
|
||||
# Safe content lines (no credentials, endpoints, or email patterns)
|
||||
safe_content_lines = st.lists(
|
||||
st.sampled_from([
|
||||
"REST nutzen",
|
||||
"Microservices",
|
||||
"Clean Architecture",
|
||||
"Modulare Struktur",
|
||||
"Domain Driven Design",
|
||||
"Event Sourcing verwenden",
|
||||
"CQRS Pattern anwenden",
|
||||
"Hexagonale Architektur",
|
||||
"Agile Methoden einsetzen",
|
||||
"Continuous Integration",
|
||||
]),
|
||||
min_size=2,
|
||||
max_size=5,
|
||||
)
|
||||
|
||||
# Scopes for the artifact (source context)
|
||||
artifact_scopes = st.sampled_from(["privat", "dhive", "bahn"])
|
||||
|
||||
# Artifact types
|
||||
artifact_types = st.sampled_from(["decision", "note", "meeting", "reference"])
|
||||
|
||||
# Updated content lines (different from initial content for update propagation)
|
||||
updated_content_lines = st.lists(
|
||||
st.sampled_from([
|
||||
"GraphQL verwenden",
|
||||
"Serverless Architektur",
|
||||
"Monolith aufbrechen",
|
||||
"API Gateway einsetzen",
|
||||
"Container Orchestrierung",
|
||||
"Feature Flags nutzen",
|
||||
"Blue Green Deployment",
|
||||
"Canary Release",
|
||||
]),
|
||||
min_size=2,
|
||||
max_size=5,
|
||||
)
|
||||
|
||||
# Safe tags (no sensitive patterns)
|
||||
safe_tags = st.lists(
|
||||
st.sampled_from(["architektur", "design", "planung", "review", "sprint", "technik"]),
|
||||
min_size=1,
|
||||
max_size=3,
|
||||
unique=True,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hilfsfunktionen
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _build_artifact_content(title: str, artifact_type: str, scope: str,
|
||||
tags: list[str], content_lines: list[str]) -> str:
|
||||
"""Builds a valid artifact file content with YAML frontmatter."""
|
||||
tags_str = ", ".join(tags)
|
||||
content_body = "\n".join(f"- {line}" for line in content_lines)
|
||||
return (
|
||||
f"---\n"
|
||||
f"type: {artifact_type}\n"
|
||||
f"title: \"{title}\"\n"
|
||||
f"tags: [{tags_str}]\n"
|
||||
f"source_context: {scope}\n"
|
||||
f"shareable: true\n"
|
||||
f"content_hash: sha256:initial\n"
|
||||
f"---\n"
|
||||
f"# {title}\n\n"
|
||||
f"{content_body}\n"
|
||||
)
|
||||
|
||||
|
||||
def _create_bridge_with_artifact(
|
||||
tmp_path: Path,
|
||||
artifact_id: str,
|
||||
scope: str,
|
||||
artifact_type: str,
|
||||
title: str,
|
||||
tags: list[str],
|
||||
content_lines: list[str],
|
||||
) -> tuple[ContextBridge, Path]:
|
||||
"""Creates a ContextBridge with a single artifact file and index entry.
|
||||
|
||||
Returns the bridge and the path to the artifact file.
|
||||
"""
|
||||
kb_path = tmp_path / "knowledge-store"
|
||||
kb_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create the artifact file on disk
|
||||
artifact_dir = kb_path / scope / artifact_type
|
||||
artifact_dir.mkdir(parents=True, exist_ok=True)
|
||||
artifact_file = artifact_dir / f"{artifact_id.split('/')[-1]}.md"
|
||||
|
||||
file_content = _build_artifact_content(title, artifact_type, scope, tags, content_lines)
|
||||
artifact_file.write_text(file_content, encoding="utf-8")
|
||||
|
||||
# Create the YAML index
|
||||
index = YAMLIndex(kb_path / "_index.yaml")
|
||||
index.load()
|
||||
|
||||
entry = IndexEntry(
|
||||
id=artifact_id,
|
||||
title=title,
|
||||
type=artifact_type,
|
||||
tags=tags,
|
||||
scope=scope,
|
||||
summary=f"Test-Artefakt: {title}",
|
||||
path=f"{scope}/{artifact_type}/{artifact_id.split('/')[-1]}.md",
|
||||
content_hash="sha256:initial",
|
||||
)
|
||||
index.update_entry(entry)
|
||||
|
||||
# Create bridge with registry
|
||||
registry_path = tmp_path / "config" / "shared-artifacts.yaml"
|
||||
bridge = ContextBridge(
|
||||
knowledge_base_path=kb_path,
|
||||
index=index,
|
||||
registry_path=registry_path,
|
||||
)
|
||||
|
||||
return bridge, artifact_file
|
||||
|
||||
|
||||
def _get_target_contexts(source_scope: str) -> list[str]:
|
||||
"""Returns all contexts that are NOT the source context (target contexts)."""
|
||||
return [ctx.value for ctx in Context if ctx.value != source_scope]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 15: Freigabe-Lebenszyklus (Share → Update → Revoke)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFreigabeLebenszyklus:
|
||||
"""Property-Tests für den vollständigen Freigabe-Lebenszyklus.
|
||||
|
||||
**Validates: Requirements 5.2, 5.5, 5.6**
|
||||
"""
|
||||
|
||||
@given(
|
||||
title=safe_titles,
|
||||
scope=artifact_scopes,
|
||||
artifact_type=artifact_types,
|
||||
tags=safe_tags,
|
||||
initial_content=safe_content_lines,
|
||||
updated_content=updated_content_lines,
|
||||
)
|
||||
@settings(
|
||||
max_examples=50,
|
||||
suppress_health_check=[HealthCheck.function_scoped_fixture],
|
||||
deadline=None,
|
||||
)
|
||||
def test_full_lifecycle_invariants(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
title: str,
|
||||
scope: str,
|
||||
artifact_type: str,
|
||||
tags: list[str],
|
||||
initial_content: list[str],
|
||||
updated_content: list[str],
|
||||
) -> None:
|
||||
"""Der vollständige Lebenszyklus (Share → Read → Update → Read → Revoke → Read fails)
|
||||
muss für beliebige Artefakt-Konfigurationen alle Invarianten einhalten.
|
||||
|
||||
**Validates: Requirements 5.2, 5.5, 5.6**
|
||||
"""
|
||||
# Artifact-ID aus scope/type/name ableiten
|
||||
name_slug = title.lower().replace(" ", "-").replace("ü", "ue")
|
||||
artifact_id = f"{scope}/{artifact_type}/{name_slug}"
|
||||
|
||||
# Setup: Bridge und Artefakt erstellen
|
||||
bridge, artifact_file = _create_bridge_with_artifact(
|
||||
tmp_path, artifact_id, scope, artifact_type, title, tags, initial_content
|
||||
)
|
||||
target_contexts = _get_target_contexts(scope)
|
||||
|
||||
# --- Phase 1: Vor der Freigabe ---
|
||||
assert bridge.is_shared(artifact_id) is False, (
|
||||
"Artefakt darf vor Freigabe nicht als geteilt markiert sein"
|
||||
)
|
||||
for ctx in target_contexts:
|
||||
assert bridge.get_shared_content(artifact_id, ctx) is None, (
|
||||
f"Vor Freigabe darf kein Inhalt für Kontext '{ctx}' geliefert werden"
|
||||
)
|
||||
|
||||
# --- Phase 2: Freigabe (Share) ---
|
||||
result = bridge.share_artifact(artifact_id, user_confirmed=True)
|
||||
assert result.success is True, (
|
||||
f"Freigabe hätte erfolgreich sein müssen, Fehler: {result.errors}"
|
||||
)
|
||||
|
||||
# Invariante 1: Nach share_artifact → is_shared == True
|
||||
assert bridge.is_shared(artifact_id) is True, (
|
||||
"Nach Freigabe muss is_shared True zurückgeben"
|
||||
)
|
||||
|
||||
# Invariante 1: get_shared_content liefert Inhalt für berechtigte Kontexte
|
||||
for ctx in target_contexts:
|
||||
content = bridge.get_shared_content(artifact_id, ctx)
|
||||
assert content is not None, (
|
||||
f"Nach Freigabe muss get_shared_content für Kontext '{ctx}' Inhalt liefern"
|
||||
)
|
||||
# Prüfe, dass der Inhalt mindestens eine der initial_content Zeilen enthält
|
||||
assert any(line in content for line in initial_content), (
|
||||
f"Der gelieferte Inhalt muss den initialen Inhalt enthalten. "
|
||||
f"Erwartet mindestens eins von {initial_content}, erhalten: {content!r}"
|
||||
)
|
||||
|
||||
# Quellkontext darf KEINEN Zugriff haben (nicht in shared_to)
|
||||
source_content = bridge.get_shared_content(artifact_id, scope)
|
||||
assert source_content is None, (
|
||||
"Quellkontext darf keinen Lesezugriff über get_shared_content haben"
|
||||
)
|
||||
|
||||
# --- Phase 3: Update der Quelldatei ---
|
||||
updated_file_content = _build_artifact_content(
|
||||
title, artifact_type, scope, tags, updated_content
|
||||
)
|
||||
artifact_file.write_text(updated_file_content, encoding="utf-8")
|
||||
|
||||
# Invariante 2: get_shared_content liefert den neuen Inhalt (Update-Propagierung)
|
||||
for ctx in target_contexts:
|
||||
content = bridge.get_shared_content(artifact_id, ctx)
|
||||
assert content is not None, (
|
||||
f"Nach Update muss get_shared_content für Kontext '{ctx}' Inhalt liefern"
|
||||
)
|
||||
assert any(line in content for line in updated_content), (
|
||||
f"Nach Update muss der neue Inhalt geliefert werden. "
|
||||
f"Erwartet mindestens eins von {updated_content}, erhalten: {content!r}"
|
||||
)
|
||||
|
||||
# --- Phase 4: Widerruf (Revoke) ---
|
||||
bridge.revoke_share(artifact_id)
|
||||
|
||||
# Invariante 3: Nach revoke → is_shared == False
|
||||
assert bridge.is_shared(artifact_id) is False, (
|
||||
"Nach Widerruf muss is_shared False zurückgeben"
|
||||
)
|
||||
|
||||
# Invariante 3: get_shared_content liefert None
|
||||
for ctx in target_contexts:
|
||||
content = bridge.get_shared_content(artifact_id, ctx)
|
||||
assert content is None, (
|
||||
f"Nach Widerruf muss get_shared_content für Kontext '{ctx}' None liefern"
|
||||
)
|
||||
|
||||
@given(
|
||||
title=safe_titles,
|
||||
scope=artifact_scopes,
|
||||
artifact_type=artifact_types,
|
||||
tags=safe_tags,
|
||||
content=safe_content_lines,
|
||||
)
|
||||
@settings(
|
||||
max_examples=50,
|
||||
suppress_health_check=[HealthCheck.function_scoped_fixture],
|
||||
deadline=None,
|
||||
)
|
||||
def test_share_makes_visible_in_all_target_contexts(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
title: str,
|
||||
scope: str,
|
||||
artifact_type: str,
|
||||
tags: list[str],
|
||||
content: list[str],
|
||||
) -> None:
|
||||
"""Nach Freigabe muss das Artefakt in ALLEN Zielkontexten als Lesereferenz sichtbar sein.
|
||||
|
||||
**Validates: Requirements 5.2**
|
||||
"""
|
||||
name_slug = title.lower().replace(" ", "-").replace("ü", "ue")
|
||||
artifact_id = f"{scope}/{artifact_type}/{name_slug}"
|
||||
|
||||
bridge, _ = _create_bridge_with_artifact(
|
||||
tmp_path, artifact_id, scope, artifact_type, title, tags, content
|
||||
)
|
||||
target_contexts = _get_target_contexts(scope)
|
||||
|
||||
bridge.share_artifact(artifact_id, user_confirmed=True)
|
||||
|
||||
# Artefakt muss in der Liste aller Zielkontexte auftauchen
|
||||
for ctx in target_contexts:
|
||||
shared_list = bridge.get_shared_artifacts(ctx)
|
||||
assert artifact_id in shared_list, (
|
||||
f"Artefakt '{artifact_id}' muss in get_shared_artifacts('{ctx}') enthalten sein"
|
||||
)
|
||||
|
||||
# Artefakt darf NICHT im Quellkontext auftauchen
|
||||
source_list = bridge.get_shared_artifacts(scope)
|
||||
assert artifact_id not in source_list, (
|
||||
f"Artefakt '{artifact_id}' darf NICHT in get_shared_artifacts('{scope}') enthalten sein"
|
||||
)
|
||||
|
||||
@given(
|
||||
title=safe_titles,
|
||||
scope=artifact_scopes,
|
||||
artifact_type=artifact_types,
|
||||
tags=safe_tags,
|
||||
content=safe_content_lines,
|
||||
)
|
||||
@settings(
|
||||
max_examples=50,
|
||||
suppress_health_check=[HealthCheck.function_scoped_fixture],
|
||||
deadline=None,
|
||||
)
|
||||
def test_revoke_removes_from_all_contexts(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
title: str,
|
||||
scope: str,
|
||||
artifact_type: str,
|
||||
tags: list[str],
|
||||
content: list[str],
|
||||
) -> None:
|
||||
"""Nach Widerruf darf das Artefakt in KEINEM Kontext mehr sichtbar sein.
|
||||
|
||||
**Validates: Requirements 5.6**
|
||||
"""
|
||||
name_slug = title.lower().replace(" ", "-").replace("ü", "ue")
|
||||
artifact_id = f"{scope}/{artifact_type}/{name_slug}"
|
||||
|
||||
bridge, _ = _create_bridge_with_artifact(
|
||||
tmp_path, artifact_id, scope, artifact_type, title, tags, content
|
||||
)
|
||||
|
||||
# Share then revoke
|
||||
bridge.share_artifact(artifact_id, user_confirmed=True)
|
||||
bridge.revoke_share(artifact_id)
|
||||
|
||||
# Must not appear in any context
|
||||
for ctx in [c.value for c in Context]:
|
||||
shared_list = bridge.get_shared_artifacts(ctx)
|
||||
assert artifact_id not in shared_list, (
|
||||
f"Nach Widerruf darf Artefakt '{artifact_id}' "
|
||||
f"nicht in get_shared_artifacts('{ctx}') enthalten sein"
|
||||
)
|
||||
content_result = bridge.get_shared_content(artifact_id, ctx)
|
||||
assert content_result is None, (
|
||||
f"Nach Widerruf muss get_shared_content('{artifact_id}', '{ctx}') None liefern"
|
||||
)
|
||||
@@ -0,0 +1,273 @@
|
||||
"""Property-Based Tests für den Sensitive-Content-Filter der ContextBridge.
|
||||
|
||||
**Validates: Requirements 5.1, 5.3**
|
||||
|
||||
Property 14: Sensitive-Content-Filter blockiert Freigabe
|
||||
- Für jeden Text, der kontextspezifische Geheimnisse (API-Keys, Tokens,
|
||||
Passwörter), Endpoint-URLs oder personenbezogene Daten (E-Mail-Adressen)
|
||||
enthält, muss der Sensitive-Content-Filter mindestens einen Treffer
|
||||
zurückgeben und die Freigabe verweigern.
|
||||
- Für jeden Text ohne sensitive Muster muss die Liste leer sein.
|
||||
- Zeilennummern in Treffern müssen korrekt sein.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from hypothesis import given, settings, assume, HealthCheck
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from monorepo.bridge import ContextBridge, SensitiveMatch
|
||||
from monorepo.knowledge.index import YAMLIndex
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hilfsfunktionen
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_bridge(tmp_path: Path) -> ContextBridge:
|
||||
"""Erstellt eine minimale ContextBridge-Instanz für Tests."""
|
||||
kb_path = tmp_path / "kb"
|
||||
kb_path.mkdir(parents=True, exist_ok=True)
|
||||
index = YAMLIndex(kb_path / "_index.yaml")
|
||||
index.load()
|
||||
return ContextBridge(knowledge_base_path=kb_path, index=index)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategien
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Credential-Keywords (case-insensitiv erkannt)
|
||||
credential_keywords = st.sampled_from([
|
||||
"api_key", "API_KEY", "Api_Key",
|
||||
"token", "Token", "TOKEN",
|
||||
"password", "Password", "PASSWORD",
|
||||
"secret", "Secret", "SECRET",
|
||||
"api-key", "API-KEY",
|
||||
])
|
||||
|
||||
# Separatoren zwischen Keyword und Wert
|
||||
separators = st.sampled_from([":", "=", ": ", "= "])
|
||||
|
||||
# Zufällige Werte für Credentials (mindestens 5 Zeichen, alphanumerisch)
|
||||
credential_values = st.text(
|
||||
st.characters(whitelist_categories=("L", "N")),
|
||||
min_size=5,
|
||||
max_size=30,
|
||||
)
|
||||
|
||||
# Strategie für eine Zeile mit Credential-Muster
|
||||
credential_lines = st.builds(
|
||||
lambda kw, sep, val: f"{kw}{sep}{val}",
|
||||
credential_keywords,
|
||||
separators,
|
||||
credential_values,
|
||||
)
|
||||
|
||||
# Strategie für Endpoint-URLs
|
||||
endpoint_hosts = st.text(
|
||||
st.sampled_from("abcdefghijklmnopqrstuvwxyz"),
|
||||
min_size=3,
|
||||
max_size=10,
|
||||
)
|
||||
|
||||
endpoint_paths = st.text(
|
||||
st.sampled_from("abcdefghijklmnopqrstuvwxyz"),
|
||||
min_size=2,
|
||||
max_size=8,
|
||||
)
|
||||
|
||||
endpoint_lines = st.builds(
|
||||
lambda host, path: f"endpoint: https://{host}.example.com/{path}",
|
||||
endpoint_hosts,
|
||||
endpoint_paths,
|
||||
)
|
||||
|
||||
# Alternative: url= pattern
|
||||
url_lines = st.builds(
|
||||
lambda host, path: f"url= https://{host}.internal.dev/{path}",
|
||||
endpoint_hosts,
|
||||
endpoint_paths,
|
||||
)
|
||||
|
||||
# Strategie für E-Mail-Adressen (PII)
|
||||
# Regex erkennt: [A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}
|
||||
# Wir generieren daher nur ASCII-konforme E-Mail-Teile.
|
||||
ascii_alnum = st.sampled_from(
|
||||
"abcdefghijklmnopqrstuvwxyz" "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "0123456789"
|
||||
)
|
||||
|
||||
email_users = st.text(ascii_alnum, min_size=3, max_size=12)
|
||||
|
||||
email_domains = st.text(
|
||||
st.sampled_from("abcdefghijklmnopqrstuvwxyz"),
|
||||
min_size=3,
|
||||
max_size=10,
|
||||
)
|
||||
|
||||
email_tlds = st.sampled_from(["de", "com", "org", "net", "io"])
|
||||
|
||||
email_lines = st.builds(
|
||||
lambda user, domain, tld: f"{user}@{domain}.{tld}",
|
||||
email_users,
|
||||
email_domains,
|
||||
email_tlds,
|
||||
)
|
||||
|
||||
# Strategie für "sauberen" Text ohne sensitive Muster
|
||||
safe_words = st.sampled_from([
|
||||
"Architektur", "Design", "Implementierung", "Modul", "Klasse",
|
||||
"Methode", "Test", "Dokumentation", "Refactoring", "Sprint",
|
||||
"Analyse", "Bewertung", "Ergebnis", "Zusammenfassung", "Planung",
|
||||
])
|
||||
|
||||
clean_text = st.lists(safe_words, min_size=3, max_size=10).map(" ".join)
|
||||
|
||||
# Umgebungstext (Zeilen ohne sensitive Inhalte)
|
||||
filler_lines = st.lists(
|
||||
st.sampled_from([
|
||||
"# Überschrift",
|
||||
"Hier ist ein normaler Satz.",
|
||||
"## Abschnitt",
|
||||
"Notizen zum Design.",
|
||||
"- Punkt eins",
|
||||
"- Punkt zwei",
|
||||
"Weitere Details folgen.",
|
||||
"",
|
||||
]),
|
||||
min_size=0,
|
||||
max_size=5,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 14: Sensitive-Content-Filter blockiert Freigabe
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSensitiveContentFilterProperties:
|
||||
"""Property-Tests für den Sensitive-Content-Filter.
|
||||
|
||||
**Validates: Requirements 5.1, 5.3**
|
||||
"""
|
||||
|
||||
@given(
|
||||
prefix_lines=filler_lines,
|
||||
credential_line=credential_lines,
|
||||
suffix_lines=filler_lines,
|
||||
)
|
||||
@settings(max_examples=100, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_credentials_always_detected(
|
||||
self, tmp_path: Path, prefix_lines: list[str], credential_line: str, suffix_lines: list[str]
|
||||
) -> None:
|
||||
"""Jeder Text mit Credential-Mustern muss mindestens einen Treffer liefern.
|
||||
|
||||
**Validates: Requirements 5.1, 5.3**
|
||||
"""
|
||||
bridge = _make_bridge(tmp_path)
|
||||
content = "\n".join(prefix_lines + [credential_line] + suffix_lines)
|
||||
|
||||
matches = bridge.check_sensitive_content(content)
|
||||
|
||||
assert len(matches) >= 1, (
|
||||
f"Credential-Muster nicht erkannt in: {credential_line!r}"
|
||||
)
|
||||
assert any(m.pattern_name == "credentials" for m in matches), (
|
||||
f"Kein Match mit pattern_name='credentials' für: {credential_line!r}"
|
||||
)
|
||||
|
||||
@given(
|
||||
prefix_lines=filler_lines,
|
||||
endpoint_line=st.one_of(endpoint_lines, url_lines),
|
||||
suffix_lines=filler_lines,
|
||||
)
|
||||
@settings(max_examples=100, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_endpoints_always_detected(
|
||||
self, tmp_path: Path, prefix_lines: list[str], endpoint_line: str, suffix_lines: list[str]
|
||||
) -> None:
|
||||
"""Jeder Text mit Endpoint-URL-Mustern muss mindestens einen Treffer liefern.
|
||||
|
||||
**Validates: Requirements 5.1, 5.3**
|
||||
"""
|
||||
bridge = _make_bridge(tmp_path)
|
||||
content = "\n".join(prefix_lines + [endpoint_line] + suffix_lines)
|
||||
|
||||
matches = bridge.check_sensitive_content(content)
|
||||
|
||||
assert len(matches) >= 1, (
|
||||
f"Endpoint-Muster nicht erkannt in: {endpoint_line!r}"
|
||||
)
|
||||
assert any(m.pattern_name == "endpoint" for m in matches), (
|
||||
f"Kein Match mit pattern_name='endpoint' für: {endpoint_line!r}"
|
||||
)
|
||||
|
||||
@given(
|
||||
prefix_lines=filler_lines,
|
||||
email_line=email_lines,
|
||||
suffix_lines=filler_lines,
|
||||
)
|
||||
@settings(max_examples=100, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_email_pii_always_detected(
|
||||
self, tmp_path: Path, prefix_lines: list[str], email_line: str, suffix_lines: list[str]
|
||||
) -> None:
|
||||
"""Jeder Text mit E-Mail-Adressen muss als PII erkannt werden.
|
||||
|
||||
**Validates: Requirements 5.1, 5.3**
|
||||
"""
|
||||
bridge = _make_bridge(tmp_path)
|
||||
content = "\n".join(prefix_lines + [email_line] + suffix_lines)
|
||||
|
||||
matches = bridge.check_sensitive_content(content)
|
||||
|
||||
assert len(matches) >= 1, (
|
||||
f"E-Mail-PII nicht erkannt in: {email_line!r}"
|
||||
)
|
||||
assert any(m.pattern_name == "pii" for m in matches), (
|
||||
f"Kein Match mit pattern_name='pii' für: {email_line!r}"
|
||||
)
|
||||
|
||||
@given(text=clean_text)
|
||||
@settings(max_examples=100, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_clean_text_no_matches(self, tmp_path: Path, text: str) -> None:
|
||||
"""Text ohne sensitive Muster darf keine Treffer liefern.
|
||||
|
||||
**Validates: Requirements 5.1, 5.3**
|
||||
"""
|
||||
bridge = _make_bridge(tmp_path)
|
||||
matches = bridge.check_sensitive_content(text)
|
||||
|
||||
assert matches == [], (
|
||||
f"Falscher Treffer in sauberem Text: {text!r} -> {matches}"
|
||||
)
|
||||
|
||||
@given(
|
||||
prefix_lines=filler_lines,
|
||||
sensitive_line=st.one_of(credential_lines, endpoint_lines, email_lines),
|
||||
suffix_lines=filler_lines,
|
||||
)
|
||||
@settings(max_examples=100, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_line_numbers_are_correct(
|
||||
self, tmp_path: Path, prefix_lines: list[str], sensitive_line: str, suffix_lines: list[str]
|
||||
) -> None:
|
||||
"""Zeilennummern in Treffern müssen der tatsächlichen Position entsprechen.
|
||||
|
||||
**Validates: Requirements 5.1, 5.3**
|
||||
"""
|
||||
bridge = _make_bridge(tmp_path)
|
||||
content = "\n".join(prefix_lines + [sensitive_line] + suffix_lines)
|
||||
expected_line_number = len(prefix_lines) + 1 # 1-basiert
|
||||
|
||||
matches = bridge.check_sensitive_content(content)
|
||||
|
||||
assert len(matches) >= 1, (
|
||||
f"Kein Treffer für: {sensitive_line!r}"
|
||||
)
|
||||
# Mindestens ein Treffer muss auf der erwarteten Zeile liegen
|
||||
match_lines = {m.line_number for m in matches}
|
||||
assert expected_line_number in match_lines, (
|
||||
f"Erwartete Zeile {expected_line_number}, aber Treffer auf Zeilen: {match_lines}. "
|
||||
f"Content:\n{content}"
|
||||
)
|
||||
@@ -0,0 +1,834 @@
|
||||
"""End-to-End Integration-Tests für den gesamten Komponentenstack.
|
||||
|
||||
Testet die folgenden Workflows über mehrere Komponenten hinweg:
|
||||
1. Projekt erstellen → Env verschlüsseln → Env laden → Secret-Zugriff prüfen
|
||||
2. Artefakt erstellen → Indexieren → Suchen → Freigeben → Widerrufen
|
||||
3. Repo einbinden → Sync → Read-Only-Schutz
|
||||
4. Migration → Validierung → Rollback
|
||||
5. Secret verschlüsseln → Maschinenkontext wechseln → Entschlüsselung fehlschlägt
|
||||
6. Team_Repo vorbereiten → Isolation prüfen → Sync → Konflikt-Auflösung
|
||||
7. Shared-Mirror in Team_Repo → Read-Only-Prüfung
|
||||
|
||||
Requirements: 1.1, 2.1, 3.2, 4.1, 5.2, 6.1, 9.1, 9.4, 10.3, 10.5, 10.12
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from monorepo.bridge import ContextBridge
|
||||
from monorepo.encryption import DecryptionResult, EncryptionResult, SecretEncryptionManager
|
||||
from monorepo.federation import FederationManager
|
||||
from monorepo.knowledge.etl import ETLPipeline
|
||||
from monorepo.knowledge.index import IndexEntry, YAMLIndex
|
||||
from monorepo.knowledge.sources.markdown import MarkdownSource
|
||||
from monorepo.knowledge.store import KnowledgeStore
|
||||
from monorepo.migration import MigrationEngine
|
||||
from monorepo.models import (
|
||||
ConflictInfo,
|
||||
MachineContext,
|
||||
MigrationPlan,
|
||||
RepoEntry,
|
||||
ScopeConfig,
|
||||
SyncResult,
|
||||
)
|
||||
from monorepo.repos import RepoManager
|
||||
from monorepo.security import ContextGuard
|
||||
from monorepo.structure import StructureManager
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def monorepo_root(tmp_path: Path) -> Path:
|
||||
"""Creates a full monorepo structure for E2E integration tests."""
|
||||
root = tmp_path / "monorepo"
|
||||
root.mkdir()
|
||||
|
||||
# Context directories
|
||||
for ctx in ("privat", "dhive", "bahn", "shared"):
|
||||
(root / ctx).mkdir()
|
||||
|
||||
# Shared subdirectories
|
||||
for sub in ("tools", "powers", "knowledge-store", "config", "mcp-servers"):
|
||||
(root / "shared" / sub).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# .env files per context
|
||||
(root / "privat" / ".env").write_text(
|
||||
"PRIVAT_SECRET=privat_val_123\nPRIVAT_API=http://localhost:3000\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(root / "dhive" / ".env").write_text(
|
||||
"DHIVE_SECRET=dhive_val_456\nDHIVE_TOKEN=tok_dhive\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(root / "bahn" / ".env").write_text(
|
||||
"BAHN_SECRET=bahn_val_789\nBAHN_ENDPOINT=https://bahn.api\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(root / "shared" / ".env").write_text(
|
||||
"SHARED_KEY=shared_val\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
# access-config.yaml
|
||||
config_dir = root / "shared" / "config"
|
||||
access_config = {
|
||||
"contexts": {
|
||||
"privat": {
|
||||
"env_file": "privat/.env",
|
||||
"allowed_shared": ["shared/tools/", "shared/config/"],
|
||||
},
|
||||
"dhive": {
|
||||
"env_file": "dhive/.env",
|
||||
"allowed_shared": ["shared/tools/", "shared/config/"],
|
||||
},
|
||||
"bahn": {
|
||||
"env_file": "bahn/.env",
|
||||
"allowed_shared": [
|
||||
"shared/tools/",
|
||||
"shared/config/",
|
||||
"shared/knowledge-store/",
|
||||
],
|
||||
},
|
||||
"shared": {
|
||||
"env_file": "shared/.env",
|
||||
"allowed_shared": ["*"],
|
||||
},
|
||||
}
|
||||
}
|
||||
(config_dir / "access-config.yaml").write_text(
|
||||
yaml.dump(access_config), encoding="utf-8"
|
||||
)
|
||||
|
||||
# machine-context.yaml (full access)
|
||||
machine_config = {
|
||||
"machine": {
|
||||
"name": "test-hauptrechner",
|
||||
"description": "Test machine with full access",
|
||||
"authorized_contexts": ["privat", "dhive", "bahn"],
|
||||
"key_source": "keyring",
|
||||
}
|
||||
}
|
||||
(config_dir / "machine-context.yaml").write_text(
|
||||
yaml.dump(machine_config), encoding="utf-8"
|
||||
)
|
||||
|
||||
# team-repos.yaml
|
||||
team_repos_config = {
|
||||
"version": "1.0",
|
||||
"federation": {
|
||||
"topology": "hub-and-spoke",
|
||||
"hub_owner": "andre",
|
||||
"conflict_strategy": "team-wins",
|
||||
},
|
||||
"team_repos": [
|
||||
{
|
||||
"context": "privat",
|
||||
"url": "https://github.com/test/privat-team.git",
|
||||
"branch": "main",
|
||||
"sync_direction": "bidirectional",
|
||||
"sync_frequency": "manual",
|
||||
"shared_mirror": {
|
||||
"enabled": True,
|
||||
"paths": ["shared/tools/common-scripts/"],
|
||||
"mode": "read-only",
|
||||
},
|
||||
},
|
||||
{
|
||||
"context": "dhive",
|
||||
"url": "https://gitlab.dhive.io/team/dhive-mono.git",
|
||||
"branch": "main",
|
||||
"sync_direction": "bidirectional",
|
||||
"sync_frequency": "on-push",
|
||||
"shared_mirror": {
|
||||
"enabled": True,
|
||||
"paths": ["shared/tools/"],
|
||||
"mode": "read-only",
|
||||
},
|
||||
},
|
||||
{
|
||||
"context": "bahn",
|
||||
"url": "https://gitlab.2700.2db.it/team/bahn-workspace.git",
|
||||
"branch": "main",
|
||||
"sync_direction": "bidirectional",
|
||||
"sync_frequency": "daily",
|
||||
"shared_mirror": {
|
||||
"enabled": True,
|
||||
"paths": ["shared/tools/"],
|
||||
"mode": "read-only",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
(config_dir / "team-repos.yaml").write_text(
|
||||
yaml.dump(team_repos_config), encoding="utf-8"
|
||||
)
|
||||
|
||||
# repos.yaml (empty registry)
|
||||
repos_config = {"repos": []}
|
||||
(config_dir / "repos.yaml").write_text(
|
||||
yaml.dump(repos_config), encoding="utf-8"
|
||||
)
|
||||
|
||||
# Knowledge store index
|
||||
ks_path = root / "shared" / "knowledge-store"
|
||||
(ks_path / "_index.yaml").write_text(
|
||||
yaml.dump({"version": "1.0", "last_updated": "", "artifacts": []}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
return root
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def full_machine_context() -> MachineContext:
|
||||
"""MachineContext authorized for all contexts."""
|
||||
return MachineContext(
|
||||
name="test-hauptrechner",
|
||||
description="Full access",
|
||||
authorized_contexts=["privat", "dhive", "bahn"],
|
||||
key_source="keyring",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def limited_machine_context() -> MachineContext:
|
||||
"""MachineContext authorized only for dhive (simulates dhive-laptop)."""
|
||||
return MachineContext(
|
||||
name="dhive-laptop",
|
||||
description="Only dhive access",
|
||||
authorized_contexts=["dhive"],
|
||||
key_source="keyring",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def scope_config() -> ScopeConfig:
|
||||
"""ScopeConfig for knowledge store."""
|
||||
return ScopeConfig(
|
||||
scopes={
|
||||
"privat": {"scope": "privat", "paths": ["privat/"]},
|
||||
"dhive": {"scope": "dhive", "paths": ["dhive/"]},
|
||||
"bahn": {"scope": "bahn", "paths": ["bahn/"]},
|
||||
"shared": {"scope": "shared", "paths": ["shared/"]},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 1: Projekt erstellen → Env verschlüsseln → Env laden → Secret-Zugriff
|
||||
# Requirements: 1.1, 2.1, 9.1, 9.4
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestProjectEncryptionSecretAccess:
|
||||
"""E2E: Project creation → Encrypt env → Load env → Cross-context denial."""
|
||||
|
||||
def test_create_project_encrypt_env_load_and_deny_cross_context(
|
||||
self, monorepo_root: Path, full_machine_context: MachineContext
|
||||
) -> None:
|
||||
"""Full flow: create project, encrypt its env, load it, deny cross-access."""
|
||||
# Step 1: Create a project in privat context
|
||||
structure_mgr = StructureManager(monorepo_root)
|
||||
project_path = structure_mgr.create_project("privat", "my-new-app")
|
||||
assert project_path.exists()
|
||||
assert project_path == monorepo_root / "privat" / "my-new-app"
|
||||
|
||||
# Step 2: Create a project-level .env for the context
|
||||
env_content = "MY_APP_SECRET=super-secret-123\nDB_URL=postgres://localhost/mydb\n"
|
||||
env_file = monorepo_root / "privat" / ".env"
|
||||
env_file.write_text(env_content, encoding="utf-8")
|
||||
|
||||
# Step 3: Create encryption manager and encrypt the file
|
||||
encryption_mgr = SecretEncryptionManager(monorepo_root, full_machine_context)
|
||||
result = encryption_mgr.encrypt_file(env_file, "privat")
|
||||
# Encryption may or may not succeed (git-crypt might not be installed)
|
||||
# but the flow should be exercised regardless
|
||||
|
||||
# Step 4: Load env via ContextGuard (with encryption integration)
|
||||
guard = ContextGuard(
|
||||
root_path=monorepo_root,
|
||||
encryption_manager=encryption_mgr,
|
||||
)
|
||||
|
||||
# Loading env for own context should succeed
|
||||
env_vars = guard.load_env("privat")
|
||||
assert "MY_APP_SECRET" in env_vars
|
||||
assert env_vars["MY_APP_SECRET"] == "super-secret-123"
|
||||
|
||||
# Step 5: Cross-context access should be denied
|
||||
access_allowed = guard.check_access("dhive", monorepo_root / "privat" / ".env")
|
||||
assert access_allowed is False
|
||||
|
||||
# dhive should not be able to access privat's .env content via its own context
|
||||
access_allowed_bahn = guard.check_access("bahn", monorepo_root / "privat" / ".env")
|
||||
assert access_allowed_bahn is False
|
||||
|
||||
def test_shared_tools_accessible_from_any_context(
|
||||
self, monorepo_root: Path, full_machine_context: MachineContext
|
||||
) -> None:
|
||||
"""Shared tools are accessible from all contexts."""
|
||||
encryption_mgr = SecretEncryptionManager(monorepo_root, full_machine_context)
|
||||
guard = ContextGuard(root_path=monorepo_root, encryption_manager=encryption_mgr)
|
||||
|
||||
# Create a shared tool file
|
||||
tool_file = monorepo_root / "shared" / "tools" / "helper.py"
|
||||
tool_file.write_text("# shared helper", encoding="utf-8")
|
||||
|
||||
# All contexts can access shared tools
|
||||
for ctx in ("privat", "dhive", "bahn"):
|
||||
assert guard.check_access(ctx, tool_file) is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 2: Artefakt erstellen → Indexieren → Suchen → Freigeben → Widerrufen
|
||||
# Requirements: 3.2, 5.2
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestArtifactIngestSearchShareRevoke:
|
||||
"""E2E: Create artifact → Ingest via ETL → Search → Share → Revoke."""
|
||||
|
||||
def test_full_knowledge_lifecycle(
|
||||
self, monorepo_root: Path, scope_config: ScopeConfig
|
||||
) -> None:
|
||||
"""Ingest markdown, search it, share it, then revoke the share."""
|
||||
ks_path = monorepo_root / "shared" / "knowledge-store"
|
||||
|
||||
# Step 1: Create a markdown artifact in bahn context source
|
||||
source_dir = monorepo_root / "bahn" / "docs"
|
||||
source_dir.mkdir(parents=True)
|
||||
artifact_md = source_dir / "api-patterns.md"
|
||||
artifact_md.write_text(
|
||||
"---\n"
|
||||
"type: decision\n"
|
||||
"title: REST API Patterns für InfraGO\n"
|
||||
"tags: [api, rest, patterns]\n"
|
||||
"---\n\n"
|
||||
"# REST API Patterns\n\n"
|
||||
"Wir verwenden RESTful API-Konventionen für alle neuen Services.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# Step 2: Ingest via KnowledgeStore
|
||||
store = KnowledgeStore(ks_path, scope_config)
|
||||
source = MarkdownSource(directory=source_dir)
|
||||
ingest_result = store.ingest(source, "bahn")
|
||||
assert ingest_result.processed >= 1
|
||||
assert ingest_result.updated >= 1
|
||||
|
||||
# Step 3: Search the index
|
||||
results = store.search("REST API", allowed_scopes=["bahn", "shared"])
|
||||
assert len(results) >= 1
|
||||
found_titles = [r.entry.title for r in results]
|
||||
assert any("REST API" in t or "api" in t.lower() for t in found_titles)
|
||||
|
||||
# Step 4: Verify scope filtering - privat should not see bahn artifacts
|
||||
privat_results = store.search("REST API", allowed_scopes=["privat"])
|
||||
bahn_artifact_in_privat = [
|
||||
r for r in privat_results if r.entry.scope == "bahn"
|
||||
]
|
||||
assert len(bahn_artifact_in_privat) == 0
|
||||
|
||||
# Step 5: Share the artifact via ContextBridge
|
||||
bridge = ContextBridge(
|
||||
knowledge_base_path=ks_path,
|
||||
index=store.index,
|
||||
)
|
||||
# Get the artifact ID from the index
|
||||
index_entries = store.get_index(scope="bahn")
|
||||
assert len(index_entries) >= 1
|
||||
artifact_id = index_entries[0].id
|
||||
|
||||
# Share with user confirmation
|
||||
share_result = bridge.share_artifact(artifact_id, user_confirmed=True)
|
||||
assert share_result.success is True
|
||||
assert bridge.is_shared(artifact_id) is True
|
||||
|
||||
# Other contexts can now access the shared content
|
||||
content = bridge.get_shared_content(artifact_id, "privat")
|
||||
assert content is not None
|
||||
assert "REST" in content or "api" in content.lower()
|
||||
|
||||
# Step 6: Revoke the share
|
||||
bridge.revoke_share(artifact_id)
|
||||
assert bridge.is_shared(artifact_id) is False
|
||||
|
||||
# After revocation, other contexts cannot access
|
||||
content_after = bridge.get_shared_content(artifact_id, "privat")
|
||||
assert content_after is None
|
||||
|
||||
def test_share_blocked_for_sensitive_content(
|
||||
self, monorepo_root: Path, scope_config: ScopeConfig
|
||||
) -> None:
|
||||
"""Sensitive artifacts cannot be shared across contexts."""
|
||||
ks_path = monorepo_root / "shared" / "knowledge-store"
|
||||
|
||||
# Create artifact with sensitive content
|
||||
source_dir = monorepo_root / "dhive" / "secrets-docs"
|
||||
source_dir.mkdir(parents=True)
|
||||
sensitive_md = source_dir / "credentials.md"
|
||||
sensitive_md.write_text(
|
||||
"---\n"
|
||||
"type: note\n"
|
||||
"title: API Credentials\n"
|
||||
"tags: [credentials]\n"
|
||||
"---\n\n"
|
||||
"# Credentials\n\n"
|
||||
"api_key: sk-12345-abcdef\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# Ingest the sensitive artifact
|
||||
store = KnowledgeStore(ks_path, scope_config)
|
||||
source = MarkdownSource(directory=source_dir)
|
||||
store.ingest(source, "dhive")
|
||||
|
||||
# Try to share - should be blocked
|
||||
bridge = ContextBridge(knowledge_base_path=ks_path, index=store.index)
|
||||
index_entries = store.get_index(scope="dhive")
|
||||
assert len(index_entries) >= 1
|
||||
artifact_id = index_entries[0].id
|
||||
|
||||
share_result = bridge.share_artifact(artifact_id, user_confirmed=True)
|
||||
assert share_result.success is False
|
||||
assert any("sensib" in e.lower() or "sensitive" in e.lower() for e in share_result.errors)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 3: Repo einbinden → Sync → Read-Only-Schutz
|
||||
# Requirements: 4.1
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRepoAddSyncReadonly:
|
||||
"""E2E: Add repo → Sync → Protect read-only."""
|
||||
|
||||
def test_add_repo_sync_and_readonly_protection(
|
||||
self, monorepo_root: Path
|
||||
) -> None:
|
||||
"""Add an external repo, sync it, and verify read-only protection."""
|
||||
config_path = monorepo_root / "shared" / "config" / "repos.yaml"
|
||||
|
||||
# Initialize a git repo in monorepo_root for hook installation
|
||||
git_dir = monorepo_root / ".git"
|
||||
git_dir.mkdir()
|
||||
hooks_dir = git_dir / "hooks"
|
||||
hooks_dir.mkdir()
|
||||
|
||||
repo_mgr = RepoManager(config_path=config_path, monorepo_root=monorepo_root)
|
||||
|
||||
# Step 1: Add a read-only repo entry (mock git operations)
|
||||
entry = RepoEntry(
|
||||
name="symphony-spec",
|
||||
url="https://github.com/openai/symphony",
|
||||
mode="read-only",
|
||||
target="shared/references/symphony",
|
||||
pinned="v1.0.0",
|
||||
mechanism="subtree",
|
||||
)
|
||||
|
||||
# Mock git subprocess to avoid network calls
|
||||
with patch("monorepo.repos.subprocess.run") as mock_run:
|
||||
mock_run.return_value = MagicMock(
|
||||
returncode=0, stdout="", stderr=""
|
||||
)
|
||||
repo_mgr.add_repo(entry)
|
||||
|
||||
# Verify repo was registered
|
||||
assert repo_mgr._find_repo("symphony-spec") is not None
|
||||
|
||||
# Step 2: Sync the repo (mocked)
|
||||
# Create the target directory to simulate existing subtree
|
||||
target_path = monorepo_root / "shared" / "references" / "symphony"
|
||||
target_path.mkdir(parents=True)
|
||||
(target_path / "README.md").write_text("# Symphony", encoding="utf-8")
|
||||
|
||||
with patch("monorepo.repos.subprocess.run") as mock_run:
|
||||
mock_run.return_value = MagicMock(
|
||||
returncode=0, stdout="1 commit pulled", stderr=""
|
||||
)
|
||||
sync_result = repo_mgr.sync("symphony-spec")
|
||||
# Sync should complete (success or graceful failure)
|
||||
assert sync_result is not None
|
||||
|
||||
# Step 3: Protect read-only
|
||||
repo_mgr.protect_readonly(target_path)
|
||||
|
||||
# Verify pre-commit hook was created/updated
|
||||
hook_path = hooks_dir / "pre-commit"
|
||||
assert hook_path.exists()
|
||||
hook_content = hook_path.read_text(encoding="utf-8")
|
||||
# The hook should reference the protected path
|
||||
assert "symphony" in hook_content or "references" in hook_content
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 4: Migration → Validierung → Rollback
|
||||
# Requirements: 6.1
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMigrationValidateRollback:
|
||||
"""E2E: Migrate repo → Validate → Rollback."""
|
||||
|
||||
def test_migrate_validate_and_rollback(self, monorepo_root: Path) -> None:
|
||||
"""Migrate a repo, validate it, then rollback."""
|
||||
# Initialize git in monorepo_root
|
||||
git_dir = monorepo_root / ".git"
|
||||
git_dir.mkdir(exist_ok=True)
|
||||
|
||||
engine = MigrationEngine(monorepo_root=monorepo_root)
|
||||
|
||||
plan = MigrationPlan(
|
||||
source_repo="https://github.com/test/old-project.git",
|
||||
target_context="dhive",
|
||||
target_name="old-project",
|
||||
mode="subtree",
|
||||
dependencies=[],
|
||||
order=1,
|
||||
)
|
||||
|
||||
# Step 1: Execute migration (mock git calls)
|
||||
with patch.object(engine, "_run_git") as mock_git:
|
||||
mock_git.return_value = MagicMock(
|
||||
returncode=0, stdout="", stderr=""
|
||||
)
|
||||
# Also mock specific detection methods
|
||||
with patch.object(engine, "_get_remote_branches", return_value=["main"]):
|
||||
with patch.object(engine, "_get_local_branches", return_value=["main"]):
|
||||
result = engine.migrate(plan)
|
||||
|
||||
# Migration should succeed or detect conflicts gracefully
|
||||
assert result is not None
|
||||
|
||||
# Step 2: Validate the migration
|
||||
if result.success:
|
||||
# Create target path to simulate successful migration
|
||||
target = monorepo_root / "dhive" / "old-project"
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
(target / "README.md").write_text("# Old Project", encoding="utf-8")
|
||||
|
||||
# Mock git validation commands
|
||||
with patch.object(engine, "_run_git") as mock_git:
|
||||
mock_git.return_value = MagicMock(
|
||||
returncode=0, stdout="5", stderr=""
|
||||
)
|
||||
validation = engine.validate("old-project")
|
||||
assert validation is not None
|
||||
|
||||
# Step 3: Rollback
|
||||
with patch.object(engine, "_run_git") as mock_git:
|
||||
mock_git.return_value = MagicMock(
|
||||
returncode=0, stdout="", stderr=""
|
||||
)
|
||||
engine.rollback("old-project")
|
||||
# After rollback, the project should no longer be in registry
|
||||
assert engine.is_migrated("old-project") is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 5: Secret verschlüsseln → Maschinenkontext wechseln → Entschlüsselung
|
||||
# fehlschlägt
|
||||
# Requirements: 9.1, 9.4
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSecretEncryptionMachineContextSwitch:
|
||||
"""E2E: Encrypt secret → Switch machine context → Decryption fails."""
|
||||
|
||||
def test_encrypt_then_switch_context_decrypt_fails(
|
||||
self,
|
||||
monorepo_root: Path,
|
||||
full_machine_context: MachineContext,
|
||||
limited_machine_context: MachineContext,
|
||||
) -> None:
|
||||
"""Encrypt with full access, then try to decrypt with limited access."""
|
||||
# Step 1: Encrypt a secret file with full machine context
|
||||
full_mgr = SecretEncryptionManager(monorepo_root, full_machine_context)
|
||||
secret_file = monorepo_root / "privat" / ".env"
|
||||
|
||||
encrypt_result = full_mgr.encrypt_file(secret_file, "privat")
|
||||
# The encryption manager should at least attempt to encrypt
|
||||
# (git-crypt may not be available but the flow is tested)
|
||||
|
||||
# Step 2: Verify full context can still decrypt
|
||||
assert full_mgr.is_authorized("privat") is True
|
||||
assert full_mgr.is_authorized("dhive") is True
|
||||
assert full_mgr.is_authorized("bahn") is True
|
||||
|
||||
# Step 3: Switch to limited machine context (only dhive)
|
||||
limited_mgr = SecretEncryptionManager(monorepo_root, limited_machine_context)
|
||||
|
||||
# Step 4: Limited context should NOT be authorized for privat
|
||||
assert limited_mgr.is_authorized("privat") is False
|
||||
assert limited_mgr.is_authorized("bahn") is False
|
||||
assert limited_mgr.is_authorized("dhive") is True
|
||||
|
||||
# Step 5: Decryption attempt for privat should fail/be denied
|
||||
decrypt_result = limited_mgr.decrypt_file(secret_file)
|
||||
# Either returns failure or raises PermissionError
|
||||
if decrypt_result is not None:
|
||||
assert decrypt_result.success is False
|
||||
|
||||
def test_context_guard_denies_load_env_on_unauthorized_machine(
|
||||
self,
|
||||
monorepo_root: Path,
|
||||
limited_machine_context: MachineContext,
|
||||
) -> None:
|
||||
"""ContextGuard denies env loading for unauthorized context on limited machine."""
|
||||
limited_mgr = SecretEncryptionManager(monorepo_root, limited_machine_context)
|
||||
guard = ContextGuard(root_path=monorepo_root, encryption_manager=limited_mgr)
|
||||
|
||||
# dhive env should load fine (authorized)
|
||||
env = guard.load_env("dhive")
|
||||
assert "DHIVE_SECRET" in env
|
||||
|
||||
# privat env should fail (not authorized on this machine)
|
||||
with pytest.raises(PermissionError):
|
||||
guard.load_env("privat")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 6: Team_Repo vorbereiten → Isolation prüfen → Sync → Konflikt-Auflösung
|
||||
# Requirements: 10.3, 10.5
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFederationPrepareIsolateSyncConflict:
|
||||
"""E2E: Prepare team repo → Verify isolation → Sync → Resolve conflicts."""
|
||||
|
||||
def test_prepare_team_repo_verify_isolation(
|
||||
self, monorepo_root: Path, full_machine_context: MachineContext
|
||||
) -> None:
|
||||
"""Prepare team repo and verify it contains only its own context."""
|
||||
encryption_mgr = SecretEncryptionManager(monorepo_root, full_machine_context)
|
||||
config_path = monorepo_root / "shared" / "config" / "team-repos.yaml"
|
||||
|
||||
fed_mgr = FederationManager(
|
||||
config_path=config_path,
|
||||
monorepo_root=monorepo_root,
|
||||
encryption_manager=encryption_mgr,
|
||||
)
|
||||
|
||||
# Create some content in privat context
|
||||
(monorepo_root / "privat" / "my-project").mkdir(parents=True)
|
||||
(monorepo_root / "privat" / "my-project" / "main.py").write_text(
|
||||
"# My private project\nprint('hello')\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
# Create content in other contexts (should NOT appear in privat team repo)
|
||||
(monorepo_root / "dhive" / "dhive-project").mkdir(parents=True)
|
||||
(monorepo_root / "dhive" / "dhive-project" / "app.py").write_text(
|
||||
"# dhive project", encoding="utf-8"
|
||||
)
|
||||
|
||||
# Step 1: Prepare the team repo for privat
|
||||
team_repo_path = fed_mgr.prepare_team_repo("privat")
|
||||
assert team_repo_path.exists()
|
||||
|
||||
# Step 2: Verify isolation - no references to other contexts
|
||||
isolation_report = fed_mgr.verify_isolation(team_repo_path, "privat")
|
||||
assert isolation_report.is_isolated is True
|
||||
assert len(isolation_report.leaks) == 0
|
||||
|
||||
# Verify privat content is present
|
||||
privat_files = list(team_repo_path.rglob("*.py"))
|
||||
assert len(privat_files) >= 1
|
||||
|
||||
# Verify dhive/bahn content is NOT present
|
||||
dhive_files = list(team_repo_path.rglob("*dhive*"))
|
||||
# Filter out any path references in filenames only
|
||||
for f in dhive_files:
|
||||
assert "dhive-project" not in str(f) or not f.exists()
|
||||
|
||||
def test_sync_and_conflict_resolution(
|
||||
self, monorepo_root: Path, full_machine_context: MachineContext
|
||||
) -> None:
|
||||
"""Sync from team repo and resolve conflicts with team-wins strategy."""
|
||||
encryption_mgr = SecretEncryptionManager(monorepo_root, full_machine_context)
|
||||
config_path = monorepo_root / "shared" / "config" / "team-repos.yaml"
|
||||
|
||||
fed_mgr = FederationManager(
|
||||
config_path=config_path,
|
||||
monorepo_root=monorepo_root,
|
||||
encryption_manager=encryption_mgr,
|
||||
)
|
||||
|
||||
# Mock the git subtree operations (no actual network calls)
|
||||
with patch.object(
|
||||
fed_mgr.sync_engine, "subtree_pull"
|
||||
) as mock_pull:
|
||||
mock_pull.return_value = SyncResult(
|
||||
success=True,
|
||||
context="privat",
|
||||
direction="pull",
|
||||
commits_synced=3,
|
||||
conflicts=[],
|
||||
)
|
||||
result = fed_mgr.sync_from_team("privat")
|
||||
assert result.success is True
|
||||
assert result.commits_synced == 3
|
||||
|
||||
# Test conflict resolution with team-wins strategy
|
||||
with patch.object(
|
||||
fed_mgr.sync_engine, "subtree_pull"
|
||||
) as mock_pull:
|
||||
mock_pull.return_value = SyncResult(
|
||||
success=False,
|
||||
context="dhive",
|
||||
direction="pull",
|
||||
commits_synced=0,
|
||||
conflicts=[
|
||||
ConflictInfo(
|
||||
file_path="dhive/config.yaml",
|
||||
conflict_type="content",
|
||||
source="team-repo",
|
||||
details="Conflicting changes in config.yaml",
|
||||
)
|
||||
],
|
||||
)
|
||||
conflict_result = fed_mgr.sync_from_team("dhive")
|
||||
assert conflict_result.success is False
|
||||
assert len(conflict_result.conflicts) >= 1
|
||||
|
||||
# Resolve conflict with team-wins strategy (mock git operations)
|
||||
with patch.object(fed_mgr.sync_engine, "_run_git") as mock_git:
|
||||
# First call: git diff to find conflicted files
|
||||
mock_git.return_value = MagicMock(
|
||||
stdout="dhive/config.yaml\n", stderr="", returncode=0
|
||||
)
|
||||
resolved = fed_mgr.resolve_conflict("dhive", strategy="team-wins")
|
||||
assert resolved is not None
|
||||
assert resolved.context == "dhive"
|
||||
assert resolved.strategy == "team-wins"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 7: Shared-Mirror in Team_Repo → Read-Only-Prüfung
|
||||
# Requirements: 10.12
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSharedMirrorReadOnly:
|
||||
"""E2E: Mirror shared files into team repo → Verify read-only."""
|
||||
|
||||
def test_mirror_shared_to_team_repo_readonly(
|
||||
self, monorepo_root: Path, full_machine_context: MachineContext
|
||||
) -> None:
|
||||
"""Shared files mirrored to team repo should be read-only."""
|
||||
encryption_mgr = SecretEncryptionManager(monorepo_root, full_machine_context)
|
||||
config_path = monorepo_root / "shared" / "config" / "team-repos.yaml"
|
||||
|
||||
fed_mgr = FederationManager(
|
||||
config_path=config_path,
|
||||
monorepo_root=monorepo_root,
|
||||
encryption_manager=encryption_mgr,
|
||||
)
|
||||
|
||||
# Create shared content to be mirrored
|
||||
shared_tools = monorepo_root / "shared" / "tools" / "common-scripts"
|
||||
shared_tools.mkdir(parents=True)
|
||||
(shared_tools / "deploy.sh").write_text(
|
||||
"#!/bin/bash\necho 'deploying...'\n", encoding="utf-8"
|
||||
)
|
||||
(shared_tools / "lint.sh").write_text(
|
||||
"#!/bin/bash\necho 'linting...'\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
# Step 1: Mirror shared files to privat team repo
|
||||
mirror_result = fed_mgr.mirror_shared(
|
||||
"privat", ["shared/tools/common-scripts/"]
|
||||
)
|
||||
assert mirror_result is not None
|
||||
assert mirror_result.success is True
|
||||
|
||||
# Step 2: Verify mirrored paths are recorded
|
||||
assert len(mirror_result.mirrored_paths) >= 1
|
||||
|
||||
# Step 3: Verify context is correct
|
||||
assert mirror_result.context == "privat"
|
||||
|
||||
def test_mirror_shared_with_multiple_paths(
|
||||
self, monorepo_root: Path, full_machine_context: MachineContext
|
||||
) -> None:
|
||||
"""Multiple shared paths can be mirrored to team repo."""
|
||||
encryption_mgr = SecretEncryptionManager(monorepo_root, full_machine_context)
|
||||
config_path = monorepo_root / "shared" / "config" / "team-repos.yaml"
|
||||
|
||||
fed_mgr = FederationManager(
|
||||
config_path=config_path,
|
||||
monorepo_root=monorepo_root,
|
||||
encryption_manager=encryption_mgr,
|
||||
)
|
||||
|
||||
# Create multiple shared resources
|
||||
(monorepo_root / "shared" / "tools" / "script-a.py").write_text(
|
||||
"# script A", encoding="utf-8"
|
||||
)
|
||||
(monorepo_root / "shared" / "tools" / "script-b.py").write_text(
|
||||
"# script B", encoding="utf-8"
|
||||
)
|
||||
|
||||
mirror_result = fed_mgr.mirror_shared("dhive", ["shared/tools/"])
|
||||
assert mirror_result is not None
|
||||
assert mirror_result.success is True
|
||||
assert len(mirror_result.mirrored_paths) >= 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test: Integration Stack Factory (create_integrated_stack)
|
||||
# Requirements: 2.2, 9.3, 9.4, 10.10
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestIntegratedStackFactory:
|
||||
"""E2E: Verify create_integrated_stack wires all components correctly."""
|
||||
|
||||
def test_create_integrated_stack_full(self, monorepo_root: Path) -> None:
|
||||
"""Full stack creation with all components wired."""
|
||||
from monorepo.integration import create_integrated_stack
|
||||
|
||||
stack = create_integrated_stack(monorepo_root)
|
||||
|
||||
# All components should be created
|
||||
assert stack["context_guard"] is not None
|
||||
assert stack["audit_logger"] is not None
|
||||
assert stack["orchestrator_adapter"] is not None
|
||||
assert stack["encryption_manager"] is not None
|
||||
assert stack["federation_manager"] is not None
|
||||
assert stack["machine_context_manager"] is not None
|
||||
|
||||
# Verify the ContextGuard has encryption wired
|
||||
guard = stack["context_guard"]
|
||||
assert guard.encryption_manager is not None
|
||||
|
||||
# Verify the guard can load env with encryption
|
||||
env = guard.load_env("privat")
|
||||
assert "PRIVAT_SECRET" in env
|
||||
|
||||
def test_create_integrated_stack_cross_context_denied(
|
||||
self, monorepo_root: Path
|
||||
) -> None:
|
||||
"""Integrated stack correctly denies cross-context access."""
|
||||
from monorepo.integration import create_integrated_stack
|
||||
|
||||
stack = create_integrated_stack(monorepo_root)
|
||||
guard = stack["context_guard"]
|
||||
|
||||
# Cross-context access should be denied
|
||||
assert guard.check_access("privat", monorepo_root / "dhive" / ".env") is False
|
||||
assert guard.check_access("dhive", monorepo_root / "bahn" / ".env") is False
|
||||
assert guard.check_access("bahn", monorepo_root / "privat" / ".env") is False
|
||||
|
||||
# Same-context access should be allowed
|
||||
assert guard.check_access("privat", monorepo_root / "privat" / ".env") is True
|
||||
assert guard.check_access("dhive", monorepo_root / "dhive" / ".env") is True
|
||||
@@ -0,0 +1,462 @@
|
||||
"""Tests für SecretEncryptionManager – Maschinenkontext-Verwaltung (Task 4.2).
|
||||
|
||||
Testet:
|
||||
- get_context_key: Schlüsselabruf aus verschiedenen Quellen
|
||||
- onboard_machine: Einrichtung neuer Maschinen
|
||||
- resolve_merge: Merge-Konflikt-Auflösung auf verschlüsselter Ebene
|
||||
- MachineContextManager: Laden der Konfiguration aus YAML
|
||||
- Passwort-Manager-Integration
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from monorepo.encryption import (
|
||||
MachineContextManager,
|
||||
PasswordManagerError,
|
||||
SecretEncryptionManager,
|
||||
)
|
||||
from monorepo.models import (
|
||||
EncryptionKey,
|
||||
MachineContext,
|
||||
OnboardingResult,
|
||||
PasswordManagerConfig,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tmp_monorepo(tmp_path: Path) -> Path:
|
||||
"""Erstellt eine minimale Monorepo-Struktur in tmp_path."""
|
||||
# Kontextordner
|
||||
for ctx in ("privat", "dhive", "bahn", "shared"):
|
||||
(tmp_path / ctx).mkdir()
|
||||
|
||||
# .git/git-crypt/keys Struktur simulieren
|
||||
keys_dir = tmp_path / ".git" / "git-crypt" / "keys"
|
||||
keys_dir.mkdir(parents=True)
|
||||
(keys_dir / "default").mkdir()
|
||||
(keys_dir / "privat").mkdir()
|
||||
(keys_dir / "dhive").mkdir()
|
||||
|
||||
# Konfigurationsdateien
|
||||
config_dir = tmp_path / "shared" / "config"
|
||||
config_dir.mkdir(parents=True)
|
||||
|
||||
machine_context_yaml = {
|
||||
"machine": {
|
||||
"name": "test-rechner",
|
||||
"description": "Testmaschine",
|
||||
"authorized_contexts": ["privat", "dhive", "bahn"],
|
||||
"key_source": "keyring",
|
||||
"password_manager": {
|
||||
"type": "bitwarden",
|
||||
"vault": "monorepo-keys",
|
||||
},
|
||||
}
|
||||
}
|
||||
(config_dir / "machine-context.yaml").write_text(
|
||||
yaml.dump(machine_context_yaml), encoding="utf-8"
|
||||
)
|
||||
|
||||
return tmp_path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def full_access_context() -> MachineContext:
|
||||
"""MachineContext mit Zugriff auf alle Kontexte."""
|
||||
return MachineContext(
|
||||
name="andre-hauptrechner",
|
||||
description="Voller Zugriff",
|
||||
authorized_contexts=["privat", "dhive", "bahn"],
|
||||
key_source="keyring",
|
||||
password_manager=PasswordManagerConfig(
|
||||
type="bitwarden",
|
||||
vault="monorepo-keys",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def limited_context() -> MachineContext:
|
||||
"""MachineContext mit eingeschränktem Zugriff (nur dhive)."""
|
||||
return MachineContext(
|
||||
name="dhive-laptop",
|
||||
description="Nur dhive-Zugriff",
|
||||
authorized_contexts=["dhive"],
|
||||
key_source="keyring",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def manager(tmp_monorepo: Path, full_access_context: MachineContext) -> SecretEncryptionManager:
|
||||
"""SecretEncryptionManager mit vollem Zugriff."""
|
||||
return SecretEncryptionManager(tmp_monorepo, full_access_context)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def limited_manager(
|
||||
tmp_monorepo: Path, limited_context: MachineContext
|
||||
) -> SecretEncryptionManager:
|
||||
"""SecretEncryptionManager mit eingeschränktem Zugriff."""
|
||||
return SecretEncryptionManager(tmp_monorepo, limited_context)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: get_context_key
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGetContextKey:
|
||||
"""Tests für die Methode get_context_key."""
|
||||
|
||||
def test_returns_key_for_authorized_context_from_keyring(
|
||||
self, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Autorisierter Kontext mit vorhandenem Schlüssel liefert EncryptionKey."""
|
||||
key = manager.get_context_key("privat")
|
||||
assert key is not None
|
||||
assert isinstance(key, EncryptionKey)
|
||||
assert key.context == "privat"
|
||||
assert key.source == "keyring"
|
||||
|
||||
def test_returns_key_for_dhive_context(
|
||||
self, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Dhive-Kontext liefert ebenfalls einen Schlüssel."""
|
||||
key = manager.get_context_key("dhive")
|
||||
assert key is not None
|
||||
assert key.context == "dhive"
|
||||
|
||||
def test_returns_none_for_unauthorized_context(
|
||||
self, limited_manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Nicht-autorisierter Kontext liefert None."""
|
||||
key = limited_manager.get_context_key("privat")
|
||||
assert key is None
|
||||
|
||||
def test_returns_none_for_nonexistent_key(
|
||||
self, tmp_monorepo: Path
|
||||
) -> None:
|
||||
"""Kontext ohne vorhandenen Schlüssel liefert None."""
|
||||
# Kein .git/git-crypt/keys/bahn Verzeichnis
|
||||
ctx = MachineContext(
|
||||
name="test",
|
||||
description="",
|
||||
authorized_contexts=["bahn"],
|
||||
key_source="keyring",
|
||||
)
|
||||
mgr = SecretEncryptionManager(tmp_monorepo, ctx)
|
||||
# bahn existiert nicht im keys-Verzeichnis, aber default existiert
|
||||
key = mgr.get_context_key("bahn")
|
||||
# Fallback auf default key
|
||||
assert key is not None
|
||||
assert "default" in key.key_id
|
||||
|
||||
def test_file_source_returns_key_when_file_exists(
|
||||
self, tmp_monorepo: Path
|
||||
) -> None:
|
||||
"""Dateiquelle liefert Schlüssel wenn Datei vorhanden."""
|
||||
# Schlüsseldatei anlegen
|
||||
key_dir = Path.home() / ".monorepo" / "keys"
|
||||
key_dir.mkdir(parents=True, exist_ok=True)
|
||||
key_file = key_dir / "privat.key"
|
||||
key_file.write_text("test-key-content")
|
||||
|
||||
try:
|
||||
ctx = MachineContext(
|
||||
name="file-test",
|
||||
description="",
|
||||
authorized_contexts=["privat"],
|
||||
key_source="file",
|
||||
)
|
||||
mgr = SecretEncryptionManager(tmp_monorepo, ctx)
|
||||
key = mgr.get_context_key("privat")
|
||||
assert key is not None
|
||||
assert key.source == "file"
|
||||
assert key.context == "privat"
|
||||
finally:
|
||||
key_file.unlink(missing_ok=True)
|
||||
|
||||
def test_file_source_returns_none_when_no_file(
|
||||
self, tmp_monorepo: Path
|
||||
) -> None:
|
||||
"""Dateiquelle liefert None wenn keine Datei vorhanden."""
|
||||
# Stelle sicher, dass keine Datei existiert
|
||||
key_file = Path.home() / ".monorepo" / "keys" / "nonexistent-ctx.key"
|
||||
key_file.unlink(missing_ok=True)
|
||||
|
||||
ctx = MachineContext(
|
||||
name="file-test",
|
||||
description="",
|
||||
authorized_contexts=["nonexistent-ctx"],
|
||||
key_source="file",
|
||||
)
|
||||
mgr = SecretEncryptionManager(tmp_monorepo, ctx)
|
||||
key = mgr.get_context_key("nonexistent-ctx")
|
||||
assert key is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: onboard_machine
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestOnboardMachine:
|
||||
"""Tests für die Methode onboard_machine."""
|
||||
|
||||
def test_successful_onboarding(self, manager: SecretEncryptionManager) -> None:
|
||||
"""Erfolgreiches Onboarding installiert Schlüssel und aktualisiert Config."""
|
||||
result = manager.onboard_machine("neuer-rechner", ["privat", "dhive"])
|
||||
assert isinstance(result, OnboardingResult)
|
||||
assert result.success is True
|
||||
assert result.machine_name == "neuer-rechner"
|
||||
assert "privat" in result.authorized_contexts
|
||||
assert "dhive" in result.authorized_contexts
|
||||
assert len(result.installed_keys) >= 1
|
||||
assert len(result.errors) == 0
|
||||
|
||||
def test_onboarding_with_invalid_context(
|
||||
self, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Ungültiger Kontext erzeugt Fehler im OnboardingResult."""
|
||||
result = manager.onboard_machine("test-rechner", ["invalid-context"])
|
||||
assert result.success is False
|
||||
assert "Ungültiger Kontext: 'invalid-context'" in result.errors
|
||||
|
||||
def test_onboarding_partial_success(
|
||||
self, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Mix aus gültigen und ungültigen Kontexten."""
|
||||
result = manager.onboard_machine("mix-rechner", ["privat", "invalid"])
|
||||
# privat sollte erfolgreich sein, invalid erzeugt Fehler
|
||||
assert "Ungültiger Kontext: 'invalid'" in result.errors
|
||||
assert "privat" in result.authorized_contexts or len(result.installed_keys) > 0
|
||||
|
||||
def test_onboarding_updates_config_file(
|
||||
self, tmp_monorepo: Path, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Onboarding aktualisiert die machine-context.yaml."""
|
||||
manager.onboard_machine("config-test-rechner", ["privat"])
|
||||
|
||||
config_path = tmp_monorepo / "shared" / "config" / "machine-context.yaml"
|
||||
assert config_path.exists()
|
||||
|
||||
with open(config_path, encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f)
|
||||
|
||||
assert data["machine"]["name"] == "config-test-rechner"
|
||||
assert "privat" in data["machine"]["authorized_contexts"]
|
||||
|
||||
def test_onboarding_empty_contexts(
|
||||
self, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Leere Kontextliste führt zu Misserfolg."""
|
||||
result = manager.onboard_machine("empty-rechner", [])
|
||||
assert result.success is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: resolve_merge
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestResolveMerge:
|
||||
"""Tests für die Methode resolve_merge."""
|
||||
|
||||
def test_identical_versions_return_same(
|
||||
self, tmp_monorepo: Path, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Identische Versionen erzeugen keinen Konflikt."""
|
||||
file_path = tmp_monorepo / "privat" / "test.env"
|
||||
file_path.write_bytes(b"content")
|
||||
|
||||
result = manager.resolve_merge(file_path, b"same", b"same")
|
||||
assert result == b"same"
|
||||
|
||||
def test_theirs_wins_for_authorized_context(
|
||||
self, tmp_monorepo: Path, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Bei autorisiertem Kontext gewinnt theirs (Team-Repo-Vorrang)."""
|
||||
file_path = tmp_monorepo / "privat" / "secret.key"
|
||||
file_path.write_bytes(b"dummy")
|
||||
|
||||
result = manager.resolve_merge(file_path, b"ours-version", b"theirs-version")
|
||||
assert result == b"theirs-version"
|
||||
|
||||
def test_ours_wins_for_unauthorized_context(
|
||||
self, tmp_monorepo: Path, limited_manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Bei nicht-autorisiertem Kontext wird ours beibehalten."""
|
||||
file_path = tmp_monorepo / "privat" / "secret.key"
|
||||
file_path.write_bytes(b"dummy")
|
||||
|
||||
result = limited_manager.resolve_merge(
|
||||
file_path, b"ours-version", b"theirs-version"
|
||||
)
|
||||
assert result == b"ours-version"
|
||||
|
||||
def test_ours_wins_for_unknown_context(
|
||||
self, tmp_monorepo: Path, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Datei außerhalb der Kontextordner: ours beibehalten."""
|
||||
file_path = tmp_monorepo / "unknown-dir" / "file.txt"
|
||||
|
||||
result = manager.resolve_merge(file_path, b"ours", b"theirs")
|
||||
assert result == b"ours"
|
||||
|
||||
def test_binary_content_handled(
|
||||
self, tmp_monorepo: Path, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Binärdaten werden korrekt verarbeitet."""
|
||||
file_path = tmp_monorepo / "dhive" / "cert.pem"
|
||||
file_path.write_bytes(b"\x00\x01\x02")
|
||||
|
||||
ours = b"\x00\x01\x02\x03"
|
||||
theirs = b"\x00\x01\x02\x04\x05"
|
||||
result = manager.resolve_merge(file_path, ours, theirs)
|
||||
assert result == theirs # dhive ist autorisiert
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: MachineContextManager
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMachineContextManager:
|
||||
"""Tests für den MachineContextManager."""
|
||||
|
||||
def test_load_valid_config(self, tmp_monorepo: Path) -> None:
|
||||
"""Lädt gültige machine-context.yaml korrekt."""
|
||||
mcm = MachineContextManager(tmp_monorepo)
|
||||
ctx = mcm.load()
|
||||
|
||||
assert ctx.name == "test-rechner"
|
||||
assert ctx.description == "Testmaschine"
|
||||
assert "privat" in ctx.authorized_contexts
|
||||
assert "dhive" in ctx.authorized_contexts
|
||||
assert "bahn" in ctx.authorized_contexts
|
||||
assert ctx.key_source == "keyring"
|
||||
assert ctx.password_manager is not None
|
||||
assert ctx.password_manager.type == "bitwarden"
|
||||
assert ctx.password_manager.vault == "monorepo-keys"
|
||||
|
||||
def test_load_missing_config_raises(self, tmp_path: Path) -> None:
|
||||
"""Fehlende Konfigurationsdatei wirft FileNotFoundError."""
|
||||
mcm = MachineContextManager(tmp_path)
|
||||
with pytest.raises(FileNotFoundError):
|
||||
mcm.load()
|
||||
|
||||
def test_load_invalid_config_raises(self, tmp_path: Path) -> None:
|
||||
"""Ungültiges YAML-Format wirft ValueError."""
|
||||
config_dir = tmp_path / "shared" / "config"
|
||||
config_dir.mkdir(parents=True)
|
||||
(config_dir / "machine-context.yaml").write_text("invalid: true")
|
||||
|
||||
mcm = MachineContextManager(tmp_path)
|
||||
with pytest.raises(ValueError, match="machine"):
|
||||
mcm.load()
|
||||
|
||||
def test_machine_context_property_before_load_raises(
|
||||
self, tmp_monorepo: Path
|
||||
) -> None:
|
||||
"""Zugriff auf machine_context vor load() wirft RuntimeError."""
|
||||
mcm = MachineContextManager(tmp_monorepo)
|
||||
with pytest.raises(RuntimeError, match="load"):
|
||||
_ = mcm.machine_context
|
||||
|
||||
def test_machine_context_property_after_load(
|
||||
self, tmp_monorepo: Path
|
||||
) -> None:
|
||||
"""Nach load() ist machine_context verfügbar."""
|
||||
mcm = MachineContextManager(tmp_monorepo)
|
||||
mcm.load()
|
||||
ctx = mcm.machine_context
|
||||
assert ctx.name == "test-rechner"
|
||||
|
||||
def test_create_encryption_manager(self, tmp_monorepo: Path) -> None:
|
||||
"""create_encryption_manager erstellt funktionsfähigen Manager."""
|
||||
mcm = MachineContextManager(tmp_monorepo)
|
||||
mgr = mcm.create_encryption_manager()
|
||||
assert isinstance(mgr, SecretEncryptionManager)
|
||||
assert mgr.machine_context.name == "test-rechner"
|
||||
|
||||
def test_custom_config_path(self, tmp_path: Path) -> None:
|
||||
"""Benutzerdefinierter config_path wird korrekt verwendet."""
|
||||
custom_config = tmp_path / "custom-config.yaml"
|
||||
custom_config.write_text(
|
||||
yaml.dump({
|
||||
"machine": {
|
||||
"name": "custom-machine",
|
||||
"description": "Custom",
|
||||
"authorized_contexts": ["privat"],
|
||||
"key_source": "file",
|
||||
}
|
||||
}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
mcm = MachineContextManager(tmp_path, config_path=custom_config)
|
||||
ctx = mcm.load()
|
||||
assert ctx.name == "custom-machine"
|
||||
assert ctx.key_source == "file"
|
||||
|
||||
def test_config_without_password_manager(self, tmp_path: Path) -> None:
|
||||
"""Konfiguration ohne password_manager-Feld funktioniert."""
|
||||
config_dir = tmp_path / "shared" / "config"
|
||||
config_dir.mkdir(parents=True)
|
||||
(config_dir / "machine-context.yaml").write_text(
|
||||
yaml.dump({
|
||||
"machine": {
|
||||
"name": "no-pm-machine",
|
||||
"description": "Ohne PM",
|
||||
"authorized_contexts": ["bahn"],
|
||||
"key_source": "keyring",
|
||||
}
|
||||
}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
mcm = MachineContextManager(tmp_path)
|
||||
ctx = mcm.load()
|
||||
assert ctx.password_manager is None
|
||||
assert ctx.key_source == "keyring"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: OnboardingResult Dataclass
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestOnboardingResultDataclass:
|
||||
"""Tests für die OnboardingResult Dataclass."""
|
||||
|
||||
def test_default_values(self) -> None:
|
||||
"""Standard-Initialwerte sind korrekt."""
|
||||
result = OnboardingResult(success=True, machine_name="test")
|
||||
assert result.success is True
|
||||
assert result.machine_name == "test"
|
||||
assert result.authorized_contexts == []
|
||||
assert result.installed_keys == []
|
||||
assert result.errors == []
|
||||
|
||||
def test_full_initialization(self) -> None:
|
||||
"""Vollständige Initialisierung funktioniert."""
|
||||
result = OnboardingResult(
|
||||
success=True,
|
||||
machine_name="full-test",
|
||||
authorized_contexts=["privat", "dhive"],
|
||||
installed_keys=["key-1", "key-2"],
|
||||
errors=[],
|
||||
)
|
||||
assert result.machine_name == "full-test"
|
||||
assert len(result.authorized_contexts) == 2
|
||||
assert len(result.installed_keys) == 2
|
||||
@@ -0,0 +1,332 @@
|
||||
"""Property-basierte Tests für SecretEncryptionManager: Maschinenkontext-Beschränkung.
|
||||
|
||||
**Validates: Requirements 9.4, 9.5, 9.8, 9.10**
|
||||
|
||||
Property 24: Maschinenkontext beschränkt Entschlüsselung auf autorisierte Kontexte
|
||||
|
||||
For any Maschinenkontext-Konfiguration mit einer definierten Menge autorisierter
|
||||
Arbeitskontexte, muss gelten:
|
||||
(a) Secrets der autorisierten Kontexte sind entschlüsselbar,
|
||||
(b) Secrets aller nicht-autorisierten Kontexte verbleiben als verschlüsselte
|
||||
Binärdaten im Working Tree,
|
||||
(c) die Fehlermeldung bei fehlgeschlagener Entschlüsselung gibt keinen Hinweis
|
||||
auf den Dateiinhalt,
|
||||
(d) das Repository bleibt vollständig funktionsfähig (Code, Konfiguration,
|
||||
Dokumentation zugänglich).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from hypothesis import given, settings, assume
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from monorepo.encryption import SecretEncryptionManager
|
||||
from monorepo.models import MachineContext, PasswordManagerConfig
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
ALL_WORK_CONTEXTS = ["privat", "dhive", "bahn"]
|
||||
SECRET_FILE_NAMES = [".env", "cert.pem", "private.key", "api-token.txt", "db-secret.yaml"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@st.composite
|
||||
def machine_context_with_subset(draw: st.DrawFn) -> tuple[MachineContext, list[str], list[str]]:
|
||||
"""Generates a MachineContext with a random non-empty subset of authorized contexts.
|
||||
|
||||
Returns:
|
||||
Tuple of (MachineContext, authorized_contexts, unauthorized_contexts)
|
||||
"""
|
||||
# Draw a non-empty subset of contexts as authorized
|
||||
authorized = draw(
|
||||
st.lists(
|
||||
st.sampled_from(ALL_WORK_CONTEXTS),
|
||||
min_size=1,
|
||||
max_size=len(ALL_WORK_CONTEXTS),
|
||||
unique=True,
|
||||
)
|
||||
)
|
||||
unauthorized = [c for c in ALL_WORK_CONTEXTS if c not in authorized]
|
||||
|
||||
machine_name = draw(st.sampled_from([
|
||||
"andre-hauptrechner", "dhive-laptop", "bahn-workstation", "test-machine"
|
||||
]))
|
||||
|
||||
ctx = MachineContext(
|
||||
name=machine_name,
|
||||
description=f"Test-Maschine mit Zugriff auf: {', '.join(authorized)}",
|
||||
authorized_contexts=authorized,
|
||||
key_source="keyring",
|
||||
)
|
||||
|
||||
return ctx, authorized, unauthorized
|
||||
|
||||
|
||||
@st.composite
|
||||
def machine_context_strict_subset(draw: st.DrawFn) -> tuple[MachineContext, list[str], list[str]]:
|
||||
"""Generates a MachineContext where authorized is a STRICT subset (not all contexts).
|
||||
|
||||
Ensures there is at least one unauthorized context.
|
||||
|
||||
Returns:
|
||||
Tuple of (MachineContext, authorized_contexts, unauthorized_contexts)
|
||||
"""
|
||||
# Draw a strict subset (1 or 2 of 3 contexts)
|
||||
authorized = draw(
|
||||
st.lists(
|
||||
st.sampled_from(ALL_WORK_CONTEXTS),
|
||||
min_size=1,
|
||||
max_size=len(ALL_WORK_CONTEXTS) - 1,
|
||||
unique=True,
|
||||
)
|
||||
)
|
||||
unauthorized = [c for c in ALL_WORK_CONTEXTS if c not in authorized]
|
||||
assume(len(unauthorized) >= 1)
|
||||
|
||||
machine_name = draw(st.sampled_from([
|
||||
"eingeschraenkt-rechner", "dhive-only", "bahn-only", "privat-dhive"
|
||||
]))
|
||||
|
||||
ctx = MachineContext(
|
||||
name=machine_name,
|
||||
description=f"Eingeschränkt: nur {', '.join(authorized)}",
|
||||
authorized_contexts=authorized,
|
||||
key_source="keyring",
|
||||
)
|
||||
|
||||
return ctx, authorized, unauthorized
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _create_monorepo_with_secrets(
|
||||
authorized_contexts: list[str],
|
||||
) -> tuple[Path, dict[str, Path]]:
|
||||
"""Creates a temporary monorepo structure with secret files in each context.
|
||||
|
||||
Returns:
|
||||
Tuple of (root_path, dict mapping context -> secret_file_path)
|
||||
"""
|
||||
tmp_dir = Path(tempfile.mkdtemp(prefix="monorepo_enc_test_"))
|
||||
|
||||
# Create context directories and secret files
|
||||
secret_files: dict[str, Path] = {}
|
||||
for ctx in ALL_WORK_CONTEXTS:
|
||||
ctx_dir = tmp_dir / ctx
|
||||
ctx_dir.mkdir(parents=True, exist_ok=True)
|
||||
# Create a .env secret file with recognizable content
|
||||
env_file = ctx_dir / ".env"
|
||||
env_file.write_bytes(f"SECRET_{ctx.upper()}=super-secret-value-{ctx}\n".encode())
|
||||
secret_files[ctx] = env_file
|
||||
|
||||
# Create shared directory (non-secret, always accessible)
|
||||
shared_dir = tmp_dir / "shared"
|
||||
shared_dir.mkdir(parents=True, exist_ok=True)
|
||||
(shared_dir / "config").mkdir(parents=True, exist_ok=True)
|
||||
(shared_dir / "tools").mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create .git/git-crypt/keys structure for authorized contexts
|
||||
keys_dir = tmp_dir / ".git" / "git-crypt" / "keys"
|
||||
keys_dir.mkdir(parents=True, exist_ok=True)
|
||||
(keys_dir / "default").mkdir(exist_ok=True)
|
||||
for ctx in authorized_contexts:
|
||||
(keys_dir / ctx).mkdir(exist_ok=True)
|
||||
|
||||
return tmp_dir, secret_files
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestProperty24MachineContextRestriction:
|
||||
"""Property 24: Maschinenkontext beschränkt Entschlüsselung auf autorisierte Kontexte.
|
||||
|
||||
**Validates: Requirements 9.4, 9.5, 9.8, 9.10**
|
||||
"""
|
||||
|
||||
@given(data=st.data())
|
||||
@settings(max_examples=100)
|
||||
def test_decryption_succeeds_for_authorized_contexts(
|
||||
self, data: st.DataObject
|
||||
) -> None:
|
||||
"""(a) Secrets der autorisierten Kontexte sind entschlüsselbar.
|
||||
|
||||
For any machine context with authorized contexts, decryption of
|
||||
secret files in those contexts must succeed.
|
||||
"""
|
||||
machine_ctx, authorized, _ = data.draw(machine_context_with_subset())
|
||||
target_ctx = data.draw(st.sampled_from(authorized))
|
||||
|
||||
root_path, secret_files = _create_monorepo_with_secrets(authorized)
|
||||
manager = SecretEncryptionManager(root_path, machine_ctx)
|
||||
|
||||
# Verify authorization check passes
|
||||
assert manager.is_authorized(target_ctx) is True
|
||||
|
||||
# Verify decryption attempt does not fail due to authorization
|
||||
# (We mock _run_gitcrypt since git-crypt binary isn't available in tests)
|
||||
with patch.object(manager, "_run_gitcrypt"):
|
||||
result = manager.decrypt_file(secret_files[target_ctx])
|
||||
# Should succeed (file exists and context is authorized)
|
||||
assert result.success is True, (
|
||||
f"Decryption should succeed for authorized context '{target_ctx}' "
|
||||
f"on machine '{machine_ctx.name}' with authorized={authorized}. "
|
||||
f"Error: {result.error}"
|
||||
)
|
||||
|
||||
@given(data=st.data())
|
||||
@settings(max_examples=100)
|
||||
def test_decryption_denied_for_unauthorized_contexts(
|
||||
self, data: st.DataObject
|
||||
) -> None:
|
||||
"""(b) Secrets aller nicht-autorisierten Kontexte verbleiben als
|
||||
verschlüsselte Binärdaten im Working Tree (inaccessible).
|
||||
|
||||
For any machine context with a strict subset of authorized contexts,
|
||||
decryption of secrets from unauthorized contexts must fail.
|
||||
"""
|
||||
machine_ctx, authorized, unauthorized = data.draw(machine_context_strict_subset())
|
||||
target_ctx = data.draw(st.sampled_from(unauthorized))
|
||||
|
||||
root_path, secret_files = _create_monorepo_with_secrets(authorized)
|
||||
manager = SecretEncryptionManager(root_path, machine_ctx)
|
||||
|
||||
# Verify authorization check fails
|
||||
assert manager.is_authorized(target_ctx) is False
|
||||
|
||||
# Attempt decryption - must fail without revealing content
|
||||
result = manager.decrypt_file(secret_files[target_ctx])
|
||||
assert result.success is False, (
|
||||
f"Decryption should be DENIED for unauthorized context '{target_ctx}' "
|
||||
f"on machine '{machine_ctx.name}' with authorized={authorized}"
|
||||
)
|
||||
# Content must NOT be returned
|
||||
assert result.content is None, (
|
||||
f"Content MUST be None for unauthorized context '{target_ctx}'. "
|
||||
f"No content should be revealed for unauthorized decryption attempts."
|
||||
)
|
||||
|
||||
@given(data=st.data())
|
||||
@settings(max_examples=100)
|
||||
def test_error_message_does_not_reveal_content(
|
||||
self, data: st.DataObject
|
||||
) -> None:
|
||||
"""(c) Die Fehlermeldung bei fehlgeschlagener Entschlüsselung gibt
|
||||
keinen Hinweis auf den Dateiinhalt.
|
||||
|
||||
For any failed decryption due to unauthorized machine context,
|
||||
the error message must not contain any portion of the file content.
|
||||
"""
|
||||
machine_ctx, authorized, unauthorized = data.draw(machine_context_strict_subset())
|
||||
target_ctx = data.draw(st.sampled_from(unauthorized))
|
||||
|
||||
root_path, secret_files = _create_monorepo_with_secrets(authorized)
|
||||
manager = SecretEncryptionManager(root_path, machine_ctx)
|
||||
|
||||
# Read the actual secret content to check it's not in the error
|
||||
secret_content = secret_files[target_ctx].read_text(encoding="utf-8")
|
||||
secret_lines = [line.strip() for line in secret_content.splitlines() if line.strip()]
|
||||
|
||||
result = manager.decrypt_file(secret_files[target_ctx])
|
||||
assert result.success is False
|
||||
|
||||
# Error message must not contain any secret content
|
||||
error_msg = result.error or ""
|
||||
for secret_line in secret_lines:
|
||||
# Check the value part (after =) is not in the error
|
||||
if "=" in secret_line:
|
||||
secret_value = secret_line.split("=", 1)[1]
|
||||
assert secret_value not in error_msg, (
|
||||
f"Error message reveals secret content! "
|
||||
f"Found '{secret_value}' in error: '{error_msg}'"
|
||||
)
|
||||
# Also check the full line is not in the error
|
||||
assert secret_line not in error_msg, (
|
||||
f"Error message reveals secret content! "
|
||||
f"Found '{secret_line}' in error: '{error_msg}'"
|
||||
)
|
||||
|
||||
@given(data=st.data())
|
||||
@settings(max_examples=100)
|
||||
def test_get_context_key_returns_none_for_unauthorized(
|
||||
self, data: st.DataObject
|
||||
) -> None:
|
||||
"""get_context_key() returns None for unauthorized contexts.
|
||||
|
||||
This ensures that the key retrieval mechanism respects the machine
|
||||
context boundaries and doesn't provide keys for unauthorized contexts.
|
||||
"""
|
||||
machine_ctx, authorized, unauthorized = data.draw(machine_context_strict_subset())
|
||||
target_ctx = data.draw(st.sampled_from(unauthorized))
|
||||
|
||||
root_path, _ = _create_monorepo_with_secrets(authorized)
|
||||
manager = SecretEncryptionManager(root_path, machine_ctx)
|
||||
|
||||
key = manager.get_context_key(target_ctx)
|
||||
assert key is None, (
|
||||
f"get_context_key() must return None for unauthorized context '{target_ctx}' "
|
||||
f"on machine '{machine_ctx.name}' with authorized={authorized}"
|
||||
)
|
||||
|
||||
@given(data=st.data())
|
||||
@settings(max_examples=100)
|
||||
def test_onboard_machine_only_installs_valid_context_keys(
|
||||
self, data: st.DataObject
|
||||
) -> None:
|
||||
"""onboard_machine() only installs keys for valid contexts.
|
||||
|
||||
When onboarding a machine, only keys for contexts in the valid set
|
||||
(privat, dhive, bahn) should be installed. Invalid contexts must
|
||||
produce errors and not be included in the result.
|
||||
"""
|
||||
machine_ctx, authorized, _ = data.draw(machine_context_with_subset())
|
||||
|
||||
# Mix valid and invalid contexts for onboarding request
|
||||
invalid_contexts = data.draw(
|
||||
st.lists(
|
||||
st.text(
|
||||
alphabet=st.characters(whitelist_categories=("Ll",)),
|
||||
min_size=3,
|
||||
max_size=10,
|
||||
).filter(lambda x: x not in ALL_WORK_CONTEXTS),
|
||||
min_size=0,
|
||||
max_size=2,
|
||||
unique=True,
|
||||
)
|
||||
)
|
||||
requested_contexts = authorized + invalid_contexts
|
||||
|
||||
root_path, _ = _create_monorepo_with_secrets(authorized)
|
||||
manager = SecretEncryptionManager(root_path, machine_ctx)
|
||||
|
||||
result = manager.onboard_machine(machine_ctx.name, requested_contexts)
|
||||
|
||||
# Invalid contexts must appear in errors
|
||||
for invalid_ctx in invalid_contexts:
|
||||
assert any(invalid_ctx in err for err in result.errors), (
|
||||
f"Invalid context '{invalid_ctx}' should produce an error in onboarding. "
|
||||
f"Errors: {result.errors}"
|
||||
)
|
||||
|
||||
# Authorized contexts in result must be subset of valid contexts
|
||||
for ctx in result.authorized_contexts:
|
||||
assert ctx in ALL_WORK_CONTEXTS, (
|
||||
f"Onboarding result contains invalid context '{ctx}'. "
|
||||
f"Only valid contexts {ALL_WORK_CONTEXTS} should be in authorized_contexts."
|
||||
)
|
||||
@@ -0,0 +1,292 @@
|
||||
"""Property-basierte Tests für SecretEncryptionManager: Verschlüsselung nur mit Kontextschlüssel.
|
||||
|
||||
**Validates: Requirements 9.3, 9.8**
|
||||
|
||||
Property 23: Verschlüsselte Secrets können nur mit autorisiertem Kontextschlüssel entschlüsselt werden
|
||||
|
||||
For any verschlüsselte Secret-Datei eines Arbeitskontexts X und für jeden
|
||||
Entschlüsselungsversuch mit einem Schlüssel des Kontexts Y (wobei X ≠ Y),
|
||||
muss die Entschlüsselung fehlschlagen und die Datei im verschlüsselten Zustand
|
||||
verbleiben. Nur der Schlüssel des zugehörigen Kontexts X darf die Datei
|
||||
erfolgreich entschlüsseln.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from hypothesis import assume, given, settings
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from monorepo.encryption import SecretEncryptionManager
|
||||
from monorepo.models import MachineContext
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Valid non-shared work contexts for encryption
|
||||
WORK_CONTEXTS = ["privat", "dhive", "bahn"]
|
||||
|
||||
# Secret file names that match the SECRET_PATTERNS
|
||||
SECRET_FILE_NAMES = [".env", "private.pem", "server.key", "api-token.txt", "my-secret.yaml"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@st.composite
|
||||
def machine_context_with_authorized(draw: st.DrawFn) -> tuple[MachineContext, list[str]]:
|
||||
"""Generates a MachineContext with a random subset of authorized contexts.
|
||||
|
||||
Returns (MachineContext, authorized_contexts_list).
|
||||
Ensures at least one context is authorized.
|
||||
"""
|
||||
# Draw a non-empty subset of contexts
|
||||
authorized = draw(
|
||||
st.lists(
|
||||
st.sampled_from(WORK_CONTEXTS),
|
||||
min_size=1,
|
||||
max_size=3,
|
||||
unique=True,
|
||||
)
|
||||
)
|
||||
name = draw(st.sampled_from(["test-rechner", "dhive-laptop", "bahn-pc", "home-pc"]))
|
||||
ctx = MachineContext(
|
||||
name=name,
|
||||
description=f"Test machine: {name}",
|
||||
authorized_contexts=authorized,
|
||||
key_source="keyring",
|
||||
)
|
||||
return ctx, authorized
|
||||
|
||||
|
||||
@st.composite
|
||||
def unauthorized_file_context(
|
||||
draw: st.DrawFn, authorized_contexts: list[str]
|
||||
) -> str:
|
||||
"""Generates a context that is NOT in the authorized list."""
|
||||
unauthorized = [c for c in WORK_CONTEXTS if c not in authorized_contexts]
|
||||
assume(len(unauthorized) > 0)
|
||||
return draw(st.sampled_from(unauthorized))
|
||||
|
||||
|
||||
@st.composite
|
||||
def secret_file_name(draw: st.DrawFn) -> str:
|
||||
"""Generates a secret file name matching common patterns."""
|
||||
return draw(st.sampled_from(SECRET_FILE_NAMES))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _create_monorepo_with_secret(
|
||||
tmp_dir: Path, context: str, filename: str, content: bytes
|
||||
) -> Path:
|
||||
"""Creates a minimal monorepo structure with a secret file in the given context.
|
||||
|
||||
Returns the path to the created secret file.
|
||||
"""
|
||||
# Create context directories
|
||||
for ctx in WORK_CONTEXTS + ["shared"]:
|
||||
(tmp_dir / ctx).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create the secret file
|
||||
secret_path = tmp_dir / context / filename
|
||||
secret_path.write_bytes(content)
|
||||
return secret_path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestProperty23EncryptionOnlyWithContextKey:
|
||||
"""Property 23: Verschlüsselte Secrets können nur mit autorisiertem Kontextschlüssel
|
||||
entschlüsselt werden.
|
||||
|
||||
**Validates: Requirements 9.3, 9.8**
|
||||
|
||||
For any encrypted file belonging to a context, decryption must succeed ONLY when
|
||||
the machine context includes the authorized key for that context. Without the
|
||||
correct key, decryption must fail without revealing file content.
|
||||
"""
|
||||
|
||||
@given(data=st.data())
|
||||
@settings(max_examples=200)
|
||||
def test_decrypt_fails_for_unauthorized_context(self, data: st.DataObject) -> None:
|
||||
"""Decryption must fail when the machine context does NOT authorize the
|
||||
file's context.
|
||||
|
||||
For any file in context X, if the machine only authorizes contexts Y
|
||||
(where X not in Y), decrypt_file must return success=False.
|
||||
"""
|
||||
# Generate a machine context with a subset of authorized contexts
|
||||
machine_ctx, authorized = data.draw(machine_context_with_authorized())
|
||||
|
||||
# Pick a file context that is NOT authorized
|
||||
unauthorized_ctxs = [c for c in WORK_CONTEXTS if c not in authorized]
|
||||
assume(len(unauthorized_ctxs) > 0)
|
||||
file_context = data.draw(st.sampled_from(unauthorized_ctxs))
|
||||
|
||||
# Generate a secret file name
|
||||
filename = data.draw(secret_file_name())
|
||||
|
||||
# Setup temporary monorepo
|
||||
tmp_dir = Path(tempfile.mkdtemp(prefix="monorepo_enc_test_"))
|
||||
secret_content = b"SUPER_SECRET_VALUE=mysecret123\nDB_PASSWORD=hunter2"
|
||||
secret_path = _create_monorepo_with_secret(tmp_dir, file_context, filename, secret_content)
|
||||
|
||||
# Create manager with the generated machine context
|
||||
manager = SecretEncryptionManager(tmp_dir, machine_ctx)
|
||||
|
||||
# Patch _run_gitcrypt so we don't need actual git-crypt installed
|
||||
with patch.object(manager, "_run_gitcrypt"):
|
||||
result = manager.decrypt_file(secret_path)
|
||||
|
||||
# Decryption MUST fail for unauthorized context
|
||||
assert result.success is False, (
|
||||
f"Decryption should fail: machine '{machine_ctx.name}' "
|
||||
f"(authorized: {authorized}) tried to decrypt file in "
|
||||
f"context '{file_context}'"
|
||||
)
|
||||
|
||||
# Content must NOT be returned on failure
|
||||
assert result.content is None, (
|
||||
f"Failed decryption must not return file content. "
|
||||
f"Got content for file in context '{file_context}'"
|
||||
)
|
||||
|
||||
@given(data=st.data())
|
||||
@settings(max_examples=200)
|
||||
def test_decrypt_succeeds_for_authorized_context(self, data: st.DataObject) -> None:
|
||||
"""Decryption must succeed when the machine context DOES authorize the
|
||||
file's context.
|
||||
|
||||
For any file in context X, if the machine authorizes X, decrypt_file
|
||||
must return success=True with the file content.
|
||||
"""
|
||||
# Generate a machine context
|
||||
machine_ctx, authorized = data.draw(machine_context_with_authorized())
|
||||
|
||||
# Pick a file context that IS authorized
|
||||
file_context = data.draw(st.sampled_from(authorized))
|
||||
|
||||
# Generate a secret file name
|
||||
filename = data.draw(secret_file_name())
|
||||
|
||||
# Setup temporary monorepo
|
||||
tmp_dir = Path(tempfile.mkdtemp(prefix="monorepo_enc_test_"))
|
||||
secret_content = b"AUTHORIZED_SECRET=value\nKEY=data"
|
||||
secret_path = _create_monorepo_with_secret(tmp_dir, file_context, filename, secret_content)
|
||||
|
||||
# Create manager with the generated machine context
|
||||
manager = SecretEncryptionManager(tmp_dir, machine_ctx)
|
||||
|
||||
# Patch _run_gitcrypt to simulate successful unlock
|
||||
with patch.object(manager, "_run_gitcrypt"):
|
||||
result = manager.decrypt_file(secret_path)
|
||||
|
||||
# Decryption MUST succeed for authorized context
|
||||
assert result.success is True, (
|
||||
f"Decryption should succeed: machine '{machine_ctx.name}' "
|
||||
f"(authorized: {authorized}) should decrypt file in "
|
||||
f"context '{file_context}'. Error: {result.error}"
|
||||
)
|
||||
|
||||
# Content must be returned on success
|
||||
assert result.content is not None, (
|
||||
f"Successful decryption must return file content for "
|
||||
f"context '{file_context}'"
|
||||
)
|
||||
assert result.content == secret_content
|
||||
|
||||
@given(data=st.data())
|
||||
@settings(max_examples=200)
|
||||
def test_failed_decryption_does_not_reveal_content(self, data: st.DataObject) -> None:
|
||||
"""When decryption fails, the error message must NOT contain
|
||||
the file's actual contents.
|
||||
|
||||
This ensures that the encryption boundary does not leak sensitive
|
||||
information through error messages (Requirement 9.8).
|
||||
"""
|
||||
# Generate a machine context with limited access
|
||||
machine_ctx, authorized = data.draw(machine_context_with_authorized())
|
||||
|
||||
# Pick a file context that is NOT authorized
|
||||
unauthorized_ctxs = [c for c in WORK_CONTEXTS if c not in authorized]
|
||||
assume(len(unauthorized_ctxs) > 0)
|
||||
file_context = data.draw(st.sampled_from(unauthorized_ctxs))
|
||||
|
||||
filename = data.draw(secret_file_name())
|
||||
|
||||
# Use recognizable secret content to check for leakage
|
||||
secret_content = b"TOP_SECRET_API_KEY=sk-abc123xyz789\nDATABASE_URL=postgres://admin:pass@host/db"
|
||||
secret_strings = [
|
||||
"TOP_SECRET_API_KEY",
|
||||
"sk-abc123xyz789",
|
||||
"DATABASE_URL",
|
||||
"postgres://admin:pass@host/db",
|
||||
]
|
||||
|
||||
tmp_dir = Path(tempfile.mkdtemp(prefix="monorepo_enc_test_"))
|
||||
secret_path = _create_monorepo_with_secret(tmp_dir, file_context, filename, secret_content)
|
||||
|
||||
manager = SecretEncryptionManager(tmp_dir, machine_ctx)
|
||||
|
||||
with patch.object(manager, "_run_gitcrypt"):
|
||||
result = manager.decrypt_file(secret_path)
|
||||
|
||||
# Must fail
|
||||
assert result.success is False
|
||||
|
||||
# Error message must NOT contain any of the secret content
|
||||
error_msg = result.error or ""
|
||||
for secret_str in secret_strings:
|
||||
assert secret_str not in error_msg, (
|
||||
f"Error message leaks file content! "
|
||||
f"Found '{secret_str}' in error: '{error_msg}'"
|
||||
)
|
||||
|
||||
@given(data=st.data())
|
||||
@settings(max_examples=200)
|
||||
def test_is_authorized_returns_true_only_for_authorized_contexts(
|
||||
self, data: st.DataObject
|
||||
) -> None:
|
||||
"""is_authorized() must return True ONLY for contexts in the machine's
|
||||
authorized_contexts list, and False for all others.
|
||||
|
||||
This is the fundamental gate that controls decryption access.
|
||||
"""
|
||||
# Generate a machine context
|
||||
machine_ctx, authorized = data.draw(machine_context_with_authorized())
|
||||
|
||||
# Pick any context to check
|
||||
check_context = data.draw(st.sampled_from(WORK_CONTEXTS))
|
||||
|
||||
tmp_dir = Path(tempfile.mkdtemp(prefix="monorepo_enc_test_"))
|
||||
for ctx in WORK_CONTEXTS + ["shared"]:
|
||||
(tmp_dir / ctx).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
manager = SecretEncryptionManager(tmp_dir, machine_ctx)
|
||||
|
||||
result = manager.is_authorized(check_context)
|
||||
|
||||
if check_context in authorized:
|
||||
assert result is True, (
|
||||
f"is_authorized('{check_context}') should be True "
|
||||
f"for machine with authorized={authorized}"
|
||||
)
|
||||
else:
|
||||
assert result is False, (
|
||||
f"is_authorized('{check_context}') should be False "
|
||||
f"for machine with authorized={authorized}"
|
||||
)
|
||||
@@ -0,0 +1,568 @@
|
||||
"""Tests für Task 17.3: Encryption-Security-Integration Verdrahtung.
|
||||
|
||||
Testet die korrekte Integration zwischen:
|
||||
- SecretEncryptionManager ↔ ContextGuard (load_env nutzt Entschlüsselung)
|
||||
- SecretEncryptionManager ↔ FederationManager (Team_Repos erhalten nur eigene Secrets)
|
||||
- SecretEncryptionManager ↔ OrchestratorAdapter (Env-Loading via Encryption)
|
||||
- create_integrated_stack Factory-Funktion
|
||||
|
||||
Requirements: 2.2, 9.3, 9.4, 10.10
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from monorepo.encryption import DecryptionResult, SecretEncryptionManager
|
||||
from monorepo.federation import FederationManager
|
||||
from monorepo.integration import create_integrated_stack
|
||||
from monorepo.models import MachineContext
|
||||
from monorepo.orchestrator import OrchestratorAdapter
|
||||
from monorepo.security import ContextGuard
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def monorepo_root(tmp_path: Path) -> Path:
|
||||
"""Creates a minimal monorepo structure for integration tests."""
|
||||
root = tmp_path / "monorepo"
|
||||
root.mkdir()
|
||||
|
||||
# Context directories
|
||||
for ctx in ("privat", "dhive", "bahn", "shared"):
|
||||
(root / ctx).mkdir()
|
||||
|
||||
# .env files per context
|
||||
(root / "privat" / ".env").write_text(
|
||||
"PRIVAT_SECRET=privat_val_123\nPRIVAT_API=http://localhost:3000\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(root / "dhive" / ".env").write_text(
|
||||
"DHIVE_SECRET=dhive_val_456\nDHIVE_TOKEN=tok_dhive\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(root / "bahn" / ".env").write_text(
|
||||
"BAHN_SECRET=bahn_val_789\nBAHN_ENDPOINT=https://bahn.api\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(root / "shared" / ".env").write_text(
|
||||
"SHARED_KEY=shared_val\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
# access-config.yaml
|
||||
config_dir = root / "shared" / "config"
|
||||
config_dir.mkdir(parents=True)
|
||||
|
||||
access_config = {
|
||||
"contexts": {
|
||||
"privat": {
|
||||
"env_file": "privat/.env",
|
||||
"allowed_shared": ["shared/tools/", "shared/config/"],
|
||||
},
|
||||
"dhive": {
|
||||
"env_file": "dhive/.env",
|
||||
"allowed_shared": ["shared/tools/", "shared/config/"],
|
||||
},
|
||||
"bahn": {
|
||||
"env_file": "bahn/.env",
|
||||
"allowed_shared": ["shared/tools/", "shared/config/"],
|
||||
},
|
||||
"shared": {
|
||||
"env_file": "shared/.env",
|
||||
"allowed_shared": ["*"],
|
||||
},
|
||||
}
|
||||
}
|
||||
(config_dir / "access-config.yaml").write_text(
|
||||
yaml.dump(access_config), encoding="utf-8"
|
||||
)
|
||||
|
||||
# machine-context.yaml
|
||||
machine_config = {
|
||||
"machine": {
|
||||
"name": "test-machine",
|
||||
"description": "Test Machine",
|
||||
"authorized_contexts": ["privat", "dhive", "bahn"],
|
||||
"key_source": "keyring",
|
||||
}
|
||||
}
|
||||
(config_dir / "machine-context.yaml").write_text(
|
||||
yaml.dump(machine_config), encoding="utf-8"
|
||||
)
|
||||
|
||||
# team-repos.yaml
|
||||
team_repos_config = {
|
||||
"version": "1.0",
|
||||
"federation": {
|
||||
"topology": "hub-and-spoke",
|
||||
"hub_owner": "andre",
|
||||
"conflict_strategy": "team-wins",
|
||||
},
|
||||
"team_repos": [
|
||||
{
|
||||
"context": "privat",
|
||||
"url": "https://github.com/test/privat.git",
|
||||
"branch": "main",
|
||||
"sync_direction": "bidirectional",
|
||||
"sync_frequency": "manual",
|
||||
"shared_mirror": {
|
||||
"enabled": True,
|
||||
"paths": ["shared/tools/"],
|
||||
"mode": "read-only",
|
||||
},
|
||||
},
|
||||
{
|
||||
"context": "dhive",
|
||||
"url": "https://github.com/test/dhive.git",
|
||||
"branch": "main",
|
||||
"sync_direction": "bidirectional",
|
||||
"sync_frequency": "on-push",
|
||||
},
|
||||
],
|
||||
}
|
||||
(config_dir / "team-repos.yaml").write_text(
|
||||
yaml.dump(team_repos_config), encoding="utf-8"
|
||||
)
|
||||
|
||||
return root
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def machine_context() -> MachineContext:
|
||||
"""Creates a MachineContext authorized for all contexts."""
|
||||
return MachineContext(
|
||||
name="test-machine",
|
||||
description="Test",
|
||||
authorized_contexts=["privat", "dhive", "bahn"],
|
||||
key_source="keyring",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def limited_machine_context() -> MachineContext:
|
||||
"""Creates a MachineContext authorized only for dhive."""
|
||||
return MachineContext(
|
||||
name="dhive-laptop",
|
||||
description="Only dhive",
|
||||
authorized_contexts=["dhive"],
|
||||
key_source="keyring",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def encryption_manager(
|
||||
monorepo_root: Path, machine_context: MachineContext
|
||||
) -> SecretEncryptionManager:
|
||||
"""Creates a SecretEncryptionManager with full authorization."""
|
||||
return SecretEncryptionManager(monorepo_root, machine_context)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def limited_encryption_manager(
|
||||
monorepo_root: Path, limited_machine_context: MachineContext
|
||||
) -> SecretEncryptionManager:
|
||||
"""Creates a SecretEncryptionManager authorized only for dhive."""
|
||||
return SecretEncryptionManager(monorepo_root, limited_machine_context)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: ContextGuard + SecretEncryptionManager Integration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestContextGuardEncryptionIntegration:
|
||||
"""Tests für die Integration von ContextGuard.load_env mit Entschlüsselung."""
|
||||
|
||||
def test_load_env_without_encryption_reads_file_directly(
|
||||
self, monorepo_root: Path
|
||||
) -> None:
|
||||
"""Ohne encryption_manager: .env wird direkt gelesen (Req 2.2)."""
|
||||
guard = ContextGuard(root_path=monorepo_root)
|
||||
env = guard.load_env("dhive")
|
||||
assert env == {"DHIVE_SECRET": "dhive_val_456", "DHIVE_TOKEN": "tok_dhive"}
|
||||
|
||||
def test_load_env_with_encryption_uses_decrypt_file(
|
||||
self, monorepo_root: Path
|
||||
) -> None:
|
||||
"""Mit encryption_manager: load_env nutzt decrypt_file (Req 2.2, 9.3)."""
|
||||
mock_enc = MagicMock()
|
||||
mock_enc.is_authorized.return_value = True
|
||||
mock_enc.machine_context.name = "test-machine"
|
||||
mock_enc.decrypt_file.return_value = DecryptionResult(
|
||||
success=True,
|
||||
file_path=monorepo_root / "dhive" / ".env",
|
||||
content=b"DECRYPTED_KEY=secret_decrypted_value\n",
|
||||
)
|
||||
|
||||
guard = ContextGuard(
|
||||
root_path=monorepo_root, encryption_manager=mock_enc
|
||||
)
|
||||
env = guard.load_env("dhive")
|
||||
|
||||
mock_enc.is_authorized.assert_called_once_with("dhive")
|
||||
mock_enc.decrypt_file.assert_called_once_with(monorepo_root / "dhive" / ".env")
|
||||
assert env == {"DECRYPTED_KEY": "secret_decrypted_value"}
|
||||
|
||||
def test_load_env_decryption_fails_falls_back_to_direct_read(
|
||||
self, monorepo_root: Path
|
||||
) -> None:
|
||||
"""Bei fehlgeschlagener Entschlüsselung: Fallback auf direktes Lesen."""
|
||||
mock_enc = MagicMock()
|
||||
mock_enc.is_authorized.return_value = True
|
||||
mock_enc.machine_context.name = "test-machine"
|
||||
mock_enc.decrypt_file.return_value = DecryptionResult(
|
||||
success=False,
|
||||
file_path=monorepo_root / "privat" / ".env",
|
||||
error="git-crypt not available",
|
||||
)
|
||||
|
||||
guard = ContextGuard(
|
||||
root_path=monorepo_root, encryption_manager=mock_enc
|
||||
)
|
||||
env = guard.load_env("privat")
|
||||
|
||||
# Fallback: liest die Datei direkt
|
||||
assert env == {"PRIVAT_SECRET": "privat_val_123", "PRIVAT_API": "http://localhost:3000"}
|
||||
|
||||
def test_load_env_not_authorized_raises_permission_error(
|
||||
self, monorepo_root: Path
|
||||
) -> None:
|
||||
"""Nicht autorisierter Maschinenkontext: PermissionError (Req 9.4)."""
|
||||
mock_enc = MagicMock()
|
||||
mock_enc.is_authorized.return_value = False
|
||||
mock_enc.machine_context.name = "dhive-laptop"
|
||||
|
||||
guard = ContextGuard(
|
||||
root_path=monorepo_root, encryption_manager=mock_enc
|
||||
)
|
||||
|
||||
with pytest.raises(PermissionError, match="nicht.*autorisiert"):
|
||||
guard.load_env("privat")
|
||||
|
||||
def test_load_env_respects_context_isolation(
|
||||
self, monorepo_root: Path
|
||||
) -> None:
|
||||
"""Jeder Kontext erhält nur seine eigenen Umgebungsvariablen (Req 2.2)."""
|
||||
guard = ContextGuard(root_path=monorepo_root)
|
||||
|
||||
privat_env = guard.load_env("privat")
|
||||
dhive_env = guard.load_env("dhive")
|
||||
bahn_env = guard.load_env("bahn")
|
||||
|
||||
assert "PRIVAT_SECRET" in privat_env
|
||||
assert "DHIVE_SECRET" not in privat_env
|
||||
assert "DHIVE_SECRET" in dhive_env
|
||||
assert "BAHN_SECRET" not in dhive_env
|
||||
assert "BAHN_SECRET" in bahn_env
|
||||
assert "PRIVAT_SECRET" not in bahn_env
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: OrchestratorAdapter + SecretEncryptionManager Integration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestOrchestratorEncryptionIntegration:
|
||||
"""Tests für die Integration von OrchestratorAdapter.load_context_env mit Entschlüsselung."""
|
||||
|
||||
def test_load_context_env_with_encryption_uses_decrypt(
|
||||
self, monorepo_root: Path
|
||||
) -> None:
|
||||
"""Mit encryption_manager: load_context_env entschlüsselt (Req 7.3, 9.3)."""
|
||||
mock_enc = MagicMock()
|
||||
mock_enc.is_authorized.return_value = True
|
||||
mock_enc.machine_context.name = "test-machine"
|
||||
mock_enc.decrypt_file.return_value = DecryptionResult(
|
||||
success=True,
|
||||
file_path=monorepo_root / "bahn" / ".env",
|
||||
content=b"BAHN_DECRYPTED=value_from_decryption\n",
|
||||
)
|
||||
|
||||
guard = ContextGuard(root_path=monorepo_root)
|
||||
adapter = OrchestratorAdapter(
|
||||
root_path=monorepo_root,
|
||||
context_guard=guard,
|
||||
encryption_manager=mock_enc,
|
||||
)
|
||||
|
||||
env = adapter.load_context_env("bahn")
|
||||
|
||||
mock_enc.decrypt_file.assert_called_once()
|
||||
assert env == {"BAHN_DECRYPTED": "value_from_decryption"}
|
||||
|
||||
def test_load_context_env_without_encryption_delegates_to_guard(
|
||||
self, monorepo_root: Path
|
||||
) -> None:
|
||||
"""Ohne encryption_manager: Delegation an ContextGuard.load_env."""
|
||||
guard = ContextGuard(root_path=monorepo_root)
|
||||
adapter = OrchestratorAdapter(
|
||||
root_path=monorepo_root,
|
||||
context_guard=guard,
|
||||
encryption_manager=None,
|
||||
)
|
||||
|
||||
env = adapter.load_context_env("dhive")
|
||||
assert env == {"DHIVE_SECRET": "dhive_val_456", "DHIVE_TOKEN": "tok_dhive"}
|
||||
|
||||
def test_load_context_env_not_authorized_raises_permission_error(
|
||||
self, monorepo_root: Path
|
||||
) -> None:
|
||||
"""Nicht autorisiert: PermissionError bei Env-Zugriff (Req 9.4)."""
|
||||
mock_enc = MagicMock()
|
||||
mock_enc.is_authorized.return_value = False
|
||||
mock_enc.machine_context.name = "bahn-only-machine"
|
||||
|
||||
guard = ContextGuard(root_path=monorepo_root)
|
||||
adapter = OrchestratorAdapter(
|
||||
root_path=monorepo_root,
|
||||
context_guard=guard,
|
||||
encryption_manager=mock_enc,
|
||||
)
|
||||
|
||||
with pytest.raises(PermissionError, match="nicht.*autorisiert"):
|
||||
adapter.load_context_env("privat")
|
||||
|
||||
def test_load_context_env_decryption_fails_falls_back(
|
||||
self, monorepo_root: Path
|
||||
) -> None:
|
||||
"""Bei fehlgeschlagener Entschlüsselung: Fallback auf guard.load_env."""
|
||||
mock_enc = MagicMock()
|
||||
mock_enc.is_authorized.return_value = True
|
||||
mock_enc.machine_context.name = "test-machine"
|
||||
mock_enc.decrypt_file.return_value = DecryptionResult(
|
||||
success=False,
|
||||
file_path=monorepo_root / "dhive" / ".env",
|
||||
error="git-crypt Fehler",
|
||||
)
|
||||
|
||||
guard = ContextGuard(root_path=monorepo_root)
|
||||
adapter = OrchestratorAdapter(
|
||||
root_path=monorepo_root,
|
||||
context_guard=guard,
|
||||
encryption_manager=mock_enc,
|
||||
)
|
||||
|
||||
env = adapter.load_context_env("dhive")
|
||||
# Fallback auf direkte Leseoperation
|
||||
assert env == {"DHIVE_SECRET": "dhive_val_456", "DHIVE_TOKEN": "tok_dhive"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: FederationManager + SecretEncryptionManager Integration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFederationEncryptionIntegration:
|
||||
"""Tests für die Integration von FederationManager._handle_team_secrets."""
|
||||
|
||||
def test_prepare_team_repo_decrypts_secrets_when_authorized(
|
||||
self, monorepo_root: Path
|
||||
) -> None:
|
||||
"""Team-Repo erhält entschlüsselte Secrets wenn autorisiert (Req 10.10)."""
|
||||
# Create secret file in privat context
|
||||
(monorepo_root / "privat" / "src").mkdir(parents=True, exist_ok=True)
|
||||
(monorepo_root / "privat" / "src" / "app.py").write_text("pass\n")
|
||||
|
||||
env_content = b"PRIVAT_API_KEY=decrypted_secret_123\n"
|
||||
mock_enc = MagicMock()
|
||||
mock_enc.is_authorized.return_value = True
|
||||
mock_enc.decrypt_file.return_value = DecryptionResult(
|
||||
success=True,
|
||||
file_path=monorepo_root / "privat" / ".env",
|
||||
content=env_content,
|
||||
)
|
||||
|
||||
config_path = monorepo_root / "shared" / "config" / "team-repos.yaml"
|
||||
fm = FederationManager(
|
||||
config_path=config_path,
|
||||
monorepo_root=monorepo_root,
|
||||
encryption_manager=mock_enc,
|
||||
)
|
||||
|
||||
team_path = fm.prepare_team_repo("privat")
|
||||
|
||||
# Verify decrypt_file was called
|
||||
assert mock_enc.decrypt_file.called
|
||||
# Verify team repo was created
|
||||
assert team_path.exists()
|
||||
|
||||
def test_prepare_team_repo_secrets_stay_encrypted_when_not_authorized(
|
||||
self, monorepo_root: Path
|
||||
) -> None:
|
||||
"""Team-Repo Secrets bleiben verschlüsselt ohne Autorisierung."""
|
||||
(monorepo_root / "dhive" / "src").mkdir(parents=True, exist_ok=True)
|
||||
(monorepo_root / "dhive" / "src" / "main.py").write_text("pass\n")
|
||||
|
||||
mock_enc = MagicMock()
|
||||
mock_enc.is_authorized.return_value = False
|
||||
|
||||
config_path = monorepo_root / "shared" / "config" / "team-repos.yaml"
|
||||
fm = FederationManager(
|
||||
config_path=config_path,
|
||||
monorepo_root=monorepo_root,
|
||||
encryption_manager=mock_enc,
|
||||
)
|
||||
|
||||
team_path = fm.prepare_team_repo("dhive")
|
||||
|
||||
# decrypt_file should NOT be called when not authorized
|
||||
mock_enc.decrypt_file.assert_not_called()
|
||||
assert team_path.exists()
|
||||
|
||||
def test_prepare_team_repo_without_encryption_manager(
|
||||
self, monorepo_root: Path
|
||||
) -> None:
|
||||
"""Ohne encryption_manager: Secrets werden nicht verarbeitet."""
|
||||
(monorepo_root / "privat" / "src").mkdir(parents=True, exist_ok=True)
|
||||
(monorepo_root / "privat" / "src" / "app.py").write_text("pass\n")
|
||||
|
||||
config_path = monorepo_root / "shared" / "config" / "team-repos.yaml"
|
||||
fm = FederationManager(
|
||||
config_path=config_path,
|
||||
monorepo_root=monorepo_root,
|
||||
encryption_manager=None,
|
||||
)
|
||||
|
||||
team_path = fm.prepare_team_repo("privat")
|
||||
assert team_path.exists()
|
||||
|
||||
def test_prepare_team_repo_filters_gitattributes_for_own_context(
|
||||
self, monorepo_root: Path
|
||||
) -> None:
|
||||
"""Team-Repo .gitattributes behält nur Filter des eigenen Kontexts."""
|
||||
(monorepo_root / "privat" / "src").mkdir(parents=True, exist_ok=True)
|
||||
(monorepo_root / "privat" / "src" / "app.py").write_text("pass\n")
|
||||
|
||||
# Create .gitattributes with filters for multiple contexts
|
||||
gitattributes_content = (
|
||||
".env filter=git-crypt-privat diff=git-crypt-privat\n"
|
||||
"*.pem filter=git-crypt-privat diff=git-crypt-privat\n"
|
||||
"# Should be removed:\n"
|
||||
".env filter=git-crypt-dhive diff=git-crypt-dhive\n"
|
||||
".env filter=git-crypt-bahn diff=git-crypt-bahn\n"
|
||||
)
|
||||
(monorepo_root / "privat" / ".gitattributes").write_text(
|
||||
gitattributes_content, encoding="utf-8"
|
||||
)
|
||||
|
||||
mock_enc = MagicMock()
|
||||
mock_enc.is_authorized.return_value = True
|
||||
mock_enc.decrypt_file.return_value = DecryptionResult(
|
||||
success=False, file_path=Path("/"), error="skip"
|
||||
)
|
||||
|
||||
config_path = monorepo_root / "shared" / "config" / "team-repos.yaml"
|
||||
fm = FederationManager(
|
||||
config_path=config_path,
|
||||
monorepo_root=monorepo_root,
|
||||
encryption_manager=mock_enc,
|
||||
)
|
||||
|
||||
team_path = fm.prepare_team_repo("privat")
|
||||
|
||||
# Check filtered .gitattributes
|
||||
ga_path = team_path / ".gitattributes"
|
||||
if ga_path.exists():
|
||||
content = ga_path.read_text(encoding="utf-8")
|
||||
assert "git-crypt-privat" in content
|
||||
assert "git-crypt-dhive" not in content
|
||||
assert "git-crypt-bahn" not in content
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: create_integrated_stack Factory
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCreateIntegratedStack:
|
||||
"""Tests für die create_integrated_stack Factory-Funktion."""
|
||||
|
||||
def test_creates_all_components(self, monorepo_root: Path) -> None:
|
||||
"""Factory erstellt alle Komponenten korrekt."""
|
||||
with patch(
|
||||
"monorepo.integration.SecretEncryptionManager._run_gitcrypt"
|
||||
):
|
||||
stack = create_integrated_stack(monorepo_root)
|
||||
|
||||
assert "encryption_manager" in stack
|
||||
assert "context_guard" in stack
|
||||
assert "audit_logger" in stack
|
||||
assert "orchestrator_adapter" in stack
|
||||
assert "federation_manager" in stack
|
||||
assert "machine_context_manager" in stack
|
||||
|
||||
def test_components_are_correctly_wired(self, monorepo_root: Path) -> None:
|
||||
"""Alle Komponenten sind korrekt verdrahtet."""
|
||||
with patch(
|
||||
"monorepo.integration.SecretEncryptionManager._run_gitcrypt"
|
||||
):
|
||||
stack = create_integrated_stack(monorepo_root)
|
||||
|
||||
# ContextGuard hat encryption_manager
|
||||
guard = stack["context_guard"]
|
||||
assert guard.encryption_manager is stack["encryption_manager"]
|
||||
|
||||
# OrchestratorAdapter hat encryption_manager
|
||||
adapter = stack["orchestrator_adapter"]
|
||||
assert adapter.encryption_manager is stack["encryption_manager"]
|
||||
assert adapter.context_guard is guard
|
||||
|
||||
# FederationManager hat encryption_manager
|
||||
fed_mgr = stack["federation_manager"]
|
||||
assert fed_mgr is not None
|
||||
assert fed_mgr.encryption_manager is stack["encryption_manager"]
|
||||
|
||||
def test_handles_missing_machine_context_yaml(self, tmp_path: Path) -> None:
|
||||
"""Ohne machine-context.yaml: Stack ohne Verschlüsselung."""
|
||||
root = tmp_path / "monorepo"
|
||||
root.mkdir()
|
||||
for ctx in ("privat", "dhive", "bahn", "shared"):
|
||||
(root / ctx).mkdir()
|
||||
|
||||
# access-config.yaml is required
|
||||
config_dir = root / "shared" / "config"
|
||||
config_dir.mkdir(parents=True)
|
||||
access_config = {
|
||||
"contexts": {
|
||||
"privat": {"env_file": "privat/.env", "allowed_shared": []},
|
||||
"dhive": {"env_file": "dhive/.env", "allowed_shared": []},
|
||||
"bahn": {"env_file": "bahn/.env", "allowed_shared": []},
|
||||
"shared": {"env_file": "shared/.env", "allowed_shared": ["*"]},
|
||||
}
|
||||
}
|
||||
(config_dir / "access-config.yaml").write_text(
|
||||
yaml.dump(access_config), encoding="utf-8"
|
||||
)
|
||||
|
||||
# No machine-context.yaml → stack works but without encryption
|
||||
stack = create_integrated_stack(root)
|
||||
|
||||
assert stack["encryption_manager"] is None
|
||||
assert stack["context_guard"] is not None
|
||||
assert stack["context_guard"].encryption_manager is None
|
||||
assert stack["orchestrator_adapter"] is not None
|
||||
assert stack["orchestrator_adapter"].encryption_manager is None
|
||||
|
||||
def test_handles_missing_team_repos_yaml(self, monorepo_root: Path) -> None:
|
||||
"""Ohne team-repos.yaml: FederationManager ist None."""
|
||||
# Remove team-repos.yaml
|
||||
team_repos_path = monorepo_root / "shared" / "config" / "team-repos.yaml"
|
||||
team_repos_path.unlink()
|
||||
|
||||
with patch(
|
||||
"monorepo.integration.SecretEncryptionManager._run_gitcrypt"
|
||||
):
|
||||
stack = create_integrated_stack(monorepo_root)
|
||||
|
||||
assert stack["federation_manager"] is None
|
||||
# Other components still work
|
||||
assert stack["context_guard"] is not None
|
||||
assert stack["orchestrator_adapter"] is not None
|
||||
@@ -0,0 +1,626 @@
|
||||
"""Unit-Tests für SecretEncryptionManager (Task 4.6).
|
||||
|
||||
Testet die folgenden Aspekte, die NICHT bereits in test_encryption.py abgedeckt sind:
|
||||
1. encrypt_file() – Erfolg, Autorisierung, fehlende Datei
|
||||
2. decrypt_file() – Erfolg, Autorisierung, fehlende Datei, Kontextauflösung
|
||||
3. is_authorized() – autorisierte und nicht-autorisierte Kontexte
|
||||
4. Fehlermeldungen enthalten KEINEN Dateiinhalt (keine Offenlegung)
|
||||
5. setup_gitcrypt_filters() – .gitattributes-Generierung pro Kontext
|
||||
6. Edge Cases: shared-Kontext, Dateien außerhalb von Kontextordnern
|
||||
|
||||
_Requirements: 9.1, 9.3, 9.4, 9.8, 9.10, 9.11_
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from monorepo.encryption import (
|
||||
DecryptionResult,
|
||||
EncryptionResult,
|
||||
GitCryptNotAvailableError,
|
||||
SecretEncryptionManager,
|
||||
)
|
||||
from monorepo.models import MachineContext
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tmp_monorepo(tmp_path: Path) -> Path:
|
||||
"""Erstellt eine minimale Monorepo-Struktur."""
|
||||
for ctx in ("privat", "dhive", "bahn", "shared"):
|
||||
(tmp_path / ctx).mkdir()
|
||||
return tmp_path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def full_context() -> MachineContext:
|
||||
"""MachineContext mit Zugriff auf alle Kontexte."""
|
||||
return MachineContext(
|
||||
name="andre-hauptrechner",
|
||||
description="Voller Zugriff",
|
||||
authorized_contexts=["privat", "dhive", "bahn"],
|
||||
key_source="keyring",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dhive_only_context() -> MachineContext:
|
||||
"""MachineContext nur für dhive autorisiert."""
|
||||
return MachineContext(
|
||||
name="dhive-laptop",
|
||||
description="Nur dhive",
|
||||
authorized_contexts=["dhive"],
|
||||
key_source="keyring",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def no_context() -> MachineContext:
|
||||
"""MachineContext ohne jede Autorisierung."""
|
||||
return MachineContext(
|
||||
name="fremder-rechner",
|
||||
description="Kein Zugriff",
|
||||
authorized_contexts=[],
|
||||
key_source="keyring",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def manager(tmp_monorepo: Path, full_context: MachineContext) -> SecretEncryptionManager:
|
||||
"""SecretEncryptionManager mit vollem Zugriff."""
|
||||
return SecretEncryptionManager(tmp_monorepo, full_context)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def limited_manager(
|
||||
tmp_monorepo: Path, dhive_only_context: MachineContext
|
||||
) -> SecretEncryptionManager:
|
||||
"""SecretEncryptionManager nur für dhive."""
|
||||
return SecretEncryptionManager(tmp_monorepo, dhive_only_context)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def unauthorized_manager(
|
||||
tmp_monorepo: Path, no_context: MachineContext
|
||||
) -> SecretEncryptionManager:
|
||||
"""SecretEncryptionManager ohne Autorisierung."""
|
||||
return SecretEncryptionManager(tmp_monorepo, no_context)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: encrypt_file()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEncryptFile:
|
||||
"""Tests für SecretEncryptionManager.encrypt_file()."""
|
||||
|
||||
def test_encrypt_success_with_authorized_context(
|
||||
self, tmp_monorepo: Path, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Verschlüsselung gelingt bei autorisiertem Kontext und existierender Datei."""
|
||||
secret_file = tmp_monorepo / "privat" / ".env"
|
||||
secret_file.write_text("SECRET_KEY=super-geheim-123")
|
||||
|
||||
# Mock git-crypt Aufruf, da git-crypt nicht installiert ist
|
||||
with patch.object(manager, "_run_gitcrypt") as mock_gc:
|
||||
mock_gc.return_value = None
|
||||
result = manager.encrypt_file(secret_file, "privat")
|
||||
|
||||
assert isinstance(result, EncryptionResult)
|
||||
assert result.success is True
|
||||
assert result.file_path == secret_file
|
||||
assert result.context == "privat"
|
||||
assert result.error is None
|
||||
|
||||
def test_encrypt_fails_with_unauthorized_context(
|
||||
self, tmp_monorepo: Path, limited_manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Verschlüsselung schlägt fehl bei nicht-autorisiertem Kontext."""
|
||||
secret_file = tmp_monorepo / "privat" / ".env"
|
||||
secret_file.write_text("SECRET=value")
|
||||
|
||||
result = limited_manager.encrypt_file(secret_file, "privat")
|
||||
|
||||
assert result.success is False
|
||||
assert "nicht" in result.error.lower() or "autorisiert" in result.error.lower()
|
||||
|
||||
def test_encrypt_fails_when_file_does_not_exist(
|
||||
self, tmp_monorepo: Path, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Verschlüsselung schlägt fehl wenn die Datei nicht existiert."""
|
||||
nonexistent = tmp_monorepo / "privat" / "nonexistent.env"
|
||||
|
||||
result = manager.encrypt_file(nonexistent, "privat")
|
||||
|
||||
assert result.success is False
|
||||
assert "existiert nicht" in result.error
|
||||
|
||||
def test_encrypt_fails_when_gitcrypt_not_available(
|
||||
self, tmp_monorepo: Path, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Verschlüsselung schlägt fehl wenn git-crypt nicht installiert ist."""
|
||||
secret_file = tmp_monorepo / "dhive" / "credentials.pem"
|
||||
secret_file.write_text("-----BEGIN PRIVATE KEY-----\n...")
|
||||
|
||||
with patch.object(manager, "_run_gitcrypt") as mock_gc:
|
||||
mock_gc.side_effect = GitCryptNotAvailableError(
|
||||
"git-crypt ist nicht installiert"
|
||||
)
|
||||
result = manager.encrypt_file(secret_file, "dhive")
|
||||
|
||||
assert result.success is False
|
||||
assert "git-crypt" in result.error.lower()
|
||||
|
||||
def test_encrypt_for_bahn_context(
|
||||
self, tmp_monorepo: Path, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Verschlüsselung funktioniert auch für Bahn-Kontext."""
|
||||
bahn_secret = tmp_monorepo / "bahn" / "api-token.key"
|
||||
bahn_secret.write_text("token-abc-123")
|
||||
|
||||
with patch.object(manager, "_run_gitcrypt"):
|
||||
result = manager.encrypt_file(bahn_secret, "bahn")
|
||||
|
||||
assert result.success is True
|
||||
assert result.context == "bahn"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: decrypt_file()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDecryptFile:
|
||||
"""Tests für SecretEncryptionManager.decrypt_file()."""
|
||||
|
||||
def test_decrypt_success_with_authorized_context(
|
||||
self, tmp_monorepo: Path, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Entschlüsselung gelingt bei autorisiertem Kontext."""
|
||||
secret_file = tmp_monorepo / "privat" / ".env"
|
||||
secret_content = b"DB_PASSWORD=geheim123"
|
||||
secret_file.write_bytes(secret_content)
|
||||
|
||||
with patch.object(manager, "_run_gitcrypt"):
|
||||
result = manager.decrypt_file(secret_file)
|
||||
|
||||
assert isinstance(result, DecryptionResult)
|
||||
assert result.success is True
|
||||
assert result.content == secret_content
|
||||
|
||||
def test_decrypt_fails_with_unauthorized_context(
|
||||
self, tmp_monorepo: Path, limited_manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Entschlüsselung schlägt fehl bei nicht-autorisiertem Kontext."""
|
||||
privat_secret = tmp_monorepo / "privat" / ".env"
|
||||
privat_secret.write_text("PRIVATE_SECRET=xyz")
|
||||
|
||||
result = limited_manager.decrypt_file(privat_secret)
|
||||
|
||||
assert result.success is False
|
||||
assert result.content is None
|
||||
assert "verweigert" in result.error.lower() or "autorisiert" in result.error.lower()
|
||||
|
||||
def test_decrypt_succeeds_for_own_context(
|
||||
self, tmp_monorepo: Path, limited_manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Eingeschränkter Manager kann eigenen Kontext entschlüsseln."""
|
||||
dhive_secret = tmp_monorepo / "dhive" / ".env"
|
||||
dhive_content = b"DHIVE_API_KEY=abc"
|
||||
dhive_secret.write_bytes(dhive_content)
|
||||
|
||||
with patch.object(limited_manager, "_run_gitcrypt"):
|
||||
result = limited_manager.decrypt_file(dhive_secret)
|
||||
|
||||
assert result.success is True
|
||||
assert result.content == dhive_content
|
||||
|
||||
def test_decrypt_fails_when_file_does_not_exist(
|
||||
self, tmp_monorepo: Path, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Entschlüsselung schlägt fehl wenn die Datei nicht existiert."""
|
||||
nonexistent = tmp_monorepo / "dhive" / "missing.key"
|
||||
|
||||
result = manager.decrypt_file(nonexistent)
|
||||
|
||||
assert result.success is False
|
||||
assert "existiert nicht" in result.error
|
||||
|
||||
def test_decrypt_resolves_context_from_path(
|
||||
self, tmp_monorepo: Path, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Entschlüsselung ermittelt den Kontext automatisch aus dem Dateipfad."""
|
||||
bahn_secret = tmp_monorepo / "bahn" / "project-x" / "secret.key"
|
||||
bahn_secret.parent.mkdir(parents=True, exist_ok=True)
|
||||
bahn_secret.write_bytes(b"key-data")
|
||||
|
||||
with patch.object(manager, "_run_gitcrypt"):
|
||||
result = manager.decrypt_file(bahn_secret)
|
||||
|
||||
assert result.success is True
|
||||
assert result.content == b"key-data"
|
||||
|
||||
def test_decrypt_fails_for_file_outside_context_dirs(
|
||||
self, tmp_monorepo: Path, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Entschlüsselung schlägt fehl für Dateien außerhalb der Kontextordner."""
|
||||
outside_file = tmp_monorepo / "unknown-dir" / "secrets.env"
|
||||
outside_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
outside_file.write_text("SHOULD_NOT_DECRYPT=true")
|
||||
|
||||
result = manager.decrypt_file(outside_file)
|
||||
|
||||
assert result.success is False
|
||||
assert "kontext" in result.error.lower()
|
||||
|
||||
def test_decrypt_fails_for_root_level_file(
|
||||
self, tmp_monorepo: Path, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Entschlüsselung schlägt fehl für Datei direkt im Root (kein Kontext)."""
|
||||
root_file = tmp_monorepo / ".env"
|
||||
root_file.write_text("ROOT_SECRET=hidden")
|
||||
|
||||
result = manager.decrypt_file(root_file)
|
||||
|
||||
# .env im Root hat keinen gültigen Kontext als erstes Pfadsegment
|
||||
assert result.success is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: is_authorized()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestIsAuthorized:
|
||||
"""Tests für SecretEncryptionManager.is_authorized()."""
|
||||
|
||||
def test_authorized_for_all_contexts_full_access(
|
||||
self, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Full-Access-Manager ist für alle drei Kontexte autorisiert."""
|
||||
assert manager.is_authorized("privat") is True
|
||||
assert manager.is_authorized("dhive") is True
|
||||
assert manager.is_authorized("bahn") is True
|
||||
|
||||
def test_authorized_only_for_own_context(
|
||||
self, limited_manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Eingeschränkter Manager ist nur für seinen Kontext autorisiert."""
|
||||
assert limited_manager.is_authorized("dhive") is True
|
||||
assert limited_manager.is_authorized("privat") is False
|
||||
assert limited_manager.is_authorized("bahn") is False
|
||||
|
||||
def test_not_authorized_for_any_context(
|
||||
self, unauthorized_manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Manager ohne Autorisierung ist für keinen Kontext autorisiert."""
|
||||
assert unauthorized_manager.is_authorized("privat") is False
|
||||
assert unauthorized_manager.is_authorized("dhive") is False
|
||||
assert unauthorized_manager.is_authorized("bahn") is False
|
||||
|
||||
def test_not_authorized_for_nonexistent_context(
|
||||
self, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Nicht existierender Kontext ist nie autorisiert."""
|
||||
assert manager.is_authorized("nonexistent") is False
|
||||
assert manager.is_authorized("") is False
|
||||
|
||||
def test_shared_context_not_in_authorized_list(
|
||||
self, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""'shared' ist per Design nicht in der Autorisierungsliste."""
|
||||
# shared wird über andere Mechanismen gesteuert
|
||||
assert manager.is_authorized("shared") is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Fehlermeldungen offenlegen KEINEN Dateiinhalt (Requirement 9.8)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestErrorMessagesSecurity:
|
||||
"""Fehlermeldungen dürfen KEINEN Dateiinhalt offenlegen."""
|
||||
|
||||
def test_encrypt_error_does_not_reveal_content(
|
||||
self, tmp_monorepo: Path, limited_manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Verschlüsselungs-Fehlermeldung enthält keinen Dateiinhalt."""
|
||||
secret_file = tmp_monorepo / "privat" / ".env"
|
||||
secret_content = "SUPER_SECRET_API_KEY=sk-12345abcde"
|
||||
secret_file.write_text(secret_content)
|
||||
|
||||
result = limited_manager.encrypt_file(secret_file, "privat")
|
||||
|
||||
assert result.success is False
|
||||
# Fehlermeldung darf KEINEN Teil des Dateiinhalts enthalten
|
||||
assert "SUPER_SECRET_API_KEY" not in (result.error or "")
|
||||
assert "sk-12345abcde" not in (result.error or "")
|
||||
assert secret_content not in (result.error or "")
|
||||
|
||||
def test_decrypt_error_does_not_reveal_content(
|
||||
self, tmp_monorepo: Path, limited_manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Entschlüsselungs-Fehlermeldung enthält keinen Dateiinhalt."""
|
||||
secret_file = tmp_monorepo / "privat" / "credentials.key"
|
||||
secret_content = "-----BEGIN RSA PRIVATE KEY-----\nMIIE...very-secret"
|
||||
secret_file.write_text(secret_content)
|
||||
|
||||
result = limited_manager.decrypt_file(secret_file)
|
||||
|
||||
assert result.success is False
|
||||
assert result.content is None
|
||||
# Fehlermeldung darf KEINEN Teil des Dateiinhalts enthalten
|
||||
assert "BEGIN RSA PRIVATE KEY" not in (result.error or "")
|
||||
assert "MIIE" not in (result.error or "")
|
||||
assert "very-secret" not in (result.error or "")
|
||||
|
||||
def test_decrypt_error_for_unauthorized_binary_file(
|
||||
self, tmp_monorepo: Path, limited_manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Fehlermeldung für verschlüsselte Binärdatei enthält keine Bytes."""
|
||||
binary_file = tmp_monorepo / "bahn" / "cert.pem"
|
||||
binary_content = b"\x00\x01\x02\x03PRIVATE_DATA\xff\xfe"
|
||||
binary_file.write_bytes(binary_content)
|
||||
|
||||
result = limited_manager.decrypt_file(binary_file)
|
||||
|
||||
assert result.success is False
|
||||
assert result.content is None
|
||||
# Keine binären Daten oder Schlüsselwörter in der Fehlermeldung
|
||||
assert "PRIVATE_DATA" not in (result.error or "")
|
||||
|
||||
def test_encrypt_error_has_useful_context_info(
|
||||
self, tmp_monorepo: Path, limited_manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Fehlermeldung enthält nützliche Kontextinformationen (ohne Secrets)."""
|
||||
secret_file = tmp_monorepo / "privat" / "token.secret"
|
||||
secret_file.write_text("geheim-token-value")
|
||||
|
||||
result = limited_manager.encrypt_file(secret_file, "privat")
|
||||
|
||||
assert result.success is False
|
||||
# Fehlermeldung sollte den Maschinennamen oder Kontext benennen
|
||||
assert "dhive-laptop" in result.error or "privat" in result.error
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: setup_gitcrypt_filters() – .gitattributes-Generierung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSetupGitcryptFilters:
|
||||
"""Tests für SecretEncryptionManager.setup_gitcrypt_filters()."""
|
||||
|
||||
def test_generates_gitattributes_for_privat(
|
||||
self, tmp_monorepo: Path, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Erzeugt .gitattributes im privat-Kontext mit korrekten Filtern."""
|
||||
manager.setup_gitcrypt_filters("privat")
|
||||
|
||||
gitattr_path = tmp_monorepo / "privat" / ".gitattributes"
|
||||
assert gitattr_path.exists()
|
||||
|
||||
content = gitattr_path.read_text(encoding="utf-8")
|
||||
assert "filter=git-crypt-privat" in content
|
||||
assert "diff=git-crypt-privat" in content
|
||||
|
||||
def test_generates_gitattributes_for_dhive(
|
||||
self, tmp_monorepo: Path, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Erzeugt .gitattributes im dhive-Kontext."""
|
||||
manager.setup_gitcrypt_filters("dhive")
|
||||
|
||||
gitattr_path = tmp_monorepo / "dhive" / ".gitattributes"
|
||||
assert gitattr_path.exists()
|
||||
|
||||
content = gitattr_path.read_text(encoding="utf-8")
|
||||
assert "filter=git-crypt-dhive" in content
|
||||
assert "diff=git-crypt-dhive" in content
|
||||
|
||||
def test_generates_gitattributes_for_bahn(
|
||||
self, tmp_monorepo: Path, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Erzeugt .gitattributes im bahn-Kontext."""
|
||||
manager.setup_gitcrypt_filters("bahn")
|
||||
|
||||
gitattr_path = tmp_monorepo / "bahn" / ".gitattributes"
|
||||
assert gitattr_path.exists()
|
||||
|
||||
content = gitattr_path.read_text(encoding="utf-8")
|
||||
assert "filter=git-crypt-bahn" in content
|
||||
assert "diff=git-crypt-bahn" in content
|
||||
|
||||
def test_contains_all_secret_patterns(
|
||||
self, tmp_monorepo: Path, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Die generierten .gitattributes enthalten alle Secret-Patterns."""
|
||||
manager.setup_gitcrypt_filters("privat")
|
||||
|
||||
content = (tmp_monorepo / "privat" / ".gitattributes").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
# Alle SECRET_PATTERNS müssen als gitattributes-Patterns vorhanden sein
|
||||
assert ".env" in content
|
||||
assert "*.pem" in content
|
||||
assert "*.key" in content
|
||||
assert "*token*" in content
|
||||
assert "*secret*" in content
|
||||
|
||||
def test_filter_names_are_context_specific(
|
||||
self, tmp_monorepo: Path, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Jeder Kontext hat seinen eigenen Filter-Namen."""
|
||||
manager.setup_gitcrypt_filters("privat")
|
||||
manager.setup_gitcrypt_filters("dhive")
|
||||
manager.setup_gitcrypt_filters("bahn")
|
||||
|
||||
privat_content = (tmp_monorepo / "privat" / ".gitattributes").read_text()
|
||||
dhive_content = (tmp_monorepo / "dhive" / ".gitattributes").read_text()
|
||||
bahn_content = (tmp_monorepo / "bahn" / ".gitattributes").read_text()
|
||||
|
||||
# Jeder Kontext referenziert nur seinen eigenen Filter
|
||||
assert "git-crypt-privat" in privat_content
|
||||
assert "git-crypt-dhive" not in privat_content
|
||||
|
||||
assert "git-crypt-dhive" in dhive_content
|
||||
assert "git-crypt-privat" not in dhive_content
|
||||
|
||||
assert "git-crypt-bahn" in bahn_content
|
||||
assert "git-crypt-privat" not in bahn_content
|
||||
|
||||
def test_creates_context_directory_if_missing(
|
||||
self, tmp_path: Path, full_context: MachineContext
|
||||
) -> None:
|
||||
"""Erstellt den Kontextordner falls er nicht existiert."""
|
||||
mgr = SecretEncryptionManager(tmp_path, full_context)
|
||||
mgr.setup_gitcrypt_filters("privat")
|
||||
|
||||
assert (tmp_path / "privat" / ".gitattributes").exists()
|
||||
|
||||
def test_overwrites_existing_gitattributes(
|
||||
self, tmp_monorepo: Path, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Überschreibt bestehende .gitattributes-Datei."""
|
||||
gitattr_path = tmp_monorepo / "privat" / ".gitattributes"
|
||||
gitattr_path.write_text("# alte Konfiguration\n*.old filter=old")
|
||||
|
||||
manager.setup_gitcrypt_filters("privat")
|
||||
|
||||
content = gitattr_path.read_text(encoding="utf-8")
|
||||
assert "alte Konfiguration" not in content
|
||||
assert "filter=git-crypt-privat" in content
|
||||
|
||||
def test_gitattributes_has_header_comment(
|
||||
self, tmp_monorepo: Path, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Die .gitattributes-Datei enthält einen informativen Header."""
|
||||
manager.setup_gitcrypt_filters("bahn")
|
||||
|
||||
content = (tmp_monorepo / "bahn" / ".gitattributes").read_text(encoding="utf-8")
|
||||
assert "# git-crypt Filter" in content
|
||||
assert "bahn" in content
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Edge Cases – shared-Kontext und Dateien außerhalb
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEdgeCases:
|
||||
"""Edge Cases für die Verschlüsselungs-Operationen."""
|
||||
|
||||
def test_shared_context_file_decrypt_unauthorized(
|
||||
self, tmp_monorepo: Path, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Datei im shared-Kontext: 'shared' ist nicht in authorized_contexts."""
|
||||
shared_file = tmp_monorepo / "shared" / "config" / "secrets.env"
|
||||
shared_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
shared_file.write_text("SHARED_SECRET=123")
|
||||
|
||||
# 'shared' ist nicht in der authorized_contexts-Liste des full_context
|
||||
result = manager.decrypt_file(shared_file)
|
||||
|
||||
# Da 'shared' nicht in authorized_contexts ist, schlägt es fehl
|
||||
assert result.success is False
|
||||
|
||||
def test_deeply_nested_file_resolves_context(
|
||||
self, tmp_monorepo: Path, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Tief verschachtelte Datei wird korrekt dem Kontext zugeordnet."""
|
||||
deep_file = tmp_monorepo / "dhive" / "proj" / "module" / "sub" / ".env"
|
||||
deep_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
deep_file.write_bytes(b"DEEP_SECRET=value")
|
||||
|
||||
with patch.object(manager, "_run_gitcrypt"):
|
||||
result = manager.decrypt_file(deep_file)
|
||||
|
||||
assert result.success is True
|
||||
assert result.content == b"DEEP_SECRET=value"
|
||||
|
||||
def test_encrypt_file_with_empty_content(
|
||||
self, tmp_monorepo: Path, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Leere Datei kann verschlüsselt werden."""
|
||||
empty_file = tmp_monorepo / "privat" / "empty.key"
|
||||
empty_file.write_text("")
|
||||
|
||||
with patch.object(manager, "_run_gitcrypt"):
|
||||
result = manager.encrypt_file(empty_file, "privat")
|
||||
|
||||
assert result.success is True
|
||||
|
||||
def test_encrypt_file_path_outside_monorepo(
|
||||
self, tmp_path: Path, full_context: MachineContext
|
||||
) -> None:
|
||||
"""Datei komplett außerhalb des Monorepo-Roots."""
|
||||
# Erstelle einen separaten Pfad als "Monorepo-Root"
|
||||
mono_root = tmp_path / "monorepo"
|
||||
mono_root.mkdir()
|
||||
(mono_root / "privat").mkdir()
|
||||
|
||||
mgr = SecretEncryptionManager(mono_root, full_context)
|
||||
|
||||
# Datei außerhalb
|
||||
outside_file = tmp_path / "outside" / "secret.env"
|
||||
outside_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
outside_file.write_text("OUTSIDE=true")
|
||||
|
||||
# encrypt_file prüft nur Existenz und Autorisierung, nicht den Pfad
|
||||
# aber es sollte nicht abstürzen
|
||||
with patch.object(mgr, "_run_gitcrypt"):
|
||||
result = mgr.encrypt_file(outside_file, "privat")
|
||||
|
||||
assert result.success is True # encrypt prüft Pfad nicht, nur Kontext-Param
|
||||
|
||||
def test_decrypt_file_path_outside_monorepo(
|
||||
self, tmp_path: Path, full_context: MachineContext
|
||||
) -> None:
|
||||
"""Datei außerhalb des Monorepo-Roots kann nicht entschlüsselt werden."""
|
||||
mono_root = tmp_path / "monorepo"
|
||||
mono_root.mkdir()
|
||||
|
||||
mgr = SecretEncryptionManager(mono_root, full_context)
|
||||
|
||||
outside_file = tmp_path / "somewhere-else" / "secret.env"
|
||||
outside_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
outside_file.write_text("OUTSIDE=true")
|
||||
|
||||
result = mgr.decrypt_file(outside_file)
|
||||
|
||||
# Kontext kann nicht aus dem Pfad ermittelt werden
|
||||
assert result.success is False
|
||||
assert "kontext" in result.error.lower()
|
||||
|
||||
def test_resolve_merge_for_shared_context_file(
|
||||
self, tmp_monorepo: Path, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""Merge-Auflösung für Datei im shared-Ordner: ours gewinnt (shared nicht autorisiert)."""
|
||||
shared_file = tmp_monorepo / "shared" / "config.yaml"
|
||||
shared_file.write_text("config: value")
|
||||
|
||||
result = manager.resolve_merge(shared_file, b"ours", b"theirs")
|
||||
|
||||
# shared ist nicht in authorized_contexts → ours beibehalten
|
||||
assert result == b"ours"
|
||||
|
||||
def test_setup_gitcrypt_filters_for_shared_context(
|
||||
self, tmp_monorepo: Path, manager: SecretEncryptionManager
|
||||
) -> None:
|
||||
"""setup_gitcrypt_filters funktioniert auch für den shared-Kontext."""
|
||||
manager.setup_gitcrypt_filters("shared")
|
||||
|
||||
gitattr_path = tmp_monorepo / "shared" / ".gitattributes"
|
||||
assert gitattr_path.exists()
|
||||
|
||||
content = gitattr_path.read_text(encoding="utf-8")
|
||||
assert "filter=git-crypt-shared" in content
|
||||
@@ -0,0 +1,506 @@
|
||||
"""Tests für ETLPipeline und MarkdownSource.
|
||||
|
||||
Validiert:
|
||||
- MarkdownSource: Rekursives Scannen, Fehlerbehandlung, Frontmatter-Parsing
|
||||
- ETLPipeline: Inkrementelle Verarbeitung, Fehlerresilienz, Index-Update
|
||||
- IngestResult: Korrekte Zähler und Fehlerlisten
|
||||
- KnowledgeStore.ingest(): Integration mit ETLPipeline
|
||||
|
||||
Requirements: 3.2, 3.6, 3.10, 3.12, 3.20, 3.21
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from monorepo.knowledge.etl import ETLPipeline, IngestError, IngestResult
|
||||
from monorepo.knowledge.index import IndexEntry, YAMLIndex
|
||||
from monorepo.knowledge.sources.markdown import MarkdownSource, SourceError
|
||||
from monorepo.knowledge.store import KnowledgeStore
|
||||
from monorepo.models import ScopeConfig
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def scope_config() -> ScopeConfig:
|
||||
"""Beispiel-ScopeConfig."""
|
||||
return ScopeConfig(scopes={
|
||||
"privat": {"scope": "privat", "paths": ["privat/"]},
|
||||
"dhive": {"scope": "dhive", "paths": ["dhive/"]},
|
||||
"bahn": {"scope": "bahn", "paths": ["bahn/"]},
|
||||
"shared": {"scope": "shared", "paths": ["shared/"]},
|
||||
})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def source_dir_with_artifacts(tmp_path: Path) -> Path:
|
||||
"""Verzeichnis mit gültigen Markdown-Dateien mit YAML-Frontmatter."""
|
||||
source_dir = tmp_path / "source"
|
||||
source_dir.mkdir()
|
||||
|
||||
# Datei 1: gültiges Artefakt
|
||||
(source_dir / "api-design.md").write_text(
|
||||
"---\n"
|
||||
"type: decision\n"
|
||||
"title: API-Designprinzipien\n"
|
||||
"tags: [api, rest]\n"
|
||||
"---\n"
|
||||
"\n# API-Designprinzipien\n\nREST-Konventionen für Services.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# Datei 2: gültiges Artefakt
|
||||
(source_dir / "cloud-patterns.md").write_text(
|
||||
"---\n"
|
||||
"type: pattern\n"
|
||||
"title: Cloud-Native Patterns\n"
|
||||
"tags: [cloud, kubernetes]\n"
|
||||
"---\n"
|
||||
"\n# Cloud-Native Patterns\n\nBest Practices für cloud-native Apps.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
return source_dir
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def source_dir_with_subdirs(tmp_path: Path) -> Path:
|
||||
"""Verzeichnis mit verschachtelten Markdown-Dateien."""
|
||||
source_dir = tmp_path / "nested_source"
|
||||
source_dir.mkdir()
|
||||
|
||||
# Top-level Datei
|
||||
(source_dir / "top-level.md").write_text(
|
||||
"---\ntype: note\ntitle: Top Level\ntags: []\n---\n\nTop level content.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# Unterverzeichnis
|
||||
sub_dir = source_dir / "decisions"
|
||||
sub_dir.mkdir()
|
||||
(sub_dir / "decision-one.md").write_text(
|
||||
"---\ntype: decision\ntitle: Decision One\ntags: [arch]\n---\n\nDecision content.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
return source_dir
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def source_dir_with_invalid(tmp_path: Path) -> Path:
|
||||
"""Verzeichnis mit gemischten gültigen und ungültigen Dateien."""
|
||||
source_dir = tmp_path / "mixed"
|
||||
source_dir.mkdir()
|
||||
|
||||
# Gültige Datei
|
||||
(source_dir / "valid.md").write_text(
|
||||
"---\ntype: note\ntitle: Valid Note\ntags: [test]\n---\n\nValid content.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# Ungültige Datei (kein Frontmatter)
|
||||
(source_dir / "no-frontmatter.md").write_text(
|
||||
"# Just a Heading\n\nNo YAML frontmatter here.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# Ungültige Datei (leeres Frontmatter)
|
||||
(source_dir / "empty-frontmatter.md").write_text(
|
||||
"---\n---\n\nEmpty frontmatter.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
return source_dir
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MarkdownSource Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMarkdownSource:
|
||||
"""Tests für MarkdownSource."""
|
||||
|
||||
def test_extract_finds_all_md_files(
|
||||
self, source_dir_with_artifacts: Path
|
||||
) -> None:
|
||||
"""extract() findet alle .md-Dateien im Verzeichnis."""
|
||||
source = MarkdownSource(directory=source_dir_with_artifacts)
|
||||
artifacts = source.extract()
|
||||
|
||||
assert len(artifacts) == 2
|
||||
titles = {a.metadata.title for a in artifacts}
|
||||
assert "API-Designprinzipien" in titles
|
||||
assert "Cloud-Native Patterns" in titles
|
||||
|
||||
def test_extract_recursive(self, source_dir_with_subdirs: Path) -> None:
|
||||
"""extract() findet Dateien in Unterverzeichnissen."""
|
||||
source = MarkdownSource(directory=source_dir_with_subdirs)
|
||||
artifacts = source.extract()
|
||||
|
||||
assert len(artifacts) == 2
|
||||
titles = {a.metadata.title for a in artifacts}
|
||||
assert "Top Level" in titles
|
||||
assert "Decision One" in titles
|
||||
|
||||
def test_extract_nonexistent_directory(self, tmp_path: Path) -> None:
|
||||
"""extract() bei nicht-existierendem Verzeichnis gibt leere Liste + Fehler."""
|
||||
source = MarkdownSource(directory=tmp_path / "nonexistent")
|
||||
artifacts = source.extract()
|
||||
|
||||
assert artifacts == []
|
||||
assert len(source.errors) == 1
|
||||
assert source.errors[0].retry is True
|
||||
|
||||
def test_extract_handles_invalid_frontmatter_gracefully(
|
||||
self, source_dir_with_invalid: Path
|
||||
) -> None:
|
||||
"""extract() überspringt Dateien ohne gültiges Frontmatter und loggt Fehler."""
|
||||
source = MarkdownSource(directory=source_dir_with_invalid)
|
||||
artifacts = source.extract()
|
||||
|
||||
# valid.md und empty-frontmatter.md parse successfully
|
||||
# (empty-frontmatter has valid --- delimiters, just empty metadata)
|
||||
# Only no-frontmatter.md is rejected (no --- at start)
|
||||
assert len(artifacts) == 2
|
||||
|
||||
# Fehler für die ungültige Datei ohne Frontmatter
|
||||
assert len(source.errors) == 1
|
||||
assert any("no-frontmatter" in e.file_path for e in source.errors)
|
||||
|
||||
def test_extract_empty_directory(self, tmp_path: Path) -> None:
|
||||
"""extract() bei leerem Verzeichnis gibt leere Liste ohne Fehler."""
|
||||
empty_dir = tmp_path / "empty"
|
||||
empty_dir.mkdir()
|
||||
|
||||
source = MarkdownSource(directory=empty_dir)
|
||||
artifacts = source.extract()
|
||||
|
||||
assert artifacts == []
|
||||
assert source.errors == []
|
||||
|
||||
def test_extract_file_instead_of_directory(self, tmp_path: Path) -> None:
|
||||
"""extract() bei Datei statt Verzeichnis gibt Fehler."""
|
||||
file_path = tmp_path / "not-a-dir.md"
|
||||
file_path.write_text("content", encoding="utf-8")
|
||||
|
||||
source = MarkdownSource(directory=file_path)
|
||||
artifacts = source.extract()
|
||||
|
||||
assert artifacts == []
|
||||
assert len(source.errors) == 1
|
||||
assert source.errors[0].retry is False
|
||||
|
||||
def test_extract_resets_errors(self, source_dir_with_invalid: Path) -> None:
|
||||
"""extract() setzt vorherige Fehler zurück bei erneutem Aufruf."""
|
||||
source = MarkdownSource(directory=source_dir_with_invalid)
|
||||
source.extract()
|
||||
first_errors = len(source.errors)
|
||||
|
||||
source.extract()
|
||||
# Fehler sollten gleich sein (kein Akkumulieren)
|
||||
assert len(source.errors) == first_errors
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ETLPipeline Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestETLPipeline:
|
||||
"""Tests für ETLPipeline."""
|
||||
|
||||
def test_ingest_processes_all_artifacts(
|
||||
self, tmp_path: Path, source_dir_with_artifacts: Path
|
||||
) -> None:
|
||||
"""ingest() verarbeitet alle gültigen Artefakte aus der Quelle."""
|
||||
store_path = tmp_path / "store"
|
||||
store_path.mkdir()
|
||||
index = YAMLIndex(store_path / "_index.yaml")
|
||||
index.load()
|
||||
|
||||
pipeline = ETLPipeline(index=index, store_path=store_path)
|
||||
source = MarkdownSource(directory=source_dir_with_artifacts)
|
||||
|
||||
result = pipeline.ingest(source, "bahn")
|
||||
|
||||
assert result.processed == 2
|
||||
assert result.updated == 2
|
||||
assert result.skipped == 0
|
||||
assert result.errors == []
|
||||
|
||||
def test_ingest_creates_index_entries(
|
||||
self, tmp_path: Path, source_dir_with_artifacts: Path
|
||||
) -> None:
|
||||
"""ingest() erstellt Index-Einträge für alle verarbeiteten Artefakte."""
|
||||
store_path = tmp_path / "store"
|
||||
store_path.mkdir()
|
||||
index = YAMLIndex(store_path / "_index.yaml")
|
||||
index.load()
|
||||
|
||||
pipeline = ETLPipeline(index=index, store_path=store_path)
|
||||
source = MarkdownSource(directory=source_dir_with_artifacts)
|
||||
pipeline.ingest(source, "bahn")
|
||||
|
||||
# Index enthält Einträge
|
||||
assert len(index.entries) == 2
|
||||
api_entry = index.get_entry("bahn/decision/api-design")
|
||||
assert api_entry is not None
|
||||
assert api_entry.title == "API-Designprinzipien"
|
||||
assert api_entry.scope == "bahn"
|
||||
assert api_entry.type == "decision"
|
||||
assert "api" in api_entry.tags
|
||||
assert api_entry.content_hash.startswith("sha256:")
|
||||
|
||||
def test_ingest_incremental_skips_unchanged(
|
||||
self, tmp_path: Path, source_dir_with_artifacts: Path
|
||||
) -> None:
|
||||
"""ingest() überspringt Artefakte mit unverändertem Content-Hash."""
|
||||
store_path = tmp_path / "store"
|
||||
store_path.mkdir()
|
||||
index = YAMLIndex(store_path / "_index.yaml")
|
||||
index.load()
|
||||
|
||||
pipeline = ETLPipeline(index=index, store_path=store_path)
|
||||
source = MarkdownSource(directory=source_dir_with_artifacts)
|
||||
|
||||
# Erster Durchlauf: alle werden verarbeitet
|
||||
result1 = pipeline.ingest(source, "bahn")
|
||||
assert result1.updated == 2
|
||||
|
||||
# Zweiter Durchlauf: alle werden übersprungen (unverändert)
|
||||
source2 = MarkdownSource(directory=source_dir_with_artifacts)
|
||||
result2 = pipeline.ingest(source2, "bahn")
|
||||
assert result2.processed == 2
|
||||
assert result2.updated == 0
|
||||
assert result2.skipped == 2
|
||||
|
||||
def test_ingest_incremental_detects_changes(
|
||||
self, tmp_path: Path, source_dir_with_artifacts: Path
|
||||
) -> None:
|
||||
"""ingest() erkennt geänderte Inhalte über Content-Hash."""
|
||||
store_path = tmp_path / "store"
|
||||
store_path.mkdir()
|
||||
index = YAMLIndex(store_path / "_index.yaml")
|
||||
index.load()
|
||||
|
||||
pipeline = ETLPipeline(index=index, store_path=store_path)
|
||||
source = MarkdownSource(directory=source_dir_with_artifacts)
|
||||
pipeline.ingest(source, "bahn")
|
||||
|
||||
# Inhalt einer Datei ändern
|
||||
api_file = source_dir_with_artifacts / "api-design.md"
|
||||
api_file.write_text(
|
||||
"---\n"
|
||||
"type: decision\n"
|
||||
"title: API-Designprinzipien\n"
|
||||
"tags: [api, rest, updated]\n"
|
||||
"---\n"
|
||||
"\n# API-Designprinzipien V2\n\nAktualisierte Konventionen.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# Zweiter Durchlauf: nur die geänderte Datei wird aktualisiert
|
||||
source2 = MarkdownSource(directory=source_dir_with_artifacts)
|
||||
result2 = pipeline.ingest(source2, "bahn")
|
||||
assert result2.updated == 1
|
||||
assert result2.skipped == 1
|
||||
|
||||
def test_ingest_error_resilience(
|
||||
self, tmp_path: Path, source_dir_with_invalid: Path
|
||||
) -> None:
|
||||
"""ingest() bewahrt erfolgreiche Artefakte bei Quellfehlern."""
|
||||
store_path = tmp_path / "store"
|
||||
store_path.mkdir()
|
||||
index = YAMLIndex(store_path / "_index.yaml")
|
||||
index.load()
|
||||
|
||||
pipeline = ETLPipeline(index=index, store_path=store_path)
|
||||
source = MarkdownSource(directory=source_dir_with_invalid)
|
||||
|
||||
result = pipeline.ingest(source, "privat")
|
||||
|
||||
# valid.md and empty-frontmatter.md are parseable (2 artifacts)
|
||||
# no-frontmatter.md is rejected by source
|
||||
assert result.processed == 2
|
||||
assert result.updated == 2
|
||||
# Fehler für ungültige Dateien wurden protokolliert
|
||||
assert len(result.errors) == 1
|
||||
# Erfolgreiche Artefakte im Index
|
||||
assert len(index.entries) == 2
|
||||
|
||||
def test_ingest_saves_index(
|
||||
self, tmp_path: Path, source_dir_with_artifacts: Path
|
||||
) -> None:
|
||||
"""ingest() speichert den Index im selben Durchlauf."""
|
||||
store_path = tmp_path / "store"
|
||||
store_path.mkdir()
|
||||
index_path = store_path / "_index.yaml"
|
||||
index = YAMLIndex(index_path)
|
||||
index.load()
|
||||
|
||||
pipeline = ETLPipeline(index=index, store_path=store_path)
|
||||
source = MarkdownSource(directory=source_dir_with_artifacts)
|
||||
pipeline.ingest(source, "bahn")
|
||||
|
||||
# Index-Datei wurde geschrieben
|
||||
assert index_path.exists()
|
||||
with open(index_path, encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f)
|
||||
assert len(data["artifacts"]) == 2
|
||||
assert data["version"] == "1.0"
|
||||
assert "last_updated" in data
|
||||
|
||||
def test_ingest_sets_metadata(
|
||||
self, tmp_path: Path, source_dir_with_artifacts: Path
|
||||
) -> None:
|
||||
"""ingest() setzt source_context und Datum in den Metadaten."""
|
||||
store_path = tmp_path / "store"
|
||||
store_path.mkdir()
|
||||
index = YAMLIndex(store_path / "_index.yaml")
|
||||
index.load()
|
||||
|
||||
pipeline = ETLPipeline(index=index, store_path=store_path)
|
||||
source = MarkdownSource(directory=source_dir_with_artifacts)
|
||||
pipeline.ingest(source, "bahn")
|
||||
|
||||
entry = index.get_entry("bahn/decision/api-design")
|
||||
assert entry is not None
|
||||
assert entry.scope == "bahn"
|
||||
# Content-Hash ist gesetzt
|
||||
assert entry.content_hash.startswith("sha256:")
|
||||
|
||||
def test_ingest_writes_artifact_files(
|
||||
self, tmp_path: Path, source_dir_with_artifacts: Path
|
||||
) -> None:
|
||||
"""ingest() schreibt Artefakt-Dateien in die Scope-basierte Ordnerstruktur."""
|
||||
store_path = tmp_path / "store"
|
||||
store_path.mkdir()
|
||||
index = YAMLIndex(store_path / "_index.yaml")
|
||||
index.load()
|
||||
|
||||
pipeline = ETLPipeline(index=index, store_path=store_path)
|
||||
source = MarkdownSource(directory=source_dir_with_artifacts)
|
||||
pipeline.ingest(source, "bahn")
|
||||
|
||||
# Artefakt-Datei existiert in der Scope-Struktur
|
||||
expected_path = store_path / "bahn" / "decision" / "api-design.md"
|
||||
assert expected_path.exists()
|
||||
content = expected_path.read_text(encoding="utf-8")
|
||||
assert "---" in content
|
||||
assert "API-Designprinzipien" in content
|
||||
|
||||
def test_ingest_nonexistent_source_directory(self, tmp_path: Path) -> None:
|
||||
"""ingest() bei nicht-existierendem Quellverzeichnis gibt Fehler zurück."""
|
||||
store_path = tmp_path / "store"
|
||||
store_path.mkdir()
|
||||
index = YAMLIndex(store_path / "_index.yaml")
|
||||
index.load()
|
||||
|
||||
pipeline = ETLPipeline(index=index, store_path=store_path)
|
||||
source = MarkdownSource(directory=tmp_path / "nonexistent")
|
||||
|
||||
result = pipeline.ingest(source, "bahn")
|
||||
|
||||
assert result.processed == 0
|
||||
assert len(result.errors) >= 1
|
||||
|
||||
def test_ingest_empty_source(self, tmp_path: Path) -> None:
|
||||
"""ingest() bei leerem Quellverzeichnis verarbeitet null Artefakte."""
|
||||
store_path = tmp_path / "store"
|
||||
store_path.mkdir()
|
||||
empty_dir = tmp_path / "empty"
|
||||
empty_dir.mkdir()
|
||||
index = YAMLIndex(store_path / "_index.yaml")
|
||||
index.load()
|
||||
|
||||
pipeline = ETLPipeline(index=index, store_path=store_path)
|
||||
source = MarkdownSource(directory=empty_dir)
|
||||
|
||||
result = pipeline.ingest(source, "bahn")
|
||||
|
||||
assert result.processed == 0
|
||||
assert result.updated == 0
|
||||
assert result.skipped == 0
|
||||
assert result.errors == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# KnowledgeStore.ingest() Integration Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestKnowledgeStoreIngest:
|
||||
"""Tests für KnowledgeStore.ingest() Integration mit ETLPipeline."""
|
||||
|
||||
def test_ingest_via_store(
|
||||
self, tmp_path: Path, scope_config: ScopeConfig, source_dir_with_artifacts: Path
|
||||
) -> None:
|
||||
"""KnowledgeStore.ingest() delegiert an ETLPipeline."""
|
||||
store = KnowledgeStore(tmp_path, scope_config)
|
||||
source = MarkdownSource(directory=source_dir_with_artifacts)
|
||||
|
||||
result = store.ingest(source, "bahn")
|
||||
|
||||
assert result.processed == 2
|
||||
assert result.updated == 2
|
||||
# Artefakte sind im Index
|
||||
assert len(store.index.entries) == 2
|
||||
|
||||
def test_ingest_via_store_incremental(
|
||||
self, tmp_path: Path, scope_config: ScopeConfig, source_dir_with_artifacts: Path
|
||||
) -> None:
|
||||
"""KnowledgeStore.ingest() unterstützt inkrementelle Verarbeitung."""
|
||||
store = KnowledgeStore(tmp_path, scope_config)
|
||||
source1 = MarkdownSource(directory=source_dir_with_artifacts)
|
||||
store.ingest(source1, "bahn")
|
||||
|
||||
# Zweiter Durchlauf: keine Änderungen
|
||||
source2 = MarkdownSource(directory=source_dir_with_artifacts)
|
||||
result2 = store.ingest(source2, "bahn")
|
||||
assert result2.skipped == 2
|
||||
assert result2.updated == 0
|
||||
|
||||
def test_ingest_via_store_search_after_ingest(
|
||||
self, tmp_path: Path, scope_config: ScopeConfig, source_dir_with_artifacts: Path
|
||||
) -> None:
|
||||
"""Nach ingest() sind Artefakte über search() auffindbar."""
|
||||
store = KnowledgeStore(tmp_path, scope_config)
|
||||
source = MarkdownSource(directory=source_dir_with_artifacts)
|
||||
store.ingest(source, "bahn")
|
||||
|
||||
results = store.search("API", ["bahn"])
|
||||
assert len(results) >= 1
|
||||
assert any(r.entry.title == "API-Designprinzipien" for r in results)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# IngestResult Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestIngestResult:
|
||||
"""Tests für IngestResult Dataclass."""
|
||||
|
||||
def test_success_with_processed(self) -> None:
|
||||
"""success ist True wenn Artefakte verarbeitet wurden."""
|
||||
result = IngestResult(processed=3, updated=2, skipped=1)
|
||||
assert result.success is True
|
||||
|
||||
def test_success_empty_no_errors(self) -> None:
|
||||
"""success ist True bei leerem Ergebnis ohne Fehler."""
|
||||
result = IngestResult()
|
||||
assert result.success is True
|
||||
|
||||
def test_defaults(self) -> None:
|
||||
"""Standardwerte sind korrekt."""
|
||||
result = IngestResult()
|
||||
assert result.processed == 0
|
||||
assert result.updated == 0
|
||||
assert result.skipped == 0
|
||||
assert result.errors == []
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,574 @@
|
||||
"""Unit-Tests für Team-Isolation und Cross-Context-Leakage-Prüfung.
|
||||
|
||||
Testet die Implementierung von Task 15.2:
|
||||
- verify_isolation: Erkennung von Pfad-Referenzen, Env-Variablen,
|
||||
Config-Referenzen und Kommentaren zu anderen Kontexten
|
||||
- prepare_team_repo: Erstellung eines Team-Repos mit nur eigenem Kontext
|
||||
- Integration mit SecretEncryptionManager für kontextspezifische Secret-Filterung
|
||||
|
||||
Validates: Requirements 10.5, 10.9, 10.10
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from monorepo.federation import FederationManager
|
||||
from monorepo.models import IsolationLeak, IsolationReport, MachineContext
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def team_repos_config(tmp_path: Path) -> Path:
|
||||
"""Erstellt eine temporäre team-repos.yaml mit Testdaten."""
|
||||
config_path = tmp_path / "config" / "team-repos.yaml"
|
||||
config_path.parent.mkdir(parents=True)
|
||||
data = {
|
||||
"version": "1.0",
|
||||
"federation": {
|
||||
"topology": "hub-and-spoke",
|
||||
"hub_owner": "andre",
|
||||
"conflict_strategy": "team-wins",
|
||||
},
|
||||
"team_repos": [
|
||||
{
|
||||
"context": "privat",
|
||||
"url": "https://github.com/test/privat-team.git",
|
||||
"branch": "main",
|
||||
"sync_direction": "bidirectional",
|
||||
"sync_frequency": "on-push",
|
||||
},
|
||||
{
|
||||
"context": "dhive",
|
||||
"url": "https://gitlab.dhive.io/team/dhive-mono.git",
|
||||
"branch": "main",
|
||||
"sync_direction": "bidirectional",
|
||||
"sync_frequency": "on-push",
|
||||
},
|
||||
{
|
||||
"context": "bahn",
|
||||
"url": "https://gitlab.example.com/bahn-workspace.git",
|
||||
"branch": "main",
|
||||
"sync_direction": "bidirectional",
|
||||
"sync_frequency": "daily",
|
||||
},
|
||||
],
|
||||
}
|
||||
with open(config_path, "w", encoding="utf-8") as f:
|
||||
yaml.dump(data, f)
|
||||
return config_path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def monorepo_root(tmp_path: Path) -> Path:
|
||||
"""Erstellt ein temporäres Monorepo-Verzeichnis mit Kontextordnern."""
|
||||
root = tmp_path / "monorepo"
|
||||
root.mkdir()
|
||||
(root / "privat").mkdir()
|
||||
(root / "dhive").mkdir()
|
||||
(root / "bahn").mkdir()
|
||||
(root / "shared").mkdir()
|
||||
return root
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def federation_manager(team_repos_config: Path, monorepo_root: Path) -> FederationManager:
|
||||
"""Erstellt einen FederationManager mit Testkonfiguration."""
|
||||
return FederationManager(
|
||||
config_path=team_repos_config,
|
||||
monorepo_root=monorepo_root,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_encryption_manager() -> MagicMock:
|
||||
"""Erstellt einen gemockten SecretEncryptionManager."""
|
||||
mock = MagicMock()
|
||||
mock.is_authorized.return_value = True
|
||||
return mock
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def federation_manager_with_encryption(
|
||||
team_repos_config: Path, monorepo_root: Path, mock_encryption_manager: MagicMock
|
||||
) -> FederationManager:
|
||||
"""Erstellt einen FederationManager mit SecretEncryptionManager."""
|
||||
return FederationManager(
|
||||
config_path=team_repos_config,
|
||||
monorepo_root=monorepo_root,
|
||||
encryption_manager=mock_encryption_manager,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: verify_isolation - Pfad-Referenzen
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestVerifyIsolationPathReferences:
|
||||
"""Tests für die Erkennung von Pfad-Referenzen auf andere Kontexte."""
|
||||
|
||||
def test_clean_repo_is_isolated(
|
||||
self, federation_manager: FederationManager, tmp_path: Path
|
||||
) -> None:
|
||||
"""Prüft dass ein sauberes Repo als isoliert erkannt wird."""
|
||||
team_repo = tmp_path / "team-repo"
|
||||
team_repo.mkdir()
|
||||
(team_repo / "src").mkdir()
|
||||
(team_repo / "src" / "main.py").write_text(
|
||||
"# My privat project\nprint('hello')\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
report = federation_manager.verify_isolation(team_repo, "privat")
|
||||
assert report.is_isolated is True
|
||||
assert report.leaks == []
|
||||
assert report.context == "privat"
|
||||
|
||||
def test_detects_path_reference_to_dhive(
|
||||
self, federation_manager: FederationManager, tmp_path: Path
|
||||
) -> None:
|
||||
"""Prüft Erkennung einer Pfad-Referenz auf dhive/."""
|
||||
team_repo = tmp_path / "team-repo"
|
||||
team_repo.mkdir()
|
||||
(team_repo / "config.yaml").write_text(
|
||||
"path: dhive/project-x/src\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
report = federation_manager.verify_isolation(team_repo, "privat")
|
||||
assert report.is_isolated is False
|
||||
assert len(report.leaks) >= 1
|
||||
leak = report.leaks[0]
|
||||
assert leak.leaked_context == "dhive"
|
||||
assert leak.leak_type == "path"
|
||||
assert leak.line_number == 1
|
||||
|
||||
def test_detects_path_reference_to_bahn(
|
||||
self, federation_manager: FederationManager, tmp_path: Path
|
||||
) -> None:
|
||||
"""Prüft Erkennung einer Pfad-Referenz auf bahn/."""
|
||||
team_repo = tmp_path / "team-repo"
|
||||
team_repo.mkdir()
|
||||
(team_repo / "script.sh").write_text(
|
||||
"#!/bin/bash\ncp bahn/data.csv .\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
report = federation_manager.verify_isolation(team_repo, "privat")
|
||||
assert report.is_isolated is False
|
||||
path_leaks = [l for l in report.leaks if l.leak_type == "path"]
|
||||
assert len(path_leaks) >= 1
|
||||
assert path_leaks[0].leaked_context == "bahn"
|
||||
|
||||
def test_own_context_path_not_detected(
|
||||
self, federation_manager: FederationManager, tmp_path: Path
|
||||
) -> None:
|
||||
"""Prüft dass Pfade des eigenen Kontexts NICHT als Leak erkannt werden."""
|
||||
team_repo = tmp_path / "team-repo"
|
||||
team_repo.mkdir()
|
||||
(team_repo / "readme.md").write_text(
|
||||
"## Project\nSee privat/notes for details.\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
report = federation_manager.verify_isolation(team_repo, "privat")
|
||||
# privat/ should NOT be a leak when context is "privat"
|
||||
assert report.is_isolated is True
|
||||
|
||||
def test_nonexistent_path_returns_isolated(
|
||||
self, federation_manager: FederationManager, tmp_path: Path
|
||||
) -> None:
|
||||
"""Prüft dass nicht-existierende Pfade als isoliert behandelt werden."""
|
||||
report = federation_manager.verify_isolation(
|
||||
tmp_path / "nonexistent", "privat"
|
||||
)
|
||||
assert report.is_isolated is True
|
||||
assert report.leaks == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: verify_isolation - Env-Variablen
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestVerifyIsolationEnvVars:
|
||||
"""Tests für die Erkennung von Env-Variablen-Referenzen anderer Kontexte."""
|
||||
|
||||
def test_detects_dhive_env_var(
|
||||
self, federation_manager: FederationManager, tmp_path: Path
|
||||
) -> None:
|
||||
"""Prüft Erkennung von DHIVE_* Umgebungsvariablen."""
|
||||
team_repo = tmp_path / "team-repo"
|
||||
team_repo.mkdir()
|
||||
(team_repo / "app.py").write_text(
|
||||
'import os\napi_key = os.environ["DHIVE_API_KEY"]\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
report = federation_manager.verify_isolation(team_repo, "privat")
|
||||
assert report.is_isolated is False
|
||||
env_leaks = [l for l in report.leaks if l.leak_type == "env_var"]
|
||||
assert len(env_leaks) >= 1
|
||||
assert env_leaks[0].leaked_context == "dhive"
|
||||
|
||||
def test_detects_bahn_env_var(
|
||||
self, federation_manager: FederationManager, tmp_path: Path
|
||||
) -> None:
|
||||
"""Prüft Erkennung von BAHN_* Umgebungsvariablen."""
|
||||
team_repo = tmp_path / "team-repo"
|
||||
team_repo.mkdir()
|
||||
(team_repo / ".env.example").write_text(
|
||||
"BAHN_TOKEN=your-token-here\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
report = federation_manager.verify_isolation(team_repo, "privat")
|
||||
assert report.is_isolated is False
|
||||
env_leaks = [l for l in report.leaks if l.leak_type == "env_var"]
|
||||
assert len(env_leaks) >= 1
|
||||
assert env_leaks[0].leaked_context == "bahn"
|
||||
|
||||
def test_own_context_env_var_not_detected(
|
||||
self, federation_manager: FederationManager, tmp_path: Path
|
||||
) -> None:
|
||||
"""Prüft dass PRIVAT_* Env-Vars im privat-Kontext NICHT als Leak gelten."""
|
||||
team_repo = tmp_path / "team-repo"
|
||||
team_repo.mkdir()
|
||||
(team_repo / "config.py").write_text(
|
||||
'PRIVAT_SECRET = "xxx"\n', encoding="utf-8"
|
||||
)
|
||||
|
||||
report = federation_manager.verify_isolation(team_repo, "privat")
|
||||
# PRIVAT_ env vars should not be leaks when context is "privat"
|
||||
assert report.is_isolated is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: verify_isolation - Config-Referenzen
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestVerifyIsolationConfigRefs:
|
||||
"""Tests für die Erkennung von Config-Referenzen auf andere Kontexte."""
|
||||
|
||||
def test_detects_git_crypt_filter_reference(
|
||||
self, federation_manager: FederationManager, tmp_path: Path
|
||||
) -> None:
|
||||
"""Prüft Erkennung von git-crypt-Filter-Referenzen anderer Kontexte."""
|
||||
team_repo = tmp_path / "team-repo"
|
||||
team_repo.mkdir()
|
||||
(team_repo / ".gitattributes").write_text(
|
||||
".env filter=git-crypt-dhive diff=git-crypt-dhive\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
report = federation_manager.verify_isolation(team_repo, "privat")
|
||||
assert report.is_isolated is False
|
||||
config_leaks = [l for l in report.leaks if l.leak_type == "config_ref"]
|
||||
assert len(config_leaks) >= 1
|
||||
assert config_leaks[0].leaked_context == "dhive"
|
||||
|
||||
def test_detects_context_yaml_reference(
|
||||
self, federation_manager: FederationManager, tmp_path: Path
|
||||
) -> None:
|
||||
"""Prüft Erkennung von context: <other> in YAML-Dateien."""
|
||||
team_repo = tmp_path / "team-repo"
|
||||
team_repo.mkdir()
|
||||
(team_repo / "settings.yaml").write_text(
|
||||
"name: project\ncontext: bahn\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
report = federation_manager.verify_isolation(team_repo, "privat")
|
||||
assert report.is_isolated is False
|
||||
config_leaks = [l for l in report.leaks if l.leak_type == "config_ref"]
|
||||
assert len(config_leaks) >= 1
|
||||
assert config_leaks[0].leaked_context == "bahn"
|
||||
|
||||
def test_own_git_crypt_filter_not_detected(
|
||||
self, federation_manager: FederationManager, tmp_path: Path
|
||||
) -> None:
|
||||
"""Prüft dass der eigene git-crypt-Filter NICHT als Leak erkannt wird."""
|
||||
team_repo = tmp_path / "team-repo"
|
||||
team_repo.mkdir()
|
||||
(team_repo / ".gitattributes").write_text(
|
||||
".env filter=git-crypt-privat diff=git-crypt-privat\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
report = federation_manager.verify_isolation(team_repo, "privat")
|
||||
assert report.is_isolated is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: verify_isolation - Kommentare
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestVerifyIsolationComments:
|
||||
"""Tests für die Erkennung von Kommentaren zu anderen Kontexten."""
|
||||
|
||||
def test_detects_comment_mentioning_other_context(
|
||||
self, federation_manager: FederationManager, tmp_path: Path
|
||||
) -> None:
|
||||
"""Prüft Erkennung von Kommentaren die andere Kontexte erwähnen."""
|
||||
team_repo = tmp_path / "team-repo"
|
||||
team_repo.mkdir()
|
||||
(team_repo / "main.py").write_text(
|
||||
"# Dieses Skript wurde vom dhive-Team erstellt\n"
|
||||
"print('hello')\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
report = federation_manager.verify_isolation(team_repo, "privat")
|
||||
assert report.is_isolated is False
|
||||
comment_leaks = [l for l in report.leaks if l.leak_type == "comment"]
|
||||
assert len(comment_leaks) >= 1
|
||||
assert comment_leaks[0].leaked_context == "dhive"
|
||||
|
||||
def test_non_comment_context_word_not_detected_as_comment(
|
||||
self, federation_manager: FederationManager, tmp_path: Path
|
||||
) -> None:
|
||||
"""Prüft dass Kontextnamen in nicht-Kommentar-Zeilen nicht als comment-Leak gemeldet werden."""
|
||||
team_repo = tmp_path / "team-repo"
|
||||
team_repo.mkdir()
|
||||
(team_repo / "data.txt").write_text(
|
||||
"Some text that mentions dhive in a regular line\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
report = federation_manager.verify_isolation(team_repo, "privat")
|
||||
# Should not have comment-type leaks (may have other types)
|
||||
comment_leaks = [l for l in report.leaks if l.leak_type == "comment"]
|
||||
assert len(comment_leaks) == 0
|
||||
|
||||
def test_js_comment_detected(
|
||||
self, federation_manager: FederationManager, tmp_path: Path
|
||||
) -> None:
|
||||
"""Prüft Erkennung in JS-Style-Kommentaren (//)."""
|
||||
team_repo = tmp_path / "team-repo"
|
||||
team_repo.mkdir()
|
||||
(team_repo / "index.js").write_text(
|
||||
"// TODO: Check bahn integration\nconst x = 1;\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
report = federation_manager.verify_isolation(team_repo, "privat")
|
||||
comment_leaks = [l for l in report.leaks if l.leak_type == "comment"]
|
||||
assert len(comment_leaks) >= 1
|
||||
assert comment_leaks[0].leaked_context == "bahn"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: verify_isolation - Mehrfache Leaks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestVerifyIsolationMultipleLeaks:
|
||||
"""Tests für die Erkennung mehrerer Leaks in einem Repo."""
|
||||
|
||||
def test_detects_multiple_leak_types(
|
||||
self, federation_manager: FederationManager, tmp_path: Path
|
||||
) -> None:
|
||||
"""Prüft Erkennung verschiedener Leak-Typen in derselben Datei."""
|
||||
team_repo = tmp_path / "team-repo"
|
||||
team_repo.mkdir()
|
||||
(team_repo / "config.py").write_text(
|
||||
"# Für das bahn-Team\n"
|
||||
'PATH = "dhive/src/module"\n'
|
||||
"TOKEN = BAHN_API_TOKEN\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
report = federation_manager.verify_isolation(team_repo, "privat")
|
||||
assert report.is_isolated is False
|
||||
assert len(report.leaks) >= 3
|
||||
|
||||
leak_types = {l.leak_type for l in report.leaks}
|
||||
assert "comment" in leak_types
|
||||
assert "path" in leak_types
|
||||
assert "env_var" in leak_types
|
||||
|
||||
def test_detects_leaks_across_multiple_files(
|
||||
self, federation_manager: FederationManager, tmp_path: Path
|
||||
) -> None:
|
||||
"""Prüft Erkennung von Leaks über mehrere Dateien."""
|
||||
team_repo = tmp_path / "team-repo"
|
||||
team_repo.mkdir()
|
||||
(team_repo / "a.py").write_text(
|
||||
'x = "dhive/config"\n', encoding="utf-8"
|
||||
)
|
||||
(team_repo / "b.yaml").write_text(
|
||||
"context: bahn\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
report = federation_manager.verify_isolation(team_repo, "privat")
|
||||
assert report.is_isolated is False
|
||||
files_with_leaks = {l.file_path for l in report.leaks}
|
||||
assert len(files_with_leaks) == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: prepare_team_repo
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPrepareTeamRepo:
|
||||
"""Tests für die Team-Repo-Erstellung mit nur eigenem Kontext."""
|
||||
|
||||
def test_prepare_creates_directory(
|
||||
self, federation_manager: FederationManager, monorepo_root: Path
|
||||
) -> None:
|
||||
"""Prüft dass ein Team-Repo-Verzeichnis erstellt wird."""
|
||||
# Erstelle Quelldateien im Kontextordner
|
||||
privat_dir = monorepo_root / "privat"
|
||||
(privat_dir / "project" / "src").mkdir(parents=True)
|
||||
(privat_dir / "project" / "src" / "main.py").write_text(
|
||||
"print('hello')\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
result = federation_manager.prepare_team_repo("privat")
|
||||
assert result.exists()
|
||||
assert result.is_dir()
|
||||
|
||||
def test_prepare_copies_context_files(
|
||||
self, federation_manager: FederationManager, monorepo_root: Path
|
||||
) -> None:
|
||||
"""Prüft dass Dateien des eigenen Kontexts kopiert werden."""
|
||||
privat_dir = monorepo_root / "privat"
|
||||
(privat_dir / "src").mkdir()
|
||||
(privat_dir / "src" / "app.py").write_text(
|
||||
"print('app')\n", encoding="utf-8"
|
||||
)
|
||||
(privat_dir / "README.md").write_text(
|
||||
"# My Project\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
result = federation_manager.prepare_team_repo("privat")
|
||||
assert (result / "src" / "app.py").exists()
|
||||
assert (result / "README.md").exists()
|
||||
|
||||
def test_prepare_invalid_context_raises(
|
||||
self, federation_manager: FederationManager
|
||||
) -> None:
|
||||
"""Prüft ValueError bei ungültigem Kontext."""
|
||||
with pytest.raises(ValueError, match="Ungültiger Kontext"):
|
||||
federation_manager.prepare_team_repo("invalid")
|
||||
|
||||
def test_prepare_unconfigured_context_raises(
|
||||
self, team_repos_config: Path, monorepo_root: Path
|
||||
) -> None:
|
||||
"""Prüft ValueError wenn kein Team-Repo konfiguriert ist."""
|
||||
# "shared" is not in VALID_CONTEXTS so it raises "Ungültiger Kontext"
|
||||
fm = FederationManager(
|
||||
config_path=team_repos_config,
|
||||
monorepo_root=monorepo_root,
|
||||
)
|
||||
with pytest.raises(ValueError, match="Ungültiger Kontext"):
|
||||
fm.prepare_team_repo("shared")
|
||||
|
||||
def test_prepare_sanitizes_cross_context_leaks(
|
||||
self, federation_manager: FederationManager, monorepo_root: Path
|
||||
) -> None:
|
||||
"""Prüft dass Cross-Context-Leaks im Team-Repo bereinigt werden."""
|
||||
privat_dir = monorepo_root / "privat"
|
||||
(privat_dir / "config.yaml").write_text(
|
||||
"path: dhive/some-project\nname: my-project\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
result = federation_manager.prepare_team_repo("privat")
|
||||
# Die Datei sollte bereinigt worden sein
|
||||
config_content = (result / "config.yaml").read_text(encoding="utf-8")
|
||||
assert "dhive/" not in config_content
|
||||
|
||||
def test_prepare_with_encryption_manager(
|
||||
self,
|
||||
federation_manager_with_encryption: FederationManager,
|
||||
monorepo_root: Path,
|
||||
) -> None:
|
||||
"""Prüft Integration mit SecretEncryptionManager."""
|
||||
privat_dir = monorepo_root / "privat"
|
||||
(privat_dir / "src").mkdir()
|
||||
(privat_dir / "src" / "main.py").write_text(
|
||||
"print('hello')\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
result = federation_manager_with_encryption.prepare_team_repo("privat")
|
||||
assert result.exists()
|
||||
# EncryptionManager.is_authorized sollte aufgerufen worden sein
|
||||
federation_manager_with_encryption.encryption_manager.is_authorized.assert_called_with("privat")
|
||||
|
||||
def test_prepare_filters_gitattributes(
|
||||
self,
|
||||
federation_manager_with_encryption: FederationManager,
|
||||
monorepo_root: Path,
|
||||
) -> None:
|
||||
"""Prüft dass .gitattributes nur eigene Filter enthält."""
|
||||
privat_dir = monorepo_root / "privat"
|
||||
(privat_dir / ".gitattributes").write_text(
|
||||
".env filter=git-crypt-privat diff=git-crypt-privat\n"
|
||||
"*.pem filter=git-crypt-dhive diff=git-crypt-dhive\n"
|
||||
"*.key filter=git-crypt-bahn diff=git-crypt-bahn\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
result = federation_manager_with_encryption.prepare_team_repo("privat")
|
||||
gitattributes = (result / ".gitattributes").read_text(encoding="utf-8")
|
||||
assert "git-crypt-privat" in gitattributes
|
||||
assert "git-crypt-dhive" not in gitattributes
|
||||
assert "git-crypt-bahn" not in gitattributes
|
||||
|
||||
def test_prepare_empty_context_folder(
|
||||
self, federation_manager: FederationManager, monorepo_root: Path
|
||||
) -> None:
|
||||
"""Prüft dass ein leerer Kontextordner ein leeres Team-Repo erzeugt."""
|
||||
# privat/ exists but is empty
|
||||
result = federation_manager.prepare_team_repo("privat")
|
||||
assert result.exists()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Integration verify_isolation + prepare_team_repo
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestIsolationIntegration:
|
||||
"""Integrationstests für Isolation-Verifikation mit Team-Repo-Vorbereitung."""
|
||||
|
||||
def test_prepared_repo_passes_isolation_check(
|
||||
self, federation_manager: FederationManager, monorepo_root: Path
|
||||
) -> None:
|
||||
"""Prüft dass ein vorbereitetes Team-Repo die Isolationsprüfung besteht."""
|
||||
privat_dir = monorepo_root / "privat"
|
||||
(privat_dir / "src").mkdir()
|
||||
(privat_dir / "src" / "clean.py").write_text(
|
||||
"# Clean code without cross-context references\n"
|
||||
"x = 42\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
team_repo_path = federation_manager.prepare_team_repo("privat")
|
||||
report = federation_manager.verify_isolation(team_repo_path, "privat")
|
||||
assert report.is_isolated is True
|
||||
|
||||
def test_prepared_repo_with_initial_leaks_is_sanitized(
|
||||
self, federation_manager: FederationManager, monorepo_root: Path
|
||||
) -> None:
|
||||
"""Prüft dass Leaks beim Vorbereiten automatisch bereinigt werden."""
|
||||
privat_dir = monorepo_root / "privat"
|
||||
(privat_dir / "leak.py").write_text(
|
||||
"# Reference to dhive/project\n"
|
||||
'path = "bahn/data"\n'
|
||||
"clean_line = True\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
team_repo_path = federation_manager.prepare_team_repo("privat")
|
||||
|
||||
# Nach Sanitisierung sollte der Report sauber sein
|
||||
report = federation_manager.verify_isolation(team_repo_path, "privat")
|
||||
assert report.is_isolated is True
|
||||
@@ -0,0 +1,390 @@
|
||||
"""Property-Based Tests für Team-Mitglieder-Isolation (Property 27).
|
||||
|
||||
**Validates: Requirements 10.5, 10.9**
|
||||
|
||||
Property 27: Team-Mitglieder können die Existenz anderer Kontexte nicht entdecken
|
||||
- Für jedes valide Member-Onboarding darf das Ergebnis keine Referenzen auf
|
||||
andere Kontexte, das Monorepo, den Hub oder Cross-Context-Daten enthalten.
|
||||
- Die team_repo_url im Ergebnis muss ausschließlich die URL des zugewiesenen
|
||||
Kontexts enthalten.
|
||||
- Die Strings "monorepo" oder "hub" dürfen niemals im Onboarding-Ergebnis erscheinen.
|
||||
- Für ungültige Mitglieder (leerer Name oder leere E-Mail) muss das Onboarding
|
||||
mit Fehlern abgelehnt werden.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
from hypothesis import given, settings, HealthCheck, assume
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from monorepo.federation import FederationManager, MemberInfo, MemberOnboardingResult
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hilfsfunktionen
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
VALID_CONTEXTS = ["privat", "dhive", "bahn"]
|
||||
OTHER_CONTEXT_MARKERS = ["privat", "dhive", "bahn"]
|
||||
MONOREPO_MARKERS = ["monorepo", "hub-and-spoke", "spoke", "federation"]
|
||||
# "hub" als eigenständiges Wort (nicht als Teil von "github" o.ä.)
|
||||
HUB_STANDALONE_MARKERS = ["hub"]
|
||||
|
||||
|
||||
def _make_team_repos_config(config_path: Path, team_repos: list[dict] | None = None) -> Path:
|
||||
"""Erstellt eine team-repos.yaml Konfiguration."""
|
||||
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if team_repos is None:
|
||||
team_repos = [
|
||||
{
|
||||
"context": "privat",
|
||||
"url": "https://github.com/test/privat-team.git",
|
||||
"branch": "main",
|
||||
"sync_direction": "bidirectional",
|
||||
"sync_frequency": "on-push",
|
||||
},
|
||||
{
|
||||
"context": "dhive",
|
||||
"url": "https://gitlab.dhive.io/team/dhive-mono.git",
|
||||
"branch": "main",
|
||||
"sync_direction": "bidirectional",
|
||||
"sync_frequency": "on-push",
|
||||
},
|
||||
{
|
||||
"context": "bahn",
|
||||
"url": "https://gitlab.example.com/bahn-workspace.git",
|
||||
"branch": "main",
|
||||
"sync_direction": "bidirectional",
|
||||
"sync_frequency": "daily",
|
||||
},
|
||||
]
|
||||
data = {
|
||||
"version": "1.0",
|
||||
"federation": {
|
||||
"topology": "hub-and-spoke",
|
||||
"hub_owner": "andre",
|
||||
"conflict_strategy": "team-wins",
|
||||
},
|
||||
"team_repos": team_repos,
|
||||
}
|
||||
with open(config_path, "w", encoding="utf-8") as f:
|
||||
yaml.dump(data, f)
|
||||
return config_path
|
||||
|
||||
|
||||
def _make_federation_manager(tmp_path: Path) -> FederationManager:
|
||||
"""Erstellt einen FederationManager mit Standard-Konfiguration."""
|
||||
config_path = tmp_path / "config" / "team-repos.yaml"
|
||||
_make_team_repos_config(config_path)
|
||||
monorepo_root = tmp_path / "monorepo"
|
||||
monorepo_root.mkdir(parents=True, exist_ok=True)
|
||||
for ctx in VALID_CONTEXTS:
|
||||
(monorepo_root / ctx).mkdir(exist_ok=True)
|
||||
(monorepo_root / "shared").mkdir(exist_ok=True)
|
||||
return FederationManager(
|
||||
config_path=config_path,
|
||||
monorepo_root=monorepo_root,
|
||||
)
|
||||
|
||||
|
||||
def _result_to_strings(result: MemberOnboardingResult) -> list[str]:
|
||||
"""Konvertiert alle String-Felder des Onboarding-Ergebnisses in eine Liste."""
|
||||
strings = [
|
||||
result.context,
|
||||
result.member_name,
|
||||
result.team_repo_url,
|
||||
]
|
||||
strings.extend(result.errors)
|
||||
return strings
|
||||
|
||||
|
||||
def _get_other_contexts(context: str) -> list[str]:
|
||||
"""Liefert alle Kontexte außer dem gegebenen."""
|
||||
return [c for c in VALID_CONTEXTS if c != context]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategien
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Valide Mitglieder-Namen (nicht-leer)
|
||||
member_names = st.text(
|
||||
alphabet=st.characters(whitelist_categories=("L", "N", "Zs", "Pd")),
|
||||
min_size=1,
|
||||
max_size=50,
|
||||
).filter(lambda s: s.strip() != "")
|
||||
|
||||
# Valide E-Mail-Adressen
|
||||
member_emails = st.from_regex(
|
||||
r"[a-z][a-z0-9]{1,10}@[a-z]{2,8}\.[a-z]{2,4}",
|
||||
fullmatch=True,
|
||||
)
|
||||
|
||||
# Rollen
|
||||
member_roles = st.sampled_from(["developer", "reviewer", "admin", "reader"])
|
||||
|
||||
# Kontexte
|
||||
contexts = st.sampled_from(VALID_CONTEXTS)
|
||||
|
||||
# Ungültige (leere) Strings für Fehlertests
|
||||
empty_or_whitespace = st.sampled_from(["", " ", " ", "\t", "\n"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 27: Team-Mitglieder können die Existenz anderer Kontexte nicht entdecken
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMemberIsolationNoOtherContexts:
|
||||
"""Property-Tests: Onboarding-Ergebnis enthält keine Referenzen auf andere Kontexte.
|
||||
|
||||
**Validates: Requirements 10.5, 10.9**
|
||||
"""
|
||||
|
||||
@given(context=contexts, name=member_names, email=member_emails, role=member_roles)
|
||||
@settings(max_examples=100, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_onboarding_result_contains_no_other_context_references(
|
||||
self, tmp_path: Path, context: str, name: str, email: str, role: str
|
||||
) -> None:
|
||||
"""Für jedes valide Onboarding darf das Ergebnis keine Referenzen
|
||||
auf andere Kontexte als den zugewiesenen enthalten.
|
||||
|
||||
**Validates: Requirements 10.5, 10.9**
|
||||
"""
|
||||
fm = _make_federation_manager(tmp_path)
|
||||
member = MemberInfo(name=name, email=email, role=role)
|
||||
|
||||
result = fm.onboard_member(context, member)
|
||||
|
||||
# Nur erfolgreiches Onboarding prüfen
|
||||
assert result.success is True, (
|
||||
f"Onboarding sollte für validen Member gelingen: {result.errors}"
|
||||
)
|
||||
|
||||
other_contexts = _get_other_contexts(context)
|
||||
all_strings = _result_to_strings(result)
|
||||
|
||||
for s in all_strings:
|
||||
for other_ctx in other_contexts:
|
||||
# Prüfe dass der Kontextname nicht als eigenständiges Wort oder
|
||||
# Pfadbestandteil in den Ergebnis-Strings auftritt
|
||||
assert other_ctx not in s.lower(), (
|
||||
f"Onboarding-Ergebnis für Kontext '{context}' enthält "
|
||||
f"Referenz auf anderen Kontext '{other_ctx}' in: '{s}'"
|
||||
)
|
||||
|
||||
|
||||
class TestMemberIsolationOnlyOwnTeamRepoUrl:
|
||||
"""Property-Tests: Onboarding-Ergebnis enthält nur die Team-Repo-URL des eigenen Kontexts.
|
||||
|
||||
**Validates: Requirements 10.5, 10.9**
|
||||
"""
|
||||
|
||||
@given(context=contexts, name=member_names, email=member_emails, role=member_roles)
|
||||
@settings(max_examples=100, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_team_repo_url_matches_configured_context_url(
|
||||
self, tmp_path: Path, context: str, name: str, email: str, role: str
|
||||
) -> None:
|
||||
"""Die team_repo_url im Ergebnis muss der konfigurierten URL des
|
||||
zugewiesenen Kontexts entsprechen.
|
||||
|
||||
**Validates: Requirements 10.5, 10.9**
|
||||
"""
|
||||
fm = _make_federation_manager(tmp_path)
|
||||
member = MemberInfo(name=name, email=email, role=role)
|
||||
|
||||
result = fm.onboard_member(context, member)
|
||||
|
||||
assert result.success is True, (
|
||||
f"Onboarding sollte für validen Member gelingen: {result.errors}"
|
||||
)
|
||||
|
||||
# Die URL muss genau die konfigurierte Team-Repo-URL für den Kontext sein
|
||||
expected_urls = {
|
||||
"privat": "https://github.com/test/privat-team.git",
|
||||
"dhive": "https://gitlab.dhive.io/team/dhive-mono.git",
|
||||
"bahn": "https://gitlab.example.com/bahn-workspace.git",
|
||||
}
|
||||
expected_url = expected_urls[context]
|
||||
assert result.team_repo_url == expected_url, (
|
||||
f"Team-Repo-URL für Kontext '{context}' sollte '{expected_url}' sein, "
|
||||
f"war aber '{result.team_repo_url}'"
|
||||
)
|
||||
|
||||
@given(context=contexts, name=member_names, email=member_emails)
|
||||
@settings(max_examples=100, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_team_repo_url_contains_no_other_context_urls(
|
||||
self, tmp_path: Path, context: str, name: str, email: str
|
||||
) -> None:
|
||||
"""Die team_repo_url darf keine URL eines anderen Kontexts enthalten.
|
||||
|
||||
**Validates: Requirements 10.5, 10.9**
|
||||
"""
|
||||
fm = _make_federation_manager(tmp_path)
|
||||
member = MemberInfo(name=name, email=email)
|
||||
|
||||
result = fm.onboard_member(context, member)
|
||||
|
||||
assert result.success is True, (
|
||||
f"Onboarding sollte für validen Member gelingen: {result.errors}"
|
||||
)
|
||||
|
||||
other_urls = {
|
||||
"privat": "https://github.com/test/privat-team.git",
|
||||
"dhive": "https://gitlab.dhive.io/team/dhive-mono.git",
|
||||
"bahn": "https://gitlab.example.com/bahn-workspace.git",
|
||||
}
|
||||
# Entferne die eigene URL
|
||||
del other_urls[context]
|
||||
|
||||
for other_ctx, other_url in other_urls.items():
|
||||
assert other_url not in result.team_repo_url, (
|
||||
f"Team-Repo-URL für '{context}' enthält URL von '{other_ctx}': "
|
||||
f"'{result.team_repo_url}'"
|
||||
)
|
||||
|
||||
|
||||
class TestMemberIsolationNoMonorepoOrHubReferences:
|
||||
"""Property-Tests: Weder 'monorepo' noch 'hub' dürfen im Onboarding-Ergebnis erscheinen.
|
||||
|
||||
**Validates: Requirements 10.5, 10.9**
|
||||
"""
|
||||
|
||||
@given(context=contexts, name=member_names, email=member_emails, role=member_roles)
|
||||
@settings(max_examples=100, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_no_monorepo_or_hub_in_onboarding_result(
|
||||
self, tmp_path: Path, context: str, name: str, email: str, role: str
|
||||
) -> None:
|
||||
"""Die Strings 'monorepo' und 'hub' dürfen niemals im
|
||||
Onboarding-Ergebnis auftauchen.
|
||||
|
||||
**Validates: Requirements 10.5, 10.9**
|
||||
|
||||
Hinweis: 'hub' wird als eigenständiges Wort geprüft (nicht als
|
||||
Teilstring in z.B. 'github.com').
|
||||
"""
|
||||
fm = _make_federation_manager(tmp_path)
|
||||
member = MemberInfo(name=name, email=email, role=role)
|
||||
|
||||
result = fm.onboard_member(context, member)
|
||||
|
||||
assert result.success is True, (
|
||||
f"Onboarding sollte für validen Member gelingen: {result.errors}"
|
||||
)
|
||||
|
||||
all_strings = _result_to_strings(result)
|
||||
|
||||
for s in all_strings:
|
||||
s_lower = s.lower()
|
||||
# Prüfe Marker die als Substring gesucht werden können
|
||||
for marker in MONOREPO_MARKERS:
|
||||
assert marker not in s_lower, (
|
||||
f"Onboarding-Ergebnis enthält verbotenen Begriff '{marker}' "
|
||||
f"in: '{s}' (Kontext: '{context}', Mitglied: '{name}')"
|
||||
)
|
||||
# Prüfe 'hub' als eigenständiges Wort (nicht Teil von 'github' o.ä.)
|
||||
for marker in HUB_STANDALONE_MARKERS:
|
||||
# Wortgrenze-Match: \bhub\b erkennt 'hub' nur alleinstehend
|
||||
if re.search(rf'\b{re.escape(marker)}\b', s_lower):
|
||||
# Doppelt prüfen: ist es wirklich alleinstehend?
|
||||
# "github" enthält "hub" aber nicht als Wortgrenze
|
||||
assert False, (
|
||||
f"Onboarding-Ergebnis enthält verbotenen eigenständigen "
|
||||
f"Begriff '{marker}' in: '{s}' "
|
||||
f"(Kontext: '{context}', Mitglied: '{name}')"
|
||||
)
|
||||
|
||||
|
||||
class TestMemberIsolationInvalidMemberRejected:
|
||||
"""Property-Tests: Ungültige Mitglieder (leerer Name oder E-Mail) werden abgelehnt.
|
||||
|
||||
**Validates: Requirements 10.5, 10.9**
|
||||
"""
|
||||
|
||||
@given(context=contexts, email=member_emails, role=member_roles)
|
||||
@settings(max_examples=100, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_empty_name_rejected_with_errors(
|
||||
self, tmp_path: Path, context: str, email: str, role: str
|
||||
) -> None:
|
||||
"""Onboarding mit leerem Namen muss fehlschlagen.
|
||||
|
||||
**Validates: Requirements 10.5, 10.9**
|
||||
"""
|
||||
fm = _make_federation_manager(tmp_path)
|
||||
|
||||
for empty_name in ["", " ", " ", "\t"]:
|
||||
member = MemberInfo(name=empty_name, email=email, role=role)
|
||||
result = fm.onboard_member(context, member)
|
||||
|
||||
assert result.success is False, (
|
||||
f"Onboarding sollte mit leerem Namen fehlschlagen "
|
||||
f"(Name='{repr(empty_name)}', Kontext='{context}')"
|
||||
)
|
||||
assert len(result.errors) > 0, (
|
||||
f"Fehlermeldung fehlt bei leerem Namen"
|
||||
)
|
||||
|
||||
@given(context=contexts, name=member_names, role=member_roles)
|
||||
@settings(max_examples=100, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_empty_email_rejected_with_errors(
|
||||
self, tmp_path: Path, context: str, name: str, role: str
|
||||
) -> None:
|
||||
"""Onboarding mit leerer E-Mail muss fehlschlagen.
|
||||
|
||||
**Validates: Requirements 10.5, 10.9**
|
||||
"""
|
||||
fm = _make_federation_manager(tmp_path)
|
||||
|
||||
for empty_email in ["", " ", " ", "\t"]:
|
||||
member = MemberInfo(name=name, email=empty_email, role=role)
|
||||
result = fm.onboard_member(context, member)
|
||||
|
||||
assert result.success is False, (
|
||||
f"Onboarding sollte mit leerer E-Mail fehlschlagen "
|
||||
f"(Email='{repr(empty_email)}', Kontext='{context}')"
|
||||
)
|
||||
assert len(result.errors) > 0, (
|
||||
f"Fehlermeldung fehlt bei leerer E-Mail"
|
||||
)
|
||||
|
||||
@given(context=contexts, name=member_names, email=member_emails)
|
||||
@settings(max_examples=100, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
||||
def test_invalid_member_error_also_isolated(
|
||||
self, tmp_path: Path, context: str, name: str, email: str
|
||||
) -> None:
|
||||
"""Auch Fehlermeldungen bei ungültigem Onboarding dürfen keine
|
||||
Cross-Context-Informationen offenlegen.
|
||||
|
||||
**Validates: Requirements 10.5, 10.9**
|
||||
"""
|
||||
fm = _make_federation_manager(tmp_path)
|
||||
|
||||
# Teste mit leerem Namen
|
||||
member = MemberInfo(name="", email=email)
|
||||
result = fm.onboard_member(context, member)
|
||||
|
||||
assert result.success is False
|
||||
|
||||
other_contexts = _get_other_contexts(context)
|
||||
# Prüfe dass Fehlermeldungen keine anderen Kontexte erwähnen
|
||||
for error in result.errors:
|
||||
for other_ctx in other_contexts:
|
||||
assert other_ctx not in error.lower(), (
|
||||
f"Fehlermeldung bei ungültigem Onboarding enthält "
|
||||
f"Referenz auf '{other_ctx}': '{error}'"
|
||||
)
|
||||
for marker in MONOREPO_MARKERS:
|
||||
assert marker not in error.lower(), (
|
||||
f"Fehlermeldung enthält verbotenen Begriff '{marker}': '{error}'"
|
||||
)
|
||||
for marker in HUB_STANDALONE_MARKERS:
|
||||
if re.search(rf'\b{re.escape(marker)}\b', error.lower()):
|
||||
assert False, (
|
||||
f"Fehlermeldung enthält verbotenen eigenständigen "
|
||||
f"Begriff '{marker}': '{error}'"
|
||||
)
|
||||
@@ -0,0 +1,706 @@
|
||||
"""Property-basierte Tests für bidirektionale Synchronisation.
|
||||
|
||||
**Validates: Requirements 10.3, 10.4, 10.6, 10.7, 10.8**
|
||||
|
||||
Property 26: Bidirektionale Synchronisation bewahrt Git-Historie und löst Konflikte korrekt
|
||||
|
||||
For any bidirectional sync operation:
|
||||
(a) Git history must be preserved in both directions (commits_synced >= 0)
|
||||
(b) When conflicts occur, sync must abort with appropriate error and conflict list
|
||||
(c) Team-Repo is Single Source of Truth — team-wins strategy resolves to team version
|
||||
(d) full_sync must abort if pre-conflicts are detected (no partial sync)
|
||||
(e) Sync direction constraints must be respected (hub-to-spoke blocks pull, etc.)
|
||||
|
||||
Testing approach:
|
||||
- Mock git operations (subprocess calls) as the existing tests do
|
||||
- Generate various sync configurations, contexts, and conflict scenarios
|
||||
- Verify invariants hold across all generated inputs
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import tempfile
|
||||
from dataclasses import field
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import yaml
|
||||
from hypothesis import given, settings, assume, note
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from monorepo.federation import FederationManager, ConflictResult, SubtreeSyncEngine
|
||||
from monorepo.models import ConflictInfo, SyncResult, TeamRepoEntry
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
VALID_CONTEXTS = st.sampled_from(["privat", "dhive", "bahn"])
|
||||
|
||||
SYNC_DIRECTIONS = st.sampled_from(["bidirectional", "hub-to-spoke", "spoke-to-hub"])
|
||||
|
||||
SYNC_FREQUENCIES = st.sampled_from(["on-push", "hourly", "daily", "manual"])
|
||||
|
||||
BRANCHES = st.sampled_from(["main", "develop", "release/1.0", "feature/sync"])
|
||||
|
||||
CONFLICT_STRATEGIES = st.sampled_from(["team-wins", "hub-wins", "manual"])
|
||||
|
||||
# Simulated commit counts from git output
|
||||
COMMIT_COUNTS = st.integers(min_value=0, max_value=50)
|
||||
|
||||
# File paths that could appear in conflicts
|
||||
CONFLICT_FILE_PATHS = st.builds(
|
||||
lambda ctx, name: f"{ctx}/{name}",
|
||||
VALID_CONTEXTS,
|
||||
st.from_regex(r"[a-z][a-z0-9_/\-]{2,20}\.[a-z]{1,4}", fullmatch=True),
|
||||
)
|
||||
|
||||
# Merge conflict messages from git
|
||||
MERGE_CONFLICT_MESSAGES = st.sampled_from([
|
||||
"CONFLICT (content): Merge conflict in {file}\nAutomatic merge failed",
|
||||
"CONFLICT (modify/delete): {file} deleted in HEAD and modified in upstream",
|
||||
"CONFLICT (rename): {file} renamed in both branches",
|
||||
])
|
||||
|
||||
# Network/auth error messages
|
||||
GIT_ERROR_MESSAGES = st.sampled_from([
|
||||
"fatal: unable to access: Could not resolve host",
|
||||
"fatal: Authentication failed",
|
||||
"fatal: Could not read from remote repository.",
|
||||
"error: failed to push some refs",
|
||||
])
|
||||
|
||||
|
||||
@st.composite
|
||||
def team_repo_configs(draw: st.DrawFn) -> dict[str, Any]:
|
||||
"""Generates a valid team-repos.yaml configuration."""
|
||||
contexts_to_include = draw(
|
||||
st.lists(VALID_CONTEXTS, min_size=1, max_size=3, unique=True)
|
||||
)
|
||||
team_repos = []
|
||||
for ctx in contexts_to_include:
|
||||
team_repos.append({
|
||||
"context": ctx,
|
||||
"url": f"https://example.com/{ctx}-team.git",
|
||||
"branch": draw(BRANCHES),
|
||||
"sync_direction": draw(SYNC_DIRECTIONS),
|
||||
"sync_frequency": draw(SYNC_FREQUENCIES),
|
||||
})
|
||||
|
||||
return {
|
||||
"version": "1.0",
|
||||
"federation": {
|
||||
"topology": "hub-and-spoke",
|
||||
"hub_owner": "andre",
|
||||
"conflict_strategy": "team-wins",
|
||||
},
|
||||
"team_repos": team_repos,
|
||||
}
|
||||
|
||||
|
||||
@st.composite
|
||||
def sync_success_output(draw: st.DrawFn) -> str:
|
||||
"""Generates stdout from a successful git subtree pull/push."""
|
||||
commits = draw(COMMIT_COUNTS)
|
||||
if commits == 0:
|
||||
return ""
|
||||
lines = []
|
||||
for i in range(commits):
|
||||
lines.append(f"abc{i:04d}f Commit message {i}")
|
||||
lines.append(f" {commits} files changed, {commits * 3} insertions(+)")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _create_config(tmp: Path, config_data: dict[str, Any]) -> Path:
|
||||
"""Creates a team-repos.yaml config file."""
|
||||
config_path = tmp / "config" / "team-repos.yaml"
|
||||
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(config_path, "w", encoding="utf-8") as f:
|
||||
yaml.dump(config_data, f)
|
||||
return config_path
|
||||
|
||||
|
||||
def _create_federation_manager(
|
||||
tmp: Path, config_data: dict[str, Any]
|
||||
) -> FederationManager:
|
||||
"""Creates a FederationManager with test configuration."""
|
||||
monorepo_root = tmp / "monorepo"
|
||||
monorepo_root.mkdir(exist_ok=True)
|
||||
for ctx in ("privat", "dhive", "bahn", "shared"):
|
||||
(monorepo_root / ctx).mkdir(exist_ok=True)
|
||||
|
||||
config_path = _create_config(tmp, config_data)
|
||||
return FederationManager(
|
||||
config_path=config_path,
|
||||
monorepo_root=monorepo_root,
|
||||
)
|
||||
|
||||
|
||||
def _get_configured_context(config_data: dict[str, Any]) -> str | None:
|
||||
"""Returns the first context from the config, or None."""
|
||||
repos = config_data.get("team_repos", [])
|
||||
if repos:
|
||||
return repos[0]["context"]
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestProperty26SyncCommitsNonNegative:
|
||||
"""For any sync operation that succeeds, commits_synced must be >= 0.
|
||||
|
||||
**Validates: Requirements 10.3, 10.4, 10.8**
|
||||
"""
|
||||
|
||||
@given(
|
||||
config=team_repo_configs(),
|
||||
git_output=sync_success_output(),
|
||||
)
|
||||
@settings(max_examples=200)
|
||||
def test_successful_sync_from_team_has_non_negative_commits(
|
||||
self,
|
||||
config: dict[str, Any],
|
||||
git_output: str,
|
||||
) -> None:
|
||||
"""After a successful sync_from_team, commits_synced >= 0."""
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
tmp = Path(tmp_dir)
|
||||
fm = _create_federation_manager(tmp, config)
|
||||
|
||||
# Find a context that allows pull (not hub-to-spoke)
|
||||
pull_ctx = None
|
||||
for repo in config["team_repos"]:
|
||||
if repo["sync_direction"] != "hub-to-spoke":
|
||||
pull_ctx = repo["context"]
|
||||
break
|
||||
|
||||
if pull_ctx is None:
|
||||
# All are hub-to-spoke, skip this scenario
|
||||
return
|
||||
|
||||
with patch.object(fm.sync_engine, "_run_git") as mock_git:
|
||||
mock_git.return_value = subprocess.CompletedProcess(
|
||||
args=[], returncode=0, stdout=git_output, stderr=""
|
||||
)
|
||||
result = fm.sync_from_team(pull_ctx)
|
||||
|
||||
if result.success:
|
||||
assert result.commits_synced >= 0, (
|
||||
f"commits_synced must be >= 0 on success, got {result.commits_synced}"
|
||||
)
|
||||
|
||||
@given(
|
||||
config=team_repo_configs(),
|
||||
git_output=sync_success_output(),
|
||||
)
|
||||
@settings(max_examples=200)
|
||||
def test_successful_sync_to_team_has_non_negative_commits(
|
||||
self,
|
||||
config: dict[str, Any],
|
||||
git_output: str,
|
||||
) -> None:
|
||||
"""After a successful sync_to_team, commits_synced >= 0."""
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
tmp = Path(tmp_dir)
|
||||
fm = _create_federation_manager(tmp, config)
|
||||
|
||||
# Find a context that allows push (not spoke-to-hub)
|
||||
push_ctx = None
|
||||
for repo in config["team_repos"]:
|
||||
if repo["sync_direction"] != "spoke-to-hub":
|
||||
push_ctx = repo["context"]
|
||||
break
|
||||
|
||||
if push_ctx is None:
|
||||
return
|
||||
|
||||
with patch.object(fm.sync_engine, "_run_git") as mock_git:
|
||||
mock_git.return_value = subprocess.CompletedProcess(
|
||||
args=[], returncode=0, stdout=git_output, stderr=""
|
||||
)
|
||||
result = fm.sync_to_team(push_ctx)
|
||||
|
||||
if result.success:
|
||||
assert result.commits_synced >= 0, (
|
||||
f"commits_synced must be >= 0 on success, got {result.commits_synced}"
|
||||
)
|
||||
|
||||
|
||||
class TestProperty26SyncDirectionConstraints:
|
||||
"""For any sync direction mismatch, the operation must fail with an error.
|
||||
|
||||
**Validates: Requirements 10.3, 10.4**
|
||||
"""
|
||||
|
||||
@given(config=team_repo_configs())
|
||||
@settings(max_examples=200)
|
||||
def test_hub_to_spoke_blocks_pull(
|
||||
self,
|
||||
config: dict[str, Any],
|
||||
) -> None:
|
||||
"""When sync_direction is hub-to-spoke, sync_from_team must fail."""
|
||||
# Find or create a hub-to-spoke context
|
||||
hub_to_spoke_ctx = None
|
||||
for repo in config["team_repos"]:
|
||||
if repo["sync_direction"] == "hub-to-spoke":
|
||||
hub_to_spoke_ctx = repo["context"]
|
||||
break
|
||||
|
||||
if hub_to_spoke_ctx is None:
|
||||
return # No hub-to-spoke config in this example
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
tmp = Path(tmp_dir)
|
||||
fm = _create_federation_manager(tmp, config)
|
||||
|
||||
result = fm.sync_from_team(hub_to_spoke_ctx)
|
||||
|
||||
assert result.success is False, (
|
||||
f"sync_from_team should fail for hub-to-spoke direction, "
|
||||
f"context='{hub_to_spoke_ctx}'"
|
||||
)
|
||||
assert len(result.conflicts) >= 1, (
|
||||
"Failed sync must have at least one conflict entry with error details"
|
||||
)
|
||||
|
||||
@given(config=team_repo_configs())
|
||||
@settings(max_examples=200)
|
||||
def test_spoke_to_hub_blocks_push(
|
||||
self,
|
||||
config: dict[str, Any],
|
||||
) -> None:
|
||||
"""When sync_direction is spoke-to-hub, sync_to_team must fail."""
|
||||
spoke_to_hub_ctx = None
|
||||
for repo in config["team_repos"]:
|
||||
if repo["sync_direction"] == "spoke-to-hub":
|
||||
spoke_to_hub_ctx = repo["context"]
|
||||
break
|
||||
|
||||
if spoke_to_hub_ctx is None:
|
||||
return
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
tmp = Path(tmp_dir)
|
||||
fm = _create_federation_manager(tmp, config)
|
||||
|
||||
result = fm.sync_to_team(spoke_to_hub_ctx)
|
||||
|
||||
assert result.success is False, (
|
||||
f"sync_to_team should fail for spoke-to-hub direction, "
|
||||
f"context='{spoke_to_hub_ctx}'"
|
||||
)
|
||||
assert len(result.conflicts) >= 1, (
|
||||
"Failed sync must have at least one conflict entry with error details"
|
||||
)
|
||||
|
||||
@given(config=team_repo_configs())
|
||||
@settings(max_examples=200)
|
||||
def test_full_sync_requires_bidirectional(
|
||||
self,
|
||||
config: dict[str, Any],
|
||||
) -> None:
|
||||
"""full_sync must fail when sync_direction is not bidirectional."""
|
||||
non_bidir_ctx = None
|
||||
for repo in config["team_repos"]:
|
||||
if repo["sync_direction"] != "bidirectional":
|
||||
non_bidir_ctx = repo["context"]
|
||||
break
|
||||
|
||||
if non_bidir_ctx is None:
|
||||
return
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
tmp = Path(tmp_dir)
|
||||
fm = _create_federation_manager(tmp, config)
|
||||
|
||||
result = fm.full_sync(non_bidir_ctx)
|
||||
|
||||
assert result.success is False, (
|
||||
f"full_sync should fail for non-bidirectional direction, "
|
||||
f"context='{non_bidir_ctx}'"
|
||||
)
|
||||
|
||||
|
||||
class TestProperty26ConflictResolutionTeamWins:
|
||||
"""For any conflict resolution with team-wins, resolved state matches team version.
|
||||
|
||||
**Validates: Requirements 10.6, 10.7**
|
||||
"""
|
||||
|
||||
@given(
|
||||
context=VALID_CONTEXTS,
|
||||
num_files=st.integers(min_value=1, max_value=5),
|
||||
)
|
||||
@settings(max_examples=200)
|
||||
def test_team_wins_uses_theirs_checkout(
|
||||
self,
|
||||
context: str,
|
||||
num_files: int,
|
||||
) -> None:
|
||||
"""With team-wins strategy, git checkout --theirs must be used for each conflict file."""
|
||||
config_data = {
|
||||
"version": "1.0",
|
||||
"federation": {
|
||||
"topology": "hub-and-spoke",
|
||||
"hub_owner": "andre",
|
||||
"conflict_strategy": "team-wins",
|
||||
},
|
||||
"team_repos": [
|
||||
{
|
||||
"context": context,
|
||||
"url": f"https://example.com/{context}-team.git",
|
||||
"branch": "main",
|
||||
"sync_direction": "bidirectional",
|
||||
"sync_frequency": "on-push",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
# Generate conflict file list within the context prefix
|
||||
conflict_files = [f"{context}/file_{i}.py" for i in range(num_files)]
|
||||
diff_output = "\n".join(conflict_files)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
tmp = Path(tmp_dir)
|
||||
fm = _create_federation_manager(tmp, config_data)
|
||||
|
||||
call_count = 0
|
||||
|
||||
def mock_run_git(args: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if args[:3] == ["diff", "--name-only", "--diff-filter=U"]:
|
||||
return subprocess.CompletedProcess(
|
||||
args=args, returncode=0, stdout=diff_output, stderr=""
|
||||
)
|
||||
# checkout --theirs and add commands
|
||||
return subprocess.CompletedProcess(
|
||||
args=args, returncode=0, stdout="", stderr=""
|
||||
)
|
||||
|
||||
with patch.object(fm.sync_engine, "_run_git", side_effect=mock_run_git):
|
||||
result = fm.resolve_conflict(context, strategy="team-wins")
|
||||
|
||||
assert result.success is True, (
|
||||
f"resolve_conflict with team-wins should succeed, errors: {result.errors}"
|
||||
)
|
||||
assert set(result.resolved_files) == set(conflict_files), (
|
||||
f"All conflict files should be resolved. "
|
||||
f"Expected: {conflict_files}, Got: {result.resolved_files}"
|
||||
)
|
||||
|
||||
@given(
|
||||
context=VALID_CONTEXTS,
|
||||
num_files=st.integers(min_value=1, max_value=5),
|
||||
)
|
||||
@settings(max_examples=200)
|
||||
def test_hub_wins_uses_ours_checkout(
|
||||
self,
|
||||
context: str,
|
||||
num_files: int,
|
||||
) -> None:
|
||||
"""With hub-wins strategy, git checkout --ours must be used."""
|
||||
config_data = {
|
||||
"version": "1.0",
|
||||
"federation": {
|
||||
"topology": "hub-and-spoke",
|
||||
"hub_owner": "andre",
|
||||
"conflict_strategy": "team-wins",
|
||||
},
|
||||
"team_repos": [
|
||||
{
|
||||
"context": context,
|
||||
"url": f"https://example.com/{context}-team.git",
|
||||
"branch": "main",
|
||||
"sync_direction": "bidirectional",
|
||||
"sync_frequency": "on-push",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
conflict_files = [f"{context}/src/module_{i}.py" for i in range(num_files)]
|
||||
diff_output = "\n".join(conflict_files)
|
||||
|
||||
checkout_flags_used: list[str] = []
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
tmp = Path(tmp_dir)
|
||||
fm = _create_federation_manager(tmp, config_data)
|
||||
|
||||
def mock_run_git(args: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]:
|
||||
if args[:3] == ["diff", "--name-only", "--diff-filter=U"]:
|
||||
return subprocess.CompletedProcess(
|
||||
args=args, returncode=0, stdout=diff_output, stderr=""
|
||||
)
|
||||
if args[0] == "checkout" and len(args) >= 2:
|
||||
checkout_flags_used.append(args[1])
|
||||
return subprocess.CompletedProcess(
|
||||
args=args, returncode=0, stdout="", stderr=""
|
||||
)
|
||||
|
||||
with patch.object(fm.sync_engine, "_run_git", side_effect=mock_run_git):
|
||||
result = fm.resolve_conflict(context, strategy="hub-wins")
|
||||
|
||||
assert result.success is True
|
||||
# All checkout calls should use --ours for hub-wins
|
||||
ours_calls = [f for f in checkout_flags_used if f == "--ours"]
|
||||
assert len(ours_calls) == num_files, (
|
||||
f"hub-wins should use --ours for all {num_files} files, "
|
||||
f"got {len(ours_calls)} --ours calls"
|
||||
)
|
||||
|
||||
|
||||
class TestProperty26FullSyncAbortsOnPreConflicts:
|
||||
"""full_sync must abort if pre-conflicts are detected (no partial sync).
|
||||
|
||||
**Validates: Requirements 10.7, 10.8**
|
||||
"""
|
||||
|
||||
@given(
|
||||
context=VALID_CONTEXTS,
|
||||
num_conflicts=st.integers(min_value=1, max_value=5),
|
||||
)
|
||||
@settings(max_examples=200)
|
||||
def test_full_sync_aborts_with_pre_conflicts(
|
||||
self,
|
||||
context: str,
|
||||
num_conflicts: int,
|
||||
) -> None:
|
||||
"""full_sync must return failure and conflicts if pre-conflicts exist."""
|
||||
config_data = {
|
||||
"version": "1.0",
|
||||
"federation": {
|
||||
"topology": "hub-and-spoke",
|
||||
"hub_owner": "andre",
|
||||
"conflict_strategy": "team-wins",
|
||||
},
|
||||
"team_repos": [
|
||||
{
|
||||
"context": context,
|
||||
"url": f"https://example.com/{context}-team.git",
|
||||
"branch": "main",
|
||||
"sync_direction": "bidirectional",
|
||||
"sync_frequency": "on-push",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
# Simulate uncommitted changes (pre-conflicts)
|
||||
status_lines = [
|
||||
f" M {context}/file_{i}.py" for i in range(num_conflicts)
|
||||
]
|
||||
status_output = "\n".join(status_lines)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
tmp = Path(tmp_dir)
|
||||
fm = _create_federation_manager(tmp, config_data)
|
||||
|
||||
def mock_run_git(args: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]:
|
||||
if "status" in args:
|
||||
return subprocess.CompletedProcess(
|
||||
args=args, returncode=0, stdout=status_output, stderr=""
|
||||
)
|
||||
# Subtree pull/push should NOT be called
|
||||
raise AssertionError(
|
||||
f"Git subtree command should not be called when pre-conflicts "
|
||||
f"exist, but got: {args}"
|
||||
)
|
||||
|
||||
with patch.object(fm.sync_engine, "_run_git", side_effect=mock_run_git):
|
||||
result = fm.full_sync(context)
|
||||
|
||||
assert result.success is False, (
|
||||
"full_sync must fail when pre-conflicts are detected"
|
||||
)
|
||||
assert result.direction == "full"
|
||||
assert result.commits_synced == 0, (
|
||||
"No commits should be synced when aborting due to pre-conflicts"
|
||||
)
|
||||
assert len(result.conflicts) >= 1, (
|
||||
"Pre-conflicts must be reported in the result"
|
||||
)
|
||||
|
||||
@given(
|
||||
context=VALID_CONTEXTS,
|
||||
pull_output=sync_success_output(),
|
||||
push_output=sync_success_output(),
|
||||
)
|
||||
@settings(max_examples=200)
|
||||
def test_full_sync_succeeds_without_conflicts(
|
||||
self,
|
||||
context: str,
|
||||
pull_output: str,
|
||||
push_output: str,
|
||||
) -> None:
|
||||
"""full_sync succeeds when no pre-conflicts and both pull/push succeed."""
|
||||
config_data = {
|
||||
"version": "1.0",
|
||||
"federation": {
|
||||
"topology": "hub-and-spoke",
|
||||
"hub_owner": "andre",
|
||||
"conflict_strategy": "team-wins",
|
||||
},
|
||||
"team_repos": [
|
||||
{
|
||||
"context": context,
|
||||
"url": f"https://example.com/{context}-team.git",
|
||||
"branch": "main",
|
||||
"sync_direction": "bidirectional",
|
||||
"sync_frequency": "on-push",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
call_index = 0
|
||||
outputs = ["", pull_output, push_output] # status(empty), pull, push
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
tmp = Path(tmp_dir)
|
||||
fm = _create_federation_manager(tmp, config_data)
|
||||
|
||||
def mock_run_git(args: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]:
|
||||
nonlocal call_index
|
||||
idx = min(call_index, len(outputs) - 1)
|
||||
call_index += 1
|
||||
return subprocess.CompletedProcess(
|
||||
args=args, returncode=0, stdout=outputs[idx], stderr=""
|
||||
)
|
||||
|
||||
with patch.object(fm.sync_engine, "_run_git", side_effect=mock_run_git):
|
||||
result = fm.full_sync(context)
|
||||
|
||||
assert result.success is True, (
|
||||
f"full_sync should succeed without conflicts, got errors: {result.conflicts}"
|
||||
)
|
||||
assert result.direction == "full"
|
||||
assert result.commits_synced >= 0
|
||||
|
||||
|
||||
class TestProperty26ResolvedFilesWithinContextPrefix:
|
||||
"""For any resolved conflict, only files within the context prefix are resolved.
|
||||
|
||||
**Validates: Requirements 10.6, 10.7**
|
||||
"""
|
||||
|
||||
@given(
|
||||
context=VALID_CONTEXTS,
|
||||
strategy=st.sampled_from(["team-wins", "hub-wins"]),
|
||||
num_context_files=st.integers(min_value=1, max_value=3),
|
||||
num_other_files=st.integers(min_value=0, max_value=3),
|
||||
)
|
||||
@settings(max_examples=200)
|
||||
def test_only_context_prefixed_files_resolved(
|
||||
self,
|
||||
context: str,
|
||||
strategy: str,
|
||||
num_context_files: int,
|
||||
num_other_files: int,
|
||||
) -> None:
|
||||
"""resolve_conflict only resolves files within the specified context prefix."""
|
||||
config_data = {
|
||||
"version": "1.0",
|
||||
"federation": {
|
||||
"topology": "hub-and-spoke",
|
||||
"hub_owner": "andre",
|
||||
"conflict_strategy": "team-wins",
|
||||
},
|
||||
"team_repos": [
|
||||
{
|
||||
"context": context,
|
||||
"url": f"https://example.com/{context}-team.git",
|
||||
"branch": "main",
|
||||
"sync_direction": "bidirectional",
|
||||
"sync_frequency": "on-push",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
# Files within the context
|
||||
context_files = [f"{context}/src/file_{i}.py" for i in range(num_context_files)]
|
||||
# Files outside the context (other contexts)
|
||||
other_contexts = [c for c in ("privat", "dhive", "bahn") if c != context]
|
||||
other_files = [
|
||||
f"{other_contexts[i % len(other_contexts)]}/file_{i}.py"
|
||||
for i in range(num_other_files)
|
||||
]
|
||||
# Combined output from git diff (shows all conflicted files)
|
||||
all_files = context_files + other_files
|
||||
diff_output = "\n".join(all_files)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
tmp = Path(tmp_dir)
|
||||
fm = _create_federation_manager(tmp, config_data)
|
||||
|
||||
def mock_run_git(args: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]:
|
||||
if args[:3] == ["diff", "--name-only", "--diff-filter=U"]:
|
||||
return subprocess.CompletedProcess(
|
||||
args=args, returncode=0, stdout=diff_output, stderr=""
|
||||
)
|
||||
return subprocess.CompletedProcess(
|
||||
args=args, returncode=0, stdout="", stderr=""
|
||||
)
|
||||
|
||||
with patch.object(fm.sync_engine, "_run_git", side_effect=mock_run_git):
|
||||
result = fm.resolve_conflict(context, strategy=strategy)
|
||||
|
||||
# All resolved files must start with the context prefix
|
||||
for resolved_file in result.resolved_files:
|
||||
assert resolved_file.startswith(f"{context}/"), (
|
||||
f"Resolved file '{resolved_file}' is outside context '{context}/' prefix"
|
||||
)
|
||||
|
||||
# Only the context-prefixed files should be resolved
|
||||
assert set(result.resolved_files) == set(context_files), (
|
||||
f"Only files with prefix '{context}/' should be resolved. "
|
||||
f"Expected: {context_files}, Got: {result.resolved_files}"
|
||||
)
|
||||
|
||||
@given(context=VALID_CONTEXTS)
|
||||
@settings(max_examples=100)
|
||||
def test_invalid_strategy_returns_error(
|
||||
self,
|
||||
context: str,
|
||||
) -> None:
|
||||
"""resolve_conflict with invalid strategy must return error."""
|
||||
config_data = {
|
||||
"version": "1.0",
|
||||
"federation": {
|
||||
"topology": "hub-and-spoke",
|
||||
"hub_owner": "andre",
|
||||
"conflict_strategy": "team-wins",
|
||||
},
|
||||
"team_repos": [
|
||||
{
|
||||
"context": context,
|
||||
"url": f"https://example.com/{context}-team.git",
|
||||
"branch": "main",
|
||||
"sync_direction": "bidirectional",
|
||||
"sync_frequency": "on-push",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
tmp = Path(tmp_dir)
|
||||
fm = _create_federation_manager(tmp, config_data)
|
||||
|
||||
result = fm.resolve_conflict(context, strategy="invalid-strategy")
|
||||
|
||||
assert result.success is False, (
|
||||
"resolve_conflict with invalid strategy must fail"
|
||||
)
|
||||
assert len(result.errors) >= 1, (
|
||||
"Invalid strategy must produce at least one error message"
|
||||
)
|
||||
@@ -0,0 +1,482 @@
|
||||
"""Property-basierte Tests für Team-Repo-Isolation (Property 25).
|
||||
|
||||
**Validates: Requirements 10.5, 10.9, 10.10, 10.12**
|
||||
|
||||
Property 25: Team-Repos enthalten ausschließlich Inhalte des eigenen Kontexts
|
||||
|
||||
For any Team-Repo prepared by the FederationManager, it must contain
|
||||
exclusively content from its own context. No references to other contexts
|
||||
(paths, env vars, config refs, comments) are allowed. Secrets from other
|
||||
contexts must not be present.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import yaml
|
||||
from hypothesis import assume, given, settings
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from monorepo.federation import FederationManager
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
WORK_CONTEXTS = ["privat", "dhive", "bahn"]
|
||||
|
||||
SECRET_FILE_PATTERNS = [".env", "private.pem", "server.key", "api-token.txt"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@st.composite
|
||||
def context_pair(draw: st.DrawFn) -> tuple[str, str]:
|
||||
"""Generates a pair (own_context, other_context) where they differ."""
|
||||
own = draw(st.sampled_from(WORK_CONTEXTS))
|
||||
other = draw(st.sampled_from([c for c in WORK_CONTEXTS if c != own]))
|
||||
return own, other
|
||||
|
||||
|
||||
@st.composite
|
||||
def cross_context_path_content(draw: st.DrawFn, other_context: str) -> str:
|
||||
"""Generates file content containing a path reference to another context.
|
||||
|
||||
Creates lines like:
|
||||
- path: dhive/some-project/src
|
||||
- import from bahn/module
|
||||
- cp privat/data.csv .
|
||||
"""
|
||||
prefix = draw(st.sampled_from([
|
||||
"path: ", "import ", "cp ", 'source = "', "target: ", "from ",
|
||||
]))
|
||||
suffix = draw(st.sampled_from([
|
||||
"/project/src", "/data.csv", "/module", "/config.yaml",
|
||||
"/secrets", "/tools/script.sh",
|
||||
]))
|
||||
# Add some clean lines around the cross-context reference
|
||||
clean_line = draw(st.sampled_from([
|
||||
"x = 42\n", "print('hello')\n", "# clean comment\n", "name: my-app\n",
|
||||
]))
|
||||
return f"{clean_line}{prefix}{other_context}{suffix}\n{clean_line}"
|
||||
|
||||
|
||||
@st.composite
|
||||
def cross_context_env_var_content(draw: st.DrawFn, other_context: str) -> str:
|
||||
"""Generates file content containing env var references to another context.
|
||||
|
||||
Creates lines like:
|
||||
- DHIVE_API_KEY=value
|
||||
- os.environ["BAHN_TOKEN"]
|
||||
"""
|
||||
var_suffix = draw(st.sampled_from([
|
||||
"_API_KEY", "_TOKEN", "_SECRET", "_PASSWORD", "_DATABASE_URL",
|
||||
]))
|
||||
ctx_upper = other_context.upper()
|
||||
style = draw(st.sampled_from(["assignment", "environ_access", "export"]))
|
||||
|
||||
if style == "assignment":
|
||||
return f"x = 1\n{ctx_upper}{var_suffix}=some_value\nprint('done')\n"
|
||||
elif style == "environ_access":
|
||||
return f'import os\nkey = os.environ["{ctx_upper}{var_suffix}"]\n'
|
||||
else:
|
||||
return f"export {ctx_upper}{var_suffix}=secret123\n"
|
||||
|
||||
|
||||
@st.composite
|
||||
def cross_context_config_ref_content(draw: st.DrawFn, other_context: str) -> str:
|
||||
"""Generates file content with config references to another context.
|
||||
|
||||
Creates lines like:
|
||||
- filter=git-crypt-dhive
|
||||
- context: bahn
|
||||
"""
|
||||
style = draw(st.sampled_from(["git_crypt", "context_yaml"]))
|
||||
|
||||
if style == "git_crypt":
|
||||
return f".env filter=git-crypt-{other_context} diff=git-crypt-{other_context}\n"
|
||||
else:
|
||||
return f"name: project\ncontext: {other_context}\nversion: 1.0\n"
|
||||
|
||||
|
||||
@st.composite
|
||||
def cross_context_comment_content(draw: st.DrawFn, other_context: str) -> str:
|
||||
"""Generates file content with comments referencing another context.
|
||||
|
||||
Creates lines like:
|
||||
- # TODO: Check dhive integration
|
||||
- // From bahn team
|
||||
"""
|
||||
comment_style = draw(st.sampled_from(["#", "//", "/*"]))
|
||||
phrase = draw(st.sampled_from([
|
||||
f"From {other_context} team",
|
||||
f"TODO: Check {other_context} integration",
|
||||
f"Originally from {other_context}",
|
||||
f"See {other_context} for reference",
|
||||
]))
|
||||
return f"x = 1\n{comment_style} {phrase}\nprint('done')\n"
|
||||
|
||||
|
||||
@st.composite
|
||||
def file_name_with_extension(draw: st.DrawFn) -> str:
|
||||
"""Generates a valid file name with a text extension."""
|
||||
base = draw(st.sampled_from([
|
||||
"main", "config", "settings", "app", "utils", "script", "readme",
|
||||
]))
|
||||
ext = draw(st.sampled_from([
|
||||
".py", ".yaml", ".yml", ".json", ".sh", ".md", ".txt", ".toml",
|
||||
]))
|
||||
return f"{base}{ext}"
|
||||
|
||||
|
||||
@st.composite
|
||||
def secret_file_for_context(draw: st.DrawFn, context: str) -> tuple[str, bytes]:
|
||||
"""Generates a secret file name and content that belongs to a specific context."""
|
||||
filename = draw(st.sampled_from(SECRET_FILE_PATTERNS))
|
||||
content = draw(st.binary(min_size=10, max_size=100))
|
||||
return filename, content
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures (as helper functions since hypothesis doesn't use pytest fixtures directly)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _create_team_repos_config(tmp_dir: Path) -> Path:
|
||||
"""Creates a team-repos.yaml config file."""
|
||||
config_path = tmp_dir / "config" / "team-repos.yaml"
|
||||
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
data = {
|
||||
"version": "1.0",
|
||||
"federation": {
|
||||
"topology": "hub-and-spoke",
|
||||
"hub_owner": "andre",
|
||||
"conflict_strategy": "team-wins",
|
||||
},
|
||||
"team_repos": [
|
||||
{
|
||||
"context": ctx,
|
||||
"url": f"https://example.com/{ctx}-team.git",
|
||||
"branch": "main",
|
||||
"sync_direction": "bidirectional",
|
||||
"sync_frequency": "on-push",
|
||||
}
|
||||
for ctx in WORK_CONTEXTS
|
||||
],
|
||||
}
|
||||
with open(config_path, "w", encoding="utf-8") as f:
|
||||
yaml.dump(data, f)
|
||||
return config_path
|
||||
|
||||
|
||||
def _create_monorepo_root(tmp_dir: Path) -> Path:
|
||||
"""Creates a minimal monorepo root with context folders."""
|
||||
root = tmp_dir / "monorepo"
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
for ctx in WORK_CONTEXTS:
|
||||
(root / ctx).mkdir(exist_ok=True)
|
||||
(root / "shared").mkdir(exist_ok=True)
|
||||
return root
|
||||
|
||||
|
||||
def _create_federation_manager(tmp_dir: Path) -> tuple[FederationManager, Path]:
|
||||
"""Creates a FederationManager instance with temp config and monorepo root."""
|
||||
config_path = _create_team_repos_config(tmp_dir)
|
||||
monorepo_root = _create_monorepo_root(tmp_dir)
|
||||
mock_encryption = MagicMock()
|
||||
mock_encryption.is_authorized.return_value = True
|
||||
fm = FederationManager(
|
||||
config_path=config_path,
|
||||
monorepo_root=monorepo_root,
|
||||
encryption_manager=mock_encryption,
|
||||
)
|
||||
return fm, monorepo_root
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestProperty25TeamRepoContainsOnlyOwnContext:
|
||||
"""Property 25: Team-Repos enthalten ausschließlich Inhalte des eigenen Kontexts.
|
||||
|
||||
**Validates: Requirements 10.5, 10.9, 10.10, 10.12**
|
||||
|
||||
For any Team-Repo prepared by the FederationManager, regardless of
|
||||
what cross-context references existed in the source files, the prepared
|
||||
team repo must pass verify_isolation (no leaks detected).
|
||||
"""
|
||||
|
||||
@given(data=st.data())
|
||||
@settings(max_examples=100, deadline=10000)
|
||||
def test_prepared_repo_passes_isolation_with_path_references(
|
||||
self, data: st.DataObject, tmp_path_factory
|
||||
) -> None:
|
||||
"""For any context and files containing cross-context path references,
|
||||
prepare_team_repo must produce a repo that passes verify_isolation.
|
||||
|
||||
Even when source files contain paths like 'dhive/project' or 'bahn/data',
|
||||
the prepared team repo must be clean.
|
||||
"""
|
||||
tmp_dir = tmp_path_factory.mktemp("prop25_path")
|
||||
fm, monorepo_root = _create_federation_manager(tmp_dir)
|
||||
|
||||
own_ctx, other_ctx = data.draw(context_pair())
|
||||
content = data.draw(cross_context_path_content(other_ctx))
|
||||
filename = data.draw(file_name_with_extension())
|
||||
|
||||
# Place a file with cross-context path references in the context folder
|
||||
ctx_dir = monorepo_root / own_ctx
|
||||
(ctx_dir / filename).write_text(content, encoding="utf-8")
|
||||
|
||||
# Prepare team repo - this should sanitize cross-context references
|
||||
team_repo_path = fm.prepare_team_repo(own_ctx)
|
||||
|
||||
# Verify isolation passes
|
||||
report = fm.verify_isolation(team_repo_path, own_ctx)
|
||||
assert report.is_isolated, (
|
||||
f"Team-Repo for '{own_ctx}' has leaks after prepare_team_repo! "
|
||||
f"Source had path refs to '{other_ctx}'. "
|
||||
f"Leaks found: {[(l.file_path, l.leak_type, l.leaked_context) for l in report.leaks]}"
|
||||
)
|
||||
|
||||
@given(data=st.data())
|
||||
@settings(max_examples=100, deadline=10000)
|
||||
def test_prepared_repo_passes_isolation_with_env_var_references(
|
||||
self, data: st.DataObject, tmp_path_factory
|
||||
) -> None:
|
||||
"""For any context and files containing cross-context env var references,
|
||||
prepare_team_repo must produce a repo that passes verify_isolation.
|
||||
|
||||
Even when source files reference DHIVE_API_KEY or BAHN_TOKEN,
|
||||
the prepared team repo must be clean.
|
||||
"""
|
||||
tmp_dir = tmp_path_factory.mktemp("prop25_env")
|
||||
fm, monorepo_root = _create_federation_manager(tmp_dir)
|
||||
|
||||
own_ctx, other_ctx = data.draw(context_pair())
|
||||
content = data.draw(cross_context_env_var_content(other_ctx))
|
||||
filename = data.draw(file_name_with_extension())
|
||||
|
||||
ctx_dir = monorepo_root / own_ctx
|
||||
(ctx_dir / filename).write_text(content, encoding="utf-8")
|
||||
|
||||
team_repo_path = fm.prepare_team_repo(own_ctx)
|
||||
|
||||
report = fm.verify_isolation(team_repo_path, own_ctx)
|
||||
assert report.is_isolated, (
|
||||
f"Team-Repo for '{own_ctx}' has leaks after prepare_team_repo! "
|
||||
f"Source had env var refs to '{other_ctx}'. "
|
||||
f"Leaks found: {[(l.file_path, l.leak_type, l.leaked_context) for l in report.leaks]}"
|
||||
)
|
||||
|
||||
@given(data=st.data())
|
||||
@settings(max_examples=100, deadline=10000)
|
||||
def test_prepared_repo_has_no_gitattributes_for_other_contexts(
|
||||
self, data: st.DataObject, tmp_path_factory
|
||||
) -> None:
|
||||
"""For any prepared team-repo, .gitattributes must not contain
|
||||
git-crypt filters for other contexts.
|
||||
|
||||
If source has filters like git-crypt-dhive or git-crypt-bahn,
|
||||
prepare_team_repo must filter them out.
|
||||
"""
|
||||
tmp_dir = tmp_path_factory.mktemp("prop25_gitattr")
|
||||
fm, monorepo_root = _create_federation_manager(tmp_dir)
|
||||
|
||||
own_ctx = data.draw(st.sampled_from(WORK_CONTEXTS))
|
||||
other_contexts = [c for c in WORK_CONTEXTS if c != own_ctx]
|
||||
|
||||
# Create .gitattributes with filters for ALL contexts (including own + others)
|
||||
lines = [f".env filter=git-crypt-{own_ctx} diff=git-crypt-{own_ctx}"]
|
||||
for other_ctx in other_contexts:
|
||||
ext = data.draw(st.sampled_from([".env", "*.pem", "*.key"]))
|
||||
lines.append(f"{ext} filter=git-crypt-{other_ctx} diff=git-crypt-{other_ctx}")
|
||||
|
||||
ctx_dir = monorepo_root / own_ctx
|
||||
(ctx_dir / ".gitattributes").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
# Also add a regular source file so the repo isn't empty
|
||||
(ctx_dir / "main.py").write_text("x = 1\n", encoding="utf-8")
|
||||
|
||||
team_repo_path = fm.prepare_team_repo(own_ctx)
|
||||
|
||||
# Check .gitattributes in the prepared repo
|
||||
gitattr_path = team_repo_path / ".gitattributes"
|
||||
if gitattr_path.exists():
|
||||
content = gitattr_path.read_text(encoding="utf-8")
|
||||
for other_ctx in other_contexts:
|
||||
assert f"git-crypt-{other_ctx}" not in content, (
|
||||
f".gitattributes in team-repo for '{own_ctx}' still contains "
|
||||
f"git-crypt filter for '{other_ctx}'. Content:\n{content}"
|
||||
)
|
||||
|
||||
@given(data=st.data())
|
||||
@settings(max_examples=100, deadline=10000)
|
||||
def test_prepared_repo_secrets_belong_only_to_own_context(
|
||||
self, data: st.DataObject, tmp_path_factory
|
||||
) -> None:
|
||||
"""For any prepared team-repo, secret files (.env, *.pem, *.key) must
|
||||
only belong to the own context. No secrets from other contexts may be present.
|
||||
|
||||
This verifies that even if the source directory somehow contains files
|
||||
referencing other contexts' secrets, the prepared repo is clean.
|
||||
"""
|
||||
tmp_dir = tmp_path_factory.mktemp("prop25_secrets")
|
||||
fm, monorepo_root = _create_federation_manager(tmp_dir)
|
||||
|
||||
own_ctx = data.draw(st.sampled_from(WORK_CONTEXTS))
|
||||
|
||||
# Create some secret files in the own context
|
||||
ctx_dir = monorepo_root / own_ctx
|
||||
secret_name = data.draw(st.sampled_from(SECRET_FILE_PATTERNS))
|
||||
secret_content = f"{own_ctx.upper()}_SECRET=my-own-secret\n"
|
||||
(ctx_dir / secret_name).write_text(secret_content, encoding="utf-8")
|
||||
|
||||
# Also place a normal source file
|
||||
(ctx_dir / "app.py").write_text("print('app')\n", encoding="utf-8")
|
||||
|
||||
team_repo_path = fm.prepare_team_repo(own_ctx)
|
||||
|
||||
# Verify no file paths in the team repo reference other contexts
|
||||
report = fm.verify_isolation(team_repo_path, own_ctx)
|
||||
assert report.is_isolated, (
|
||||
f"Team-Repo for '{own_ctx}' has cross-context leaks even for secrets. "
|
||||
f"Leaks: {[(l.file_path, l.leak_type, l.leaked_context) for l in report.leaks]}"
|
||||
)
|
||||
|
||||
@given(data=st.data())
|
||||
@settings(max_examples=100, deadline=10000)
|
||||
def test_no_file_path_in_prepared_repo_references_other_context_folders(
|
||||
self, data: st.DataObject, tmp_path_factory
|
||||
) -> None:
|
||||
"""For any prepared team-repo, no file path within the repo should
|
||||
contain references to other context folders.
|
||||
|
||||
The directory structure itself must not expose other context names
|
||||
as folder paths.
|
||||
"""
|
||||
tmp_dir = tmp_path_factory.mktemp("prop25_dirpaths")
|
||||
fm, monorepo_root = _create_federation_manager(tmp_dir)
|
||||
|
||||
own_ctx = data.draw(st.sampled_from(WORK_CONTEXTS))
|
||||
other_contexts = [c for c in WORK_CONTEXTS if c != own_ctx]
|
||||
|
||||
# Create a directory structure with clean names
|
||||
ctx_dir = monorepo_root / own_ctx
|
||||
subdir = data.draw(st.sampled_from(["src", "lib", "docs", "tests", "config"]))
|
||||
(ctx_dir / subdir).mkdir(parents=True, exist_ok=True)
|
||||
(ctx_dir / subdir / "module.py").write_text("pass\n", encoding="utf-8")
|
||||
|
||||
team_repo_path = fm.prepare_team_repo(own_ctx)
|
||||
|
||||
# Collect all file/directory paths in the prepared repo
|
||||
all_paths = [
|
||||
str(p.relative_to(team_repo_path))
|
||||
for p in team_repo_path.rglob("*")
|
||||
]
|
||||
|
||||
for path_str in all_paths:
|
||||
for other_ctx in other_contexts:
|
||||
# Path segments should not be named after other contexts
|
||||
path_parts = Path(path_str).parts
|
||||
assert other_ctx not in path_parts, (
|
||||
f"Directory structure in team-repo for '{own_ctx}' contains "
|
||||
f"path segment '{other_ctx}': {path_str}"
|
||||
)
|
||||
|
||||
@given(data=st.data())
|
||||
@settings(max_examples=80, deadline=10000)
|
||||
def test_prepared_repo_with_multiple_cross_context_leak_types(
|
||||
self, data: st.DataObject, tmp_path_factory
|
||||
) -> None:
|
||||
"""For any context and files containing MULTIPLE types of cross-context
|
||||
references (paths, env vars, config refs, comments simultaneously),
|
||||
prepare_team_repo must still produce a fully isolated repo.
|
||||
"""
|
||||
tmp_dir = tmp_path_factory.mktemp("prop25_multi")
|
||||
fm, monorepo_root = _create_federation_manager(tmp_dir)
|
||||
|
||||
own_ctx, other_ctx = data.draw(context_pair())
|
||||
ctx_upper = other_ctx.upper()
|
||||
|
||||
# Create a file that has ALL types of leaks
|
||||
multi_leak_content = (
|
||||
f"# Reference to {other_ctx} team project\n"
|
||||
f'import_path = "{other_ctx}/module/utils"\n'
|
||||
f"{ctx_upper}_API_KEY=secret123\n"
|
||||
f"context: {other_ctx}\n"
|
||||
f"clean_value = 42\n"
|
||||
)
|
||||
|
||||
ctx_dir = monorepo_root / own_ctx
|
||||
(ctx_dir / "leaky_file.py").write_text(multi_leak_content, encoding="utf-8")
|
||||
|
||||
team_repo_path = fm.prepare_team_repo(own_ctx)
|
||||
|
||||
report = fm.verify_isolation(team_repo_path, own_ctx)
|
||||
assert report.is_isolated, (
|
||||
f"Team-Repo for '{own_ctx}' still has leaks after sanitizing "
|
||||
f"multiple leak types referencing '{other_ctx}'. "
|
||||
f"Leaks: {[(l.file_path, l.leak_type, l.leaked_context) for l in report.leaks]}"
|
||||
)
|
||||
|
||||
@given(data=st.data())
|
||||
@settings(max_examples=80, deadline=10000)
|
||||
def test_prepared_repo_with_config_references_passes_isolation(
|
||||
self, data: st.DataObject, tmp_path_factory
|
||||
) -> None:
|
||||
"""For any context and files containing config references to other
|
||||
contexts (like context: bahn in YAML), prepare_team_repo must produce
|
||||
a repo that passes verify_isolation.
|
||||
"""
|
||||
tmp_dir = tmp_path_factory.mktemp("prop25_config")
|
||||
fm, monorepo_root = _create_federation_manager(tmp_dir)
|
||||
|
||||
own_ctx, other_ctx = data.draw(context_pair())
|
||||
content = data.draw(cross_context_config_ref_content(other_ctx))
|
||||
filename = data.draw(file_name_with_extension())
|
||||
|
||||
ctx_dir = monorepo_root / own_ctx
|
||||
(ctx_dir / filename).write_text(content, encoding="utf-8")
|
||||
|
||||
team_repo_path = fm.prepare_team_repo(own_ctx)
|
||||
|
||||
report = fm.verify_isolation(team_repo_path, own_ctx)
|
||||
assert report.is_isolated, (
|
||||
f"Team-Repo for '{own_ctx}' has config ref leaks to '{other_ctx}'. "
|
||||
f"Leaks: {[(l.file_path, l.leak_type, l.leaked_context) for l in report.leaks]}"
|
||||
)
|
||||
|
||||
@given(data=st.data())
|
||||
@settings(max_examples=80, deadline=10000)
|
||||
def test_prepared_repo_with_comment_references_passes_isolation(
|
||||
self, data: st.DataObject, tmp_path_factory
|
||||
) -> None:
|
||||
"""For any context and files containing comments referencing other
|
||||
contexts, prepare_team_repo must produce a repo that passes verify_isolation.
|
||||
"""
|
||||
tmp_dir = tmp_path_factory.mktemp("prop25_comment")
|
||||
fm, monorepo_root = _create_federation_manager(tmp_dir)
|
||||
|
||||
own_ctx, other_ctx = data.draw(context_pair())
|
||||
content = data.draw(cross_context_comment_content(other_ctx))
|
||||
filename = data.draw(file_name_with_extension())
|
||||
|
||||
ctx_dir = monorepo_root / own_ctx
|
||||
(ctx_dir / filename).write_text(content, encoding="utf-8")
|
||||
|
||||
team_repo_path = fm.prepare_team_repo(own_ctx)
|
||||
|
||||
report = fm.verify_isolation(team_repo_path, own_ctx)
|
||||
assert report.is_isolated, (
|
||||
f"Team-Repo for '{own_ctx}' has comment leaks to '{other_ctx}'. "
|
||||
f"Leaks: {[(l.file_path, l.leak_type, l.leaked_context) for l in report.leaks]}"
|
||||
)
|
||||
@@ -0,0 +1,812 @@
|
||||
"""Ergänzende Unit-Tests für FederationManager (Task 15.7).
|
||||
|
||||
Deckt Edge-Cases und Szenarien ab, die in test_federation.py und
|
||||
test_federation_isolation.py noch nicht vorhanden sind:
|
||||
|
||||
- Subtree-Pull/Push mit verschiedenen Fehlertypen (rename, delete-modify)
|
||||
- full_sync: Push-Fehler nach erfolgreichem Pull
|
||||
- resolve_conflict mit konfigurierbarer Strategie
|
||||
- mirror_shared: Gemischte Erfolge/Fehler
|
||||
- Onboarding: Kein Leak von Monorepo-Struktur-Details
|
||||
- Sync-Abbruch bei Merge-Konflikten mit Protokollierung
|
||||
|
||||
Validates: Requirements 10.1, 10.3, 10.5, 10.6, 10.7, 10.9, 10.12
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from monorepo.federation import (
|
||||
ConflictResult,
|
||||
FederationManager,
|
||||
MemberInfo,
|
||||
MirrorResult,
|
||||
SubtreeSyncEngine,
|
||||
)
|
||||
from monorepo.models import ConflictInfo, SyncResult
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def team_repos_config(tmp_path: Path) -> Path:
|
||||
"""Erstellt eine temporäre team-repos.yaml."""
|
||||
config_path = tmp_path / "config" / "team-repos.yaml"
|
||||
config_path.parent.mkdir(parents=True)
|
||||
data = {
|
||||
"version": "1.0",
|
||||
"federation": {
|
||||
"topology": "hub-and-spoke",
|
||||
"hub_owner": "andre",
|
||||
"conflict_strategy": "team-wins",
|
||||
},
|
||||
"team_repos": [
|
||||
{
|
||||
"context": "privat",
|
||||
"url": "https://github.com/test/privat-team.git",
|
||||
"branch": "main",
|
||||
"sync_direction": "bidirectional",
|
||||
"sync_frequency": "on-push",
|
||||
"shared_mirror": {
|
||||
"enabled": True,
|
||||
"paths": [
|
||||
"shared/tools/common/",
|
||||
"shared/config/base-config.yaml",
|
||||
],
|
||||
"mode": "read-only",
|
||||
},
|
||||
},
|
||||
{
|
||||
"context": "dhive",
|
||||
"url": "https://gitlab.dhive.io/team/dhive-mono.git",
|
||||
"branch": "main",
|
||||
"sync_direction": "bidirectional",
|
||||
"sync_frequency": "on-push",
|
||||
"shared_mirror": {
|
||||
"enabled": True,
|
||||
"paths": ["shared/tools/", "shared/powers/"],
|
||||
"mode": "read-only",
|
||||
},
|
||||
},
|
||||
{
|
||||
"context": "bahn",
|
||||
"url": "https://gitlab.example.com/bahn-workspace.git",
|
||||
"branch": "develop",
|
||||
"sync_direction": "bidirectional",
|
||||
"sync_frequency": "daily",
|
||||
"shared_mirror": {
|
||||
"enabled": True,
|
||||
"paths": ["shared/tools/", "shared/knowledge-store/"],
|
||||
"mode": "read-only",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
with open(config_path, "w", encoding="utf-8") as f:
|
||||
yaml.dump(data, f)
|
||||
return config_path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def monorepo_root(tmp_path: Path) -> Path:
|
||||
"""Erstellt ein temporäres Monorepo-Verzeichnis mit Struktur."""
|
||||
root = tmp_path / "monorepo"
|
||||
root.mkdir()
|
||||
(root / "privat").mkdir()
|
||||
(root / "dhive").mkdir()
|
||||
(root / "bahn").mkdir()
|
||||
(root / "shared" / "tools" / "common").mkdir(parents=True)
|
||||
(root / "shared" / "config").mkdir(parents=True)
|
||||
(root / "shared" / "powers").mkdir(parents=True)
|
||||
(root / "shared" / "knowledge-store").mkdir(parents=True)
|
||||
return root
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def federation_manager(
|
||||
team_repos_config: Path, monorepo_root: Path
|
||||
) -> FederationManager:
|
||||
"""Erstellt einen FederationManager mit Testkonfiguration."""
|
||||
return FederationManager(
|
||||
config_path=team_repos_config,
|
||||
monorepo_root=monorepo_root,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def engine(monorepo_root: Path) -> SubtreeSyncEngine:
|
||||
"""Erstellt eine SubtreeSyncEngine mit Test-Root."""
|
||||
return SubtreeSyncEngine(monorepo_root=monorepo_root)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: SubtreeSyncEngine - Erweiterte Konflikt-Typen
|
||||
# Validates: Requirements 10.1, 10.3
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSubtreeSyncEngineConflictTypes:
|
||||
"""Tests für verschiedene Konflikt-Typen bei Subtree-Operationen."""
|
||||
|
||||
def test_subtree_pull_rename_conflict(self, engine: SubtreeSyncEngine) -> None:
|
||||
"""Prüft Erkennung von Rename-Konflikten beim Pull."""
|
||||
with patch.object(engine, "_run_git") as mock_git, \
|
||||
patch.object(engine, "_abort_merge"):
|
||||
error = subprocess.CalledProcessError(1, "git")
|
||||
error.stdout = (
|
||||
"CONFLICT (rename/rename): Rename dhive/old.py->dhive/new.py "
|
||||
"in HEAD. Rename dhive/old.py->dhive/other.py in remote."
|
||||
)
|
||||
error.stderr = ""
|
||||
mock_git.side_effect = error
|
||||
|
||||
result = engine.subtree_pull(
|
||||
remote="https://example.com/repo.git",
|
||||
prefix="dhive",
|
||||
branch="main",
|
||||
)
|
||||
|
||||
assert result.success is False
|
||||
assert len(result.conflicts) >= 1
|
||||
assert result.conflicts[0].conflict_type == "rename"
|
||||
|
||||
def test_subtree_pull_delete_modify_conflict(
|
||||
self, engine: SubtreeSyncEngine
|
||||
) -> None:
|
||||
"""Prüft Erkennung von Delete-Modify-Konflikten beim Pull."""
|
||||
with patch.object(engine, "_run_git") as mock_git, \
|
||||
patch.object(engine, "_abort_merge"):
|
||||
error = subprocess.CalledProcessError(1, "git")
|
||||
error.stdout = (
|
||||
"CONFLICT (modify/delete): bahn/removed.py deleted in remote "
|
||||
"and modified in HEAD."
|
||||
)
|
||||
error.stderr = ""
|
||||
mock_git.side_effect = error
|
||||
|
||||
result = engine.subtree_pull(
|
||||
remote="https://example.com/repo.git",
|
||||
prefix="bahn",
|
||||
branch="develop",
|
||||
)
|
||||
|
||||
assert result.success is False
|
||||
assert len(result.conflicts) >= 1
|
||||
assert result.conflicts[0].conflict_type == "delete-modify"
|
||||
|
||||
def test_subtree_pull_multiple_conflicts(
|
||||
self, engine: SubtreeSyncEngine
|
||||
) -> None:
|
||||
"""Prüft Erkennung mehrerer Konflikte in einem Pull."""
|
||||
with patch.object(engine, "_run_git") as mock_git, \
|
||||
patch.object(engine, "_abort_merge"):
|
||||
error = subprocess.CalledProcessError(1, "git")
|
||||
error.stdout = (
|
||||
"CONFLICT (content): Merge conflict in privat/a.py\n"
|
||||
"CONFLICT (content): Merge conflict in privat/b.py\n"
|
||||
"CONFLICT (rename/rename): Rename privat/c.py\n"
|
||||
"Automatic merge failed; fix conflicts and then commit."
|
||||
)
|
||||
error.stderr = ""
|
||||
mock_git.side_effect = error
|
||||
|
||||
result = engine.subtree_pull(
|
||||
remote="https://example.com/repo.git",
|
||||
prefix="privat",
|
||||
branch="main",
|
||||
)
|
||||
|
||||
assert result.success is False
|
||||
assert len(result.conflicts) == 3
|
||||
content_conflicts = [
|
||||
c for c in result.conflicts if c.conflict_type == "content"
|
||||
]
|
||||
rename_conflicts = [
|
||||
c for c in result.conflicts if c.conflict_type == "rename"
|
||||
]
|
||||
assert len(content_conflicts) == 2
|
||||
assert len(rename_conflicts) == 1
|
||||
|
||||
def test_subtree_push_auth_error(self, engine: SubtreeSyncEngine) -> None:
|
||||
"""Prüft Fehlerbehandlung bei Authentifizierungsfehler."""
|
||||
with patch.object(engine, "_run_git") as mock_git:
|
||||
error = subprocess.CalledProcessError(128, "git")
|
||||
error.stdout = ""
|
||||
error.stderr = "fatal: Authentication failed for 'https://...'"
|
||||
mock_git.side_effect = error
|
||||
|
||||
result = engine.subtree_push(
|
||||
remote="https://example.com/repo.git",
|
||||
prefix="dhive",
|
||||
branch="main",
|
||||
)
|
||||
|
||||
assert result.success is False
|
||||
assert result.conflicts[0].source == "monorepo"
|
||||
assert "fehlgeschlagen" in result.conflicts[0].details
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: full_sync - Push-Fehler nach erfolgreichem Pull
|
||||
# Validates: Requirements 10.3, 10.7
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFullSyncPushFailure:
|
||||
"""Tests für full_sync wenn Push nach Pull fehlschlägt."""
|
||||
|
||||
def test_full_sync_push_fails_after_pull_success(
|
||||
self, federation_manager: FederationManager
|
||||
) -> None:
|
||||
"""Prüft dass Push-Fehler nach erfolgreichem Pull korrekt gemeldet wird."""
|
||||
with patch.object(
|
||||
federation_manager.sync_engine, "subtree_pull"
|
||||
) as mock_pull, patch.object(
|
||||
federation_manager.sync_engine, "subtree_push"
|
||||
) as mock_push, patch.object(
|
||||
federation_manager.sync_engine, "detect_conflicts",
|
||||
return_value=[],
|
||||
):
|
||||
mock_pull.return_value = SyncResult(
|
||||
success=True, context="dhive", direction="pull",
|
||||
commits_synced=5, conflicts=[], timestamp=datetime.now(),
|
||||
)
|
||||
mock_push.return_value = SyncResult(
|
||||
success=False, context="dhive", direction="push",
|
||||
commits_synced=0,
|
||||
conflicts=[ConflictInfo(
|
||||
file_path="",
|
||||
conflict_type="content",
|
||||
source="monorepo",
|
||||
details="Push rejected: non-fast-forward",
|
||||
)],
|
||||
timestamp=datetime.now(),
|
||||
)
|
||||
result = federation_manager.full_sync("dhive")
|
||||
|
||||
assert result.success is False
|
||||
assert result.direction == "full"
|
||||
# Pull commits are counted even though push failed
|
||||
assert result.commits_synced == 5
|
||||
assert len(result.conflicts) == 1
|
||||
assert "Push rejected" in result.conflicts[0].details
|
||||
|
||||
def test_full_sync_uses_correct_branch_per_context(
|
||||
self, federation_manager: FederationManager
|
||||
) -> None:
|
||||
"""Prüft dass full_sync den konfigurierten Branch verwendet."""
|
||||
with patch.object(
|
||||
federation_manager.sync_engine, "subtree_pull"
|
||||
) as mock_pull, patch.object(
|
||||
federation_manager.sync_engine, "subtree_push"
|
||||
) as mock_push, patch.object(
|
||||
federation_manager.sync_engine, "detect_conflicts",
|
||||
return_value=[],
|
||||
):
|
||||
mock_pull.return_value = SyncResult(
|
||||
success=True, context="bahn", direction="pull",
|
||||
commits_synced=1, conflicts=[], timestamp=datetime.now(),
|
||||
)
|
||||
mock_push.return_value = SyncResult(
|
||||
success=True, context="bahn", direction="push",
|
||||
commits_synced=1, conflicts=[], timestamp=datetime.now(),
|
||||
)
|
||||
federation_manager.full_sync("bahn")
|
||||
|
||||
# bahn uses "develop" branch in our config
|
||||
mock_pull.assert_called_once_with(
|
||||
remote="https://gitlab.example.com/bahn-workspace.git",
|
||||
prefix="bahn",
|
||||
branch="develop",
|
||||
)
|
||||
mock_push.assert_called_once_with(
|
||||
remote="https://gitlab.example.com/bahn-workspace.git",
|
||||
prefix="bahn",
|
||||
branch="develop",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Sync-Abbruch bei Merge-Konflikten mit Protokollierung
|
||||
# Validates: Requirements 10.7
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSyncAbortOnConflict:
|
||||
"""Tests für Sync-Abbruch und Konflikt-Protokollierung."""
|
||||
|
||||
def test_pull_conflict_aborts_merge_cleanly(
|
||||
self, engine: SubtreeSyncEngine
|
||||
) -> None:
|
||||
"""Prüft dass bei Merge-Konflikt ein abort durchgeführt wird."""
|
||||
with patch.object(engine, "_run_git") as mock_git, \
|
||||
patch.object(engine, "_abort_merge") as mock_abort:
|
||||
error = subprocess.CalledProcessError(1, "git")
|
||||
error.stdout = "CONFLICT (content): Merge conflict in file.py"
|
||||
error.stderr = ""
|
||||
mock_git.side_effect = error
|
||||
|
||||
result = engine.subtree_pull(
|
||||
remote="https://example.com/repo.git",
|
||||
prefix="privat",
|
||||
branch="main",
|
||||
)
|
||||
|
||||
# Merge abort muss aufgerufen werden
|
||||
mock_abort.assert_called_once()
|
||||
assert result.success is False
|
||||
assert result.commits_synced == 0
|
||||
|
||||
def test_conflict_details_contain_file_info(
|
||||
self, engine: SubtreeSyncEngine
|
||||
) -> None:
|
||||
"""Prüft dass Konflikt-Details Dateiinformationen enthalten."""
|
||||
with patch.object(engine, "_run_git") as mock_git, \
|
||||
patch.object(engine, "_abort_merge"):
|
||||
error = subprocess.CalledProcessError(1, "git")
|
||||
error.stdout = (
|
||||
"CONFLICT (content): Merge conflict in privat/src/main.py\n"
|
||||
"Automatic merge failed; fix conflicts and then commit."
|
||||
)
|
||||
error.stderr = ""
|
||||
mock_git.side_effect = error
|
||||
|
||||
result = engine.subtree_pull(
|
||||
remote="https://example.com/repo.git",
|
||||
prefix="privat",
|
||||
branch="main",
|
||||
)
|
||||
|
||||
assert result.conflicts[0].file_path == "privat/src/main.py"
|
||||
assert result.conflicts[0].source == "team-repo"
|
||||
assert "CONFLICT" in result.conflicts[0].details
|
||||
|
||||
def test_full_sync_abort_preserves_pull_commit_count(
|
||||
self, federation_manager: FederationManager
|
||||
) -> None:
|
||||
"""Prüft dass bei Abbruch die bisher gezählten Commits erhalten bleiben."""
|
||||
with patch.object(
|
||||
federation_manager.sync_engine, "detect_conflicts",
|
||||
return_value=[ConflictInfo(
|
||||
file_path="privat/dirty.py",
|
||||
conflict_type="content",
|
||||
source="monorepo",
|
||||
details="Uncommitted lokale Änderungen erkannt",
|
||||
)],
|
||||
):
|
||||
result = federation_manager.full_sync("privat")
|
||||
|
||||
assert result.success is False
|
||||
assert result.commits_synced == 0
|
||||
assert result.conflicts[0].file_path == "privat/dirty.py"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Konflikt-Auflösung mit team-wins Strategie (erweitert)
|
||||
# Validates: Requirements 10.6
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestResolveConflictExtended:
|
||||
"""Erweiterte Tests für resolve_conflict mit team-wins."""
|
||||
|
||||
def test_team_wins_is_configured_default_strategy(
|
||||
self, federation_manager: FederationManager
|
||||
) -> None:
|
||||
"""Prüft dass team-wins die konfigurierte Standard-Strategie ist."""
|
||||
assert federation_manager.get_conflict_strategy() == "team-wins"
|
||||
|
||||
def test_resolve_uses_theirs_for_team_wins(
|
||||
self, federation_manager: FederationManager
|
||||
) -> None:
|
||||
"""Prüft dass team-wins git checkout --theirs für jede Datei verwendet."""
|
||||
calls_made: list[list[str]] = []
|
||||
|
||||
def capture_calls(args: list[str]) -> MagicMock:
|
||||
calls_made.append(args)
|
||||
if args[0] == "diff":
|
||||
return MagicMock(stdout="privat/file1.py\n", stderr="")
|
||||
return MagicMock(stdout="", stderr="")
|
||||
|
||||
with patch.object(
|
||||
federation_manager.sync_engine, "_run_git",
|
||||
side_effect=capture_calls,
|
||||
):
|
||||
result = federation_manager.resolve_conflict(
|
||||
"privat", strategy="team-wins"
|
||||
)
|
||||
|
||||
assert result.success is True
|
||||
# Verify --theirs was used
|
||||
checkout_calls = [c for c in calls_made if "checkout" in c]
|
||||
assert len(checkout_calls) == 1
|
||||
assert "--theirs" in checkout_calls[0]
|
||||
|
||||
def test_resolve_uses_ours_for_hub_wins(
|
||||
self, federation_manager: FederationManager
|
||||
) -> None:
|
||||
"""Prüft dass hub-wins git checkout --ours verwendet."""
|
||||
calls_made: list[list[str]] = []
|
||||
|
||||
def capture_calls(args: list[str]) -> MagicMock:
|
||||
calls_made.append(args)
|
||||
if args[0] == "diff":
|
||||
return MagicMock(stdout="dhive/app.py\n", stderr="")
|
||||
return MagicMock(stdout="", stderr="")
|
||||
|
||||
with patch.object(
|
||||
federation_manager.sync_engine, "_run_git",
|
||||
side_effect=capture_calls,
|
||||
):
|
||||
result = federation_manager.resolve_conflict(
|
||||
"dhive", strategy="hub-wins"
|
||||
)
|
||||
|
||||
assert result.success is True
|
||||
checkout_calls = [c for c in calls_made if "checkout" in c]
|
||||
assert "--ours" in checkout_calls[0]
|
||||
|
||||
def test_resolve_stages_files_after_resolution(
|
||||
self, federation_manager: FederationManager
|
||||
) -> None:
|
||||
"""Prüft dass aufgelöste Dateien mit git add gestaged werden."""
|
||||
calls_made: list[list[str]] = []
|
||||
|
||||
def capture_calls(args: list[str]) -> MagicMock:
|
||||
calls_made.append(args)
|
||||
if args[0] == "diff":
|
||||
return MagicMock(stdout="privat/resolved.py\n", stderr="")
|
||||
return MagicMock(stdout="", stderr="")
|
||||
|
||||
with patch.object(
|
||||
federation_manager.sync_engine, "_run_git",
|
||||
side_effect=capture_calls,
|
||||
):
|
||||
federation_manager.resolve_conflict("privat")
|
||||
|
||||
add_calls = [c for c in calls_made if c[0] == "add"]
|
||||
assert len(add_calls) == 1
|
||||
assert "privat/resolved.py" in add_calls[0]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Isolation-Check erweitert
|
||||
# Validates: Requirements 10.5, 10.9
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestIsolationCheckExtended:
|
||||
"""Erweiterte Isolation-Check-Tests für Leakage-Erkennung."""
|
||||
|
||||
def test_isolation_detects_mixed_path_and_env_leaks(
|
||||
self, federation_manager: FederationManager, tmp_path: Path
|
||||
) -> None:
|
||||
"""Prüft Erkennung gemischter Leaks (Pfad + Env-Var) in einer Datei."""
|
||||
team_repo = tmp_path / "team-repo"
|
||||
team_repo.mkdir()
|
||||
(team_repo / "settings.py").write_text(
|
||||
'import os\n'
|
||||
'DB_PATH = "bahn/database"\n'
|
||||
'KEY = os.getenv("DHIVE_SECRET_KEY")\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
report = federation_manager.verify_isolation(team_repo, "privat")
|
||||
assert report.is_isolated is False
|
||||
leak_types = {l.leak_type for l in report.leaks}
|
||||
assert "path" in leak_types
|
||||
assert "env_var" in leak_types
|
||||
|
||||
def test_isolation_binary_files_skipped(
|
||||
self, federation_manager: FederationManager, tmp_path: Path
|
||||
) -> None:
|
||||
"""Prüft dass Binärdateien übersprungen werden (kein Crash)."""
|
||||
team_repo = tmp_path / "team-repo"
|
||||
team_repo.mkdir()
|
||||
# Schreibe Binär-Inhalt (nicht-UTF8)
|
||||
(team_repo / "image.png").write_bytes(b'\x89PNG\r\n\x1a\n\x00\x00')
|
||||
# Schreibe saubere Textdatei
|
||||
(team_repo / "clean.py").write_text("x = 1\n", encoding="utf-8")
|
||||
|
||||
report = federation_manager.verify_isolation(team_repo, "privat")
|
||||
assert report.is_isolated is True
|
||||
|
||||
def test_isolation_git_directory_skipped(
|
||||
self, federation_manager: FederationManager, tmp_path: Path
|
||||
) -> None:
|
||||
"""Prüft dass .git-Verzeichnisse nicht gescannt werden."""
|
||||
team_repo = tmp_path / "team-repo"
|
||||
team_repo.mkdir()
|
||||
git_dir = team_repo / ".git" / "refs"
|
||||
git_dir.mkdir(parents=True)
|
||||
# .git enthält Referenzen die wie Leaks aussehen
|
||||
(git_dir / "heads").write_text("dhive/feature\n", encoding="utf-8")
|
||||
# Saubere Hauptdatei
|
||||
(team_repo / "main.py").write_text("# clean\n", encoding="utf-8")
|
||||
|
||||
report = federation_manager.verify_isolation(team_repo, "privat")
|
||||
assert report.is_isolated is True
|
||||
|
||||
def test_isolation_deep_nested_files_checked(
|
||||
self, federation_manager: FederationManager, tmp_path: Path
|
||||
) -> None:
|
||||
"""Prüft dass tief verschachtelte Dateien auch gescannt werden."""
|
||||
team_repo = tmp_path / "team-repo"
|
||||
deep_dir = team_repo / "src" / "core" / "utils" / "helpers"
|
||||
deep_dir.mkdir(parents=True)
|
||||
(deep_dir / "config.yaml").write_text(
|
||||
"remote: bahn/api-gateway\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
report = federation_manager.verify_isolation(team_repo, "privat")
|
||||
assert report.is_isolated is False
|
||||
# Path separator is OS-dependent, check with Path for platform compatibility
|
||||
expected_parts = ("src", "core", "utils", "helpers", "config.yaml")
|
||||
assert any(
|
||||
all(part in l.file_path for part in expected_parts)
|
||||
for l in report.leaks
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Shared-Mirror erweitert
|
||||
# Validates: Requirements 10.12
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMirrorSharedExtended:
|
||||
"""Erweiterte Tests für mirror_shared – Read-Only-Spiegelung."""
|
||||
|
||||
def test_mirror_mixed_success_and_failure(
|
||||
self, federation_manager: FederationManager, monorepo_root: Path
|
||||
) -> None:
|
||||
"""Prüft dass bei gemischten Pfaden Teilfehler korrekt gemeldet werden."""
|
||||
# Existierender Pfad
|
||||
tools_dir = monorepo_root / "shared" / "tools" / "common"
|
||||
tools_dir.mkdir(parents=True, exist_ok=True)
|
||||
(tools_dir / "helper.sh").write_text("#!/bin/bash\n", encoding="utf-8")
|
||||
|
||||
result = federation_manager.mirror_shared("privat", [
|
||||
"shared/tools/common/helper.sh", # existiert
|
||||
"shared/tools/common/missing.sh", # existiert nicht
|
||||
])
|
||||
|
||||
# Teilerfolg: eine Datei gespiegelt, eine fehlgeschlagen
|
||||
assert result.success is False # Fehler vorhanden
|
||||
assert "shared/tools/common/helper.sh" in result.mirrored_paths
|
||||
assert any("existiert nicht" in e for e in result.errors)
|
||||
|
||||
def test_mirror_preserves_file_content(
|
||||
self, federation_manager: FederationManager, monorepo_root: Path
|
||||
) -> None:
|
||||
"""Prüft dass der Dateiinhalt exakt gespiegelt wird."""
|
||||
tools_dir = monorepo_root / "shared" / "tools" / "common"
|
||||
tools_dir.mkdir(parents=True, exist_ok=True)
|
||||
original_content = "#!/bin/bash\necho 'Version 2.0'\nexit 0\n"
|
||||
(tools_dir / "deploy.sh").write_text(original_content, encoding="utf-8")
|
||||
|
||||
federation_manager.mirror_shared("privat", ["shared/tools/common/deploy.sh"])
|
||||
|
||||
mirror_file = (
|
||||
monorepo_root / "privat" / ".shared-mirror"
|
||||
/ "shared" / "tools" / "common" / "deploy.sh"
|
||||
)
|
||||
assert mirror_file.read_text(encoding="utf-8") == original_content
|
||||
|
||||
def test_mirror_multiple_allowed_paths(
|
||||
self, federation_manager: FederationManager, monorepo_root: Path
|
||||
) -> None:
|
||||
"""Prüft Spiegelung mehrerer erlaubter Pfade."""
|
||||
# Erstelle Dateien in verschiedenen erlaubten Pfaden
|
||||
(monorepo_root / "shared" / "tools" / "common" / "a.sh").write_text(
|
||||
"a", encoding="utf-8"
|
||||
)
|
||||
config_file = monorepo_root / "shared" / "config" / "base-config.yaml"
|
||||
config_file.write_text("key: val", encoding="utf-8")
|
||||
|
||||
result = federation_manager.mirror_shared("privat", [
|
||||
"shared/tools/common/a.sh",
|
||||
"shared/config/base-config.yaml",
|
||||
])
|
||||
|
||||
assert result.success is True
|
||||
assert len(result.mirrored_paths) == 2
|
||||
|
||||
def test_mirror_read_only_prevents_write(
|
||||
self, federation_manager: FederationManager, monorepo_root: Path
|
||||
) -> None:
|
||||
"""Prüft dass gespiegelte Dateien nicht beschrieben werden können."""
|
||||
import stat
|
||||
|
||||
tools_dir = monorepo_root / "shared" / "tools" / "common"
|
||||
tools_dir.mkdir(parents=True, exist_ok=True)
|
||||
(tools_dir / "readonly.sh").write_text("content", encoding="utf-8")
|
||||
|
||||
federation_manager.mirror_shared("privat", ["shared/tools/common/readonly.sh"])
|
||||
|
||||
mirror_file = (
|
||||
monorepo_root / "privat" / ".shared-mirror"
|
||||
/ "shared" / "tools" / "common" / "readonly.sh"
|
||||
)
|
||||
mode = mirror_file.stat().st_mode
|
||||
# Prüfe 0o444: nur Leserechte
|
||||
assert mode & stat.S_IRUSR # Lesen für Owner
|
||||
assert not (mode & stat.S_IWUSR) # Kein Schreiben
|
||||
assert not (mode & stat.S_IXUSR) # Kein Execute
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Onboarding ohne Monorepo-Kenntnis
|
||||
# Validates: Requirements 10.9
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestOnboardingNoMonorepoKnowledge:
|
||||
"""Tests dass Onboarding keine Monorepo-Struktur offenbart."""
|
||||
|
||||
def test_onboard_returns_only_team_repo_url(
|
||||
self, federation_manager: FederationManager
|
||||
) -> None:
|
||||
"""Prüft dass nur die Team-Repo-URL zurückgegeben wird."""
|
||||
member = MemberInfo(
|
||||
name="Neue Kollegin", email="kollegin@dhive.io", role="developer"
|
||||
)
|
||||
result = federation_manager.onboard_member("dhive", member)
|
||||
|
||||
assert result.success is True
|
||||
assert result.team_repo_url == "https://gitlab.dhive.io/team/dhive-mono.git"
|
||||
# Keine Erwähnung des Monorepos oder anderer Kontexte
|
||||
assert "monorepo" not in result.team_repo_url.lower()
|
||||
assert "privat" not in result.team_repo_url
|
||||
assert "bahn" not in result.team_repo_url
|
||||
|
||||
def test_onboard_each_context_provides_correct_url(
|
||||
self, federation_manager: FederationManager
|
||||
) -> None:
|
||||
"""Prüft dass jeder Kontext die korrekte Team-Repo-URL bekommt."""
|
||||
expected_urls = {
|
||||
"privat": "https://github.com/test/privat-team.git",
|
||||
"dhive": "https://gitlab.dhive.io/team/dhive-mono.git",
|
||||
"bahn": "https://gitlab.example.com/bahn-workspace.git",
|
||||
}
|
||||
|
||||
for context, expected_url in expected_urls.items():
|
||||
member = MemberInfo(
|
||||
name=f"User {context}",
|
||||
email=f"user@{context}.io",
|
||||
)
|
||||
result = federation_manager.onboard_member(context, member)
|
||||
assert result.success is True
|
||||
assert result.team_repo_url == expected_url
|
||||
assert result.context == context
|
||||
|
||||
def test_onboard_does_not_reveal_hub_topology(
|
||||
self, federation_manager: FederationManager
|
||||
) -> None:
|
||||
"""Prüft dass Hub-and-Spoke-Topologie nicht offenbart wird."""
|
||||
member = MemberInfo(name="Team Member", email="member@bahn.de")
|
||||
result = federation_manager.onboard_member("bahn", member)
|
||||
|
||||
# Serialisiere Ergebnis um nach Leaks zu suchen
|
||||
result_repr = repr(result)
|
||||
assert "hub" not in result_repr.lower()
|
||||
assert "spoke" not in result_repr.lower()
|
||||
assert "andre" not in result_repr.lower()
|
||||
|
||||
def test_onboard_invalid_email_format_still_rejected(
|
||||
self, federation_manager: FederationManager
|
||||
) -> None:
|
||||
"""Prüft dass leere E-Mail abgelehnt wird."""
|
||||
member = MemberInfo(name="Valid Name", email=" ")
|
||||
result = federation_manager.onboard_member("privat", member)
|
||||
assert result.success is False
|
||||
assert any("E-Mail" in e for e in result.errors)
|
||||
|
||||
def test_onboard_preserves_member_info(
|
||||
self, federation_manager: FederationManager
|
||||
) -> None:
|
||||
"""Prüft dass Member-Info im Ergebnis korrekt zurückgegeben wird."""
|
||||
member = MemberInfo(
|
||||
name="Dr. Anna Becker",
|
||||
email="anna.becker@dhive.io",
|
||||
role="architect",
|
||||
)
|
||||
result = federation_manager.onboard_member("dhive", member)
|
||||
|
||||
assert result.success is True
|
||||
assert result.member_name == "Dr. Anna Becker"
|
||||
assert result.context == "dhive"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: SubtreeSyncEngine Hilfsmethoden
|
||||
# Validates: Requirements 10.1
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSyncEngineHelpers:
|
||||
"""Tests für interne Hilfsmethoden der SubtreeSyncEngine."""
|
||||
|
||||
def test_is_merge_conflict_various_indicators(
|
||||
self, engine: SubtreeSyncEngine
|
||||
) -> None:
|
||||
"""Prüft Erkennung verschiedener Merge-Konflikt-Indikatoren."""
|
||||
indicators = [
|
||||
"CONFLICT (content): Merge conflict in file.py",
|
||||
"Automatic merge failed; fix conflicts and then commit.",
|
||||
"merge conflict detected",
|
||||
"Merge conflict in path/to/file",
|
||||
"fix conflicts and then commit the result",
|
||||
]
|
||||
for msg in indicators:
|
||||
error = subprocess.CalledProcessError(1, "git")
|
||||
error.stdout = msg
|
||||
error.stderr = ""
|
||||
assert engine._is_merge_conflict(error) is True, (
|
||||
f"Should detect merge conflict in: {msg}"
|
||||
)
|
||||
|
||||
def test_is_merge_conflict_false_for_other_errors(
|
||||
self, engine: SubtreeSyncEngine
|
||||
) -> None:
|
||||
"""Prüft dass andere Fehler nicht als Merge-Konflikt erkannt werden."""
|
||||
non_conflict_msgs = [
|
||||
"fatal: Could not read from remote repository.",
|
||||
"error: failed to push some refs",
|
||||
"Permission denied (publickey).",
|
||||
"fatal: not a git repository",
|
||||
]
|
||||
for msg in non_conflict_msgs:
|
||||
error = subprocess.CalledProcessError(1, "git")
|
||||
error.stdout = ""
|
||||
error.stderr = msg
|
||||
assert engine._is_merge_conflict(error) is False, (
|
||||
f"Should NOT detect conflict in: {msg}"
|
||||
)
|
||||
|
||||
def test_count_commits_empty_output(
|
||||
self, engine: SubtreeSyncEngine
|
||||
) -> None:
|
||||
"""Prüft Commit-Zählung bei leerer Ausgabe."""
|
||||
assert engine._count_commits("") == 0
|
||||
|
||||
def test_count_commits_with_changes(
|
||||
self, engine: SubtreeSyncEngine
|
||||
) -> None:
|
||||
"""Prüft Commit-Zählung bei typischer Git-Ausgabe."""
|
||||
output = "abc1234 First commit\ndef5678 Second commit\n1 file changed\n"
|
||||
count = engine._count_commits(output)
|
||||
assert count >= 1
|
||||
|
||||
def test_extract_conflicts_content_type(
|
||||
self, engine: SubtreeSyncEngine
|
||||
) -> None:
|
||||
"""Prüft Extraktion von Content-Konflikten."""
|
||||
error = subprocess.CalledProcessError(1, "git")
|
||||
error.stdout = "CONFLICT (content): Merge conflict in src/main.py"
|
||||
error.stderr = ""
|
||||
conflicts = engine._extract_conflicts(error)
|
||||
assert len(conflicts) == 1
|
||||
assert conflicts[0].conflict_type == "content"
|
||||
assert "src/main.py" in conflicts[0].file_path
|
||||
|
||||
def test_extract_conflicts_no_conflict_lines_fallback(
|
||||
self, engine: SubtreeSyncEngine
|
||||
) -> None:
|
||||
"""Prüft Fallback wenn keine CONFLICT-Zeilen gefunden werden."""
|
||||
error = subprocess.CalledProcessError(1, "git")
|
||||
error.stdout = "Automatic merge failed"
|
||||
error.stderr = ""
|
||||
conflicts = engine._extract_conflicts(error)
|
||||
assert len(conflicts) == 1
|
||||
assert "Merge-Konflikt erkannt" in conflicts[0].details
|
||||
@@ -0,0 +1,547 @@
|
||||
"""Unit-Tests für das hooks-Modul.
|
||||
|
||||
Tests für:
|
||||
- HookManager: Installation, Deinstallation, Validierung
|
||||
- Read-Only-Schutz: Prüfung gestaged Dateien in geschützten Pfaden
|
||||
- Secret-Verschlüsselungs-Prüfung: Unverschlüsselte Secrets blockieren
|
||||
- Integration mit ContextGuard und SecretEncryptionManager
|
||||
- Hook-Installation bei `init`
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import stat
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from monorepo.hooks import HookManager, install_hooks, uninstall_hooks
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def monorepo_root(tmp_path: Path) -> Path:
|
||||
"""Erstellt ein minimales Monorepo mit .git-Verzeichnis und Konfiguration."""
|
||||
root = tmp_path / "monorepo"
|
||||
root.mkdir()
|
||||
|
||||
# Git-Verzeichnis simulieren
|
||||
git_dir = root / ".git"
|
||||
git_dir.mkdir()
|
||||
(git_dir / "hooks").mkdir()
|
||||
|
||||
# Kontextordner
|
||||
for ctx in ["privat", "dhive", "bahn", "shared"]:
|
||||
(root / ctx).mkdir()
|
||||
|
||||
# shared/config
|
||||
config_dir = root / "shared" / "config"
|
||||
config_dir.mkdir(parents=True)
|
||||
|
||||
# repos.yaml mit einem Read-Only-Repo
|
||||
repos_config = {
|
||||
"repos": [
|
||||
{
|
||||
"name": "symphony-spec",
|
||||
"url": "https://github.com/openai/symphony",
|
||||
"mode": "read-only",
|
||||
"target": "shared/references/symphony",
|
||||
"pinned": "v1.0.0",
|
||||
"mechanism": "subtree",
|
||||
},
|
||||
{
|
||||
"name": "db-wissensdatenbank",
|
||||
"url": "https://gitlab.2700.2db.it/team/db-wissen.git",
|
||||
"mode": "upstream",
|
||||
"target": "bahn/db-wissensdatenbank",
|
||||
"pinned": "main",
|
||||
"mechanism": "subtree",
|
||||
},
|
||||
]
|
||||
}
|
||||
with open(config_dir / "repos.yaml", "w", encoding="utf-8") as f:
|
||||
yaml.dump(repos_config, f)
|
||||
|
||||
# access-config.yaml (für ContextGuard)
|
||||
access_config = {
|
||||
"contexts": {
|
||||
"privat": {
|
||||
"env_file": "privat/.env",
|
||||
"allowed_shared": ["shared/tools/", "shared/powers/", "shared/config/"],
|
||||
},
|
||||
"dhive": {
|
||||
"env_file": "dhive/.env",
|
||||
"allowed_shared": ["shared/tools/", "shared/powers/", "shared/config/"],
|
||||
},
|
||||
"bahn": {
|
||||
"env_file": "bahn/.env",
|
||||
"allowed_shared": ["shared/tools/", "shared/powers/", "shared/config/"],
|
||||
},
|
||||
"shared": {
|
||||
"env_file": "shared/.env",
|
||||
"allowed_shared": ["*"],
|
||||
},
|
||||
}
|
||||
}
|
||||
with open(config_dir / "access-config.yaml", "w", encoding="utf-8") as f:
|
||||
yaml.dump(access_config, f)
|
||||
|
||||
# .gitattributes im privat-Kontext (mit git-crypt-Filter)
|
||||
privat_ga = root / "privat" / ".gitattributes"
|
||||
privat_ga.write_text(
|
||||
"# git-crypt Filter für Kontext: privat\n"
|
||||
".env filter=git-crypt-privat diff=git-crypt-privat\n"
|
||||
"*.pem filter=git-crypt-privat diff=git-crypt-privat\n"
|
||||
"*.key filter=git-crypt-privat diff=git-crypt-privat\n"
|
||||
"*token* filter=git-crypt-privat diff=git-crypt-privat\n"
|
||||
"*secret* filter=git-crypt-privat diff=git-crypt-privat\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
return root
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hook_manager(monorepo_root: Path) -> HookManager:
|
||||
"""Erstellt einen HookManager für das Test-Monorepo."""
|
||||
return HookManager(monorepo_root=monorepo_root)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Installation und Deinstallation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestHookInstallation:
|
||||
"""Tests für die Hook-Installation und -Deinstallation."""
|
||||
|
||||
def test_install_creates_hook_file(self, hook_manager: HookManager, monorepo_root: Path):
|
||||
"""Hook-Installation erstellt die pre-commit Datei."""
|
||||
hook_manager.install_hooks()
|
||||
hook_path = monorepo_root / ".git" / "hooks" / "pre-commit"
|
||||
assert hook_path.exists()
|
||||
|
||||
def test_hook_is_executable(self, hook_manager: HookManager, monorepo_root: Path):
|
||||
"""Installierter Hook hat Ausführungsrechte (Unix) bzw. existiert (Windows)."""
|
||||
hook_manager.install_hooks()
|
||||
hook_path = monorepo_root / ".git" / "hooks" / "pre-commit"
|
||||
if os.name != "nt":
|
||||
mode = hook_path.stat().st_mode
|
||||
assert mode & stat.S_IXUSR # Owner execute
|
||||
else:
|
||||
# Windows unterstützt keine Unix-Permissions, nur Existenz prüfen
|
||||
assert hook_path.exists()
|
||||
|
||||
def test_hook_contains_shebang(self, hook_manager: HookManager, monorepo_root: Path):
|
||||
"""Installierter Hook beginnt mit Shebang."""
|
||||
hook_manager.install_hooks()
|
||||
hook_path = monorepo_root / ".git" / "hooks" / "pre-commit"
|
||||
content = hook_path.read_text(encoding="utf-8")
|
||||
assert content.startswith("#!/bin/sh")
|
||||
|
||||
def test_hook_contains_markers(self, hook_manager: HookManager, monorepo_root: Path):
|
||||
"""Installierter Hook enthält Start- und End-Marker."""
|
||||
hook_manager.install_hooks()
|
||||
hook_path = monorepo_root / ".git" / "hooks" / "pre-commit"
|
||||
content = hook_path.read_text(encoding="utf-8")
|
||||
assert "# === Monorepo Integrated Pre-Commit Hook (auto-generiert) ===" in content
|
||||
assert "# === Ende Monorepo Integrated Pre-Commit Hook ===" in content
|
||||
|
||||
def test_hook_contains_readonly_section(self, hook_manager: HookManager, monorepo_root: Path):
|
||||
"""Installierter Hook enthält Read-Only-Schutz-Abschnitt."""
|
||||
hook_manager.install_hooks()
|
||||
hook_path = monorepo_root / ".git" / "hooks" / "pre-commit"
|
||||
content = hook_path.read_text(encoding="utf-8")
|
||||
assert "Read-Only-Repos Schutz" in content
|
||||
assert "shared/references/symphony" in content
|
||||
|
||||
def test_hook_contains_secrets_section(self, hook_manager: HookManager, monorepo_root: Path):
|
||||
"""Installierter Hook enthält Secret-Prüfungs-Abschnitt."""
|
||||
hook_manager.install_hooks()
|
||||
hook_path = monorepo_root / ".git" / "hooks" / "pre-commit"
|
||||
content = hook_path.read_text(encoding="utf-8")
|
||||
assert "Unverschlüsselte Secrets Prüfung" in content
|
||||
assert "check_encrypted" in content
|
||||
assert "is_secret_file" in content
|
||||
|
||||
def test_hook_preserves_existing_content(self, hook_manager: HookManager, monorepo_root: Path):
|
||||
"""Installation bewahrt bestehende Hook-Inhalte."""
|
||||
hook_path = monorepo_root / ".git" / "hooks" / "pre-commit"
|
||||
hook_path.write_text("#!/bin/sh\n# Custom Hook Logic\necho 'custom'\n", encoding="utf-8")
|
||||
|
||||
hook_manager.install_hooks()
|
||||
content = hook_path.read_text(encoding="utf-8")
|
||||
|
||||
# Bestehender Inhalt muss erhalten bleiben
|
||||
assert "# Custom Hook Logic" in content
|
||||
assert "echo 'custom'" in content
|
||||
# Neuer Monorepo-Abschnitt muss vorhanden sein
|
||||
assert "Monorepo Integrated Pre-Commit Hook" in content
|
||||
|
||||
def test_hook_update_replaces_existing_section(self, hook_manager: HookManager, monorepo_root: Path):
|
||||
"""Erneute Installation aktualisiert nur den Monorepo-Abschnitt."""
|
||||
hook_manager.install_hooks()
|
||||
|
||||
# Zweite Installation (simuliert Update)
|
||||
hook_manager.install_hooks()
|
||||
|
||||
hook_path = monorepo_root / ".git" / "hooks" / "pre-commit"
|
||||
content = hook_path.read_text(encoding="utf-8")
|
||||
|
||||
# Sollte genau einmal den Marker enthalten (kein doppelter Abschnitt)
|
||||
assert content.count("# === Monorepo Integrated Pre-Commit Hook") == 1
|
||||
|
||||
def test_uninstall_removes_section(self, hook_manager: HookManager, monorepo_root: Path):
|
||||
"""Deinstallation entfernt den Monorepo-Abschnitt."""
|
||||
hook_manager.install_hooks()
|
||||
hook_manager.uninstall_hooks()
|
||||
|
||||
hook_path = monorepo_root / ".git" / "hooks" / "pre-commit"
|
||||
# Hook-Datei sollte entfernt sein (war nur Monorepo-Inhalt)
|
||||
assert not hook_path.exists()
|
||||
|
||||
def test_uninstall_preserves_custom_content(self, hook_manager: HookManager, monorepo_root: Path):
|
||||
"""Deinstallation bewahrt benutzerdefinierte Hook-Inhalte."""
|
||||
hook_path = monorepo_root / ".git" / "hooks" / "pre-commit"
|
||||
hook_path.write_text("#!/bin/sh\n# Custom Logic\necho 'keep'\n", encoding="utf-8")
|
||||
|
||||
hook_manager.install_hooks()
|
||||
hook_manager.uninstall_hooks()
|
||||
|
||||
assert hook_path.exists()
|
||||
content = hook_path.read_text(encoding="utf-8")
|
||||
assert "# Custom Logic" in content
|
||||
assert "Monorepo Integrated Pre-Commit Hook" not in content
|
||||
|
||||
def test_is_hook_installed(self, hook_manager: HookManager):
|
||||
"""is_hook_installed gibt korrekt True/False zurück."""
|
||||
assert not hook_manager.is_hook_installed()
|
||||
hook_manager.install_hooks()
|
||||
assert hook_manager.is_hook_installed()
|
||||
|
||||
def test_uninstall_when_no_hook(self, hook_manager: HookManager):
|
||||
"""Deinstallation ohne vorhandenen Hook läuft ohne Fehler."""
|
||||
# Sollte keine Exception werfen
|
||||
hook_manager.uninstall_hooks()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Read-Only-Schutz
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestReadOnlyProtection:
|
||||
"""Tests für die Read-Only-Repo-Schutzprüfung."""
|
||||
|
||||
def test_readonly_path_is_detected(self, hook_manager: HookManager):
|
||||
"""Gestagte Dateien in Read-Only-Repos werden blockiert."""
|
||||
errors = hook_manager.validate_staged_files([
|
||||
"shared/references/symphony/README.md",
|
||||
])
|
||||
assert len(errors) == 1
|
||||
assert "read-only" in errors[0]
|
||||
assert "symphony" in errors[0]
|
||||
|
||||
def test_readonly_subdirectory_is_protected(self, hook_manager: HookManager):
|
||||
"""Dateien in Unterverzeichnissen von Read-Only-Repos werden blockiert."""
|
||||
errors = hook_manager.validate_staged_files([
|
||||
"shared/references/symphony/src/main.py",
|
||||
])
|
||||
assert len(errors) == 1
|
||||
assert "read-only" in errors[0]
|
||||
|
||||
def test_non_readonly_repo_is_allowed(self, hook_manager: HookManager):
|
||||
"""Dateien in Upstream-Repos werden nicht blockiert (nur Read-Only)."""
|
||||
errors = hook_manager.validate_staged_files([
|
||||
"bahn/db-wissensdatenbank/docs/README.md",
|
||||
])
|
||||
# Upstream-Repos sollten nicht blockiert werden
|
||||
readonly_errors = [e for e in errors if "read-only" in e]
|
||||
assert len(readonly_errors) == 0
|
||||
|
||||
def test_normal_files_are_allowed(self, hook_manager: HookManager):
|
||||
"""Normale Dateien außerhalb von Read-Only-Repos werden durchgelassen."""
|
||||
errors = hook_manager.validate_staged_files([
|
||||
"privat/my-project/src/main.py",
|
||||
"dhive/tooling/config.yaml",
|
||||
])
|
||||
assert len(errors) == 0
|
||||
|
||||
def test_multiple_violations_reported(self, hook_manager: HookManager):
|
||||
"""Mehrere Verstöße werden alle gemeldet."""
|
||||
errors = hook_manager.validate_staged_files([
|
||||
"shared/references/symphony/file1.py",
|
||||
"shared/references/symphony/file2.py",
|
||||
])
|
||||
assert len(errors) == 2
|
||||
|
||||
def test_protected_paths_returned(self, hook_manager: HookManager):
|
||||
"""get_protected_paths gibt die konfigurierten Read-Only-Pfade zurück."""
|
||||
paths = hook_manager.get_protected_paths()
|
||||
assert "shared/references/symphony" in paths
|
||||
# Upstream-Repos sollten NICHT in der geschützten Liste sein
|
||||
assert "bahn/db-wissensdatenbank" not in paths
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Secret-Verschlüsselungs-Prüfung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSecretEncryptionCheck:
|
||||
"""Tests für die Prüfung unverschlüsselter Secrets."""
|
||||
|
||||
def test_env_file_without_encryption_blocked(self, hook_manager: HookManager, monorepo_root: Path):
|
||||
"""Unverschlüsselte .env-Datei wird blockiert (kein Filter konfiguriert)."""
|
||||
# Erstelle eine .env im dhive-Kontext OHNE git-crypt-Filter
|
||||
env_path = monorepo_root / "dhive" / ".env"
|
||||
env_path.write_text("SECRET_KEY=abc123\n", encoding="utf-8")
|
||||
|
||||
errors = hook_manager.validate_staged_files(["dhive/.env"])
|
||||
assert len(errors) == 1
|
||||
assert "nicht verschlüsselt" in errors[0]
|
||||
|
||||
def test_env_file_with_filter_allowed(self, hook_manager: HookManager, monorepo_root: Path):
|
||||
"""Unverschlüsselte .env-Datei mit git-crypt-Filter wird durchgelassen."""
|
||||
# privat/.env hat einen git-crypt-Filter in .gitattributes
|
||||
env_path = monorepo_root / "privat" / ".env"
|
||||
env_path.write_text("MY_SECRET=xyz\n", encoding="utf-8")
|
||||
|
||||
errors = hook_manager.validate_staged_files(["privat/.env"])
|
||||
assert len(errors) == 0
|
||||
|
||||
def test_pem_file_without_filter_blocked(self, hook_manager: HookManager, monorepo_root: Path):
|
||||
"""Unverschlüsselte .pem-Datei ohne Filter wird blockiert."""
|
||||
pem_path = monorepo_root / "dhive" / "cert.pem"
|
||||
pem_path.write_text("-----BEGIN CERTIFICATE-----\n...\n", encoding="utf-8")
|
||||
|
||||
errors = hook_manager.validate_staged_files(["dhive/cert.pem"])
|
||||
assert len(errors) == 1
|
||||
assert "nicht verschlüsselt" in errors[0]
|
||||
|
||||
def test_pem_file_with_filter_allowed(self, hook_manager: HookManager, monorepo_root: Path):
|
||||
"""Unverschlüsselte .pem-Datei mit git-crypt-Filter wird durchgelassen."""
|
||||
pem_path = monorepo_root / "privat" / "server.pem"
|
||||
pem_path.write_text("-----BEGIN RSA PRIVATE KEY-----\n...\n", encoding="utf-8")
|
||||
|
||||
errors = hook_manager.validate_staged_files(["privat/server.pem"])
|
||||
assert len(errors) == 0
|
||||
|
||||
def test_key_file_detected(self, hook_manager: HookManager, monorepo_root: Path):
|
||||
"""*.key-Dateien werden als Secrets erkannt."""
|
||||
key_path = monorepo_root / "dhive" / "private.key"
|
||||
key_path.write_text("key-data\n", encoding="utf-8")
|
||||
|
||||
errors = hook_manager.validate_staged_files(["dhive/private.key"])
|
||||
assert len(errors) == 1
|
||||
|
||||
def test_token_file_detected(self, hook_manager: HookManager, monorepo_root: Path):
|
||||
"""Dateien mit 'token' im Namen werden als Secrets erkannt."""
|
||||
token_path = monorepo_root / "dhive" / "api-token.txt"
|
||||
token_path.write_text("token123\n", encoding="utf-8")
|
||||
|
||||
errors = hook_manager.validate_staged_files(["dhive/api-token.txt"])
|
||||
assert len(errors) == 1
|
||||
|
||||
def test_secret_file_detected(self, hook_manager: HookManager, monorepo_root: Path):
|
||||
"""Dateien mit 'secret' im Namen werden als Secrets erkannt."""
|
||||
secret_path = monorepo_root / "dhive" / "my-secret-config.yaml"
|
||||
secret_path.write_text("data: secret\n", encoding="utf-8")
|
||||
|
||||
errors = hook_manager.validate_staged_files(["dhive/my-secret-config.yaml"])
|
||||
assert len(errors) == 1
|
||||
|
||||
def test_encrypted_file_is_allowed(self, hook_manager: HookManager, monorepo_root: Path):
|
||||
"""Verschlüsselte Dateien (mit git-crypt-Header) werden durchgelassen."""
|
||||
# Simuliere eine git-crypt-verschlüsselte Datei
|
||||
enc_path = monorepo_root / "dhive" / ".env"
|
||||
enc_path.write_bytes(b"\x00GITCRYPT\x00" + b"encrypted-data")
|
||||
|
||||
errors = hook_manager.validate_staged_files(["dhive/.env"])
|
||||
assert len(errors) == 0
|
||||
|
||||
def test_normal_files_not_flagged(self, hook_manager: HookManager, monorepo_root: Path):
|
||||
"""Normale Dateien (nicht Secret-Patterns) werden nicht geprüft."""
|
||||
py_path = monorepo_root / "privat" / "main.py"
|
||||
py_path.write_text("print('hello')\n", encoding="utf-8")
|
||||
|
||||
errors = hook_manager.validate_staged_files(["privat/main.py"])
|
||||
assert len(errors) == 0
|
||||
|
||||
def test_gitattributes_pattern_matching(self, hook_manager: HookManager):
|
||||
"""Pattern-Matching für .gitattributes funktioniert korrekt."""
|
||||
# Interne Methode testen
|
||||
assert hook_manager._pattern_matches_file(".env", ".env")
|
||||
assert hook_manager._pattern_matches_file("*.pem", "server.pem")
|
||||
assert hook_manager._pattern_matches_file("*.key", "private.key")
|
||||
assert hook_manager._pattern_matches_file("*token*", "api-token.txt")
|
||||
assert hook_manager._pattern_matches_file("*secret*", "my-secret-config.yaml")
|
||||
|
||||
# Negative Cases
|
||||
assert not hook_manager._pattern_matches_file(".env", ".envrc")
|
||||
assert not hook_manager._pattern_matches_file("*.pem", "file.txt")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Convenience-Funktionen und CLI-Integration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestConvenienceFunctions:
|
||||
"""Tests für install_hooks/uninstall_hooks Convenience-Funktionen."""
|
||||
|
||||
def test_install_hooks_function(self, monorepo_root: Path):
|
||||
"""install_hooks()-Funktion erstellt den Hook erfolgreich."""
|
||||
install_hooks(monorepo_root)
|
||||
hook_path = monorepo_root / ".git" / "hooks" / "pre-commit"
|
||||
assert hook_path.exists()
|
||||
content = hook_path.read_text(encoding="utf-8")
|
||||
assert "Monorepo Integrated Pre-Commit Hook" in content
|
||||
|
||||
def test_uninstall_hooks_function(self, monorepo_root: Path):
|
||||
"""uninstall_hooks()-Funktion entfernt den Hook."""
|
||||
install_hooks(monorepo_root)
|
||||
uninstall_hooks(monorepo_root)
|
||||
hook_path = monorepo_root / ".git" / "hooks" / "pre-commit"
|
||||
assert not hook_path.exists()
|
||||
|
||||
def test_install_without_access_config(self, tmp_path: Path):
|
||||
"""Installation funktioniert auch ohne access-config.yaml."""
|
||||
root = tmp_path / "bare-mono"
|
||||
root.mkdir()
|
||||
(root / ".git" / "hooks").mkdir(parents=True)
|
||||
(root / "shared" / "config").mkdir(parents=True)
|
||||
|
||||
# Nur repos.yaml, keine access-config.yaml
|
||||
repos = {"repos": []}
|
||||
with open(root / "shared" / "config" / "repos.yaml", "w") as f:
|
||||
yaml.dump(repos, f)
|
||||
|
||||
install_hooks(root)
|
||||
hook_path = root / ".git" / "hooks" / "pre-commit"
|
||||
assert hook_path.exists()
|
||||
|
||||
def test_install_without_repos_yaml(self, tmp_path: Path):
|
||||
"""Installation funktioniert auch ohne repos.yaml (leere Schutzliste)."""
|
||||
root = tmp_path / "bare-mono"
|
||||
root.mkdir()
|
||||
(root / ".git" / "hooks").mkdir(parents=True)
|
||||
(root / "shared" / "config").mkdir(parents=True)
|
||||
|
||||
install_hooks(root)
|
||||
hook_path = root / ".git" / "hooks" / "pre-commit"
|
||||
assert hook_path.exists()
|
||||
content = hook_path.read_text(encoding="utf-8")
|
||||
assert "Keine Read-Only-Repos konfiguriert" in content
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: ContextGuard-Integration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestContextGuardIntegration:
|
||||
"""Tests für die Integration mit dem ContextGuard."""
|
||||
|
||||
def test_hook_manager_with_context_guard(self, monorepo_root: Path):
|
||||
"""HookManager funktioniert mit eingebundenem ContextGuard."""
|
||||
from monorepo.security import ContextGuard
|
||||
|
||||
guard = ContextGuard(monorepo_root)
|
||||
hm = HookManager(
|
||||
monorepo_root=monorepo_root,
|
||||
context_guard=guard,
|
||||
)
|
||||
|
||||
# Normale Datei sollte kein Problem sein
|
||||
errors = hm.validate_staged_files(["privat/project/main.py"])
|
||||
assert len(errors) == 0
|
||||
|
||||
def test_hook_manager_without_context_guard(self, monorepo_root: Path):
|
||||
"""HookManager funktioniert auch ohne ContextGuard."""
|
||||
hm = HookManager(monorepo_root=monorepo_root)
|
||||
errors = hm.validate_staged_files(["privat/project/main.py"])
|
||||
assert len(errors) == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: SecretEncryptionManager-Integration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEncryptionManagerIntegration:
|
||||
"""Tests für die Integration mit dem SecretEncryptionManager."""
|
||||
|
||||
def test_hook_manager_with_encryption_manager(self, monorepo_root: Path):
|
||||
"""HookManager funktioniert mit eingebundenem SecretEncryptionManager."""
|
||||
from monorepo.encryption import SecretEncryptionManager
|
||||
from monorepo.models import MachineContext
|
||||
|
||||
mc = MachineContext(
|
||||
name="test-machine",
|
||||
description="Test",
|
||||
authorized_contexts=["privat", "dhive", "bahn"],
|
||||
key_source="keyring",
|
||||
)
|
||||
enc = SecretEncryptionManager(monorepo_root, mc)
|
||||
|
||||
hm = HookManager(
|
||||
monorepo_root=monorepo_root,
|
||||
encryption_manager=enc,
|
||||
)
|
||||
hm.install_hooks()
|
||||
assert hm.is_hook_installed()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: is_secret_file Erkennung
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSecretFileDetection:
|
||||
"""Tests für die _is_secret_file Methode."""
|
||||
|
||||
def test_env_detected(self, hook_manager: HookManager):
|
||||
"""Standard .env wird erkannt."""
|
||||
assert hook_manager._is_secret_file("privat/.env")
|
||||
assert hook_manager._is_secret_file("dhive/.env")
|
||||
assert hook_manager._is_secret_file(".env")
|
||||
|
||||
def test_env_prefix_detected(self, hook_manager: HookManager):
|
||||
""".env.local und ähnliche werden erkannt."""
|
||||
assert hook_manager._is_secret_file("privat/.env.local")
|
||||
assert hook_manager._is_secret_file(".env.production")
|
||||
|
||||
def test_pem_detected(self, hook_manager: HookManager):
|
||||
""".pem-Dateien werden erkannt."""
|
||||
assert hook_manager._is_secret_file("certs/server.pem")
|
||||
assert hook_manager._is_secret_file("privat/my.pem")
|
||||
|
||||
def test_key_detected(self, hook_manager: HookManager):
|
||||
""".key-Dateien werden erkannt."""
|
||||
assert hook_manager._is_secret_file("privat/ssl.key")
|
||||
assert hook_manager._is_secret_file("private.key")
|
||||
|
||||
def test_token_detected(self, hook_manager: HookManager):
|
||||
"""Dateien mit 'token' werden erkannt."""
|
||||
assert hook_manager._is_secret_file("api-token.txt")
|
||||
assert hook_manager._is_secret_file("github_token")
|
||||
assert hook_manager._is_secret_file("TOKEN_FILE")
|
||||
|
||||
def test_secret_detected(self, hook_manager: HookManager):
|
||||
"""Dateien mit 'secret' werden erkannt."""
|
||||
assert hook_manager._is_secret_file("my-secret.yaml")
|
||||
assert hook_manager._is_secret_file("SECRET_CONFIG")
|
||||
|
||||
def test_normal_files_not_detected(self, hook_manager: HookManager):
|
||||
"""Normale Dateien werden nicht als Secrets erkannt."""
|
||||
assert not hook_manager._is_secret_file("main.py")
|
||||
assert not hook_manager._is_secret_file("README.md")
|
||||
assert not hook_manager._is_secret_file("package.json")
|
||||
assert not hook_manager._is_secret_file("config.yaml")
|
||||
@@ -0,0 +1,360 @@
|
||||
"""Property-basierte Tests: ETL-Fehlerresilienz bewahrt erfolgreiche Artefakte.
|
||||
|
||||
**Validates: Requirements 3.20**
|
||||
|
||||
Property 10: ETL-Fehlerresilienz bewahrt erfolgreiche Artefakte
|
||||
- IF die ETL-Pipeline bei der Verarbeitung einer Quelle fehlschlägt,
|
||||
THEN muss sie: den Fehler mit Quellidentifikator und Zeitstempel protokollieren,
|
||||
bereits erfolgreich verarbeitete Artefakte beibehalten und die fehlgeschlagene
|
||||
Quelle beim nächsten Durchlauf erneut verarbeiten.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
from hypothesis import given, settings
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from monorepo.knowledge.etl import ETLPipeline, IngestResult
|
||||
from monorepo.knowledge.index import YAMLIndex
|
||||
from monorepo.knowledge.sources.markdown import MarkdownSource
|
||||
from monorepo.knowledge.store import KnowledgeStore
|
||||
from monorepo.models import ScopeConfig
|
||||
|
||||
|
||||
# --- Constants ---
|
||||
|
||||
ARTIFACT_TYPES = ["decision", "note", "meeting", "reference", "pattern"]
|
||||
CONTEXTS = ["privat", "dhive", "bahn", "shared"]
|
||||
SAMPLE_TAGS = ["api", "architecture", "python", "devops", "testing", "design"]
|
||||
|
||||
|
||||
# --- Strategies ---
|
||||
|
||||
|
||||
@st.composite
|
||||
def valid_artifact_content(draw: st.DrawFn) -> str:
|
||||
"""Generates content for a valid artifact (with some non-empty text)."""
|
||||
num_lines = draw(st.integers(min_value=1, max_value=5))
|
||||
lines = []
|
||||
for _ in range(num_lines):
|
||||
line = draw(st.text(
|
||||
alphabet=st.characters(whitelist_categories=("L", "N", "Zs")),
|
||||
min_size=5,
|
||||
max_size=50,
|
||||
))
|
||||
lines.append(line)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
@st.composite
|
||||
def valid_artifact_metadata(draw: st.DrawFn) -> dict[str, object]:
|
||||
"""Generates valid YAML frontmatter metadata for an artifact."""
|
||||
title = draw(st.text(
|
||||
alphabet=st.characters(whitelist_categories=("L", "N", "Zs")),
|
||||
min_size=3,
|
||||
max_size=40,
|
||||
))
|
||||
artifact_type = draw(st.sampled_from(ARTIFACT_TYPES))
|
||||
tags = draw(st.lists(st.sampled_from(SAMPLE_TAGS), min_size=1, max_size=3, unique=True))
|
||||
return {
|
||||
"type": artifact_type,
|
||||
"title": title,
|
||||
"tags": tags,
|
||||
}
|
||||
|
||||
|
||||
@st.composite
|
||||
def invalid_file_content(draw: st.DrawFn) -> str:
|
||||
"""Generates content that will fail frontmatter parsing.
|
||||
|
||||
Produces files without valid YAML frontmatter:
|
||||
- No --- delimiters at all (plain text or heading only)
|
||||
- Only one --- delimiter (no closing)
|
||||
|
||||
These are the cases that MarkdownSource handles gracefully
|
||||
(ValueError from parse_frontmatter), keeping other artifacts intact.
|
||||
"""
|
||||
strategy_choice = draw(st.integers(min_value=0, max_value=1))
|
||||
|
||||
if strategy_choice == 0:
|
||||
# No frontmatter at all - just plain text
|
||||
text = draw(st.text(
|
||||
alphabet=st.characters(whitelist_categories=("L", "N", "Zs")),
|
||||
min_size=10,
|
||||
max_size=100,
|
||||
))
|
||||
return f"# Just a heading\n{text}"
|
||||
else:
|
||||
# Only opening delimiter, no closing
|
||||
text = draw(st.text(
|
||||
alphabet=st.characters(whitelist_categories=("L", "N", "Zs")),
|
||||
min_size=10,
|
||||
max_size=50,
|
||||
))
|
||||
return f"---\ntitle: broken\n{text}"
|
||||
|
||||
|
||||
@st.composite
|
||||
def mixed_artifact_batch(draw: st.DrawFn) -> tuple[int, int]:
|
||||
"""Generates counts for valid and invalid artifacts.
|
||||
|
||||
Returns (valid_count, invalid_count) with at least 1 valid and 1 invalid.
|
||||
"""
|
||||
valid_count = draw(st.integers(min_value=1, max_value=5))
|
||||
invalid_count = draw(st.integers(min_value=1, max_value=5))
|
||||
return (valid_count, invalid_count)
|
||||
|
||||
|
||||
# --- Helpers ---
|
||||
|
||||
|
||||
def _create_mixed_source_directory(
|
||||
tmp_dir: Path,
|
||||
valid_count: int,
|
||||
invalid_count: int,
|
||||
draw_valid_meta: list[dict[str, object]],
|
||||
draw_valid_content: list[str],
|
||||
draw_invalid_content: list[str],
|
||||
) -> Path:
|
||||
"""Creates a source directory with a mix of valid and invalid files.
|
||||
|
||||
Valid files have proper YAML frontmatter.
|
||||
Invalid files have corrupted/missing frontmatter.
|
||||
"""
|
||||
source_dir = tmp_dir / "source"
|
||||
source_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create valid files
|
||||
for i in range(valid_count):
|
||||
meta = draw_valid_meta[i]
|
||||
content = draw_valid_content[i]
|
||||
frontmatter = yaml.dump(meta, default_flow_style=False, allow_unicode=True, sort_keys=False)
|
||||
file_content = f"---\n{frontmatter}---\n{content}"
|
||||
file_path = source_dir / f"valid-{i}.md"
|
||||
file_path.write_text(file_content, encoding="utf-8")
|
||||
|
||||
# Create invalid files
|
||||
for i in range(invalid_count):
|
||||
content = draw_invalid_content[i]
|
||||
file_path = source_dir / f"invalid-{i}.md"
|
||||
file_path.write_text(content, encoding="utf-8")
|
||||
|
||||
return source_dir
|
||||
|
||||
|
||||
def _create_knowledge_store(tmp_dir: Path) -> KnowledgeStore:
|
||||
"""Creates a KnowledgeStore in a temporary directory."""
|
||||
store_path = tmp_dir / "knowledge-store"
|
||||
store_path.mkdir(parents=True, exist_ok=True)
|
||||
scope_config = ScopeConfig(scopes={
|
||||
"privat": {"scope": "privat", "paths": ["privat/"]},
|
||||
"dhive": {"scope": "dhive", "paths": ["dhive/"]},
|
||||
"bahn": {"scope": "bahn", "paths": ["bahn/"]},
|
||||
"shared": {"scope": "shared", "paths": ["shared/"]},
|
||||
})
|
||||
return KnowledgeStore(base_path=store_path, scope_config=scope_config)
|
||||
|
||||
|
||||
# --- Tests ---
|
||||
|
||||
|
||||
class TestProperty10ETLErrorResilience:
|
||||
"""Property 10: ETL-Fehlerresilienz bewahrt erfolgreiche Artefakte.
|
||||
|
||||
**Validates: Requirements 3.20**
|
||||
|
||||
If the ETL pipeline fails when processing a source, it must:
|
||||
1. Log the error with source identifier (file path)
|
||||
2. Retain already-successfully-processed artifacts
|
||||
3. Mark the failed source for retry on the next run
|
||||
"""
|
||||
|
||||
@given(
|
||||
counts=mixed_artifact_batch(),
|
||||
context=st.sampled_from(CONTEXTS),
|
||||
)
|
||||
@settings(max_examples=50, deadline=None)
|
||||
def test_valid_artifacts_retained_despite_invalid_files(
|
||||
self,
|
||||
counts: tuple[int, int],
|
||||
context: str,
|
||||
) -> None:
|
||||
"""All N valid artifacts must be present in the index even when
|
||||
M invalid artifacts also exist in the source directory."""
|
||||
valid_count, invalid_count = counts
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="monorepo_pbt_") as tmp:
|
||||
tmp_dir = Path(tmp)
|
||||
store = _create_knowledge_store(tmp_dir)
|
||||
|
||||
# Generate valid artifacts
|
||||
valid_metas = []
|
||||
valid_contents = []
|
||||
for i in range(valid_count):
|
||||
meta = {"type": "note", "title": f"Valid Article {i}", "tags": ["testing"]}
|
||||
valid_metas.append(meta)
|
||||
valid_contents.append(f"Content for valid article {i}\nSome more text here.")
|
||||
|
||||
# Generate invalid file contents (no valid frontmatter)
|
||||
invalid_contents = []
|
||||
for i in range(invalid_count):
|
||||
# Alternate between different kinds of invalid content
|
||||
if i % 2 == 0:
|
||||
invalid_contents.append(f"# No frontmatter\nJust plain text {i}")
|
||||
else:
|
||||
invalid_contents.append(f"---\ntitle: broken {i}\nUnclosed frontmatter")
|
||||
|
||||
source_dir = _create_mixed_source_directory(
|
||||
tmp_dir, valid_count, invalid_count,
|
||||
valid_metas, valid_contents, invalid_contents,
|
||||
)
|
||||
|
||||
# Run the ETL pipeline
|
||||
source = MarkdownSource(directory=source_dir)
|
||||
result = store.ingest(source, context=context)
|
||||
|
||||
# Verify: All N valid artifacts are present in the index
|
||||
index_entries = store.get_index(scope=context)
|
||||
assert len(index_entries) == valid_count, (
|
||||
f"Expected {valid_count} entries in index, got {len(index_entries)}. "
|
||||
f"Valid artifacts must be retained despite {invalid_count} invalid files."
|
||||
)
|
||||
|
||||
@given(
|
||||
counts=mixed_artifact_batch(),
|
||||
context=st.sampled_from(CONTEXTS),
|
||||
)
|
||||
@settings(max_examples=50, deadline=None)
|
||||
def test_errors_list_has_entries_for_invalid_files(
|
||||
self,
|
||||
counts: tuple[int, int],
|
||||
context: str,
|
||||
) -> None:
|
||||
"""The errors list must have M entries, one per invalid file."""
|
||||
valid_count, invalid_count = counts
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="monorepo_pbt_") as tmp:
|
||||
tmp_dir = Path(tmp)
|
||||
store = _create_knowledge_store(tmp_dir)
|
||||
|
||||
# Generate valid artifacts
|
||||
valid_metas = []
|
||||
valid_contents = []
|
||||
for i in range(valid_count):
|
||||
meta = {"type": "decision", "title": f"Good Doc {i}", "tags": ["api"]}
|
||||
valid_metas.append(meta)
|
||||
valid_contents.append(f"Decision content {i}.")
|
||||
|
||||
# Generate invalid file contents
|
||||
invalid_contents = []
|
||||
for i in range(invalid_count):
|
||||
invalid_contents.append(f"# No frontmatter at all for file {i}\nPlain text.")
|
||||
|
||||
source_dir = _create_mixed_source_directory(
|
||||
tmp_dir, valid_count, invalid_count,
|
||||
valid_metas, valid_contents, invalid_contents,
|
||||
)
|
||||
|
||||
# Run the ETL pipeline
|
||||
source = MarkdownSource(directory=source_dir)
|
||||
result = store.ingest(source, context=context)
|
||||
|
||||
# Verify: Errors list has M entries (one per invalid file)
|
||||
assert len(result.errors) == invalid_count, (
|
||||
f"Expected {invalid_count} errors, got {len(result.errors)}. "
|
||||
f"Each invalid file must produce exactly one error entry."
|
||||
)
|
||||
|
||||
@given(
|
||||
counts=mixed_artifact_batch(),
|
||||
context=st.sampled_from(CONTEXTS),
|
||||
)
|
||||
@settings(max_examples=50, deadline=None)
|
||||
def test_each_error_has_file_path(
|
||||
self,
|
||||
counts: tuple[int, int],
|
||||
context: str,
|
||||
) -> None:
|
||||
"""Each error entry must contain a file path identifying the failed source."""
|
||||
valid_count, invalid_count = counts
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="monorepo_pbt_") as tmp:
|
||||
tmp_dir = Path(tmp)
|
||||
store = _create_knowledge_store(tmp_dir)
|
||||
|
||||
valid_metas = []
|
||||
valid_contents = []
|
||||
for i in range(valid_count):
|
||||
meta = {"type": "pattern", "title": f"Pattern {i}", "tags": ["design"]}
|
||||
valid_metas.append(meta)
|
||||
valid_contents.append(f"Pattern description {i}.")
|
||||
|
||||
invalid_contents = []
|
||||
for i in range(invalid_count):
|
||||
invalid_contents.append(f"No valid frontmatter here {i}")
|
||||
|
||||
source_dir = _create_mixed_source_directory(
|
||||
tmp_dir, valid_count, invalid_count,
|
||||
valid_metas, valid_contents, invalid_contents,
|
||||
)
|
||||
|
||||
source = MarkdownSource(directory=source_dir)
|
||||
result = store.ingest(source, context=context)
|
||||
|
||||
# Verify: Each error has a file_path that is non-empty
|
||||
for error in result.errors:
|
||||
assert error.file_path, (
|
||||
f"Error entry must have a non-empty file_path. "
|
||||
f"Got error with empty path: {error}"
|
||||
)
|
||||
# file_path should point to an actual path (contains path separator or filename)
|
||||
assert len(error.file_path) > 0, (
|
||||
f"Error file_path must identify the source. Got: '{error.file_path}'"
|
||||
)
|
||||
|
||||
@given(
|
||||
counts=mixed_artifact_batch(),
|
||||
context=st.sampled_from(CONTEXTS),
|
||||
)
|
||||
@settings(max_examples=50, deadline=None)
|
||||
def test_processed_count_equals_valid_count(
|
||||
self,
|
||||
counts: tuple[int, int],
|
||||
context: str,
|
||||
) -> None:
|
||||
"""result.processed must equal N (only valid ones counted as processed)."""
|
||||
valid_count, invalid_count = counts
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="monorepo_pbt_") as tmp:
|
||||
tmp_dir = Path(tmp)
|
||||
store = _create_knowledge_store(tmp_dir)
|
||||
|
||||
valid_metas = []
|
||||
valid_contents = []
|
||||
for i in range(valid_count):
|
||||
meta = {"type": "reference", "title": f"Ref {i}", "tags": ["cloud"]}
|
||||
valid_metas.append(meta)
|
||||
valid_contents.append(f"Reference material {i}.")
|
||||
|
||||
invalid_contents = []
|
||||
for i in range(invalid_count):
|
||||
invalid_contents.append(f"---\ntitle: unclosed {i}\nBroken file content")
|
||||
|
||||
source_dir = _create_mixed_source_directory(
|
||||
tmp_dir, valid_count, invalid_count,
|
||||
valid_metas, valid_contents, invalid_contents,
|
||||
)
|
||||
|
||||
source = MarkdownSource(directory=source_dir)
|
||||
result = store.ingest(source, context=context)
|
||||
|
||||
# Verify: result.processed == N (only valid ones counted)
|
||||
assert result.processed == valid_count, (
|
||||
f"Expected processed={valid_count} (valid files only), "
|
||||
f"got processed={result.processed}. "
|
||||
f"Invalid files must not be counted as processed."
|
||||
)
|
||||
@@ -0,0 +1,296 @@
|
||||
"""Property-basierte Tests für die Volltextsuche im KnowledgeStore.
|
||||
|
||||
**Validates: Requirements 3.7**
|
||||
|
||||
Property 9: Volltextsuche findet indexierte Inhalte
|
||||
- For any indexed artifact with content, a search query matching part of its
|
||||
title, tags, or summary must return that artifact in the results (provided
|
||||
the artifact's scope is in allowed_scopes).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from hypothesis import given, settings, assume
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from monorepo.knowledge.index import IndexEntry, YAMLIndex
|
||||
from monorepo.knowledge.store import KnowledgeStore, SearchResult
|
||||
from monorepo.models import ScopeConfig
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
ALL_SCOPES = ["privat", "dhive", "bahn", "shared"]
|
||||
|
||||
ARTIFACT_TYPES = ["decision", "note", "meeting", "reference", "pattern"]
|
||||
|
||||
# Words that form valid, searchable content (lowercase, no whitespace)
|
||||
SAMPLE_WORDS = [
|
||||
"api", "kubernetes", "docker", "python", "design",
|
||||
"architecture", "cloud", "deployment", "service", "testing",
|
||||
"database", "security", "network", "monitoring", "logging",
|
||||
"pipeline", "migration", "config", "infra", "tooling",
|
||||
"workflow", "automation", "integration", "analytics", "platform",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@st.composite
|
||||
def index_entry_strategy(draw: st.DrawFn) -> IndexEntry:
|
||||
"""Generates a random but well-formed IndexEntry."""
|
||||
scope = draw(st.sampled_from(ALL_SCOPES))
|
||||
artifact_type = draw(st.sampled_from(ARTIFACT_TYPES))
|
||||
|
||||
# Generate title from sample words (guaranteed searchable content)
|
||||
title_words = draw(st.lists(
|
||||
st.sampled_from(SAMPLE_WORDS), min_size=1, max_size=4
|
||||
))
|
||||
title = " ".join(title_words).title()
|
||||
|
||||
# Generate tags from sample words
|
||||
tags = draw(st.lists(
|
||||
st.sampled_from(SAMPLE_WORDS), min_size=1, max_size=5, unique=True
|
||||
))
|
||||
|
||||
# Generate summary from sample words
|
||||
summary_words = draw(st.lists(
|
||||
st.sampled_from(SAMPLE_WORDS), min_size=2, max_size=8
|
||||
))
|
||||
summary = " ".join(summary_words)
|
||||
|
||||
# Generate unique ID
|
||||
name_suffix = draw(st.integers(min_value=1, max_value=9999))
|
||||
entry_id = f"{scope}/{artifact_type}s/entry-{name_suffix}"
|
||||
|
||||
return IndexEntry(
|
||||
id=entry_id,
|
||||
title=title,
|
||||
type=artifact_type,
|
||||
tags=tags,
|
||||
scope=scope,
|
||||
summary=summary,
|
||||
path=f"{entry_id}.md",
|
||||
content_hash=f"sha256:{draw(st.text(alphabet='0123456789abcdef', min_size=8, max_size=8))}",
|
||||
links=[],
|
||||
)
|
||||
|
||||
|
||||
@st.composite
|
||||
def entry_with_query_from_title(draw: st.DrawFn) -> tuple[IndexEntry, str]:
|
||||
"""Generates an IndexEntry and a query substring drawn from its title."""
|
||||
entry = draw(index_entry_strategy())
|
||||
# The title is composed of sample words joined with spaces and title-cased.
|
||||
# Pick a substring: use a word that appears in the title (lowercased for search).
|
||||
title_lower = entry.title.lower()
|
||||
# Pick a starting position and extract a substring of at least 3 chars
|
||||
assume(len(title_lower) >= 3)
|
||||
start = draw(st.integers(min_value=0, max_value=len(title_lower) - 3))
|
||||
end = draw(st.integers(min_value=start + 3, max_value=len(title_lower)))
|
||||
query = title_lower[start:end]
|
||||
# Ensure the query is non-empty and non-whitespace
|
||||
assume(query.strip() != "")
|
||||
return entry, query.strip()
|
||||
|
||||
|
||||
@st.composite
|
||||
def entry_with_query_from_tag(draw: st.DrawFn) -> tuple[IndexEntry, str]:
|
||||
"""Generates an IndexEntry and a query substring drawn from one of its tags."""
|
||||
entry = draw(index_entry_strategy())
|
||||
assume(len(entry.tags) > 0)
|
||||
tag = draw(st.sampled_from(entry.tags))
|
||||
# Tags are single words from SAMPLE_WORDS - use as query directly
|
||||
assume(len(tag) >= 2)
|
||||
return entry, tag
|
||||
|
||||
|
||||
@st.composite
|
||||
def entry_with_query_from_summary(draw: st.DrawFn) -> tuple[IndexEntry, str]:
|
||||
"""Generates an IndexEntry and a query substring drawn from its summary."""
|
||||
entry = draw(index_entry_strategy())
|
||||
summary_lower = entry.summary.lower()
|
||||
assume(len(summary_lower) >= 3)
|
||||
start = draw(st.integers(min_value=0, max_value=len(summary_lower) - 3))
|
||||
end = draw(st.integers(min_value=start + 3, max_value=len(summary_lower)))
|
||||
query = summary_lower[start:end]
|
||||
assume(query.strip() != "")
|
||||
return entry, query.strip()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _build_scope_config() -> ScopeConfig:
|
||||
"""Creates a standard ScopeConfig for testing."""
|
||||
return ScopeConfig(scopes={
|
||||
"privat": {"scope": "privat", "paths": ["privat/"]},
|
||||
"dhive": {"scope": "dhive", "paths": ["dhive/"]},
|
||||
"bahn": {"scope": "bahn", "paths": ["bahn/"]},
|
||||
"shared": {"scope": "shared", "paths": ["shared/"]},
|
||||
})
|
||||
|
||||
|
||||
def _build_store(entries: list[IndexEntry]) -> KnowledgeStore:
|
||||
"""Creates a KnowledgeStore populated with given entries in-memory."""
|
||||
scope_config = _build_scope_config()
|
||||
tmp = Path(tempfile.mkdtemp())
|
||||
store = KnowledgeStore(tmp, scope_config)
|
||||
for entry in entries:
|
||||
store.index.update_entry(entry)
|
||||
return store
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestProperty9FulltextSearchFindsIndexedContent:
|
||||
"""Property 9: Volltextsuche findet indexierte Inhalte.
|
||||
|
||||
**Validates: Requirements 3.7**
|
||||
|
||||
For any indexed artifact with content, a search query matching part
|
||||
of its title, tags, or summary must return that artifact in the results
|
||||
(when the artifact's scope is in allowed_scopes).
|
||||
"""
|
||||
|
||||
@given(data=st.data())
|
||||
@settings(max_examples=200)
|
||||
def test_search_finds_entry_by_title_substring(self, data: st.DataObject) -> None:
|
||||
"""A substring from an entry's title must return that entry in results.
|
||||
|
||||
**Validates: Requirements 3.7**
|
||||
"""
|
||||
entry, query = data.draw(entry_with_query_from_title())
|
||||
# Include the entry's scope in allowed_scopes so it's findable
|
||||
allowed_scopes = [entry.scope]
|
||||
|
||||
store = _build_store([entry])
|
||||
results = store.search(query, allowed_scopes)
|
||||
|
||||
result_ids = {r.entry.id for r in results}
|
||||
assert entry.id in result_ids, (
|
||||
f"Entry '{entry.id}' with title '{entry.title}' was NOT found "
|
||||
f"when searching for query '{query}' (from title). "
|
||||
f"Allowed scopes: {allowed_scopes}, entry scope: {entry.scope}"
|
||||
)
|
||||
|
||||
@given(data=st.data())
|
||||
@settings(max_examples=200)
|
||||
def test_search_finds_entry_by_tag(self, data: st.DataObject) -> None:
|
||||
"""A tag from an entry must return that entry in search results.
|
||||
|
||||
**Validates: Requirements 3.7**
|
||||
"""
|
||||
entry, query = data.draw(entry_with_query_from_tag())
|
||||
allowed_scopes = [entry.scope]
|
||||
|
||||
store = _build_store([entry])
|
||||
results = store.search(query, allowed_scopes)
|
||||
|
||||
result_ids = {r.entry.id for r in results}
|
||||
assert entry.id in result_ids, (
|
||||
f"Entry '{entry.id}' with tags {entry.tags} was NOT found "
|
||||
f"when searching for query '{query}' (from tags). "
|
||||
f"Allowed scopes: {allowed_scopes}, entry scope: {entry.scope}"
|
||||
)
|
||||
|
||||
@given(data=st.data())
|
||||
@settings(max_examples=200)
|
||||
def test_search_finds_entry_by_summary_substring(self, data: st.DataObject) -> None:
|
||||
"""A substring from an entry's summary must return that entry in results.
|
||||
|
||||
**Validates: Requirements 3.7**
|
||||
"""
|
||||
entry, query = data.draw(entry_with_query_from_summary())
|
||||
allowed_scopes = [entry.scope]
|
||||
|
||||
store = _build_store([entry])
|
||||
results = store.search(query, allowed_scopes)
|
||||
|
||||
result_ids = {r.entry.id for r in results}
|
||||
assert entry.id in result_ids, (
|
||||
f"Entry '{entry.id}' with summary '{entry.summary}' was NOT found "
|
||||
f"when searching for query '{query}' (from summary). "
|
||||
f"Allowed scopes: {allowed_scopes}, entry scope: {entry.scope}"
|
||||
)
|
||||
|
||||
@given(data=st.data())
|
||||
@settings(max_examples=200)
|
||||
def test_search_finds_entry_when_scope_included(self, data: st.DataObject) -> None:
|
||||
"""When multiple entries exist, an entry is found if its scope is allowed
|
||||
and the query matches its content.
|
||||
|
||||
**Validates: Requirements 3.7**
|
||||
"""
|
||||
# Generate multiple entries with various scopes
|
||||
entries = data.draw(st.lists(
|
||||
index_entry_strategy(), min_size=3, max_size=20, unique_by=lambda e: e.id
|
||||
))
|
||||
assume(len(entries) >= 3)
|
||||
|
||||
# Pick a target entry and derive a query from its title
|
||||
target_idx = data.draw(st.integers(min_value=0, max_value=len(entries) - 1))
|
||||
target = entries[target_idx]
|
||||
|
||||
# Use first word of title as query (guaranteed to be in title)
|
||||
title_lower = target.title.lower()
|
||||
first_word = title_lower.split()[0] if title_lower.split() else title_lower
|
||||
assume(len(first_word) >= 3)
|
||||
|
||||
# Include target's scope in allowed_scopes
|
||||
allowed_scopes = list({target.scope} | {
|
||||
e.scope for e in data.draw(
|
||||
st.lists(st.sampled_from(entries), min_size=0, max_size=3)
|
||||
)
|
||||
})
|
||||
|
||||
store = _build_store(entries)
|
||||
results = store.search(first_word, allowed_scopes)
|
||||
|
||||
result_ids = {r.entry.id for r in results}
|
||||
assert target.id in result_ids, (
|
||||
f"Target entry '{target.id}' (scope='{target.scope}', title='{target.title}') "
|
||||
f"was NOT found when searching for '{first_word}' with "
|
||||
f"allowed_scopes={allowed_scopes}."
|
||||
)
|
||||
|
||||
@given(data=st.data())
|
||||
@settings(max_examples=200)
|
||||
def test_search_excludes_entry_when_scope_not_included(self, data: st.DataObject) -> None:
|
||||
"""Even when content matches, entries are excluded if their scope is not allowed.
|
||||
|
||||
This verifies the interaction between fulltext search and scope filtering:
|
||||
The search finds the content, but the scope gate prevents it from appearing.
|
||||
|
||||
**Validates: Requirements 3.7**
|
||||
"""
|
||||
entry, query = data.draw(entry_with_query_from_title())
|
||||
|
||||
# Use scopes that do NOT include the entry's scope
|
||||
other_scopes = [s for s in ALL_SCOPES if s != entry.scope]
|
||||
assume(len(other_scopes) > 0)
|
||||
allowed_scopes = data.draw(st.lists(
|
||||
st.sampled_from(other_scopes), min_size=1, max_size=len(other_scopes), unique=True
|
||||
))
|
||||
|
||||
store = _build_store([entry])
|
||||
results = store.search(query, allowed_scopes)
|
||||
|
||||
result_ids = {r.entry.id for r in results}
|
||||
assert entry.id not in result_ids, (
|
||||
f"Entry '{entry.id}' (scope='{entry.scope}') appeared in results "
|
||||
f"even though allowed_scopes={allowed_scopes} does not include "
|
||||
f"'{entry.scope}'. Query was '{query}'."
|
||||
)
|
||||
@@ -0,0 +1,312 @@
|
||||
"""Property-basierte Tests für den Wissensspeicher: Graph-Verknüpfungen bidirektional.
|
||||
|
||||
**Validates: Requirements 3.8**
|
||||
|
||||
Property 11: Graph-Verknüpfungen werden in beiden Artefakten reflektiert
|
||||
- For any zwei Artefakte, wenn eine gerichtete Verknüpfung von A nach B erstellt wird,
|
||||
muss die Kante sowohl im YAML-Frontmatter von A als auch im Frontmatter von B erscheinen.
|
||||
- Die Verlinkung ist bidirektional im Index (source.links enthält target_id, target.links enthält source_id).
|
||||
- Wiederholtes Aufrufen von link_artifacts ist idempotent (keine Duplikat-Links).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from hypothesis import given, settings
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from monorepo.knowledge.index import IndexEntry, YAMLIndex
|
||||
from monorepo.knowledge.store import KnowledgeStore
|
||||
from monorepo.models import ArtifactType, Context, ScopeConfig
|
||||
|
||||
|
||||
# --- Constants ---
|
||||
|
||||
VALID_ARTIFACT_TYPES = [t.value for t in ArtifactType]
|
||||
VALID_SCOPES = [c.value for c in Context]
|
||||
|
||||
VALID_RELATIONS = [
|
||||
"implements",
|
||||
"references",
|
||||
"extends",
|
||||
"relates-to",
|
||||
"depends-on",
|
||||
"contradicts",
|
||||
"supersedes",
|
||||
]
|
||||
|
||||
|
||||
# --- Strategies ---
|
||||
|
||||
|
||||
@st.composite
|
||||
def valid_entry_id(draw: st.DrawFn) -> str:
|
||||
"""Generates a valid artifact ID in scope/type/name format."""
|
||||
scope = draw(st.sampled_from(VALID_SCOPES))
|
||||
artifact_type = draw(st.sampled_from(VALID_ARTIFACT_TYPES))
|
||||
name = draw(st.from_regex(r"[a-z][a-z0-9\-]{1,15}[a-z0-9]", fullmatch=True))
|
||||
return f"{scope}/{artifact_type}/{name}"
|
||||
|
||||
|
||||
@st.composite
|
||||
def valid_index_entry(draw: st.DrawFn) -> IndexEntry:
|
||||
"""Generates a valid IndexEntry with unique ID."""
|
||||
entry_id = draw(valid_entry_id())
|
||||
title = draw(
|
||||
st.text(
|
||||
alphabet=st.characters(whitelist_categories=("L", "N", "Z"), whitelist_characters="-_ "),
|
||||
min_size=3,
|
||||
max_size=40,
|
||||
).filter(lambda t: t.strip() != "")
|
||||
)
|
||||
tags = draw(
|
||||
st.lists(
|
||||
st.from_regex(r"[a-z][a-z0-9]{1,10}", fullmatch=True),
|
||||
min_size=0,
|
||||
max_size=3,
|
||||
unique=True,
|
||||
)
|
||||
)
|
||||
# Extract scope from the ID
|
||||
scope = entry_id.split("/")[0]
|
||||
artifact_type = entry_id.split("/")[1]
|
||||
|
||||
return IndexEntry(
|
||||
id=entry_id,
|
||||
title=title,
|
||||
type=artifact_type,
|
||||
tags=tags,
|
||||
scope=scope,
|
||||
summary=f"Summary for {title[:20]}",
|
||||
path=f"{entry_id}.md",
|
||||
content_hash=f"sha256:{'a' * 64}",
|
||||
links=[],
|
||||
)
|
||||
|
||||
|
||||
@st.composite
|
||||
def valid_entries_with_link_pairs(
|
||||
draw: st.DrawFn,
|
||||
) -> tuple[list[IndexEntry], list[tuple[int, int, str]]]:
|
||||
"""Generates N random IndexEntry objects and random link pairs from them.
|
||||
|
||||
Returns:
|
||||
A tuple of (entries, link_pairs) where link_pairs is a list of
|
||||
(source_index, target_index, relation) tuples.
|
||||
"""
|
||||
# Generate between 2 and 8 unique entries
|
||||
entries = draw(
|
||||
st.lists(valid_index_entry(), min_size=2, max_size=8, unique_by=lambda e: e.id)
|
||||
)
|
||||
n = len(entries)
|
||||
|
||||
# Generate random link pairs (source_idx, target_idx) from the entries
|
||||
# Ensure source != target
|
||||
possible_pairs = [(i, j) for i in range(n) for j in range(n) if i != j]
|
||||
|
||||
if not possible_pairs:
|
||||
return entries, []
|
||||
|
||||
num_links = draw(st.integers(min_value=1, max_value=min(len(possible_pairs), 6)))
|
||||
selected_pairs = draw(
|
||||
st.lists(
|
||||
st.sampled_from(possible_pairs),
|
||||
min_size=num_links,
|
||||
max_size=num_links,
|
||||
)
|
||||
)
|
||||
relations = draw(
|
||||
st.lists(
|
||||
st.sampled_from(VALID_RELATIONS),
|
||||
min_size=num_links,
|
||||
max_size=num_links,
|
||||
)
|
||||
)
|
||||
|
||||
link_pairs = [
|
||||
(pair[0], pair[1], rel) for pair, rel in zip(selected_pairs, relations)
|
||||
]
|
||||
return entries, link_pairs
|
||||
|
||||
|
||||
def _make_scope_config() -> ScopeConfig:
|
||||
"""Creates a standard ScopeConfig for testing."""
|
||||
return ScopeConfig(
|
||||
scopes={
|
||||
"privat": {"scope": "privat", "paths": ["privat/"]},
|
||||
"dhive": {"scope": "dhive", "paths": ["dhive/"]},
|
||||
"bahn": {"scope": "bahn", "paths": ["bahn/"]},
|
||||
"shared": {"scope": "shared", "paths": ["shared/"]},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# --- Property Tests ---
|
||||
|
||||
|
||||
class TestProperty11GraphLinksBidirectional:
|
||||
"""Property 11: Graph-Verknüpfungen werden in beiden Artefakten reflektiert.
|
||||
|
||||
**Validates: Requirements 3.8**
|
||||
|
||||
For any two artifacts, when a directed link from A to B is created:
|
||||
1. source_entry.links must contain target_id
|
||||
2. target_entry.links must contain source_id (bidirectional)
|
||||
3. Calling link_artifacts again is idempotent (no duplicate links)
|
||||
4. Works with various relation types
|
||||
"""
|
||||
|
||||
@given(data=valid_entries_with_link_pairs())
|
||||
@settings(max_examples=100, deadline=10000)
|
||||
def test_link_artifacts_bidirectional(
|
||||
self,
|
||||
data: tuple[list[IndexEntry], list[tuple[int, int, str]]],
|
||||
) -> None:
|
||||
"""After linking, both artifacts' index entries contain each other's ID."""
|
||||
entries, link_pairs = data
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tmp = Path(tmpdir)
|
||||
store = KnowledgeStore(tmp, _make_scope_config())
|
||||
|
||||
# Add all entries to the store
|
||||
for entry in entries:
|
||||
store.index.update_entry(entry)
|
||||
|
||||
# Apply all link pairs
|
||||
for source_idx, target_idx, relation in link_pairs:
|
||||
source_id = entries[source_idx].id
|
||||
target_id = entries[target_idx].id
|
||||
|
||||
store.link_artifacts(source_id, target_id, relation)
|
||||
|
||||
# Verify: source_entry.links contains target_id
|
||||
source_entry = store.index.get_entry(source_id)
|
||||
assert source_entry is not None, (
|
||||
f"Source entry '{source_id}' not found in index after linking"
|
||||
)
|
||||
assert target_id in source_entry.links, (
|
||||
f"Target '{target_id}' not found in source '{source_id}' links. "
|
||||
f"Links: {source_entry.links}"
|
||||
)
|
||||
|
||||
# Verify: target_entry.links contains source_id (bidirectional)
|
||||
target_entry = store.index.get_entry(target_id)
|
||||
assert target_entry is not None, (
|
||||
f"Target entry '{target_id}' not found in index after linking"
|
||||
)
|
||||
assert source_id in target_entry.links, (
|
||||
f"Source '{source_id}' not found in target '{target_id}' links "
|
||||
f"(bidirectional check failed). Links: {target_entry.links}"
|
||||
)
|
||||
|
||||
@given(data=valid_entries_with_link_pairs())
|
||||
@settings(max_examples=100, deadline=10000)
|
||||
def test_link_artifacts_idempotent(
|
||||
self,
|
||||
data: tuple[list[IndexEntry], list[tuple[int, int, str]]],
|
||||
) -> None:
|
||||
"""Calling link_artifacts multiple times does not create duplicate links."""
|
||||
entries, link_pairs = data
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tmp = Path(tmpdir)
|
||||
store = KnowledgeStore(tmp, _make_scope_config())
|
||||
|
||||
# Add all entries to the store
|
||||
for entry in entries:
|
||||
store.index.update_entry(entry)
|
||||
|
||||
# Apply each link pair TWICE to test idempotency
|
||||
for source_idx, target_idx, relation in link_pairs:
|
||||
source_id = entries[source_idx].id
|
||||
target_id = entries[target_idx].id
|
||||
|
||||
store.link_artifacts(source_id, target_id, relation)
|
||||
store.link_artifacts(source_id, target_id, relation)
|
||||
|
||||
# Verify: no duplicate links in any entry
|
||||
for entry in entries:
|
||||
current = store.index.get_entry(entry.id)
|
||||
assert current is not None
|
||||
# Each link target should appear at most once
|
||||
assert len(current.links) == len(set(current.links)), (
|
||||
f"Duplicate links found in entry '{entry.id}': {current.links}"
|
||||
)
|
||||
|
||||
@given(
|
||||
entry_a=valid_index_entry(),
|
||||
entry_b=valid_index_entry(),
|
||||
relation=st.sampled_from(VALID_RELATIONS),
|
||||
)
|
||||
@settings(max_examples=100, deadline=10000)
|
||||
def test_link_artifacts_various_relations(
|
||||
self,
|
||||
entry_a: IndexEntry,
|
||||
entry_b: IndexEntry,
|
||||
relation: str,
|
||||
) -> None:
|
||||
"""Bidirectional linking works correctly with all supported relation types."""
|
||||
# Ensure distinct IDs
|
||||
if entry_a.id == entry_b.id:
|
||||
return # skip when IDs collide (rare due to generation)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tmp = Path(tmpdir)
|
||||
store = KnowledgeStore(tmp, _make_scope_config())
|
||||
|
||||
store.index.update_entry(entry_a)
|
||||
store.index.update_entry(entry_b)
|
||||
|
||||
store.link_artifacts(entry_a.id, entry_b.id, relation)
|
||||
|
||||
# Verify bidirectional
|
||||
a = store.index.get_entry(entry_a.id)
|
||||
b = store.index.get_entry(entry_b.id)
|
||||
assert a is not None and entry_b.id in a.links
|
||||
assert b is not None and entry_a.id in b.links
|
||||
|
||||
@given(data=valid_entries_with_link_pairs())
|
||||
@settings(max_examples=50, deadline=10000)
|
||||
def test_link_count_consistent(
|
||||
self,
|
||||
data: tuple[list[IndexEntry], list[tuple[int, int, str]]],
|
||||
) -> None:
|
||||
"""After linking, the total link references are consistent across all entries.
|
||||
|
||||
For every link (A -> B), both A and B must have each other in their links.
|
||||
This verifies the bidirectional invariant holds for the entire graph.
|
||||
"""
|
||||
entries, link_pairs = data
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tmp = Path(tmpdir)
|
||||
store = KnowledgeStore(tmp, _make_scope_config())
|
||||
|
||||
for entry in entries:
|
||||
store.index.update_entry(entry)
|
||||
|
||||
# Apply all links
|
||||
for source_idx, target_idx, relation in link_pairs:
|
||||
source_id = entries[source_idx].id
|
||||
target_id = entries[target_idx].id
|
||||
store.link_artifacts(source_id, target_id, relation)
|
||||
|
||||
# Verify global bidirectional invariant:
|
||||
# For every entry X, if Y is in X.links, then X must be in Y.links
|
||||
for entry in entries:
|
||||
current = store.index.get_entry(entry.id)
|
||||
assert current is not None
|
||||
for linked_id in current.links:
|
||||
linked_entry = store.index.get_entry(linked_id)
|
||||
assert linked_entry is not None, (
|
||||
f"Linked entry '{linked_id}' from '{entry.id}' not found in index"
|
||||
)
|
||||
assert entry.id in linked_entry.links, (
|
||||
f"Bidirectional invariant violated: "
|
||||
f"'{entry.id}' links to '{linked_id}', "
|
||||
f"but '{linked_id}' does not link back. "
|
||||
f"'{linked_id}'.links = {linked_entry.links}"
|
||||
)
|
||||
@@ -0,0 +1,292 @@
|
||||
"""Property-basierte Tests für inkrementelle Verarbeitung der ETL-Pipeline.
|
||||
|
||||
**Validates: Requirements 3.12**
|
||||
|
||||
Property 8: Inkrementelle Verarbeitung erkennt Änderungen über Content-Hash
|
||||
- For any source, if the content hash has not changed since the last processing,
|
||||
no update must occur. If the hash has changed, exactly the affected artifact
|
||||
must be updated.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import string
|
||||
from pathlib import Path
|
||||
|
||||
from hypothesis import given, settings, assume
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from monorepo.knowledge.etl import ETLPipeline, IngestResult
|
||||
from monorepo.knowledge.index import YAMLIndex
|
||||
from monorepo.knowledge.sources.markdown import MarkdownSource
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Characters suitable for artifact names (kebab-case, simple)
|
||||
NAME_CHARS = string.ascii_lowercase + string.digits
|
||||
|
||||
# Valid artifact types
|
||||
ARTIFACT_TYPES = ["note", "decision", "pattern", "reference", "meeting"]
|
||||
|
||||
|
||||
@st.composite
|
||||
def artifact_content(draw: st.DrawFn) -> str:
|
||||
"""Generate non-empty markdown content for an artifact body."""
|
||||
# Generate 1-5 lines of text content
|
||||
lines = draw(st.lists(
|
||||
st.text(
|
||||
alphabet=st.characters(whitelist_categories=("L", "N", "P", "Z"), min_codepoint=32, max_codepoint=126),
|
||||
min_size=5,
|
||||
max_size=80,
|
||||
),
|
||||
min_size=1,
|
||||
max_size=5,
|
||||
))
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
@st.composite
|
||||
def artifact_name(draw: st.DrawFn) -> str:
|
||||
"""Generate a unique artifact filename stem (kebab-case, 3-20 chars)."""
|
||||
length = draw(st.integers(min_value=3, max_value=15))
|
||||
name = draw(st.text(alphabet=NAME_CHARS, min_size=length, max_size=length))
|
||||
return name
|
||||
|
||||
|
||||
@st.composite
|
||||
def artifact_spec(draw: st.DrawFn) -> dict[str, str]:
|
||||
"""Generate a specification for an artifact: name, type, title, content."""
|
||||
name = draw(artifact_name())
|
||||
art_type = draw(st.sampled_from(ARTIFACT_TYPES))
|
||||
title = f"Title {name}"
|
||||
tags = draw(st.lists(st.sampled_from(["test", "prop", "etl", "data"]), min_size=0, max_size=3))
|
||||
content = draw(artifact_content())
|
||||
return {
|
||||
"name": name,
|
||||
"type": art_type,
|
||||
"title": title,
|
||||
"tags": tags,
|
||||
"content": content,
|
||||
}
|
||||
|
||||
|
||||
@st.composite
|
||||
def artifact_set(draw: st.DrawFn, min_size: int = 2, max_size: int = 8) -> list[dict[str, str]]:
|
||||
"""Generate a set of unique artifacts (unique names)."""
|
||||
count = draw(st.integers(min_value=min_size, max_value=max_size))
|
||||
specs: list[dict[str, str]] = []
|
||||
names_used: set[str] = set()
|
||||
|
||||
for _ in range(count):
|
||||
spec = draw(artifact_spec())
|
||||
# Ensure unique names
|
||||
attempt = 0
|
||||
while spec["name"] in names_used and attempt < 20:
|
||||
spec = draw(artifact_spec())
|
||||
attempt += 1
|
||||
if spec["name"] not in names_used:
|
||||
names_used.add(spec["name"])
|
||||
specs.append(spec)
|
||||
|
||||
assume(len(specs) >= min_size)
|
||||
return specs
|
||||
|
||||
|
||||
def _write_artifact_file(directory: Path, spec: dict[str, str]) -> Path:
|
||||
"""Write a markdown artifact file based on spec dict."""
|
||||
tags_str = ", ".join(spec["tags"]) if spec["tags"] else ""
|
||||
frontmatter = (
|
||||
f"---\n"
|
||||
f"type: {spec['type']}\n"
|
||||
f"title: \"{spec['title']}\"\n"
|
||||
f"tags: [{tags_str}]\n"
|
||||
f"---\n"
|
||||
)
|
||||
file_path = directory / f"{spec['name']}.md"
|
||||
file_path.write_text(frontmatter + spec["content"], encoding="utf-8")
|
||||
return file_path
|
||||
|
||||
|
||||
def _setup_pipeline(tmp_path: Path) -> tuple[ETLPipeline, YAMLIndex, Path]:
|
||||
"""Create an ETLPipeline with a fresh index and store path."""
|
||||
store_path = tmp_path / "store"
|
||||
store_path.mkdir(parents=True, exist_ok=True)
|
||||
index = YAMLIndex(store_path / "_index.yaml")
|
||||
index.load()
|
||||
pipeline = ETLPipeline(index=index, store_path=store_path)
|
||||
return pipeline, index, store_path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property 8 Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestProperty8IncrementalProcessing:
|
||||
"""Property 8: Inkrementelle Verarbeitung erkennt Änderungen über Content-Hash.
|
||||
|
||||
**Validates: Requirements 3.12**
|
||||
"""
|
||||
|
||||
@given(specs=artifact_set(min_size=2, max_size=8))
|
||||
@settings(max_examples=50, deadline=None)
|
||||
def test_unchanged_content_skips_all(self, specs: list[dict[str, str]], tmp_path_factory) -> None:
|
||||
"""For any N artifacts ingested, re-running ingestion with SAME content
|
||||
must yield updated==0 and skipped==N."""
|
||||
tmp_path = tmp_path_factory.mktemp("unchanged")
|
||||
source_dir = tmp_path / "source"
|
||||
source_dir.mkdir()
|
||||
|
||||
# Write N artifacts
|
||||
for spec in specs:
|
||||
_write_artifact_file(source_dir, spec)
|
||||
|
||||
pipeline, index, store_path = _setup_pipeline(tmp_path)
|
||||
n = len(specs)
|
||||
|
||||
# First ingestion: all N should be updated
|
||||
source1 = MarkdownSource(directory=source_dir)
|
||||
result1 = pipeline.ingest(source1, "bahn")
|
||||
assert result1.processed == n
|
||||
assert result1.updated == n
|
||||
assert result1.skipped == 0
|
||||
|
||||
# Second ingestion (same content): all N should be skipped
|
||||
source2 = MarkdownSource(directory=source_dir)
|
||||
result2 = pipeline.ingest(source2, "bahn")
|
||||
assert result2.processed == n, (
|
||||
f"Expected {n} processed, got {result2.processed}"
|
||||
)
|
||||
assert result2.updated == 0, (
|
||||
f"Expected 0 updated (content unchanged), got {result2.updated}"
|
||||
)
|
||||
assert result2.skipped == n, (
|
||||
f"Expected {n} skipped, got {result2.skipped}"
|
||||
)
|
||||
|
||||
@given(
|
||||
specs=artifact_set(min_size=3, max_size=8),
|
||||
modification_seed=st.randoms(use_true_random=False),
|
||||
)
|
||||
@settings(max_examples=50, deadline=None)
|
||||
def test_modified_subset_updates_only_changed(
|
||||
self, specs: list[dict[str, str]], modification_seed, tmp_path_factory
|
||||
) -> None:
|
||||
"""For any N artifacts, modifying a random subset M means:
|
||||
re-ingestion yields updated==M, skipped==(N-M)."""
|
||||
tmp_path = tmp_path_factory.mktemp("modified")
|
||||
source_dir = tmp_path / "source"
|
||||
source_dir.mkdir()
|
||||
|
||||
# Write N artifacts
|
||||
for spec in specs:
|
||||
_write_artifact_file(source_dir, spec)
|
||||
|
||||
pipeline, index, store_path = _setup_pipeline(tmp_path)
|
||||
n = len(specs)
|
||||
|
||||
# First ingestion
|
||||
source1 = MarkdownSource(directory=source_dir)
|
||||
result1 = pipeline.ingest(source1, "bahn")
|
||||
assert result1.processed == n
|
||||
assert result1.updated == n
|
||||
|
||||
# Choose a random non-empty subset to modify (at least 1, at most N-1)
|
||||
# This ensures we have both modified and unmodified artifacts
|
||||
max_modify = max(1, n - 1)
|
||||
m = modification_seed.randint(1, max_modify)
|
||||
indices_to_modify = modification_seed.sample(range(n), m)
|
||||
|
||||
# Modify the chosen subset by changing their content
|
||||
for idx in indices_to_modify:
|
||||
spec = specs[idx]
|
||||
# Change the content to something different
|
||||
new_content = f"Modified content for {spec['name']} - unique {modification_seed.randint(1, 999999)}\n"
|
||||
spec_modified = dict(spec)
|
||||
spec_modified["content"] = new_content
|
||||
_write_artifact_file(source_dir, spec_modified)
|
||||
|
||||
# Second ingestion
|
||||
source2 = MarkdownSource(directory=source_dir)
|
||||
result2 = pipeline.ingest(source2, "bahn")
|
||||
|
||||
assert result2.processed == n, (
|
||||
f"Expected {n} processed, got {result2.processed}"
|
||||
)
|
||||
assert result2.updated == m, (
|
||||
f"Expected {m} updated (modified subset), got {result2.updated}"
|
||||
)
|
||||
assert result2.skipped == n - m, (
|
||||
f"Expected {n - m} skipped, got {result2.skipped}"
|
||||
)
|
||||
|
||||
@given(
|
||||
specs=artifact_set(min_size=3, max_size=8),
|
||||
modification_seed=st.randoms(use_true_random=False),
|
||||
)
|
||||
@settings(max_examples=50, deadline=None)
|
||||
def test_only_modified_artifacts_have_changed_hash(
|
||||
self, specs: list[dict[str, str]], modification_seed, tmp_path_factory
|
||||
) -> None:
|
||||
"""After modifying a subset, only those artifacts' content_hash in the
|
||||
index must have changed. Unmodified artifacts retain their original hash."""
|
||||
tmp_path = tmp_path_factory.mktemp("hash_check")
|
||||
source_dir = tmp_path / "source"
|
||||
source_dir.mkdir()
|
||||
|
||||
# Write N artifacts
|
||||
for spec in specs:
|
||||
_write_artifact_file(source_dir, spec)
|
||||
|
||||
pipeline, index, store_path = _setup_pipeline(tmp_path)
|
||||
n = len(specs)
|
||||
|
||||
# First ingestion
|
||||
source1 = MarkdownSource(directory=source_dir)
|
||||
pipeline.ingest(source1, "bahn")
|
||||
|
||||
# Capture original hashes from index
|
||||
original_hashes: dict[str, str] = {}
|
||||
for entry_id, entry in index.entries.items():
|
||||
original_hashes[entry_id] = entry.content_hash
|
||||
|
||||
# Choose a random subset to modify
|
||||
max_modify = max(1, n - 1)
|
||||
m = modification_seed.randint(1, max_modify)
|
||||
indices_to_modify = set(modification_seed.sample(range(n), m))
|
||||
|
||||
# Track which artifact IDs should change
|
||||
modified_names: set[str] = set()
|
||||
for idx in indices_to_modify:
|
||||
spec = specs[idx]
|
||||
modified_names.add(spec["name"])
|
||||
new_content = f"Changed hash content {spec['name']} - {modification_seed.randint(1, 999999)}\n"
|
||||
spec_modified = dict(spec)
|
||||
spec_modified["content"] = new_content
|
||||
_write_artifact_file(source_dir, spec_modified)
|
||||
|
||||
# Second ingestion
|
||||
source2 = MarkdownSource(directory=source_dir)
|
||||
pipeline.ingest(source2, "bahn")
|
||||
|
||||
# Verify: only modified artifacts have changed hashes
|
||||
for entry_id, entry in index.entries.items():
|
||||
# Extract the artifact name from the entry id (format: "bahn/{type}/{name}")
|
||||
parts = entry_id.split("/")
|
||||
name = parts[-1] if len(parts) >= 3 else entry_id
|
||||
|
||||
if name in modified_names:
|
||||
# Hash must have changed
|
||||
assert entry.content_hash != original_hashes.get(entry_id, ""), (
|
||||
f"Artifact '{entry_id}' was modified but hash did not change. "
|
||||
f"Old: {original_hashes.get(entry_id)}, New: {entry.content_hash}"
|
||||
)
|
||||
else:
|
||||
# Hash must remain the same
|
||||
assert entry.content_hash == original_hashes.get(entry_id, ""), (
|
||||
f"Artifact '{entry_id}' was NOT modified but hash changed. "
|
||||
f"Old: {original_hashes.get(entry_id)}, New: {entry.content_hash}"
|
||||
)
|
||||
@@ -0,0 +1,458 @@
|
||||
"""Unit-Tests für NoteGraphMigrator.
|
||||
|
||||
Testet die Migration der NoteGraph-Verzeichnisstruktur in den KnowledgeStore:
|
||||
- Directory-Typ-Mapping (decisions→decision, inbox→note, meetings→meeting, etc.)
|
||||
- Frontmatter-Generierung für Dateien ohne Frontmatter
|
||||
- Frontmatter-Anreicherung für Dateien mit bestehendem Frontmatter
|
||||
- Scope-Konfiguration laden
|
||||
- Validierung nach Migration
|
||||
- Fehlerbehandlung
|
||||
|
||||
Requirements: 3.5, 3.14
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from monorepo.knowledge.migration import (
|
||||
NOTEGRAPH_DIR_TYPE_MAP,
|
||||
MigrationResult,
|
||||
NoteGraphMigrator,
|
||||
ValidationResult,
|
||||
load_scope_config,
|
||||
)
|
||||
from monorepo.knowledge.store import KnowledgeStore
|
||||
from monorepo.models import ScopeConfig
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def scope_config_file(tmp_path: Path) -> Path:
|
||||
"""Erstellt eine temporäre scopes.yaml Datei."""
|
||||
config_path = tmp_path / "scopes.yaml"
|
||||
config_path.write_text(
|
||||
"""\
|
||||
version: "1.0"
|
||||
scopes:
|
||||
- name: privat
|
||||
context: privat
|
||||
description: "Persönlicher Wissensbereich"
|
||||
knowledge_path: shared/knowledge-store/privat/
|
||||
- name: bahn
|
||||
context: bahn
|
||||
description: "DB InfraGO Wissensbereich"
|
||||
knowledge_path: shared/knowledge-store/bahn/
|
||||
- name: shared
|
||||
context: shared
|
||||
description: "Kontextübergreifender Wissensbereich"
|
||||
knowledge_path: shared/knowledge-store/shared/
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return config_path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def knowledge_store(tmp_path: Path) -> KnowledgeStore:
|
||||
"""Erstellt einen KnowledgeStore in einem temporären Verzeichnis."""
|
||||
store_path = tmp_path / "knowledge-store"
|
||||
store_path.mkdir()
|
||||
scope_config = ScopeConfig(scopes={
|
||||
"privat": {"scope": "privat"},
|
||||
"bahn": {"scope": "bahn"},
|
||||
"shared": {"scope": "shared"},
|
||||
})
|
||||
return KnowledgeStore(base_path=store_path, scope_config=scope_config)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def notegraph_dir(tmp_path: Path) -> Path:
|
||||
"""Erstellt eine NoteGraph-Verzeichnisstruktur mit Testdaten."""
|
||||
notegraph = tmp_path / "notegraph"
|
||||
notegraph.mkdir()
|
||||
|
||||
# decisions/
|
||||
decisions = notegraph / "decisions"
|
||||
decisions.mkdir()
|
||||
(decisions / "api-design.md").write_text(
|
||||
"""\
|
||||
---
|
||||
type: decision
|
||||
title: "API-Design Entscheidung"
|
||||
tags: [api, architecture]
|
||||
source_context: bahn
|
||||
---
|
||||
|
||||
# API-Design Entscheidung
|
||||
|
||||
Wir verwenden REST-APIs.
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# inbox/ – Datei ohne Frontmatter
|
||||
inbox = notegraph / "inbox"
|
||||
inbox.mkdir()
|
||||
(inbox / "quick-note.md").write_text(
|
||||
"""\
|
||||
# Quick Note
|
||||
|
||||
Eine schnelle Notiz ohne Frontmatter.
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# meetings/
|
||||
meetings = notegraph / "meetings"
|
||||
meetings.mkdir()
|
||||
(meetings / "sprint-review.md").write_text(
|
||||
"""\
|
||||
---
|
||||
title: "Sprint Review 42"
|
||||
tags: [sprint, review]
|
||||
---
|
||||
|
||||
# Sprint Review 42
|
||||
|
||||
Besprechung des aktuellen Sprints.
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# people/
|
||||
people = notegraph / "people"
|
||||
people.mkdir()
|
||||
(people / "max-mustermann.md").write_text(
|
||||
"""\
|
||||
---
|
||||
type: reference
|
||||
title: "Max Mustermann"
|
||||
tags: [team, developer]
|
||||
source_context: bahn
|
||||
---
|
||||
|
||||
# Max Mustermann
|
||||
|
||||
Backend-Entwickler im Team.
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# projects/
|
||||
projects = notegraph / "projects"
|
||||
projects.mkdir()
|
||||
(projects / "wissensdatenbank.md").write_text(
|
||||
"""\
|
||||
---
|
||||
type: project
|
||||
title: "DB Wissensdatenbank"
|
||||
tags: [knowledge, etl]
|
||||
source_context: bahn
|
||||
---
|
||||
|
||||
# DB Wissensdatenbank
|
||||
|
||||
ETL-Pipeline für DB-InfraGO.
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
return notegraph
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: load_scope_config
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestLoadScopeConfig:
|
||||
"""Tests für das Laden der Scope-Konfiguration."""
|
||||
|
||||
def test_load_valid_config(self, scope_config_file: Path) -> None:
|
||||
"""Lädt eine gültige scopes.yaml erfolgreich."""
|
||||
config = load_scope_config(scope_config_file)
|
||||
assert isinstance(config, ScopeConfig)
|
||||
assert "privat" in config.scopes
|
||||
assert "bahn" in config.scopes
|
||||
assert config.scopes["privat"]["scope"] == "privat"
|
||||
|
||||
def test_load_missing_file_raises(self, tmp_path: Path) -> None:
|
||||
"""FileNotFoundError wenn die Datei nicht existiert."""
|
||||
with pytest.raises(FileNotFoundError):
|
||||
load_scope_config(tmp_path / "missing.yaml")
|
||||
|
||||
def test_load_invalid_format_raises(self, tmp_path: Path) -> None:
|
||||
"""ValueError bei ungültigem Format."""
|
||||
bad_file = tmp_path / "bad.yaml"
|
||||
bad_file.write_text("just a string", encoding="utf-8")
|
||||
with pytest.raises(ValueError):
|
||||
load_scope_config(bad_file)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: NoteGraphMigrator.migrate
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestNoteGraphMigration:
|
||||
"""Tests für die NoteGraph-Migration."""
|
||||
|
||||
def test_migrate_full_notegraph(
|
||||
self,
|
||||
knowledge_store: KnowledgeStore,
|
||||
scope_config_file: Path,
|
||||
notegraph_dir: Path,
|
||||
) -> None:
|
||||
"""Migriert eine vollständige NoteGraph-Struktur."""
|
||||
migrator = NoteGraphMigrator(
|
||||
store=knowledge_store, scope_config_path=scope_config_file
|
||||
)
|
||||
result = migrator.migrate(source_path=notegraph_dir, context="bahn")
|
||||
|
||||
assert isinstance(result, MigrationResult)
|
||||
assert result.migrated_count >= 4 # Mindestens 4 neue Artefakte
|
||||
assert result.success
|
||||
|
||||
def test_migrate_adds_frontmatter_to_plain_files(
|
||||
self,
|
||||
knowledge_store: KnowledgeStore,
|
||||
scope_config_file: Path,
|
||||
notegraph_dir: Path,
|
||||
) -> None:
|
||||
"""Dateien ohne Frontmatter erhalten generiertes Frontmatter."""
|
||||
migrator = NoteGraphMigrator(
|
||||
store=knowledge_store, scope_config_path=scope_config_file
|
||||
)
|
||||
migrator.migrate(source_path=notegraph_dir, context="bahn")
|
||||
|
||||
# Die inbox/quick-note.md sollte jetzt Frontmatter haben
|
||||
inbox_file = notegraph_dir / "inbox" / "quick-note.md"
|
||||
content = inbox_file.read_text(encoding="utf-8")
|
||||
assert content.startswith("---")
|
||||
assert "type: note" in content
|
||||
assert "source_context: bahn" in content
|
||||
|
||||
def test_migrate_enriches_existing_frontmatter(
|
||||
self,
|
||||
knowledge_store: KnowledgeStore,
|
||||
scope_config_file: Path,
|
||||
notegraph_dir: Path,
|
||||
) -> None:
|
||||
"""Existierendes Frontmatter wird um fehlende Felder ergänzt."""
|
||||
migrator = NoteGraphMigrator(
|
||||
store=knowledge_store, scope_config_path=scope_config_file
|
||||
)
|
||||
migrator.migrate(source_path=notegraph_dir, context="bahn")
|
||||
|
||||
# meetings/sprint-review.md hatte keinen type und keinen source_context
|
||||
meeting_file = notegraph_dir / "meetings" / "sprint-review.md"
|
||||
content = meeting_file.read_text(encoding="utf-8")
|
||||
assert "type: meeting" in content
|
||||
assert "source_context: bahn" in content
|
||||
|
||||
def test_migrate_preserves_existing_metadata(
|
||||
self,
|
||||
knowledge_store: KnowledgeStore,
|
||||
scope_config_file: Path,
|
||||
notegraph_dir: Path,
|
||||
) -> None:
|
||||
"""Vorhandene Metadaten (Tags, Title) bleiben erhalten."""
|
||||
migrator = NoteGraphMigrator(
|
||||
store=knowledge_store, scope_config_path=scope_config_file
|
||||
)
|
||||
migrator.migrate(source_path=notegraph_dir, context="bahn")
|
||||
|
||||
# decisions/api-design.md hatte bereits vollständige Metadaten
|
||||
decision_file = notegraph_dir / "decisions" / "api-design.md"
|
||||
content = decision_file.read_text(encoding="utf-8")
|
||||
assert "API-Design Entscheidung" in content
|
||||
assert "api" in content
|
||||
|
||||
def test_migrate_directory_type_mapping(
|
||||
self,
|
||||
knowledge_store: KnowledgeStore,
|
||||
scope_config_file: Path,
|
||||
notegraph_dir: Path,
|
||||
) -> None:
|
||||
"""Verzeichnisse werden korrekt auf Artefakt-Typen gemappt."""
|
||||
migrator = NoteGraphMigrator(
|
||||
store=knowledge_store, scope_config_path=scope_config_file
|
||||
)
|
||||
migrator.migrate(source_path=notegraph_dir, context="bahn")
|
||||
|
||||
# Index prüfen
|
||||
index_entries = knowledge_store.get_index(scope="bahn")
|
||||
types_found = {entry.type for entry in index_entries}
|
||||
|
||||
# Mindestens decision, note, meeting, reference, project
|
||||
assert "decision" in types_found
|
||||
assert "note" in types_found
|
||||
assert "meeting" in types_found
|
||||
assert "reference" in types_found
|
||||
assert "project" in types_found
|
||||
|
||||
def test_migrate_nonexistent_path_returns_error(
|
||||
self,
|
||||
knowledge_store: KnowledgeStore,
|
||||
scope_config_file: Path,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Migration mit nicht-existierendem Pfad gibt Fehler zurück."""
|
||||
migrator = NoteGraphMigrator(
|
||||
store=knowledge_store, scope_config_path=scope_config_file
|
||||
)
|
||||
result = migrator.migrate(
|
||||
source_path=tmp_path / "nonexistent", context="bahn"
|
||||
)
|
||||
assert len(result.errors) > 0
|
||||
assert result.migrated_count == 0
|
||||
|
||||
def test_migrate_empty_notegraph(
|
||||
self,
|
||||
knowledge_store: KnowledgeStore,
|
||||
scope_config_file: Path,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Migration eines leeren NoteGraph-Verzeichnisses."""
|
||||
empty_dir = tmp_path / "empty-notegraph"
|
||||
empty_dir.mkdir()
|
||||
|
||||
migrator = NoteGraphMigrator(
|
||||
store=knowledge_store, scope_config_path=scope_config_file
|
||||
)
|
||||
result = migrator.migrate(source_path=empty_dir, context="bahn")
|
||||
|
||||
assert result.migrated_count == 0
|
||||
assert result.skipped_count == 0
|
||||
assert len(result.errors) == 0
|
||||
|
||||
def test_migrate_partial_notegraph(
|
||||
self,
|
||||
knowledge_store: KnowledgeStore,
|
||||
scope_config_file: Path,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Migration mit nur einigen der NoteGraph-Verzeichnisse vorhanden."""
|
||||
partial = tmp_path / "partial-notegraph"
|
||||
partial.mkdir()
|
||||
|
||||
# Nur decisions/ anlegen
|
||||
decisions = partial / "decisions"
|
||||
decisions.mkdir()
|
||||
(decisions / "one-decision.md").write_text(
|
||||
"""\
|
||||
---
|
||||
type: decision
|
||||
title: "Eine Entscheidung"
|
||||
tags: [test]
|
||||
source_context: privat
|
||||
---
|
||||
|
||||
# Eine Entscheidung
|
||||
|
||||
Inhalt der Entscheidung.
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
migrator = NoteGraphMigrator(
|
||||
store=knowledge_store, scope_config_path=scope_config_file
|
||||
)
|
||||
result = migrator.migrate(source_path=partial, context="privat")
|
||||
|
||||
assert result.migrated_count == 1
|
||||
assert len(result.errors) == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: NoteGraphMigrator.validate
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestNoteGraphValidation:
|
||||
"""Tests für die Migrations-Validierung."""
|
||||
|
||||
def test_validate_after_successful_migration(
|
||||
self,
|
||||
knowledge_store: KnowledgeStore,
|
||||
scope_config_file: Path,
|
||||
notegraph_dir: Path,
|
||||
) -> None:
|
||||
"""Validierung nach erfolgreicher Migration ist erfolgreich."""
|
||||
migrator = NoteGraphMigrator(
|
||||
store=knowledge_store, scope_config_path=scope_config_file
|
||||
)
|
||||
migrator.migrate(source_path=notegraph_dir, context="bahn")
|
||||
|
||||
validation = migrator.validate(context="bahn")
|
||||
|
||||
assert isinstance(validation, ValidationResult)
|
||||
assert validation.valid
|
||||
assert validation.total >= 4
|
||||
assert validation.found_in_index == validation.total
|
||||
assert len(validation.missing) == 0
|
||||
|
||||
def test_validate_empty_context_is_valid(
|
||||
self,
|
||||
knowledge_store: KnowledgeStore,
|
||||
scope_config_file: Path,
|
||||
) -> None:
|
||||
"""Validierung eines leeren Kontexts ist gültig (0 Artefakte)."""
|
||||
migrator = NoteGraphMigrator(
|
||||
store=knowledge_store, scope_config_path=scope_config_file
|
||||
)
|
||||
validation = migrator.validate(context="privat")
|
||||
|
||||
assert validation.valid
|
||||
assert validation.total == 0
|
||||
|
||||
def test_validate_returns_details(
|
||||
self,
|
||||
knowledge_store: KnowledgeStore,
|
||||
scope_config_file: Path,
|
||||
notegraph_dir: Path,
|
||||
) -> None:
|
||||
"""Validierung enthält detaillierte Ergebnisse pro Artefakt."""
|
||||
migrator = NoteGraphMigrator(
|
||||
store=knowledge_store, scope_config_path=scope_config_file
|
||||
)
|
||||
migrator.migrate(source_path=notegraph_dir, context="bahn")
|
||||
|
||||
validation = migrator.validate(context="bahn")
|
||||
|
||||
assert len(validation.details) > 0
|
||||
for detail in validation.details:
|
||||
assert detail.file_name
|
||||
assert detail.found_in_index
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: NOTEGRAPH_DIR_TYPE_MAP
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDirectoryTypeMapping:
|
||||
"""Tests für das Verzeichnis-Typ-Mapping."""
|
||||
|
||||
def test_all_standard_directories_mapped(self) -> None:
|
||||
"""Alle Standard-NoteGraph-Verzeichnisse sind gemappt."""
|
||||
assert "decisions" in NOTEGRAPH_DIR_TYPE_MAP
|
||||
assert "inbox" in NOTEGRAPH_DIR_TYPE_MAP
|
||||
assert "meetings" in NOTEGRAPH_DIR_TYPE_MAP
|
||||
assert "people" in NOTEGRAPH_DIR_TYPE_MAP
|
||||
assert "projects" in NOTEGRAPH_DIR_TYPE_MAP
|
||||
|
||||
def test_correct_type_assignments(self) -> None:
|
||||
"""Verzeichnisse sind den korrekten Typen zugeordnet."""
|
||||
assert NOTEGRAPH_DIR_TYPE_MAP["decisions"] == "decision"
|
||||
assert NOTEGRAPH_DIR_TYPE_MAP["inbox"] == "note"
|
||||
assert NOTEGRAPH_DIR_TYPE_MAP["meetings"] == "meeting"
|
||||
assert NOTEGRAPH_DIR_TYPE_MAP["people"] == "reference"
|
||||
assert NOTEGRAPH_DIR_TYPE_MAP["projects"] == "project"
|
||||
@@ -0,0 +1,340 @@
|
||||
"""Property-basierte Tests: Progressive-Disclosure-Index enthält nur kompakte Einträge.
|
||||
|
||||
**Validates: Requirements 3.16, 3.18**
|
||||
|
||||
Property 7: Progressive-Disclosure-Index enthält nur kompakte Einträge
|
||||
- For any Abfrage des YAML-Index (Schicht 1) müssen die zurückgegebenen Einträge
|
||||
Titel, Tags, Beziehungen und Kurzbeschreibungen enthalten, aber niemals den
|
||||
vollständigen Dokumentinhalt. Suchergebnisse liefern ausschließlich Pfade zurück.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
from hypothesis import given, settings
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from monorepo.knowledge.index import IndexEntry, YAMLIndex
|
||||
from monorepo.knowledge.etl import ETLPipeline
|
||||
from monorepo.knowledge.sources.markdown import MarkdownSource
|
||||
from monorepo.knowledge.store import KnowledgeStore
|
||||
from monorepo.models import ScopeConfig
|
||||
|
||||
|
||||
# --- Constants ---
|
||||
|
||||
ARTIFACT_TYPES = ["decision", "note", "meeting", "reference", "pattern"]
|
||||
CONTEXTS = ["privat", "dhive", "bahn", "shared"]
|
||||
SAMPLE_TAGS = ["api", "architecture", "python", "devops", "testing", "design", "cloud"]
|
||||
|
||||
|
||||
# --- Strategies ---
|
||||
|
||||
|
||||
@st.composite
|
||||
def artifact_content(draw: st.DrawFn) -> str:
|
||||
"""Generates random artifact content of varying length.
|
||||
|
||||
Some short (< 150 chars), some long (> 150 chars) to test
|
||||
that long content never leaks into the index.
|
||||
"""
|
||||
# Choose between short and long content
|
||||
is_long = draw(st.booleans())
|
||||
if is_long:
|
||||
# Generate long content (200-1000 chars)
|
||||
num_lines = draw(st.integers(min_value=5, max_value=20))
|
||||
lines = []
|
||||
for _ in range(num_lines):
|
||||
line = draw(st.text(
|
||||
alphabet=st.characters(whitelist_categories=("L", "N", "Zs")),
|
||||
min_size=20,
|
||||
max_size=80,
|
||||
))
|
||||
lines.append(line)
|
||||
return "\n".join(lines)
|
||||
else:
|
||||
# Generate short content (10-100 chars)
|
||||
return draw(st.text(
|
||||
alphabet=st.characters(whitelist_categories=("L", "N", "Zs")),
|
||||
min_size=10,
|
||||
max_size=100,
|
||||
))
|
||||
|
||||
|
||||
@st.composite
|
||||
def artifact_metadata(draw: st.DrawFn) -> dict[str, object]:
|
||||
"""Generates random artifact metadata for YAML frontmatter."""
|
||||
title = draw(st.text(
|
||||
alphabet=st.characters(whitelist_categories=("L", "N", "Zs")),
|
||||
min_size=3,
|
||||
max_size=60,
|
||||
))
|
||||
artifact_type = draw(st.sampled_from(ARTIFACT_TYPES))
|
||||
tags = draw(st.lists(st.sampled_from(SAMPLE_TAGS), min_size=1, max_size=4, unique=True))
|
||||
return {
|
||||
"type": artifact_type,
|
||||
"title": title,
|
||||
"tags": tags,
|
||||
}
|
||||
|
||||
|
||||
@st.composite
|
||||
def artifact_batch(draw: st.DrawFn) -> list[tuple[dict[str, object], str]]:
|
||||
"""Generates a batch of 1-5 artifacts with metadata and content."""
|
||||
count = draw(st.integers(min_value=1, max_value=5))
|
||||
artifacts = []
|
||||
for _ in range(count):
|
||||
meta = draw(artifact_metadata())
|
||||
content = draw(artifact_content())
|
||||
artifacts.append((meta, content))
|
||||
return artifacts
|
||||
|
||||
|
||||
# --- Helpers ---
|
||||
|
||||
|
||||
def _create_source_directory(tmp_dir: Path, artifacts: list[tuple[dict[str, object], str]]) -> Path:
|
||||
"""Creates a source directory with markdown files containing YAML frontmatter."""
|
||||
source_dir = tmp_dir / "source"
|
||||
source_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for i, (meta, content) in enumerate(artifacts):
|
||||
frontmatter = yaml.dump(meta, default_flow_style=False, allow_unicode=True, sort_keys=False)
|
||||
file_content = f"---\n{frontmatter}---\n{content}"
|
||||
file_path = source_dir / f"artifact-{i}.md"
|
||||
file_path.write_text(file_content, encoding="utf-8")
|
||||
|
||||
return source_dir
|
||||
|
||||
|
||||
def _create_knowledge_store(tmp_dir: Path) -> KnowledgeStore:
|
||||
"""Creates a KnowledgeStore in a temporary directory."""
|
||||
store_path = tmp_dir / "knowledge-store"
|
||||
store_path.mkdir(parents=True, exist_ok=True)
|
||||
scope_config = ScopeConfig(scopes={
|
||||
"privat": {"scope": "privat", "paths": ["privat/"]},
|
||||
"dhive": {"scope": "dhive", "paths": ["dhive/"]},
|
||||
"bahn": {"scope": "bahn", "paths": ["bahn/"]},
|
||||
"shared": {"scope": "shared", "paths": ["shared/"]},
|
||||
})
|
||||
return KnowledgeStore(base_path=store_path, scope_config=scope_config)
|
||||
|
||||
|
||||
# Index entry required fields per specification
|
||||
REQUIRED_INDEX_FIELDS = {"id", "title", "type", "tags", "scope", "summary", "path", "content_hash", "links"}
|
||||
|
||||
|
||||
# --- Tests ---
|
||||
|
||||
|
||||
class TestProperty7ProgressiveDisclosureIndex:
|
||||
"""Property 7: Progressive-Disclosure-Index enthält nur kompakte Einträge.
|
||||
|
||||
**Validates: Requirements 3.16, 3.18**
|
||||
|
||||
For any query of the YAML index (Layer 1), the returned entries must
|
||||
contain title, tags, relationships, and short descriptions, but NEVER
|
||||
the full document content. Search results must exclusively return paths.
|
||||
"""
|
||||
|
||||
@given(artifacts=artifact_batch())
|
||||
@settings(max_examples=50, deadline=None)
|
||||
def test_persisted_yaml_index_has_no_content_field(
|
||||
self, artifacts: list[tuple[dict[str, object], str]]
|
||||
) -> None:
|
||||
"""After ingestion, the persisted YAML index file must NOT contain
|
||||
a 'content' field in any entry."""
|
||||
with tempfile.TemporaryDirectory(prefix="monorepo_pbt_") as tmp:
|
||||
tmp_dir = Path(tmp)
|
||||
store = _create_knowledge_store(tmp_dir)
|
||||
source_dir = _create_source_directory(tmp_dir, artifacts)
|
||||
|
||||
# Ingest via ETLPipeline
|
||||
source = MarkdownSource(directory=source_dir)
|
||||
store.ingest(source, context="bahn")
|
||||
|
||||
# Load the persisted YAML index file directly
|
||||
index_path = store.base_path / "_index.yaml"
|
||||
if not index_path.exists():
|
||||
# No artifacts were ingested (possible if all had errors)
|
||||
return
|
||||
|
||||
raw_data = yaml.safe_load(index_path.read_text(encoding="utf-8"))
|
||||
if raw_data is None or "artifacts" not in raw_data:
|
||||
return
|
||||
|
||||
for entry_data in raw_data["artifacts"]:
|
||||
assert "content" not in entry_data, (
|
||||
f"YAML index entry '{entry_data.get('id', '?')}' contains a "
|
||||
f"'content' field – Progressive Disclosure violated!"
|
||||
)
|
||||
|
||||
@given(artifacts=artifact_batch())
|
||||
@settings(max_examples=50, deadline=None)
|
||||
def test_persisted_yaml_index_entries_have_required_fields(
|
||||
self, artifacts: list[tuple[dict[str, object], str]]
|
||||
) -> None:
|
||||
"""Each persisted YAML index entry must have all required compact fields:
|
||||
id, title, type, tags, scope, summary, path, content_hash, links."""
|
||||
with tempfile.TemporaryDirectory(prefix="monorepo_pbt_") as tmp:
|
||||
tmp_dir = Path(tmp)
|
||||
store = _create_knowledge_store(tmp_dir)
|
||||
source_dir = _create_source_directory(tmp_dir, artifacts)
|
||||
|
||||
source = MarkdownSource(directory=source_dir)
|
||||
store.ingest(source, context="bahn")
|
||||
|
||||
index_path = store.base_path / "_index.yaml"
|
||||
if not index_path.exists():
|
||||
return
|
||||
|
||||
raw_data = yaml.safe_load(index_path.read_text(encoding="utf-8"))
|
||||
if raw_data is None or "artifacts" not in raw_data:
|
||||
return
|
||||
|
||||
for entry_data in raw_data["artifacts"]:
|
||||
entry_keys = set(entry_data.keys())
|
||||
missing = REQUIRED_INDEX_FIELDS - entry_keys
|
||||
assert not missing, (
|
||||
f"YAML index entry '{entry_data.get('id', '?')}' is missing "
|
||||
f"required fields: {missing}"
|
||||
)
|
||||
|
||||
@given(artifacts=artifact_batch())
|
||||
@settings(max_examples=50, deadline=None)
|
||||
def test_persisted_yaml_index_summary_within_limit(
|
||||
self, artifacts: list[tuple[dict[str, object], str]]
|
||||
) -> None:
|
||||
"""Each entry's summary must be <= 150 chars (not full content)."""
|
||||
with tempfile.TemporaryDirectory(prefix="monorepo_pbt_") as tmp:
|
||||
tmp_dir = Path(tmp)
|
||||
store = _create_knowledge_store(tmp_dir)
|
||||
source_dir = _create_source_directory(tmp_dir, artifacts)
|
||||
|
||||
source = MarkdownSource(directory=source_dir)
|
||||
store.ingest(source, context="privat")
|
||||
|
||||
index_path = store.base_path / "_index.yaml"
|
||||
if not index_path.exists():
|
||||
return
|
||||
|
||||
raw_data = yaml.safe_load(index_path.read_text(encoding="utf-8"))
|
||||
if raw_data is None or "artifacts" not in raw_data:
|
||||
return
|
||||
|
||||
for entry_data in raw_data["artifacts"]:
|
||||
summary = entry_data.get("summary", "")
|
||||
assert len(summary) <= 150, (
|
||||
f"YAML index entry '{entry_data.get('id', '?')}' has summary "
|
||||
f"of {len(summary)} chars (max 150): '{summary[:60]}...'"
|
||||
)
|
||||
|
||||
@given(artifacts=artifact_batch())
|
||||
@settings(max_examples=50, deadline=None)
|
||||
def test_search_results_contain_no_full_content(
|
||||
self, artifacts: list[tuple[dict[str, object], str]]
|
||||
) -> None:
|
||||
"""KnowledgeStore.search() results must not contain full document content.
|
||||
IndexEntry objects returned have only compact metadata."""
|
||||
with tempfile.TemporaryDirectory(prefix="monorepo_pbt_") as tmp:
|
||||
tmp_dir = Path(tmp)
|
||||
store = _create_knowledge_store(tmp_dir)
|
||||
source_dir = _create_source_directory(tmp_dir, artifacts)
|
||||
|
||||
source = MarkdownSource(directory=source_dir)
|
||||
store.ingest(source, context="dhive")
|
||||
|
||||
# Search with a broad query that should match something
|
||||
results = store.search("", allowed_scopes=["dhive"])
|
||||
# Empty query returns no results by design, try with a common term
|
||||
results = store.search("a", allowed_scopes=["dhive"])
|
||||
|
||||
for result in results:
|
||||
entry = result.entry
|
||||
# IndexEntry must not have a "content" attribute with actual content
|
||||
assert not hasattr(entry, "content"), (
|
||||
f"SearchResult entry '{entry.id}' has a 'content' attribute – "
|
||||
f"Progressive Disclosure violated!"
|
||||
)
|
||||
# Verify the entry has only compact fields
|
||||
entry_dict = entry.to_dict()
|
||||
assert "content" not in entry_dict, (
|
||||
f"SearchResult entry '{entry.id}' dict contains 'content' field!"
|
||||
)
|
||||
# Summary must be compact
|
||||
assert len(entry.summary) <= 150, (
|
||||
f"SearchResult entry '{entry.id}' has summary > 150 chars"
|
||||
)
|
||||
|
||||
@given(artifacts=artifact_batch())
|
||||
@settings(max_examples=50, deadline=None)
|
||||
def test_get_index_returns_no_full_content(
|
||||
self, artifacts: list[tuple[dict[str, object], str]]
|
||||
) -> None:
|
||||
"""KnowledgeStore.get_index() must return IndexEntry objects without
|
||||
full document content – only compact metadata."""
|
||||
with tempfile.TemporaryDirectory(prefix="monorepo_pbt_") as tmp:
|
||||
tmp_dir = Path(tmp)
|
||||
store = _create_knowledge_store(tmp_dir)
|
||||
source_dir = _create_source_directory(tmp_dir, artifacts)
|
||||
|
||||
source = MarkdownSource(directory=source_dir)
|
||||
store.ingest(source, context="bahn")
|
||||
|
||||
# Get all index entries
|
||||
entries = store.get_index()
|
||||
|
||||
for entry in entries:
|
||||
# IndexEntry must not have "content" attribute
|
||||
assert not hasattr(entry, "content"), (
|
||||
f"get_index() entry '{entry.id}' has a 'content' attribute – "
|
||||
f"Progressive Disclosure violated!"
|
||||
)
|
||||
# Serialized dict must not contain "content"
|
||||
entry_dict = entry.to_dict()
|
||||
assert "content" not in entry_dict, (
|
||||
f"get_index() entry '{entry.id}' dict contains 'content' field!"
|
||||
)
|
||||
# Summary must be compact (<= 150 chars)
|
||||
assert len(entry.summary) <= 150, (
|
||||
f"get_index() entry '{entry.id}' has summary > 150 chars: "
|
||||
f"'{entry.summary[:60]}...'"
|
||||
)
|
||||
# Entry must have required fields
|
||||
assert entry.id, "Entry missing 'id'"
|
||||
assert entry.title is not None, "Entry missing 'title'"
|
||||
assert entry.type is not None, "Entry missing 'type'"
|
||||
assert entry.path, "Entry missing 'path'"
|
||||
assert entry.content_hash, "Entry missing 'content_hash'"
|
||||
|
||||
@given(artifacts=artifact_batch())
|
||||
@settings(max_examples=50, deadline=None)
|
||||
def test_get_index_scoped_returns_no_full_content(
|
||||
self, artifacts: list[tuple[dict[str, object], str]]
|
||||
) -> None:
|
||||
"""KnowledgeStore.get_index(scope=...) also must not expose content."""
|
||||
with tempfile.TemporaryDirectory(prefix="monorepo_pbt_") as tmp:
|
||||
tmp_dir = Path(tmp)
|
||||
store = _create_knowledge_store(tmp_dir)
|
||||
source_dir = _create_source_directory(tmp_dir, artifacts)
|
||||
|
||||
source = MarkdownSource(directory=source_dir)
|
||||
store.ingest(source, context="shared")
|
||||
|
||||
# Get entries filtered by scope
|
||||
entries = store.get_index(scope="shared")
|
||||
|
||||
for entry in entries:
|
||||
assert not hasattr(entry, "content"), (
|
||||
f"get_index(scope) entry '{entry.id}' has 'content' attribute!"
|
||||
)
|
||||
entry_dict = entry.to_dict()
|
||||
assert "content" not in entry_dict, (
|
||||
f"get_index(scope) entry '{entry.id}' dict has 'content' field!"
|
||||
)
|
||||
assert len(entry.summary) <= 150, (
|
||||
f"get_index(scope) entry '{entry.id}' summary > 150 chars"
|
||||
)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user