commit 2f2b29553170b514006903099de2ed90f23ff8fe Author: DoctoDre Date: Tue Jun 30 20:37:40 2026 +0200 Initial monorepo structure diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a9fdea0 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,13 @@ +# ============================================================================= +# Root-Level git-crypt-Regeln +# Kontextspezifische Regeln befinden sich in den jeweiligen Kontextordnern +# (privat/.gitattributes, dhive/.gitattributes, bahn/.gitattributes) +# ============================================================================= + +# Shared Secrets (falls vorhanden) mit Basis-Filter verschlüsseln +shared/.env filter=git-crypt diff=git-crypt +shared/**/*.pem filter=git-crypt diff=git-crypt +shared/**/*.key filter=git-crypt diff=git-crypt + +# Maschinenkontext-Konfiguration enthält keine Secrets, nur Mapping-Informationen +# (Schlüssel werden aus Keyring/Passwort-Manager geladen, nicht aus Dateien) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6dab269 --- /dev/null +++ b/.gitignore @@ -0,0 +1,73 @@ +# ============================================================================= +# Monorepo .gitignore +# Nur Build-Artefakte, Caches und temporäre Dateien. +# Secrets werden NICHT ausgeschlossen – sie werden via git-crypt verschlüsselt. +# ============================================================================= + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +*.egg-info/ +*.egg +dist/ +build/ +.eggs/ +*.whl + +# Virtual Environments +.venv/ +venv/ +ENV/ +env/ + +# IDE / Editor +.idea/ +*.swp +*.swo +*~ +.project +.classpath +.settings/ + +# OS-Dateien +.DS_Store +Thumbs.db +Desktop.ini + +# Build-Artefakte +*.o +*.a +*.lib +*.dll +*.dylib + +# Test & Coverage +.coverage +htmlcov/ +.pytest_cache/ +.hypothesis/ +.tox/ +.nox/ + +# Node.js (falls relevant für Tools) +node_modules/ +npm-debug.log* +yarn-error.log* + +# Logs (außer Audit-Logs) +*.log +!.audit/*.log + +# Temporäre Dateien +*.tmp +*.temp +*.bak +*.orig + +# Docker +.docker/ + +# Misc +.cache/ diff --git a/.kiro/mcp-servers/orgmylife-api/requirements.txt b/.kiro/mcp-servers/orgmylife-api/requirements.txt new file mode 100644 index 0000000..31ae1bf --- /dev/null +++ b/.kiro/mcp-servers/orgmylife-api/requirements.txt @@ -0,0 +1,2 @@ +mcp[cli]>=1.0.0 +httpx>=0.27.0 diff --git a/.kiro/mcp-servers/orgmylife-api/server.py b/.kiro/mcp-servers/orgmylife-api/server.py new file mode 100644 index 0000000..f1e0280 --- /dev/null +++ b/.kiro/mcp-servers/orgmylife-api/server.py @@ -0,0 +1,230 @@ +"""OrgMyLife MCP Server — provides tools to interact with the OrgMyLife API. + +Tools: +- list_projects: Get all project tasks from the kanban board +- update_project: Move a project task to a different status column +- create_project: Create a new project task +- list_tasks: Get open tasks (with optional filters) +- list_calls: Get pending call items +""" + +import os +import httpx +from mcp.server.fastmcp import FastMCP + +# Configuration from environment +BASE_URL = os.environ.get("ORGMYLIFE_BASE_URL", "https://api.andreknie.de") +API_SECRET = os.environ.get("ORGMYLIFE_API_SECRET", "") + +mcp = FastMCP("orgmylife-api", instructions="OrgMyLife project management API. Use these tools to read and update the projects board, task list, and call list.") + + +def _headers(): + """Build auth headers for the OrgMyLife API.""" + return { + "Authorization": f"Bearer {API_SECRET}", + "X-API-Key": API_SECRET, + "Content-Type": "application/json", + } + + +def _get(path: str, params: dict = None) -> dict: + """Make an authenticated GET request to the OrgMyLife API.""" + r = httpx.get(f"{BASE_URL}{path}", headers=_headers(), params=params, timeout=15) + r.raise_for_status() + return r.json() + + +def _post(path: str, json_body: dict = None) -> dict: + """Make an authenticated POST request to the OrgMyLife API.""" + r = httpx.post(f"{BASE_URL}{path}", headers=_headers(), json=json_body, timeout=15) + r.raise_for_status() + return r.json() + + +def _put(path: str, json_body: dict = None) -> dict: + """Make an authenticated PUT request to the OrgMyLife API.""" + r = httpx.put(f"{BASE_URL}{path}", headers=_headers(), json=json_body, timeout=15) + r.raise_for_status() + return r.json() + + +def _delete(path: str) -> dict: + """Make an authenticated DELETE request to the OrgMyLife API.""" + r = httpx.delete(f"{BASE_URL}{path}", headers=_headers(), timeout=15) + r.raise_for_status() + return r.json() + + +# --- Projects Board --- + +@mcp.tool() +def list_projects() -> str: + """Get all project tasks from the kanban board, grouped by status column.""" + data = _get("/api/projects") + tasks = data.get("tasks", []) + if not tasks: + return "No project tasks found." + + # Group by status + columns = {} + for t in tasks: + status = t.get("status", "backlog") + columns.setdefault(status, []).append(t) + + lines = [] + status_order = ["backlog", "ready", "in_progress", "blocked", "review", "done"] + status_labels = { + "backlog": "📋 Backlog", + "ready": "🟢 Ready", + "in_progress": "⚙️ In Progress", + "blocked": "🚫 Blocked", + "review": "👁 Review", + "done": "✅ Done", + } + for status in status_order: + items = columns.get(status, []) + lines.append(f"\n## {status_labels.get(status, status)} ({len(items)})") + if items: + for t in items: + repo = f" [{t['repo']}]" if t.get("repo") else "" + lines.append(f" - #{t['id']}: {t['title']}{repo}") + else: + lines.append(" (empty)") + + return "\n".join(lines) + + +@mcp.tool() +def update_project(task_id: int, status: str) -> str: + """Move a project task to a different status column. + + Args: + task_id: The project task ID (from list_projects) + status: Target column — one of: backlog, ready, in_progress, blocked, review, done + """ + valid = ["backlog", "ready", "in_progress", "blocked", "review", "done"] + if status not in valid: + return f"Invalid status '{status}'. Must be one of: {', '.join(valid)}" + result = _put(f"/api/projects/{task_id}", {"status": status}) + return f"Moved project task #{task_id} to '{status}'. Result: {result.get('status', 'unknown')}" + + +@mcp.tool() +def create_project(title: str, repo: str = "") -> str: + """Create a new project task on the kanban board (starts in Backlog). + + Args: + title: Task title + repo: Repository name (e.g., "OrgMyLife", "AI-Orchestrator") + """ + result = _post("/api/projects/create", {"title": title, "repo": repo}) + if result.get("status") == "success": + return f"Created project task: '{title}' (ID: {result.get('id')})" + return f"Failed to create project task: {result}" + + +# --- Tasks --- + +@mcp.tool() +def list_tasks(filter_origin: str = "", filter_label: str = "", sort: str = "priority") -> str: + """Get open tasks from the task list. + + Args: + filter_origin: Filter by source — "manual", "nextcloud_todo", "dhive_email", "gmail", or "" for all + filter_label: Filter by label name, or "" for all + sort: Sort order — "priority", "due_date", "created", or "" for manual order + """ + params = {} + if sort: + params["sort"] = sort + if filter_origin: + params["filter_origin"] = filter_origin + if filter_label: + params["filter_label"] = filter_label + + data = _get("/api/tasks", params) + tasks = data.get("tasks", []) + total = data.get("total", len(tasks)) + + if not tasks: + return "No open tasks." + + lines = [f"**{total} open tasks:**\n"] + for t in tasks[:30]: # Limit to 30 to avoid huge output + priority = t.get("priority", "") + due = f" (due: {t['due_date']})" if t.get("due_date") else "" + labels = f" [{t['labels']}]" if t.get("labels") else "" + conflict = " ⚠️CONFLICT" if t.get("has_conflict") else "" + lines.append(f" #{t['id']}: {t['title']} — {priority}{due}{labels}{conflict}") + + if total > 30: + lines.append(f"\n ... and {total - 30} more") + + return "\n".join(lines) + + +# --- Calls --- + +@mcp.tool() +def list_calls() -> str: + """Get all pending call items from the call list.""" + data = _get("/api/calls") + calls = data.get("calls", []) + + if not calls: + return "No pending calls." + + lines = [f"**{len(calls)} pending calls:**\n"] + for c in calls: + phone = f" ({c['phone_number']})" if c.get("phone_number") else "" + lines.append(f" #{c['id']}: {c['person']}{phone} — {c['reason']}") + + return "\n".join(lines) + + +# --- Daily Digest --- + +@mcp.tool() +def get_digest() -> str: + """Get the daily digest summary (overdue, due today, top priorities, schedule).""" + data = _get("/api/digest") + + lines = [f"**{data.get('greeting', 'Hello')}** — {data.get('date', '')}"] + lines.append(f"Total open tasks: {data.get('total_open', '?')}\n") + + overdue = data.get("overdue", []) + if overdue: + lines.append(f"⚠️ **Overdue ({len(overdue)}):**") + for t in overdue: + lines.append(f" - {t['title']} ({t['days_overdue']}d overdue)") + + due_today = data.get("due_today", []) + if due_today: + lines.append(f"\n📅 **Due Today ({len(due_today)}):**") + for t in due_today: + lines.append(f" - {t['title']}") + + top = data.get("top_tasks", []) + if top: + lines.append(f"\n🎯 **Top Priorities:**") + for t in top: + lines.append(f" - {t['title']} ({t['priority']})") + + events = data.get("events_today", []) + if events: + lines.append(f"\n📆 **Today's Schedule:**") + for e in events: + lines.append(f" - {e['time']} {e['title']}") + + calls = data.get("pending_calls", []) + if calls: + lines.append(f"\n📞 **Calls to make ({len(calls)}):**") + for c in calls: + lines.append(f" - {c['person']} — {c['reason']}") + + return "\n".join(lines) + + +if __name__ == "__main__": + mcp.run() diff --git a/.kiro/settings/mcp.json b/.kiro/settings/mcp.json new file mode 100644 index 0000000..9e881ef --- /dev/null +++ b/.kiro/settings/mcp.json @@ -0,0 +1,54 @@ +{ + "mcpServers": { + "orgmylife-api": { + "command": "python", + "args": [ + ".kiro/mcp-servers/orgmylife-api/server.py" + ], + "env": { + "ORGMYLIFE_BASE_URL": "https://api.andreknie.de", + "ORGMYLIFE_API_SECRET": "!Z5l5vUcm!&qfhUuE=Nf" + }, + "disabled": false, + "autoApprove": [ + "list_projects", + "list_tasks", + "list_calls", + "get_digest", + "create_project", + "update_project" + ] + }, + "atlassian": { + "url": "https://taros-playground-mcp-atlassian.apps.dbcs-prag.comp.db.de/mcp", + "headers": { + "Authorization": "Bearer NTMwMjQ0MzY2NzQ0OkpIZBEiwwYfES+c80xqOpsjVdpx", + "X-Jira-Host": "arija-jira.jaas.service.deutschebahn.com", + "X-Jira-Projects": "O2CO2C,O2CAAPN,O2CAABS,O2CAABN,O2CAEB,O2CADX", + "X-Jira-Mode": "read" + }, + "disabled": false, + "autoApprove": [ + "search_issues", + "get_issue", + "get_projects", + "get_project" + ] + }, + "dbctx": { + "url": "https://taros-playground-dbctx.apps.dbcs-prag.comp.db.de/mcp", + "disabled": false, + "autoApprove": [] + }, + "beam-mcp": { + "command": "node", + "args": ["beam-mcp/dist/index.js"], + "env": { + "BEAM_BASE_URL": "https://db.leanix.net", + "BEAM_WORKSPACE": "beam", + "BEAM_HEADLESS": "true" + }, + "disabled": true + } + } +} diff --git a/.kiro/skills/check-ai-tasks.md b/.kiro/skills/check-ai-tasks.md new file mode 100644 index 0000000..54edf5b --- /dev/null +++ b/.kiro/skills/check-ai-tasks.md @@ -0,0 +1,41 @@ +--- +inclusion: manual +--- + +# Check AI Tasks + +When this skill is activated, perform the following routine: + +## Steps + +1. **Read `OrgMyLife/AI_TASKS.md`** to see all open AI issues (auto-updated every 30 min by GitHub Actions) + - This file contains the issue number, title, description, and URL for each open `ai-task` + - If the file says "No open AI tasks" — report that and stop + +2. **For each open issue**, check if the work described has already been implemented: + - Read the issue title to identify the task (format: `[AI] OML-XX: Task title`) + - Check the OrgMyLife codebase to see if the feature/fix already exists + - Check recent git commits for related work + +3. **If the task is already done:** + - Report it as completed + - Remind the user to close the GitHub issue (which auto-marks it done in OrgMyLife) + +4. **If the task is NOT done:** + - Implement it in the OrgMyLife repo + - Commit and push (auto-deploys via GitHub Actions) + - Report what was done + - Remind the user to close the issue after verifying + +5. **Summary:** After processing all issues, give a brief status report: + - How many issues were open + - How many were already done + - How many were just implemented + - Any issues that need user input before implementation + +## Notes +- The OrgMyLife repo is at: `./OrgMyLife/` +- The AI-Orchestrator repo is at: `./AI-Orchestrator/` +- Auto-deploy is configured: push to main triggers deployment to the VPS +- The live app is at: https://api.andreknie.de +- Tasks are marked done by closing the GitHub issue (triggers complete-task.yml workflow) diff --git a/.kiro/specs/call-list/.config.kiro b/.kiro/specs/call-list/.config.kiro new file mode 100644 index 0000000..5351837 --- /dev/null +++ b/.kiro/specs/call-list/.config.kiro @@ -0,0 +1 @@ +{"specId": "e33fa6e5-89e0-4d8d-ad9e-d43d6bf23daa", "workflowType": "requirements-first", "specType": "feature"} diff --git a/.kiro/specs/call-list/design.md b/.kiro/specs/call-list/design.md new file mode 100644 index 0000000..56cd7a1 --- /dev/null +++ b/.kiro/specs/call-list/design.md @@ -0,0 +1,343 @@ +# Design Document: Call List Feature + +## Overview + +The Call List feature adds a dedicated phone-call tracking system to OrgMyLife. It introduces a new `CallItem` model, a CRUD REST API at `/api/calls`, a phone number extraction service for German formats, a new "📞 Calls" frontend tab, and integration with the existing Daily Digest. + +The design follows existing OrgMyLife patterns: SQLAlchemy model in `app/models.py`, FastAPI routes in `app/main.py`, service logic in `app/services/`, and vanilla JS frontend in `frontend/js/app.js`. + +## Architecture + +```mermaid +graph TD + subgraph Frontend + A[Task List View] -->|📞 button click| B[POST /api/calls] + C[Calls Tab] -->|load| D[GET /api/calls] + C -->|mark done| E[PUT /api/calls/:id] + C -->|delete| F[DELETE /api/calls/:id] + end + + subgraph Backend API + B --> G[Call List Service] + D --> G + E --> G + F --> G + end + + subgraph Services + G --> H[Phone Extractor] + G --> I[Task Completion Flow] + J[Daily Digest Service] --> G + end + + subgraph Database + G --> K[(call_items table)] + I --> L[(tasks table)] + end +``` + +### Request Flow: Create Call from Task + +1. User clicks 📞 on a task in the Task List +2. Frontend sends `POST /api/calls` with `{ task_id: }` +3. Backend loads the task, extracts sender name from description +4. Phone Extractor scans the task description for German phone numbers +5. CallItem is created with person, reason, phone_number, original_task_id +6. Response returns the created CallItem + +### Request Flow: Complete Call with Task Resolution + +1. User clicks "Done" on a call item that has `original_task_id` +2. Frontend sends `PUT /api/calls/:id` with `{ status: "done", resolve_task: true }` +3. Backend marks CallItem as done +4. If `resolve_task` is true, backend marks the original task as "completed" and triggers existing side effects (email archiving, Nextcloud sync) + +## Components and Interfaces + +### 1. CallItem Model (`app/models.py`) + +New SQLAlchemy model added to the existing models file: + +```python +class CallItem(Base): + __tablename__ = "call_items" + + id = Column(Integer, primary_key=True, index=True) + person = Column(String(200), nullable=False) + reason = Column(String(500), nullable=False) + phone_number = Column(String(30), nullable=True) + original_task_id = Column(Integer, ForeignKey("tasks.id"), nullable=True) + priority = Column(Integer, default=3) # 1=Very High, 2=High, 3=Medium, 4=Low + status = Column(String, default="pending") # "pending" or "done" + created_at = Column(DateTime(timezone=True), server_default=func.now()) +``` + +### 2. Phone Extractor Service (`app/services/phone_extractor.py`) + +Pure function module with no external dependencies: + +```python +def extract_phone_number(text: str) -> str | None: + """Extract the first German phone number from text. + + Supports formats: + - +49 xxx xxxx xxxx (international) + - 0xxx xxxxxxx (domestic with area code) + - (0xxx) xxxxxxx (domestic with parenthesized area code) + + Returns the first match or None. + """ + +def normalize_phone_number(raw: str) -> str: + """Normalize a phone number string by removing separators and standardizing format.""" + +def parse_phone_number(text: str) -> dict | None: + """Parse a phone number string into components: country_code, area_code, subscriber.""" +``` + +### 3. Call List API Routes (`app/main.py`) + +New endpoints following existing patterns (session auth, Pydantic models): + +| Method | Path | Description | +|--------|------|-------------| +| POST | `/api/calls` | Create call item (from task or manual) | +| GET | `/api/calls` | List pending call items | +| PUT | `/api/calls/{id}` | Update call item (status, phone, priority) | +| DELETE | `/api/calls/{id}` | Remove call item | + +### 4. Pydantic Request/Response Models + +```python +class CallItemCreate(BaseModel): + task_id: Optional[int] = None # Create from existing task + person: Optional[str] = None # Required if no task_id + reason: Optional[str] = None # Required if no task_id + phone_number: Optional[str] = None + priority: Optional[int] = 3 + +class CallItemUpdate(BaseModel): + phone_number: Optional[str] = None + priority: Optional[int] = None + status: Optional[str] = None # "pending" or "done" + resolve_task: Optional[bool] = False # Also complete original task? + +class CallItemResponse(BaseModel): + id: int + person: str + reason: str + phone_number: Optional[str] + original_task_id: Optional[int] + priority: int + status: str + created_at: str +``` + +### 5. Daily Digest Integration + +Extend `generate_daily_digest()` in `app/services/digest.py` to query pending CallItems and include a `"pending_calls"` section in the response dict. + +### 6. Frontend Components + +- **Navigation**: Add "📞 Calls" button between "Projects" and "Daily Digest" +- **Calls View** (`#calls-view`): New section with call item list, inline phone edit +- **Task Item**: Add 📞 button to existing task action buttons +- **Digest View**: Render the `pending_calls` section when present + +## Data Models + +### CallItem Table Schema + +| Column | Type | Constraints | Default | +|--------|------|-------------|---------| +| id | INTEGER | PRIMARY KEY, INDEX | auto | +| person | VARCHAR(200) | NOT NULL | — | +| reason | VARCHAR(500) | NOT NULL | — | +| phone_number | VARCHAR(30) | NULLABLE | NULL | +| original_task_id | INTEGER | FK → tasks.id, NULLABLE | NULL | +| priority | INTEGER | — | 3 | +| status | VARCHAR | — | "pending" | +| created_at | DATETIME(tz) | — | CURRENT_TIMESTAMP | + +### Alembic Migration + +New migration file `005_add_call_items.py`: + +```python +def upgrade() -> None: + op.create_table('call_items', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('person', sa.String(200), nullable=False), + sa.Column('reason', sa.String(500), nullable=False), + sa.Column('phone_number', sa.String(30), nullable=True), + sa.Column('original_task_id', sa.Integer(), nullable=True), + sa.Column('priority', sa.Integer(), server_default='3'), + sa.Column('status', sa.String(), server_default="'pending'"), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('(CURRENT_TIMESTAMP)')), + sa.ForeignKeyConstraint(['original_task_id'], ['tasks.id']), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_call_items_id'), 'call_items', ['id'], unique=False) + op.create_index(op.f('ix_call_items_status'), 'call_items', ['status'], unique=False) + +def downgrade() -> None: + op.drop_index(op.f('ix_call_items_status'), table_name='call_items') + op.drop_index(op.f('ix_call_items_id'), table_name='call_items') + op.drop_table('call_items') +``` + +### Relationship to Existing Models + +```mermaid +erDiagram + tasks { + int id PK + string title + text description + int priority + string status + string origin + string source_id + } + call_items { + int id PK + string person + string reason + string phone_number + int original_task_id FK + int priority + string status + datetime created_at + } + tasks ||--o{ call_items : "original_task_id" +``` + +## Correctness Properties + +*A property is a characteristic or behavior that should hold true across all valid executions of a system — essentially, a formal statement about what the system should do. Properties serve as the bridge between human-readable specifications and machine-verifiable correctness guarantees.* + +### Property 1: Create-from-task field mapping + +*For any* valid Task with origin "gmail" or "dhive_email" and a description starting with "From: {name}", creating a CallItem from that task SHALL produce a CallItem where: person equals the extracted sender name, reason equals the task title, original_task_id equals the task ID, priority is 3, and status is "pending". + +**Validates: Requirements 1.1, 1.2, 1.3, 1.5** + +### Property 2: Phone number extraction from German formats + +*For any* text string containing a valid German phone number (in +49, 0xxx, or (0xxx) format), the Phone_Extractor SHALL return a non-null result matching the first phone number occurrence in the text. + +**Validates: Requirements 4.1, 4.2, 4.3, 4.4** + +### Property 3: Phone number parse-format round-trip + +*For any* valid German phone number string, parsing then formatting then parsing again SHALL produce an equivalent normalized result to parsing once. + +**Validates: Requirements 4.6** + +### Property 4: Call list ordering + +*For any* set of pending CallItems with varying priorities and creation dates, the GET /api/calls response SHALL return items sorted by priority ascending, then by created_at ascending within the same priority. + +**Validates: Requirements 2.2** + +### Property 5: Field length truncation + +*For any* string input to person (>200 chars), reason (>500 chars), or phone_number (>30 chars) fields, the stored value SHALL be truncated to the respective maximum length, and the stored value SHALL equal the first N characters of the input. + +**Validates: Requirements 9.1, 9.2, 9.3** + +### Property 6: Priority clamping + +*For any* integer value provided as priority, the stored priority SHALL be clamped to the range [1, 4] — values below 1 become 1, values above 4 become 4. + +**Validates: Requirements 9.4** + +### Property 7: Status constraint + +*For any* status value provided to a CallItem update, only "pending" and "done" SHALL be accepted; any other value SHALL be rejected or ignored. + +**Validates: Requirements 3.3** + +### Property 8: Digest includes all pending calls + +*For any* non-empty set of CallItems with status "pending", the generated daily digest SHALL include a "pending_calls" section containing an entry for each pending item with its person name and reason. + +**Validates: Requirements 7.1, 7.2** + +### Property 9: Call completion without task resolution + +*For any* CallItem with an original_task_id, marking it as "done" with resolve_task=false SHALL update only the CallItem status to "done" without modifying the Original_Task status. + +**Validates: Requirements 8.3** + +## Error Handling + +### API Error Responses + +| Scenario | HTTP Status | Response Body | +|----------|-------------|---------------| +| Unauthenticated request | 401 | `{"detail": "Not authenticated"}` | +| CallItem not found | 404 | `{"detail": "Call item not found"}` | +| Missing required field (person/reason) | 422 | Pydantic validation error | +| Invalid status value | 400 | `{"detail": "Status must be 'pending' or 'done'"}` | +| Referenced task_id not found | 404 | `{"detail": "Task not found"}` | + +### Phone Extractor Error Handling + +- Empty or None input → returns None (no exception) +- Text with no phone numbers → returns None +- Malformed numbers (too short/long) → skipped by regex, not returned +- Non-string input → TypeError raised (caller responsibility) + +### Task Resolution Errors + +When completing a call with `resolve_task=true`: +- If the original task no longer exists → log warning, complete call item only +- If email archiving fails → log error, still mark task as completed locally +- Follows the same error handling pattern as existing `update_task` endpoint + +## Testing Strategy + +### Property-Based Tests (using Hypothesis) + +The project uses Python, so **Hypothesis** is the property-based testing library. Each property test runs a minimum of 100 iterations. + +Tests will be placed in `OrgMyLife/tests/test_call_list.py` and `OrgMyLife/tests/test_phone_extractor.py`. + +**Property tests cover:** +- Phone number extraction (Properties 2, 3) +- Field truncation and priority clamping (Properties 5, 6) +- Status constraint validation (Property 7) +- Create-from-task field mapping (Property 1) +- Call list ordering (Property 4) +- Digest integration (Property 8) +- Completion without resolution (Property 9) + +Each test is tagged with: `# Feature: call-list, Property {N}: {description}` + +Configuration: `@settings(max_examples=100)` + +### Unit Tests (example-based) + +- Authentication enforcement (401 for all endpoints) +- 404 for non-existent IDs +- DELETE removes item from database +- POST without person/reason returns 422 +- Phone extractor returns None for text without numbers +- Digest omits calls section when no pending items exist +- Frontend renders tel: link for items with phone numbers + +### Integration Tests + +- Create call from task → verify task side effects (email archiving) on completion +- Full flow: create from task → mark done with resolve_task=true → verify original task completed +- Alembic migration up/down cycle + +### Test File Structure + +``` +OrgMyLife/tests/ +├── test_phone_extractor.py # Property tests for extraction + round-trip +├── test_call_list.py # Property + unit tests for API and service logic +└── test_call_digest.py # Property tests for digest integration +``` diff --git a/.kiro/specs/call-list/requirements.md b/.kiro/specs/call-list/requirements.md new file mode 100644 index 0000000..c6003ea --- /dev/null +++ b/.kiro/specs/call-list/requirements.md @@ -0,0 +1,122 @@ +# Requirements Document + +## Introduction + +The Call List feature adds a dedicated "📞 Calls" tab to OrgMyLife that acts as a separate funnel for phone-call tasks. Users can promote any existing task to the Call List, where it is tracked with call-specific metadata (person, phone number, reason). Phone numbers are auto-extracted from email bodies using German format patterns. The Call List integrates with the Daily Digest to surface pending calls. + +## Glossary + +- **Call_List_Service**: The backend service responsible for managing call items, including creation, retrieval, updates, and deletion via the REST API. +- **Call_Item**: A database entity representing a phone call to be made, containing person name, reason, phone number, priority, status, and a reference to the originating task. +- **Phone_Extractor**: The component that scans email body text and extracts phone numbers matching German telephone formats. +- **Frontend_Call_View**: The UI tab ("📞 Calls") that displays pending and completed call items with sorting and action controls. +- **Task_List_View**: The existing "All Tasks" UI view showing open tasks with action buttons. +- **Daily_Digest_Service**: The existing service that generates the daily summary of overdue tasks, priorities, and schedule. +- **Original_Task**: The existing Task entity from which a Call_Item is derived. + +## Requirements + +### Requirement 1: Create Call Item from Task + +**User Story:** As a user, I want to move a task to the Call List by clicking a phone icon, so that I can track phone calls separately from regular tasks. + +#### Acceptance Criteria + +1. WHEN the user clicks the phone icon button on a task in the Task_List_View, THE Call_List_Service SHALL create a new Call_Item linked to the Original_Task. +2. WHEN a Call_Item is created from an Original_Task, THE Call_List_Service SHALL populate the Call_Item person field with the task sender name if the origin is "gmail" or "dhive_email". +3. WHEN a Call_Item is created from an Original_Task, THE Call_List_Service SHALL populate the Call_Item reason field with the Original_Task title. +4. WHEN a Call_Item is created from an Original_Task that has an email body, THE Phone_Extractor SHALL scan the email body and populate the Call_Item phone_number field with the first extracted number. +5. WHEN a Call_Item is created, THE Call_List_Service SHALL assign a default priority of 3 (Medium) and a status of "pending". + +### Requirement 2: Call List REST API + +**User Story:** As a developer, I want a complete CRUD API for call items, so that the frontend and external integrations can manage the Call List. + +#### Acceptance Criteria + +1. WHEN a POST request is sent to /api/calls with valid call item data, THE Call_List_Service SHALL create a new Call_Item and return the created item with its ID. +2. WHEN a GET request is sent to /api/calls, THE Call_List_Service SHALL return all Call_Items with status "pending" ordered by priority then creation date. +3. WHEN a PUT request is sent to /api/calls/{id} with update data, THE Call_List_Service SHALL update the specified Call_Item fields and return the updated item. +4. WHEN a DELETE request is sent to /api/calls/{id}, THE Call_List_Service SHALL remove the specified Call_Item from the database. +5. WHEN an unauthenticated request is sent to any /api/calls endpoint, THE Call_List_Service SHALL return HTTP 401. +6. WHEN a request references a non-existent Call_Item ID, THE Call_List_Service SHALL return HTTP 404. + +### Requirement 3: Call Item Data Model + +**User Story:** As a developer, I want a dedicated CallItem model with call-specific fields, so that call metadata is stored separately from general tasks. + +#### Acceptance Criteria + +1. THE Call_Item SHALL store the following fields: id (integer, primary key), person (string, required), reason (string, required), phone_number (string, nullable), original_task_id (integer, nullable foreign key to tasks), priority (integer, default 3), status (string, default "pending"), created_at (timestamp with timezone). +2. WHEN a Call_Item references an Original_Task, THE Call_Item original_task_id SHALL be a valid foreign key to the tasks table. +3. THE Call_Item status field SHALL accept only the values "pending" and "done". + +### Requirement 4: Phone Number Extraction from Email + +**User Story:** As a user, I want phone numbers automatically extracted from email bodies, so that I do not have to manually look up contact numbers. + +#### Acceptance Criteria + +1. WHEN an email body is provided, THE Phone_Extractor SHALL identify phone numbers in the format +49 followed by 9 to 12 digits with optional spaces or hyphens. +2. WHEN an email body is provided, THE Phone_Extractor SHALL identify phone numbers in the format 0 followed by 3 to 5 digit area code and 4 to 8 digit subscriber number with optional separators. +3. WHEN an email body is provided, THE Phone_Extractor SHALL identify phone numbers in the format (0xxx) followed by subscriber digits with optional separators. +4. WHEN multiple phone numbers are found in an email body, THE Phone_Extractor SHALL return the first match. +5. WHEN no phone number is found in an email body, THE Phone_Extractor SHALL return null. +6. FOR ALL valid German phone number strings, parsing then formatting then parsing SHALL produce an equivalent normalized result (round-trip property). + +### Requirement 5: Call List Frontend Tab + +**User Story:** As a user, I want a dedicated "📞 Calls" tab in the navigation, so that I can view and manage my pending calls in one place. + +#### Acceptance Criteria + +1. THE Frontend_Call_View SHALL display a "📞 Calls" navigation button in the top navigation bar between "Projects" and "Daily Digest". +2. WHEN the user navigates to the Calls tab, THE Frontend_Call_View SHALL display all pending Call_Items showing person name, reason, phone number, and priority. +3. WHEN a Call_Item has a phone_number, THE Frontend_Call_View SHALL display the phone number as a clickable tel: link. +4. THE Frontend_Call_View SHALL provide a button to mark a Call_Item as done. +5. THE Frontend_Call_View SHALL provide a button to delete a Call_Item. +6. THE Frontend_Call_View SHALL allow editing the phone_number field inline. +7. THE Frontend_Call_View SHALL order Call_Items by priority (ascending) then by creation date (ascending). + +### Requirement 6: Phone Icon on Task Items + +**User Story:** As a user, I want a phone icon button on each task, so that I can quickly send any task to the Call List. + +#### Acceptance Criteria + +1. THE Task_List_View SHALL display a 📞 icon button on each task item alongside the existing action buttons. +2. WHEN the user clicks the 📞 button on a task, THE Frontend_Call_View SHALL send a POST request to /api/calls with the task data. +3. WHEN the Call_Item is successfully created from a task, THE Frontend_Call_View SHALL display a confirmation indicator to the user. + +### Requirement 7: Daily Digest Integration + +**User Story:** As a user, I want to see pending calls in my Daily Digest, so that I am reminded of calls I need to make today. + +#### Acceptance Criteria + +1. WHEN the Daily Digest is generated, THE Daily_Digest_Service SHALL include a "📞 Calls to make" section listing all Call_Items with status "pending". +2. WHEN there are pending Call_Items, THE Daily_Digest_Service SHALL display each item with person name and reason. +3. WHEN there are no pending Call_Items, THE Daily_Digest_Service SHALL omit the calls section from the digest. + +### Requirement 8: Complete Call and Resolve Original Task + +**User Story:** As a user, I want the option to mark the original task as done when I complete a call, so that I do not have to update both items manually. + +#### Acceptance Criteria + +1. WHEN a Call_Item is marked as done and the Call_Item has an original_task_id, THE Call_List_Service SHALL offer the option to also mark the Original_Task as "completed". +2. WHEN the user confirms marking the Original_Task as completed, THE Call_List_Service SHALL update the Original_Task status to "completed" and trigger the same side effects as the existing task completion flow (email archiving, Nextcloud sync). +3. WHEN the user declines marking the Original_Task as completed, THE Call_List_Service SHALL only update the Call_Item status to "done" without modifying the Original_Task. + +### Requirement 9: Input Validation + +**User Story:** As a developer, I want call item inputs validated, so that the database contains only well-formed data. + +#### Acceptance Criteria + +1. WHEN a POST or PUT request is sent with a person field exceeding 200 characters, THE Call_List_Service SHALL truncate the value to 200 characters. +2. WHEN a POST or PUT request is sent with a reason field exceeding 500 characters, THE Call_List_Service SHALL truncate the value to 500 characters. +3. WHEN a POST or PUT request is sent with a phone_number field exceeding 30 characters, THE Call_List_Service SHALL truncate the value to 30 characters. +4. WHEN a POST or PUT request is sent with a priority value outside the range 1 to 4, THE Call_List_Service SHALL clamp the value to the nearest valid bound. +5. WHEN a POST request is sent without a person field, THE Call_List_Service SHALL return HTTP 422 with a validation error. +6. WHEN a POST request is sent without a reason field, THE Call_List_Service SHALL return HTTP 422 with a validation error. diff --git a/.kiro/specs/call-list/tasks.md b/.kiro/specs/call-list/tasks.md new file mode 100644 index 0000000..11da9ac --- /dev/null +++ b/.kiro/specs/call-list/tasks.md @@ -0,0 +1,194 @@ +# Implementation Plan: Call List Feature + +## Overview + +This plan implements the Call List feature for OrgMyLife — a dedicated phone-call tracking system with a new `CallItem` model, CRUD API, phone number extraction service, frontend "📞 Calls" tab, and Daily Digest integration. Tasks are ordered so each step builds on the previous, with no orphaned code. + +## Tasks + +- [x] 1. Database model and migration + - [x] 1.1 Add CallItem model to `app/models.py` + - Add `CallItem` SQLAlchemy class with fields: id, person, reason, phone_number, original_task_id (FK → tasks.id), priority, status, created_at + - Follow existing model patterns (Column types, server_default for timestamps) + - _Requirements: 3.1, 3.2, 3.3_ + + - [x] 1.2 Create Alembic migration `005_add_call_items.py` + - Create `call_items` table with all columns, FK constraint, and indexes on id and status + - Include `downgrade()` that drops the table and indexes + - _Requirements: 3.1, 3.2_ + +- [x] 2. Phone extractor service + - [x] 2.1 Create `app/services/phone_extractor.py` + - Implement `extract_phone_number(text: str) -> str | None` with regex patterns for: +49 international, 0xxx domestic, (0xxx) parenthesized area code + - Implement `normalize_phone_number(raw: str) -> str` to strip separators and standardize + - Implement `parse_phone_number(text: str) -> dict | None` to decompose into country_code, area_code, subscriber + - Return first match when multiple numbers found; return None when no match + - _Requirements: 4.1, 4.2, 4.3, 4.4, 4.5_ + + - [ ]* 2.2 Write property test: Phone number extraction (Property 2) + - **Property 2: Phone number extraction from German formats** + - Generate random valid German phone numbers in +49, 0xxx, and (0xxx) formats embedded in arbitrary text + - Assert `extract_phone_number` returns a non-null result matching the first occurrence + - Use Hypothesis with `@settings(max_examples=100)` + - Place in `OrgMyLife/tests/test_phone_extractor.py` + - **Validates: Requirements 4.1, 4.2, 4.3, 4.4** + + - [ ]* 2.3 Write property test: Phone number round-trip (Property 3) + - **Property 3: Phone number parse-format round-trip** + - Generate valid German phone number strings, parse → normalize → parse again + - Assert the two parse results are equivalent + - Use Hypothesis with `@settings(max_examples=100)` + - Place in `OrgMyLife/tests/test_phone_extractor.py` + - **Validates: Requirements 4.6** + +- [x] 3. Call List API routes and service logic + - [x] 3.1 Add Pydantic models for Call List in `app/main.py` + - `CallItemCreate`: task_id (optional), person (optional), reason (optional), phone_number (optional), priority (default 3) + - `CallItemUpdate`: phone_number, priority, status, resolve_task (bool) + - `CallItemResponse`: id, person, reason, phone_number, original_task_id, priority, status, created_at + - Implement field truncation (person≤200, reason≤500, phone_number≤30) and priority clamping [1,4] in `model_post_init` + - _Requirements: 9.1, 9.2, 9.3, 9.4, 9.5, 9.6_ + + - [x] 3.2 Implement POST `/api/calls` endpoint + - If `task_id` provided: load task, extract sender name from description, extract phone number via `phone_extractor`, create CallItem with person=sender, reason=task.title, original_task_id=task.id + - If no `task_id`: require person and reason fields, return 422 if missing + - Assign default priority=3, status="pending" + - Require session auth (401 if unauthenticated) + - _Requirements: 1.1, 1.2, 1.3, 1.4, 1.5, 2.1, 2.5_ + + - [x] 3.3 Implement GET `/api/calls` endpoint + - Return all CallItems with status="pending", ordered by priority ASC then created_at ASC + - Require session auth + - _Requirements: 2.2, 2.5_ + + - [x] 3.4 Implement PUT `/api/calls/{id}` endpoint + - Update specified fields (phone_number, priority, status) + - Validate status is "pending" or "done" only; reject others with 400 + - If status="done" and resolve_task=true and original_task_id exists: mark original task as "completed" and trigger existing side effects (email archiving, Nextcloud sync) + - If status="done" and resolve_task=false: only update CallItem + - Return 404 if ID not found; require session auth + - _Requirements: 2.3, 2.5, 2.6, 3.3, 8.1, 8.2, 8.3_ + + - [x] 3.5 Implement DELETE `/api/calls/{id}` endpoint + - Remove CallItem from database + - Return 404 if ID not found; require session auth + - _Requirements: 2.4, 2.5, 2.6_ + + - [ ]* 3.6 Write property test: Field truncation (Property 5) + - **Property 5: Field length truncation** + - Generate strings longer than max lengths for person, reason, phone_number + - Assert stored values are truncated to 200, 500, 30 chars respectively and equal the first N chars of input + - Use Hypothesis with `@settings(max_examples=100)` + - Place in `OrgMyLife/tests/test_call_list.py` + - **Validates: Requirements 9.1, 9.2, 9.3** + + - [ ]* 3.7 Write property test: Priority clamping (Property 6) + - **Property 6: Priority clamping** + - Generate arbitrary integers, assert stored priority is clamped to [1, 4] + - Use Hypothesis with `@settings(max_examples=100)` + - Place in `OrgMyLife/tests/test_call_list.py` + - **Validates: Requirements 9.4** + + - [ ]* 3.8 Write property test: Status constraint (Property 7) + - **Property 7: Status constraint** + - Generate arbitrary status strings, assert only "pending" and "done" are accepted + - Use Hypothesis with `@settings(max_examples=100)` + - Place in `OrgMyLife/tests/test_call_list.py` + - **Validates: Requirements 3.3** + +- [x] 4. Checkpoint - Ensure all tests pass + - Ensure all tests pass, ask the user if questions arise. + +- [x] 5. Frontend: Calls tab and phone icon on tasks + - [x] 5.1 Add "📞 Calls" navigation button to `frontend/index.html` + - Insert new nav button between "Projects" and "Daily Digest" + - Add `#calls-view` section with call items list container + - _Requirements: 5.1_ + + - [x] 5.2 Implement Calls tab view in `frontend/js/app.js` + - Add `loadCalls()` function: fetch GET `/api/calls`, render list with person, reason, phone (as `tel:` link), priority badge + - Add "Mark Done" button per item (sends PUT with status="done", prompts about resolving original task if original_task_id exists) + - Add "Delete" button per item (sends DELETE) + - Add inline edit for phone_number field (sends PUT on blur/enter) + - Order items by priority then creation date + - Wire "📞 Calls" nav button to `switchTab("calls")` + - _Requirements: 5.2, 5.3, 5.4, 5.5, 5.6, 5.7_ + + - [x] 5.3 Add 📞 icon button to task items in Task List view + - Add phone icon button to existing task action buttons in `loadAllTasks()` render + - On click: send POST `/api/calls` with `{ task_id: }` + - Show confirmation indicator (brief flash or text) on success + - _Requirements: 6.1, 6.2, 6.3_ + + - [x] 5.4 Add CSS styles for Calls view in `frontend/css/style.css` + - Style call item cards, tel: links, inline phone edit, priority badges + - Match existing card/list styling patterns + - _Requirements: 5.2, 5.3_ + +- [x] 6. Daily Digest integration + - [x] 6.1 Extend `generate_daily_digest()` in `app/services/digest.py` + - Import CallItem model + - Query all CallItems with status="pending" + - Add `"pending_calls"` section to response dict with person and reason for each item + - Omit section if no pending calls exist + - _Requirements: 7.1, 7.2, 7.3_ + + - [x] 6.2 Render pending calls in Digest frontend view + - In `loadDigest()`, render the `pending_calls` section when present in API response + - Display as a "📞 Calls to make" card with person name and reason per item + - _Requirements: 7.1, 7.2_ + + - [ ]* 6.3 Write property test: Digest includes all pending calls (Property 8) + - **Property 8: Digest includes all pending calls** + - Generate a non-empty set of pending CallItems, run `generate_daily_digest` + - Assert response contains `pending_calls` with an entry for each pending item (person + reason) + - Use Hypothesis with `@settings(max_examples=100)` + - Place in `OrgMyLife/tests/test_call_digest.py` + - **Validates: Requirements 7.1, 7.2** + +- [x] 7. Create-from-task logic and completion flow tests + - [ ]* 7.1 Write property test: Create-from-task field mapping (Property 1) + - **Property 1: Create-from-task field mapping** + - Generate tasks with origin "gmail"/"dhive_email" and description starting with "From: {name}" + - Assert created CallItem has: person=extracted sender, reason=task.title, original_task_id=task.id, priority=3, status="pending" + - Use Hypothesis with `@settings(max_examples=100)` + - Place in `OrgMyLife/tests/test_call_list.py` + - **Validates: Requirements 1.1, 1.2, 1.3, 1.5** + + - [ ]* 7.2 Write property test: Call list ordering (Property 4) + - **Property 4: Call list ordering** + - Generate sets of pending CallItems with varying priorities and creation dates + - Assert GET `/api/calls` returns items sorted by priority ASC, then created_at ASC + - Use Hypothesis with `@settings(max_examples=100)` + - Place in `OrgMyLife/tests/test_call_list.py` + - **Validates: Requirements 2.2** + + - [ ]* 7.3 Write property test: Completion without task resolution (Property 9) + - **Property 9: Call completion without task resolution** + - Generate CallItems with original_task_id, mark as done with resolve_task=false + - Assert only CallItem status changes; original task status remains unchanged + - Use Hypothesis with `@settings(max_examples=100)` + - Place in `OrgMyLife/tests/test_call_list.py` + - **Validates: Requirements 8.3** + + - [ ]* 7.4 Write unit tests for API edge cases + - Test 401 for unauthenticated requests to all /api/calls endpoints + - Test 404 for non-existent call item IDs + - Test 422 for POST without person/reason (when no task_id) + - Test DELETE removes item from database + - Test phone extractor returns None for text without numbers + - Place in `OrgMyLife/tests/test_call_list.py` + - _Requirements: 2.5, 2.6, 4.5, 9.5, 9.6_ + +- [x] 8. Final checkpoint - Ensure all tests pass + - Ensure all tests pass, ask the user if questions arise. + +## Notes + +- Tasks marked with `*` are optional and can be skipped for faster MVP +- Each task references specific requirements for traceability +- Checkpoints ensure incremental validation +- Property tests validate universal correctness properties from the design document (Hypothesis, min 100 examples) +- Unit tests validate specific examples and edge cases +- The project uses Python (FastAPI + SQLAlchemy + Alembic) for backend and vanilla HTML/CSS/JS for frontend +- Test files: `tests/test_phone_extractor.py`, `tests/test_call_list.py`, `tests/test_call_digest.py` diff --git a/.kiro/specs/jury-voting/.config.kiro b/.kiro/specs/jury-voting/.config.kiro new file mode 100644 index 0000000..5351837 --- /dev/null +++ b/.kiro/specs/jury-voting/.config.kiro @@ -0,0 +1 @@ +{"specId": "e33fa6e5-89e0-4d8d-ad9e-d43d6bf23daa", "workflowType": "requirements-first", "specType": "feature"} diff --git a/.kiro/specs/jury-voting/design.md b/.kiro/specs/jury-voting/design.md new file mode 100644 index 0000000..7e8ea7e --- /dev/null +++ b/.kiro/specs/jury-voting/design.md @@ -0,0 +1,475 @@ +# Design Document: Jury Voting + +## Overview + +The Jury Voting app is a standalone web application that digitizes the paper-based evaluation process for the **15. UNIKAT Ideenwettbewerb**. It enables 6 jurors to score 11 teams on 5 criteria (1–10 each) via their smartphones, while a beamer/projector view shows live-updating rankings to the audience. + +The system is designed for a single-event use case with minimal infrastructure requirements — deployable on a laptop connected to the projector, or a simple server. The architecture prioritizes simplicity, reliability, and a premium visual experience matching the Projekt-KIQ-HP design language. + +**Key design decisions:** +- **Monorepo structure**: Single `Jury-Voting/` directory containing both frontend and backend +- **Express.js + JSON file backend**: Lightweight server with file-based persistence (no database needed for 6 jurors × 11 teams) +- **Server-Sent Events (SSE)** for real-time updates to the beamer view — simpler than WebSockets, no library needed, unidirectional (server → client) which is exactly what the results view needs +- **Optimistic UI with local fallback**: Scores are saved locally first, then synced to server. Handles brief connectivity loss gracefully. +- **No authentication complexity**: Simple name-based juror selection (no passwords) — appropriate for a trusted, in-person event + +## Architecture + +```mermaid +graph TB + subgraph "Client Devices" + J1[Juror Phone 1] + J2[Juror Phone 2] + J6[Juror Phone 6] + B[Beamer Browser] + A[Admin Laptop] + end + + subgraph "Server (single process)" + API[Express.js REST API] + SSE[SSE Endpoint] + FS[JSON File Store] + end + + J1 -->|POST /api/scores| API + J2 -->|POST /api/scores| API + J6 -->|POST /api/scores| API + A -->|Admin API calls| API + API -->|Read/Write| FS + API -->|Notify| SSE + SSE -->|EventStream| B + SSE -->|EventStream| A +``` + +### Architecture Decisions + +| Decision | Choice | Rationale | +|----------|--------|-----------| +| Backend | Express.js | Minimal setup, same language as frontend, easy to bundle | +| Persistence | JSON file | 6×11×5 = 330 score cells — trivial data volume, no DB overhead | +| Real-time | SSE (Server-Sent Events) | Unidirectional push is sufficient; no WebSocket library needed | +| Frontend | React 19 + Vite | Matches Projekt-KIQ-HP stack, fast HMR for development | +| Styling | CSS variables + glassmorphism | Reuses KIQ-HP design tokens, no CSS framework needed | +| Deployment | docker-compose OR `npm run dev` | Flexible: production-like or quick laptop setup | +| Offline handling | localStorage queue | Scores saved locally, synced on reconnect | + +### Deployment Topology + +```mermaid +graph LR + subgraph "Event Setup (laptop or server)" + V[Vite Dev Server :5173] + E[Express API :3001] + end + + subgraph "Network (local WiFi)" + P1[Phone 1] --> V + P6[Phone 6] --> V + BM[Beamer PC] --> V + end + + V --> E +``` + +For production, Vite builds static assets served by Express on a single port. For development, Vite proxies API calls to Express. + +## Components and Interfaces + +### Route Structure + +| Route | Component | Purpose | +|-------|-----------|---------| +| `/` | `JurorSelect` | Juror name selection (landing page) | +| `/vote` | `VotingDashboard` | Team list with scoring status | +| `/vote/:teamId` | `TeamScoring` | Score input for a specific team | +| `/results` | `ResultsView` | Beamer-optimized live rankings | +| `/admin` | `AdminPanel` | Session management, progress, export | + +### Frontend Component Tree + +```mermaid +graph TD + App --> Router + Router --> JurorSelect + Router --> VotingDashboard + Router --> TeamScoring + Router --> ResultsView + Router --> AdminPanel + + VotingDashboard --> TeamCard[TeamCard × 11] + TeamScoring --> ScoreSlider[ScoreSlider × 5] + TeamScoring --> TeamHeader + ResultsView --> RankingTable + ResultsView --> SessionStatus + AdminPanel --> ProgressMatrix + AdminPanel --> SessionControls + AdminPanel --> ExportButton +``` + +### Key Components + +#### `JurorSelect` +- Displays 6 juror names as large tap targets (glass-panel cards) +- On selection, stores juror identity in localStorage and navigates to `/vote` +- Shows warning if juror is already active in another session + +#### `VotingDashboard` +- Lists all 11 teams as cards +- Each card shows: team name, description snippet, scoring status (✓ complete / partial / not started) +- Tap navigates to `/vote/:teamId` + +#### `TeamScoring` +- Shows team name + description at top +- 5 score inputs (one per criterion), each a slider or number stepper (1–10) +- Auto-saves on change (debounced 300ms) +- Visual confirmation on save (green flash) +- Navigation: prev/next team buttons + +#### `ResultsView` (Beamer) +- Full-screen, no navigation chrome +- Animated ranking table with team names, scores, ranks +- Updates via SSE — smooth transitions when scores change +- Shows "Voting in Progress" or "Final Results" status +- Designed for 1920×1080 landscape + +#### `AdminPanel` +- Session controls: Start/End voting session +- Progress matrix: 6 jurors × 11 teams grid showing completion +- Export button (CSV download) +- Juror/team configuration (pre-populated) + +### API Endpoints + +| Method | Endpoint | Purpose | +|--------|----------|---------| +| `GET` | `/api/session` | Get current session state | +| `POST` | `/api/session/start` | Start voting session | +| `POST` | `/api/session/end` | End voting session | +| `GET` | `/api/teams` | List all teams | +| `GET` | `/api/jurors` | List all jurors | +| `GET` | `/api/scores` | Get all scores (admin) | +| `GET` | `/api/scores/:jurorId` | Get scores for a juror | +| `POST` | `/api/scores` | Submit/update a score | +| `GET` | `/api/results` | Get aggregated results | +| `GET` | `/api/results/stream` | SSE endpoint for live updates | +| `GET` | `/api/export/csv` | Download CSV export | +| `GET` | `/api/progress` | Get juror completion matrix | + +### Score Submission Payload + +```json +{ + "jurorId": "finke", + "teamId": "dual", + "criterionId": "gesamtidee", + "score": 8 +} +``` + +### SSE Event Format + +```json +{ + "event": "scores_updated", + "data": { + "results": [ + { "teamId": "dual", "teamName": "DUAL", "total": 187, "average": 31.2, "rank": 1 } + ], + "timestamp": "2026-05-19T18:30:00Z" + } +} +``` + +## Data Models + +### Session + +```typescript +interface Session { + id: string; + status: 'setup' | 'active' | 'ended'; + startedAt: string | null; // ISO timestamp + endedAt: string | null; // ISO timestamp + config: SessionConfig; +} + +interface SessionConfig { + criteria: Criterion[]; + scaleMin: number; // default: 1 + scaleMax: number; // default: 10 +} +``` + +### Team + +```typescript +interface Team { + id: string; // kebab-case slug, e.g. "dual" + number: number; // original number from submission + name: string; // display name, e.g. "DUAL" + description: string; // short description + contact?: string; // contact person (optional) +} +``` + +### Juror + +```typescript +interface Juror { + id: string; // lowercase name, e.g. "finke" + name: string; // display name, e.g. "Finke" + active: boolean; // currently has an active session +} +``` + +### Criterion + +```typescript +interface Criterion { + id: string; // kebab-case, e.g. "gesamtidee" + nameDE: string; // German name, e.g. "Gesamtidee" + nameEN: string; // English name, e.g. "Overall Idea" +} +``` + +### Score + +```typescript +interface Score { + jurorId: string; + teamId: string; + criterionId: string; + value: number; // 1–10 + updatedAt: string; // ISO timestamp +} +``` + +### Aggregated Result + +```typescript +interface TeamResult { + teamId: string; + teamName: string; + total: number; // sum of all juror scores across all criteria + average: number; // total / number of jurors who scored + rank: number; // 1-based, ties get same rank + jurorCount: number; // how many jurors have scored this team + complete: boolean; // all 6 jurors scored all 5 criteria +} +``` + +### JSON File Structure (persistence) + +```json +{ + "session": { "id": "unikat-2026", "status": "active", "startedAt": "...", "endedAt": null, "config": { ... } }, + "teams": [ ... ], + "jurors": [ ... ], + "scores": [ + { "jurorId": "finke", "teamId": "dual", "criterionId": "gesamtidee", "value": 8, "updatedAt": "..." } + ] +} +``` + +The entire state fits in a single JSON file (~50KB max). The server reads/writes this file atomically on each score submission. + +### Score Aggregation Logic + +``` +Per team: + total = SUM(score.value) for all scores where score.teamId == team.id + jurorCount = COUNT(DISTINCT score.jurorId) where score.teamId == team.id AND all 5 criteria submitted + average = total / jurorCount (or total / 6 if all jurors scored) + +Ranking: + Sort teams by total DESC + Assign rank 1, 2, 3... with ties getting the same rank (dense ranking) + Example: scores [250, 240, 240, 230] → ranks [1, 2, 2, 4] +``` + +### Offline Queue (localStorage) + +```typescript +interface PendingScore { + jurorId: string; + teamId: string; + criterionId: string; + value: number; + timestamp: string; + synced: boolean; +} +``` + +When offline, scores are pushed to a `pendingScores` array in localStorage. On reconnect, the app replays them to the server in order. The server uses `updatedAt` to resolve conflicts (last-write-wins). + + +## Correctness Properties + +*A property is a characteristic or behavior that should hold true across all valid executions of a system — essentially, a formal statement about what the system should do. Properties serve as the bridge between human-readable specifications and machine-verifiable correctness guarantees.* + +### Property 1: Team Completion Status + +*For any* juror and any set of submitted scores, a team's completion status SHALL be "complete" if and only if exactly 5 scores exist (one per criterion) for that juror-team pair, "partial" if between 1 and 4 scores exist, and "pending" if zero scores exist. + +**Validates: Requirements 2.4, 7.1** + +### Property 2: Offline Queue Preservation + +*For any* sequence of scores submitted while offline, after synchronization completes, the server state SHALL contain all scores from the offline queue with their original values and timestamps preserved. + +**Validates: Requirements 3.2** + +### Property 3: Last-Write-Wins Score Resolution + +*For any* sequence of score updates to the same (jurorId, teamId, criterionId) triple, the final persisted value SHALL equal the value of the most recent update (by timestamp). + +**Validates: Requirements 3.3** + +### Property 4: Score Aggregation Correctness + +*For any* team and any set of valid scores (each 1–10), the total score SHALL equal the arithmetic sum of all individual score values, and the average SHALL equal the total divided by the number of distinct jurors who have submitted at least one score for that team. + +**Validates: Requirements 4.2, 4.3** + +### Property 5: Ranking Correctness with Ties + +*For any* set of team totals, the ranking SHALL be sorted in descending order by total score, and any two teams with equal total scores SHALL receive the same rank number. + +**Validates: Requirements 4.4, 4.5** + +### Property 6: Session Gate + +*For any* score submission attempt, the system SHALL accept the score if and only if the current session status is "active". Submissions when session status is "setup" or "ended" SHALL be rejected. + +**Validates: Requirements 6.4, 6.5** + +### Property 7: Progress Privacy + +*For any* call to the progress endpoint, the response SHALL contain juror completion counts (number of fully-scored teams per juror) but SHALL NOT contain any individual score values. + +**Validates: Requirements 7.3** + +### Property 8: Export Round-Trip + +*For any* complete set of scores, exporting to CSV and parsing the result SHALL recover: all individual scores per (juror, team, criterion), correct aggregated totals, correct averages, correct rankings, with teams as rows and (juror × criterion) as columns. + +**Validates: Requirements 8.1, 8.2, 8.3, 8.4** + +### Property 9: Results Display Completeness + +*For any* TeamResult object, the rendered results view SHALL include the team name, total score, average score, and current rank. + +**Validates: Requirements 5.4** + +## Error Handling + +### Network Errors (Juror Phones) + +| Scenario | Handling | +|----------|----------| +| Score submission fails (timeout/5xx) | Store in localStorage queue, show subtle "offline" indicator, retry on reconnect | +| Server unreachable | Switch to offline mode, continue scoring locally, sync when back | +| Partial sync failure | Retry individual failed scores with exponential backoff (max 3 retries) | + +### Data Validation Errors + +| Scenario | Handling | +|----------|----------| +| Score out of range (< 1 or > 10) | Reject at client and server, show validation error | +| Unknown jurorId or teamId | Return 400 with descriptive error message | +| Score submission when session not active | Return 403 with "Session not active" message | +| Duplicate score without overwrite | Return 409 with current value, client prompts for confirmation | + +### Server Errors + +| Scenario | Handling | +|----------|----------| +| JSON file write failure | Return 500, log error, scores remain in client queue | +| JSON file corruption | Keep backup copy (`.bak`), restore from backup on startup | +| SSE connection dropped | Client auto-reconnects with `EventSource` built-in retry | + +### Admin Errors + +| Scenario | Handling | +|----------|----------| +| End session with incomplete scores | Show warning with progress summary, require confirmation | +| Export with no scores | Return empty CSV with headers only | + +## Testing Strategy + +### Property-Based Testing + +**Library:** [fast-check](https://github.com/dubzzz/fast-check) (JavaScript/TypeScript PBT library) + +Property-based tests will validate the 9 correctness properties defined above. Each test runs a minimum of **100 iterations** with randomly generated inputs. + +**Test configuration:** +```typescript +fc.assert(fc.property(...), { numRuns: 100 }) +``` + +**Tag format:** Each test file includes a comment referencing the design property: +```typescript +// Feature: jury-voting, Property 4: Score Aggregation Correctness +``` + +**Properties to test:** +1. Team completion status computation (pure function) +2. Offline queue sync logic (mock server) +3. Last-write-wins resolution (pure function) +4. Score aggregation (pure function — sum and average) +5. Ranking with ties (pure function — sort + rank assignment) +6. Session gate logic (pure function — accept/reject based on status) +7. Progress privacy (API response shape validation) +8. Export round-trip (generate scores → export CSV → parse → compare) +9. Results display completeness (render → check fields present) + +### Unit Tests (Example-Based) + +Unit tests cover specific scenarios, edge cases, and integration points: + +- **JurorSelect**: Renders 6 names, handles selection, shows conflict warning +- **TeamScoring**: Renders 5 criteria, auto-saves on change, shows confirmation +- **VotingDashboard**: Shows correct status icons for complete/partial/pending teams +- **ResultsView**: Shows "Voting in Progress" vs "Final Results" based on session state +- **AdminPanel**: Session start/end controls, export button +- **API endpoints**: Correct HTTP status codes, validation errors, auth checks +- **SSE**: Connection, reconnection, event parsing + +### Integration Tests + +- Full scoring flow: juror selects name → scores a team → results update +- Offline/online cycle: score while offline → reconnect → verify sync +- Session lifecycle: create → start → score → end → export +- Concurrent jurors: 6 simultaneous score submissions + +### Test Structure + +``` +Jury-Voting/ +├── src/ +│ ├── __tests__/ # Unit tests (co-located) +│ └── ... +├── server/ +│ ├── __tests__/ # Server unit tests +│ └── ... +└── tests/ + ├── properties/ # Property-based tests + │ ├── aggregation.property.test.ts + │ ├── ranking.property.test.ts + │ ├── completion.property.test.ts + │ ├── session-gate.property.test.ts + │ ├── offline-queue.property.test.ts + │ ├── last-write-wins.property.test.ts + │ ├── progress-privacy.property.test.ts + │ ├── export-roundtrip.property.test.ts + │ └── results-display.property.test.ts + └── integration/ # Integration tests + ├── scoring-flow.test.ts + └── session-lifecycle.test.ts +``` + +### Test Runner + +- **Vitest** — fast, Vite-native, supports TypeScript, works with fast-check +- Run: `npx vitest --run` (single execution, no watch mode) diff --git a/.kiro/specs/jury-voting/requirements.md b/.kiro/specs/jury-voting/requirements.md new file mode 100644 index 0000000..af3fa5b --- /dev/null +++ b/.kiro/specs/jury-voting/requirements.md @@ -0,0 +1,192 @@ +# Requirements Document + +## Introduction + +This feature replaces the paper-based jury evaluation process (Bewertungsbogen and Auswertungstabelle) for the **15. UNIKAT Ideenwettbewerb** with a digital, mobile-friendly voting application. A panel of **6 jurors** evaluates **11 teams** via their smartphones during the Prämierungsfeier on **19. Mai 2026, 18 Uhr** at **Science Park Kassel, Universitätsplatz 12** (hosted by Universität Kassel). Results are displayed in real time on a beamer/projector for the audience. + +**Scoring system:** 5 equally-weighted criteria, each scored on a 1–10 scale (configurable). Maximum score per juror per team: 50. Aggregation: simple sum across all jurors, average = total / number of jurors. + +**Visual design basis:** Projekt-KIQ-HP (React 19 + Vite) — dark theme (#0a0a0a background), neon green accents (#00FF00), glassmorphism panels, Outfit font, lucide-react icons. Premium event look. + +**Tech stack:** React 19 + Vite, react-router-dom, lucide-react icons. NO Supabase — use a lightweight real-time backend (e.g., simple JSON server, or localStorage + polling for MVP since only 6 jurors). + +**Reference documents (in repo):** +- `Jury-Voting/Teams/` — folder containing team/project descriptions (11 teams) +- `Jury-Voting/UNIKAT_IDEENWETTBEWERB_2026_Einladung_Prämierungsfeier.pdf` — event invitation +- `Jury-Voting/Bewertungsbogen Jury 2026.doc` — the paper scoring form (5 criteria, 1–10 scale) +- `Jury-Voting/Auswertungstabelle 2026.xlsx` — the paper results table (simple sum aggregation) + +## Glossary + +- **Voting_App**: The web application that jurors and the audience interact with +- **Juror**: A registered panel member who submits scores for each team (6 jurors this year: Cielejewski, Finke, Freyer, Knie, Meyer, Trieschmann) +- **Team**: A team/product/concept being evaluated (11 total in the current event) +- **Score**: A numeric rating (1–10) a juror assigns to a team for a given criterion +- **Criterion**: A dimension on which a team is evaluated (5 criteria: Gesamtidee, Kundennutzen, Markt, Realisierbarkeit, Präsentation) +- **Voting_Session**: A time-bounded event during which jurors submit their scores +- **Results_View**: The projector-optimized page that displays aggregated scores and rankings +- **Admin**: The person who configures the session, manages jurors, and controls the results display + +## Concrete Event Data + +### Jurors (6) + +| # | Name | +|---|------| +| 1 | Cielejewski | +| 2 | Finke | +| 3 | Freyer | +| 4 | Knie | +| 5 | Meyer | +| 6 | Trieschmann | + +### Teams (11) + +| # | Team Name | Description | Contact | +|---|-----------|-------------|---------| +| 1 | Anima Lektor | Lektor for game language/text | Lukas Fleck | +| 2 | Das Any-Thing | Physical AI automaton that adapts to context | — | +| 3 | Doppelrad-Sortierer | AI-powered seed sorting machine for niche producers | — | +| 4 | DUAL | Fully reversible wood construction system | Nicole Kozlewski, Sebastian Görs | +| 5 | fömo | Funding management platform for social institutions | — | +| 6 | ISOmedi | Compact rechargeable medication case with protective micro-environment | — | +| 7 | Lian's Tempeh Chips | Tempeh-based snack product | Dr. Berlianti Puteri | +| 8 | Lumpen | Sustainable upholstery material from old clothing textiles | — | +| 9 | Mobile Sprinkleranlage | Mobile sprinkler system for fire protection | Björn Knoke | +| 10 | SafeSupply OS | Lightweight supplier risk & audit system for agri-food SMEs | Jesvin Jaksan | +| 11 | Second Nose | Digital smell detection module for smartphones | — | + +### Scoring Criteria (5, equally weighted) + +| # | Criterion (DE) | Criterion (EN) | +|---|----------------|----------------| +| 1 | Gesamtidee | Overall Idea | +| 2 | Kundennutzen | Customer Benefit | +| 3 | Markt | Market | +| 4 | Realisierbarkeit | Feasibility | +| 5 | Präsentation | Presentation | + +**Scale:** 1–10 per criterion (configurable, default 1–10) +**Max per juror per team:** 50 points +**Aggregation:** Simple sum. Per-team total = sum of all juror scores. Average = total / 6. + +### Presentation Schedule (partial, from Gesamtauswertung) + +| Time | Team | +|------|------| +| 15:10–15:30 | DUAL | +| 16:10–16:30 | fömo | +| 16:30–16:50 | Lumpen (inferred) | +| 16:50–17:10 | Das Any-Thing | +| 17:10–17:30 | Doppelrad-Sortierer | +| 17:45–18:05 | Anima Lektor | +| 18:05–18:25 | Mobile Sprinkleranlage | + +## Requirements + +### Requirement 1: Juror Authentication + +**User Story:** As a juror, I want a simple way to identify myself when I open the app on my phone, so that my votes are attributed to me without a complex login flow. + +#### Acceptance Criteria + +1. WHEN a juror opens the Voting_App on a mobile device, THE Voting_App SHALL present a juror selection list showing the 6 registered juror names (Cielejewski, Finke, Freyer, Knie, Meyer, Trieschmann) +2. WHEN a juror selects their name from the list, THE Voting_App SHALL grant access to the voting interface within 3 seconds +3. IF a juror selects a name that is already in use by another active session, THEN THE Voting_App SHALL display a warning and allow override or retry +4. THE Voting_App SHALL support 6 concurrent jurors in a single Voting_Session + +### Requirement 2: Mobile-Friendly Voting Interface + +**User Story:** As a juror, I want to score each team on my smartphone quickly and intuitively, so that the evaluation process is faster than the paper form. + +#### Acceptance Criteria + +1. THE Voting_App SHALL render a responsive voting interface optimized for mobile viewports (320px–428px width) +2. WHEN a juror navigates to a team, THE Voting_App SHALL display all 5 scoring criteria (Gesamtidee, Kundennutzen, Markt, Realisierbarkeit, Präsentation) with a 1–10 input for each +3. WHEN a juror selects a score for a criterion, THE Voting_App SHALL persist the score immediately without requiring a separate save action +4. THE Voting_App SHALL display all 11 teams in a navigable list with clear indication of which teams have been scored and which are pending +5. WHILE a juror is scoring a team, THE Voting_App SHALL display the team name and short description + +### Requirement 3: Score Persistence and Conflict Handling + +**User Story:** As a juror, I want my scores to be saved reliably even if my phone loses connection briefly, so that no votes are lost. + +#### Acceptance Criteria + +1. WHEN a juror submits a score, THE Voting_App SHALL persist the score to the server within 2 seconds under normal network conditions +2. IF the network connection is lost, THEN THE Voting_App SHALL store scores locally and synchronize them when connectivity is restored +3. WHEN a juror changes a previously submitted score, THE Voting_App SHALL overwrite the previous value with the new score +4. THE Voting_App SHALL prevent a juror from submitting scores for the same criterion on the same team more than once without explicit overwrite intent + +### Requirement 4: Real-Time Results Aggregation + +**User Story:** As an admin, I want the results to update in real time as jurors submit scores, so that the audience sees a live leaderboard on the projector. + +#### Acceptance Criteria + +1. WHEN a new score is persisted, THE Voting_App SHALL recalculate the aggregated result for the affected team within 1 second +2. THE Voting_App SHALL calculate a total score per team by summing all juror scores across all 5 criteria (simple sum, max possible = 6 jurors × 5 criteria × 10 points = 300) +3. THE Voting_App SHALL calculate an average score per team (total / number of jurors who have scored) +4. THE Voting_App SHALL rank teams from highest to lowest total score +5. IF two or more teams have the same total score, THEN THE Voting_App SHALL display them with equal rank + +### Requirement 5: Beamer/Projector Results Display + +**User Story:** As an event organizer, I want a dedicated results page optimized for projection on a large screen, so that the audience can follow the voting live. + +#### Acceptance Criteria + +1. THE Results_View SHALL be optimized for landscape display at 1920×1080 resolution +2. THE Results_View SHALL use the dark theme (#0a0a0a background) with neon green (#00FF00) accents, glassmorphism panels, and Outfit font for a premium event look +3. WHEN aggregated scores change, THE Results_View SHALL update the displayed rankings within 2 seconds without requiring a page refresh +4. THE Results_View SHALL display each team name, its total score, its average score, and its current rank +5. WHILE a Voting_Session is active, THE Results_View SHALL indicate that voting is in progress +6. WHEN a Voting_Session ends, THE Results_View SHALL display the final rankings with a clear "Final Results" indicator + +### Requirement 6: Session Management + +**User Story:** As an admin, I want to configure and control the voting session, so that I can set up teams, criteria, and start/stop voting at the right time. + +#### Acceptance Criteria + +1. THE Voting_App SHALL provide an admin interface to create and configure a Voting_Session +2. WHEN an admin creates a Voting_Session, THE Voting_App SHALL allow configuration of team names and descriptions (pre-populated with the 11 teams for this event) +3. WHEN an admin creates a Voting_Session, THE Voting_App SHALL allow configuration of scoring criteria and their scale (pre-populated with 5 criteria, 1–10 scale) +4. WHEN an admin starts a Voting_Session, THE Voting_App SHALL enable score submission for all 6 registered jurors +5. WHEN an admin ends a Voting_Session, THE Voting_App SHALL disable further score submission and mark results as final +6. THE Voting_App SHALL allow the admin to register jurors by name (pre-populated with the 6 jurors for this event) + +### Requirement 7: Voting Progress Visibility + +**User Story:** As an admin, I want to see which jurors have completed their scoring, so that I know when all votes are in and I can close the session. + +#### Acceptance Criteria + +1. THE Voting_App SHALL display a progress overview showing how many of the 11 teams each juror has fully scored (all 5 criteria submitted) +2. WHEN all 6 jurors have submitted scores for all 11 teams and all 5 criteria, THE Voting_App SHALL indicate that voting is complete +3. THE Voting_App SHALL allow the admin to view individual juror completion status without revealing their specific scores + +### Requirement 8: Data Export + +**User Story:** As an admin, I want to export the final results, so that I have a digital record comparable to the previous paper-based Auswertungstabelle. + +#### Acceptance Criteria + +1. WHEN a Voting_Session has ended, THE Voting_App SHALL provide an export of all scores in a tabular format (CSV or XLSX) +2. THE Voting_App SHALL include in the export: team names, juror names, individual scores per criterion (Gesamtidee, Kundennutzen, Markt, Realisierbarkeit, Präsentation), and aggregated totals +3. THE Voting_App SHALL include the final ranking and average scores in the export +4. THE Voting_App SHALL format the export to match the structure of the original Auswertungstabelle (teams as rows, jurors × criteria as columns) + +--- + +## Open Questions (resolved) + +| # | Question | Resolution | +|---|----------|------------| +| 1 | Scoring scale | **1–10 per criterion** (configurable, default 1–10) | +| 2 | Criteria | **5 criteria:** Gesamtidee, Kundennutzen, Markt, Realisierbarkeit, Präsentation | +| 3 | Weighting | **Equally weighted** — simple sum, no weighting factors | +| 4 | Aggregation | **Simple sum** across all jurors. Average = total / number of jurors | +| 5 | Juror anonymity | **TBD** — not specified in paper documents. Default: scores visible only to admin | +| 6 | Idea metadata | Team name + short description shown to jurors (from Teams/ folder) | +| 7 | Comments | **Not included** — paper form has no free-text field; digital version omits it too | diff --git a/.kiro/specs/jury-voting/tasks.md b/.kiro/specs/jury-voting/tasks.md new file mode 100644 index 0000000..34a248d --- /dev/null +++ b/.kiro/specs/jury-voting/tasks.md @@ -0,0 +1,294 @@ +# Implementation Plan: Jury Voting + +## Overview + +Build a full-stack web application for the 15. UNIKAT Ideenwettbewerb jury voting process. The app enables 6 jurors to score 11 teams on 5 criteria (1–10) via smartphones, with real-time results displayed on a beamer. Stack: React 19 + Vite frontend, Express.js backend, JSON file persistence, SSE for live updates. TypeScript throughout. + +## Tasks + +- [x] 1. Project scaffolding and configuration + - [x] 1.1 Initialize Vite + React 19 + TypeScript project in `Jury-Voting/` + - Run `npm create vite@latest` with React + TypeScript template + - Configure `vite.config.ts` with proxy to Express backend (port 3001) + - Add dependencies: `react-router-dom`, `lucide-react` + - Add dev dependencies: `vitest`, `fast-check`, `@testing-library/react`, `jsdom` + - _Requirements: 5.2 (dark theme, Outfit font, glassmorphism)_ + + - [x] 1.2 Set up Express.js backend in `Jury-Voting/server/` + - Create `server/` directory with TypeScript configuration + - Install: `express`, `cors`, `tsx` (for dev), `typescript` + - Create `server/index.ts` entry point with basic Express app on port 3001 + - Add `npm run dev:server` script using `tsx watch` + - _Requirements: 3.1 (score persistence)_ + + - [x] 1.3 Define shared TypeScript interfaces in `Jury-Voting/shared/types.ts` + - Create `Session`, `Team`, `Juror`, `Criterion`, `Score`, `TeamResult`, `PendingScore` interfaces + - Create `SessionConfig` interface with criteria and scale settings + - Export all types for use by both frontend and backend + - _Requirements: 6.2, 6.3 (configurable teams, criteria, scale)_ + + - [x] 1.4 Create seed data file `Jury-Voting/server/data/seed.json` + - Pre-populate 11 teams from `Teams/` folder (id, number, name, description) + - Pre-populate 6 jurors (Cielejewski, Finke, Freyer, Knie, Meyer, Trieschmann) + - Pre-populate 5 criteria (Gesamtidee, Kundennutzen, Markt, Realisierbarkeit, Präsentation) + - Set default session config (scale 1–10) + - _Requirements: 6.2, 6.3, 6.6_ + +- [x] 2. Backend data layer and core logic + - [x] 2.1 Implement JSON file store (`server/store.ts`) + - Read/write `server/data/session.json` atomically (write to temp, rename) + - Create `.bak` backup before each write + - Initialize from `seed.json` if `session.json` doesn't exist + - Expose typed read/write functions for session, teams, jurors, scores + - _Requirements: 3.1, 3.2 (persistence, reliability)_ + + - [x] 2.2 Implement score aggregation logic (`server/aggregation.ts`) + - `calculateTeamResult(teamId, scores[])` → total, average, jurorCount, complete + - `calculateRankings(teamResults[])` → sorted results with dense ranking (ties get same rank) + - `getCompletionStatus(jurorId, teamId, scores[])` → 'complete' | 'partial' | 'pending' + - Pure functions, no side effects — easy to test + - _Requirements: 4.2, 4.3, 4.4, 4.5_ + + - [ ]* 2.3 Write property test: Score Aggregation Correctness + - **Property 4: Score Aggregation Correctness** + - Generate random valid scores (1–10) for arbitrary juror/team/criterion combinations + - Assert: total equals arithmetic sum of all score values for a team + - Assert: average equals total / distinct jurors who scored + - **Validates: Requirements 4.2, 4.3** + + - [ ]* 2.4 Write property test: Ranking Correctness with Ties + - **Property 5: Ranking Correctness with Ties** + - Generate random team totals, compute rankings + - Assert: rankings sorted descending by total + - Assert: equal totals receive equal rank numbers + - **Validates: Requirements 4.4, 4.5** + + - [ ]* 2.5 Write property test: Team Completion Status + - **Property 1: Team Completion Status** + - Generate random subsets of criteria scores for a juror-team pair + - Assert: 5 scores → 'complete', 1–4 → 'partial', 0 → 'pending' + - **Validates: Requirements 2.4, 7.1** + + - [x] 2.6 Implement last-write-wins conflict resolution (`server/conflict.ts`) + - Compare timestamps on incoming score vs stored score + - Accept score with later `updatedAt` timestamp + - Return conflict info (409) if client sends older timestamp without force flag + - _Requirements: 3.3, 3.4_ + + - [ ]* 2.7 Write property test: Last-Write-Wins Score Resolution + - **Property 3: Last-Write-Wins Score Resolution** + - Generate sequences of score updates with varying timestamps for same (jurorId, teamId, criterionId) + - Assert: final persisted value equals the value with the most recent timestamp + - **Validates: Requirements 3.3** + +- [ ] 3. Backend API endpoints + - [x] 3.1 Implement session endpoints + - `GET /api/session` — return current session state + - `POST /api/session/start` — set status to 'active', record startedAt + - `POST /api/session/end` — set status to 'ended', record endedAt + - Validate: cannot start if already active, cannot end if not active + - _Requirements: 6.4, 6.5_ + + - [x] 3.2 Implement team and juror endpoints + - `GET /api/teams` — return all teams + - `GET /api/jurors` — return all jurors + - _Requirements: 6.2, 6.6_ + + - [x] 3.3 Implement score endpoints + - `POST /api/scores` — submit/update a score (validate: session active, score 1–10, valid jurorId/teamId/criterionId) + - `GET /api/scores/:jurorId` — return all scores for a juror + - `GET /api/scores` — return all scores (admin only) + - On successful score submission, trigger SSE broadcast + - _Requirements: 2.3, 3.1, 3.3_ + + - [ ]* 3.4 Write property test: Session Gate + - **Property 6: Session Gate** + - Generate score submissions with random session states ('setup', 'active', 'ended') + - Assert: accepted if and only if session status is 'active' + - **Validates: Requirements 6.4, 6.5** + + - [x] 3.5 Implement results and progress endpoints + - `GET /api/results` — return aggregated TeamResult[] with rankings + - `GET /api/progress` — return juror completion matrix (jurorId → count of fully-scored teams) + - `GET /api/export/csv` — generate and return CSV file + - Progress endpoint must NOT include individual score values + - _Requirements: 4.1, 7.1, 7.3, 8.1_ + + - [ ]* 3.6 Write property test: Progress Privacy + - **Property 7: Progress Privacy** + - Generate random score sets, call progress endpoint logic + - Assert: response contains juror completion counts but no individual score values + - **Validates: Requirements 7.3** + +- [x] 4. SSE real-time infrastructure + - [x] 4.1 Implement SSE endpoint (`server/sse.ts`) + - `GET /api/results/stream` — establish SSE connection + - Maintain list of connected clients + - On score change, broadcast `scores_updated` event with full results payload + - Handle client disconnect (remove from list) + - Set appropriate headers: `Content-Type: text/event-stream`, `Cache-Control: no-cache`, `Connection: keep-alive` + - _Requirements: 4.1, 5.3_ + + - [x] 4.2 Create frontend SSE hook (`src/hooks/useSSE.ts`) + - Custom React hook that connects to `/api/results/stream` + - Parse incoming events and update local state + - Auto-reconnect on connection loss (EventSource built-in) + - Return: `{ results, connected, lastUpdate }` + - _Requirements: 5.3 (update within 2 seconds without page refresh)_ + +- [x] 5. Checkpoint - Backend complete + - Ensure all tests pass, ask the user if questions arise. + +- [x] 6. Frontend: Juror selection and routing + - [x] 6.1 Set up React Router with route structure + - Configure routes: `/`, `/vote`, `/vote/:teamId`, `/results`, `/admin` + - Create `App.tsx` with `BrowserRouter` and route definitions + - Create layout wrapper with dark theme base styles + - _Requirements: 1.1, 2.1_ + + - [x] 6.2 Implement `JurorSelect` page (`src/pages/JurorSelect.tsx`) + - Fetch juror list from `GET /api/jurors` + - Display 6 juror names as large glass-panel cards (tap targets for mobile) + - On selection: store jurorId in localStorage, navigate to `/vote` + - Show warning if juror already has active session (allow override) + - _Requirements: 1.1, 1.2, 1.3, 1.4_ + + - [ ]* 6.3 Write unit tests for JurorSelect + - Test: renders 6 juror names + - Test: selection stores jurorId and navigates + - Test: shows conflict warning for active juror + - _Requirements: 1.1, 1.2, 1.3_ + +- [ ] 7. Frontend: Voting interface + - [x] 7.1 Implement `VotingDashboard` page (`src/pages/VotingDashboard.tsx`) + - Fetch teams from `GET /api/teams` and scores from `GET /api/scores/:jurorId` + - Display 11 `TeamCard` components in a scrollable list + - Each card shows: team name, description snippet, status icon (✓ complete / ◐ partial / ○ pending) + - Tap card navigates to `/vote/:teamId` + - _Requirements: 2.1, 2.4, 2.5_ + + - [x] 7.2 Implement `TeamScoring` page (`src/pages/TeamScoring.tsx`) + - Display team name and description at top + - Render 5 `ScoreSlider` components (one per criterion) + - Each slider: range 1–10, shows current value, large touch target + - Auto-save on change with 300ms debounce + - Show green flash confirmation on successful save + - Prev/Next team navigation buttons + - _Requirements: 2.2, 2.3, 2.5_ + + - [x] 7.3 Implement offline score queue (`src/hooks/useOfflineQueue.ts`) + - On score change: save to localStorage `pendingScores` array immediately + - Attempt POST to `/api/scores` — on success, mark as synced + - On network failure: keep in queue, show subtle "offline" indicator + - On reconnect: replay pending scores in order + - Exponential backoff for retries (max 3 attempts per score) + - _Requirements: 3.1, 3.2_ + + - [ ]* 7.4 Write property test: Offline Queue Preservation + - **Property 2: Offline Queue Preservation** + - Generate random sequences of scores submitted while "offline" + - Simulate sync, assert all scores present on server with original values/timestamps + - **Validates: Requirements 3.2** + + - [ ]* 7.5 Write unit tests for TeamScoring + - Test: renders 5 criteria with correct labels + - Test: auto-saves on slider change (debounced) + - Test: shows confirmation flash on save + - Test: prev/next navigation works + - _Requirements: 2.2, 2.3_ + +- [ ] 8. Frontend: Results view (beamer) + - [x] 8.1 Implement `ResultsView` page (`src/pages/ResultsView.tsx`) + - Full-screen layout, no navigation chrome + - Connect to SSE via `useSSE` hook + - Display `RankingTable` component with animated transitions + - Show "Voting in Progress" or "Final Results" based on session status + - Optimized for 1920×1080 landscape projection + - _Requirements: 5.1, 5.2, 5.3, 5.4, 5.5, 5.6_ + + - [x] 8.2 Implement `RankingTable` component (`src/components/RankingTable.tsx`) + - Display ranked list: rank, team name, total score, average score + - Animate rank changes (CSS transitions on position) + - Highlight top 3 teams with special styling + - Handle ties (same rank number displayed) + - _Requirements: 4.4, 4.5, 5.4_ + + - [ ]* 8.3 Write property test: Results Display Completeness + - **Property 9: Results Display Completeness** + - Generate random TeamResult objects, render ResultsView + - Assert: each team's name, total, average, and rank are present in rendered output + - **Validates: Requirements 5.4** + +- [ ] 9. Frontend: Admin panel + - [x] 9.1 Implement `AdminPanel` page (`src/pages/AdminPanel.tsx`) + - Session controls: Start Voting / End Voting buttons with confirmation dialogs + - Show current session status and timestamps + - Warning when ending session with incomplete scores + - _Requirements: 6.1, 6.4, 6.5_ + + - [x] 9.2 Implement `ProgressMatrix` component (`src/components/ProgressMatrix.tsx`) + - Fetch from `GET /api/progress` + - Display 6×11 grid (jurors × teams) with completion indicators + - Show per-juror totals (X/11 teams complete) + - Indicate when all voting is complete (6/6 jurors × 11/11 teams) + - No individual scores shown — only completion status + - _Requirements: 7.1, 7.2, 7.3_ + + - [x] 9.3 Implement CSV export functionality + - Export button calls `GET /api/export/csv` and triggers download + - CSV format: teams as rows, (juror × criterion) as columns, plus totals/averages/ranks + - Match structure of original Auswertungstabelle + - _Requirements: 8.1, 8.2, 8.3, 8.4_ + + - [ ]* 9.4 Write property test: Export Round-Trip + - **Property 8: Export Round-Trip** + - Generate complete score sets, export to CSV, parse CSV back + - Assert: all individual scores recovered, totals correct, averages correct, rankings correct + - **Validates: Requirements 8.1, 8.2, 8.3, 8.4** + +- [x] 10. Checkpoint - Core features complete + - Ensure all tests pass, ask the user if questions arise. + +- [ ] 11. Styling and visual polish + - [x] 11.1 Implement global theme and CSS variables + - Define CSS custom properties: `--bg-primary: #0a0a0a`, `--accent: #00FF00`, glassmorphism values + - Import Outfit font from Google Fonts + - Set up glassmorphism utility classes (backdrop-filter, border, shadow) + - Dark scrollbar styling, smooth transitions + - _Requirements: 5.2_ + + - [x] 11.2 Style all components to match Projekt-KIQ-HP design + - JurorSelect: glass-panel cards with hover/tap glow effect + - VotingDashboard: team cards with status indicators, neon green accents + - TeamScoring: custom slider styling (neon green track), score display + - ResultsView: large typography, animated rank transitions, glassmorphism table + - AdminPanel: clean grid layout, status badges + - Mobile-first responsive design (320px–428px primary, scales up) + - _Requirements: 2.1, 5.1, 5.2_ + +- [ ] 12. Deployment configuration + - [x] 12.1 Create production build setup + - Configure Vite to build static assets to `dist/` + - Configure Express to serve static files from `dist/` in production + - Single-port deployment: Express serves both API and frontend + - Add `npm run build` and `npm start` scripts + - _Requirements: all (deployment readiness)_ + + - [x] 12.2 Create `docker-compose.yml` for containerized deployment + - Single container: Node.js serving Express + static frontend + - Volume mount for `server/data/` (persist scores across restarts) + - Expose port 3001 (or configurable via env) + - Add `.dockerignore` and `Dockerfile` + - _Requirements: all (deployment readiness)_ + +- [x] 13. Final checkpoint - All features complete + - Ensure all tests pass, ask the user if questions arise. + +## Notes + +- Tasks marked with `*` are optional property-based tests — skip for faster MVP delivery given the tight timeline (event: 19. Mai 2026) +- Each task references specific requirements for traceability +- Checkpoints at tasks 5, 10, and 13 ensure incremental validation +- Property tests validate the 9 correctness properties from the design document +- The app is designed for single-event use — simplicity over scalability +- All seed data (teams, jurors, criteria) is pre-populated; no manual setup needed at event time diff --git a/.kiro/specs/monorepo-consolidation/.config.kiro b/.kiro/specs/monorepo-consolidation/.config.kiro new file mode 100644 index 0000000..74c1e6e --- /dev/null +++ b/.kiro/specs/monorepo-consolidation/.config.kiro @@ -0,0 +1 @@ +{"specId": "ff9eec9c-4758-4190-8766-743007d13d53", "workflowType": "requirements-first", "specType": "feature"} diff --git a/.kiro/specs/monorepo-consolidation/design.md b/.kiro/specs/monorepo-consolidation/design.md new file mode 100644 index 0000000..a183394 --- /dev/null +++ b/.kiro/specs/monorepo-consolidation/design.md @@ -0,0 +1,1083 @@ +# Design Document: Monorepo-Consolidation + +## Overview + +Dieses Design beschreibt die technische Architektur für die Konsolidierung von 18+ einzelnen Repositories in eine einheitliche Monorepo-Struktur. Das System vereint Projekte aus drei Arbeitskontexten (privat, dhive, bahn) mit einem shared-Bereich, implementiert Sicherheitsgrenzen zwischen Kontexten, einen übergreifenden Wissensspeicher basierend auf der DB-Wissensdatenbank-ETL-Pipeline, integriert den bestehenden AI-Orchestrator, speichert Secrets verschlüsselt im Repository mittels git-crypt, und ermöglicht eine föderierte Zusammenarbeit über eigenständige Team-Repositories in einer Hub-and-Spoke-Topologie. + +### Kernentscheidungen + +1. **Git-Subtrees statt Submodules** für externe Repos und Team-Repos: Subtrees ermöglichen flache Checkouts ohne rekursive Init-Schritte und erlauben bidirektionale Synchronisation mit Upstream und Team-Repositories. +2. **Dateibasierte Zugriffskontrolle** statt OS-Level-Permissions: Ein Wrapper-Script (`ctx-guard`) prüft Kontextzugehörigkeit und kontrolliert den Zugriff auf .env-Dateien und Secrets. +3. **ETL-Pipeline-Fork** der DB-Wissensdatenbank als Basis für den Wissensspeicher: Wiederverwendung der bewährten Quellstrategien und Erweiterung um persönliche Arbeitskontexte. +4. **YAML-Frontmatter + YAML-Index** als Progressive-Disclosure-System: Agenten lesen zuerst den kompakten Index und laden nur bei Bedarf vollständige Dokumente. +5. **git-crypt für Encryption-at-Rest**: Secrets werden verschlüsselt im Repository gespeichert statt via .gitignore ausgeschlossen. Pro Arbeitskontext ein eigener GPG-/symmetrischer Schlüssel ermöglicht granulare Entschlüsselung per Maschinenkontext. +6. **Hub-and-Spoke-Föderation** für Multi-Team-Zusammenarbeit: Das Monorepo ist der zentrale Hub (nur Andre sieht alles), Team-Repos sind unabhängige Spokes. Synchronisation via Git-Subtrees bewahrt vollständige Historie. + +## Architecture + +```mermaid +graph TB + subgraph Monorepo["Monorepo Root (Hub)"] + direction TB + + subgraph Contexts["Arbeitskontexte"] + privat["privat/"] + dhive["dhive/"] + bahn["bahn/"] + shared["shared/"] + end + + subgraph SharedArea["shared/ Bereich"] + tools["tools/"] + powers["powers/"] + knowledge["knowledge-store/"] + config["config/"] + mcpservers["mcp-servers/"] + end + + subgraph Security["Sicherheitsschicht"] + ctxguard["ctx-guard"] + envfiles[".env pro Kontext (verschlüsselt)"] + accessconfig["access-config.yaml"] + auditlog["audit.log"] + end + + subgraph Encryption["Verschlüsselungsschicht"] + gitcrypt["git-crypt"] + contextkeys["Kontext-Schlüssel (privat/dhive/bahn)"] + machineconfig["machine-context.yaml"] + end + + subgraph Federation["Föderationsschicht"] + fedmanager["Federation-Manager"] + syncengine["Sync-Engine (git-subtree)"] + teamconfig["team-repos.yaml"] + end + end + + subgraph External["Externe Repos"] + extro1["Read-Only Repos"] + upstream["Upstream Repos"] + end + + subgraph TeamRepos["Team-Repos (Spokes)"] + teamPrivat["Team-Repo: privat"] + teamDhive["Team-Repo: dhive"] + teamBahn["Team-Repo: bahn"] + end + + subgraph Orchestrator["AI-Orchestrator"] + taskrouter["Task-Router"] + ctxresolver["Context-Resolver"] + agentrunner["Agent-Runner"] + end + + Orchestrator --> Security + Security --> Encryption + Encryption --> Contexts + External --> Contexts + knowledge --> Contexts + Federation -.->|"bidirektionaler Sync"| TeamRepos + teamPrivat -.-> privat + teamDhive -.-> dhive + teamBahn -.-> bahn +``` + +### Schichtenarchitektur + +```mermaid +graph LR + subgraph L1["Schicht 1: Dateisystem"] + folders["Ordnerstruktur"] + gitconfig["Git-Konfiguration"] + end + + subgraph L2["Schicht 2: Verschlüsselung"] + gitcrypt["git-crypt"] + keymanagement["Schlüsselverwaltung"] + machinecontext["Maschinenkontext"] + end + + subgraph L3["Schicht 3: Zugriffskontrolle"] + guard["ctx-guard Wrapper"] + hooks["Git-Hooks"] + envloader["Env-Loader"] + end + + subgraph L4["Schicht 4: Wissensspeicher"] + etl["ETL-Pipeline"] + index["YAML-Index"] + search["Suche"] + end + + subgraph L5["Schicht 5: Integration"] + orch["Orchestrator-Adapter"] + bridge["Kontextbrücke"] + sync["Repo-Sync"] + end + + subgraph L6["Schicht 6: Föderation"] + fedmgr["Federation-Manager"] + subtreesync["Subtree-Sync"] + teamisolation["Team-Isolation"] + end + + L1 --> L2 --> L3 --> L4 --> L5 --> L6 +``` + +## Components and Interfaces + +### 1. Ordnerstruktur-Manager (`structure-manager`) + +**Verantwortung:** Anlegen, Validieren und Verwalten der Monorepo-Ordnerstruktur. + +```python +class StructureManager: + """Verwaltet die 3-Ebenen-Ordnerhierarchie.""" + + CONTEXTS = ("privat", "dhive", "bahn", "shared") + NAME_PATTERN = re.compile(r'^[a-z0-9][a-z0-9\-]{0,48}[a-z0-9]$') + + def create_project(self, context: str, name: str) -> Path: + """Erstellt ein neues Projekt im gegebenen Kontext.""" + ... + + def validate_name(self, name: str) -> bool: + """Prüft kebab-case Namenskonvention (2-50 Zeichen).""" + ... + + def list_projects(self, context: str | None = None) -> list[ProjectInfo]: + """Listet alle Projekte, optional gefiltert nach Kontext.""" + ... + + def resolve_context(self, project_path: Path) -> str: + """Ermittelt den Arbeitskontext eines Projekts anhand seines Pfads.""" + ... +``` + +### 2. Sicherheits-Guard (`ctx-guard`) + +**Verantwortung:** Zugriffskontrolle auf Secrets und .env-Dateien basierend auf Ausführungskontext. + +```python +class ContextGuard: + """Erzwingt Sicherheitsgrenzen zwischen Arbeitskontexten.""" + + def __init__(self, config_path: Path): + self.config = self._load_access_config(config_path) + self.audit_log = AuditLogger() + + def check_access(self, requesting_context: str, target_path: Path) -> bool: + """Prüft ob der Zugriff auf target_path vom requesting_context erlaubt ist.""" + ... + + def load_env(self, context: str) -> dict[str, str]: + """Lädt die .env-Datei des gegebenen Kontexts.""" + ... + + def log_violation(self, event: SecurityEvent) -> None: + """Protokolliert einen Zugriffsverletzungs-Versuch.""" + ... +``` + +**Konfigurationsdatei (`access-config.yaml`):** +```yaml +contexts: + privat: + env_file: privat/.env + allowed_shared: + - shared/tools/ + - shared/powers/ + - shared/config/ + dhive: + env_file: dhive/.env + allowed_shared: + - shared/tools/ + - shared/powers/ + - shared/config/ + bahn: + env_file: bahn/.env + allowed_shared: + - shared/tools/ + - shared/powers/ + - shared/config/ + - shared/knowledge-store/ + shared: + env_file: shared/.env + allowed_shared: ["*"] +``` + +### 3. Wissensspeicher (`knowledge-store`) + +**Verantwortung:** ETL-Pipeline für kontextübergreifendes Wissen, basierend auf der DB-Wissensdatenbank-Architektur. + +```python +class KnowledgeStore: + """Zentraler Wissensspeicher mit ETL-Pipeline.""" + + def __init__(self, base_path: Path, scope_config: ScopeConfig): + self.base_path = base_path + self.scope_config = scope_config + self.index = YAMLIndex(base_path / "_index.yaml") + + def ingest(self, source: Source, context: str) -> list[Artifact]: + """Verarbeitet eine Quelle und legt Artefakte ab.""" + ... + + def search(self, query: str, allowed_scopes: list[str]) -> list[SearchResult]: + """Volltextsuche über den Index, gefiltert nach Berechtigung.""" + ... + + def get_index(self, scope: str | None = None) -> IndexData: + """Liefert den kompakten YAML-Index (Progressive Disclosure Schicht 1).""" + ... + + def link_artifacts(self, source_id: str, target_id: str, relation: str) -> None: + """Verknüpft zwei Artefakte als gerichtete Graph-Kante.""" + ... +``` + +**ETL-Pipeline-Komponenten:** + +```mermaid +graph LR + Sources["Quellen"] --> Extract["Extract"] + Extract --> Transform["Transform"] + Transform --> Load["Load"] + Load --> Index["Index Update"] + + Sources --- Confluence + Sources --- Webseiten + Sources --- PDFs + Sources --- Markdown + Sources --- GitLab +``` + +### 4. Externes-Repo-Manager (`repo-manager`) + +**Verantwortung:** Einbindung, Synchronisation und Schutzmechanismen für externe Repositories. + +```python +class RepoManager: + """Verwaltet externe und Upstream-Repository-Einbindungen.""" + + def __init__(self, config_path: Path): + self.config = self._load_repos_config(config_path) + + def add_repo(self, entry: RepoEntry) -> None: + """Bindet ein externes Repo ein (Subtree oder Submodule).""" + ... + + def sync(self, repo_name: str) -> SyncResult: + """Synchronisiert ein Repo mit seinem Remote.""" + ... + + def protect_readonly(self, repo_path: Path) -> None: + """Installiert Git-Hooks zum Schutz vor Schreibzugriffen.""" + ... +``` + +**Konfigurationsdatei (`repos.yaml`):** +```yaml +repos: + - name: db-wissensdatenbank + url: https://gitlab.2700.2db.it/... + mode: upstream + target: bahn/db-wissensdatenbank + pinned: main + + - name: symphony-spec + url: https://github.com/openai/symphony + mode: read-only + target: shared/references/symphony + pinned: v1.0.0 +``` + +### 5. Kontextbrücke (`context-bridge`) + +**Verantwortung:** Kontextübergreifendes Teilen von Wissensartefakten unter Wahrung der Sicherheitsgrenzen. + +```python +class ContextBridge: + """Ermöglicht kontextübergreifenden Wissenstransfer.""" + + SENSITIVE_PATTERNS = [ + r'(?i)(api[_-]?key|token|password|secret)\s*[:=]', + r'(?i)(endpoint|url)\s*[:=]\s*https?://', + r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', + ] + + def share_artifact(self, artifact_id: str, user_confirmed: bool = False) -> ShareResult: + """Gibt ein Artefakt kontextübergreifend frei.""" + ... + + def check_sensitive_content(self, content: str) -> list[SensitiveMatch]: + """Prüft Inhalt auf kontextspezifische Geheimnisse und PII.""" + ... + + def revoke_share(self, artifact_id: str) -> None: + """Widerruft die Freigabe eines geteilten Artefakts.""" + ... +``` + +### 6. Migrations-Engine (`migration-engine`) + +**Verantwortung:** Inkrementelle Migration bestehender Repositories unter Bewahrung der Git-Historie. + +```python +class MigrationEngine: + """Migriert bestehende Repos in die Monorepo-Struktur.""" + + def migrate(self, plan: MigrationPlan) -> MigrationResult: + """Führt die Migration eines einzelnen Repos durch.""" + ... + + def validate(self, repo_name: str) -> ValidationResult: + """Validiert eine abgeschlossene Migration.""" + ... + + def rollback(self, repo_name: str) -> None: + """Macht die Migration eines Repos rückgängig.""" + ... +``` + +### 7. Orchestrator-Adapter (`orchestrator-adapter`) + +**Verantwortung:** Integration des AI-Orchestrators mit der Monorepo-Struktur. + +```python +class OrchestratorAdapter: + """Adaptiert den AI-Orchestrator für die Monorepo-Struktur.""" + + def resolve_context(self, task: Task) -> str: + """Ermittelt den Arbeitskontext aus Task-Metadaten.""" + ... + + def create_workspace(self, task: Task, context: str) -> Path: + """Erstellt ein kontextgebundenes Workspace.""" + ... + + def build_prompt(self, task: Task, context: str) -> str: + """Erstellt den Agenten-Prompt mit Wissenskontext.""" + ... + + def inject_knowledge(self, prompt: str, context: str) -> str: + """Fügt relevante Artefakt-Pfade aus dem YAML-Index hinzu.""" + ... +``` + +### 8. Secret-Encryption-Manager (`secret-encryption`) + +**Verantwortung:** Verschlüsselung von Secrets im Repository mittels git-crypt, Schlüsselverwaltung pro Arbeitskontext, und Maschinenkontext-basierte Entschlüsselung. + +```python +class SecretEncryptionManager: + """Verwaltet git-crypt-basierte Verschlüsselung pro Arbeitskontext.""" + + SUPPORTED_TOOLS = ("git-crypt", "sops", "age") + SECRET_PATTERNS = [ + "**/.env", + "**/*.pem", + "**/*.key", + "**/*token*", + "**/*secret*", + ] + + def __init__(self, config_path: Path, machine_context: MachineContext): + self.config = self._load_encryption_config(config_path) + self.machine_context = machine_context + self.tool = self._init_encryption_tool() + + def encrypt_file(self, file_path: Path, context: str) -> EncryptionResult: + """Verschlüsselt eine Datei mit dem Schlüssel des gegebenen Kontexts.""" + ... + + def decrypt_file(self, file_path: Path) -> DecryptionResult: + """Entschlüsselt eine Datei, sofern der Maschinenkontext autorisiert ist.""" + ... + + def is_authorized(self, context: str) -> bool: + """Prüft ob der aktuelle Maschinenkontext für den Kontext autorisiert ist.""" + return context in self.machine_context.authorized_contexts + + def get_context_key(self, context: str) -> Optional[EncryptionKey]: + """Liefert den Schlüssel für einen Kontext (aus Keyring/Passwort-Manager).""" + ... + + def setup_gitcrypt_filters(self, context: str) -> None: + """Installiert git-crypt-Filter für den gegebenen Kontext.""" + ... + + def onboard_machine(self, machine_name: str, authorized_contexts: list[str]) -> OnboardingResult: + """Richtet eine neue Maschine mit den autorisierten Schlüsseln ein.""" + ... + + def resolve_merge(self, file_path: Path, ours: bytes, theirs: bytes) -> bytes: + """Löst Merge-Konflikte auf verschlüsselter Ebene.""" + ... +``` + +**Maschinenkontext-Konfiguration (`machine-context.yaml`):** +```yaml +machine: + name: "andre-hauptrechner" + description: "Andres Hauptrechner mit vollem Zugriff" + authorized_contexts: + - privat + - dhive + - bahn + key_source: "keyring" # keyring | file | password-manager + password_manager: + type: "bitwarden" # bitwarden | 1password | keepass + vault: "monorepo-keys" +``` + +**git-crypt-Konfiguration (`.gitattributes` pro Kontext):** +```gitattributes +# privat/.gitattributes +.env filter=git-crypt-privat diff=git-crypt-privat +*.pem filter=git-crypt-privat diff=git-crypt-privat +*.key filter=git-crypt-privat diff=git-crypt-privat + +# dhive/.gitattributes +.env filter=git-crypt-dhive diff=git-crypt-dhive +*.pem filter=git-crypt-dhive diff=git-crypt-dhive + +# bahn/.gitattributes +.env filter=git-crypt-bahn diff=git-crypt-bahn +*.pem filter=git-crypt-bahn diff=git-crypt-bahn +``` + +### 9. Federation-Manager (`federation-manager`) + +**Verantwortung:** Verwaltung der Hub-and-Spoke-Topologie, bidirektionale Synchronisation zwischen Monorepo und Team-Repos via Git-Subtrees, Team-Isolation und Shared-Bereich-Spiegelung. + +```python +class FederationManager: + """Verwaltet die föderierte Repo-Struktur (Hub-and-Spoke).""" + + def __init__(self, config_path: Path, encryption_manager: SecretEncryptionManager): + self.config = self._load_team_repos_config(config_path) + self.encryption = encryption_manager + self.sync_engine = SubtreeSyncEngine() + + def sync_from_team(self, context: str) -> SyncResult: + """Synchronisiert Änderungen vom Team_Repo in den Monorepo-Kontextordner.""" + ... + + def sync_to_team(self, context: str) -> SyncResult: + """Synchronisiert Änderungen vom Monorepo-Kontextordner zum Team_Repo.""" + ... + + def full_sync(self, context: str) -> SyncResult: + """Bidirektionale Synchronisation mit Konflikt-Erkennung.""" + ... + + def verify_isolation(self, team_repo_path: Path, context: str) -> IsolationReport: + """Prüft ob ein Team_Repo keine Referenzen auf andere Kontexte enthält.""" + ... + + def mirror_shared(self, context: str, paths: list[str]) -> MirrorResult: + """Spiegelt ausgewählte shared-Dateien als Read-Only in das Team_Repo.""" + ... + + def prepare_team_repo(self, context: str) -> Path: + """Erzeugt ein Team_Repo mit nur dem eigenen Kontext (Secrets entschlüsselt/re-keyed).""" + ... + + def resolve_conflict(self, context: str, strategy: str = "team-wins") -> ConflictResult: + """Löst Sync-Konflikte auf (Standard: Team_Repo hat Vorrang).""" + ... + + def onboard_member(self, context: str, member_info: MemberInfo) -> OnboardingResult: + """Erteilt Zugang nur zum Team_Repo ohne Kenntnis des Monorepos.""" + ... + + +class SubtreeSyncEngine: + """Git-Subtree-basierte Synchronisation mit Historie-Bewahrung.""" + + def subtree_pull(self, remote: str, prefix: str, branch: str) -> SyncResult: + """Zieht Änderungen vom Team_Repo per git subtree pull.""" + ... + + def subtree_push(self, remote: str, prefix: str, branch: str) -> SyncResult: + """Pusht Änderungen zum Team_Repo per git subtree push.""" + ... + + def detect_conflicts(self, context: str) -> list[ConflictInfo]: + """Erkennt Merge-Konflikte vor der Synchronisation.""" + ... + + def preserve_history(self, repo_path: Path) -> bool: + """Verifiziert, dass die Git-Historie nach Sync vollständig ist.""" + ... +``` + +**Team-Repos-Konfiguration (`team-repos.yaml`):** +```yaml +version: "1.0" +federation: + topology: "hub-and-spoke" + hub_owner: "andre" + conflict_strategy: "team-wins" # Team_Repo hat Vorrang + +team_repos: + - context: privat + url: "https://github.com/andreknie/privat-team.git" + branch: main + sync_direction: bidirectional + sync_frequency: "on-push" # on-push | hourly | daily | manual + shared_mirror: + enabled: true + paths: + - "shared/tools/common-scripts/" + - "shared/config/base-config.yaml" + mode: read-only + + - context: dhive + url: "https://gitlab.dhive.io/team/dhive-mono.git" + branch: main + sync_direction: bidirectional + sync_frequency: "on-push" + shared_mirror: + enabled: true + paths: + - "shared/tools/" + - "shared/powers/db-dxp-platform/" + - "shared/mcp-servers/" + mode: read-only + + - context: bahn + url: "https://gitlab.2700.2db.it/team/bahn-workspace.git" + branch: main + sync_direction: bidirectional + sync_frequency: "daily" + shared_mirror: + enabled: true + paths: + - "shared/tools/" + - "shared/powers/" + - "shared/knowledge-store/_index.yaml" + mode: read-only +``` + +## Data Models + +### Ordnerstruktur + +``` +monorepo-root/ +├── privat/ +│ ├── project-a/ +│ │ ├── module-1/ +│ │ └── module-2/ +│ ├── project-b/ +│ ├── .env (verschlüsselt via git-crypt-privat) +│ └── .gitattributes (git-crypt-Filter-Regeln) +├── dhive/ +│ ├── project-c/ +│ ├── .env (verschlüsselt via git-crypt-dhive) +│ └── .gitattributes +├── bahn/ +│ ├── db-wissensdatenbank/ (Upstream) +│ ├── project-d/ +│ ├── .env (verschlüsselt via git-crypt-bahn) +│ └── .gitattributes +├── shared/ +│ ├── tools/ +│ ├── powers/ +│ ├── knowledge-store/ +│ ├── config/ +│ │ ├── access-config.yaml +│ │ ├── repos.yaml +│ │ ├── scopes.yaml +│ │ ├── machine-context.yaml (Maschinenkontext-Mapping) +│ │ └── team-repos.yaml (Föderations-Konfiguration) +│ ├── mcp-servers/ +│ └── .env +├── .gitattributes (Root-Level git-crypt-Regeln) +├── .gitignore (nur Build-Artefakte, Caches, Temp-Dateien) +├── .gitmodules +└── monorepo.yaml (Zentrale Monorepo-Konfiguration) +``` + +### Wissensartefakt (Markdown mit YAML-Frontmatter) + +```yaml +--- +type: decision # decision | note | meeting | reference | pattern +title: "API-Designprinzipien für Microservices" +tags: [api, microservices, architecture, rest] +source_context: bahn # Quellkontext +created: 2024-12-15 +updated: 2025-01-10 +shareable: true # Kontextübergreifend teilbar? +links: + - target: "privat/notes/rest-patterns.md" + relation: "implements" + - target: "shared/knowledge-store/patterns/api-versioning.md" + relation: "references" +content_hash: "sha256:abc123..." +--- + +# API-Designprinzipien für Microservices + +... +``` + +### YAML-Index (Progressive Disclosure – Schicht 1) + +```yaml +# shared/knowledge-store/_index.yaml +version: "1.0" +last_updated: "2025-01-15T10:30:00Z" +artifacts: + - id: "bahn/decisions/api-design" + title: "API-Designprinzipien für Microservices" + type: decision + tags: [api, microservices, architecture] + scope: bahn + summary: "REST-API-Konventionen für DB-InfraGO-Dienste" + path: "bahn/decisions/api-design.md" + content_hash: "sha256:abc123..." + links: ["shared/patterns/api-versioning"] + - id: "privat/notes/rest-patterns" + title: "REST-Patterns Notizen" + type: note + tags: [api, rest, patterns] + scope: privat + summary: "Sammlung bewährter REST-Patterns" + path: "privat/notes/rest-patterns.md" + content_hash: "sha256:def456..." + links: ["bahn/decisions/api-design"] +``` + +### Zentrale Monorepo-Konfiguration (`monorepo.yaml`) + +```yaml +version: "1.0" +contexts: + - name: privat + description: "Persönliche Projekte" + - name: dhive + description: "dhive GmbH Projekte" + - name: bahn + description: "DB InfraGO Projekte" + - name: shared + description: "Kontextübergreifende Tools und Wissen" + +naming: + pattern: "^[a-z0-9][a-z0-9\\-]{0,48}[a-z0-9]$" + min_length: 2 + max_length: 50 + +security: + config: shared/config/access-config.yaml + audit_log: .audit/access.log + encryption: + tool: git-crypt + machine_context_config: shared/config/machine-context.yaml + key_source: keyring + federation: + config: shared/config/team-repos.yaml + conflict_strategy: team-wins +``` + +### Repository-Eintrag (für `repos.yaml`) + +```python +@dataclass +class RepoEntry: + name: str # Eindeutiger Name + url: str # Repository-URL + mode: Literal["read-only", "upstream"] + target: str # Zielpfad im Monorepo + pinned: str # Git-SHA, Tag oder Branch + mechanism: Literal["subtree", "submodule"] = "subtree" +``` + +### Migrations-Plan + +```python +@dataclass +class MigrationPlan: + source_repo: str # Pfad zum Quell-Repository + target_context: str # Ziel-Kontextordner + target_name: str # Projektname im Ziel + mode: Literal["direct", "subtree", "upstream"] + dependencies: list[str] # Abhängigkeiten zu anderen Repos + order: int # Migrationsreihenfolge +``` + +### Sicherheits-Event + +```python +@dataclass +class SecurityEvent: + timestamp: datetime + requesting_context: str + target_context: str + resource: str + action: str # read | write | execute + outcome: Literal["denied", "allowed"] +``` + +### Maschinenkontext + +```python +@dataclass +class MachineContext: + name: str # z.B. "andre-hauptrechner", "dhive-laptop" + description: str + authorized_contexts: list[str] # ["privat", "dhive", "bahn"] oder Subset + key_source: Literal["keyring", "file", "password-manager"] + password_manager: Optional[PasswordManagerConfig] = None + +@dataclass +class PasswordManagerConfig: + type: Literal["bitwarden", "1password", "keepass"] + vault: str # Name des Vaults/der Datenbank + entry_prefix: str = "monorepo-key-" # Prefix für Schlüssel-Einträge + +@dataclass +class EncryptionKey: + context: str # Zugehöriger Arbeitskontext + key_id: str # GPG Key-ID oder symmetrischer Key-Name + key_type: Literal["gpg", "symmetric"] + source: Literal["keyring", "file", "password-manager"] +``` + +### Team-Repo-Konfiguration + +```python +@dataclass +class TeamRepoEntry: + context: str # Zugehöriger Arbeitskontext (privat/dhive/bahn) + url: str # Repository-URL + branch: str # Sync-Branch (z.B. "main") + sync_direction: Literal["bidirectional", "hub-to-spoke", "spoke-to-hub"] + sync_frequency: Literal["on-push", "hourly", "daily", "manual"] + shared_mirror: Optional[SharedMirrorConfig] = None + +@dataclass +class SharedMirrorConfig: + enabled: bool + paths: list[str] # Pfade aus shared/ die gespiegelt werden + mode: Literal["read-only"] # Immer read-only im Team_Repo + +@dataclass +class SyncResult: + success: bool + context: str + direction: str # "pull" | "push" | "full" + commits_synced: int + conflicts: list[ConflictInfo] + timestamp: datetime + +@dataclass +class ConflictInfo: + file_path: str + conflict_type: Literal["content", "rename", "delete-modify"] + source: str # "monorepo" | "team-repo" + details: str + +@dataclass +class IsolationReport: + context: str + is_isolated: bool + leaks: list[IsolationLeak] # Gefundene Referenzen auf andere Kontexte + +@dataclass +class IsolationLeak: + file_path: str + line_number: int + leaked_context: str # Welcher fremde Kontext referenziert wird + leak_type: Literal["path", "env_var", "config_ref", "comment"] +``` + + + +## Correctness Properties + +*A property is a characteristic or behavior that should hold true across all valid executions of a system — essentially, a formal statement about what the system should do. Properties serve as the bridge between human-readable specifications and machine-verifiable correctness guarantees.* + +### Property 1: Namensvalidierung akzeptiert nur gültiges kebab-case + +*For any* String, die `validate_name`-Funktion soll genau dann `true` zurückgeben, wenn der String ausschließlich aus Kleinbuchstaben, Ziffern und Bindestrichen besteht, zwischen 2 und 50 Zeichen lang ist und nicht mit einem Bindestrich beginnt oder endet. + +**Validates: Requirements 1.3** + +### Property 2: Namenskollision verhindert doppelte Projekterstellung + +*For any* Sequenz von Projekt-Erstellungsaufrufen im gleichen Kontext, wenn ein Projektname bereits existiert, muss der zweite Aufruf mit diesem Namen fehlschlagen und die bestehende Projektstruktur unverändert lassen. + +**Validates: Requirements 1.6** + +### Property 3: Kontextübergreifender Secret-Zugriff wird verweigert + +*For any* Kombination aus anfragendem Kontext A und Zielkontext B (wobei A ≠ B und B ≠ shared), muss der Zugriff auf .env-Dateien und Secret-Dateien von B verweigert werden, und es muss ein Audit-Log-Eintrag mit Zeitstempel, anfragendem Kontext, Zielkontext und Ressource erzeugt werden. + +**Validates: Requirements 2.1, 2.3, 2.5, 2.7** + +### Property 4: Shared-Tool-Isolation + +*For any* Tool aus dem shared-Bereich, das in einem bestimmten Arbeitskontext ausgeführt wird, darf es ausschließlich auf die Secrets des aktiven Kontexts und auf explizit freigegebene shared-Ressourcen zugreifen. Jeder andere Zugriff muss blockiert werden. + +**Validates: Requirements 2.7, 8.5** + +### Property 5: Wissensartefakt-Ingestion erzeugt vollständige Metadaten im Index + +*For any* gültiges Wissensartefakt aus einem beliebigen Kontext, nach der Verarbeitung durch die ETL-Pipeline muss es im YAML-Index erscheinen mit korrektem YAML-Frontmatter (mindestens Typ, Titel, Tags, Quellkontext, Erstelldatum, Content-Hash) und der Pfad muss der Scope-basierten Ordnerstruktur entsprechen. + +**Validates: Requirements 3.2, 3.4, 3.6, 3.11, 3.13, 3.17, 3.21** + +### Property 6: Suche respektiert Scope-Berechtigungen + +*For any* Suchanfrage und beliebige Menge autorisierter Scopes, dürfen die Ergebnisse ausschließlich Artefakte enthalten, deren Scope in der autorisierten Menge liegt. Artefakte aus nicht-autorisierten Scopes dürfen weder in der Ergebnisliste erscheinen noch darf deren Existenz offengelegt werden. + +**Validates: Requirements 3.3, 3.9** + +### Property 7: Progressive-Disclosure-Index enthält nur kompakte Einträge + +*For any* Abfrage des YAML-Index (Schicht 1) müssen die zurückgegebenen Einträge Titel, Tags, Beziehungen und Kurzbeschreibungen enthalten, aber niemals den vollständigen Dokumentinhalt. Suchergebnisse liefern ausschließlich Pfade zurück. + +**Validates: Requirements 3.16, 3.18** + +### Property 8: Inkrementelle Verarbeitung erkennt Änderungen über Content-Hash + +*For any* Quelle, wenn der Content-Hash seit der letzten Verarbeitung unverändert ist, darf kein Update erfolgen. Wenn sich der Hash geändert hat, muss genau das betroffene Artefakt aktualisiert werden. + +**Validates: Requirements 3.12** + +### Property 9: Volltextsuche findet indexierte Inhalte + +*For any* indexiertes Artefakt mit einem bestimmten Suchbegriff im Inhalt, muss eine Volltextsuche nach diesem Begriff das Artefakt in den Ergebnissen zurückliefern (sofern der Scope autorisiert ist), sortiert nach Relevanz. + +**Validates: Requirements 3.7** + +### Property 10: ETL-Fehlerresilienz bewahrt erfolgreiche Artefakte + +*For any* ETL-Durchlauf, in dem eine Quelle fehlschlägt, müssen alle zuvor erfolgreich verarbeiteten Artefakte erhalten bleiben, der Fehler mit Quellidentifikator und Zeitstempel protokolliert werden, und die fehlgeschlagene Quelle beim nächsten Durchlauf erneut verarbeitet werden. + +**Validates: Requirements 3.20** + +### Property 11: Graph-Verknüpfungen werden in beiden Artefakten reflektiert + +*For any* zwei Artefakte, wenn eine gerichtete Verknüpfung von A nach B erstellt wird, muss die Kante sowohl im YAML-Frontmatter von A als auch im Frontmatter von B erscheinen. + +**Validates: Requirements 3.8** + +### Property 12: Read-Only-Repos blockieren Schreibzugriffe + +*For any* Schreiboperation (Datei anlegen, ändern oder löschen) auf ein als read-only konfiguriertes eingebundenes Repository muss die Operation blockiert werden und eine Fehlermeldung ausgegeben werden, die den Read-Only-Status und den Repository-Namen enthält. + +**Validates: Requirements 4.2, 4.5** + +### Property 13: Fehlgeschlagene Synchronisation bewahrt lokalen Stand + +*For any* fehlgeschlagene Synchronisation mit einem Upstream-Repository (Netzwerkfehler, Authentifizierungsfehler, Merge-Konflikt) muss der lokale Dateizustand identisch zum Zustand vor dem Sync-Versuch sein. + +**Validates: Requirements 4.9** + +### Property 14: Sensitive-Content-Filter blockiert Freigabe + +*For any* Wissensartefakt, das Muster für kontextspezifische Geheimnisse, Zugangsdaten oder personenbezogene Daten enthält, muss die kontextübergreifende Freigabe verweigert werden mit einer Fehlermeldung, die den Ablehnungsgrund benennt. + +**Validates: Requirements 5.1, 5.3** + +### Property 15: Freigabe-Lebenszyklus (Share → Update → Revoke) + +*For any* geteiltes Artefakt gilt: (a) nach Freigabe ist es in allen Kontexten als schreibgeschützte Lesereferenz sichtbar, (b) Aktualisierungen im Quellkontext werden beim nächsten Lesezugriff reflektiert, (c) nach Widerruf ist es in keinem Zielkontext mehr sichtbar. + +**Validates: Requirements 5.2, 5.5, 5.6** + +### Property 16: Freigabe erfordert explizite Nutzerbestätigung + +*For any* Versuch ein Artefakt kontextübergreifend zu teilen, ohne dass `user_confirmed=True` gesetzt ist, muss die Operation fehlschlagen. + +**Validates: Requirements 5.4** + +### Property 17: Migrations-Validierung prüft Vollständigkeit + +*For any* migrierten Repository-Zustand muss die Validierungsfunktion korrekt prüfen: Übereinstimmung der Commit-Anzahl, Vorhandensein aller Branches und Tags, Vollständigkeit des Dateibaums. + +**Validates: Requirements 6.4** + +### Property 18: Migrations-Rollback stellt Vor-Zustand wieder her + +*For any* fehlgeschlagene Migration eines einzelnen Repositories muss der Rollback den exakten Zustand vor der Migration wiederherstellen, ohne andere bereits migrierte Repositories zu beeinflussen. + +**Validates: Requirements 6.7** + +### Property 19: Orchestrator-Kontextauflösung und Workspace-Isolation + +*For any* Task mit gültigen Kontext-Metadaten muss der Orchestrator (a) das Workspace unter dem korrekten Kontextordner anlegen und (b) ausschließlich die .env-Datei dieses Kontexts laden. Für Tasks ohne gültigen Kontext muss der Start verweigert werden. + +**Validates: Requirements 7.1, 7.2, 7.3, 7.6** + +### Property 20: Orchestrator-Kontextverletzung bricht Task ab + +*For any* Zugriff des Orchestrators auf Dateien oder Secrets außerhalb des zugewiesenen Kontextordners muss der laufende Task abgebrochen, der Vorfall protokolliert und der Nutzer benachrichtigt werden. + +**Validates: Requirements 7.5** + +### Property 21: MCP-Konfiguration-Merge mit Kontext-Vorrang + +*For any* MCP-Server-Konfiguration, bei der sowohl eine shared-Konfiguration als auch eine kontextspezifische Konfiguration existiert, muss die effektive Konfiguration die kontextspezifischen Werte bevorzugen (Merge mit Override). + +**Validates: Requirements 8.4, 8.6** + +### Property 22: Shared-Tool-Versionierung ohne manuelle Synchronisation + +*For any* Aktualisierung eines Tools im shared-Bereich müssen alle Kontexte bei ihrer nächsten Ausführung die aktualisierte Version verwenden, ohne manuelle Schritte in einzelnen Kontexten. + +**Validates: Requirements 8.2** + +### Property 23: Verschlüsselte Secrets können nur mit autorisiertem Kontextschlüssel entschlüsselt werden + +*For any* verschlüsselte Secret-Datei eines Arbeitskontexts X und für jeden Entschlüsselungsversuch mit einem Schlüssel des Kontexts Y (wobei X ≠ Y), muss die Entschlüsselung fehlschlagen und die Datei im verschlüsselten Zustand verbleiben. Nur der Schlüssel des zugehörigen Kontexts X darf die Datei erfolgreich entschlüsseln. + +**Validates: Requirements 9.3, 9.8** + +### Property 24: Maschinenkontext beschränkt Entschlüsselung auf autorisierte Kontexte + +*For any* Maschinenkontext-Konfiguration mit einer definierten Menge autorisierter Arbeitskontexte, muss gelten: (a) Secrets der autorisierten Kontexte sind entschlüsselbar, (b) Secrets aller nicht-autorisierten Kontexte verbleiben als verschlüsselte Binärdaten im Working Tree, (c) die Fehlermeldung bei fehlgeschlagener Entschlüsselung gibt keinen Hinweis auf den Dateiinhalt, (d) das Repository bleibt vollständig funktionsfähig (Code, Konfiguration, Dokumentation zugänglich). + +**Validates: Requirements 9.4, 9.5, 9.8, 9.10** + +### Property 25: Team-Repos enthalten ausschließlich Inhalte des eigenen Kontexts + +*For any* Team_Repo für einen Arbeitskontext X muss gelten: (a) es enthält keine Dateien, Pfade, Konfigurationsreferenzen oder Umgebungsvariablen, die auf einen anderen Arbeitskontext Y (Y ≠ X) verweisen, (b) es enthält keine Secrets anderer Kontexte (weder verschlüsselt noch unverschlüsselt), (c) optional gespiegelte Dateien aus dem shared-Bereich sind als Read-Only markiert und enthalten keine kontextfremden Referenzen. + +**Validates: Requirements 10.5, 10.9, 10.10, 10.12** + +### Property 26: Bidirektionale Synchronisation bewahrt Git-Historie und löst Konflikte korrekt + +*For any* Sequenz von Commits in einem Team_Repo oder einem Monorepo-Kontextordner, nach einer bidirektionalen Synchronisation muss gelten: (a) alle Commits erscheinen mit vollständiger Autoren- und Zeitstempel-Information auf beiden Seiten, (b) bei Merge-Konflikten wird die Synchronisation abgebrochen, der Konflikt mit betroffenen Dateien und Quellen protokolliert, und dem Hub-Besitzer manuelle Auflösung ermöglicht, (c) das Team_Repo gilt als Single Source of Truth — bei Konflikten hat es Vorrang. + +**Validates: Requirements 10.3, 10.4, 10.6, 10.7, 10.8** + +### Property 27: Team-Mitglieder können die Existenz anderer Kontexte nicht entdecken + +*For any* Team_Repo und jeden Dateipfad, Konfigurationseintrag, Git-Remote-URL, Commit-Message oder Metadaten-Feld innerhalb des Team_Repos darf kein Hinweis auf die Existenz des Monorepos, anderer Arbeitskontexte oder anderer Team_Repos enthalten sein. Das Team_Repo muss als vollständig eigenständiges Repository erscheinen. + +**Validates: Requirements 10.5, 10.9** + +## Error Handling + +### Sicherheitsgrenzen + +| Fehlerfall | Verhalten | +|---|---| +| Zugriff auf fremden Kontext | Zugriff blockiert, Audit-Log-Eintrag, Operation gibt Fehler zurück | +| Ungültige access-config.yaml | System startet nicht, Fehlermeldung mit Validierungsdetails | +| Shared-Tool ohne gültige Autorisierung | Ausführung wird verhindert, Fehler mit Kontextinfo | + +### Wissensspeicher + +| Fehlerfall | Verhalten | +|---|---| +| ETL-Quellenfehler | Fehler protokolliert (Quelle + Zeitstempel), vorhandene Artefakte bewahrt, Retry beim nächsten Lauf | +| Ungültiges YAML-Frontmatter | Artefakt als fehlerhaft markiert, nicht indexiert, Fehler protokolliert | +| Index-Korruption | Vollständiger Index-Rebuild aus den Artefakt-Dateien | +| Suchanfrage-Timeout (>5s) | Abbruch mit Teilergebnis-Warnung | +| Duplikat-Content-Hash | Vorhandenes Artefakt beibehalten, neues als Konflikt melden | + +### Repository-Management + +| Fehlerfall | Verhalten | +|---|---| +| Sync fehlgeschlagen (Netzwerk) | Lokaler Stand unverändert, Fehlergrund protokolliert, Nutzer informiert | +| Merge-Konflikt bei Upstream-Sync | Sync abgebrochen, Konflikt protokolliert, manuelle Auflösung ermöglicht | +| Schreibversuch auf Read-Only-Repo | Operation blockiert, Fehlermeldung mit Repo-Name und Status | +| Submodule/Subtree-Init-Fehler | Fehlgeschlagenes Repo übersprungen, andere Repos weiter verfügbar | + +### Migration + +| Fehlerfall | Verhalten | +|---|---| +| Pfadkollision | Migration pausiert, Konflikt mit Dateipfad und Quell-Repos protokolliert | +| Branch-Namenskonflikt | Migration pausiert, betroffene Branches gelistet | +| Validierung fehlgeschlagen | Ergebnis dokumentiert, Rollback angeboten | +| Migration fehlgeschlagen | Automatischer Rollback auf Vor-Zustand, andere Repos unberührt | + +### Orchestrator + +| Fehlerfall | Verhalten | +|---|---| +| Kein Kontext ermittelbar | Task nicht gestartet, Fehlermeldung an Nutzer | +| Out-of-Context-Zugriff | Task abgebrochen, Vorfall protokolliert, Nutzer über OrgMyLife benachrichtigt | +| Wissensspeicher-Index nicht verfügbar | Task ohne Wissenskontext fortsetzen, Warnung protokolliert | + +### Verschlüsselung (Secret Encryption) + +| Fehlerfall | Verhalten | +|---|---| +| Entschlüsselung ohne autorisierten Schlüssel | Zugriff verweigert, Datei bleibt verschlüsselt, keine inhaltsbezogene Fehlermeldung | +| git-crypt nicht installiert | Setup-Fehler mit Installationsanleitung ausgeben, Repository im Nur-Lesen-Modus | +| Passwort-Manager nicht erreichbar | Fallback auf lokalen Keyring versuchen, Warnung protokollieren | +| Beschädigter Schlüssel im Keyring | Schlüssel als ungültig markieren, Neu-Import aus Passwort-Manager anbieten | +| Merge-Konflikt bei verschlüsselten Dateien | git-crypt-Merge-Treiber verwenden, bei Scheitern: Konflikt protokollieren, manuelle Auflösung | +| Maschinenkontext-Konfiguration fehlt | Alle Secrets verschlüsselt belassen, Warnung mit Setup-Verweis ausgeben | +| Ungültiger Maschinenkontext (unbekannter Kontext referenziert) | Konfiguration ablehnen, Validierungsfehler mit gültigen Kontexten ausgeben | + +### Föderation (Team-Repos) + +| Fehlerfall | Verhalten | +|---|---| +| Sync-Konflikt (Merge-Conflict) | Synchronisation abgebrochen, Konflikt mit Dateien und Quellen protokolliert, Hub-Besitzer informiert | +| Team_Repo nicht erreichbar (Netzwerk/Auth) | Sync übersprungen, Fehler protokolliert, nächster Versuch bei nächstem Trigger | +| Cross-Context-Leakage erkannt bei Isolation-Check | Sync blockiert, betroffene Dateien gelistet, manuelle Bereinigung erforderlich | +| Shared-Mirror-Quelle nicht vorhanden | Mirror-Schritt übersprungen, Warnung protokolliert, restlicher Sync fortgesetzt | +| Team_Repo-Branch divergiert stark (>100 Commits Differenz) | Warnung an Hub-Besitzer, Sync nur nach expliziter Bestätigung | +| Subtree-Push fehlgeschlagen | Lokaler Monorepo-Stand unverändert, Fehler mit Remote-Details protokolliert | +| Neues Team-Mitglied erhält falschen Kontext-Zugang | Sicherheitswarnung, Zugang sofort revoken, Vorfall im Audit-Log | + +## Testing Strategy + +### Testebenen + +1. **Property-Based Tests (Hypothesis)**: Universelle Eigenschaften über alle gültigen Eingaben (Minimum 100 Iterationen pro Property) +2. **Unit Tests (pytest)**: Spezifische Beispiele, Edge Cases, Fehlerbehandlung +3. **Integration Tests**: Git-Operationen, Dateisystem-Interaktionen, ETL-Pipeline-Durchläufe +4. **Smoke Tests**: Strukturprüfungen (Ordner existieren, Konfigurationen parsebar) + +### Property-Based Testing + +**Library:** Hypothesis (Python) + +Jeder Property-Test wird mit mindestens 100 Iterationen konfiguriert und referenziert die zugehörige Design-Property: + +```python +from hypothesis import given, settings +from hypothesis import strategies as st + +@settings(max_examples=100) +@given(name=st.text(min_size=1, max_size=60)) +def test_name_validation_property(name): + """Feature: monorepo-consolidation, Property 1: Namensvalidierung akzeptiert nur gültiges kebab-case""" + result = validate_name(name) + expected = bool(re.match(r'^[a-z0-9][a-z0-9\-]{0,48}[a-z0-9]$', name)) + assert result == expected +``` + +### Testabdeckung nach Komponente + +| Komponente | Property Tests | Unit Tests | Integration Tests | +|---|---|---|---| +| StructureManager | P1, P2 | Beispiel-Projekte, Edge Cases | — | +| ContextGuard | P3, P4 | Konfigurationsvalidierung | Dateisystem-Zugriffe | +| KnowledgeStore | P5–P11 | Frontmatter-Parsing, Fehlerszenarien | ETL-Pipeline mit Mock-Quellen | +| RepoManager | P12, P13 | Git-Hook-Installation | Subtree/Submodule-Operationen | +| ContextBridge | P14–P16 | Regex-Muster, Freigabelogik | — | +| MigrationEngine | P17, P18 | Migrationsplan-Validierung | git filter-repo-Operationen | +| OrchestratorAdapter | P19–P22 | YAML-Index-Abfrage, Prompt-Build | Workspace-Erstellung | +| SecretEncryptionManager | P23, P24 | Schlüssel-Validierung, Konfigurationsschema | git-crypt-Operationen, Passwort-Manager-Integration | +| FederationManager | P25, P26, P27 | Isolation-Check-Logik, Konfigurationsschema | Git-Subtree-Operationen, Sync-Szenarien | + +### Abgrenzung + +- **Kein PBT für**: Git-Remote-Operationen (Push/Pull/Sync), CI/CD-Konfigurationen, GitLab-Pages-Deployment, git-crypt-Tooling-Installation, Passwort-Manager-API +- **Integration Tests für**: Tatsächliche Git-Operationen, Dateisystem-Berechtigungen, Netzwerk-Sync, git-crypt encrypt/decrypt-Zyklen, Subtree-Push/Pull gegen Test-Repos +- **Smoke Tests für**: Konfigurationsdateien existieren und sind parsebar, Ordnerstruktur ist korrekt initialisiert, git-crypt ist installiert, Maschinenkontext-Konfiguration vorhanden diff --git a/.kiro/specs/monorepo-consolidation/requirements.md b/.kiro/specs/monorepo-consolidation/requirements.md new file mode 100644 index 0000000..5377b12 --- /dev/null +++ b/.kiro/specs/monorepo-consolidation/requirements.md @@ -0,0 +1,193 @@ +# Requirements Document + +## Introduction + +Dieses Dokument beschreibt die Anforderungen für die Konsolidierung von 18+ einzelnen Repositories in eine einheitliche Monorepo-Struktur. Das Ziel ist es, alle Projekte in einem System zu vereinen, dabei klare Sicherheitsgrenzen zwischen Arbeitskontexten (Privat, dhive, Bahn) zu wahren, einen übergreifenden Wissensspeicher aufzubauen, externe Repos als Read-Only einzubinden, Secrets verschlüsselt im Repository zu speichern (statt sie auszuschließen), und eine föderierte Zusammenarbeit mit verschiedenen Teams über eigenständige Team-Repositories zu ermöglichen. + +## Glossary + +- **Monorepo**: Ein einzelnes Repository, das mehrere Projekte und Arbeitskontexte in einer gemeinsamen Ordnerstruktur vereint +- **Arbeitskontext**: Eine logische Sicherheitsdomäne, die zusammengehörige Projekte gruppiert (Privat, dhive, Bahn) +- **Wissensspeicher**: Ein übergreifendes System zur Erfassung, Strukturierung und kontextübergreifenden Abfrage von Wissen (Nachfolger des NoteGraph) +- **Sicherheitsgrenze**: Eine Konfiguration, die den Zugriff auf Dateien und Geheimnisse eines Arbeitskontexts auf berechtigte Prozesse beschränkt +- **Externes_Repo**: Ein Repository eines Dritten oder eines Team-Kontexts, das als unveränderbare Referenz oder bidirektionale Synchronisationsquelle in die Monorepo-Struktur eingebunden wird +- **Kontextbrücke**: Ein Mechanismus, der es erlaubt, Erkenntnisse aus einem Arbeitskontext in einem anderen nutzbar zu machen, ohne Sicherheitsgrenzen zu verletzen +- **Ordnerstruktur**: Die hierarchische Organisation von Projekten und Modulen innerhalb des Monorepos +- **Agent_Harness**: Ein AI-Coding-Tool oder -System (z.B. Kiro, Codex, Claude Code, Antigravity), das als Ausführungsumgebung für AI-gestützte Entwicklungsaufgaben dient +- **Agent_Erweiterung**: Ein Plugin, Skill, Power oder ähnliches Konfigurationsartefakt, das einem Agent_Harness zusätzliche Fähigkeiten verleiht (z.B. Kiro Powers, Claude Code Slash-Commands, Codex-Plugins) +- **Orchestrator**: Der bestehende AI-Orchestrator-Service, der Tasks aus OrgMyLife verarbeitet und an verschiedene Agent_Harnesses dispatcht, wobei die Harness-Auswahl pro Arbeitskontext oder pro Task konfigurierbar ist +- **Secret_Encryption**: Die Verschlüsselung von Geheimnissen im Repository mittels git-crypt, SOPS oder age, sodass Secrets verschlüsselt committed werden und nur mit autorisierten Schlüsseln entschlüsselt werden können +- **Maschinenkontext**: Die einem bestimmten Rechner zugeordnete Kombination aus autorisierten Arbeitskontexten und den entsprechenden Entschlüsselungsschlüsseln (z.B. "dhive-Rechner" hat nur dhive-Schlüssel) +- **Team_Repo**: Ein eigenständiges Repository, das einem bestimmten Team (dhive, bahn, privat/Familie) gehört und bidirektional mit dem entsprechenden Kontextordner im Monorepo synchronisiert wird +- **Föderierte_Struktur**: Eine Hub-and-Spoke-Topologie, in der das Monorepo als zentraler Hub fungiert und die Team-Repos als unabhängige Spokes, wobei nur der Hub-Besitzer (Andre) Zugriff auf alle Kontexte hat + +## Requirements + +### Requirement 1: Einheitliche Ordnerstruktur + +**User Story:** Als Entwickler möchte ich alle meine Projekte in einer klar strukturierten Ordnerhierarchie organisiert haben, damit ich schnell navigieren und den Überblick behalten kann. + +#### Acceptance Criteria + +1. THE Monorepo SHALL alle Projekte in einer flachen Kontext-Hierarchie mit exakt drei Ebenen organisieren: Ebene 1 (Kontextordner) → Ebene 2 (Projektordner) → Ebene 3 (Modulordner), wobei unterhalb der Modulebene keine weitere strukturelle Verschachtelung von Projekten oder Modulen erlaubt ist +2. WHEN ein neues Projekt angelegt wird, THE Ordnerstruktur SHALL das Projekt genau einem der definierten Kontextordner (privat, dhive, bahn, shared) zuordnen +3. THE Monorepo SHALL eine einheitliche Namenskonvention in kebab-case für alle Ordner auf Projekt- und Modulebene verwenden, wobei Ordnernamen ausschließlich aus Kleinbuchstaben, Ziffern und Bindestrichen bestehen und zwischen 2 und 50 Zeichen lang sein dürfen +4. THE Ordnerstruktur SHALL genau folgende vier Kontextordner auf oberster Ebene bereitstellen: privat, dhive, bahn, shared +5. WHEN ein Projekt Funktionalität bereitstellt, die von Projekten in mindestens zwei unterschiedlichen Arbeitskontexten genutzt wird, THE Ordnerstruktur SHALL das Projekt im shared-Kontext platzieren +6. IF ein Projektordner-Name bereits im Ziel-Kontext existiert, THEN THE Ordnerstruktur SHALL die Anlage verweigern und eine Fehlermeldung ausgeben, die den Namenskonflikt benennt + +### Requirement 2: Sicherheitsgrenzen zwischen Arbeitskontexten + +**User Story:** Als Entwickler möchte ich sicherstellen, dass Geheimnisse und sensible Daten eines Arbeitskontexts nicht versehentlich in einen anderen Kontext gelangen, damit die Trennung meiner beruflichen Realitäten gewahrt bleibt. + +#### Acceptance Criteria + +1. THE Sicherheitsgrenze SHALL verhindern, dass Secrets eines Arbeitskontexts für Prozesse eines anderen Arbeitskontexts über Dateisystemzugriff oder Umgebungsvariablen lesbar sind, wobei die Zugriffskontrolle auf dem Ausführungskontext des Prozesses und den verfügbaren Entschlüsselungsschlüsseln basiert +2. THE Sicherheitsgrenze SHALL für jeden Arbeitskontext eine eigene .env-Datei im jeweiligen Kontextordner bereitstellen, die ausschließlich die Geheimnisse dieses Kontexts enthält und im Repository verschlüsselt gespeichert wird (siehe Requirement 9) +3. WHEN ein Prozess in einem beliebigen Arbeitskontext ausgeführt wird, THE Sicherheitsgrenze SHALL den Zugriff auf entschlüsselte .env-Dateien und Secret-Dateien aller anderen Arbeitskontexte unterbinden +4. THE Sicherheitsgrenze SHALL Secret-Dateien aller Kontexte über Verschlüsselung im Repository schützen (Encryption-at-Rest), wobei die bisherige .gitignore-basierte Ausschlussstrategie durch die kontextspezifische Verschlüsselung gemäß Requirement 9 ersetzt wird; unverschlüsselte temporäre Dateien (Build-Artefakte, Caches) werden weiterhin via .gitignore ausgeschlossen +5. IF ein Prozess auf Secrets eines nicht-autorisierten Kontexts zugreift, THEN THE Sicherheitsgrenze SHALL einen Eintrag in eine Protokolldatei schreiben, der Zeitstempel, den anfragenden Kontext, den Zielkontext und die angefragte Ressource enthält +6. THE Sicherheitsgrenze SHALL eine zentrale Konfigurationsdatei bereitstellen, die für jeden Arbeitskontext definiert, auf welche .env-Dateien, Secret-Dateien und shared-Ressourcen dieser Kontext zugreifen darf +7. WHILE ein Tool aus dem shared-Bereich in einem bestimmten Arbeitskontext ausgeführt wird, THE Sicherheitsgrenze SHALL proaktiv korrekte Autorisierungszustände aufrechterhalten und dem Tool ausschließlich Zugriff auf die Secrets des aktiven Kontexts und auf explizit freigegebene shared-Ressourcen gewähren + +### Requirement 3: Übergreifender Wissensspeicher + +**User Story:** Als Wissensarbeiter möchte ich einen zentralen Wissensspeicher haben, der auf der bewährten ETL-Architektur der DB-Wissensdatenbank basiert und Informationen aus allen Kontexten aufnimmt, damit ich relevantes Wissen jederzeit kontextübergreifend abrufen kann. + +#### Acceptance Criteria + +1. THE Wissensspeicher SHALL als Fork/Adaption der DB-Wissensdatenbank-ETL-Pipeline im shared-Bereich des Monorepos implementiert werden +2. THE Wissensspeicher SHALL Wissensartefakte aus allen Arbeitskontexten aufnehmen und indexieren +3. WHEN eine Suchanfrage gestellt wird, THE Wissensspeicher SHALL Ergebnisse aus allen Kontexten liefern, für die der Nutzer berechtigt ist, und die Ergebnisliste innerhalb von 5 Sekunden zurückgeben +4. THE Wissensspeicher SHALL Wissensartefakte mit Metadaten versehen: Quellkontext, Erstelldatum, Tags und verknüpfte Projekte +5. THE Wissensspeicher SHALL bestehende NoteGraph-Daten migrieren können, wobei die Migration als erfolgreich gilt, wenn alle Artefakte mit ihren Metadaten, Verknüpfungen und der Verzeichnisstruktur (decisions, inbox, meetings, people, projects) im Zielsystem vorhanden und über den Index auffindbar sind +6. WHEN ein neues Wissensartefakt erstellt wird, THE Wissensspeicher SHALL automatisch den Quellkontext und das Erstelldatum erfassen +7. THE Wissensspeicher SHALL eine Volltextsuche über alle indexierten Artefakte bereitstellen, die Treffer nach Relevanz sortiert zurückliefert +8. THE Wissensspeicher SHALL Verknüpfungen zwischen Artefakten unterschiedlicher Kontexte als gerichtete Graph-Kanten im YAML-Frontmatter beider beteiligter Artefakte unterstützen +9. WHILE ein Wissensartefakt einem vertraulichen Kontext zugeordnet ist, THE Wissensspeicher SHALL den Inhalt nur für berechtigte Abfragen bereitstellen und bei unberechtigtem Zugriff das Artefakt aus den Suchergebnissen ausschließen, ohne dessen Existenz offenzulegen +10. THE Wissensspeicher SHALL die ETL-Quellstrategien der DB-Wissensdatenbank wiederverwenden (Confluence-Seiten, Webseiten, PDFs, Markdown-Dateien, GitLab-Repos) +11. THE Wissensspeicher SHALL verarbeitetes Wissen in einer Scope-basierten Ordnerstruktur ablegen, wobei jeder Arbeitskontext einem Scope entspricht (privat, dhive, bahn, shared) +12. WHEN eine Quelle inkrementell verarbeitet wird, THE Wissensspeicher SHALL nur geänderte oder neue Inhalte aktualisieren, wobei Änderungen anhand von Content-Hashes erkannt werden +13. THE Wissensspeicher SHALL einen maschinenlesbaren Index bereitstellen, der den gesamten Wissensbestand beschreibt (Dokumente, Metadaten, Content-Hashes) +14. THE Wissensspeicher SHALL die Scope-Konfiguration um die persönlichen Arbeitskontexte erweitern, wobei das Mapping Arbeitskontext → Scope in einer zentralen Konfigurationsdatei definiert wird +15. WHERE die DB-Wissensdatenbank als aktives Bahn-Projekt eingebunden ist, THE Wissensspeicher SHALL Upstream-Änderungen der ETL-Pipeline übernehmen können, auch wenn der Wissensspeicher noch nicht vollständig implementiert ist +16. THE Wissensspeicher SHALL ein Progressive-Disclosure-Indexsystem bereitstellen, das Agenten ermöglicht auf Schicht 1 nur eine kompakte YAML-Index-Datei pro Domäne zu lesen (Titel, Tags, Beziehungen, Kurzbeschreibungen), ohne vollständige Markdown-Dateien laden zu müssen +17. THE Wissensspeicher SHALL jedes Wissensartefakt mit YAML-Frontmatter versehen, das mindestens Typ, Titel, Tags, Quellkontext und Verknüpfungen zu anderen Artefakten enthält +18. WHEN ein Agent eine Suchanfrage stellt, THE Wissensspeicher SHALL über den YAML-Index die relevanten Dokumente identifizieren und nur deren Pfade zurückliefern, sodass der Agent gezielt einzelne Dateien nachladen kann +19. THE Wissensspeicher SHALL sich am Google Open Knowledge Format (OKF) orientieren: ein Konzept pro Datei, Markdown mit YAML-Frontmatter, Cross-Links als Graph-Kanten, index.md pro Verzeichnis für Inhaltsübersichten +20. IF die ETL-Pipeline bei der Verarbeitung einer Quelle fehlschlägt, THEN THE Wissensspeicher SHALL den Fehler mit Quellidentifikator und Zeitstempel protokollieren, bereits erfolgreich verarbeitete Artefakte beibehalten und die fehlgeschlagene Quelle beim nächsten Durchlauf erneut verarbeiten +21. WHEN die ETL-Pipeline einen Verarbeitungsdurchlauf abschließt, THE Wissensspeicher SHALL den YAML-Index innerhalb desselben Durchlaufs aktualisieren, sodass neu verarbeitete Artefakte sofort über den Index auffindbar sind + +### Requirement 4: Einbindung externer und geteilter Repos + +**User Story:** Als Entwickler möchte ich externe Repositories als unveränderbare Referenz, geteilte Repos mit Upstream-Anbindung und Team-Repos mit bidirektionaler Synchronisation in mein Monorepo einbinden, damit ich deren Inhalte nutzen, bei Bedarf Änderungen zurückspielen und mit verschiedenen Teams in ihrem jeweiligen Kontext zusammenarbeiten kann. + +#### Acceptance Criteria + +1. THE Monorepo SHALL externe Repositories als Read-Only-Referenz einbinden können +2. WHEN ein externes Repo eingebunden wird, THE Monorepo SHALL dessen Inhalte als unveränderbar behandeln, indem lokale Commits und Dateiänderungen im eingebundenen Verzeichnis durch Git-Hooks oder Dateisystem-Schutzmechanismen verhindert werden +3. THE Monorepo SHALL eine Konfigurationsdatei bereitstellen, die alle eingebundenen externen Repos mit Repository-URL, gepinnter Version (Git-Commit-SHA oder Tag), Einbindungsmodus (read-only oder upstream) und Zielpfad im Monorepo auflistet +4. WHEN der Nutzer eine Aktualisierung eines externen Repos auslöst, THE Monorepo SHALL die lokale Kopie auf die in der Konfigurationsdatei angegebene Version aktualisieren und stets eine Ergebnismeldung anzeigen (Erfolg, Misserfolg oder 'Aktualisierung abgeschlossen – Status unbekannt') +5. IF ein Schreibzugriff auf ein als read-only konfiguriertes Repo versucht wird (Datei anlegen, ändern oder löschen im eingebundenen Verzeichnis), THEN THE Monorepo SHALL den Zugriff blockieren und eine Fehlermeldung ausgeben, die den Read-Only-Status und den Namen des betroffenen Repos benennt +6. THE Monorepo SHALL Git-Submodules oder Git-Subtrees für die Einbindung externer Repos verwenden +7. WHERE ein Repo als aktives Upstream-Projekt eingebunden ist, THE Monorepo SHALL bidirektionale Synchronisation mit dem entfernten Repository ermöglichen (Pull von Upstream, Push von lokalen Änderungen), wobei bei Merge-Konflikten die Synchronisation abgebrochen, der Konflikt protokolliert und eine manuelle Auflösung ermöglicht wird; für Read-Only-Repos SHALL das Monorepo Konflikte erkennen und protokollieren, ohne die Synchronisation abzubrechen (da keine tatsächliche Synchronisation stattfindet) +8. THE Monorepo SHALL die DB-Wissensdatenbank als aktives Bahn-Projekt mit Upstream-Anbindung zum DB-GitLab einbinden können +9. IF eine Synchronisation mit einem Upstream-Repository fehlschlägt (Netzwerkfehler, Authentifizierungsfehler oder Merge-Konflikt), THEN THE Monorepo SHALL den lokalen Stand unverändert beibehalten, den Fehlergrund protokollieren und den Nutzer über den fehlgeschlagenen Vorgang informieren + +### Requirement 5: Kontextübergreifender Wissenstransfer + +**User Story:** Als Entwickler möchte ich, dass meine verschiedenen Arbeitskontexte voneinander profitieren können, ohne Sicherheitsgrenzen zu verletzen, damit Erkenntnisse und Patterns wiederverwendbar werden. + +#### Acceptance Criteria + +1. THE Kontextbrücke SHALL es ermöglichen, Wissensartefakte eines Kontexts in anderen Kontexten als Lesereferenz bereitzustellen, wobei beliebige Artefakte teilbar sind, die Kontextbrücke jedoch Artefakte mit sensiblen Inhalten (kontextspezifische Credentials, Endpoints, personenbezogene Daten) vor der Freigabe filtert und blockiert +2. WHEN ein Wissensartefakt als "teilbar" markiert wird, THE Kontextbrücke SHALL es in allen Kontexten als schreibgeschützte Lesereferenz bereitstellen +3. IF ein als "teilbar" markiertes Artefakt kontextspezifische Geheimnisse, Zugangsdaten oder personenbezogene Daten enthält, THEN THE Kontextbrücke SHALL die Freigabe verweigern und eine Fehlermeldung ausgeben, die den Grund der Ablehnung benennt +4. THE Kontextbrücke SHALL eine explizite Freigabe durch den Nutzer erfordern, bevor Information kontextübergreifend geteilt wird +5. WHILE ein geteiltes Artefakt im Quellkontext aktualisiert wird, THE Kontextbrücke SHALL die Änderungen beim nächsten Lesezugriff in allen verknüpften Kontexten sichtbar machen +6. WHEN ein Nutzer die Freigabe eines geteilten Artefakts widerruft, THE Kontextbrücke SHALL das Artefakt aus allen Zielkontexten als Lesereferenz entfernen + +### Requirement 6: Migration bestehender Repositories + +**User Story:** Als Entwickler möchte ich meine bestehenden 18+ Repositories in die neue Monorepo-Struktur überführen können, ohne Git-Historie oder bestehende Funktionalität zu verlieren. + +#### Acceptance Criteria + +1. THE Monorepo SHALL die vollständige Git-Historie aller migrierten Repositories bewahren, einschließlich aller Commits, aller Branches, aller Tags und der zugehörigen Autoreninformationen +2. WHEN ein Repository migriert wird, THE Monorepo SHALL sicherstellen, dass alle existierenden CI/CD-Konfigurationen entweder unverändert lauffähig bleiben oder mit einer dokumentierten Anpassungsanleitung (Grund der Änderung, neue Konfiguration, Testnachweis) versehen werden +3. THE Monorepo SHALL einen dokumentierten Migrationsplan bereitstellen, der für jedes der 18+ Repositories den Ziel-Kontextordner, die Migrationsreihenfolge, die Abhängigkeiten zu anderen Repositories und den erwarteten Einbindungsmodus (direkt, Submodule, Upstream) beschreibt +4. WHEN die Migration eines einzelnen Repositories abgeschlossen ist, THE Monorepo SHALL eine Validierung durchführen, die mindestens folgende Prüfungen umfasst: Übereinstimmung der Commit-Anzahl, Vorhandensein aller Branches und Tags, Vollständigkeit des Dateibaums im letzten Commit und erfolgreicher Durchlauf vorhandener Tests, wobei die Validierung für alle abgeschlossenen Migrationen durchgeführt wird (unabhängig davon ob sie erfolgreich waren oder fehlschlugen) und die Migration auch bei Nicht-Ausführbarkeit des Validierungsschritts als erfolgreich gilt, sofern alle zugrundeliegenden Prüfungen bestanden wurden +5. IF während der Migration ein Konflikt auftritt (Pfadkollision, Branch-Namenskonflikt oder Namenskonvention-Verletzung), THEN THE Monorepo SHALL den Konflikt mit betroffener Datei, Konflikttyp und beteiligten Quell-Repositories protokollieren und die Migration des betroffenen Repositories pausieren, bis eine manuelle Auflösung erfolgt ist +6. THE Monorepo SHALL die Migration inkrementell unterstützen, sodass Repositories einzeln und unabhängig voneinander überführt werden können, während bereits migrierte und noch nicht migrierte Repositories parallel funktionsfähig bleiben +7. IF die Migration eines einzelnen Repositories fehlschlägt, THEN THE Monorepo SHALL den Zustand vor der Migration dieses Repositories wiederherstellen können, ohne bereits erfolgreich migrierte Repositories zu beeinflussen + +### Requirement 7: Orchestrator-Integration und Multi-Harness-Dispatch + +**User Story:** Als Entwickler möchte ich, dass der bestehende AI-Orchestrator nahtlos mit der neuen Monorepo-Struktur zusammenarbeitet und Tasks an verschiedene Agent_Harnesses (Kiro, Codex, Claude Code, Antigravity u.a.) dispatchen kann, damit ich pro Kontext oder Task den optimalen Agenten einsetzen kann. + +#### Acceptance Criteria + +1. WHEN der Orchestrator einen Task bearbeitet, THE Orchestrator SHALL den zugehörigen Arbeitskontext anhand der Task-Metadaten (Labels oder Projektzuordnung) ermitteln und das Arbeitsverzeichnis des Agent_Harness auf den entsprechenden Kontextordner innerhalb der Monorepo-Struktur beschränken +2. THE Orchestrator SHALL seinen Workspace-Root-Pfad in der WORKFLOW.md-Konfiguration so auflösen, dass pro Task ein Arbeitsverzeichnis unterhalb des ermittelten Kontextordners (privat, dhive, bahn oder shared) angelegt wird +3. WHEN der Orchestrator in einem Kontext arbeitet, THE Orchestrator SHALL ausschließlich die .env-Datei des jeweiligen Kontextordners laden und Umgebungsvariablen anderer Kontexte weder lesen noch an den Agent_Harness-Prozess weitergeben +4. WHEN der Orchestrator einen Task-Prompt zusammenstellt, THE Orchestrator SHALL den YAML-Index des Wissensspeichers abfragen und relevante Artefakt-Pfade als Kontextinformation in den Agenten-Prompt einfügen +5. IF der Orchestrator einen Dateizugriff oder Secret-Zugriff außerhalb des zugewiesenen Kontextordners versucht, THEN THE Orchestrator SHALL den laufenden Task abbrechen, den Vorfall mit Task-Identifier und Zugriffspfad protokollieren und den Nutzer über den bestehenden OrgMyLife-Benachrichtigungskanal informieren +6. WHEN kein Arbeitskontext aus den Task-Metadaten ermittelt werden kann, THE Orchestrator SHALL den Task nicht starten und eine Fehlermeldung mit dem fehlenden Kontext-Mapping an den Nutzer senden +7. THE Orchestrator SHALL eine Konfiguration bereitstellen, die pro Arbeitskontext und optional pro Task-Typ den zu verwendenden Agent_Harness definiert (z.B. dhive → Codex, bahn → Kiro, privat → Claude Code) +8. THE Orchestrator SHALL eine einheitliche Schnittstelle für alle Agent_Harnesses bereitstellen, die mindestens Prompt-Injection, Workspace-Verzeichnis, Umgebungsvariablen-Übergabe und Agent_Erweiterungen-Konfiguration umfasst +9. WHEN ein Task einem Agent_Harness zugewiesen wird, THE Orchestrator SHALL die harness-spezifische Startsequenz ausführen (CLI-Aufruf, API-Call oder Prozessstart), wobei die Startsequenz pro Harness in der Konfiguration definiert ist +10. IF der konfigurierte Agent_Harness für einen Task nicht verfügbar ist (nicht installiert, nicht lizenziert, Netzwerkfehler), THEN THE Orchestrator SHALL den Task als nicht-ausführbar markieren, den Fehlergrund protokollieren und den Nutzer benachrichtigen + +### Requirement 8: Shared Tooling und Agent-Erweiterungen + +**User Story:** Als Entwickler möchte ich übergreifende Tools, Agent-Erweiterungen (Powers, Skills, Plugins) und Steering-Konfigurationen an einer zentralen Stelle verwalten, damit sie allen Kontexten und allen eingesetzten Agent_Harnesses zur Verfügung stehen und nicht dupliziert werden müssen. + +#### Acceptance Criteria + +1. THE Ordnerstruktur SHALL einen dedizierten shared-Bereich für kontextübergreifende Tools, Agent-Erweiterungen und Konfigurationen bereitstellen +2. WHEN ein Tool im shared-Bereich aktualisiert wird, THE Monorepo SHALL sicherstellen, dass alle Kontexte bei ihrer nächsten Ausführung die aktualisierte Version des Tools verwenden, ohne manuelle Synchronisation in den einzelnen Kontexten +3. THE Monorepo SHALL Agent-Erweiterungen (Steering-Dateien, Prompt-Vorlagen, Kontext-Definitionen, Skills) zentral im shared-Bereich ablegen und harness-agnostisch organisieren, sodass sie von verschiedenen Agent_Harnesses (Kiro, Codex, Claude Code, Antigravity u.a.) genutzt werden können +4. THE Monorepo SHALL MCP-Server-Konfigurationen zentral im shared-Bereich bereitstellen, wobei kontextspezifische Konfigurationen die zentrale Konfiguration überschreiben (Kontext-Konfiguration hat Vorrang vor shared-Konfiguration) +5. WHILE ein geteiltes Tool in einem bestimmten Kontext ausgeführt wird, THE Sicherheitsgrenze SHALL sicherstellen, dass das Tool nur auf die Dateien und Secrets des aktiven Kontexts sowie auf den shared-Bereich zugreift; IF die Sicherheitsgrenzen nicht durchgesetzt werden können, THEN SHALL die Ausführung des geteilten Tools verhindert werden +6. IF ein geteiltes Tool oder eine zentrale Konfiguration einen Konflikt mit einer kontextspezifischen Überschreibung erzeugt, THEN THE Monorepo SHALL den Konflikt protokollieren und die kontextspezifische Konfiguration beibehalten +7. THE Monorepo SHALL eine Konfigurationsdatei bereitstellen, die pro Arbeitskontext den eingesetzten Agent_Harness und dessen harness-spezifische Konfiguration definiert (z.B. Kiro-Powers-Pfad, Codex-Agenten-Config, Claude-Code-Projektdatei) +8. THE Monorepo SHALL Agent-Erweiterungen in einem harness-unabhängigen Format ablegen (Markdown-Steering-Dateien, YAML-Konfigurationen, Prompt-Templates), die bei Bedarf in harness-spezifische Formate transformiert werden können +9. WHEN ein neuer Agent_Harness konfiguriert wird, THE Monorepo SHALL einen Adapter-Mechanismus bereitstellen, der die zentral abgelegten Agent-Erweiterungen für den neuen Harness verfügbar macht, ohne die bestehenden Erweiterungen zu duplizieren + + +### Requirement 9: Verschlüsselte Secrets im Repository + +**User Story:** Als Entwickler möchte ich Secrets verschlüsselt im Repository speichern (statt sie via .gitignore auszuschließen), damit das Repo vollständig selbstbeschreibend und portabel über mehrere Maschinen hinweg ist, ohne dass Secrets manuell transportiert oder separat gesichert werden müssen. + +#### Acceptance Criteria + +1. THE Secret_Encryption SHALL alle Secret-Dateien (.env-Dateien, private Schlüssel, Token-Dateien) verschlüsselt im Repository speichern, sodass sie committet und gepusht werden können, ohne im Klartext im Git-Verlauf oder Working Tree sichtbar zu sein +2. THE Secret_Encryption SHALL ein bewährtes Verschlüsselungstool (git-crypt, SOPS oder age) für die Encryption-at-Rest verwenden +3. THE Secret_Encryption SHALL für jeden Arbeitskontext (privat, dhive, bahn) einen eigenen Verschlüsselungsschlüssel verwenden, sodass der Besitz eines Schlüssels nur den zugehörigen Kontext entschlüsseln kann +4. WHEN eine Maschine einem bestimmten Maschinenkontext zugeordnet ist, THE Secret_Encryption SHALL ausschließlich die Secrets der autorisierten Arbeitskontexte dieser Maschine entschlüsseln +5. THE Secret_Encryption SHALL eine Konfigurationsdatei bereitstellen, die das Mapping von Maschinenkontext zu autorisierten Arbeitskontexten und Schlüsseln definiert (z.B. "dhive-Laptop" → nur dhive-Schlüssel, "Andre-Hauptrechner" → alle Schlüssel) +6. WHEN ein neuer Rechner eingerichtet wird, THE Secret_Encryption SHALL einen dokumentierten Onboarding-Prozess bereitstellen, der die Installation der Entschlüsselungsschlüssel für die autorisierten Kontexte beschreibt +7. THE Secret_Encryption SHALL mit einem Passwort-Manager integrierbar sein, sodass Entschlüsselungsschlüssel sicher gespeichert und bei Bedarf abgerufen werden können +8. IF ein Entschlüsselungsversuch mangels autorisiertem Schlüssel fehlschlägt, THEN THE Secret_Encryption SHALL den Zugriff verweigern und die betroffenen Dateien im verschlüsselten Zustand belassen, ohne eine Fehlermeldung auszugeben, die den Inhalt der Datei offenlegt +9. THE Secret_Encryption SHALL die bisherige .gitignore-basierte Ausschlussstrategie für Secret-Dateien ablösen, wobei .gitignore weiterhin für Build-Artefakte, Caches und temporäre Dateien verwendet wird +10. WHEN ein Nutzer auf einer nicht-autorisierten Maschine das Repository klont, THE Secret_Encryption SHALL alle nicht-autorisierten Secret-Dateien als verschlüsselte Binärdaten im Working Tree belassen, sodass das Repository funktionsfähig bleibt (Code, Konfiguration, Dokumentation sind verfügbar), aber Secrets unzugänglich sind +11. THE Secret_Encryption SHALL sicherstellen, dass verschlüsselte Dateien problemlos gemergt werden können, indem der Merge auf der verschlüsselten Ebene stattfindet oder ein git-crypt-kompatibler Merge-Treiber verwendet wird + +### Requirement 10: Föderierte Repo-Struktur und Multi-Team-Zusammenarbeit + +**User Story:** Als Entwickler möchte ich mit verschiedenen Teams (dhive-Kollegen, Bahn-Kollegen, Familie) zusammenarbeiten, wobei jedes Team ein eigenständiges Repository für seinen Kontext hat und nur ich als Hub-Besitzer das kombinierte Monorepo sehe, damit Sicherheitsgrenzen automatisch durchgesetzt werden und Teams unabhängig arbeiten können. + +#### Acceptance Criteria + +1. THE Föderierte_Struktur SHALL für jeden Arbeitskontext (privat, dhive, bahn) ein eigenständiges Team_Repo unterstützen, das unabhängig vom Monorepo als vollständiges Git-Repository funktioniert +2. THE Föderierte_Struktur SHALL eine Hub-and-Spoke-Topologie implementieren, in der das Monorepo als zentraler Hub fungiert und jedes Team_Repo als unabhängiger Spoke +3. WHEN Änderungen in einem Team_Repo vorgenommen werden, THE Föderierte_Struktur SHALL diese Änderungen bidirektional mit dem entsprechenden Kontextordner im Monorepo synchronisieren können +4. WHEN Änderungen im Monorepo in einem Kontextordner vorgenommen werden, THE Föderierte_Struktur SHALL diese Änderungen zum zugehörigen Team_Repo synchronisieren können +5. THE Föderierte_Struktur SHALL sicherstellen, dass Team-Mitglieder eines Kontexts ausschließlich Zugriff auf das Team_Repo ihres eigenen Kontexts haben und weder den Inhalt noch die Existenz anderer Kontexte im Monorepo erkennen können +6. THE Föderierte_Struktur SHALL das Team_Repo als Single Source of Truth für den jeweiligen Kontext behandeln, wobei bei Konflikten zwischen Team_Repo und Monorepo das Team_Repo Vorrang hat +7. IF eine bidirektionale Synchronisation zwischen Monorepo und Team_Repo einen Merge-Konflikt erzeugt, THEN THE Föderierte_Struktur SHALL die Synchronisation abbrechen, den Konflikt mit betroffenen Dateien und Quellen protokollieren und dem Hub-Besitzer (Andre) eine manuelle Auflösung ermöglichen +8. THE Föderierte_Struktur SHALL die Synchronisation über Git-Subtrees oder einen vergleichbaren Mechanismus realisieren, der die vollständige Git-Historie des Team_Repos im Monorepo bewahrt +9. WHEN ein neues Team-Mitglied Zugang benötigt, THE Föderierte_Struktur SHALL ausschließlich Zugriff auf das entsprechende Team_Repo erteilen, ohne Kenntnis des Monorepos oder der Existenz anderer Kontexte preiszugeben +10. THE Föderierte_Struktur SHALL die verschlüsselten Secrets gemäß Requirement 9 so handhaben, dass in den Team_Repos nur die Secrets des jeweiligen Kontexts enthalten sind (unverschlüsselt oder mit dem Team-Schlüssel verschlüsselt), während das Monorepo alle Kontexte mit kontextspezifischen Schlüsseln enthält +11. THE Föderierte_Struktur SHALL eine Konfigurationsdatei bereitstellen, die für jeden Arbeitskontext das zugehörige Team_Repo (URL, Branch, Sync-Richtung, Sync-Frequenz) definiert +12. WHILE die Synchronisation zwischen Monorepo und Team_Repo aktiv ist, THE Föderierte_Struktur SHALL sicherstellen, dass Dateien aus dem shared-Bereich, die für den jeweiligen Kontext relevant sind, optional als Read-Only-Kopie in das Team_Repo gespiegelt werden können diff --git a/.kiro/specs/monorepo-consolidation/tasks.md b/.kiro/specs/monorepo-consolidation/tasks.md new file mode 100644 index 0000000..9d38800 --- /dev/null +++ b/.kiro/specs/monorepo-consolidation/tasks.md @@ -0,0 +1,475 @@ +# Implementation Plan: Monorepo-Consolidation + +## Overview + +Inkrementelle Implementierung der Monorepo-Konsolidierung in Python. Die Umsetzung folgt einem Bottom-Up-Ansatz: Zunächst werden die Basis-Komponenten (Ordnerstruktur, Sicherheit, Verschlüsselung) erstellt, darauf aufbauend der Wissensspeicher und die Repo-Verwaltung, dann die Integrationsschicht (Orchestrator-Adapter, Kontextbrücke, Migration), und schließlich die Föderationsschicht (Team-Repos, Synchronisation). + +## Tasks + +- [x] 1. Projektstruktur und Basiskonfiguration + - [x] 1.1 Monorepo-Grundstruktur und zentrale Konfigurationsdateien erstellen + - Erstelle die Ordnerstruktur: `privat/`, `dhive/`, `bahn/`, `shared/` mit Unterordnern `shared/tools/`, `shared/powers/`, `shared/knowledge-store/`, `shared/config/`, `shared/mcp-servers/` + - Erstelle `monorepo.yaml` mit Kontext-Definitionen, Naming-Regeln, Security- und Encryption-Konfiguration + - Erstelle `shared/config/access-config.yaml` mit Zugriffskonfiguration pro Kontext + - Erstelle `shared/config/repos.yaml` als leere Repo-Registry + - Erstelle `shared/config/scopes.yaml` mit Scope-Mapping für den Wissensspeicher + - Erstelle `shared/config/machine-context.yaml` mit Maschinenkontext-Mapping + - Erstelle `shared/config/team-repos.yaml` mit Föderations-Konfiguration + - Erstelle `.gitignore` mit Patterns für Build-Artefakte, Caches, temporäre Dateien (NICHT für Secrets) + - Erstelle `.gitattributes` mit Root-Level git-crypt-Regeln + - _Requirements: 1.1, 1.4, 2.2, 2.4, 2.6, 8.1, 9.5, 9.9, 10.11_ + + + - [x] 1.2 Python-Paketstruktur und gemeinsame Datenmodelle anlegen + - Erstelle `shared/tools/monorepo-cli/` mit `pyproject.toml` und `src/monorepo/` Package + - Definiere Dataclasses: `ProjectInfo`, `RepoEntry`, `MigrationPlan`, `SecurityEvent`, `ScopeConfig`, `MachineContext`, `EncryptionKey`, `PasswordManagerConfig`, `TeamRepoEntry`, `SharedMirrorConfig`, `SyncResult`, `ConflictInfo`, `IsolationReport`, `IsolationLeak` + - Definiere Enums: `Context`, `RepoMode`, `ArtifactType`, `KeySource`, `SyncDirection`, `SyncFrequency`, `ConflictStrategy` + - Erstelle `src/monorepo/__init__.py`, `src/monorepo/models.py`, `src/monorepo/config.py` + - _Requirements: 1.1, 4.3, 9.5, 10.11_ + +- [ ] 2. Ordnerstruktur-Manager (StructureManager) + - [x] 2.1 StructureManager-Klasse implementieren + - Implementiere `StructureManager` in `src/monorepo/structure.py` + - Methode `validate_name(name: str) -> bool`: Prüfung auf kebab-case, 2-50 Zeichen, Regex `^[a-z0-9][a-z0-9\-]{0,48}[a-z0-9]$` + - Methode `create_project(context: str, name: str) -> Path`: Erstellt Projektordner, prüft Namenskonflikt, gibt Fehlermeldung bei Duplikat + - Methode `list_projects(context: str | None) -> list[ProjectInfo]`: Auflistung aller Projekte + - Methode `resolve_context(project_path: Path) -> str`: Kontexterkennung anhand Pfad + - _Requirements: 1.1, 1.2, 1.3, 1.5, 1.6_ + + - [-] 2.2 Property-Test: Namensvalidierung (Property 1) + - **Property 1: Namensvalidierung akzeptiert nur gültiges kebab-case** + - **Validates: Requirements 1.3** + + - [-] 2.3 Property-Test: Namenskollision (Property 2) + - **Property 2: Namenskollision verhindert doppelte Projekterstellung** + - **Validates: Requirements 1.6** + + - [-] 2.4 Unit-Tests für StructureManager + - Teste Erstellung in allen vier Kontexten + - Teste Fehlermeldung bei existierendem Projektnamen + - Teste Edge Cases: Minimum-/Maximum-Länge, Sonderzeichen, Unicode + - _Requirements: 1.1, 1.2, 1.3, 1.6_ + +- [ ] 3. Sicherheits-Guard (ContextGuard) + - [x] 3.1 ContextGuard-Klasse mit Zugriffskontrolle implementieren + - Implementiere `ContextGuard` in `src/monorepo/security.py` + - Methode `check_access(requesting_context: str, target_path: Path) -> bool`: Prüft Zugriffsberechtigung basierend auf access-config.yaml + - Methode `load_env(context: str) -> dict[str, str]`: Lädt nur die .env des eigenen Kontexts (entschlüsselt via SecretEncryptionManager) + - Shared-Zugriff: Nur auf explizit freigegebene Pfade gemäß Konfiguration + - _Requirements: 2.1, 2.3, 2.6, 2.7_ + + - [-] 3.2 AuditLogger und Violation-Protokollierung implementieren + - Implementiere `AuditLogger` in `src/monorepo/audit.py` + - Methode `log_violation(event: SecurityEvent) -> None`: Schreibt Zeitstempel, anfragender Kontext, Zielkontext, Ressource in Protokolldatei + - Konfiguration des Audit-Log-Pfads aus monorepo.yaml + - _Requirements: 2.5_ + + - [x] 3.3 Property-Test: Kontextübergreifender Secret-Zugriff (Property 3) + - **Property 3: Kontextübergreifender Secret-Zugriff wird verweigert** + - **Validates: Requirements 2.1, 2.3, 2.5, 2.7** + + - [x] 3.4 Property-Test: Shared-Tool-Isolation (Property 4) + - **Property 4: Shared-Tool-Isolation** + - **Validates: Requirements 2.7, 8.5** + + - [x] 3.5 Unit-Tests für ContextGuard und AuditLogger + - Teste Zugriff innerhalb des eigenen Kontexts (erlaubt) + - Teste Zugriff auf fremden Kontext (verweigert) + - Teste Zugriff auf shared-Bereich (erlaubt gemäß Konfiguration) + - Teste Audit-Log-Format und Vollständigkeit + - _Requirements: 2.1, 2.3, 2.5, 2.6, 2.7_ + +- [ ] 4. Secret-Encryption-Manager (SecretEncryptionManager) + - [-] 4.1 SecretEncryptionManager-Basisklasse mit git-crypt-Integration implementieren + - Implementiere `SecretEncryptionManager` in `src/monorepo/encryption.py` + - Methode `encrypt_file(file_path: Path, context: str) -> EncryptionResult`: Verschlüsselt Datei mit Kontext-Schlüssel + - Methode `decrypt_file(file_path: Path) -> DecryptionResult`: Entschlüsselt nur bei autorisiertem Maschinenkontext + - Methode `is_authorized(context: str) -> bool`: Prüft Maschinenkontext-Autorisierung + - Methode `setup_gitcrypt_filters(context: str) -> None`: Installiert git-crypt-Filter pro Kontext (.gitattributes) + - _Requirements: 9.1, 9.2, 9.3, 9.4, 9.8_ + + - [x] 4.2 Maschinenkontext-Verwaltung und Schlüssel-Management implementieren + - Implementiere `MachineContext`-Logik in `src/monorepo/encryption.py` + - Methode `get_context_key(context: str) -> Optional[EncryptionKey]`: Schlüsselabruf aus Keyring oder Passwort-Manager + - Methode `onboard_machine(machine_name, authorized_contexts) -> OnboardingResult`: Einrichtung neuer Maschinen mit autorisierten Schlüsseln + - Methode `resolve_merge(file_path, ours, theirs) -> bytes`: Merge-Konflikt-Auflösung auf verschlüsselter Ebene + - Lade Konfiguration aus `shared/config/machine-context.yaml` + - Passwort-Manager-Integration (Bitwarden, 1Password, KeePass) als optionaler Key-Source + - _Requirements: 9.4, 9.5, 9.6, 9.7, 9.10, 9.11_ + + - [x] 4.3 .gitattributes pro Kontext und git-crypt-Konfiguration erstellen + - Erstelle `privat/.gitattributes` mit git-crypt-privat-Filter-Regeln + - Erstelle `dhive/.gitattributes` mit git-crypt-dhive-Filter-Regeln + - Erstelle `bahn/.gitattributes` mit git-crypt-bahn-Filter-Regeln + - Definiere Secret-Patterns: .env, *.pem, *.key, *token*, *secret* + - Dokumentiere Onboarding-Prozess für neue Maschinen + - _Requirements: 9.1, 9.2, 9.6, 9.9_ + + - [x] 4.4 Property-Test: Verschlüsselung nur mit Kontext-Schlüssel (Property 23) + - **Property 23: Verschlüsselte Secrets können nur mit autorisiertem Kontextschlüssel entschlüsselt werden** + - **Validates: Requirements 9.3, 9.8** + + - [x] 4.5 Property-Test: Maschinenkontext-Beschränkung (Property 24) + - **Property 24: Maschinenkontext beschränkt Entschlüsselung auf autorisierte Kontexte** + - **Validates: Requirements 9.4, 9.5, 9.8, 9.10** + + - [x] 4.6 Unit-Tests für SecretEncryptionManager + - Teste Verschlüsselung/Entschlüsselung mit korrektem Schlüssel + - Teste Fehlschlag bei falschem Kontext-Schlüssel + - Teste Maschinenkontext-Autorisierungsprüfung + - Teste Fehlermeldung ohne Inhalt-Offenlegung + - Teste .gitattributes-Generierung pro Kontext + - Teste Merge-Auflösung für verschlüsselte Dateien + - _Requirements: 9.1, 9.3, 9.4, 9.8, 9.10, 9.11_ + +- [x] 5. Checkpoint - Basis-Sicherheits- und Verschlüsselungsschicht + - Ensure all tests pass, ask the user if questions arise. + +- [x] 6. Wissensspeicher – ETL-Pipeline und Indexierung + - [x] 6.1 KnowledgeStore-Basisklasse und YAML-Index implementieren + - Implementiere `KnowledgeStore` in `src/monorepo/knowledge/store.py` + - Implementiere `YAMLIndex` in `src/monorepo/knowledge/index.py` mit Methoden: `load()`, `update_entry()`, `search()`, `get_by_scope()` + - Index-Format gemäß Design: version, last_updated, artifacts-Liste mit id, title, type, tags, scope, summary, path, content_hash, links + - Progressive-Disclosure: Index liefert nur kompakte Einträge (kein vollständiger Inhalt) + - _Requirements: 3.13, 3.16, 3.18, 3.19_ + + - [x] 6.2 Wissensartefakt-Modell mit YAML-Frontmatter implementieren + - Implementiere `Artifact` und `ArtifactMetadata` in `src/monorepo/knowledge/artifact.py` + - YAML-Frontmatter-Parsing und -Generierung (type, title, tags, source_context, created, updated, shareable, links, content_hash) + - Content-Hash-Berechnung (SHA-256) + - Scope-basierte Ordnerstruktur: Artefakt-Pfad = `{scope}/{type}/{name}.md` + - _Requirements: 3.4, 3.6, 3.11, 3.17, 3.19_ + + - [x] 6.3 ETL-Pipeline-Grundstruktur und Markdown-Quellstrategie implementieren + - Implementiere `ETLPipeline` in `src/monorepo/knowledge/etl.py` + - Implementiere `MarkdownSource` als erste Quellstrategie in `src/monorepo/knowledge/sources/markdown.py` + - Methode `ingest(source, context) -> list[Artifact]`: Verarbeitet Quelle, erstellt Artefakte mit Metadaten + - Inkrementelle Verarbeitung: Content-Hash-Vergleich, nur geänderte Inhalte aktualisieren + - Fehlerresilienz: Bei Quellenfehler → Fehler protokollieren, erfolgreiche Artefakte beibehalten, Retry-Markierung + - Index-Update im selben Durchlauf + - _Requirements: 3.2, 3.6, 3.10, 3.12, 3.20, 3.21_ + + - [x] 6.4 Volltextsuche und Scope-basierte Filterung implementieren + - Implementiere `search(query, allowed_scopes) -> list[SearchResult]` in KnowledgeStore + - Volltextsuche über indexierte Artefakte mit Relevanz-Sortierung + - Scope-Filterung: Nur Ergebnisse aus autorisierten Scopes zurückgeben + - Existenz nicht-autorisierter Artefakte nicht offenlegen + - Timeout-Handling: Abbruch nach 5 Sekunden mit Teilergebnis-Warnung + - _Requirements: 3.3, 3.7, 3.9_ + + - [x] 6.5 Graph-Verknüpfungen zwischen Artefakten implementieren + - Methode `link_artifacts(source_id, target_id, relation) -> None` + - Bidirektionale Verknüpfung: Eintrag im YAML-Frontmatter beider Artefakte + - Aktualisierung des YAML-Index mit Link-Informationen + - _Requirements: 3.8_ + + - [x] 6.6 NoteGraph-Migration und Scope-Konfiguration implementieren + - Implementiere `NoteGraphMigrator` in `src/monorepo/knowledge/migration.py` + - Migration der NoteGraph-Verzeichnisstruktur (decisions, inbox, meetings, people, projects) + - Scope-Konfiguration: Mapping Arbeitskontext → Scope in scopes.yaml + - Validierung: Alle Artefakte, Metadaten und Verknüpfungen im Zielsystem vorhanden und auffindbar + - _Requirements: 3.5, 3.14_ + + - [x] 6.7 Property-Test: Artefakt-Ingestion erzeugt vollständige Metadaten (Property 5) + - **Property 5: Wissensartefakt-Ingestion erzeugt vollständige Metadaten im Index** + - **Validates: Requirements 3.2, 3.4, 3.6, 3.11, 3.13, 3.17, 3.21** + + - [x] 6.8 Property-Test: Suche respektiert Scope-Berechtigungen (Property 6) + - **Property 6: Suche respektiert Scope-Berechtigungen** + - **Validates: Requirements 3.3, 3.9** + + - [x] 6.9 Property-Test: Progressive-Disclosure-Index (Property 7) + - **Property 7: Progressive-Disclosure-Index enthält nur kompakte Einträge** + - **Validates: Requirements 3.16, 3.18** + + - [x] 6.10 Property-Test: Inkrementelle Verarbeitung (Property 8) + - **Property 8: Inkrementelle Verarbeitung erkennt Änderungen über Content-Hash** + - **Validates: Requirements 3.12** + + - [x] 6.11 Property-Test: Volltextsuche (Property 9) + - **Property 9: Volltextsuche findet indexierte Inhalte** + - **Validates: Requirements 3.7** + + - [x] 6.12 Property-Test: ETL-Fehlerresilienz (Property 10) + - **Property 10: ETL-Fehlerresilienz bewahrt erfolgreiche Artefakte** + - **Validates: Requirements 3.20** + + - [x] 6.13 Property-Test: Graph-Verknüpfungen bidirektional (Property 11) + - **Property 11: Graph-Verknüpfungen werden in beiden Artefakten reflektiert** + - **Validates: Requirements 3.8** + +- [x] 7. Checkpoint - Wissensspeicher-Kern + - Ensure all tests pass, ask the user if questions arise. + +- [x] 8. Externes-Repo-Manager (RepoManager) + - [x] 8.1 RepoManager-Klasse mit Subtree/Submodule-Einbindung implementieren + - Implementiere `RepoManager` in `src/monorepo/repos.py` + - Methode `add_repo(entry: RepoEntry) -> None`: Bindet Repo via Git-Subtree (Standard) oder Submodule ein + - Methode `sync(repo_name: str) -> SyncResult`: Aktualisiert auf gepinnte Version, gibt Ergebnismeldung (Erfolg/Misserfolg/unbekannt) + - Bidirektionale Sync für Upstream-Repos (Pull/Push), Abbruch bei Merge-Konflikten + - _Requirements: 4.1, 4.3, 4.4, 4.6, 4.7, 4.8_ + + - [x] 8.2 Read-Only-Schutzmechanismus implementieren + - Methode `protect_readonly(repo_path: Path) -> None`: Installiert Git pre-commit Hook + - Hook verhindert Commits die Dateien im geschützten Verzeichnis ändern + - Fehlermeldung bei Schreibversuch mit Repo-Name und Read-Only-Status + - Fehlerprotokollierung bei fehlgeschlagener Synchronisation (Netzwerk, Auth, Merge) + - _Requirements: 4.2, 4.5, 4.9_ + + - [x] 8.3 Property-Test: Read-Only-Repos blockieren Schreibzugriffe (Property 12) + - **Property 12: Read-Only-Repos blockieren Schreibzugriffe** + - **Validates: Requirements 4.2, 4.5** + + - [x] 8.4 Property-Test: Fehlgeschlagene Synchronisation (Property 13) + - **Property 13: Fehlgeschlagene Synchronisation bewahrt lokalen Stand** + - **Validates: Requirements 4.9** + + - [x] 8.5 Unit-Tests für RepoManager + - Teste Subtree-Add und Submodule-Add + - Teste Sync-Ergebnismeldungen (Erfolg, Fehler, unbekannt) + - Teste Read-Only-Hook-Blockierung + - Teste Fehlerprotokollierung bei Sync-Fehlschlägen + - _Requirements: 4.1, 4.2, 4.4, 4.5, 4.9_ + +- [x] 9. Kontextbrücke (ContextBridge) + - [x] 9.1 ContextBridge-Klasse mit Sensitive-Content-Filter implementieren + - Implementiere `ContextBridge` in `src/monorepo/bridge.py` + - Methode `share_artifact(artifact_id, user_confirmed) -> ShareResult`: Freigabe nur mit expliziter Nutzerbestätigung + - Methode `check_sensitive_content(content: str) -> list[SensitiveMatch]`: Regex-basierte Prüfung auf Secrets, Endpoints, PII + - Methode `revoke_share(artifact_id) -> None`: Entfernt Artefakt aus allen Zielkontexten als Lesereferenz + - Update-Propagierung: Änderungen im Quellkontext beim nächsten Lesezugriff sichtbar + - _Requirements: 5.1, 5.2, 5.3, 5.4, 5.5, 5.6_ + + - [x] 9.2 Property-Test: Sensitive-Content-Filter (Property 14) + - **Property 14: Sensitive-Content-Filter blockiert Freigabe** + - **Validates: Requirements 5.1, 5.3** + + - [x] 9.3 Property-Test: Freigabe-Lebenszyklus (Property 15) + - **Property 15: Freigabe-Lebenszyklus (Share → Update → Revoke)** + - **Validates: Requirements 5.2, 5.5, 5.6** + + - [x] 9.4 Property-Test: Explizite Nutzerbestätigung (Property 16) + - **Property 16: Freigabe erfordert explizite Nutzerbestätigung** + - **Validates: Requirements 5.4** + +- [x] 10. Checkpoint - Repo-Manager und Kontextbrücke + - Ensure all tests pass, ask the user if questions arise. + +- [ ] 11. Migrations-Engine + - [x] 11.1 MigrationEngine-Klasse mit Git-Historie-Bewahrung implementieren + - Implementiere `MigrationEngine` in `src/monorepo/migration.py` + - Methode `migrate(plan: MigrationPlan) -> MigrationResult`: Führt Migration via `git filter-repo` oder `git subtree add` durch, bewahrt alle Commits, Branches, Tags, Autoreninformationen + - Inkrementelle Migration: Repositories einzeln und unabhängig migrierbar + - Konflikterkennung: Pfadkollision, Branch-Namenskonflikt, Namenskonvention-Verletzung → Migration pausieren + - _Requirements: 6.1, 6.3, 6.5, 6.6_ + + - [-] 11.2 Migrations-Validierung und Rollback implementieren + - Methode `validate(repo_name: str) -> ValidationResult`: Prüft Commit-Anzahl, Branches, Tags, Dateibaum-Vollständigkeit, Test-Durchlauf + - Methode `rollback(repo_name: str) -> None`: Stellt Vor-Migrations-Zustand wieder her, ohne andere Repos zu beeinflussen + - Dokumentierter Migrationsplan: Für jedes Repo Ziel-Kontext, Reihenfolge, Abhängigkeiten, Einbindungsmodus + - _Requirements: 6.1, 6.4, 6.7_ + + - [-] 11.3 Property-Test: Migrations-Validierung (Property 17) + - **Property 17: Migrations-Validierung prüft Vollständigkeit** + - **Validates: Requirements 6.4** + + - [-] 11.4 Property-Test: Migrations-Rollback (Property 18) + - **Property 18: Migrations-Rollback stellt Vor-Zustand wieder her** + - **Validates: Requirements 6.7** + + - [x] 11.5 Unit-Tests für MigrationEngine + - Teste Migration eines einfachen Repos (Commits, Branches, Tags bewahrt) + - Teste Konflikterkennung (Pfadkollision, Branch-Namenskonflikt) + - Teste Rollback-Wiederherstellung + - Teste inkrementelle Migration (parallel funktionsfähig) + - _Requirements: 6.1, 6.4, 6.5, 6.6, 6.7_ + +- [ ] 12. Orchestrator-Adapter + - [x] 12.1 OrchestratorAdapter-Klasse mit Kontextauflösung implementieren + - Implementiere `OrchestratorAdapter` in `src/monorepo/orchestrator.py` + - Methode `resolve_context(task: Task) -> str`: Ermittelt Arbeitskontext aus Task-Metadaten (Labels/Projektzuordnung) + - Methode `create_workspace(task, context) -> Path`: Erstellt Arbeitsverzeichnis unter dem korrekten Kontextordner + - Methode `build_prompt(task, context) -> str`: Erstellt Agenten-Prompt mit Wissenskontext + - Fehlerbehandlung: Kein Kontext → Task nicht starten, Fehlermeldung; Out-of-Context-Zugriff → Task abbrechen, protokollieren, Nutzer benachrichtigen + - _Requirements: 7.1, 7.2, 7.3, 7.5, 7.6_ + + - [-] 12.2 Wissensinjection und Env-Loading im Orchestrator implementieren + - Methode `inject_knowledge(prompt, context) -> str`: Abfrage des YAML-Index, relevante Artefakt-Pfade einfügen + - Env-Loading: Ausschließlich .env des zugewiesenen Kontexts laden (entschlüsselt via SecretEncryptionManager), keine Umgebungsvariablen anderer Kontexte + - Integration mit bestehender WORKFLOW.md-Konfiguration: workspace_root pro Task + - _Requirements: 7.3, 7.4_ + + - [-] 12.3 Multi-Harness-Dispatch implementieren + - Implementiere Harness-Konfiguration: Pro Kontext/Task-Typ den Agent_Harness definieren + - Einheitliche Schnittstelle: Prompt-Injection, Workspace-Verzeichnis, Env-Übergabe, Agent_Erweiterungen + - Harness-spezifische Startsequenz (CLI, API, Prozessstart) aus Konfiguration laden + - Fehlerbehandlung: Harness nicht verfügbar → Task als nicht-ausführbar markieren, Nutzer benachrichtigen + - _Requirements: 7.7, 7.8, 7.9, 7.10_ + + - [x] 12.4 Property-Test: Orchestrator-Kontextauflösung (Property 19) + - **Property 19: Orchestrator-Kontextauflösung und Workspace-Isolation** + - **Validates: Requirements 7.1, 7.2, 7.3, 7.6** + + - [x] 12.5 Property-Test: Orchestrator-Kontextverletzung (Property 20) + - **Property 20: Orchestrator-Kontextverletzung bricht Task ab** + - **Validates: Requirements 7.5** + + - [x] 12.6 Unit-Tests für OrchestratorAdapter + - Teste Kontextauflösung aus Task-Labels + - Teste Workspace-Erstellung im korrekten Kontextordner + - Teste Env-Isolation (nur eigene .env geladen, via Encryption entschlüsselt) + - Teste Wissensinjektion in Prompt + - Teste Task-Abbruch bei Out-of-Context-Zugriff + - Teste Multi-Harness-Dispatch (Konfiguration, Startsequenz, Fehlerbehandlung) + - _Requirements: 7.1, 7.2, 7.3, 7.4, 7.5, 7.6, 7.7, 7.9, 7.10_ + +- [x] 13. Shared Tooling und MCP-Konfiguration + - [x] 13.1 MCP-Konfiguration-Merge und Shared-Tool-Versionierung implementieren + - Implementiere `ConfigMerger` in `src/monorepo/shared_config.py` + - Methode für MCP-Server-Konfiguration-Merge: Shared als Basis, kontextspezifische Überschreibungen mit Vorrang + - Konflikt-Protokollierung bei Überschreibungskonflikten + - Shared-Tool-Versionierung: Symlink- oder Import-basierter Mechanismus, sodass Kontexte automatisch die aktuelle Version verwenden + - Kiro Powers und Skills zentral im shared-Bereich ablegen, Read-Only-Referenz für alle Kontexte + - Agent-Erweiterungen in harness-unabhängigem Format (Markdown, YAML, Templates) mit Adapter-Mechanismus + - _Requirements: 8.2, 8.3, 8.4, 8.5, 8.6, 8.7, 8.8, 8.9_ + + - [x] 13.2 Property-Test: MCP-Konfiguration-Merge (Property 21) + - **Property 21: MCP-Konfiguration-Merge mit Kontext-Vorrang** + - **Validates: Requirements 8.4, 8.6** + + - [x] 13.3 Property-Test: Shared-Tool-Versionierung (Property 22) + - **Property 22: Shared-Tool-Versionierung ohne manuelle Synchronisation** + - **Validates: Requirements 8.2** + +- [x] 14. Checkpoint - Orchestrator, Shared Tooling und Migration + - Ensure all tests pass, ask the user if questions arise. + +- [x] 15. Federation-Manager (FederationManager) + - [x] 15.1 FederationManager-Basisklasse und SubtreeSyncEngine implementieren + - Implementiere `FederationManager` in `src/monorepo/federation.py` + - Implementiere `SubtreeSyncEngine` in `src/monorepo/federation.py` + - Methode `sync_from_team(context) -> SyncResult`: Pull vom Team_Repo via git subtree pull + - Methode `sync_to_team(context) -> SyncResult`: Push zum Team_Repo via git subtree push + - Methode `full_sync(context) -> SyncResult`: Bidirektionale Synchronisation mit Konflikt-Erkennung + - Methode `detect_conflicts(context) -> list[ConflictInfo]`: Erkennt Merge-Konflikte vor Sync + - Lade Konfiguration aus `shared/config/team-repos.yaml` + - _Requirements: 10.1, 10.2, 10.3, 10.4, 10.8_ + + - [x] 15.2 Team-Isolation und Cross-Context-Leakage-Prüfung implementieren + - Methode `verify_isolation(team_repo_path, context) -> IsolationReport`: Prüft ob Team_Repo keine Referenzen auf andere Kontexte enthält + - Prüfung auf: Pfad-Referenzen, Env-Variablen, Config-Referenzen, Kommentare zu anderen Kontexten + - Methode `prepare_team_repo(context) -> Path`: Erzeugt Team_Repo mit nur eigenem Kontext + - Secrets-Handling: Team_Repo enthält nur Secrets des eigenen Kontexts (entschlüsselt oder re-keyed mit Team-Schlüssel) + - Integration mit SecretEncryptionManager für kontextspezifische Secret-Filterung + - _Requirements: 10.5, 10.9, 10.10_ + + - [x] 15.3 Shared-Bereich-Spiegelung und Konflikt-Auflösung implementieren + - Methode `mirror_shared(context, paths) -> MirrorResult`: Spiegelt shared-Dateien als Read-Only in Team_Repo + - Methode `resolve_conflict(context, strategy) -> ConflictResult`: Löst Sync-Konflikte auf (Standard: team-wins) + - Methode `onboard_member(context, member_info) -> OnboardingResult`: Erteilt Zugang nur zum Team_Repo + - Single-Source-of-Truth-Logik: Team_Repo hat bei Konflikten Vorrang + - Konfigurierbare Sync-Frequenz: on-push, hourly, daily, manual + - _Requirements: 10.6, 10.7, 10.9, 10.11, 10.12_ + + - [x] 15.4 Property-Test: Team-Repos enthalten nur eigenen Kontext (Property 25) + - **Property 25: Team-Repos enthalten ausschließlich Inhalte des eigenen Kontexts** + - **Validates: Requirements 10.5, 10.9, 10.10, 10.12** + + - [x] 15.5 Property-Test: Bidirektionale Synchronisation (Property 26) + - **Property 26: Bidirektionale Synchronisation bewahrt Git-Historie und löst Konflikte korrekt** + - **Validates: Requirements 10.3, 10.4, 10.6, 10.7, 10.8** + + - [x] 15.6 Property-Test: Team-Mitglieder-Isolation (Property 27) + - **Property 27: Team-Mitglieder können die Existenz anderer Kontexte nicht entdecken** + - **Validates: Requirements 10.5, 10.9** + + - [x] 15.7 Unit-Tests für FederationManager + - Teste Subtree-Pull und -Push Operationen + - Teste Konflikt-Erkennung und -Auflösung (team-wins-Strategie) + - Teste Isolation-Check (Leakage-Erkennung für Pfade, Env-Vars, Config-Refs) + - Teste Shared-Mirror als Read-Only + - Teste Onboarding ohne Monorepo-Kenntnis + - Teste Sync-Abbruch bei Merge-Konflikten mit Protokollierung + - _Requirements: 10.1, 10.3, 10.5, 10.6, 10.7, 10.9, 10.12_ + +- [x] 16. Checkpoint - Föderationsschicht + - Ensure all tests pass, ask the user if questions arise. + +- [x] 17. Integration und Verdrahtung + - [x] 17.1 CLI-Tool (`ctx-guard`) als Wrapper-Script erstellen + - Erstelle CLI-Einstiegspunkt in `shared/tools/monorepo-cli/src/monorepo/cli.py` + - Kommandos: `init` (Monorepo initialisieren), `create-project`, `list-projects`, `add-repo`, `sync-repo`, `migrate`, `validate`, `search`, `encrypt`, `decrypt`, `onboard`, `fed-sync`, `fed-status` + - Verdrahte alle Komponenten: StructureManager, ContextGuard, KnowledgeStore, RepoManager, ContextBridge, MigrationEngine, OrchestratorAdapter, SecretEncryptionManager, FederationManager + - Konfiguration aus `monorepo.yaml` und `shared/config/` laden + - _Requirements: 1.1, 1.2, 4.4, 6.3, 9.6, 10.11_ + + - [x] 17.2 Git-Hooks und Dateisystem-Schutz integrieren + - Pre-commit Hook: Prüft Read-Only-Repos nicht verändert werden, unverschlüsselte Secrets nicht committet werden + - git-crypt-Filter-Integration: Sicherstellt dass Secrets automatisch verschlüsselt/entschlüsselt werden + - Integriere ContextGuard und SecretEncryptionManager in Hook-Chain + - Installationsskript für Hooks bei `init` + - _Requirements: 2.4, 4.2, 4.5, 9.1, 9.9_ + + - [x] 17.3 Encryption-Security-Integration verdrahten + - Integriere SecretEncryptionManager mit ContextGuard: load_env nutzt Entschlüsselung + - Integriere SecretEncryptionManager mit FederationManager: Team_Repos erhalten nur eigene Secrets + - Integriere SecretEncryptionManager mit OrchestratorAdapter: Env-Loading via Encryption + - _Requirements: 2.2, 9.3, 9.4, 10.10_ + + - [x] 17.4 Integration-Tests für End-to-End-Workflows + - Teste: Projekt erstellen → Env verschlüsseln → Env laden → Secret-Zugriff prüfen + - Teste: Artefakt erstellen → Indexieren → Suchen → Freigeben → Widerrufen + - Teste: Repo einbinden → Sync → Read-Only-Schutz + - Teste: Migration → Validierung → Rollback + - Teste: Secret verschlüsseln → Maschinenkontext wechseln → Entschlüsselung fehlschlägt + - Teste: Team_Repo vorbereiten → Isolation prüfen → Sync → Konflikt-Auflösung + - Teste: Shared-Mirror in Team_Repo → Read-Only-Prüfung + - _Requirements: 1.1, 2.1, 3.2, 4.1, 5.2, 6.1, 9.1, 9.4, 10.3, 10.5, 10.12_ + +- [x] 18. Final Checkpoint - Alle Komponenten integriert + - Ensure all tests pass, ask the user if questions arise. + +## Notes + +- Tasks marked with `*` are optional and can be skipped for faster MVP +- Each task references specific requirements for traceability +- Checkpoints ensure incremental validation +- Property tests validate universal correctness properties from the design document (P1-P27) +- Unit tests validate specific examples and edge cases +- Die ETL-Pipeline wird initial nur mit der Markdown-Quellstrategie implementiert; weitere Strategien (Confluence, PDF, Webseiten, GitLab) können später ergänzt werden +- Die DB-Wissensdatenbank-Integration (Requirement 3.1, 3.15, 4.8) wird über den RepoManager als Upstream-Repo eingebunden; die ETL-Pipeline-Fork-Adaption erfolgt im Wissensspeicher +- Git-Remote-Operationen (Push/Pull/Sync) werden in Integration-Tests mit gemockten Repos getestet +- git-crypt-Operationen (encrypt/decrypt) werden in Integration-Tests mit temporären Repos und Test-Schlüsseln getestet +- Team-Repo-Synchronisation wird mit lokalen Bare-Repos als Test-Remotes getestet +- Die Föderationsschicht (Task 15) baut auf dem SecretEncryptionManager (Task 4) auf und setzt diesen als Abhängigkeit voraus +- Passwort-Manager-Integration (Bitwarden, 1Password, KeePass) ist optional und kann durch lokalen Keyring ersetzt werden + +## Task Dependency Graph + +```json +{ + "waves": [ + { "id": 0, "tasks": ["1.1"] }, + { "id": 1, "tasks": ["1.2"] }, + { "id": 2, "tasks": ["2.1", "3.1"] }, + { "id": 3, "tasks": ["2.2", "2.3", "2.4", "3.2", "4.1"] }, + { "id": 4, "tasks": ["3.3", "3.4", "3.5", "4.2"] }, + { "id": 5, "tasks": ["4.3", "4.4", "4.5", "4.6"] }, + { "id": 6, "tasks": ["6.1", "6.2"] }, + { "id": 7, "tasks": ["6.3", "6.4", "6.5"] }, + { "id": 8, "tasks": ["6.6", "6.7", "6.8", "6.9", "6.10"] }, + { "id": 9, "tasks": ["6.11", "6.12", "6.13"] }, + { "id": 10, "tasks": ["8.1", "9.1"] }, + { "id": 11, "tasks": ["8.2", "8.3", "8.4", "8.5", "9.2", "9.3", "9.4"] }, + { "id": 12, "tasks": ["11.1", "12.1"] }, + { "id": 13, "tasks": ["11.2", "11.3", "11.4", "11.5", "12.2", "12.3", "13.1"] }, + { "id": 14, "tasks": ["12.4", "12.5", "12.6", "13.2", "13.3"] }, + { "id": 15, "tasks": ["15.1"] }, + { "id": 16, "tasks": ["15.2", "15.3"] }, + { "id": 17, "tasks": ["15.4", "15.5", "15.6", "15.7"] }, + { "id": 18, "tasks": ["17.1"] }, + { "id": 19, "tasks": ["17.2", "17.3"] }, + { "id": 20, "tasks": ["17.4"] } + ] +} +``` diff --git a/.kiro/specs/notegraph-ingestion/.config.kiro b/.kiro/specs/notegraph-ingestion/.config.kiro new file mode 100644 index 0000000..5351837 --- /dev/null +++ b/.kiro/specs/notegraph-ingestion/.config.kiro @@ -0,0 +1 @@ +{"specId": "e33fa6e5-89e0-4d8d-ad9e-d43d6bf23daa", "workflowType": "requirements-first", "specType": "feature"} diff --git a/.kiro/specs/notegraph-ingestion/design.md b/.kiro/specs/notegraph-ingestion/design.md new file mode 100644 index 0000000..ea5f6fb --- /dev/null +++ b/.kiro/specs/notegraph-ingestion/design.md @@ -0,0 +1,542 @@ +# Design Document: NoteGraph Ingestion & Auto-Reference + +## Overview + +The NoteGraph Ingestion service is a Python package (`NoteGraph/ingestion/`) that provides a CLI-driven pipeline for importing documents (PDFs, images, markdown, text, DOCX) into the NoteGraph knowledge base. The pipeline extracts text (via OCR or direct extraction), enriches it using an LLM to detect entities (people, projects, dates, action items), generates wiki-links, and outputs structured markdown files with YAML frontmatter into the appropriate NoteGraph folder. + +The system is designed around three principles: +1. **Model-agnostic** — A unified LLM interface (via LiteLLM) supports OpenAI, Anthropic, Google, Mistral, and local Ollama without code changes. +2. **CLI-first** — All operations are accessible via `python -m ingestion.cli` with subcommands for bulk import and continuous watching. +3. **Markdown-native** — Output is always SilverBullet-compatible markdown with YAML frontmatter and wiki-links. + +## Architecture + +```mermaid +graph TD + subgraph Input + CLI[CLI Interface] + WATCH[Drop-Folder Watcher] + end + + subgraph Pipeline + DISC[File Discovery] + EXT[Text Extraction] + OCR[OCR Engine] + ENR[Enrichment Agent] + LINK[Reference Linker] + RENDER[Markdown Renderer] + end + + subgraph Output + MD[Markdown Files] + STUB[Stub Pages] + GIT[Git Commit] + TASK[OrgMyLife Tasks] + end + + subgraph External + LLM[LLM Provider] + ORG[OrgMyLife API] + end + + CLI --> DISC + WATCH --> DISC + DISC --> EXT + EXT --> OCR + EXT --> ENR + ENR --> LLM + ENR --> LINK + LINK --> RENDER + RENDER --> MD + RENDER --> STUB + MD --> GIT + STUB --> GIT + ENR --> TASK + TASK --> ORG +``` + +### Data Flow + +``` +Input File → File Discovery → Text Extraction (OCR if needed) + → Enrichment Agent (LLM call) → Entity Detection + → Reference Linker (wiki-links) → Markdown Renderer (frontmatter + body) + → File Placement (folder routing) → Git Commit + → (optional) OrgMyLife Task Creation +``` + +## Components and Interfaces + +### Package Structure + +``` +NoteGraph/ingestion/ +├── __init__.py +├── __main__.py # python -m ingestion entry point +├── cli.py # Click-based CLI (import, watch subcommands) +├── config.py # Configuration from .env +├── pipeline.py # Main orchestration pipeline +├── discovery.py # File discovery and filtering +├── extraction/ +│ ├── __init__.py +│ ├── base.py # Extractor protocol +│ ├── text.py # Plain text / markdown extractor +│ ├── pdf.py # PDF text extraction (PyPDF2/pdfplumber) +│ ├── ocr.py # OCR via Tesseract (pytesseract) +│ └── docx.py # DOCX extraction (python-docx) +├── enrichment/ +│ ├── __init__.py +│ ├── agent.py # Enrichment orchestration +│ ├── prompts.py # Structured prompts for entity detection +│ └── models.py # Pydantic models for enrichment response +├── linking/ +│ ├── __init__.py +│ ├── linker.py # Reference linker (wiki-link generation) +│ └── stubs.py # Stub page creation +├── output/ +│ ├── __init__.py +│ ├── renderer.py # Markdown + frontmatter rendering +│ ├── router.py # Folder routing logic +│ └── naming.py # Filename generation with collision avoidance +├── integrations/ +│ ├── __init__.py +│ ├── llm.py # LiteLLM wrapper (model-agnostic) +│ ├── orgmylife.py # OrgMyLife API client +│ └── git.py # Git commit operations +├── watcher.py # Drop-folder watcher (watchdog) +└── tests/ + ├── __init__.py + ├── conftest.py + ├── test_discovery.py + ├── test_extraction.py + ├── test_enrichment.py + ├── test_linking.py + ├── test_output.py + ├── test_pipeline.py + └── test_properties.py # Property-based tests +``` + +### Core Interfaces + +```python +# ingestion/extraction/base.py +from typing import Protocol + +class TextExtractor(Protocol): + """Protocol for all text extractors.""" + + def can_handle(self, file_path: Path) -> bool: + """Return True if this extractor handles the given file type.""" + ... + + def extract(self, file_path: Path) -> ExtractionResult: + """Extract text content from the file.""" + ... + +@dataclass +class ExtractionResult: + text: str + source_file: Path + extraction_method: str # "direct", "ocr", "mixed" + metadata: dict[str, Any] = field(default_factory=dict) +``` + +```python +# ingestion/enrichment/models.py +from pydantic import BaseModel, Field + +class Entity(BaseModel): + type: Literal["person", "project", "date", "action_item"] + value: str + confidence: float = Field(ge=0.0, le=1.0) + position: int | None = None # character offset in source text + +class ActionItem(BaseModel): + description: str + assignee: str | None = None + deadline: str | None = None # ISO 8601 + +class EnrichmentResult(BaseModel): + title: str + category: Literal["meeting", "project", "decision", "inbox"] + tags: list[str] + entities: list[Entity] + action_items: list[ActionItem] + summary: str | None = None +``` + +```python +# ingestion/integrations/llm.py +from litellm import completion + +class LLMClient: + """Model-agnostic LLM client using LiteLLM.""" + + def __init__(self, config: IngestionConfig): + self.model = config.llm_model_string # e.g. "openai/gpt-4o" + + def complete(self, messages: list[dict], response_format: type[BaseModel]) -> BaseModel: + """Send a structured completion request.""" + ... +``` + +```python +# ingestion/integrations/orgmylife.py +import httpx + +class OrgMyLifeClient: + """HTTP client for OrgMyLife task creation.""" + + def __init__(self, base_url: str, api_key: str): + self.base_url = base_url + self.api_key = api_key + + def create_task(self, title: str, source_url: str, + description: str = "", priority: int = 4) -> dict: + """Create a task via POST /api/tasks.""" + ... +``` + +### Configuration + +```python +# ingestion/config.py +from pydantic_settings import BaseSettings + +class IngestionConfig(BaseSettings): + # LLM Provider + llm_provider: str = "openai" # openai, anthropic, google, mistral, ollama + llm_model: str = "gpt-4o" + openai_api_key: str = "" + anthropic_api_key: str = "" + google_api_key: str = "" + mistral_api_key: str = "" + ollama_base_url: str = "http://localhost:11434" + + # OrgMyLife Integration + orgmylife_api_url: str = "https://api.andreknie.de" + orgmylife_api_key: str = "" + + # Paths + notes_dir: str = "./notes" + inbox_dir: str = "./ingestion/inbox" + + # OCR + ocr_enabled: bool = True + ocr_language: str = "deu+eng" # German + English + + # Behavior + confidence_threshold: float = 0.7 + auto_commit: bool = True + + class Config: + env_file = ".env" + env_prefix = "" + + @property + def llm_model_string(self) -> str: + """LiteLLM model string (e.g., 'openai/gpt-4o').""" + if self.llm_provider == "ollama": + return f"ollama/{self.llm_model}" + return f"{self.llm_provider}/{self.llm_model}" + + def validate_api_key(self) -> None: + """Raise if required API key is missing.""" + ... +``` + +## Data Models + +### Enrichment Prompt (Structured Output) + +The LLM receives extracted text and a system prompt requesting structured JSON output: + +```python +ENRICHMENT_SYSTEM_PROMPT = """You are a knowledge management assistant. Analyze the following text +extracted from a document and return structured metadata. + +Return a JSON object with: +- title: A concise, descriptive title for this note (max 80 chars) +- category: One of "meeting", "project", "decision", "inbox" +- tags: List of relevant topic tags (lowercase, no #) +- entities: List of detected entities, each with: + - type: "person", "project", "date", or "action_item" + - value: The extracted text + - confidence: Float 0.0-1.0 + - position: Character offset where entity appears (approximate) +- action_items: List of action items, each with: + - description: What needs to be done + - assignee: Person responsible (if mentioned) + - deadline: Date in ISO 8601 format (if mentioned) +- summary: Optional 1-2 sentence summary + +Context: This is a personal knowledge base for a project manager working across +multiple business projects. Common projects include: {known_projects}. +Known people: {known_people}. +""" +``` + +### Output Markdown Format + +```yaml +--- +title: "Meeting with Beier GmbH about Druckluft project" +date: 2024-03-15 +tags: + - meeting + - druckluft + - customer +people: + - André Knierim + - Thomas Beier +projects: + - Beier GmbH + - Druckluft +source: + file: "Beier GmbH.pdf" + imported: "2024-03-20T14:30:00" +--- + +# Meeting with Beier GmbH about Druckluft project + +Discussion about the [[projects/druckluft]] project with [[people/thomas-beier]]. + +Key points: +- Delivery timeline confirmed for Q2 +- Budget approved at €45,000 +- Next review meeting scheduled for April 10 + +## Action Items + +- [ ] Send updated proposal to Thomas Beier (deadline: 2024-03-22) +- [ ] Schedule follow-up meeting for April 10 +- [ ] Update project timeline in planning tool +``` + +### Entity-to-WikiLink Mapping + +| Entity Type | Wiki-Link Format | Example | +|-------------|-----------------|---------| +| Person | `[[people/firstname-lastname]]` | `[[people/thomas-beier]]` | +| Project | `[[projects/project-slug]]` | `[[projects/druckluft]]` | +| Date | Frontmatter `date` field | `2024-03-15` | +| Action Item | `## Action Items` section | `- [ ] Send proposal` | + +### File Routing Rules + +| Category | Target Folder | Condition | +|----------|--------------|-----------| +| Meeting | `notes/meetings/` | Category = "meeting" | +| Project | `notes/projects/` | Category = "project" with single dominant project | +| Decision | `notes/decisions/` | Category = "decision" | +| Default | `notes/inbox/` | No clear category or mixed | + +## Correctness Properties + +*A property is a characteristic or behavior that should hold true across all valid executions of a system — essentially, a formal statement about what the system should do. Properties serve as the bridge between human-readable specifications and machine-verifiable correctness guarantees.* + +### Property 1: File discovery returns only supported extensions + +*For any* directory tree containing files with arbitrary extensions, the discovery function SHALL return exactly and only those files whose extensions are in the supported set (`.pdf`, `.jpg`, `.jpeg`, `.png`, `.md`, `.txt`, `.docx`). + +**Validates: Requirements 1.1, 1.6** + +### Property 2: Bulk import resilience — failed files do not block remaining processing + +*For any* batch of N files where a subset F fails during processing, the pipeline SHALL successfully process all N-F non-failing files and the summary SHALL report success count = N-F and failed count = |F|. + +**Validates: Requirements 1.4, 1.5** + +### Property 3: Text extraction preserves content for text-based formats + +*For any* valid UTF-8 string written to a markdown or plain text file, extracting it via the Text_Extractor SHALL return the identical string content. + +**Validates: Requirements 3.1, 3.2, 3.4** + +### Property 4: Entity parsing extracts all entities with valid structure + +*For any* valid enrichment response JSON containing entities, each parsed entity SHALL have a non-empty `type` (one of person/project/date/action_item), a non-empty `value`, and a `confidence` score in the range [0.0, 1.0]. + +**Validates: Requirements 4.1, 4.2, 4.3, 4.4, 4.5** + +### Property 5: Confidence threshold filtering + +*For any* list of entities with arbitrary confidence scores, the filtered output SHALL contain exactly those entities with confidence >= 0.7 and no others. + +**Validates: Requirements 4.6** + +### Property 6: Wiki-link generation follows correct format per entity type + +*For any* person name, the generated wiki-link SHALL match the pattern `[[people/slugified-name]]`. *For any* project name, the generated wiki-link SHALL match the pattern `[[projects/slugified-name]]`. + +**Validates: Requirements 5.1, 5.2** + +### Property 7: Only first entity occurrence is linked + +*For any* text containing N >= 2 occurrences of the same entity, the Reference_Linker SHALL convert exactly the first occurrence to a wiki-link and leave all subsequent occurrences as plain text. + +**Validates: Requirements 5.6** + +### Property 8: Frontmatter round-trip validity + +*For any* valid `EnrichmentResult`, rendering it to markdown with YAML frontmatter and then parsing the frontmatter back SHALL produce a valid YAML block containing all required fields (title, date, tags, people, projects, source). + +**Validates: Requirements 6.1, 6.3, 6.4, 6.6, 6.7** + +### Property 9: Provider routing from configuration + +*For any* valid provider name from the supported set (openai, anthropic, google, mistral, ollama), configuring `LLM_PROVIDER` to that value SHALL cause the LLM client to route requests to the corresponding provider. + +**Validates: Requirements 7.3, 8.1** + +### Property 10: Missing API key produces descriptive error + +*For any* provider that requires an API key, when the corresponding environment variable is unset, the configuration validation SHALL raise an error that names the missing variable. + +**Validates: Requirements 7.4, 8.6** + +### Property 11: Action items appear in output markdown section + +*For any* list of extracted action items, the rendered markdown SHALL contain a `## Action Items` section listing every action item description. + +**Validates: Requirements 9.2** + +### Property 12: OrgMyLife task payload correctness + +*For any* action item description and note file path, the task creation payload SHALL set `title` to the action item description and `source_url` to a URL referencing the note path. + +**Validates: Requirements 9.4** + +### Property 13: Category-based folder routing + +*For any* enrichment result with category "meeting", the output path SHALL be within `notes/meetings/`. *For any* result with category "project", the path SHALL be within `notes/projects/`. *For any* result with no clear category, the path SHALL be within `notes/inbox/`. + +**Validates: Requirements 10.1, 10.2, 10.3** + +### Property 14: Output filename follows date-slug pattern + +*For any* date and title string, the generated filename SHALL match the pattern `YYYY-MM-DD-slugified-title.md` where the slug contains only lowercase alphanumeric characters and hyphens. + +**Validates: Requirements 10.5** + +### Property 15: Filename collision avoidance + +*For any* set of existing files, when a new file would have the same name as an existing file, the system SHALL append a numeric suffix (`-2`, `-3`, etc.) producing a unique filename. + +**Validates: Requirements 10.6** + +### Property 16: Temporary file filtering + +*For any* filename, the watcher ignore logic SHALL return true if and only if the filename starts with `.` or ends with `.tmp`. + +**Validates: Requirements 11.5** + +### Property 17: Dry-run produces no file writes + +*For any* input processed with the `--dry-run` flag, the system SHALL create zero files on disk. + +**Validates: Requirements 13.6** + +### Property 18: Git commit message format + +*For any* count N and source type string, the generated commit message SHALL match the format `ingestion: import N notes from [source-type]`. + +**Validates: Requirements 12.2** + +## Error Handling + +### Error Categories and Strategies + +| Error Type | Strategy | User Impact | +|-----------|----------|-------------| +| File read failure | Log + skip file, continue batch | Warning in summary | +| OCR failure (no text) | Create note with empty body + source ref | Note created, flagged | +| Encoding detection failure | Log + skip file | Warning in summary | +| LLM API error | Retry 3x with exponential backoff, then fail file | Warning per file | +| LLM response parse error | Retry once with stricter prompt, then use defaults | Degraded metadata | +| OrgMyLife API unreachable | Log warning, continue without task creation | Tasks not created | +| Git commit failure | Log warning, files still written | No version history | +| Disk full / write failure | Fail immediately with clear error | Process stops | +| Invalid config (missing key) | Exit with descriptive error before processing | Immediate feedback | + +### Retry Policy + +```python +RETRY_CONFIG = { + "max_retries": 3, + "base_delay_seconds": 1.0, + "max_delay_seconds": 30.0, + "backoff_factor": 2.0, + "retryable_errors": [ + "rate_limit", + "timeout", + "server_error", # 5xx + ] +} +``` + +### Logging + +- **Level INFO**: File processing progress, summary counts +- **Level WARNING**: Skipped files, API failures (non-fatal), git failures +- **Level ERROR**: Configuration errors, unrecoverable failures +- **Level DEBUG**: LLM prompts/responses, entity detection details + +All logs include timestamps and structured context (file path, provider, operation). + +## Testing Strategy + +### Property-Based Tests (Hypothesis) + +The project uses **Hypothesis** (Python's standard PBT library) for property-based testing. Each property test runs a minimum of 100 iterations. + +Property tests cover the core transformation logic: +- File discovery filtering (Property 1) +- Text extraction identity (Property 3) +- Entity parsing and filtering (Properties 4, 5) +- Wiki-link generation (Properties 6, 7) +- Frontmatter round-trip (Property 8) +- Folder routing (Property 13) +- Filename generation and collision avoidance (Properties 14, 15) +- Temporary file filtering (Property 16) + +Each property test is tagged with: +```python +# Feature: notegraph-ingestion, Property N: [property text] +``` + +Configuration: minimum 100 examples per test via `@settings(max_examples=100)`. + +### Unit Tests (pytest) + +Example-based tests for: +- CLI argument parsing and subcommand routing +- Configuration loading from environment variables +- Specific enrichment response scenarios +- OrgMyLife client request formatting +- Git commit message generation +- Dry-run mode behavior + +### Integration Tests + +Integration tests (marked with `@pytest.mark.integration`) for: +- OCR extraction with sample images +- PDF text extraction with sample PDFs +- DOCX extraction with sample files +- LLM API calls (with real provider, gated by env var) +- OrgMyLife API task creation (against test instance) +- Drop-folder watcher file detection +- Git operations in a test repository + +### Test Dependencies + +```toml +[project.optional-dependencies] +dev = [ + "pytest>=8.0", + "pytest-asyncio>=0.23", + "hypothesis>=6.100", + "pytest-mock>=3.12", + "respx>=0.21", # httpx mocking +] +``` diff --git a/.kiro/specs/notegraph-ingestion/requirements.md b/.kiro/specs/notegraph-ingestion/requirements.md new file mode 100644 index 0000000..ef8dafa --- /dev/null +++ b/.kiro/specs/notegraph-ingestion/requirements.md @@ -0,0 +1,195 @@ +# Requirements Document + +## Introduction + +The NoteGraph Ingestion & Auto-Reference feature adds a document import pipeline and AI-powered entity linking system to NoteGraph. It enables bulk import of PDF files (from reMarkable tablet), images (photos of sticky notes, whiteboards, handwritten notes), and legacy text files (from OneNote, Android Notes, Obsidian, scattered file systems). Imported content is OCR-processed, enriched by an LLM to detect entities (people, projects, dates, TODOs), and output as structured markdown with wiki-links and frontmatter into the appropriate NoteGraph folder. The system uses a model-agnostic LLM abstraction layer supporting multiple providers via simple `.env` configuration. + +## Glossary + +- **Ingestion_Service**: The Python service responsible for orchestrating the full import pipeline from file input through OCR, enrichment, and markdown output. +- **OCR_Engine**: The component that extracts raw text from image files (JPG, PNG) and scanned PDF pages using optical character recognition. +- **Text_Extractor**: The component that extracts text content from text-based PDFs and legacy file formats (markdown, plain text, OneNote exports). +- **LLM_Provider**: An abstraction layer that routes enrichment requests to a configured language model provider (OpenAI, Anthropic, Google, Mistral, Ollama/Gemma) without coupling to any specific API. +- **Enrichment_Agent**: The AI-powered component that processes raw extracted text to detect entities, generate metadata, create wiki-links, and structure the output as markdown. +- **Entity_Detector**: The sub-component of the Enrichment_Agent responsible for identifying person names, project references, dates, and action items in extracted text. +- **Reference_Linker**: The sub-component that converts detected entities into SilverBullet-compatible wiki-links (e.g., `[[people/name]]`, `[[projects/name]]`). +- **Drop_Folder_Watcher**: A background process that monitors a designated inbox directory for new files and triggers the Ingestion_Service automatically. +- **Inbox_Folder**: The designated directory where files are placed for automatic ingestion (default: `NoteGraph/ingestion/inbox/`). +- **Output_Folder**: The target directory within `NoteGraph/notes/` where processed markdown files are placed. +- **Provider_Config**: The `.env`-based configuration that specifies which LLM_Provider to use and supplies the corresponding API keys. +- **OrgMyLife_Client**: The HTTP client component that creates tasks in the OrgMyLife API from extracted action items. +- **Frontmatter**: YAML metadata block at the top of a markdown file containing structured fields (title, date, tags, people, projects). + +## Requirements + +### Requirement 1: Bulk File Import via CLI + +**User Story:** As a user, I want to import many files at once from a folder using a CLI command, so that I can quickly ingest large collections of notes from various sources. + +#### Acceptance Criteria + +1. WHEN the user runs the CLI import command with a directory path, THE Ingestion_Service SHALL recursively discover all supported files (PDF, JPG, PNG, MD, TXT) in that directory. +2. WHEN the user runs the CLI import command with a single file path, THE Ingestion_Service SHALL process that individual file. +3. WHEN processing multiple files, THE Ingestion_Service SHALL report progress showing the current file number, total count, and file name. +4. WHEN a file fails to process during bulk import, THE Ingestion_Service SHALL log the error with the file path and reason, skip the file, and continue processing remaining files. +5. WHEN bulk import completes, THE Ingestion_Service SHALL output a summary showing total files processed, successful count, and failed count. +6. THE Ingestion_Service SHALL support the following file extensions: `.pdf`, `.jpg`, `.jpeg`, `.png`, `.md`, `.txt`, `.docx`. + +### Requirement 2: OCR and Text Extraction from Images + +**User Story:** As a user, I want photos of handwritten notes and whiteboards converted to text, so that I can search and reference them in my knowledge base. + +#### Acceptance Criteria + +1. WHEN a JPG or PNG file is provided, THE OCR_Engine SHALL extract text content from the image and return it as a UTF-8 string. +2. WHEN a PDF file contains scanned pages (image-based content without embedded text), THE OCR_Engine SHALL apply OCR to extract text from those pages. +3. WHEN a PDF file contains embedded text, THE Text_Extractor SHALL extract the text directly without OCR. +4. WHEN a PDF file contains a mix of text pages and scanned pages, THE Ingestion_Service SHALL apply the appropriate extraction method per page and concatenate results. +5. IF the OCR_Engine cannot extract any readable text from an image, THEN THE Ingestion_Service SHALL create the output note with an empty body and attach the original file reference in the frontmatter. +6. WHEN processing handwritten content, THE OCR_Engine SHALL preserve paragraph structure by detecting line breaks and groupings. + +### Requirement 3: Legacy File Format Import + +**User Story:** As a user, I want to import notes from old tools like OneNote, Obsidian, and scattered text files, so that I can consolidate all my knowledge in one place. + +#### Acceptance Criteria + +1. WHEN a markdown file is provided, THE Text_Extractor SHALL read the file content preserving existing formatting, links, and structure. +2. WHEN a plain text file is provided, THE Text_Extractor SHALL read the file content as UTF-8 text. +3. WHEN a DOCX file is provided, THE Text_Extractor SHALL extract the text content preserving heading structure and paragraph breaks. +4. WHEN an imported markdown file contains Obsidian-style wiki-links (`[[page name]]`), THE Text_Extractor SHALL preserve them for later resolution by the Reference_Linker. +5. WHEN a file uses a non-UTF-8 encoding, THE Text_Extractor SHALL attempt to detect the encoding and convert to UTF-8. +6. IF a file cannot be decoded to text, THEN THE Ingestion_Service SHALL log a warning and skip the file. + +### Requirement 4: AI-Powered Entity Detection + +**User Story:** As a user, I want the system to automatically detect people, projects, dates, and action items in my notes, so that I do not have to manually tag and link everything. + +#### Acceptance Criteria + +1. WHEN extracted text is provided to the Enrichment_Agent, THE Entity_Detector SHALL identify person names mentioned in the text. +2. WHEN extracted text is provided to the Enrichment_Agent, THE Entity_Detector SHALL identify project references mentioned in the text. +3. WHEN extracted text is provided to the Enrichment_Agent, THE Entity_Detector SHALL identify dates and temporal references in the text and normalize them to ISO 8601 format. +4. WHEN extracted text is provided to the Enrichment_Agent, THE Entity_Detector SHALL identify action items and TODO statements in the text. +5. WHEN the Entity_Detector identifies entities, THE Entity_Detector SHALL return each entity with its type, extracted value, and confidence score between 0.0 and 1.0. +6. THE Entity_Detector SHALL only include entities with a confidence score of 0.7 or higher in the final output. + +### Requirement 5: Auto-Reference Wiki-Link Generation + +**User Story:** As a user, I want detected entities automatically converted to wiki-links, so that my imported notes are immediately connected to my knowledge graph. + +#### Acceptance Criteria + +1. WHEN a person entity is detected, THE Reference_Linker SHALL generate a wiki-link in the format `[[people/firstname-lastname]]`. +2. WHEN a project entity is detected, THE Reference_Linker SHALL generate a wiki-link in the format `[[projects/project-name]]`. +3. WHEN a date entity is detected, THE Reference_Linker SHALL add the normalized date to the frontmatter `date` field. +4. WHEN the Reference_Linker generates a wiki-link for a person or project that does not yet exist in NoteGraph, THE Reference_Linker SHALL create a stub page with the entity name as title. +5. WHEN the Reference_Linker generates a wiki-link, THE Reference_Linker SHALL insert the link inline in the note body at the position where the entity was detected. +6. WHEN multiple references to the same entity exist in one note, THE Reference_Linker SHALL link only the first occurrence and leave subsequent mentions as plain text. + +### Requirement 6: Structured Markdown Output with Frontmatter + +**User Story:** As a user, I want imported notes output as structured markdown with proper metadata, so that they integrate seamlessly with SilverBullet's features. + +#### Acceptance Criteria + +1. THE Ingestion_Service SHALL output each processed file as a markdown file with YAML frontmatter containing: title, date, tags, people, projects, and source fields. +2. WHEN the Enrichment_Agent generates a title from the content, THE Ingestion_Service SHALL use it as the frontmatter `title` field. +3. WHEN entities are detected, THE Ingestion_Service SHALL populate the frontmatter `people` field as a list of detected person names. +4. WHEN entities are detected, THE Ingestion_Service SHALL populate the frontmatter `projects` field as a list of detected project names. +5. WHEN entities are detected, THE Ingestion_Service SHALL populate the frontmatter `tags` field with relevant topic tags derived from the content. +6. THE Ingestion_Service SHALL include a `source` field in the frontmatter containing the original file name and import timestamp. +7. FOR ALL valid extracted text inputs, processing through the Enrichment_Agent and then rendering to markdown and then parsing the frontmatter SHALL produce a valid YAML frontmatter block (round-trip property). + +### Requirement 7: Model-Agnostic LLM Provider Layer + +**User Story:** As a user, I want to easily switch between LLM providers and try different models, so that I can compare behavior and avoid vendor lock-in. + +#### Acceptance Criteria + +1. THE LLM_Provider SHALL expose a unified interface for text completion requests regardless of the underlying provider. +2. THE LLM_Provider SHALL support at minimum the following providers: OpenAI, Anthropic, Google Gemini, Mistral, Gemma (local), and Ollama (local). +3. WHEN a provider is specified in the Provider_Config, THE LLM_Provider SHALL route requests to that provider using the configured API key. +4. WHEN the configured provider is unavailable or returns an error, THE LLM_Provider SHALL raise a descriptive exception including the provider name and error details. +5. THE LLM_Provider SHALL read provider selection and API keys from environment variables defined in a `.env` file. +6. THE LLM_Provider SHALL support configuring the model name per provider (e.g., `gpt-4o`, `claude-sonnet-4-20250514`, `gemini-pro`). +7. WHEN switching providers, THE LLM_Provider SHALL require only a change to environment variables without code modifications. + +### Requirement 8: Provider Configuration via Environment Variables + +**User Story:** As a user, I want simple `.env`-based configuration for LLM providers, so that I can set up and switch providers without editing code. + +#### Acceptance Criteria + +1. THE Provider_Config SHALL use the environment variable `LLM_PROVIDER` to select the active provider (values: `openai`, `anthropic`, `google`, `mistral`, `ollama`). +2. THE Provider_Config SHALL use the environment variable `LLM_MODEL` to specify the model name for the active provider. +3. THE Provider_Config SHALL use provider-specific environment variables for API keys: `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GOOGLE_API_KEY`, `MISTRAL_API_KEY`. +4. WHERE the Ollama provider is selected, THE Provider_Config SHALL use the environment variable `OLLAMA_BASE_URL` to specify the local Ollama endpoint (default: `http://localhost:11434`). Ollama SHALL support running Google Gemma models locally. +5. THE Provider_Config SHALL NOT store API keys or secrets directly in git-tracked files. The `.env` file SHALL be listed in `.gitignore` and a `.env.example` file SHALL document all variables without actual secret values. +6. WHEN a required API key is missing for the selected provider, THE Ingestion_Service SHALL exit with a clear error message naming the missing variable. +7. THE Ingestion_Service SHALL provide a `.env.example` file documenting all supported environment variables with descriptions. + +### Requirement 9: TODO Extraction and OrgMyLife Integration + +**User Story:** As a user, I want action items extracted from my notes and optionally created as tasks in OrgMyLife, so that I do not lose track of commitments found in imported notes. + +#### Acceptance Criteria + +1. WHEN the Entity_Detector identifies action items in the text, THE Enrichment_Agent SHALL extract each action item with its description and any associated person or deadline. +2. WHEN action items are extracted, THE Ingestion_Service SHALL list them in a `## Action Items` section at the end of the output markdown. +3. WHEN the `--create-tasks` flag is provided on the CLI, THE OrgMyLife_Client SHALL create a task in OrgMyLife for each extracted action item via POST to `/api/tasks`. +4. WHEN creating a task in OrgMyLife, THE OrgMyLife_Client SHALL set the task title to the action item description and include a `source_url` linking back to the imported note. +5. IF the OrgMyLife API is unreachable when `--create-tasks` is specified, THEN THE Ingestion_Service SHALL log a warning and continue processing without creating tasks. +6. THE Provider_Config SHALL use the environment variable `ORGMYLIFE_API_URL` for the OrgMyLife endpoint and `ORGMYLIFE_API_KEY` for authentication. + +### Requirement 10: File Placement and Folder Routing + +**User Story:** As a user, I want imported notes automatically placed in the correct NoteGraph folder based on their content, so that my knowledge base stays organized. + +#### Acceptance Criteria + +1. WHEN the Enrichment_Agent detects meeting-related content, THE Ingestion_Service SHALL place the output file in `NoteGraph/notes/meetings/`. +2. WHEN the Enrichment_Agent detects project-specific content with a single dominant project, THE Ingestion_Service SHALL place the output file in `NoteGraph/notes/projects/`. +3. WHEN the Enrichment_Agent cannot determine a specific category, THE Ingestion_Service SHALL place the output file in `NoteGraph/notes/inbox/`. +4. WHEN the `--output-dir` flag is provided on the CLI, THE Ingestion_Service SHALL place all output files in the specified directory regardless of content analysis. +5. THE Ingestion_Service SHALL generate output file names using the pattern `YYYY-MM-DD-slugified-title.md` based on the detected or current date and the generated title. +6. IF an output file with the same name already exists, THEN THE Ingestion_Service SHALL append a numeric suffix (e.g., `-2`, `-3`) to avoid overwriting. + +### Requirement 11: Drop-Folder Watcher for Continuous Import + +**User Story:** As a user, I want to drop files into a folder and have them automatically processed, so that I can import notes without running CLI commands. + +#### Acceptance Criteria + +1. WHEN the watcher mode is started, THE Drop_Folder_Watcher SHALL monitor the Inbox_Folder for new files. +2. WHEN a new file appears in the Inbox_Folder, THE Drop_Folder_Watcher SHALL trigger the Ingestion_Service to process that file. +3. WHEN a file is successfully processed, THE Drop_Folder_Watcher SHALL move the original file to an `archive/` subdirectory within the Inbox_Folder. +4. IF a file fails to process, THEN THE Drop_Folder_Watcher SHALL move the file to a `failed/` subdirectory and log the error. +5. THE Drop_Folder_Watcher SHALL ignore temporary files (names starting with `.` or ending with `.tmp`). +6. WHEN the watcher mode is started, THE Drop_Folder_Watcher SHALL process any files already present in the Inbox_Folder before entering watch mode. + +### Requirement 12: Git Auto-Commit for Imported Notes + +**User Story:** As a user, I want imported notes automatically committed to git, so that my knowledge base history is preserved without manual intervention. + +#### Acceptance Criteria + +1. WHEN one or more files are successfully imported, THE Ingestion_Service SHALL stage the new markdown files and commit them to the NoteGraph git repository. +2. THE Ingestion_Service SHALL use the commit message format: `ingestion: import N notes from [source-type]` where N is the count and source-type describes the input (e.g., "pdf", "images", "bulk"). +3. WHEN stub pages are created by the Reference_Linker, THE Ingestion_Service SHALL include them in the same git commit. +4. WHEN the `--no-commit` flag is provided on the CLI, THE Ingestion_Service SHALL skip the git commit step. +5. IF the git commit fails, THEN THE Ingestion_Service SHALL log a warning but not fail the overall import process. + +### Requirement 13: CLI Interface Design + +**User Story:** As a user, I want a clear and consistent CLI interface, so that I can easily run imports with different options. + +#### Acceptance Criteria + +1. THE Ingestion_Service SHALL provide a CLI entry point via `python -m ingestion.cli`. +2. THE Ingestion_Service SHALL support the subcommand `import` accepting a path argument (file or directory). +3. THE Ingestion_Service SHALL support the subcommand `watch` to start the Drop_Folder_Watcher. +4. THE Ingestion_Service SHALL support the following global flags: `--verbose` for detailed logging, `--dry-run` to preview without writing files, `--no-commit` to skip git commits. +5. THE Ingestion_Service SHALL support the following import flags: `--output-dir` to override output location, `--create-tasks` to enable OrgMyLife task creation, `--provider` to override the LLM provider for this run. +6. WHEN the `--dry-run` flag is provided, THE Ingestion_Service SHALL display what would be created (file paths, detected entities, proposed links) without writing any files. +7. WHEN invoked without arguments, THE Ingestion_Service SHALL display usage help with examples. diff --git a/.kiro/specs/notegraph-ingestion/tasks.md b/.kiro/specs/notegraph-ingestion/tasks.md new file mode 100644 index 0000000..31e00e2 --- /dev/null +++ b/.kiro/specs/notegraph-ingestion/tasks.md @@ -0,0 +1,318 @@ +# Implementation Plan: NoteGraph Ingestion & Auto-Reference + +## Overview + +This plan implements the NoteGraph ingestion pipeline as a Python package at `NoteGraph/ingestion/`. Tasks are ordered so each builds on the previous: scaffolding → extraction → LLM → enrichment → linking → output → CLI → web upload → Nextcloud sync → OrgMyLife → git → Docker/deploy → tests. Property-based tests use Hypothesis and are placed close to the code they validate. + +## Tasks + +- [x] 1. Project scaffolding and configuration + - [x] 1.1 Create `NoteGraph/ingestion/` package structure with `__init__.py`, `__main__.py`, and all subpackages (`extraction/`, `enrichment/`, `linking/`, `output/`, `integrations/`, `tests/`) + - Create directory tree matching the design's package structure + - Add empty `__init__.py` files in each subpackage + - Create `__main__.py` with `from ingestion.cli import cli; cli()` entry point + - _Requirements: 13.1_ + + - [x] 1.2 Create `NoteGraph/ingestion/pyproject.toml` with all dependencies + - Include: click, watchdog, pytesseract, PyPDF2, pdfplumber, python-docx, litellm, httpx, pydantic, pydantic-settings, fastapi, uvicorn, python-multipart + - Add dev dependencies: pytest, pytest-asyncio, hypothesis, pytest-mock, respx + - Configure package metadata and entry points + - _Requirements: 7.1, 8.7_ + + - [x] 1.3 Implement `NoteGraph/ingestion/config.py` with `IngestionConfig` (pydantic-settings) + - Define all env vars: `LLM_PROVIDER`, `LLM_MODEL`, provider API keys, `ORGMYLIFE_API_URL`, `ORGMYLIFE_API_KEY`, `NEXTCLOUD_INBOX_URL`, `NEXTCLOUD_INBOX_USER`, `NEXTCLOUD_INBOX_PASSWORD`, `SB_USER`, paths, OCR settings, confidence threshold + - Implement `llm_model_string` property for LiteLLM routing + - Implement `validate_api_key()` that exits with clear error naming the missing variable + - _Requirements: 8.1, 8.2, 8.3, 8.4, 8.5, 8.6_ + + - [x] 1.4 Create `NoteGraph/.env.example` with all supported environment variables and descriptions + - Document every variable with comments explaining purpose and valid values + - Include placeholders for: LLM keys, OrgMyLife, Nextcloud inbox, SB_USER + - Ensure `.env` is in `.gitignore` + - _Requirements: 8.5, 8.7_ + +- [x] 2. Text extraction layer + - [x] 2.1 Implement `extraction/base.py` with `TextExtractor` protocol and `ExtractionResult` dataclass + - Define `can_handle(file_path: Path) -> bool` and `extract(file_path: Path) -> ExtractionResult` + - `ExtractionResult` fields: text, source_file, extraction_method, metadata + - _Requirements: 2.1, 3.1_ + + - [x] 2.2 Implement `extraction/text.py` for plain text and markdown files + - Handle `.md` and `.txt` extensions + - Read as UTF-8, attempt encoding detection for non-UTF-8 files (chardet/charset-normalizer) + - Preserve existing formatting, links, and Obsidian-style wiki-links + - _Requirements: 3.1, 3.2, 3.4, 3.5, 3.6_ + + - [x] 2.3 Implement `extraction/pdf.py` for PDF text extraction + - Use pdfplumber for text-based PDFs + - Detect scanned pages (pages with no extractable text) and delegate to OCR + - Handle mixed PDFs (text + scanned pages) by concatenating results + - _Requirements: 2.2, 2.3, 2.4_ + + - [x] 2.4 Implement `extraction/ocr.py` for image OCR via pytesseract + - Handle `.jpg`, `.jpeg`, `.png` files + - Configure language as `deu+eng` (German + English) + - Preserve paragraph structure by detecting line breaks + - Return empty text (not error) when OCR cannot extract readable content + - _Requirements: 2.1, 2.2, 2.5, 2.6_ + + - [x] 2.5 Implement `extraction/docx.py` for DOCX files + - Use python-docx to extract text preserving heading structure and paragraph breaks + - Handle `.docx` extension + - _Requirements: 3.3_ + + - [ ]* 2.6 Write property test for text extraction identity (Property 3) + - **Property 3: Text extraction preserves content for text-based formats** + - For any valid UTF-8 string written to a .md or .txt file, extraction returns identical content + - **Validates: Requirements 3.1, 3.2, 3.4** + +- [x] 3. File discovery + - [x] 3.1 Implement `discovery.py` with recursive file discovery and extension filtering + - Support extensions: `.pdf`, `.jpg`, `.jpeg`, `.png`, `.md`, `.txt`, `.docx` + - Accept both single file path and directory path + - Recursive directory traversal + - _Requirements: 1.1, 1.2, 1.6_ + + - [ ]* 3.2 Write property test for file discovery filtering (Property 1) + - **Property 1: File discovery returns only supported extensions** + - For any directory tree with arbitrary extensions, discovery returns exactly those with supported extensions + - **Validates: Requirements 1.1, 1.6** + +- [ ] 4. LLM provider layer + - [x] 4.1 Implement `integrations/llm.py` with LiteLLM wrapper + - `LLMClient` class with `complete(messages, response_format)` method + - Route to provider based on `config.llm_model_string` (e.g., `openai/gpt-4o`, `anthropic/claude-sonnet-4-20250514`) + - Implement retry logic: 3 retries with exponential backoff for rate limits, timeouts, 5xx errors + - Raise descriptive exception on failure including provider name and error details + - _Requirements: 7.1, 7.2, 7.3, 7.4, 7.5, 7.6, 7.7_ + + - [ ]* 4.2 Write property test for provider routing (Property 9) + - **Property 9: Provider routing from configuration** + - For any valid provider name, configuring `LLM_PROVIDER` causes the LLM client to route to the corresponding provider + - **Validates: Requirements 7.3, 8.1** + + - [ ]* 4.3 Write property test for missing API key error (Property 10) + - **Property 10: Missing API key produces descriptive error** + - For any provider requiring an API key, when the env var is unset, validation raises an error naming the missing variable + - **Validates: Requirements 7.4, 8.6** + +- [x] 5. Enrichment agent + - [x] 5.1 Implement `enrichment/models.py` with Pydantic models + - `Entity` model: type (person/project/date/action_item), value, confidence (0.0-1.0), position + - `ActionItem` model: description, assignee, deadline (ISO 8601) + - `EnrichmentResult` model: title, category, tags, entities, action_items, summary + - _Requirements: 4.1, 4.2, 4.3, 4.4, 4.5_ + + - [x] 5.2 Implement `enrichment/prompts.py` with structured LLM prompts + - System prompt requesting structured JSON output for entity detection + - Include context about known projects and people (read from existing NoteGraph pages) + - Request confidence scores for each entity + - _Requirements: 4.1, 4.2, 4.3, 4.4, 4.5_ + + - [x] 5.3 Implement `enrichment/agent.py` with enrichment orchestration + - Call LLM with extracted text and structured prompt + - Parse response into `EnrichmentResult` + - Filter entities by confidence threshold (>= 0.7) + - Handle LLM parse errors: retry once with stricter prompt, then use defaults + - _Requirements: 4.1, 4.2, 4.3, 4.4, 4.5, 4.6_ + + - [ ]* 5.4 Write property tests for entity parsing (Properties 4, 5) + - **Property 4: Entity parsing extracts all entities with valid structure** + - For any valid enrichment response JSON, each entity has valid type, non-empty value, confidence in [0.0, 1.0] + - **Property 5: Confidence threshold filtering** + - For any list of entities, filtered output contains exactly those with confidence >= 0.7 + - **Validates: Requirements 4.1, 4.2, 4.3, 4.4, 4.5, 4.6** + +- [x] 6. Checkpoint - Ensure all tests pass + - Ensure all tests pass, ask the user if questions arise. + +- [x] 7. Reference linker + - [x] 7.1 Implement `linking/linker.py` with wiki-link generation + - Person entities → `[[people/firstname-lastname]]` (slugified) + - Project entities → `[[projects/project-slug]]` (slugified) + - Insert wiki-link inline at entity position (first occurrence only) + - Leave subsequent mentions of the same entity as plain text + - Add dates to frontmatter `date` field + - _Requirements: 5.1, 5.2, 5.3, 5.5, 5.6_ + + - [x] 7.2 Implement `linking/stubs.py` for stub page creation + - Check if target page exists in NoteGraph notes directory + - Create stub page with entity name as title if it doesn't exist + - Stub pages go in `notes/people/` or `notes/projects/` as appropriate + - _Requirements: 5.4_ + + - [ ]* 7.3 Write property tests for wiki-link generation (Properties 6, 7) + - **Property 6: Wiki-link generation follows correct format per entity type** + - For any person name → `[[people/slugified-name]]`; for any project name → `[[projects/slugified-name]]` + - **Property 7: Only first entity occurrence is linked** + - For any text with N >= 2 occurrences of the same entity, only the first is converted to a wiki-link + - **Validates: Requirements 5.1, 5.2, 5.6** + +- [x] 8. Output renderer and folder routing + - [x] 8.1 Implement `output/renderer.py` for markdown + YAML frontmatter rendering + - Render frontmatter with: title, date, tags, people, projects, source (file + timestamp) + - Render body with wiki-links inserted + - Render `## Action Items` section at the end with extracted action items + - _Requirements: 6.1, 6.2, 6.3, 6.4, 6.5, 6.6, 9.2_ + + - [x] 8.2 Implement `output/router.py` for category-based folder routing + - "meeting" → `notes/meetings/` + - "project" → `notes/projects/` + - "decision" → `notes/decisions/` + - Default/inbox → `notes/inbox/` + - Support `--output-dir` override + - _Requirements: 10.1, 10.2, 10.3, 10.4_ + + - [x] 8.3 Implement `output/naming.py` for filename generation with collision avoidance + - Pattern: `YYYY-MM-DD-slugified-title.md` + - Slug: lowercase alphanumeric + hyphens only + - Append numeric suffix (`-2`, `-3`) if file already exists + - _Requirements: 10.5, 10.6_ + + - [ ]* 8.4 Write property tests for output (Properties 8, 11, 13, 14, 15) + - **Property 8: Frontmatter round-trip validity** + - For any valid EnrichmentResult, render to markdown then parse frontmatter → valid YAML with all required fields + - **Property 11: Action items appear in output markdown section** + - For any list of action items, rendered markdown contains `## Action Items` section listing every item + - **Property 13: Category-based folder routing** + - For any category, output path is within the correct folder + - **Property 14: Output filename follows date-slug pattern** + - For any date and title, filename matches `YYYY-MM-DD-slugified-title.md` + - **Property 15: Filename collision avoidance** + - When a file with the same name exists, a numeric suffix is appended + - **Validates: Requirements 6.1, 6.3, 6.4, 6.6, 6.7, 9.2, 10.1, 10.2, 10.3, 10.5, 10.6** + +- [x] 9. Pipeline orchestration + - [x] 9.1 Implement `pipeline.py` with main processing pipeline + - Orchestrate: discovery → extraction → enrichment → linking → rendering → file write → git commit + - Track progress: current file number, total count, file name + - Handle errors per file: log + skip + continue + - Output summary: total processed, success count, failed count + - Support `--dry-run` mode (display what would be created without writing) + - _Requirements: 1.3, 1.4, 1.5, 13.6_ + + - [ ]* 9.2 Write property tests for pipeline resilience and dry-run (Properties 2, 17) + - **Property 2: Bulk import resilience — failed files do not block remaining processing** + - For any batch where a subset fails, remaining files are processed successfully + - **Property 17: Dry-run produces no file writes** + - For any input with `--dry-run`, zero files are created on disk + - **Validates: Requirements 1.4, 1.5, 13.6** + +- [x] 10. CLI interface + - [x] 10.1 Implement `cli.py` with Click-based CLI + - `import` subcommand: accepts path argument (file or directory) + - `watch` subcommand: starts drop-folder watcher + - Global flags: `--verbose`, `--dry-run`, `--no-commit` + - Import flags: `--output-dir`, `--create-tasks`, `--provider` + - Display usage help with examples when invoked without arguments + - _Requirements: 13.1, 13.2, 13.3, 13.4, 13.5, 13.6, 13.7_ + +- [x] 11. Checkpoint - Ensure all tests pass + - Ensure all tests pass, ask the user if questions arise. + +- [ ] 12. Web upload endpoint (FastAPI) + - [x] 12.1 Implement `web/upload.py` with FastAPI app for file upload + - HTML form with file upload accepting PDF, JPG, PNG + - Mobile-friendly responsive HTML page (inline CSS, no external dependencies) + - Basic auth protection using `SB_USER` env var (same credentials as SilverBullet) + - On upload: save file to inbox folder, trigger processing pipeline + - Return success/error response to the user + - _Requirements: 11.2 (extends with web input)_ + + - [x] 12.2 Create `web/templates/upload.html` mobile-friendly upload page + - Simple, clean form with drag-and-drop support + - Responsive design for Android "Share to URL" workflow + - Show upload progress and result feedback + - _Requirements: (additional requirement: web upload for mobile)_ + +- [ ] 13. Nextcloud WebDAV inbox sync + - [x] 13.1 Implement `integrations/nextcloud.py` with WebDAV folder polling + - Poll `NEXTCLOUD_INBOX_URL` for new files using httpx WebDAV (PROPFIND/GET) + - Download new files to local inbox for processing + - After successful processing, move files to a `processed/` subfolder on Nextcloud (MOVE request) + - Auth via `NEXTCLOUD_INBOX_USER` and `NEXTCLOUD_INBOX_PASSWORD` env vars + - _Requirements: (additional requirement: Nextcloud WebDAV sync)_ + + - [x] 13.2 Add `sync` subcommand to CLI for Nextcloud polling + - One-shot sync: poll once, process new files, exit + - Can be called from cron or systemd timer + - _Requirements: (additional requirement: Nextcloud WebDAV sync)_ + +- [x] 14. Drop-folder watcher + - [x] 14.1 Implement `watcher.py` with watchdog-based folder monitoring + - Monitor inbox folder for new files + - Trigger pipeline on new file detection + - Move processed files to `archive/` subdirectory + - Move failed files to `failed/` subdirectory with error log + - Ignore temporary files (names starting with `.` or ending with `.tmp`) + - Process existing files in inbox on startup before entering watch mode + - _Requirements: 11.1, 11.2, 11.3, 11.4, 11.5, 11.6_ + + - [ ]* 14.2 Write property test for temporary file filtering (Property 16) + - **Property 16: Temporary file filtering** + - For any filename, ignore logic returns true iff name starts with `.` or ends with `.tmp` + - **Validates: Requirements 11.5** + +- [x] 15. OrgMyLife integration + - [x] 15.1 Implement `integrations/orgmylife.py` with HTTP client for task creation + - POST to `/api/tasks` with title, source_url, description + - Auth via `ORGMYLIFE_API_KEY` header + - Handle unreachable API gracefully: log warning, continue without creating tasks + - _Requirements: 9.3, 9.4, 9.5, 9.6_ + + - [ ]* 15.2 Write property test for OrgMyLife task payload (Property 12) + - **Property 12: OrgMyLife task payload correctness** + - For any action item description and note path, payload has correct title and source_url + - **Validates: Requirements 9.4** + +- [x] 16. Git auto-commit + - [x] 16.1 Implement `integrations/git.py` for git commit operations + - Stage new markdown files and stub pages + - Commit with message format: `ingestion: import N notes from [source-type]` + - Support `--no-commit` flag to skip + - Handle git failures gracefully: log warning, don't fail the import + - _Requirements: 12.1, 12.2, 12.3, 12.4, 12.5_ + + - [ ]* 16.2 Write property test for git commit message format (Property 18) + - **Property 18: Git commit message format** + - For any count N and source type, message matches `ingestion: import N notes from [source-type]` + - **Validates: Requirements 12.2** + +- [x] 17. Checkpoint - Ensure all tests pass + - Ensure all tests pass, ask the user if questions arise. + +- [x] 18. Docker integration and deployment + - [x] 18.1 Create `NoteGraph/ingestion/Dockerfile` for the ingestion service + - Python 3.11+ base image + - Install system dependencies: tesseract-ocr, tesseract-ocr-deu + - Install Python package with all dependencies + - Run FastAPI upload server (uvicorn) as default command + - Also support CLI entry point for import/watch/sync commands + - _Requirements: 13.1_ + + - [x] 18.2 Update `NoteGraph/docker-compose.yml` to add ingestion service + - Add `ingestion` service with build context `./ingestion` + - Mount `./notes` volume for output + - Mount `./ingestion/inbox` for drop-folder input + - Expose upload endpoint port (8001) + - Pass all env vars from `.env` + - Add watcher mode as default or separate service + - _Requirements: 11.1_ + + - [x] 18.3 Update `NoteGraph/.github/workflows/deploy.yml` with all secrets + - Write ALL secrets from GitHub secrets into `.env` on VPS: LLM keys (OpenAI, Anthropic, Google, Mistral), Nextcloud inbox creds, OrgMyLife API key, SB_USER + - Follow same pattern as OrgMyLife deploy workflow (ENV_CONTENT with printf) + - Add `docker compose up -d --build` to rebuild with ingestion service + - _Requirements: 8.5 (additional requirement: secrets via GitHub Actions)_ + +- [x] 19. Final checkpoint - Ensure all tests pass + - Ensure all tests pass, ask the user if questions arise. + +## Notes + +- Tasks marked with `*` are optional property-based tests and can be skipped for faster MVP +- Each task references specific requirements for traceability +- Checkpoints ensure incremental validation +- Property tests use Hypothesis with `@settings(max_examples=100)` +- The web upload endpoint and Nextcloud sync are additional input methods beyond the original CLI/watcher +- All secrets are managed via GitHub Actions → `.env` on VPS, never committed to git diff --git a/.kiro/specs/pat-renewal/.config.kiro b/.kiro/specs/pat-renewal/.config.kiro new file mode 100644 index 0000000..0112a52 --- /dev/null +++ b/.kiro/specs/pat-renewal/.config.kiro @@ -0,0 +1 @@ +{"specId": "d26ccd62-45f2-47f0-8fbb-e15f4f79916d", "workflowType": "requirements-first", "specType": "feature"} diff --git a/.kiro/specs/pat-renewal/design.md b/.kiro/specs/pat-renewal/design.md new file mode 100644 index 0000000..865d177 --- /dev/null +++ b/.kiro/specs/pat-renewal/design.md @@ -0,0 +1,579 @@ +# Design Document: PAT Renewal + +## Overview + +The PAT Renewal system automates Personal Access Token lifecycle management across GitLab, Jira, Confluence, and OrgMyLife services. It runs as a GitHub Actions scheduled workflow (weekly, Mondays at 06:00 UTC) that: + +1. Reads a PAT Registry (JSON) tracking all managed tokens +2. Checks each token's expiry status via service-specific API calls +3. Auto-rotates GitLab PATs using the GitLab Personal Access Tokens API +4. Creates OrgMyLife tasks or sends email alerts for tokens requiring manual renewal +5. Produces a summary report in the GitHub Actions run log + +The system integrates with the existing GitHub Actions infrastructure already used in the workspace (deploy workflows, OrgMyLife sync) and leverages the OrgMyLife API for task creation alerts. + +**Key Design Decisions:** +- **Python-based workflow script**: Matches the workspace's primary language (Python 3.11+) and allows reuse of patterns from the AI-Orchestrator project (httpx for HTTP, pytest for testing) +- **JSON registry file**: Simple, version-controllable, human-readable format stored in the repository +- **GitHub Actions secrets for token storage**: Leverages existing secure secret management without introducing new infrastructure +- **Idempotent alerting**: Prevents duplicate OrgMyLife tasks by checking for existing open tasks before creating new ones + +## Architecture + +```mermaid +graph TD + subgraph "GitHub Actions" + CRON[Cron Trigger
Monday 06:00 UTC] --> WF[pat-check Workflow] + DISPATCH[workflow_dispatch] --> WF + WF --> SCRIPT[pat_manager.py] + end + + subgraph "PAT Manager Script" + SCRIPT --> LOAD[Load PAT Registry] + LOAD --> CHECK[Expiry Checker] + CHECK --> CLASSIFY{Classify PATs} + CLASSIFY -->|expiring soon + auto| ROTATE[GitLab Rotator] + CLASSIFY -->|expiring soon + manual| ALERT[Alert Sender] + CLASSIFY -->|expired| ALERT + CLASSIFY -->|healthy| REPORT[Summary Reporter] + ROTATE --> SECRET_UPDATE[Update GitHub Secret] + ROTATE -->|failure| ALERT + SECRET_UPDATE --> REPORT + ALERT --> REPORT + end + + subgraph "External Services" + CHECK --> GITLAB_API[GitLab API] + CHECK --> JIRA_API[Jira API] + CHECK --> CONFLUENCE_API[Confluence API] + CHECK --> ORGMYLIFE_API[OrgMyLife API] + ROTATE --> GITLAB_API + SECRET_UPDATE --> GH_API[GitHub API] + ALERT --> ORGMYLIFE_API + ALERT --> EMAIL[Email / SMTP] + end + + subgraph "Storage" + LOAD --> REGISTRY[pat-registry.json] + REPORT --> LOG[Workflow Run Log] + end +``` + +### Component Interaction Flow + +```mermaid +sequenceDiagram + participant GHA as GitHub Actions + participant PM as pat_manager.py + participant REG as pat-registry.json + participant GL as GitLab API + participant JIRA as Jira API + participant OML as OrgMyLife API + participant GH as GitHub API + + GHA->>PM: Trigger (cron/dispatch) + PM->>REG: Load registry + REG-->>PM: PAT entries + + loop For each PAT + alt GitLab PAT + PM->>GL: GET /personal_access_tokens + GL-->>PM: expires_at + else Jira/Confluence/OrgMyLife PAT + PM->>JIRA: Authenticated API call + JIRA-->>PM: 200 OK / 401 Unauthorized + end + PM->>PM: Classify (healthy/expiring/expired/check failed) + end + + alt GitLab PAT expiring soon + PM->>GL: POST /personal_access_tokens/{id}/rotate + GL-->>PM: New token + expiry + PM->>GH: Update secret + PM->>REG: Update expiry_date + end + + alt Manual token expiring/expired + PM->>OML: POST /api/tasks (create alert task) + end + + PM->>GHA: Summary report (log output) +``` + +## Components and Interfaces + +### 1. PAT Registry Module (`registry.py`) + +Responsible for loading, validating, and persisting the PAT registry JSON file. + +```python +@dataclass +class PatEntry: + service: str # "GitLab" | "Jira" | "Confluence" | "OrgMyLife" + token_name: str # max 128 chars + expiry_date: str | None # ISO 8601 (YYYY-MM-DD) or None + renewal_method: str # "auto" | "manual" + +class PatRegistry: + def load(self, path: Path) -> list[PatEntry]: ... + def validate_entry(self, entry: dict) -> PatEntry | ValidationError: ... + def save(self, path: Path, entries: list[PatEntry]) -> None: ... + def create_empty(self, path: Path) -> None: ... +``` + +**Validation Rules:** +- `service` must be one of: `GitLab`, `Jira`, `Confluence`, `OrgMyLife` +- `token_name` must be non-empty, max 128 characters +- `expiry_date` must be valid ISO 8601 (YYYY-MM-DD) or null +- `renewal_method` must match service: GitLab → "auto", others → "manual" +- No duplicate `(service, token_name)` combinations + +### 2. Expiry Checker Module (`checker.py`) + +Queries each service API to determine PAT status. + +```python +class PatStatus(Enum): + HEALTHY = "healthy" + EXPIRING_SOON = "expiring soon" + EXPIRED = "expired" + CHECK_FAILED = "check failed" + +@dataclass +class CheckResult: + entry: PatEntry + status: PatStatus + days_remaining: int | None + error_message: str | None + +class ExpiryChecker: + def __init__(self, expiry_window_days: int = 14, timeout: int = 30, max_retries: int = 2): ... + async def check_all(self, entries: list[PatEntry], secrets: dict[str, str]) -> list[CheckResult]: ... + async def check_gitlab(self, entry: PatEntry, token: str) -> CheckResult: ... + async def check_jira(self, entry: PatEntry, token: str) -> CheckResult: ... + async def check_confluence(self, entry: PatEntry, token: str) -> CheckResult: ... + async def check_orgmylife(self, entry: PatEntry, token: str) -> CheckResult: ... +``` + +**Service-Specific Logic:** +- **GitLab**: `GET /personal_access_tokens` → read `expires_at` field, compare with window +- **Jira**: Authenticated call → HTTP 401 = expired +- **Confluence**: Authenticated call → HTTP 401 = expired +- **OrgMyLife**: Authenticated call → HTTP 401/403 = expired + +### 3. GitLab Rotator Module (`rotator.py`) + +Handles automatic token rotation for GitLab PATs. + +```python +@dataclass +class RotationResult: + success: bool + new_expiry_date: str | None + error_message: str | None + +class GitLabRotator: + def __init__(self, timeout: int = 30): ... + async def rotate(self, token_id: str, current_token: str) -> RotationResult: ... +``` + +### 4. Secret Updater Module (`secret_updater.py`) + +Updates GitHub Actions secrets with new token values. + +```python +class SecretUpdater: + def __init__(self, github_token: str, repo_owner: str, repo_name: str): ... + async def update_secret(self, secret_name: str, secret_value: str) -> bool: ... +``` + +### 5. Alert Sender Module (`alerter.py`) + +Sends alerts via OrgMyLife task creation or email. + +```python +class AlertChannel(Enum): + ORGMYLIFE = "orgmylife" + EMAIL = "email" + +@dataclass +class AlertResult: + sent: bool + channel: AlertChannel + error_message: str | None + +class AlertSender: + def __init__(self, channel: AlertChannel, config: dict): ... + async def send_alert(self, result: CheckResult) -> AlertResult: ... + async def check_existing_task(self, service: str, token_name: str, expiry_date: str) -> bool: ... +``` + +**OrgMyLife Integration:** +- Uses `POST /api/tasks` with `X-API-Key` header for authentication +- Task title format: `[PAT Renewal] {service} - {token_name}` +- Includes days remaining and renewal instructions in description +- Checks for existing open tasks before creating duplicates + +### 6. Summary Reporter Module (`reporter.py`) + +Produces the workflow summary log. + +```python +class SummaryReporter: + def report(self, results: list[CheckResult], rotations: list[RotationResult]) -> str: ... +``` + +### 7. Main Orchestrator (`pat_manager.py`) + +Entry point that coordinates all modules. + +```python +async def main() -> int: + """Returns exit code: 0 for success, 1 for failures.""" + ... +``` + +## Data Models + +### PAT Registry JSON Schema + +```json +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "version": { "type": "string", "const": "1.0" }, + "tokens": { + "type": "array", + "items": { + "type": "object", + "properties": { + "service": { + "type": "string", + "enum": ["GitLab", "Jira", "Confluence", "OrgMyLife"] + }, + "token_name": { + "type": "string", + "maxLength": 128, + "minLength": 1 + }, + "expiry_date": { + "type": ["string", "null"], + "pattern": "^\\d{4}-\\d{2}-\\d{2}$" + }, + "renewal_method": { + "type": "string", + "enum": ["auto", "manual"] + } + }, + "required": ["service", "token_name", "renewal_method"], + "additionalProperties": false + } + } + }, + "required": ["version", "tokens"] +} +``` + +### Example PAT Registry + +```json +{ + "version": "1.0", + "tokens": [ + { + "service": "GitLab", + "token_name": "ci-pipeline-token", + "expiry_date": "2025-08-15", + "renewal_method": "auto" + }, + { + "service": "Jira", + "token_name": "jira-api-access", + "expiry_date": "2025-07-20", + "renewal_method": "manual" + }, + { + "service": "Confluence", + "token_name": "confluence-bot", + "expiry_date": "2025-09-01", + "renewal_method": "manual" + }, + { + "service": "OrgMyLife", + "token_name": "orchestrator-api-key", + "expiry_date": null, + "renewal_method": "manual" + } + ] +} +``` + +### Check Result Data Model + +```python +@dataclass +class CheckResult: + entry: PatEntry + status: PatStatus # healthy | expiring soon | expired | check failed + days_remaining: int | None # None if check failed or no expiry date + error_message: str | None # Populated only for check_failed + stale_warning: bool = False # True if API succeeds but stored date has passed +``` + +### GitHub Actions Workflow Configuration + +```yaml +name: PAT Lifecycle Check + +on: + schedule: + - cron: '0 6 * * 1' # Every Monday at 06:00 UTC + workflow_dispatch: {} + +jobs: + check-pats: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install dependencies + run: pip install httpx + - name: Run PAT Manager + env: + GITLAB_PAT: ${{ secrets.GITLAB_PAT }} + JIRA_PAT: ${{ secrets.JIRA_PAT }} + CONFLUENCE_PAT: ${{ secrets.CONFLUENCE_PAT }} + ORGMYLIFE_API_KEY: ${{ secrets.ORGMYLIFE_API_SECRET }} + ORGMYLIFE_USER: ${{ secrets.ORGMYLIFE_USER }} + ORGMYLIFE_PASS: ${{ secrets.ORGMYLIFE_PASS }} + GH_TOKEN: ${{ secrets.PAT_MANAGER_TOKEN }} + ALERT_CHANNEL: orgmylife + run: python scripts/pat_manager.py +``` + +### Environment Variables / Secrets Mapping + +| Secret Name | Purpose | Used By | +|---|---|---| +| `GITLAB_PAT` | GitLab API access + rotation source | Checker, Rotator | +| `JIRA_PAT` | Jira authenticated check | Checker | +| `CONFLUENCE_PAT` | Confluence authenticated check | Checker | +| `ORGMYLIFE_API_SECRET` | OrgMyLife API key for task creation | Alerter | +| `ORGMYLIFE_USER` | OrgMyLife basic auth username | Alerter | +| `ORGMYLIFE_PASS` | OrgMyLife basic auth password | Alerter | +| `PAT_MANAGER_TOKEN` | GitHub API token for secret updates | SecretUpdater | + +## Correctness Properties + +*A property is a characteristic or behavior that should hold true across all valid executions of a system — essentially, a formal statement about what the system should do. Properties serve as the bridge between human-readable specifications and machine-verifiable correctness guarantees.* + +### Property 1: Registry serialization round-trip + +*For any* valid list of PAT entries, serializing the registry to JSON and deserializing it back SHALL produce an equivalent list of entries with identical field values. + +**Validates: Requirements 1.1** + +### Property 2: Validation accepts only valid entries + +*For any* dictionary representing a PAT entry, the validator SHALL accept it if and only if it contains all required fields (`service`, `token_name`, `expiry_date`, `renewal_method`), the `service` is one of the four supported values, the `token_name` is non-empty and at most 128 characters, the `expiry_date` is either null or a valid ISO 8601 date string, and the `renewal_method` matches the service (GitLab → "auto", others → "manual"). + +**Validates: Requirements 1.2, 1.4** + +### Property 3: Invalid entries and duplicates preserve registry state + +*For any* PAT registry and any entry that is either invalid (fails validation) or a duplicate (same service + token_name already exists), attempting to add that entry SHALL leave the registry unchanged — the entry count and all existing entries remain identical. + +**Validates: Requirements 1.3, 1.5** + +### Property 4: Expiry classification is deterministic and correct + +*For any* PAT entry with an expiry date and any reference date, the classification SHALL be: +- "expired" if `expiry_date < reference_date` +- "expiring soon" if `reference_date <= expiry_date <= reference_date + expiry_window` +- "healthy" if `expiry_date > reference_date + expiry_window` + +This holds regardless of whether the expiry date comes from the service API or the stored registry value. + +**Validates: Requirements 2.2, 2.3, 2.4, 6.5** + +### Property 5: Classification completeness + +*For any* PAT registry with N entries, after running the expiry check, the system SHALL produce exactly N classification results, each with exactly one status from the set {"healthy", "expiring soon", "expired", "check failed"}. + +**Validates: Requirements 2.7** + +### Property 6: Registry update after successful rotation + +*For any* GitLab PAT entry and any valid rotation response containing a new expiry date, after successful rotation the registry entry's `expiry_date` SHALL equal the new expiry date from the rotation response. + +**Validates: Requirements 3.3** + +### Property 7: Alert triggering correctness + +*For any* set of classified PATs: +- If ALL PATs have status "healthy", THEN zero alerts SHALL be sent +- For each manual-renewal PAT with status "expiring soon" or "expired", exactly one alert SHALL be triggered + +**Validates: Requirements 4.1, 5.4** + +### Property 8: Alert message completeness + +*For any* PAT entry requiring an alert: +- If the alert channel is OrgMyLife, the task title SHALL contain both the service name and token identifier +- If the alert channel is email, the email body SHALL contain the service name, token identifier, expiration date, and a renewal instructions URL + +**Validates: Requirements 4.2, 4.3** + +### Property 9: Days remaining calculation + +*For any* expiry date and reference date, the computed `days_remaining` SHALL equal `max(0, (expiry_date - reference_date).days)` — yielding 0 for already-expired tokens and the correct positive integer for future dates. + +**Validates: Requirements 4.4** + +### Property 10: Alert idempotence + +*For any* PAT that requires an alert, if an open OrgMyLife task already exists for the same service, token_name, and expiry_date, THEN no new task SHALL be created. Running the alert flow twice for the same PAT state SHALL produce at most one task. + +**Validates: Requirements 4.5** + +### Property 11: Summary report completeness + +*For any* list of check results, the generated summary report SHALL contain one line per result, and each line SHALL include the service name, token identifier, and classification status. + +**Validates: Requirements 5.3** + +### Property 12: Non-auth HTTP errors yield "check failed" + +*For any* HTTP response status code that is not 200-299, 401, or 403, the classification SHALL be "check failed". This includes 4xx errors (other than 401/403), 5xx errors, and timeout conditions. + +**Validates: Requirements 6.6** + +## Error Handling + +### Retry Strategy + +| Operation | Max Retries | Backoff | Fallback | +|---|---|---|---| +| Service API health check | 2 | 5s linear | Classify as "check failed" | +| GitLab token rotation | 1 | None | Manual alert flow | +| GitHub secret update | 0 | N/A | Alert + non-zero exit | +| OrgMyLife task creation | 1 | 2s | Log error + non-zero exit | +| Email sending | 1 | 2s | Log error + non-zero exit | + +### Error Classification + +```python +class PatManagerError(Exception): + """Base exception for PAT Manager errors.""" + pass + +class RegistryValidationError(PatManagerError): + """Raised when a registry entry fails validation.""" + def __init__(self, entry: dict, failed_fields: list[str]): ... + +class ServiceCheckError(PatManagerError): + """Raised when a service API check fails after retries.""" + def __init__(self, service: str, token_name: str, reason: str): ... + +class RotationError(PatManagerError): + """Raised when GitLab token rotation fails.""" + def __init__(self, token_name: str, http_status: int | None, reason: str): ... + +class SecretUpdateError(PatManagerError): + """Raised when GitHub secret update fails.""" + def __init__(self, secret_name: str, reason: str): ... + +class AlertError(PatManagerError): + """Raised when alert delivery fails.""" + def __init__(self, channel: str, reason: str): ... +``` + +### Exit Codes + +| Code | Meaning | +|---|---| +| 0 | All checks passed, all actions succeeded | +| 1 | One or more alerts could not be delivered | +| 2 | Registry validation failed (malformed registry file) | + +### Security Error Handling + +- Token values are masked via `::add-mask::{value}` before any operation that might log output +- On rotation failure, the old token remains valid — no service disruption +- On secret update failure, the new token exists in GitLab but the workflow still uses the old one — alert is critical +- All HTTP errors are logged without including request/response bodies that might contain tokens + +## Testing Strategy + +### Property-Based Tests (Hypothesis) + +The project uses Python 3.11+ with pytest. Property-based tests will use the **Hypothesis** library, which is the standard PBT framework for Python. + +**Configuration:** +- Minimum 100 examples per property test (`@settings(max_examples=100)`) +- Each test tagged with a comment referencing the design property +- Tag format: `# Feature: pat-renewal, Property {number}: {property_text}` + +**Properties to implement:** +1. Registry round-trip serialization +2. Validation correctness (accepts valid, rejects invalid) +3. Invalid/duplicate entries preserve state +4. Expiry classification determinism +5. Classification completeness +6. Registry update after rotation +7. Alert triggering correctness +8. Alert message completeness +9. Days remaining calculation +10. Alert idempotence +11. Summary report completeness +12. Non-auth HTTP errors → "check failed" + +**Generators needed:** +- `pat_entries()` — generates valid PatEntry instances with random services, token names, dates +- `invalid_entries()` — generates dictionaries missing fields or with invalid values +- `expiry_dates(relative_to)` — generates dates in various ranges (past, within window, beyond window) +- `http_status_codes()` — generates non-success, non-auth status codes +- `check_results()` — generates lists of CheckResult with various statuses + +### Unit Tests (pytest) + +Example-based tests for specific scenarios: +- Empty registry file creation (Req 1.6) +- Successful rotation logging format (Req 3.6) +- Expired GitLab PAT skips rotation (Req 3.7) +- Stale expiry_date warning (Req 6.7) +- Token masking with `::add-mask::` (Req 7.2) + +### Integration Tests (pytest + httpx mock) + +Mock-based tests for external service interactions: +- GitLab API expiry check with mocked response (Req 6.1) +- Jira/Confluence 401 interpretation (Req 6.2, 6.3) +- OrgMyLife 401/403 interpretation (Req 6.4) +- GitLab rotation API call + retry on failure (Req 3.1, 3.4) +- GitHub secret update after rotation (Req 3.2) +- Rotation success + secret update failure → alert (Req 3.5) +- Alert channel unreachable → non-zero exit (Req 4.6) +- Service timeout after retries → "check failed" (Req 2.5) +- Null expiry_date classification (Req 2.6, 6.8) + +### Smoke Tests + +- Workflow YAML contains correct cron schedule (Req 5.1) +- Workflow YAML contains workflow_dispatch trigger (Req 5.2) +- All API URLs use HTTPS scheme (Req 7.3) +- No token values written to non-secret locations (Req 7.1) + +### Test Dependencies + +``` +hypothesis>=6.100 +pytest>=8.0 +pytest-asyncio>=0.23 +pytest-cov>=5.0 +respx>=0.21 # httpx mock library +``` + diff --git a/.kiro/specs/pat-renewal/requirements.md b/.kiro/specs/pat-renewal/requirements.md new file mode 100644 index 0000000..ecf4b92 --- /dev/null +++ b/.kiro/specs/pat-renewal/requirements.md @@ -0,0 +1,117 @@ +# Requirements Document + +## Introduction + +This feature provides automated Personal Access Token (PAT) lifecycle management across all services used in the workspace: GitLab, Jira, Confluence, and OrgMyLife. The system tracks PAT expiration dates, auto-renews tokens where the service API supports it, and alerts the user via OrgMyLife task creation or email when manual intervention is required. + +**Recommended Architecture:** A GitHub Actions scheduled workflow (runs weekly) that: +- Checks PAT expiry via each service's API +- Auto-rotates GitLab PATs (GitLab API supports this natively) +- Creates OrgMyLife tasks or sends email alerts for tokens that cannot be auto-renewed (Jira, Confluence) +- Stores a PAT registry (JSON) tracking token names, services, and expiration dates + +This approach keeps the logic centralized, runs on schedule without needing the AI-Orchestrator to be online, and integrates with existing GitHub Actions infrastructure already in use. + +## Glossary + +- **PAT_Manager**: The automated system (GitHub Actions workflow) responsible for checking, renewing, and alerting about PAT status +- **PAT_Registry**: A JSON configuration file that stores metadata about all managed PATs (service, token name, expiration date, renewal method) +- **Token_Expiry_Window**: The number of days before expiration at which the PAT_Manager begins renewal or alerting (default: 14 days) +- **Auto_Renewable_Token**: A PAT whose service API supports programmatic rotation (e.g., GitLab PATs via the Personal Access Tokens API) +- **Manual_Renewal_Token**: A PAT whose service does not support programmatic rotation and requires human intervention (e.g., Jira, Confluence) +- **Alert_Channel**: The notification destination for expiry warnings — either OrgMyLife task creation or email + +## Requirements + +### Requirement 1: PAT Registry Management + +**User Story:** As a developer, I want a central registry of all my PATs with their metadata, so that I have a single source of truth for token lifecycle information. + +#### Acceptance Criteria + +1. THE PAT_Manager SHALL maintain a PAT_Registry JSON file containing the service name, token identifier (maximum 128 characters), expiration date in ISO 8601 format (YYYY-MM-DD), and renewal method (value of "auto" or "manual") for each managed PAT +2. WHEN a new PAT is added to the PAT_Registry, THE PAT_Manager SHALL validate that the entry contains all required fields (service, token_name, expiry_date in ISO 8601 format, renewal_method) and that the service value matches one of the supported services +3. IF a PAT_Registry entry contains invalid or missing fields, THEN THE PAT_Manager SHALL reject the entry, leave the PAT_Registry unchanged, and output a validation error to the workflow log identifying the malformed entry and which fields failed validation +4. THE PAT_Manager SHALL accept only the following service values in the PAT_Registry: GitLab (renewal_method: "auto"), Jira (renewal_method: "manual"), Confluence (renewal_method: "manual"), and OrgMyLife (renewal_method: "manual") +5. IF a PAT entry is added with a service and token_name combination that already exists in the PAT_Registry, THEN THE PAT_Manager SHALL reject the duplicate entry and output a validation error to the workflow log indicating the conflict +6. WHEN the PAT_Registry file does not exist at workflow execution time, THE PAT_Manager SHALL create an empty PAT_Registry file with a valid JSON structure + +### Requirement 2: PAT Expiry Checking + +**User Story:** As a developer, I want the system to check whether any of my PATs are expired or approaching expiration, so that I can take action before services break. + +#### Acceptance Criteria + +1. WHEN the scheduled check runs, THE PAT_Manager SHALL query each service API to determine the current expiration status of each registered PAT, applying a timeout of 30 seconds per service call +2. WHEN a PAT expiration date is within the Token_Expiry_Window (default: 14 days), THE PAT_Manager SHALL classify the PAT as "expiring soon" +3. WHEN a PAT expiration date has passed, THE PAT_Manager SHALL classify the PAT as "expired" +4. WHEN a PAT is not expired and its expiration date is beyond the Token_Expiry_Window, THE PAT_Manager SHALL classify the PAT as "healthy" +5. IF a service API call fails due to connection timeout, DNS resolution failure, or HTTP 5xx response after 2 retry attempts, THEN THE PAT_Manager SHALL log the failure reason and classify the PAT as "check failed" +6. IF a PAT_Registry entry has no expiration date configured, THEN THE PAT_Manager SHALL classify the PAT as "healthy" provided the authenticated API call succeeds, or "expired" if the API call returns an authentication failure +7. WHEN the expiry check completes, THE PAT_Manager SHALL produce a classification result for every PAT in the PAT_Registry such that each PAT is assigned exactly one status: "healthy", "expiring soon", "expired", or "check failed" + +### Requirement 3: Automatic PAT Renewal (GitLab) + +**User Story:** As a developer, I want GitLab PATs to be automatically rotated before they expire, so that my CI/CD pipelines and scripts continue working without manual intervention. + +#### Acceptance Criteria + +1. WHEN a GitLab PAT is classified as "expiring soon", THE PAT_Manager SHALL call the GitLab Personal Access Tokens API (`POST /personal_access_tokens/{id}/rotate`) to rotate the token +2. WHEN the GitLab API returns a new token value after rotation, THE PAT_Manager SHALL update the stored token in GitHub Actions secrets using the GitHub API within 30 seconds of receiving the new value +3. WHEN the GitLab API returns a new token value after rotation, THE PAT_Manager SHALL update the PAT_Registry entry with the new expiration date as returned by the GitLab API response +4. IF the GitLab token rotation API call fails with an HTTP error status, a network timeout after 30 seconds, or a connection error, THEN THE PAT_Manager SHALL retry the rotation once and, if the retry also fails, fall back to the manual renewal alert flow +5. IF the GitLab token rotation succeeds but the subsequent GitHub Actions secret update fails, THEN THE PAT_Manager SHALL immediately trigger the manual renewal alert flow indicating that the token was rotated but the secret could not be updated +6. WHEN a GitLab PAT is successfully rotated and the secret is updated, THE PAT_Manager SHALL log the rotation event with the old and new expiration dates +7. IF a GitLab PAT is classified as "expired", THEN THE PAT_Manager SHALL skip automatic rotation and fall back to the manual renewal alert flow + +### Requirement 4: Manual Renewal Alerting + +**User Story:** As a developer, I want to be alerted when PATs that cannot be auto-renewed are approaching expiration, so that I can manually renew them in time. + +#### Acceptance Criteria + +1. WHEN a Manual_Renewal_Token has 14 or fewer days until expiration or is already expired, THE PAT_Manager SHALL send an alert via the configured Alert_Channel +2. IF the configured Alert_Channel is OrgMyLife, THEN THE PAT_Manager SHALL create a task with the title containing the service name and token identifier +3. IF the configured Alert_Channel is email, THEN THE PAT_Manager SHALL send an email containing the service name, token identifier, expiration date, and renewal instructions URL +4. THE PAT_Manager SHALL include the remaining days until expiration (or indicate already expired with 0 days remaining) in each alert message +5. THE PAT_Manager SHALL not create a new OrgMyLife task for a PAT if an open task for the same PAT and the same expiration date already exists +6. IF the configured Alert_Channel is unreachable or returns an error, THEN THE PAT_Manager SHALL log the failure and exit the workflow run with a non-zero exit code + +### Requirement 5: Scheduled Execution + +**User Story:** As a developer, I want the PAT checks to run automatically on a weekly schedule, so that I do not need to remember to check token status manually. + +#### Acceptance Criteria + +1. THE PAT_Manager SHALL execute as a GitHub Actions workflow on a weekly cron schedule, running once every Monday at 06:00 UTC +2. THE PAT_Manager SHALL support manual triggering via workflow_dispatch, performing the same check-and-report sequence as the scheduled run +3. WHEN the workflow completes (whether triggered by schedule or workflow_dispatch), THE PAT_Manager SHALL produce a summary report in the GitHub Actions run log listing each managed PAT with its service name, token identifier, and classification status (healthy, expiring soon, expired, or check failed) +4. IF all PATs are classified as "healthy", THEN THE PAT_Manager SHALL complete without creating OrgMyLife tasks or sending email alerts, while still logging the summary report +5. IF one or more PATs are classified as "expiring soon", "expired", or "check failed", THEN THE PAT_Manager SHALL proceed with the applicable renewal or alerting flow and still produce the summary report + +### Requirement 6: Service-Specific Expiry Detection + +**User Story:** As a developer, I want the system to correctly detect expiry for each service type, so that checks work regardless of how each platform reports token status. + +#### Acceptance Criteria + +1. WHEN checking a GitLab PAT, THE PAT_Manager SHALL use the GitLab Personal Access Tokens API (`GET /personal_access_tokens`) to retrieve the `expires_at` field and compare it against the current date and Token_Expiry_Window to classify the PAT as "healthy", "expiring soon", or "expired" +2. WHEN checking a Jira PAT, THE PAT_Manager SHALL attempt an authenticated API call within a timeout of 30 seconds and interpret HTTP 401 as an expired or revoked token +3. WHEN checking a Confluence PAT, THE PAT_Manager SHALL attempt an authenticated API call within a timeout of 30 seconds and interpret HTTP 401 as an expired or revoked token +4. WHEN checking an OrgMyLife token, THE PAT_Manager SHALL attempt an authenticated API call within a timeout of 30 seconds and interpret HTTP 401 or HTTP 403 as an expired or revoked token +5. WHEN a service does not expose an expiry date via API and the authenticated call succeeds, THE PAT_Manager SHALL compare the expiry_date stored in the PAT_Registry against the current date and Token_Expiry_Window to classify the PAT +6. IF a service API returns an HTTP error other than 401 or 403 or the request times out, THEN THE PAT_Manager SHALL classify the PAT as "check failed" and log an error message indicating the service name, token identifier, and the response status or timeout condition +7. IF the authenticated API call succeeds but the expiry_date stored in the PAT_Registry has passed, THEN THE PAT_Manager SHALL classify the PAT as "expired" based on the stored date and include a warning that the stored expiry_date may be stale +8. IF a Manual_Renewal_Token has no expiry_date in the PAT_Registry, THEN THE PAT_Manager SHALL classify the PAT based solely on the authenticated API call result — "healthy" if the call succeeds, or "expired" if the call returns 401 or 403 + +### Requirement 7: Security + +**User Story:** As a developer, I want PAT values to remain secure throughout the renewal process, so that tokens are not exposed in logs or artifacts. + +#### Acceptance Criteria + +1. THE PAT_Manager SHALL store PAT values exclusively in GitHub Actions secrets or encrypted environment variables and SHALL NOT write token values to workflow artifacts, step summaries, or PR comments +2. THE PAT_Manager SHALL mask token values by registering them with the workflow log masking mechanism so that any occurrence in stdout, stderr, or workflow annotations is replaced with `***` +3. THE PAT_Manager SHALL transmit PAT values only over HTTPS connections +4. IF a token rotation produces a new value, THEN THE PAT_Manager SHALL update the GitHub Actions secret using the GitHub API with a token that has the `admin:org` or `repo` scope required for secrets write access +5. IF the GitHub API secret update fails, THEN THE PAT_Manager SHALL retain the previous secret value unchanged, report the failure in the workflow log without exposing the new token value, and exit the workflow run with a non-success status diff --git a/.kiro/specs/pat-renewal/tasks.md b/.kiro/specs/pat-renewal/tasks.md new file mode 100644 index 0000000..969388d --- /dev/null +++ b/.kiro/specs/pat-renewal/tasks.md @@ -0,0 +1,204 @@ +# Implementation Plan: PAT Renewal + +## Overview + +Implement an automated PAT lifecycle management system as a Python script (`scripts/pat_manager.py`) that runs via GitHub Actions. The implementation follows a modular architecture with 7 components: Registry, Expiry Checker, GitLab Rotator, Secret Updater, Alert Sender, Summary Reporter, and Main Orchestrator. All modules use Python 3.11+, httpx for HTTP, and pytest with Hypothesis for property-based testing. + +## Tasks + +- [x] 1. Set up project structure and core data models + - [x] 1.1 Create directory structure and dependencies + - Create `scripts/pat_manager/` package directory with `__init__.py` + - Create `scripts/pat_manager/models.py` with `PatEntry` dataclass, `PatStatus` enum, `CheckResult` dataclass, `RotationResult` dataclass, `AlertResult` dataclass, and `AlertChannel` enum + - Create `scripts/pat_manager/errors.py` with all custom exception classes (`PatManagerError`, `RegistryValidationError`, `ServiceCheckError`, `RotationError`, `SecretUpdateError`, `AlertError`) + - Add `requirements.txt` with `httpx>=0.27` and `requirements-dev.txt` with `hypothesis>=6.100`, `pytest>=8.0`, `pytest-asyncio>=0.23`, `pytest-cov>=5.0`, `respx>=0.21` + - _Requirements: 1.1, 1.4_ + + - [x] 1.2 Create PAT Registry JSON schema and example file + - Create `pat-registry.json` with the JSON schema structure (version "1.0", empty tokens array) + - Create `pat-registry.example.json` with example entries for all four services + - _Requirements: 1.1, 1.6_ + +- [x] 2. Implement PAT Registry module + - [x] 2.1 Implement registry loading and validation (`scripts/pat_manager/registry.py`) + - Implement `PatRegistry` class with `load(path)` method that reads and parses JSON + - Implement `validate_entry(entry)` that checks all required fields, service enum values, token_name length (max 128 chars), expiry_date ISO 8601 format, and renewal_method matching service + - Implement duplicate detection for `(service, token_name)` combinations + - Implement `create_empty(path)` for creating a new empty registry file + - Implement `save(path, entries)` for persisting registry changes + - Raise `RegistryValidationError` with specific failed fields on invalid entries + - _Requirements: 1.1, 1.2, 1.3, 1.4, 1.5, 1.6_ + + - [x] 2.2 Write property test: Registry serialization round-trip + - **Property 1: Registry serialization round-trip** + - **Validates: Requirements 1.1** + - Use Hypothesis to generate valid `PatEntry` lists and verify `save` → `load` produces equivalent entries + + - [x] 2.3 Write property test: Validation accepts only valid entries + - **Property 2: Validation accepts only valid entries** + - **Validates: Requirements 1.2, 1.4** + - Use Hypothesis to generate both valid and invalid entry dicts, verify validator accepts/rejects correctly + + - [x] 2.4 Write property test: Invalid entries and duplicates preserve registry state + - **Property 3: Invalid entries and duplicates preserve registry state** + - **Validates: Requirements 1.3, 1.5** + - Use Hypothesis to generate registries and invalid/duplicate entries, verify registry unchanged after rejected add + +- [x] 3. Implement Expiry Checker module + - [x] 3.1 Implement expiry classification logic (`scripts/pat_manager/checker.py`) + - Implement `ExpiryChecker` class with configurable `expiry_window_days` (default 14), `timeout` (default 30s), and `max_retries` (default 2) + - Implement `classify_by_date(expiry_date, reference_date, window)` pure function for date-based classification + - Implement `check_all(entries, secrets)` async method that iterates all entries and returns `CheckResult` list + - _Requirements: 2.2, 2.3, 2.4, 2.7_ + + - [x] 3.2 Implement service-specific check methods + - Implement `check_gitlab(entry, token)` — calls `GET /personal_access_tokens`, reads `expires_at`, classifies + - Implement `check_jira(entry, token)` — authenticated call, HTTP 401 = expired + - Implement `check_confluence(entry, token)` — authenticated call, HTTP 401 = expired + - Implement `check_orgmylife(entry, token)` — authenticated call, HTTP 401/403 = expired + - Handle timeout (30s), retries (2 attempts with 5s backoff), and fallback to "check failed" + - Handle null `expiry_date` entries: classify based on API call result only + - Handle stale `expiry_date` warning when API succeeds but stored date has passed + - _Requirements: 2.1, 2.5, 2.6, 6.1, 6.2, 6.3, 6.4, 6.5, 6.6, 6.7, 6.8_ + + - [x] 3.3 Write property test: Expiry classification is deterministic and correct + - **Property 4: Expiry classification is deterministic and correct** + - **Validates: Requirements 2.2, 2.3, 2.4, 6.5** + - Use Hypothesis to generate expiry dates and reference dates, verify classification matches expected rules + + - [x] 3.4 Write property test: Classification completeness + - **Property 5: Classification completeness** + - **Validates: Requirements 2.7** + - Use Hypothesis to generate N-entry registries, verify exactly N results each with exactly one valid status + + - [x] 3.5 Write property test: Non-auth HTTP errors yield "check failed" + - **Property 12: Non-auth HTTP errors yield "check failed"** + - **Validates: Requirements 6.6** + - Use Hypothesis to generate HTTP status codes not in {200-299, 401, 403}, verify classification is "check failed" + +- [x] 4. Checkpoint - Ensure all tests pass + - Ensure all tests pass, ask the user if questions arise. + +- [x] 5. Implement GitLab Rotator module + - [x] 5.1 Implement GitLab token rotation (`scripts/pat_manager/rotator.py`) + - Implement `GitLabRotator` class with `rotate(token_id, current_token)` async method + - Call `POST /personal_access_tokens/{id}/rotate` with the current token + - Return `RotationResult` with new expiry date on success + - Implement retry logic: retry once on failure, then return error result + - Skip rotation for expired tokens (return error indicating manual intervention needed) + - _Requirements: 3.1, 3.4, 3.7_ + + - [x] 5.2 Write property test: Registry update after successful rotation + - **Property 6: Registry update after successful rotation** + - **Validates: Requirements 3.3** + - Use Hypothesis to generate rotation responses with new expiry dates, verify registry entry updated correctly + +- [x] 6. Implement Secret Updater module + - [x] 6.1 Implement GitHub secret update (`scripts/pat_manager/secret_updater.py`) + - Implement `SecretUpdater` class with `update_secret(secret_name, secret_value)` async method + - Use GitHub API to encrypt and update repository secrets (libsodium sealed box encryption) + - Handle failure: retain previous secret value, report error without exposing token + - _Requirements: 3.2, 3.5, 7.1, 7.4, 7.5_ + +- [x] 7. Implement Alert Sender module + - [x] 7.1 Implement alert sending (`scripts/pat_manager/alerter.py`) + - Implement `AlertSender` class with configurable `AlertChannel` (OrgMyLife or email) + - Implement `send_alert(result)` that creates OrgMyLife task or sends email based on channel + - OrgMyLife task title format: `[PAT Renewal] {service} - {token_name}` + - Include days remaining (0 for expired) in alert message + - Implement `check_existing_task(service, token_name, expiry_date)` for idempotent alerting + - Handle alert channel unreachable: log failure, signal non-zero exit + - Implement retry logic: 1 retry with 2s backoff + - _Requirements: 4.1, 4.2, 4.3, 4.4, 4.5, 4.6_ + + - [x] 7.2 Write property test: Alert triggering correctness + - **Property 7: Alert triggering correctness** + - **Validates: Requirements 4.1, 5.4** + - Use Hypothesis to generate classified PAT sets, verify zero alerts when all healthy, one alert per expiring/expired manual token + + - [x] 7.3 Write property test: Alert message completeness + - **Property 8: Alert message completeness** + - **Validates: Requirements 4.2, 4.3** + - Use Hypothesis to generate PAT entries requiring alerts, verify OrgMyLife title contains service+token, email contains all required fields + + - [x] 7.4 Write property test: Days remaining calculation + - **Property 9: Days remaining calculation** + - **Validates: Requirements 4.4** + - Use Hypothesis to generate expiry/reference date pairs, verify `days_remaining == max(0, (expiry - ref).days)` + + - [x] 7.5 Write property test: Alert idempotence + - **Property 10: Alert idempotence** + - **Validates: Requirements 4.5** + - Use Hypothesis to generate PAT alert scenarios, verify no duplicate task created when open task exists + +- [x] 8. Implement Summary Reporter module + - [x] 8.1 Implement summary report generation (`scripts/pat_manager/reporter.py`) + - Implement `SummaryReporter` class with `report(results, rotations)` method + - Generate one line per PAT with service name, token identifier, and classification status + - Include rotation results (old/new expiry dates) for rotated tokens + - _Requirements: 5.3_ + + - [x] 8.2 Write property test: Summary report completeness + - **Property 11: Summary report completeness** + - **Validates: Requirements 5.3** + - Use Hypothesis to generate check result lists, verify report contains one line per result with service, token_name, and status + +- [x] 9. Checkpoint - Ensure all tests pass + - Ensure all tests pass, ask the user if questions arise. + +- [x] 10. Implement Main Orchestrator and GitHub Actions workflow + - [x] 10.1 Implement main orchestrator (`scripts/pat_manager/__main__.py`) + - Implement `main()` async function that coordinates all modules + - Load registry → check all → rotate GitLab tokens → update secrets → send alerts → report summary + - Mask all token values with `::add-mask::{value}` before any operations + - Handle rotation success + secret update failure → trigger alert flow + - Return exit code: 0 (success), 1 (alert delivery failure), 2 (registry validation failure) + - Log rotation events with old and new expiration dates (Req 3.6) + - _Requirements: 3.5, 3.6, 5.4, 5.5, 7.1, 7.2_ + + - [x] 10.2 Create GitHub Actions workflow file + - Create `.github/workflows/pat-check.yml` with cron schedule `0 6 * * 1` (Monday 06:00 UTC) + - Add `workflow_dispatch` trigger + - Configure Python 3.11 setup, dependency installation, and script execution + - Map all required secrets as environment variables (GITLAB_PAT, JIRA_PAT, CONFLUENCE_PAT, ORGMYLIFE_API_SECRET, ORGMYLIFE_USER, ORGMYLIFE_PASS, PAT_MANAGER_TOKEN, ALERT_CHANNEL) + - Ensure all API URLs use HTTPS + - _Requirements: 5.1, 5.2, 7.3_ + + - [x] 10.3 Write integration tests for orchestrator flow + - Test full happy path: all healthy → no alerts, summary only + - Test GitLab rotation flow with mocked APIs (respx) + - Test rotation success + secret update failure → alert triggered + - Test alert channel unreachable → non-zero exit code + - Test service timeout after retries → "check failed" classification + - Test null expiry_date classification scenarios + - _Requirements: 2.5, 2.6, 3.2, 3.4, 3.5, 4.6, 5.4, 5.5, 6.8_ + +- [x] 11. Final checkpoint - Ensure all tests pass + - Ensure all tests pass, ask the user if questions arise. + +## Notes + +- Tasks marked with `*` are optional and can be skipped for faster MVP +- Each task references specific requirements for traceability +- Checkpoints ensure incremental validation +- Property tests validate universal correctness properties from the design document using Hypothesis +- Unit/integration tests validate specific examples and edge cases using pytest + respx +- The implementation uses Python 3.11+, httpx for HTTP calls, and async/await throughout +- All token values must be masked in logs via GitHub Actions `::add-mask::` mechanism + +## Task Dependency Graph + +```json +{ + "waves": [ + { "id": 0, "tasks": ["1.1", "1.2"] }, + { "id": 1, "tasks": ["2.1"] }, + { "id": 2, "tasks": ["2.2", "2.3", "2.4", "3.1"] }, + { "id": 3, "tasks": ["3.2", "3.3", "3.4", "3.5"] }, + { "id": 4, "tasks": ["5.1", "6.1", "7.1", "8.1"] }, + { "id": 5, "tasks": ["5.2", "7.2", "7.3", "7.4", "7.5", "8.2"] }, + { "id": 6, "tasks": ["10.1", "10.2"] }, + { "id": 7, "tasks": ["10.3"] } + ] +} +``` diff --git a/.kiro/steering/mcp-servers.md b/.kiro/steering/mcp-servers.md new file mode 100644 index 0000000..96bb228 --- /dev/null +++ b/.kiro/steering/mcp-servers.md @@ -0,0 +1,71 @@ +--- +inclusion: auto +--- + +# MCP Server Configuration + +## Available MCP Servers (require VPN) + +These MCP servers are hosted on DB's OpenShift cluster (dbcs-prag) and available when connected to the corporate network: + +### Jira + Confluence (Atlassian MCP — NEUER kombinierter Server) +- URL: `https://taros-playground-mcp-atlassian.apps.dbcs-prag.comp.db.de/mcp` +- Capabilities: 72+ Tools — Jira issues, transitions, comments + Confluence pages (read/write) +- Headers required: + - `Authorization: Bearer ` + - `X-Jira-Host: arija-jira.jaas.service.deutschebahn.com` (ohne https://) + - `X-Jira-Projects: O2CO2C,O2CAAPN,*` (komma-separiert, Wildcards möglich) + - `X-Jira-Mode: read` (oder `write`) +- Replaces: mcp-jira-v2 (deprecated) and mcp-confluence (separate) +- Use for: Team velocity, sprint data, bug tracking, Confluence pages + +### Jira MCP v2 (DEPRECATED) +- URL: `https://taros-playground-mcp-jira-v2.apps.dbcs-prag.comp.db.de/mcp` +- Status: DEPRECATED — use mcp-atlassian instead + +### Confluence MCP (standalone, still works) +- URL: `https://taros-playground-mcp-confluence.apps.dbcs-prag.comp.db.de/mcp` +- Still functional but mcp-atlassian is the preferred replacement + +### DB Context (dbctx) +- URL: `https://taros-playground-dbctx.apps.dbcs-prag.comp.db.de/mcp` +- Capabilities: DB-specific context and tooling +- Use for: General DB knowledge, terminology, organizational context + +### Teamlandkarte MCP (local, stdio) +- Repo: https://git.tech.rz.db.de/ThomasHandke/teamlandkarte-mcp +- Type: Local stdio server (Python, requires uv + Data Lake credentials) +- Capabilities: Match tasks to employees/teams by competences, browse free capacities, team profiles +- Use for: Finding teams (INKA, Data Science), capacity analysis, competence mapping +- Requires: DATA_LAKE_USERNAME, DATA_LAKE_PASSWORD, AZURE_OPENAI_EMBEDDING_API_KEY +- NOT yet configured (needs credentials) + +### BEAM MCP (local, stdio) — EDUCATIONAL USE ONLY +- Repo: https://git.tech.rz.db.de/beta/ai/mcp/beam-mcp +- Type: Local stdio server (Node.js, requires Edge browser for login) +- Capabilities: Enterprise Architecture (LeanIX/BEAM) — applications, domains, components, diagrams +- Use for: System landscape, application dependencies, architecture diagrams +- ⚠️ ACTIVE USE NOT PERMITTED due to data protection concerns. Educational reference only. + +## Other Noteworthy Servers (not yet configured) + +| Server | Use Case | Priority | +|--------|----------|----------| +| DataLake MCP | SQL on Trino (quantitative data) | Medium — when we need metrics | +| Foundations MCP | ADRs, Standards, Capabilities | Medium — for architecture comparison | +| gitlab-mcp | GitLab integration via glab | Low — we use direct API calls | +| sonarqube-mcp | Code quality metrics | Low — for team quality comparison | +| proclib-mcp | Process library access | Medium — for process analysis | +| mcp-agilehive | AgileHive SAFe tool | High — PI-Planning data, team capacity | + +## Fallback (without VPN) + +When MCP servers are unreachable, use direct REST API calls: +- Confluence: Bearer token auth against `https://arija-confluence.jaas.service.deutschebahn.com/rest/api` +- GitLab: PRIVATE-TOKEN header against `https://git.tech.rz.db.de/api/v4` +- Jira: Not available without VPN + +## Local MCP Server + +- OrgMyLife: Running locally, accessible via Kiro's MCP integration +- Provides: tasks, calls, digest, projects board diff --git a/.kiro/steering/session-context.md b/.kiro/steering/session-context.md new file mode 100644 index 0000000..d981d82 --- /dev/null +++ b/.kiro/steering/session-context.md @@ -0,0 +1,100 @@ +--- +inclusion: auto +--- + +# Session Context — Next Session Priorities + +## Immediate Tasks + +1. **Verify Jury-Voting** — `unikat.andreknie.de` is live and working +2. **Verify NoteGraph browse** — `notes.andreknie.de/browse?token=...` (clear SB service worker first) +3. **KIQ Docker deploy** — Docker not set up on KIQ VPS yet. Need someone with root to install Docker or find alternative +4. **Rechnungen prüfen** — 8 receipts labeled in OrgMyLife, forward to rechnungen@d-hive.de + +## Key Technical Notes + +### Git on this machine +- Git path: `C:\Users\AndreKnie\AppData\Local\GitHubDesktop\app-3.5.8\resources\app\git\cmd\git.exe` +- Always use: `$gitExe = "C:\Users\AndreKnie\AppData\Local\GitHubDesktop\app-3.5.8\resources\app\git\cmd\git.exe"` then `& $gitExe ` +- `gh` CLI is NOT available + +### OrgMyLife API +- Endpoint: `https://api.andreknie.de` +- Auth: `Authorization: Bearer !Z5l5vUcm!&qfhUuE=Nf` +- Projects board: `GET/PUT /api/projects`, `POST /api/projects/create` +- Tasks: `GET /api/tasks`, agent-ready filter: `?filter_agent=true` + +### NoteGraph +- SilverBullet: `https://notes.andreknie.de` (now serves browse UI as primary) +- Browse: `https://notes.andreknie.de/browse?token=PASSWORD` +- Upload: `https://notes.andreknie.de/upload?token=PASSWORD` +- Ingestion uses Mistral (`mistral-small-latest`) for enrichment and Vision OCR +- Repo: `NoteGraph/` with `ingestion/` Python package +- Docker on VPS: Ingestion (8001), SilverBullet (3000, internal only) +- **When creating notes for the user, always add `AI-Source` to the tags array in frontmatter** + +### Jury-Voting App +- Repo: `Jury-Voting/` (private, GitHub: DoctoDre/Jury-Voting) +- URL: `https://unikat.andreknie.de` (password: `UNIKAT_jury`) +- Stack: React 19 + Vite frontend, Express.js backend, JSON file persistence, SSE real-time +- Port: 3002 on VPS, systemd service `jury-voting` +- Caddy config: in `NoteGraph/Caddyfile` (block for `unikat.andreknie.de`) +- Event: 15. UNIKAT Ideenwettbewerb, 19. Mai 2026, Science Park Kassel +- 6 jurors, 11 teams, 5 criteria (1-10 scale) +- Admin: `/admin`, Results/Beamer: `/results` +- Future: move to `unikat.d-hive.de` (add DNS + Caddy block on KIQ VPS) + +### Repos in workspace +- `OrgMyLife/` — Personal task/coordination system (FastAPI + PostgreSQL) +- `AI-Orchestrator/` — Agent orchestration (Python) +- `NoteGraph/` — Knowledge base (SilverBullet + ingestion pipeline) +- `Confluence_Bot/` — Confluence automation (PowerShell) +- `Projekt-KIQ-HP/` — KIQ homepage (React/Vite, currently uses Supabase) +- `Jury-Voting/` — UNIKAT Jury Voting (React/Vite + Express) +- `project-audit/` — DB enterprise project audit data (GitLab) + +### Deploy pattern +- Push to main → GitHub Actions → SSH to VPS → deploy +- Secrets via GitHub repo secrets +- Caddy reverse proxy managed via `NoteGraph/Caddyfile` + +### Deploy pattern for private repos (no git on VPS) +- Build in GitHub Actions (checkout → npm ci → npm run build) +- Create tar.gz archive of built artifacts +- Upload via `appleboy/scp-action` to VPS `/tmp/` +- Extract via `appleboy/ssh-action`, install deps, restart systemd service +- This avoids needing git credentials on the VPS + +### VPS (217.160.174.2) +- Caddy handles HTTPS for all domains (auto Let's Encrypt) +- OrgMyLife on port 8000 (`api.andreknie.de`) +- SilverBullet on port 3000 (`notes.andreknie.de`) +- Ingestion on port 8001 (`notes.andreknie.de/upload`) +- Jury-Voting on port 3002 (`unikat.andreknie.de`) +- SSH: root@217.160.174.2, key stored as `DEPLOY_SSH_KEY` in GitHub repos +- Node.js 20 installed via nodesource +- No rsync installed — use tar+scp for file transfer +- No docker on this VPS for Jury-Voting (runs as systemd service with tsx) + +### Projekt-KIQ-HP (separate VPS: projekt-kiq.d-hive.de) +- Deploy user: `github-deploy` +- Static site deployed via rsync to `/opt/projekt-kiq/site` +- Caddy config at `/etc/caddy/Caddyfile` (managed by KIQ deploy workflow) +- Has rsync available +- SSH key: `DEPLOY_SSH_KEY` in KIQ GitHub repo + +### Express 5 Gotchas +- Do NOT pass `app` to functions typed as `Express` — route registration silently fails at runtime +- Use `Router()` and `app.use(router)` pattern instead +- The `Express` type from `@types/express` doesn't match Express 5 runtime correctly +- SPA catch-all: use middleware (`app.use`) not `app.get('/{*splat}')` — the splat syntax can interfere + +### DB GitLab (project-audit) +- Requires `scm-info.yaml` in repo root for security scans +- Format: `version: v3`, `license: LicenseRef-DBISL`, `contacts:`, `confidentiality: internal`, `reference-ids: none` +- GitLab URL: `git.tech.rz.db.de/AndreKnie/project-audit` + +### DNS +- `andreknie.de` DNS managed externally (takes ~5 min to propagate new subdomains) +- Add A record pointing to 217.160.174.2 for new subdomains +- Caddy auto-provisions HTTPS certificates once DNS resolves diff --git a/.kiro/steering/workspace-context.md b/.kiro/steering/workspace-context.md new file mode 100644 index 0000000..65989b0 --- /dev/null +++ b/.kiro/steering/workspace-context.md @@ -0,0 +1,108 @@ +--- +inclusion: auto +--- + +# Workspace Context & Conventions + +## Git Access + +- Git is NOT in the system PATH on this machine. +- Use GitHub Desktop's bundled git at: `C:\Users\AndreKnie\AppData\Local\GitHubDesktop\app-3.5.8\resources\app\git\cmd\git.exe` +- Private GitHub repos (e.g. DoctoDre/CV) are accessible via the locally configured GitHub credentials — no separate PAT needed. +- For git commands in shell, always use the full path: `& "C:\Users\AndreKnie\AppData\Local\GitHubDesktop\app-3.5.8\resources\app\git\cmd\git.exe" ` +- DB GitLab: `https://git.tech.rz.db.de` — PAT auth via PRIVATE-TOKEN header or inline URL for push. + +## Repo Landscape + +All repos live under: `c:\Users\AndreKnie\OneDrive - Deutsche Bahn\Coden\Orchestrator\` + +| Repo | Language | Purpose | Status | +|------|----------|---------|--------| +| CV | TypeScript/Node | Personal Knowledge Graph + CV generation | Best-structured, partially implemented | +| AI-Orchestrator | Python | Autonomous agent dispatcher (Symphony spec) | PAT-management done, orchestrator blocked on server | +| OrgMyLife | Python | Personal task/call/digest system | Productive, MCP server running | +| project-audit | Python | Jira/Confluence/GitLab analysis (DB internal) | Functional, script-based | +| Confluence_Bot | Python | CI/CD for Confluence pages | Minimal | +| Projekt-KIQ-HP | Node/Web | KIQ Landing Page | Frontend project | +| Analyse-O2C-C2S | Markdown/HTML | Strategic team analysis O2C+C2S | Active, own GitLab repo | +| NoteGraph | TBD | Note organization | Early stage | + +## Secrets + +- Located in `.secrets` at workspace root (gitignored) +- Contains: GITLAB_TOKEN_BESTELLSYSTEM, GITLAB_TOKEN_AUDIT, JIRA_TOKEN, JIRA_TOKEN_SAB, CONFLUENCE_TOKEN, CONFLUENCE_SAB, ORGMYLIFE_API_SECRET +- NEVER echo secret values in responses + +## Conventional Commits (all repos) + +All commit messages follow Conventional Commits format: +``` +feat(scope): description +fix(scope): description +docs(scope): description +test(scope): description +ci(scope): description +refactor(scope): description +``` + +## Quality Gates (before every commit) + +1. **Security scan**: Run `gitleaks detect --source . --no-git` (or equivalent) — no secrets in code +2. **Lint**: Run the project's linter (ruff for Python, eslint for Node, tsc --noEmit for TS) +3. **Test**: Run the project's test suite — minimum 80% coverage target +4. **Dependency check**: On Node projects, `npm audit` should show 0 vulnerabilities + +## Autonomous Mode Rules + +When working autonomously (orchestrator, sub-agents): +- Make decisions based on best practices and existing code patterns +- Never push directly to main/master — always use feature branches +- Create a Merge/Pull Request for human review +- If stuck after 3 attempts: stop and document the issue clearly +- Report progress: `PROGRESS: X% - what's happening (ETA: Xmin)` +- Signal help needed: `HELP: clear description of the problem` + +## Testing Standards + +- Minimum 80% line coverage for new code +- Use Given-When-Then structure for test cases +- Property-based testing where applicable (fast-check for TS, hypothesis for Python) +- Test happy path, edge cases, and error cases + +## MCP Servers (available when VPN connected) + +```json +{ + "jira": { "url": "https://taros-playground-mcp-jira-v2.apps.dbcs-prag.comp.db.de/mcp" }, + "confluence": { "url": "https://taros-playground-mcp-confluence.apps.dbcs-prag.comp.db.de/mcp" } +} +``` + +## DB Internal Services + +- Jira (arija): `https://arija-jira.jaas.service.deutschebahn.com` — PAT auth via Bearer token. Requires VPN. +- Jira (systelone/SAB): PAT auth works, but requires VPN. +- Confluence (arija): `https://arija-confluence.jaas.service.deutschebahn.com` — PAT auth via Bearer token works. +- Confluence (systelone/SAB): NO API/PAT support. Only accessible via HTTPS + HTML. +- GitLab: `https://git.tech.rz.db.de` — PAT auth via PRIVATE-TOKEN header. + +## Token Status (last checked: 2026-05-28) + +| Token | Service | Status | +|-------|---------|--------| +| GITLAB_TOKEN_AUDIT | GitLab (project-audit) | ✅ OK | +| GITLAB_TOKEN_BESTELLSYSTEM | GitLab (Bestellsystem) | ✅ OK (renewed 2026-05-28) | +| CONFLUENCE_TOKEN | Confluence (arija) | ✅ OK | +| CONFLUENCE_SAB | Confluence (systelone) | ⚠️ No API support — HTML only | +| JIRA_TOKEN | Jira (arija) | ❓ Not testable without VPN | +| JIRA_TOKEN_SAB | Jira (systelone) | ❓ Not testable without VPN | +| ORGMYLIFE_API_SECRET | OrgMyLife | ✅ OK (local) | +| GitHub (Desktop) | GitHub | ✅ OK | + +## OneDrive Compatibility + +All filenames must be OneDrive-safe: +- No brackets `()[]{}`, no `#%&@$+=~` +- No single quotes or backticks +- Replace umlauts in filenames: ä→ae, ö→oe, ü→ue, ß→ss +- Use hyphens instead of special characters diff --git a/README.md b/README.md new file mode 100644 index 0000000..cb797e3 --- /dev/null +++ b/README.md @@ -0,0 +1,293 @@ +# Wissensdatenbank ETL-Pipeline + +## Worum geht es? (in einfach) + +Dies ist die **zentrale Wissenssammlung** fuer den 1st-Level-Support und Chatbots rund um +DB InfraGO. Wir **sammeln hier das Wissen** aus vielen Quellen (Confluence, Webseiten, +PDFs), pruefen es und legen es **versioniert im Git-Repo** ab. Damit ist dieses Repo die +**Single Source of Truth**: Wenn etwas hier steht und freigegeben ist, gilt es - und genau +das lesen die Chatbots. + +**Was ist „ETL"?** Drei einfache Schritte, die bei jeder Aufnahme passieren: + +1. **Extract = Sammeln.** Wissen aus den Quellen holen (Confluence-Seiten, Webseiten, PDFs). +2. **Transform = Aufbereiten.** In einheitliches Markdown umwandeln, mit Schlagworten/Scope + versehen und vertrauliche Inhalte herausfiltern. +3. **Load = Ablegen.** Das geprueefte Wissen versioniert nach `data/processed/` schreiben - + die zentrale Wahrheit, aus der Chatbots/Vektor-DB lesen. + +Kurz: **Quellen rein → aufbereiten/pruefen → als Single Source of Truth ablegen.** + +## Ziel + +Klar definieren, **welches Wissen in Chatbots darf und welches nicht.** Wissen wird bei +der Aufnahme speziell verarbeitet und bereitgestellt, sodass Chatbots es nutzen koennen. +Klassifikation nach Scope: + +- **allgemein** – darf intern **und** extern genutzt werden; nicht toolspezifisch + (z.B. Regulierung/INB, Kundeninformationen). +- **intern** – nur DB InfraGO intern. +- **extern** – public im Internet bzw. fuer EVUs/EIUs. + +Technisch eine **ETL-Pipeline fuer LLM/RAG**: holt Wissen aus Confluence, Webseiten und +PDFs, klassifiziert es nach Scope, filtert vertrauliche Inhalte und gibt nur geprueftes +Wissen frei (`data/processed//...` = Feed fuer die Vektor-DB). + +> Architektur & vollstaendiger Wissensfluss inkl. Diagramme: +> [`.kiro/steering/architecture.md`](.kiro/steering/architecture.md) +> · Manuelle Einrichtungsschritte: [`docs/SETUP.md`](docs/SETUP.md) + +## Denkmodell + +``` +Domaene -> Scope (intern|extern|allgemein) -> Tool -> Dokument(e) +``` + +- **Domaene**: fachlicher Bereich (jedes Tool ist eigene Domaene; `regulierung`, `kundeninfo` fuer allgemeines Wissen). +- **Scope**: `intern` | `extern` | `allgemein` (allgemein gilt fuer intern UND extern). +- **Tool**: eine Fachanwendung aus dem 1st-Level-Support (Katalog automatisch aus Seite 428909879). +- **Strategie pro Quelle** – siehe Tabelle unten. + +### Strategien (Verarbeitung pro Quelle) + +| Strategie | Was sie tut | Wichtige Options | +|-----------|-------------|------------------| +| `confluence_page` | genau diese eine Seite | `scope` (auch `intern,extern`), `incremental`, `attachments` | +| `confluence_tree` | Seite inkl. Unterseiten | `max_depth` (-1=alle, 0=nur Seite), `incremental`, `attachments` | +| `confluence_faq` | FAQs aus Tabellen | `incremental` | +| `crawler` | Webseite: Index → Detailseiten | `detail_pattern`, `selector`, `max_pages` | +| `sitemap` | Detailseiten aus (gz-)Sitemap | `sitemap_url`, `url_pattern`, `limit`, `incremental` | +| `gitlab_md` | Markdown-Dateien aus einem GitLab-Repo | `ref`, `path`, `max_depth` (Token: `GITLAB_TOKEN`) | +| `file` | Dateien (md/pdf) aus dem Repo-Ordner `files/` | `max_depth` | +| `pdf` | PDFs einer Seite → Markdown (1→n) | `split: headings\|pages`, `heading_pattern`, `keep_raw`, `redact` | + +### Confluence-Anhaenge (eingebundene PDFs) + +Bei `confluence_page`/`confluence_tree` werden **eingebundene PDF-Anhaenge** +(`view-file`/`viewpdf`-Makros) automatisch geparst und als **eigene Dokumente** im +selben Feed abgelegt (Frontmatter `kind: attachment`, `parent_url` zeigt auf die Seite). +Scope/Domain/Owner werden von der Seite geerbt. Inkrementell ueber die Attachment-Version +(unveraenderter Anhang -> kein Re-Parse). Default an, pro Quelle ueber +`options: { attachments: false }` abschaltbar. + +Bilder werden nicht binaer uebernommen; der **alt-Text/Dateiname** bleibt als +`[Bild: ...]` im Markdown - gibt dem RAG Kontext ohne Volumen. Datei-Embeds ohne +Parsing-Pfad (z.B. Office-Docs) tauchen als `[Anhang: ...]`-Marker im Seitentext auf. + +### Mehrere Quellen pro Tool + +Ein Tool kann beliebig viele, gemischte Quellen haben – einfach weitere Eintraege +unter `sources:` anhaengen (z.B. mehrere Confluence-Quellen mit `max_depth`, eine +interne und eine externe Seite, oder zusaetzlich eine FAQ-Seite). Siehe die Tools +`pathos` (mehrere Quellen inkl. `confluence_faq` und `intern,extern`) und `nur`. + +### intern vs. extern – wie wird klassifiziert? + +Es gibt **keine Inhaltstrennung innerhalb einer Seite** mehr. Klassifiziert wird pro +Tool und pro Quelle: + +**Tool-`scope`** (in `config/tools.yaml`): + +| Tool-`scope` | Bedeutung | +|--------------|-----------| +| `intern` | alles intern (Default, restriktiv – z.B. Salesforce, NuR) | +| `extern` | alles extern | +| `allgemein` | gilt fuer intern UND extern | +| `mixed` | Tool hat Quellen mit **unterschiedlichem** Scope (z.B. eine interne und eine externe Seite) | + +**Source-`scope`** (pro Quelle, gilt fuer die ganze Seite): +- `intern` | `extern` | `allgemein` +- `"intern,extern"` → die Seite wird fuer **beide** Scopes genutzt (dupliziert). + +Quellen ohne eigenen `scope` erben den Tool-Scope (bei `mixed` => intern). +**Wichtig:** Hat ein Tool gemischtes Wissen, braucht es im Zweifel **zwei Seiten** +(eine interne, eine externe) als zwei Quellen – statt einer gemischten Seite. + +Allgemeines, tool-uebergreifendes Wissen steht separat in **`config/general.yaml`**. + +Der Tool-Katalog wird **initial** mit `scripts/bootstrap_tools.py` erzeugt und danach +**manuell** in `config/tools.yaml` gepflegt. Uebersicht aller Quellen: GitLab-Pages-Seite +**„Wissensquellen"** (`config.html`, geparst aus tools.yaml/general.yaml/approvals.yaml). + +## Neues Wissen hinzufuegen (Workflow) + +Jedes Wissen hat **owners** (intern Verantwortliche, Accountability) und einen +**contact** (herausgebbare Kontaktadresse, Default `einfachbahn@deutschebahn.com`). +Beide stehen im Frontmatter jedes Dokuments und auf der Pages-Seite, damit klar ist, +wer verantwortlich ist und an wen man sich wenden darf. + +1. **Issue anlegen** mit der Vorlage „Neues Wissen" (`.gitlab/issue_templates/`): + Tool/Domaene, **Quellen je Zeile** (`URL | Strategie | Scope`), **Verantwortliche** + (`owners`) und optional **Kontakt** (Default `einfachbahn@deutschebahn.com`). +2. **Reviewer uebernimmt** die Angaben nach `config/tools.yaml` bzw. `config/general.yaml` + (neuer Tool-/Quellen-Eintrag inkl. `owners:`) – per GitLab Web-IDE (Variante A) oder + manuell (Variante B). +3. **Merge Request** öffnen. Die **MR-Freigabe ist das Quality Gate** (4-Augen, via + `CODEOWNERS` + Protected Branch). +4. **Vorschau** im MR: den `preview`-Job manuell starten und `PREVIEW_ONLY` auf + Tool-Id/Domaene/URL der neuen Quelle setzen. Er verarbeitet nur diese Quelle und + stellt die erzeugten Markdowns als Job-Artefakt bereit. Lokal alternativ + `python -m src.main --only "" --data preview`. +5. **Merge** → der ETL-Schedule verarbeitet die Quelle (stuendlich Mo–Fr, inkrementell) und + committet das Wissen; die Vektor-DB liest direkt aus `data/processed//...`. + +## Freigabe-Prozess + +``` +MR mergen (Mensch) -> ETL: Extract -> Transform -> Auto-Filter -> data/processed/ -> Vektor-DB +``` + +- **Freigabe = Merge Request (Mensch, das eigentliche Gate):** Wer eine Quelle + (Link + Strategie + Scope) in `config/tools.yaml`/`general.yaml` eintraegt, stellt einen + MR. Beim **Merge** (4-Augen via `CODEOWNERS` + Protected Branch) wird entschieden, + welche Quelle mit welchem Scope aufgenommen wird. Danach laeuft der ETL automatisch. +- **Auto-Filter im ETL (Sicherheitsnetz, kein Freigabe-Knopf):** setzt pro Dokument + `review_status` – standardmaessig **approved**: + - `approved` -> `data/processed/...` (**direkt live**, sobald gemergt + ETL gelaufen) + - `pending` -> `data/staging/pending/` (Blacklist-Treffer, Scope-Warnung, oder + Inhalt < 50 Zeichen – Grund im Frontmatter `review_notes`) + - `trusted: true` (z.B. INB) bleibt trotz Treffer approved; `redact: false` schaltet + die Redaction pro Quelle ab. +- **`config/approvals.yaml` (nachtraegliche Korrektur):** hebt einzelne + `pending`-Dokumente per URL oder `hash:` doch auf `approved`. + Kein Schritt, den jedes Dokument durchlaeuft. Den Hash zeigt die Pages-Seite (Detail-Fenster). +- **Vektor-DB liest direkt aus `data/processed///`** (implementiert): + Nur freigegebenes Wissen liegt dort, der Scope steckt im Ordnerpfad **und** im + Frontmatter (inkl. `owners`/`contact`). Kein separater Export-/Ingestion-Schritt – + `processed/` IST der Feed. Der externe Index nimmt nur `extern/` + `allgemein/`. +- **`data/_meta.json`**: globale Statusdatei fuer nachgelagerte Systeme mit + `last_run` (letzter ETL-Lauf), `last_change` (wann sich der Bestand zuletzt + inhaltlich/metadatenseitig geaendert hat), `documents`, `by_scope` und einer + `content_signature`. Ein Index-/RAG-Consumer kann daran erkennen, ob ein erneutes + Einlesen ueberhaupt noetig ist. +- **`data/_index.json`**: dokument-genauer Katalog des gesamten Bestands. + Pro Voll-Dokument: `domain`/`tool`/`scope`, `url`, `path`, `content_hash`, + `last_updated` und - falls vorhanden - die zugehoerigen `chunks` (Anzahl + Pfad) + und `attachments` (Anhang-Dokumente: Name + Pfad). Anhang-Dokumente haben zudem + `kind: "attachment"` und `parent_url`. Aggregate: `documents_total`, + `chunks_total`, `attachments_total`, `by_domain` (mit `documents`/`chunks`/`attachments`), + `by_scope`. So sieht ein Anschliesser auf einen Blick, was pro Domaene/Tool wo liegt + und welche Dokumente zusaetzlich als Chunks oder Anhaenge vorliegen (z.B. INB = + Voll-Dokument **und** Chunks; pathOS-Seite = Voll-Dokument **und** Anhang-PDF). + Deterministisch (kein Zeitstempel) -> aendert sich nur bei echten Bestandsaenderungen. +- **`data/run_log.jsonl`**: append-only **Lauf-Historie** (1 JSON-Zeile je ETL-Lauf): + Zeitstempel, verarbeitete Dokumente, Status-Counts und **Fehler je Quelle** + (z.B. fehlgeschlagene Confluence-Abrufe). Auf die letzten 500 Laeufe gekappt. So sieht + man - auch historisch - ob ein Lauf sauber durchlief, ohne die fluechtigen CI-Job-Logs + zu durchsuchen. Die Uebersichtsseite zeigt zusaetzlich eine kompakte Health-Zeile + („N Dok verarbeitet, M Quellen mit Fehler") in der Fusszeile. + +### Wo liegt das final freigegebene Wissen? + +Ausschliesslich unter **`data/processed///[/]`** im Repo +(versioniert, = Single Source of Truth). `pending`-Dokumente liegen in +`data/staging/pending/` und werden zur **Transparenz ebenfalls committet** (auf der +Pages-Seite einsehbar). Der Audit-Trail steht in `data/review_report.json` und in +der Git-Historie. + +## Lokal testen + +```bash +python3 -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt + +ruff check src tests # Lint +python -m pytest -q # Offline-Tests +python -m src.main --config config/tools.yaml --data data # Live-Lauf +python -m src.chunk --data data # Chunks erzeugen (offline, optional) +``` + +## Confluence aktivieren + +```bash +export CONFLUENCE_URL="https://arija-confluence.jaas.service.deutschebahn.com" +export CONFLUENCE_TOKEN="" # Bearer-PAT (Server/DC) +pip install atlassian-python-api +``` + +Ohne Credentials werden Confluence-Quellen sauber uebersprungen. + +## Betrieb (GitLab CI) + +- `quality`-Stage: `ruff` (Lint) + `gitleaks` (Secret-Scan) bei jedem Push/MR. +- `etl`-Stage: **stuendlich Mo–Fr 8–17 Uhr** (Cron `0 8-17 * * 1-5`), baut Wissen, + committet `data/` nach `main` (per `GIT_PUSH_TOKEN`); der Commit loest automatisch + einen `pages`-Refresh aus. +- `pages`-Stage: baut `public/` (Uebersicht, Hilfe, Chatbot-Anschluss, Wissensquellen) + fuer GitLab Pages auf `main`. +- `preview`-Job (MR, manuell): verarbeitet eine neue/geaenderte Quelle via + `--only` und stellt die erzeugten Markdowns als Job-Artefakt bereit (zum Pruefen + vor dem Merge). `PREVIEW_ONLY` beim Start auf Tool-Id/Domaene/URL setzen. +- **Kundeninfos inkrementell**: `incremental: true` crawlt nur NEUE Detailseiten und + ueberschreibt vorhandene nicht – Re-Runs bleiben schnell. +- **Confluence inkrementell**: `options: { incremental: true }` prueft pro Seite die + Confluence-Versionsnummer (`source_version` im Frontmatter). Unveraenderte Seiten + werden uebersprungen, nur geaenderte/neue Seiten neu verarbeitet – spart API-Last. +- **Re-Tagging bei Metadaten-Aenderung**: Aenderst du in `tools.yaml`/`general.yaml` + nur `tags`/`owners`/`contact` einer Quelle, weicht deren `meta_fingerprint` + (im Frontmatter) vom gespeicherten ab. Der naechste inkrementelle Lauf passt dann + die bestehenden Dateien dieser Quelle in-place an (nur Frontmatter, kein erneuter + Download/API-Call). +- **Scope-Wechsel**: Aenderst du den `scope` einer Quelle, wird die Seite neu + verarbeitet und an `data/processed//...` geschrieben; die alte Ablage + wird nach erfolgreichem Schreiben **automatisch geloescht** (domaenen-intern, damit + eine URL, die legitim unter mehreren Domaenen liegt – z.B. eine pathOS-Seite auch + unter `web` – nicht versehentlich entfernt wird). +- **Domain-Umbenennung**: Die neue Ablage entsteht automatisch, die alte Domain-Ablage + bleibt aber stehen (domaenenuebergreifendes Loeschen waere wegen geteilter URLs + unsicher). Den alten `data/processed///`-Ordner daher bei einer + Domain-Umbenennung manuell loeschen. +- CI/CD-Variablen: `GIT_PUSH_TOKEN`, `CONFLUENCE_URL`, `CONFLUENCE_TOKEN` (Masked). +- `scm-info.yaml`: Pflichtdatei fuer DB-GitLab-Compliance. + +## Konfiguration + +- `config/tools.yaml` – Tools, Domaenen, Quellen, Strategien, Scopes, Tags +- `config/filter_rules.json` – Blacklist-Keywords, Regex-Redaction +- `config/chunking.yaml` – gemeinsame Chunk-Defaults (pro Quelle ueber `options.chunk` ueberschreibbar) + +### Chunking (optional, Default aus) + +Grosse, stark gegliederte Dokumente (v.a. **INB**) werden zusaetzlich zur Voll-Datei in +**Chunks entlang der Ueberschriften** zerlegt – ein **abgeleitetes, jederzeit neu +erzeugbares** Artefakt unter `data/chunks////`. Das Voll-Dokument +in `data/processed/` bleibt unangetastet (Parent-Document-Muster): ein Anschliesser waehlt +Voll-Dokument, Chunks oder beides. Es werden bewusst nur **deterministische, embedding-freie** +Strategien angeboten (`headings | faq | recursive`); semantisches Chunking gehoert in die +Vektor-DB des Anschliessers. Jeder Chunk bekommt eine kurze **Contextual-Retrieval**-Zeile +(`> Kontext: > `). Aktivierung pro Quelle via `options: { chunk: headings }`. + +**Inkrementell (kein Voll-Rebuild):** Chunking laeuft am Ende jedes ETL-Laufs automatisch +mit (`src.main`) und ist Teil von `data/`. Pro Dokument wird nur dann neu gechunkt, wenn +sich der **Inhalt** (`parent_hash`) ODER die **wirksamen Optionen/Strategie** +(`chunk_fingerprint`) geaendert haben - sonst wird das Dokument uebersprungen (keine +Datei-Aenderung, kein Git-Churn). Aenderst du also bei einer Quelle z.B. `chunk_level` +oder von `headings` auf `recursive`, werden **nur die Dokumente dieser Ablage** neu +erzeugt. Wird `chunk` wieder auf `off` gestellt oder ein Voll-Dokument geloescht, werden +die zugehoerigen Chunks automatisch entfernt. + +**Feinsteuerung (Default aus, pro Quelle ueberschreibbar):** + +- `chunk_min_doc_chars`: Dokumente unter dieser Zeichenzahl bleiben **ganz** (kleine + FAQ-/How-to-Seiten muessen nicht zwingend gechunkt werden). +- `chunk_kind`: chunkt nur Dokumente mit passendem `kind` (`""` = alle; `attachment` = + nur Anhang-PDFs; `document` = nur Voll-Seiten). + +Beispiel **pathOS** (an der Tool-Tree-Quelle in `config/tools.yaml`): grosse Anhang-PDFs +(z.B. EVU-Schnittstellen-Doku, 2,8 MB) werden gechunkt, Seiten und FAQs bleiben ganz. + +```yaml +options: + incremental: true + chunk: headings + chunk_kind: attachment # nur Anhang-PDFs chunken + chunk_min_doc_chars: 3000 # < 3k Zeichen bleiben ganz (~750 Token) +``` + +## Version & Changelog + +Aktuelle Version steht in **`VERSION`** (SemVer), die Historie in **`CHANGELOG.md`** +(Format „Keep a Changelog"). Beides wird auf der GitLab-Pages-Seite **„Changelog"** +angezeigt (Version auch in der Navigationsleiste). Pflege-Regeln: +`.kiro/steering/changelog-versioning.md`. diff --git a/bahn/.gitattributes b/bahn/.gitattributes new file mode 100644 index 0000000..a4fbb0d --- /dev/null +++ b/bahn/.gitattributes @@ -0,0 +1,12 @@ +# ============================================================================= +# git-crypt Filter für Kontext: bahn +# Alle hier definierten Patterns werden mit dem Schlüssel 'git-crypt-bahn' +# verschlüsselt. Nur Maschinen mit dem bahn-Schlüssel können diese Dateien +# im Klartext lesen. +# ============================================================================= + +.env filter=git-crypt-bahn diff=git-crypt-bahn +*.pem filter=git-crypt-bahn diff=git-crypt-bahn +*.key filter=git-crypt-bahn diff=git-crypt-bahn +*token* filter=git-crypt-bahn diff=git-crypt-bahn +*secret* filter=git-crypt-bahn diff=git-crypt-bahn diff --git a/bahn/.gitkeep b/bahn/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/dhive/.gitattributes b/dhive/.gitattributes new file mode 100644 index 0000000..115ea92 --- /dev/null +++ b/dhive/.gitattributes @@ -0,0 +1,12 @@ +# ============================================================================= +# git-crypt Filter für Kontext: dhive +# Alle hier definierten Patterns werden mit dem Schlüssel 'git-crypt-dhive' +# verschlüsselt. Nur Maschinen mit dem dhive-Schlüssel können diese Dateien +# im Klartext lesen. +# ============================================================================= + +.env filter=git-crypt-dhive diff=git-crypt-dhive +*.pem filter=git-crypt-dhive diff=git-crypt-dhive +*.key filter=git-crypt-dhive diff=git-crypt-dhive +*token* filter=git-crypt-dhive diff=git-crypt-dhive +*secret* filter=git-crypt-dhive diff=git-crypt-dhive diff --git a/dhive/.gitkeep b/dhive/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/monorepo.yaml b/monorepo.yaml new file mode 100644 index 0000000..6b3e9b7 --- /dev/null +++ b/monorepo.yaml @@ -0,0 +1,26 @@ +version: "1.0" +contexts: + - name: privat + description: "Persönliche Projekte" + - name: dhive + description: "dhive GmbH Projekte" + - name: bahn + description: "DB InfraGO Projekte" + - name: shared + description: "Kontextübergreifende Tools und Wissen" + +naming: + pattern: "^[a-z0-9][a-z0-9\\-]{0,48}[a-z0-9]$" + min_length: 2 + max_length: 50 + +security: + config: shared/config/access-config.yaml + audit_log: .audit/access.log + encryption: + tool: git-crypt + machine_context_config: shared/config/machine-context.yaml + key_source: keyring + federation: + config: shared/config/team-repos.yaml + conflict_strategy: team-wins diff --git a/privat/.gitattributes b/privat/.gitattributes new file mode 100644 index 0000000..ef82841 --- /dev/null +++ b/privat/.gitattributes @@ -0,0 +1,12 @@ +# ============================================================================= +# git-crypt Filter für Kontext: privat +# Alle hier definierten Patterns werden mit dem Schlüssel 'git-crypt-privat' +# verschlüsselt. Nur Maschinen mit dem privat-Schlüssel können diese Dateien +# im Klartext lesen. +# ============================================================================= + +.env filter=git-crypt-privat diff=git-crypt-privat +*.pem filter=git-crypt-privat diff=git-crypt-privat +*.key filter=git-crypt-privat diff=git-crypt-privat +*token* filter=git-crypt-privat diff=git-crypt-privat +*secret* filter=git-crypt-privat diff=git-crypt-privat diff --git a/privat/.gitkeep b/privat/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/shared/config/ONBOARDING.md b/shared/config/ONBOARDING.md new file mode 100644 index 0000000..32ff09c --- /dev/null +++ b/shared/config/ONBOARDING.md @@ -0,0 +1,189 @@ +# Onboarding: Neue Maschine einrichten + +Dieses Dokument beschreibt den Prozess, um eine neue Maschine für die Arbeit mit dem Monorepo und seinen verschlüsselten Secrets einzurichten. + +## Voraussetzungen + +- Git ist installiert (>= 2.25) +- [git-crypt](https://github.com/AGWA/git-crypt) ist installiert +- Zugang zum Passwort-Manager (Bitwarden) oder direkter Schlüsselzugang + +## Übersicht: Verschlüsselungsarchitektur + +Das Monorepo verwendet **git-crypt** mit kontextspezifischen Filtern: + +| Kontext | Filter-Name | Verschlüsselte Patterns | +|---------|--------------------|-------------------------------------------------| +| privat | git-crypt-privat | .env, *.pem, *.key, *token*, *secret* | +| dhive | git-crypt-dhive | .env, *.pem, *.key, *token*, *secret* | +| bahn | git-crypt-bahn | .env, *.pem, *.key, *token*, *secret* | +| shared | git-crypt (basis) | shared/.env, shared/**/*.pem, shared/**/*.key | + +Jeder Kontext hat einen eigenen symmetrischen Schlüssel. Eine Maschine kann nur die Secrets der Kontexte entschlüsseln, für die sie autorisiert ist. + +## Schritt 1: Repository klonen + +```bash +git clone monorepo +cd monorepo +``` + +Nach dem Klonen sind alle Secret-Dateien als verschlüsselte Binärdaten vorhanden. Code, Konfiguration und Dokumentation sind sofort lesbar. + +## Schritt 2: Maschinenkontext konfigurieren + +Bearbeite `shared/config/machine-context.yaml` und passe sie an die neue Maschine an: + +```yaml +machine: + name: "mein-neuer-rechner" + description: "Beschreibung des Rechners und seines Einsatzzwecks" + authorized_contexts: + - privat # Nur Kontexte eintragen, die entschlüsselt werden sollen + - dhive + # - bahn # Auskommentiert = kein Zugriff auf bahn-Secrets + key_source: "keyring" # Siehe Abschnitt "Schlüsselquellen" + password_manager: + type: "bitwarden" + vault: "monorepo-keys" +``` + +### Felder + +| Feld | Beschreibung | +|------------------------|--------------------------------------------------------------------| +| `name` | Eindeutiger Name der Maschine (z.B. "dhive-laptop", "home-pc") | +| `description` | Kurze Beschreibung des Einsatzzwecks | +| `authorized_contexts` | Liste der Kontexte, deren Secrets entschlüsselt werden dürfen | +| `key_source` | Woher die Schlüssel geladen werden (siehe unten) | +| `password_manager` | Optionale Passwort-Manager-Konfiguration | + +## Schritt 3: Entschlüsselungsschlüssel installieren + +### Schlüsselquellen (`key_source`) + +Das System unterstützt drei Quellen für Entschlüsselungsschlüssel: + +#### 1. Keyring (`key_source: "keyring"`) + +Schlüssel werden im OS-Keyring gespeichert (empfohlen für Entwicklermaschinen). + +```bash +# Schlüssel aus sicherer Quelle importieren +git-crypt unlock /pfad/zum/schlüssel-privat.key +git-crypt unlock /pfad/zum/schlüssel-dhive.key +git-crypt unlock /pfad/zum/schlüssel-bahn.key + +# Alternativ: Schlüssel im Keyring speichern (via monorepo-cli) +monorepo-cli onboard --context privat --key-file /pfad/zum/schlüssel.key +monorepo-cli onboard --context dhive --key-file /pfad/zum/schlüssel.key +``` + +#### 2. Datei (`key_source: "file"`) + +Schlüssel liegen als Dateien auf der Festplatte (nur für isolierte Maschinen). + +```bash +# Schlüsseldateien an erwarteter Stelle ablegen +mkdir -p ~/.monorepo-keys/ +cp schlüssel-privat.key ~/.monorepo-keys/git-crypt-privat.key +cp schlüssel-dhive.key ~/.monorepo-keys/git-crypt-dhive.key +cp schlüssel-bahn.key ~/.monorepo-keys/git-crypt-bahn.key + +# Berechtigungen einschränken +chmod 600 ~/.monorepo-keys/*.key +``` + +#### 3. Passwort-Manager (`key_source: "password-manager"`) + +Schlüssel werden bei Bedarf aus einem Passwort-Manager abgerufen. + +Unterstützte Manager: +- **Bitwarden** (`type: "bitwarden"`) +- **1Password** (`type: "1password"`) +- **KeePass** (`type: "keepass"`) + +```bash +# Bitwarden-Beispiel: Schlüssel sind im Vault "monorepo-keys" gespeichert +# Einträge haben das Format: monorepo-key-{context} +# z.B. monorepo-key-privat, monorepo-key-dhive, monorepo-key-bahn + +# Login in Bitwarden (einmalig pro Session) +bw login + +# Entschlüsselung wird automatisch über den SecretEncryptionManager gesteuert +monorepo-cli decrypt --context privat +``` + +## Schritt 4: Secrets entschlüsseln + +Nach Installation der Schlüssel werden die Secrets für autorisierte Kontexte entschlüsselt: + +```bash +# Alle autorisierten Kontexte entschlüsseln +monorepo-cli decrypt --all + +# Oder einzelne Kontexte +monorepo-cli decrypt --context privat +monorepo-cli decrypt --context dhive +``` + +### Verhalten bei nicht-autorisierten Kontexten + +Wenn eine Maschine keinen Schlüssel für einen Kontext besitzt: +- Die Secret-Dateien bleiben als verschlüsselte Binärdaten im Working Tree +- Der restliche Code und die Konfiguration des Kontexts sind lesbar +- Es wird keine Fehlermeldung ausgegeben, die den Inhalt offenlegt +- Das Repository bleibt voll funktionsfähig für die autorisierten Kontexte + +## Schritt 5: Einrichtung verifizieren + +```bash +# Status der Verschlüsselung prüfen +monorepo-cli encrypt --status + +# Erwartete Ausgabe für "dhive-laptop" (nur dhive autorisiert): +# privat: verschlüsselt (kein Schlüssel) +# dhive: entschlüsselt ✓ +# bahn: verschlüsselt (kein Schlüssel) +# shared: entschlüsselt ✓ +``` + +## Typische Maschinenkontexte + +| Maschine | Autorisierte Kontexte | key_source | +|----------------------|-----------------------|------------------| +| andre-hauptrechner | privat, dhive, bahn | keyring | +| dhive-laptop | dhive | password-manager | +| bahn-arbeitsrechner | bahn | keyring | +| familie-nas | privat | file | + +## Sicherheitshinweise + +1. **Schlüssel niemals committen**: Die Schlüsseldateien selbst gehören NICHT ins Repository +2. **Maschinenkontext-Datei ist kein Secret**: `machine-context.yaml` enthält nur das Mapping, keine Schlüssel +3. **Minimaler Zugriff**: Jede Maschine erhält nur die Schlüssel, die sie benötigt +4. **Schlüsselrotation**: Bei Kompromittierung eines Schlüssels → neuen Schlüssel generieren, alle autorisierten Maschinen aktualisieren, Secrets neu verschlüsseln + +## Troubleshooting + +### Dateien erscheinen als Binärdaten + +**Ursache:** Schlüssel für den betreffenden Kontext nicht installiert. + +**Lösung:** Schlüssel gemäß Schritt 3 installieren und `monorepo-cli decrypt` ausführen. + +### git-crypt meldet "not a git-crypt repo" + +**Ursache:** git-crypt wurde im Repository noch nicht initialisiert. + +**Lösung:** `git-crypt init` (nur beim erstmaligen Setup des Repositories nötig). + +### Merge-Konflikte in verschlüsselten Dateien + +**Ursache:** Zwei Branches haben dieselbe Secret-Datei geändert. + +**Lösung:** Der SecretEncryptionManager löst Merges auf verschlüsselter Ebene auf. Falls dies fehlschlägt: +```bash +monorepo-cli encrypt --resolve-merge +``` diff --git a/shared/config/access-config.yaml b/shared/config/access-config.yaml new file mode 100644 index 0000000..f6e057e --- /dev/null +++ b/shared/config/access-config.yaml @@ -0,0 +1,23 @@ +contexts: + privat: + env_file: privat/.env + allowed_shared: + - shared/tools/ + - shared/powers/ + - shared/config/ + dhive: + env_file: dhive/.env + allowed_shared: + - shared/tools/ + - shared/powers/ + - shared/config/ + bahn: + env_file: bahn/.env + allowed_shared: + - shared/tools/ + - shared/powers/ + - shared/config/ + - shared/knowledge-store/ + shared: + env_file: shared/.env + allowed_shared: ["*"] diff --git a/shared/config/machine-context.yaml b/shared/config/machine-context.yaml new file mode 100644 index 0000000..5344cc7 --- /dev/null +++ b/shared/config/machine-context.yaml @@ -0,0 +1,11 @@ +machine: + name: "andre-hauptrechner" + description: "Andres Hauptrechner mit vollem Zugriff" + authorized_contexts: + - privat + - dhive + - bahn + key_source: "keyring" + password_manager: + type: "bitwarden" + vault: "monorepo-keys" diff --git a/shared/config/repos.yaml b/shared/config/repos.yaml new file mode 100644 index 0000000..4e6a92c --- /dev/null +++ b/shared/config/repos.yaml @@ -0,0 +1 @@ +repos: [] diff --git a/shared/config/scopes.yaml b/shared/config/scopes.yaml new file mode 100644 index 0000000..d09f832 --- /dev/null +++ b/shared/config/scopes.yaml @@ -0,0 +1,18 @@ +version: "1.0" +scopes: + - name: privat + context: privat + description: "Persönlicher Wissensbereich" + knowledge_path: shared/knowledge-store/privat/ + - name: dhive + context: dhive + description: "dhive GmbH Wissensbereich" + knowledge_path: shared/knowledge-store/dhive/ + - name: bahn + context: bahn + description: "DB InfraGO Wissensbereich" + knowledge_path: shared/knowledge-store/bahn/ + - name: shared + context: shared + description: "Kontextübergreifender Wissensbereich" + knowledge_path: shared/knowledge-store/shared/ diff --git a/shared/config/team-repos.yaml b/shared/config/team-repos.yaml new file mode 100644 index 0000000..5933a8c --- /dev/null +++ b/shared/config/team-repos.yaml @@ -0,0 +1,44 @@ +version: "1.0" +federation: + topology: "hub-and-spoke" + hub_owner: "andre" + conflict_strategy: "team-wins" + +team_repos: + - context: privat + url: "https://github.com/andreknie/privat-team.git" + branch: main + sync_direction: bidirectional + sync_frequency: "on-push" + shared_mirror: + enabled: true + paths: + - "shared/tools/common-scripts/" + - "shared/config/base-config.yaml" + mode: read-only + + - context: dhive + url: "https://gitlab.dhive.io/team/dhive-mono.git" + branch: main + sync_direction: bidirectional + sync_frequency: "on-push" + shared_mirror: + enabled: true + paths: + - "shared/tools/" + - "shared/powers/db-dxp-platform/" + - "shared/mcp-servers/" + mode: read-only + + - context: bahn + url: "https://gitlab.2700.2db.it/team/bahn-workspace.git" + branch: main + sync_direction: bidirectional + sync_frequency: "daily" + shared_mirror: + enabled: true + paths: + - "shared/tools/" + - "shared/powers/" + - "shared/knowledge-store/_index.yaml" + mode: read-only diff --git a/shared/knowledge-store/.gitkeep b/shared/knowledge-store/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/shared/mcp-servers/.gitkeep b/shared/mcp-servers/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/shared/powers/.gitkeep b/shared/powers/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/shared/tools/.gitkeep b/shared/tools/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/shared/tools/monorepo-cli/README.md b/shared/tools/monorepo-cli/README.md new file mode 100644 index 0000000..7613205 --- /dev/null +++ b/shared/tools/monorepo-cli/README.md @@ -0,0 +1,15 @@ +# monorepo-cli + +CLI-Tool für die Monorepo-Verwaltung: Kontextgrenzen, Wissensspeicher, Repo-Sync, Verschlüsselung und Föderation. + +## Installation + +```bash +pip install -e ".[dev]" +``` + +## Verwendung + +```bash +ctx-guard --help +``` diff --git a/shared/tools/monorepo-cli/pyproject.toml b/shared/tools/monorepo-cli/pyproject.toml new file mode 100644 index 0000000..603b53e --- /dev/null +++ b/shared/tools/monorepo-cli/pyproject.toml @@ -0,0 +1,50 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "monorepo-cli" +version = "0.1.0" +description = "CLI-Tool für Monorepo-Verwaltung: Kontextgrenzen, Wissensspeicher, Repo-Sync, Verschlüsselung und Föderation" +readme = "README.md" +license = "Apache-2.0" +requires-python = ">=3.11" +authors = [ + { name = "Andre Knie" }, +] + +dependencies = [ + "pyyaml>=6.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0", + "pytest-cov>=5.0", + "hypothesis>=6.100", + "ruff>=0.4.0", + "mypy>=1.10", + "types-PyYAML>=6.0", +] + +[project.scripts] +ctx-guard = "monorepo.cli:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/monorepo"] + +[tool.ruff] +target-version = "py311" +line-length = 100 + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "W", "UP"] + +[tool.mypy] +python_version = "3.11" +strict = true +warn_return_any = true +warn_unused_configs = true + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/shared/tools/monorepo-cli/src/monorepo/__init__.py b/shared/tools/monorepo-cli/src/monorepo/__init__.py new file mode 100644 index 0000000..be59c3e --- /dev/null +++ b/shared/tools/monorepo-cli/src/monorepo/__init__.py @@ -0,0 +1,136 @@ +"""monorepo – CLI-Tool für Monorepo-Verwaltung. + +Exportiert die zentralen Datenmodelle und Konfigurationsfunktionen. +""" + +from monorepo.audit import AuditLogger +from monorepo.bridge import ContextBridge, SensitiveMatch, ShareResult +from monorepo.config import MonorepoConfig, load_monorepo_config +from monorepo.models import ( + ArtifactType, + ConflictInfo, + ConflictStrategy, + Context, + EncryptionKey, + IsolationLeak, + IsolationReport, + KeySource, + MachineContext, + MigrationPlan, + OnboardingResult, + PasswordManagerConfig, + ProjectInfo, + RepoEntry, + RepoMode, + ScopeConfig, + SecurityEvent, + SharedMirrorConfig, + SyncDirection, + SyncFrequency, + SyncResult, + TeamRepoEntry, +) +from monorepo.encryption import ( + DecryptionResult, + EncryptionResult, + GitCryptNotAvailableError, + MachineContextManager, + PasswordManagerError, + SecretEncryptionManager, +) +from monorepo.federation import ( + ConflictResult, + FederationManager, + FederationTopologyConfig, + MemberInfo, + MemberOnboardingResult, + MirrorResult, + SubtreeSyncEngine, +) +from monorepo.integration import create_integrated_stack +from monorepo.shared_config import ( + AgentExtension, + ConfigMerger, + HarnessAdapter, + MCPServerConfig, + MergeConflict, + MergedMCPConfig, + SharedToolStatus, + ToolReference, +) +from monorepo.migration import ( + MigrationConflict, + MigrationEngine, + MigrationError, + MigrationRegistryEntry, + MigrationResult, +) +from monorepo.security import ContextGuard +from monorepo.structure import StructureManager + +__all__ = [ + # Audit + "AuditLogger", + # Bridge + "ContextBridge", + "SensitiveMatch", + "ShareResult", + # Enums + "ArtifactType", + "ConflictStrategy", + "Context", + "KeySource", + "RepoMode", + "SyncDirection", + "SyncFrequency", + # Dataclasses + "ConflictInfo", + "EncryptionKey", + "IsolationLeak", + "IsolationReport", + "MachineContext", + "MigrationPlan", + "MonorepoConfig", + "OnboardingResult", + "PasswordManagerConfig", + "ProjectInfo", + "RepoEntry", + "ScopeConfig", + "SecurityEvent", + "SharedMirrorConfig", + "SyncResult", + "TeamRepoEntry", + # Classes + "AgentExtension", + "ConfigMerger", + "ConflictResult", + "ContextGuard", + "DecryptionResult", + "EncryptionResult", + "FederationManager", + "FederationTopologyConfig", + "GitCryptNotAvailableError", + "HarnessAdapter", + "MachineContextManager", + "MCPServerConfig", + "MemberInfo", + "MemberOnboardingResult", + "MergeConflict", + "MergedMCPConfig", + "MigrationConflict", + "MigrationEngine", + "MigrationError", + "MigrationRegistryEntry", + "MigrationResult", + "MirrorResult", + "PasswordManagerError", + "SecretEncryptionManager", + "SharedToolStatus", + "StructureManager", + "SubtreeSyncEngine", + "ToolReference", + # Config + "load_monorepo_config", + # Integration + "create_integrated_stack", +] diff --git a/shared/tools/monorepo-cli/src/monorepo/audit.py b/shared/tools/monorepo-cli/src/monorepo/audit.py new file mode 100644 index 0000000..49fcbb9 --- /dev/null +++ b/shared/tools/monorepo-cli/src/monorepo/audit.py @@ -0,0 +1,71 @@ +"""Audit-Logger für die Protokollierung von Zugriffsverletzungen. + +Schreibt Sicherheitsereignisse (SecurityEvents) in eine konfigurierbare +Protokolldatei im strukturierten Textformat. Thread-sicher durch Nutzung +eines Locks für Dateizugriffe. +""" + +from __future__ import annotations + +import threading +from pathlib import Path + +from monorepo.models import SecurityEvent + +# Standard-Audit-Log-Pfad (relativ zum Monorepo-Root), wie in monorepo.yaml definiert +_DEFAULT_AUDIT_LOG_PATH = Path(".audit/access.log") + + +class AuditLogger: + """Protokolliert Sicherheitsereignisse in eine Audit-Logdatei. + + Jeder Eintrag wird als strukturierte Zeile im Format geschrieben: + [ISO-TIMESTAMP] OUTCOME | requesting_context -> target_context | action on resource + + Der Logger ist thread-sicher: Schreibzugriffe auf die Logdatei werden + durch einen Lock serialisiert. + + Args: + log_path: Pfad zur Audit-Logdatei. Standard: `.audit/access.log` + (aus monorepo.yaml security.audit_log). + """ + + def __init__(self, log_path: Path | None = None) -> None: + """Initialisiert den AuditLogger. + + Args: + log_path: Pfad zur Audit-Logdatei. Falls None, wird der + Standard-Pfad `.audit/access.log` verwendet. + """ + self._log_path = log_path if log_path is not None else _DEFAULT_AUDIT_LOG_PATH + self._lock = threading.Lock() + # Verzeichnis erstellen, falls es nicht existiert + self._log_path.parent.mkdir(parents=True, exist_ok=True) + + @property + def log_path(self) -> Path: + """Gibt den konfigurierten Pfad zur Audit-Logdatei zurück.""" + return self._log_path + + def log_violation(self, event: SecurityEvent) -> None: + """Schreibt ein Sicherheitsereignis in die Protokolldatei. + + Format pro Zeile: + [2025-01-15T10:30:00] DENIED | dhive -> privat | read on privat/.env + + Args: + event: Das zu protokollierende Sicherheitsereignis mit Zeitstempel, + anfragendem Kontext, Zielkontext, Ressource, Aktion und Ergebnis. + """ + timestamp_iso = event.timestamp.isoformat() + outcome_upper = event.outcome.upper() + + line = ( + f"[{timestamp_iso}] {outcome_upper} | " + f"{event.requesting_context} -> {event.target_context} | " + f"{event.action} on {event.resource}\n" + ) + + with self._lock: + with open(self._log_path, "a", encoding="utf-8") as f: + f.write(line) diff --git a/shared/tools/monorepo-cli/src/monorepo/bridge.py b/shared/tools/monorepo-cli/src/monorepo/bridge.py new file mode 100644 index 0000000..c0754c4 --- /dev/null +++ b/shared/tools/monorepo-cli/src/monorepo/bridge.py @@ -0,0 +1,461 @@ +"""Kontextbrücke für kontextübergreifenden Wissenstransfer. + +Ermöglicht das sichere Teilen von Wissensartefakten zwischen Arbeitskontexten +unter Wahrung der Sicherheitsgrenzen. Implementiert: +- Sensitive-Content-Filterung (Secrets, Endpoints, PII) via Regex +- Explizite Nutzerbestätigung vor Freigabe +- Read-only-Referenzen in Zielkontexten +- Update-Propagierung beim nächsten Lesezugriff +- Widerruf von Freigaben + +Requirements: 5.1, 5.2, 5.3, 5.4, 5.5, 5.6 +""" + +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import yaml + +from monorepo.knowledge.artifact import Artifact, parse_frontmatter +from monorepo.knowledge.index import IndexEntry, YAMLIndex +from monorepo.models import Context + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Datenmodelle +# --------------------------------------------------------------------------- + + +@dataclass +class SensitiveMatch: + """Ein gefundener sensitiver Inhalt in einem Artefakt. + + Attributes: + pattern_name: Name des Musters, das getroffen hat (z.B. "credentials", "endpoint", "pii"). + matched_text: Der gefundene Text-Ausschnitt. + line_number: Zeilennummer im Inhalt (1-basiert). + reason: Menschenlesbare Begründung, warum der Inhalt als sensitiv gilt. + """ + + pattern_name: str + matched_text: str + line_number: int + reason: str + + +@dataclass +class ShareResult: + """Ergebnis einer Freigabe-Operation. + + Attributes: + success: True wenn die Freigabe erfolgreich war. + artifact_id: ID des betroffenen Artefakts. + shared_to: Liste der Kontexte, in denen das Artefakt nun verfügbar ist. + errors: Liste von Fehlermeldungen bei gescheiterter Freigabe. + """ + + success: bool + artifact_id: str + shared_to: list[str] = field(default_factory=list) + errors: list[str] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Shared-Artifacts-Registry +# --------------------------------------------------------------------------- + + +@dataclass +class SharedArtifactEntry: + """Eintrag in der Shared-Artifacts-Registry. + + Attributes: + artifact_id: ID des geteilten Artefakts. + source_context: Ursprungskontext des Artefakts. + shared_to: Zielkontexte, in denen das Artefakt verfügbar ist. + content_hash: Hash des Inhalts zum Zeitpunkt der Freigabe (für Update-Erkennung). + """ + + artifact_id: str + source_context: str + shared_to: list[str] = field(default_factory=list) + content_hash: str = "" + + +# --------------------------------------------------------------------------- +# ContextBridge +# --------------------------------------------------------------------------- + + +class ContextBridge: + """Ermöglicht kontextübergreifenden Wissenstransfer. + + Die ContextBridge prüft Artefakte auf sensitive Inhalte, erfordert + explizite Nutzerbestätigung und stellt geteilte Artefakte als + schreibgeschützte Lesereferenzen in Zielkontexten bereit. + + Update-Propagierung: Wird ein Artefakt im Quellkontext aktualisiert, + sind die Änderungen beim nächsten Lesezugriff über get_shared_content() + sichtbar, da immer vom Quell-Artefakt gelesen wird. + + Requirements: 5.1, 5.2, 5.3, 5.4, 5.5, 5.6 + """ + + SENSITIVE_PATTERNS: list[tuple[str, str, str]] = [ + ( + r"(?i)(api[_-]?key|token|password|secret)\s*[:=]", + "credentials", + "Enthält kontextspezifische Zugangsdaten (API-Key, Token, Passwort oder Secret)", + ), + ( + r"(?i)(endpoint|url)\s*[:=]\s*https?://", + "endpoint", + "Enthält kontextspezifische Endpoint-URL", + ), + ( + r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", + "pii", + "Enthält personenbezogene Daten (E-Mail-Adresse)", + ), + ] + + def __init__( + self, + knowledge_base_path: Path, + index: YAMLIndex, + registry_path: Path | None = None, + ) -> None: + """Initialisiert die ContextBridge. + + Args: + knowledge_base_path: Basispfad des Wissensspeichers. + index: Der YAML-Index des Wissensspeichers. + registry_path: Pfad zur Shared-Artifacts-Registry (YAML). + Falls None, wird in-memory gearbeitet. + """ + self.knowledge_base_path = knowledge_base_path + self.index = index + self.registry_path = registry_path + self._registry: dict[str, SharedArtifactEntry] = {} + + if registry_path and registry_path.exists(): + self._load_registry() + + # --------------------------------------------------------------------------- + # Öffentliche API + # --------------------------------------------------------------------------- + + def share_artifact( + self, artifact_id: str, user_confirmed: bool = False + ) -> ShareResult: + """Gibt ein Artefakt kontextübergreifend frei. + + Prüft: + 1. Existenz des Artefakts im Index + 2. Explizite Nutzerbestätigung (Req 5.4) + 3. Sensitive-Content-Filter (Req 5.1, 5.3) + + Bei Erfolg wird das Artefakt als Lesereferenz in allen anderen + Kontexten bereitgestellt (Req 5.2). + + Args: + artifact_id: ID des freizugebenden Artefakts. + user_confirmed: Muss True sein für explizite Nutzerbestätigung. + + Returns: + ShareResult mit Erfolgs-/Fehlerstatus. + """ + # Req 5.4: Explizite Nutzerbestätigung erforderlich + if not user_confirmed: + return ShareResult( + success=False, + artifact_id=artifact_id, + errors=[ + "Explizite Nutzerbestätigung erforderlich. " + "Setze user_confirmed=True um die Freigabe zu bestätigen." + ], + ) + + # Artefakt im Index finden + entry = self.index.get_entry(artifact_id) + if entry is None: + return ShareResult( + success=False, + artifact_id=artifact_id, + errors=[f"Artefakt nicht im Index gefunden: {artifact_id}"], + ) + + # Inhalt laden und auf sensitive Inhalte prüfen (Req 5.1, 5.3) + content = self._load_artifact_content(entry) + if content is None: + return ShareResult( + success=False, + artifact_id=artifact_id, + errors=[ + f"Artefakt-Datei nicht gefunden oder nicht lesbar: {entry.path}" + ], + ) + + sensitive_matches = self.check_sensitive_content(content) + if sensitive_matches: + reasons = [ + f"Zeile {m.line_number}: {m.reason} ('{m.matched_text}')" + for m in sensitive_matches + ] + return ShareResult( + success=False, + artifact_id=artifact_id, + errors=[ + "Freigabe verweigert: Artefakt enthält sensible Inhalte.", + *reasons, + ], + ) + + # Zielkontexte ermitteln (alle Kontexte außer dem Quellkontext) + source_context = entry.scope + target_contexts = [ + ctx.value for ctx in Context if ctx.value != source_context + ] + + # In Registry eintragen + self._registry[artifact_id] = SharedArtifactEntry( + artifact_id=artifact_id, + source_context=source_context, + shared_to=target_contexts, + content_hash=entry.content_hash, + ) + + # Registry persistieren + self._save_registry() + + logger.info( + "Artefakt '%s' aus Kontext '%s' für Kontexte %s freigegeben.", + artifact_id, + source_context, + target_contexts, + ) + + return ShareResult( + success=True, + artifact_id=artifact_id, + shared_to=target_contexts, + ) + + def check_sensitive_content(self, content: str) -> list[SensitiveMatch]: + """Prüft Inhalt auf kontextspezifische Geheimnisse und PII. + + Verwendet Regex-Muster zur Erkennung von: + - Credentials (API-Keys, Tokens, Passwörter, Secrets) + - Endpoint-URLs + - Personenbezogene Daten (E-Mail-Adressen) + + Args: + content: Der zu prüfende Textinhalt. + + Returns: + Liste der gefundenen sensitiven Stellen. Leere Liste wenn + kein sensitiver Inhalt gefunden wurde. + + Requirements: 5.1, 5.3 + """ + matches: list[SensitiveMatch] = [] + lines = content.splitlines() + + for line_num, line in enumerate(lines, start=1): + for pattern_str, pattern_name, reason in self.SENSITIVE_PATTERNS: + pattern = re.compile(pattern_str) + for match in pattern.finditer(line): + matched_text = match.group(0) + # Text auf sinnvolle Länge kürzen + if len(matched_text) > 60: + matched_text = matched_text[:57] + "..." + matches.append( + SensitiveMatch( + pattern_name=pattern_name, + matched_text=matched_text, + line_number=line_num, + reason=reason, + ) + ) + + return matches + + def revoke_share(self, artifact_id: str) -> None: + """Widerruft die Freigabe eines geteilten Artefakts. + + Entfernt das Artefakt aus allen Zielkontexten als Lesereferenz. + Nach dem Widerruf ist das Artefakt nur noch im Quellkontext sichtbar. + + Args: + artifact_id: ID des Artefakts, dessen Freigabe widerrufen wird. + + Raises: + ValueError: Wenn das Artefakt nicht in der Freigabe-Registry ist. + + Requirements: 5.6 + """ + if artifact_id not in self._registry: + raise ValueError( + f"Artefakt '{artifact_id}' ist nicht freigegeben " + f"und kann daher nicht widerrufen werden." + ) + + entry = self._registry.pop(artifact_id) + self._save_registry() + + logger.info( + "Freigabe von Artefakt '%s' widerrufen. " + "Entfernt aus Kontexten: %s", + artifact_id, + entry.shared_to, + ) + + def get_shared_content(self, artifact_id: str, requesting_context: str) -> str | None: + """Liest den aktuellen Inhalt eines geteilten Artefakts. + + Liefert immer den aktuellen Stand aus dem Quellkontext, + sodass Updates automatisch beim nächsten Lesezugriff sichtbar sind. + + Args: + artifact_id: ID des geteilten Artefakts. + requesting_context: Der Kontext, aus dem der Zugriff erfolgt. + + Returns: + Den aktuellen Inhalt des Artefakts als String, oder None + wenn der Zugriff nicht erlaubt ist. + + Requirements: 5.2, 5.5 + """ + # Prüfe ob Artefakt freigegeben ist + registry_entry = self._registry.get(artifact_id) + if registry_entry is None: + return None + + # Prüfe ob der anfragende Kontext berechtigt ist + if requesting_context not in registry_entry.shared_to: + return None + + # Lade den aktuellen Inhalt aus dem Quellkontext (Req 5.5: Update-Propagierung) + index_entry = self.index.get_entry(artifact_id) + if index_entry is None: + return None + + return self._load_artifact_content(index_entry) + + def is_shared(self, artifact_id: str) -> bool: + """Prüft ob ein Artefakt aktuell freigegeben ist. + + Args: + artifact_id: ID des zu prüfenden Artefakts. + + Returns: + True wenn das Artefakt freigegeben ist. + """ + return artifact_id in self._registry + + def get_shared_artifacts(self, context: str) -> list[str]: + """Liefert alle Artefakt-IDs, die in einem bestimmten Kontext sichtbar sind. + + Args: + context: Der Kontext, für den die sichtbaren Artefakte ermittelt werden. + + Returns: + Liste der Artefakt-IDs, die im angegebenen Kontext als + Lesereferenz verfügbar sind. + """ + return [ + entry.artifact_id + for entry in self._registry.values() + if context in entry.shared_to + ] + + # --------------------------------------------------------------------------- + # Interne Methoden + # --------------------------------------------------------------------------- + + def _load_artifact_content(self, entry: IndexEntry) -> str | None: + """Lädt den Textinhalt eines Artefakts aus dem Dateisystem. + + Args: + entry: Der IndexEntry des Artefakts. + + Returns: + Den Inhalt als String oder None bei Fehler. + """ + if not entry.path: + return None + + artifact_path = self.knowledge_base_path / entry.path + if not artifact_path.exists(): + logger.warning("Artefakt-Datei nicht gefunden: %s", artifact_path) + return None + + try: + artifact = parse_frontmatter(artifact_path) + return artifact.content + except (ValueError, FileNotFoundError, OSError) as e: + logger.warning( + "Fehler beim Laden von Artefakt '%s': %s", entry.id, e + ) + return None + + def _load_registry(self) -> None: + """Lädt die Shared-Artifacts-Registry von der Festplatte.""" + if self.registry_path is None or not self.registry_path.exists(): + return + + try: + with open(self.registry_path, encoding="utf-8") as f: + data = yaml.safe_load(f) + except (yaml.YAMLError, OSError) as e: + logger.warning("Fehler beim Laden der Registry: %s", e) + return + + if not isinstance(data, dict): + return + + for artifact_data in data.get("shared_artifacts", []): + if isinstance(artifact_data, dict): + entry = SharedArtifactEntry( + artifact_id=str(artifact_data.get("artifact_id", "")), + source_context=str(artifact_data.get("source_context", "")), + shared_to=list(artifact_data.get("shared_to", [])), + content_hash=str(artifact_data.get("content_hash", "")), + ) + if entry.artifact_id: + self._registry[entry.artifact_id] = entry + + def _save_registry(self) -> None: + """Persistiert die Shared-Artifacts-Registry auf die Festplatte.""" + if self.registry_path is None: + return + + data: dict[str, Any] = { + "shared_artifacts": [ + { + "artifact_id": entry.artifact_id, + "source_context": entry.source_context, + "shared_to": entry.shared_to, + "content_hash": entry.content_hash, + } + for entry in self._registry.values() + ] + } + + self.registry_path.parent.mkdir(parents=True, exist_ok=True) + + with open(self.registry_path, "w", encoding="utf-8") as f: + yaml.dump( + data, + f, + default_flow_style=False, + allow_unicode=True, + sort_keys=False, + ) diff --git a/shared/tools/monorepo-cli/src/monorepo/cli.py b/shared/tools/monorepo-cli/src/monorepo/cli.py new file mode 100644 index 0000000..141e592 --- /dev/null +++ b/shared/tools/monorepo-cli/src/monorepo/cli.py @@ -0,0 +1,735 @@ +"""CLI-Einstiegspunkt für das Monorepo-Verwaltungstool (ctx-guard). + +Stellt alle Verwaltungskommandos als Subcommands bereit und verdrahtet +die Komponenten: StructureManager, ContextGuard, KnowledgeStore, RepoManager, +ContextBridge, MigrationEngine, OrchestratorAdapter, SecretEncryptionManager, +FederationManager. + +Konfiguration wird aus `monorepo.yaml` und `shared/config/` geladen. + +Entry-Point: ctx-guard (pyproject.toml → [project.scripts]) +Requirements: 1.1, 1.2, 4.4, 6.3, 9.6, 10.11 +""" + +from __future__ import annotations + +import argparse +import logging +import sys +from pathlib import Path + +from monorepo.config import MonorepoConfig, load_monorepo_config + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Hilfsfunktionen für Komponenteninitialisierung +# --------------------------------------------------------------------------- + + +def _find_monorepo_root() -> Path: + """Sucht das Monorepo-Root anhand von monorepo.yaml. + + Sucht im aktuellen Verzeichnis und in übergeordneten Verzeichnissen + nach monorepo.yaml und gibt das Verzeichnis zurück. + + Returns: + Pfad zum Monorepo-Root. + + Raises: + SystemExit: Wenn kein monorepo.yaml gefunden wird. + """ + current = Path.cwd() + for parent in [current, *current.parents]: + if (parent / "monorepo.yaml").exists(): + return parent + print("FEHLER: monorepo.yaml nicht gefunden.", file=sys.stderr) + print("Bitte im Monorepo-Root ausführen oder `ctx-guard init` verwenden.", file=sys.stderr) + sys.exit(1) + + +def _load_config_safe() -> MonorepoConfig: + """Lädt die Monorepo-Konfiguration mit Fehlerbehandlung. + + Returns: + Geladene MonorepoConfig. + + Raises: + SystemExit: Bei Konfigurationsfehlern. + """ + try: + return load_monorepo_config() + except FileNotFoundError as e: + print(f"FEHLER: {e}", file=sys.stderr) + sys.exit(1) + except Exception as e: + print(f"FEHLER beim Laden der Konfiguration: {e}", file=sys.stderr) + sys.exit(1) + + +def _get_structure_manager(root: Path) -> "StructureManager": + """Erstellt einen StructureManager für das gegebene Root.""" + from monorepo.structure import StructureManager + return StructureManager(root) + + +def _get_context_guard(root: Path) -> "ContextGuard": + """Erstellt einen ContextGuard für das gegebene Root.""" + from monorepo.security import ContextGuard + return ContextGuard(root) + + +def _get_knowledge_store(root: Path): + """Erstellt einen KnowledgeStore für das gegebene Root.""" + from monorepo.knowledge.store import KnowledgeStore + from monorepo.models import ScopeConfig + + ks_path = root / "shared" / "knowledge-store" + scope_config = ScopeConfig(scopes={ + "privat": {"scope": "privat", "paths": ["privat/"]}, + "dhive": {"scope": "dhive", "paths": ["dhive/"]}, + "bahn": {"scope": "bahn", "paths": ["bahn/"]}, + "shared": {"scope": "shared", "paths": ["shared/"]}, + }) + return KnowledgeStore(base_path=ks_path, scope_config=scope_config) + + +def _get_repo_manager(root: Path) -> "RepoManager": + """Erstellt einen RepoManager für das gegebene Root.""" + from monorepo.repos import RepoManager + config_path = root / "shared" / "config" / "repos.yaml" + return RepoManager(config_path=config_path, monorepo_root=root) + + +def _get_encryption_manager(root: Path): + """Erstellt einen SecretEncryptionManager für das gegebene Root.""" + from monorepo.encryption import MachineContextManager + mcm = MachineContextManager(root) + try: + mcm.load() + except FileNotFoundError: + from monorepo.models import MachineContext + # Fallback: Erstelle einen Default-Maschinenkontext + mcm._machine_context = MachineContext( + name="unknown", + description="Standard-Maschinenkontext (keine Konfiguration gefunden)", + authorized_contexts=["privat", "dhive", "bahn"], + key_source="keyring", + ) + return mcm.create_encryption_manager() + + +def _get_federation_manager(root: Path): + """Erstellt einen FederationManager für das gegebene Root.""" + from monorepo.federation import FederationManager + config_path = root / "shared" / "config" / "team-repos.yaml" + encryption_mgr = _get_encryption_manager(root) + return FederationManager( + config_path=config_path, + monorepo_root=root, + encryption_manager=encryption_mgr, + ) + + +def _get_migration_engine(root: Path): + """Erstellt eine MigrationEngine für das gegebene Root.""" + from monorepo.migration import MigrationEngine + return MigrationEngine(monorepo_root=root) + + +# --------------------------------------------------------------------------- +# Subcommand-Handler +# --------------------------------------------------------------------------- + + +def _cmd_init(args: argparse.Namespace) -> int: + """Initialisiert die Monorepo-Grundstruktur. + + Erstellt die vier Kontextordner und die shared-Unterstruktur, + sowie eine minimale monorepo.yaml falls sie nicht existiert. + """ + target = Path(args.path).resolve() if args.path else Path.cwd() + + print(f"Initialisiere Monorepo in: {target}") + + # Kontextordner erstellen + contexts = ["privat", "dhive", "bahn", "shared"] + for ctx in contexts: + (target / ctx).mkdir(parents=True, exist_ok=True) + + # Shared-Unterstruktur + shared_dirs = [ + "shared/tools", + "shared/powers", + "shared/knowledge-store", + "shared/config", + "shared/mcp-servers", + ] + for d in shared_dirs: + (target / d).mkdir(parents=True, exist_ok=True) + + # monorepo.yaml erstellen falls nicht vorhanden + config_path = target / "monorepo.yaml" + if not config_path.exists(): + import yaml + config_data = { + "version": "1.0", + "contexts": [ + {"name": "privat", "description": "Persönliche Projekte"}, + {"name": "dhive", "description": "dhive GmbH Projekte"}, + {"name": "bahn", "description": "DB InfraGO Projekte"}, + {"name": "shared", "description": "Kontextübergreifende Tools und Wissen"}, + ], + "naming": { + "pattern": r"^[a-z0-9][a-z0-9\-]{0,48}[a-z0-9]$", + "min_length": 2, + "max_length": 50, + }, + "security": { + "config": "shared/config/access-config.yaml", + "audit_log": ".audit/access.log", + "encryption": { + "tool": "git-crypt", + "machine_context_config": "shared/config/machine-context.yaml", + "key_source": "keyring", + }, + "federation": { + "config": "shared/config/team-repos.yaml", + "conflict_strategy": "team-wins", + }, + }, + } + with open(config_path, "w", encoding="utf-8") as f: + yaml.dump(config_data, f, default_flow_style=False, allow_unicode=True) + print(f" Erstellt: {config_path}") + + # Audit-Verzeichnis + (target / ".audit").mkdir(parents=True, exist_ok=True) + + # Git-Hooks installieren (pre-commit mit Read-Only-Schutz + Secret-Prüfung) + git_dir = target / ".git" + if git_dir.exists(): + from monorepo.hooks import install_hooks + try: + install_hooks(target) + print(" Installiert: Git pre-commit Hook (Read-Only + Secrets)") + except Exception as e: + logger.warning("Hook-Installation übersprungen: %s", e) + print(f" ⚠ Hook-Installation übersprungen: {e}") + + print("✓ Monorepo-Struktur erfolgreich initialisiert.") + return 0 + + +def _cmd_create_project(args: argparse.Namespace) -> int: + """Erstellt ein neues Projekt in einem Arbeitskontext.""" + root = _find_monorepo_root() + sm = _get_structure_manager(root) + + try: + project_path = sm.create_project(context=args.context, name=args.name) + print(f"✓ Projekt erstellt: {project_path}") + return 0 + except ValueError as e: + print(f"FEHLER: {e}", file=sys.stderr) + return 1 + + +def _cmd_list_projects(args: argparse.Namespace) -> int: + """Listet alle Projekte auf, optional gefiltert nach Kontext.""" + root = _find_monorepo_root() + sm = _get_structure_manager(root) + + try: + projects = sm.list_projects(context=args.context) + except ValueError as e: + print(f"FEHLER: {e}", file=sys.stderr) + return 1 + + if not projects: + ctx_info = f" im Kontext '{args.context}'" if args.context else "" + print(f"Keine Projekte{ctx_info} gefunden.") + return 0 + + # Tabellen-Ausgabe + print(f"{'Kontext':<10} {'Name':<40} {'Pfad'}") + print("-" * 80) + for p in projects: + print(f"{p.context:<10} {p.name:<40} {p.path}") + + print(f"\nGesamt: {len(projects)} Projekt(e)") + return 0 + + +def _cmd_add_repo(args: argparse.Namespace) -> int: + """Bindet ein externes Repository ein.""" + root = _find_monorepo_root() + rm = _get_repo_manager(root) + + from monorepo.models import RepoEntry + entry = RepoEntry( + name=args.name, + url=args.url, + mode=args.mode, + target=args.target, + pinned=args.pinned, + mechanism=args.mechanism, + ) + + try: + rm.add_repo(entry) + print(f"✓ Repo '{args.name}' eingebunden via {args.mechanism} → {args.target}") + return 0 + except (ValueError, Exception) as e: + print(f"FEHLER: {e}", file=sys.stderr) + return 1 + + +def _cmd_sync_repo(args: argparse.Namespace) -> int: + """Synchronisiert ein eingebundenes Repository.""" + root = _find_monorepo_root() + rm = _get_repo_manager(root) + + result = rm.sync(args.name) + + if result.success: + print(f"✓ Repo '{args.name}' synchronisiert ({result.commits_synced} Commits).") + return 0 + else: + print(f"FEHLER bei Synchronisation von '{args.name}':", file=sys.stderr) + for conflict in result.conflicts: + print(f" - {conflict.details}", file=sys.stderr) + return 1 + + +def _cmd_migrate(args: argparse.Namespace) -> int: + """Migriert ein bestehendes Repository ins Monorepo.""" + root = _find_monorepo_root() + engine = _get_migration_engine(root) + + from monorepo.models import MigrationPlan + plan = MigrationPlan( + source_repo=args.source, + target_context=args.context, + target_name=args.name, + mode=args.mode, + dependencies=[], + order=0, + ) + + print(f"Migriere '{args.source}' → {args.context}/{args.name} (Modus: {args.mode})...") + result = engine.migrate(plan) + + if result.success: + print(f"✓ Migration erfolgreich abgeschlossen.") + if result.commits_migrated: + print(f" Commits: {result.commits_migrated}") + if result.branches_migrated: + print(f" Branches: {', '.join(result.branches_migrated)}") + if result.tags_migrated: + print(f" Tags: {', '.join(result.tags_migrated)}") + return 0 + else: + print(f"FEHLER bei Migration:", file=sys.stderr) + if result.error_message: + print(f" {result.error_message}", file=sys.stderr) + if result.conflicts: + print(f"\nKonflikte ({len(result.conflicts)}):", file=sys.stderr) + for c in result.conflicts: + print(f" [{c.conflict_type}] {c.description}", file=sys.stderr) + return 1 + + +def _cmd_validate(args: argparse.Namespace) -> int: + """Validiert eine abgeschlossene Migration.""" + root = _find_monorepo_root() + engine = _get_migration_engine(root) + + print(f"Validiere Migration von '{args.name}'...") + result = engine.validate(args.name) + + if result.success: + print(f"✓ Validierung erfolgreich für '{args.name}'.") + for check_name, passed in result.checks.items(): + status = "✓" if passed else "✗" + print(f" {status} {check_name}") + return 0 + else: + print(f"FEHLER: Validierung fehlgeschlagen für '{args.name}':", file=sys.stderr) + if result.error_message: + print(f" {result.error_message}", file=sys.stderr) + for detail in result.details: + print(f" - {detail}", file=sys.stderr) + return 1 + + +def _cmd_search(args: argparse.Namespace) -> int: + """Durchsucht den Wissensspeicher.""" + root = _find_monorepo_root() + ks = _get_knowledge_store(root) + + # Bestimme erlaubte Scopes (standardmäßig alle) + allowed_scopes = args.scopes.split(",") if args.scopes else ["privat", "dhive", "bahn", "shared"] + + results = ks.search(query=args.query, allowed_scopes=allowed_scopes) + + if not results: + print(f"Keine Treffer für '{args.query}'.") + return 0 + + print(f"Treffer für '{args.query}' ({len(results)} Ergebnis(se)):\n") + for r in results: + score_bar = "█" * int(r.relevance_score * 10) + print(f" [{r.entry.scope}] {r.entry.title}") + print(f" Pfad: {r.entry.path}") + print(f" Tags: {', '.join(r.entry.tags)}") + print(f" Relevanz: {score_bar} ({r.relevance_score:.2f})") + print() + + if results and results[0].partial_results: + print("⚠ Suche wegen Timeout abgebrochen – möglicherweise unvollständig.") + + return 0 + + +def _cmd_encrypt(args: argparse.Namespace) -> int: + """Verschlüsselt eine Datei mit dem Kontext-Schlüssel.""" + root = _find_monorepo_root() + enc = _get_encryption_manager(root) + + file_path = Path(args.file).resolve() + context = args.context + + if not context: + # Kontext aus Pfad ableiten + try: + rel = file_path.relative_to(root) + context = rel.parts[0] if rel.parts else None + except ValueError: + pass + + if not context: + print("FEHLER: Kontext konnte nicht ermittelt werden. Bitte --context angeben.", file=sys.stderr) + return 1 + + result = enc.encrypt_file(file_path, context) + + if result.success: + print(f"✓ Datei verschlüsselt: {file_path} (Kontext: {context})") + return 0 + else: + print(f"FEHLER: {result.error}", file=sys.stderr) + return 1 + + +def _cmd_decrypt(args: argparse.Namespace) -> int: + """Entschlüsselt eine Datei.""" + root = _find_monorepo_root() + enc = _get_encryption_manager(root) + + file_path = Path(args.file).resolve() + result = enc.decrypt_file(file_path) + + if result.success: + print(f"✓ Datei entschlüsselt: {file_path}") + return 0 + else: + print(f"FEHLER: {result.error}", file=sys.stderr) + return 1 + + +def _cmd_onboard(args: argparse.Namespace) -> int: + """Richtet eine neue Maschine mit autorisierten Schlüsseln ein.""" + root = _find_monorepo_root() + enc = _get_encryption_manager(root) + + contexts = [c.strip() for c in args.contexts.split(",")] + print(f"Onboarding Maschine '{args.machine_name}' für Kontexte: {', '.join(contexts)}...") + + result = enc.onboard_machine(args.machine_name, contexts) + + if result.success: + print(f"✓ Maschine '{args.machine_name}' erfolgreich eingerichtet.") + print(f" Autorisierte Kontexte: {', '.join(result.authorized_contexts)}") + print(f" Installierte Schlüssel: {len(result.installed_keys)}") + return 0 + else: + print(f"FEHLER beim Onboarding:", file=sys.stderr) + for err in result.errors: + print(f" - {err}", file=sys.stderr) + return 1 + + +def _cmd_fed_sync(args: argparse.Namespace) -> int: + """Synchronisiert mit einem Team-Repository.""" + root = _find_monorepo_root() + fm = _get_federation_manager(root) + + context = args.context + direction = args.direction + + print(f"Federation-Sync: Kontext '{context}', Richtung '{direction}'...") + + if direction == "pull": + result = fm.sync_from_team(context) + elif direction == "push": + result = fm.sync_to_team(context) + else: + result = fm.full_sync(context) + + if result.success: + print(f"✓ Sync erfolgreich ({result.commits_synced} Commits, Richtung: {result.direction}).") + return 0 + else: + print(f"FEHLER bei Federation-Sync:", file=sys.stderr) + for conflict in result.conflicts: + print(f" - [{conflict.conflict_type}] {conflict.details}", file=sys.stderr) + return 1 + + +def _cmd_fed_status(args: argparse.Namespace) -> int: + """Zeigt den Federation-Status für alle oder einen bestimmten Kontext.""" + root = _find_monorepo_root() + fm = _get_federation_manager(root) + + contexts = [args.context] if args.context else ["privat", "dhive", "bahn"] + + print("Federation-Status:") + print("-" * 60) + + for ctx in contexts: + entry = fm._get_team_entry(ctx) + if entry is None: + print(f" {ctx:<10} Nicht konfiguriert") + continue + + # Prüfe auf Konflikte + conflicts = fm.detect_conflicts(ctx) + conflict_status = f"⚠ {len(conflicts)} Konflikt(e)" if conflicts else "✓ Keine Konflikte" + + print(f" {ctx:<10} URL: {entry.url}") + print(f" {'':10} Branch: {entry.branch}") + print(f" {'':10} Sync: {entry.sync_direction} ({entry.sync_frequency})") + print(f" {'':10} Status: {conflict_status}") + if entry.shared_mirror: + mirror_status = "aktiv" if entry.shared_mirror.enabled else "deaktiviert" + print(f" {'':10} Shared-Mirror: {mirror_status}") + print() + + return 0 + + +def _cmd_install_hooks(args: argparse.Namespace) -> int: + """Installiert oder aktualisiert die Monorepo Git-Hooks.""" + root = _find_monorepo_root() + + from monorepo.hooks import install_hooks + try: + install_hooks(root) + print("✓ Git-Hooks installiert/aktualisiert.") + return 0 + except Exception as e: + print(f"FEHLER bei Hook-Installation: {e}", file=sys.stderr) + return 1 + + +def _cmd_uninstall_hooks(args: argparse.Namespace) -> int: + """Entfernt die Monorepo Git-Hooks.""" + root = _find_monorepo_root() + + from monorepo.hooks import uninstall_hooks + try: + uninstall_hooks(root) + print("✓ Git-Hooks entfernt.") + return 0 + except Exception as e: + print(f"FEHLER bei Hook-Deinstallation: {e}", file=sys.stderr) + return 1 + + +# --------------------------------------------------------------------------- +# Argument-Parser-Aufbau +# --------------------------------------------------------------------------- + + +def _build_parser() -> argparse.ArgumentParser: + """Baut den vollständigen Argument-Parser mit allen Subcommands auf.""" + parser = argparse.ArgumentParser( + prog="ctx-guard", + description="Monorepo-Verwaltungstool: Kontextgrenzen, Wissensspeicher, " + "Repo-Sync, Verschlüsselung und Föderation.", + ) + parser.add_argument( + "-v", "--verbose", + action="store_true", + help="Ausführliche Logging-Ausgabe aktivieren.", + ) + + subparsers = parser.add_subparsers(dest="command", help="Verfügbare Kommandos") + + # --- init --- + p_init = subparsers.add_parser("init", help="Monorepo-Struktur initialisieren") + p_init.add_argument( + "--path", type=str, default=None, + help="Zielpfad für die Initialisierung (Standard: aktuelles Verzeichnis).", + ) + + # --- create-project --- + p_create = subparsers.add_parser("create-project", help="Neues Projekt anlegen") + p_create.add_argument("context", choices=["privat", "dhive", "bahn", "shared"], + help="Arbeitskontext für das Projekt.") + p_create.add_argument("name", help="Projektname (kebab-case, 2-50 Zeichen).") + + # --- list-projects --- + p_list = subparsers.add_parser("list-projects", help="Projekte auflisten") + p_list.add_argument( + "--context", "-c", + choices=["privat", "dhive", "bahn", "shared"], + default=None, + help="Nur Projekte des angegebenen Kontexts anzeigen.", + ) + + # --- add-repo --- + p_add = subparsers.add_parser("add-repo", help="Externes Repository einbinden") + p_add.add_argument("name", help="Eindeutiger Name für das Repo.") + p_add.add_argument("url", help="Repository-URL.") + p_add.add_argument("target", help="Zielpfad im Monorepo (z.B. 'bahn/db-wissen').") + p_add.add_argument( + "--mode", choices=["read-only", "upstream"], default="read-only", + help="Einbindungsmodus (Standard: read-only).", + ) + p_add.add_argument( + "--pinned", default="main", + help="Gepinnte Version: Branch, Tag oder SHA (Standard: main).", + ) + p_add.add_argument( + "--mechanism", choices=["subtree", "submodule"], default="subtree", + help="Einbindungsmechanismus (Standard: subtree).", + ) + + # --- sync-repo --- + p_sync = subparsers.add_parser("sync-repo", help="Repository synchronisieren") + p_sync.add_argument("name", help="Name des zu synchronisierenden Repos.") + + # --- migrate --- + p_migrate = subparsers.add_parser("migrate", help="Repository ins Monorepo migrieren") + p_migrate.add_argument("source", help="Pfad zum Quell-Repository.") + p_migrate.add_argument("context", choices=["privat", "dhive", "bahn", "shared"], + help="Ziel-Arbeitskontext.") + p_migrate.add_argument("name", help="Projektname im Ziel.") + p_migrate.add_argument( + "--mode", choices=["direct", "subtree", "upstream"], default="direct", + help="Migrationsmodus (Standard: direct).", + ) + + # --- validate --- + p_validate = subparsers.add_parser("validate", help="Migration validieren") + p_validate.add_argument("name", help="Name des migrierten Repos/Projekts.") + + # --- search --- + p_search = subparsers.add_parser("search", help="Wissensspeicher durchsuchen") + p_search.add_argument("query", help="Suchanfrage.") + p_search.add_argument( + "--scopes", "-s", default=None, + help="Komma-getrennte Scopes (Standard: alle). Z.B. 'privat,bahn'.", + ) + + # --- encrypt --- + p_encrypt = subparsers.add_parser("encrypt", help="Datei verschlüsseln") + p_encrypt.add_argument("file", help="Pfad zur zu verschlüsselnden Datei.") + p_encrypt.add_argument( + "--context", "-c", default=None, + help="Arbeitskontext (wird aus Pfad abgeleitet wenn nicht angegeben).", + ) + + # --- decrypt --- + p_decrypt = subparsers.add_parser("decrypt", help="Datei entschlüsseln") + p_decrypt.add_argument("file", help="Pfad zur zu entschlüsselnden Datei.") + + # --- onboard --- + p_onboard = subparsers.add_parser("onboard", help="Neue Maschine einrichten") + p_onboard.add_argument("machine_name", help="Name der Maschine.") + p_onboard.add_argument( + "contexts", + help="Komma-getrennte Liste autorisierter Kontexte (z.B. 'privat,dhive,bahn').", + ) + + # --- fed-sync --- + p_fedsync = subparsers.add_parser("fed-sync", help="Team-Repo synchronisieren") + p_fedsync.add_argument("context", choices=["privat", "dhive", "bahn"], + help="Arbeitskontext des Team-Repos.") + p_fedsync.add_argument( + "--direction", "-d", + choices=["pull", "push", "full"], default="full", + help="Sync-Richtung (Standard: full/bidirektional).", + ) + + # --- fed-status --- + p_fedstatus = subparsers.add_parser("fed-status", help="Federation-Status anzeigen") + p_fedstatus.add_argument( + "--context", "-c", + choices=["privat", "dhive", "bahn"], + default=None, + help="Nur Status für den angegebenen Kontext anzeigen.", + ) + + # --- install-hooks --- + subparsers.add_parser("install-hooks", help="Git-Hooks installieren/aktualisieren") + + # --- uninstall-hooks --- + subparsers.add_parser("uninstall-hooks", help="Git-Hooks entfernen") + + return parser + + +# --------------------------------------------------------------------------- +# Dispatch +# --------------------------------------------------------------------------- + +_COMMAND_HANDLERS = { + "init": _cmd_init, + "create-project": _cmd_create_project, + "list-projects": _cmd_list_projects, + "add-repo": _cmd_add_repo, + "sync-repo": _cmd_sync_repo, + "migrate": _cmd_migrate, + "validate": _cmd_validate, + "search": _cmd_search, + "encrypt": _cmd_encrypt, + "decrypt": _cmd_decrypt, + "onboard": _cmd_onboard, + "fed-sync": _cmd_fed_sync, + "fed-status": _cmd_fed_status, + "install-hooks": _cmd_install_hooks, + "uninstall-hooks": _cmd_uninstall_hooks, +} + + +def main() -> None: + """Haupt-Einstiegspunkt für das CLI-Tool ctx-guard.""" + parser = _build_parser() + args = parser.parse_args() + + # Logging konfigurieren + log_level = logging.DEBUG if args.verbose else logging.WARNING + logging.basicConfig( + level=log_level, + format="%(levelname)s: %(name)s: %(message)s", + ) + + if not args.command: + parser.print_help() + sys.exit(0) + + handler = _COMMAND_HANDLERS.get(args.command) + if handler is None: + parser.print_help() + sys.exit(1) + + exit_code = handler(args) + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/shared/tools/monorepo-cli/src/monorepo/config.py b/shared/tools/monorepo-cli/src/monorepo/config.py new file mode 100644 index 0000000..90bfcb5 --- /dev/null +++ b/shared/tools/monorepo-cli/src/monorepo/config.py @@ -0,0 +1,173 @@ +"""Konfigurationsmodul für das Monorepo-CLI. + +Lädt und validiert die zentrale monorepo.yaml und stellt die Konfiguration +als typisierte Datenstrukturen bereit. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import yaml + + +@dataclass +class NamingConfig: + """Namenskonventions-Konfiguration.""" + + pattern: str = r"^[a-z0-9][a-z0-9\-]{0,48}[a-z0-9]$" + min_length: int = 2 + max_length: int = 50 + + +@dataclass +class EncryptionConfig: + """Verschlüsselungs-Konfiguration.""" + + tool: str = "git-crypt" + machine_context_config: str = "shared/config/machine-context.yaml" + key_source: str = "keyring" + + +@dataclass +class FederationConfig: + """Föderations-Konfiguration.""" + + config: str = "shared/config/team-repos.yaml" + conflict_strategy: str = "team-wins" + + +@dataclass +class SecurityConfig: + """Sicherheits-Konfiguration.""" + + config: str = "shared/config/access-config.yaml" + audit_log: str = ".audit/access.log" + encryption: EncryptionConfig = field(default_factory=EncryptionConfig) + federation: FederationConfig = field(default_factory=FederationConfig) + + +@dataclass +class ContextDefinition: + """Definition eines Arbeitskontexts.""" + + name: str + description: str + + +@dataclass +class MonorepoConfig: + """Zentrale Monorepo-Konfiguration (monorepo.yaml).""" + + version: str = "1.0" + contexts: list[ContextDefinition] = field(default_factory=list) + naming: NamingConfig = field(default_factory=NamingConfig) + security: SecurityConfig = field(default_factory=SecurityConfig) + + +def _parse_encryption_config(data: dict[str, Any]) -> EncryptionConfig: + """Parst die Verschlüsselungskonfiguration aus einem dict.""" + return EncryptionConfig( + tool=data.get("tool", "git-crypt"), + machine_context_config=data.get("machine_context_config", "shared/config/machine-context.yaml"), + key_source=data.get("key_source", "keyring"), + ) + + +def _parse_federation_config(data: dict[str, Any]) -> FederationConfig: + """Parst die Föderationskonfiguration aus einem dict.""" + return FederationConfig( + config=data.get("config", "shared/config/team-repos.yaml"), + conflict_strategy=data.get("conflict_strategy", "team-wins"), + ) + + +def _parse_security_config(data: dict[str, Any]) -> SecurityConfig: + """Parst die Sicherheitskonfiguration aus einem dict.""" + encryption_data = data.get("encryption", {}) + federation_data = data.get("federation", {}) + return SecurityConfig( + config=data.get("config", "shared/config/access-config.yaml"), + audit_log=data.get("audit_log", ".audit/access.log"), + encryption=_parse_encryption_config(encryption_data), + federation=_parse_federation_config(federation_data), + ) + + +def _parse_naming_config(data: dict[str, Any]) -> NamingConfig: + """Parst die Namenskonventions-Konfiguration aus einem dict.""" + return NamingConfig( + pattern=data.get("pattern", r"^[a-z0-9][a-z0-9\-]{0,48}[a-z0-9]$"), + min_length=data.get("min_length", 2), + max_length=data.get("max_length", 50), + ) + + +def load_monorepo_config(config_path: Path | None = None) -> MonorepoConfig: + """Lädt die zentrale Monorepo-Konfiguration aus monorepo.yaml. + + Args: + config_path: Pfad zur monorepo.yaml. Falls None, wird im aktuellen + Verzeichnis und darüber gesucht. + + Returns: + MonorepoConfig mit allen geladenen Einstellungen. + + Raises: + FileNotFoundError: Wenn die Konfigurationsdatei nicht gefunden wird. + yaml.YAMLError: Wenn die YAML-Datei nicht geparst werden kann. + """ + if config_path is None: + config_path = _find_config_file() + + if not config_path.exists(): + raise FileNotFoundError(f"Monorepo-Konfiguration nicht gefunden: {config_path}") + + with open(config_path, encoding="utf-8") as f: + data = yaml.safe_load(f) + + if not isinstance(data, dict): + raise ValueError(f"Ungültiges monorepo.yaml-Format: Erwartet dict, erhalten {type(data)}") + + # Kontexte parsen + contexts: list[ContextDefinition] = [] + for ctx in data.get("contexts", []): + contexts.append(ContextDefinition( + name=ctx.get("name", ""), + description=ctx.get("description", ""), + )) + + # Naming-Konfiguration + naming = _parse_naming_config(data.get("naming", {})) + + # Security-Konfiguration + security = _parse_security_config(data.get("security", {})) + + return MonorepoConfig( + version=data.get("version", "1.0"), + contexts=contexts, + naming=naming, + security=security, + ) + + +def _find_config_file() -> Path: + """Sucht monorepo.yaml im aktuellen Verzeichnis und darüber. + + Returns: + Pfad zur gefundenen monorepo.yaml. + + Raises: + FileNotFoundError: Wenn keine monorepo.yaml gefunden wird. + """ + current = Path.cwd() + for parent in [current, *current.parents]: + candidate = parent / "monorepo.yaml" + if candidate.exists(): + return candidate + raise FileNotFoundError( + "monorepo.yaml nicht gefunden. Bitte im Monorepo-Root ausführen " + "oder config_path explizit angeben." + ) diff --git a/shared/tools/monorepo-cli/src/monorepo/encryption.py b/shared/tools/monorepo-cli/src/monorepo/encryption.py new file mode 100644 index 0000000..ebfa957 --- /dev/null +++ b/shared/tools/monorepo-cli/src/monorepo/encryption.py @@ -0,0 +1,826 @@ +"""Verschlüsselungsmodul für das Monorepo-CLI. + +Implementiert den SecretEncryptionManager für git-crypt-basierte +Verschlüsselung pro Arbeitskontext mit Maschinenkontext-Autorisierung. + +Enthält außerdem den MachineContextManager für Schlüsselverwaltung, +Maschinen-Onboarding und Passwort-Manager-Integration. +""" + +from __future__ import annotations + +import logging +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +import yaml + +from monorepo.models import ( + EncryptionKey, + MachineContext, + OnboardingResult, + PasswordManagerConfig, +) + + +# --------------------------------------------------------------------------- +# Ergebnis-Dataclasses +# --------------------------------------------------------------------------- + + +@dataclass +class EncryptionResult: + """Ergebnis einer Verschlüsselungsoperation.""" + + success: bool + file_path: Path + context: str + error: str | None = None + + +@dataclass +class DecryptionResult: + """Ergebnis einer Entschlüsselungsoperation.""" + + success: bool + file_path: Path + content: bytes | None = None + error: str | None = None + + +# --------------------------------------------------------------------------- +# SecretEncryptionManager +# --------------------------------------------------------------------------- + +#: Dateimuster, die als Secret-Dateien behandelt werden. +SECRET_PATTERNS: list[str] = [ + "**/.env", + "**/*.pem", + "**/*.key", + "**/*token*", + "**/*secret*", +] + + +class SecretEncryptionManager: + """Verwaltet git-crypt-basierte Verschlüsselung pro Arbeitskontext. + + Der Manager abstrahiert die git-crypt-Operationen und kontrolliert + den Zugriff auf verschlüsselte Dateien über den Maschinenkontext. + + Args: + root_path: Pfad zum Monorepo-Root-Verzeichnis. + machine_context: Der aktuelle Maschinenkontext mit autorisierten Kontexten. + """ + + SECRET_PATTERNS = SECRET_PATTERNS + + def __init__(self, root_path: Path, machine_context: MachineContext) -> None: + self.root_path = root_path + self.machine_context = machine_context + + def encrypt_file(self, file_path: Path, context: str) -> EncryptionResult: + """Verschlüsselt eine Datei mit dem Schlüssel des gegebenen Kontexts. + + Die Verschlüsselung erfolgt über git-crypt-Filter. Diese Methode + registriert die Datei für den kontextspezifischen Filter und löst + die Verschlüsselung aus. + + Args: + file_path: Pfad zur zu verschlüsselnden Datei. + context: Arbeitskontext, dessen Schlüssel verwendet werden soll. + + Returns: + EncryptionResult mit Erfolgs-/Fehlerstatus. + """ + if not file_path.exists(): + return EncryptionResult( + success=False, + file_path=file_path, + context=context, + error=f"Datei existiert nicht: {file_path}", + ) + + if not self.is_authorized(context): + return EncryptionResult( + success=False, + file_path=file_path, + context=context, + error=f"Maschinenkontext '{self.machine_context.name}' ist nicht " + f"für Kontext '{context}' autorisiert", + ) + + try: + self._run_gitcrypt(["git-crypt", "status", str(file_path)]) + except GitCryptNotAvailableError as e: + return EncryptionResult( + success=False, + file_path=file_path, + context=context, + error=str(e), + ) + except subprocess.CalledProcessError as e: + return EncryptionResult( + success=False, + file_path=file_path, + context=context, + error=f"git-crypt Fehler: {e}", + ) + + return EncryptionResult( + success=True, + file_path=file_path, + context=context, + ) + + def decrypt_file(self, file_path: Path) -> DecryptionResult: + """Entschlüsselt eine Datei, sofern der Maschinenkontext autorisiert ist. + + Ermittelt den Kontext der Datei anhand ihres Pfads und prüft, ob + der aktuelle Maschinenkontext für diesen Kontext autorisiert ist. + Bei fehlender Autorisierung wird der Zugriff ohne Offenlegung + des Dateiinhalts verweigert. + + Args: + file_path: Pfad zur zu entschlüsselnden Datei. + + Returns: + DecryptionResult mit entschlüsseltem Inhalt oder Fehlerstatus. + """ + if not file_path.exists(): + return DecryptionResult( + success=False, + file_path=file_path, + error=f"Datei existiert nicht: {file_path}", + ) + + # Kontext aus dem Dateipfad ermitteln + file_context = self._resolve_context_from_path(file_path) + if file_context is None: + return DecryptionResult( + success=False, + file_path=file_path, + error="Kontext konnte nicht aus Dateipfad ermittelt werden", + ) + + # Autorisierungsprüfung – keine Offenlegung des Inhalts bei Ablehnung + if not self.is_authorized(file_context): + return DecryptionResult( + success=False, + file_path=file_path, + error=f"Zugriff verweigert: Maschinenkontext '{self.machine_context.name}' " + f"ist nicht für Kontext '{file_context}' autorisiert", + ) + + try: + self._run_gitcrypt(["git-crypt", "unlock"]) + content = file_path.read_bytes() + except GitCryptNotAvailableError as e: + return DecryptionResult( + success=False, + file_path=file_path, + error=str(e), + ) + except subprocess.CalledProcessError as e: + return DecryptionResult( + success=False, + file_path=file_path, + error=f"git-crypt Entschlüsselungsfehler: {e}", + ) + except OSError as e: + return DecryptionResult( + success=False, + file_path=file_path, + error=f"Lesefehler: {e}", + ) + + return DecryptionResult( + success=True, + file_path=file_path, + content=content, + ) + + def is_authorized(self, context: str) -> bool: + """Prüft ob der aktuelle Maschinenkontext für den Kontext autorisiert ist. + + Args: + context: Zu prüfender Arbeitskontext (z.B. 'privat', 'dhive', 'bahn'). + + Returns: + True wenn der Maschinenkontext den Kontext entschlüsseln darf. + """ + return context in self.machine_context.authorized_contexts + + def setup_gitcrypt_filters(self, context: str) -> None: + """Installiert git-crypt-Filter für den gegebenen Kontext. + + Schreibt oder aktualisiert die `.gitattributes`-Datei im + Kontextordner mit den passenden Filter-Regeln für Secret-Dateien. + + Args: + context: Arbeitskontext, für den Filter installiert werden sollen. + """ + context_dir = self.root_path / context + context_dir.mkdir(parents=True, exist_ok=True) + + gitattributes_path = context_dir / ".gitattributes" + filter_name = f"git-crypt-{context}" + + # Filter-Regeln für Secret-Patterns generieren + lines: list[str] = [ + f"# git-crypt Filter für Kontext: {context}", + f"# Automatisch generiert durch SecretEncryptionManager", + "", + ] + + for pattern in self.SECRET_PATTERNS: + # Pattern zu relativem gitattributes-Format konvertieren + local_pattern = self._pattern_to_gitattributes(pattern) + lines.append( + f"{local_pattern} filter={filter_name} diff={filter_name}" + ) + + lines.append("") # Abschließende Leerzeile + + gitattributes_path.write_text("\n".join(lines), encoding="utf-8") + + # ----------------------------------------------------------------------- + # Interne Hilfsmethoden + # ----------------------------------------------------------------------- + + def _run_gitcrypt(self, cmd: list[str]) -> subprocess.CompletedProcess[bytes]: + """Führt einen git-crypt-Befehl aus. + + Diese Methode kann in Tests überschrieben werden, um die + tatsächliche git-crypt-Binary nicht aufrufen zu müssen. + + Args: + cmd: Befehl und Argumente als Liste. + + Returns: + CompletedProcess-Objekt mit dem Ergebnis. + + Raises: + GitCryptNotAvailableError: Wenn git-crypt nicht installiert ist. + subprocess.CalledProcessError: Wenn der Befehl fehlschlägt. + """ + try: + return subprocess.run( + cmd, + cwd=self.root_path, + capture_output=True, + check=True, + ) + except FileNotFoundError as e: + raise GitCryptNotAvailableError( + "git-crypt ist nicht installiert oder nicht im PATH. " + "Bitte git-crypt installieren: https://github.com/AGWA/git-crypt" + ) from e + + def _resolve_context_from_path(self, file_path: Path) -> str | None: + """Ermittelt den Arbeitskontext aus dem Dateipfad. + + Der Kontext entspricht dem ersten Pfad-Segment relativ zum Root. + Gültige Kontexte: privat, dhive, bahn, shared. + + Args: + file_path: Absoluter oder relativer Pfad zur Datei. + + Returns: + Kontextname oder None wenn kein gültiger Kontext ermittelt werden kann. + """ + valid_contexts = {"privat", "dhive", "bahn", "shared"} + + try: + rel_path = file_path.resolve().relative_to(self.root_path.resolve()) + except ValueError: + return None + + if not rel_path.parts: + return None + + first_part = rel_path.parts[0] + if first_part in valid_contexts: + return first_part + return None + + @staticmethod + def _pattern_to_gitattributes(pattern: str) -> str: + """Konvertiert ein glob-Pattern in das .gitattributes-Format. + + Entfernt den rekursiven '**/' Prefix, da .gitattributes relativ + zum Verzeichnis der Datei wirkt. + + Args: + pattern: Glob-Pattern (z.B. '**/.env', '**/*.pem'). + + Returns: + .gitattributes-kompatibles Pattern. + """ + # '**/' am Anfang entfernen – .gitattributes matcht relativ + if pattern.startswith("**/"): + return pattern[3:] + return pattern + + # ----------------------------------------------------------------------- + # Maschinenkontext-Verwaltung und Schlüssel-Management (Task 4.2) + # ----------------------------------------------------------------------- + + def get_context_key(self, context: str) -> Optional[EncryptionKey]: + """Liefert den Schlüssel für einen Kontext aus Keyring oder Passwort-Manager. + + Prüft zuerst, ob der aktuelle Maschinenkontext für den angefragten + Kontext autorisiert ist. Ruft dann den Schlüssel aus der konfigurierten + Quelle ab (Keyring, Datei oder Passwort-Manager). + + Args: + context: Arbeitskontext, für den der Schlüssel benötigt wird. + + Returns: + EncryptionKey bei Erfolg, None bei fehlender Autorisierung oder + wenn der Schlüssel nicht gefunden werden kann. + """ + if not self.is_authorized(context): + return None + + key_source = self.machine_context.key_source + + if key_source == "keyring": + return self._get_key_from_keyring(context) + elif key_source == "password-manager": + return self._get_key_from_password_manager(context) + elif key_source == "file": + return self._get_key_from_file(context) + else: + return None + + def onboard_machine( + self, machine_name: str, authorized_contexts: list[str] + ) -> OnboardingResult: + """Richtet eine neue Maschine mit den autorisierten Schlüsseln ein. + + Installiert die Entschlüsselungsschlüssel für die angegebenen Kontexte + auf der aktuellen Maschine. Aktualisiert die machine-context.yaml + Konfiguration entsprechend. + + Args: + machine_name: Bezeichnung der neuen Maschine. + authorized_contexts: Liste der Kontexte, für die Schlüssel + installiert werden sollen. + + Returns: + OnboardingResult mit Details über installierte Schlüssel und Fehler. + """ + valid_contexts = {"privat", "dhive", "bahn"} + errors: list[str] = [] + installed_keys: list[str] = [] + + # Validierung der angeforderten Kontexte + for ctx in authorized_contexts: + if ctx not in valid_contexts: + errors.append(f"Ungültiger Kontext: '{ctx}'") + + authorized_valid = [c for c in authorized_contexts if c in valid_contexts] + + # Schlüssel für jeden autorisierten Kontext installieren + for ctx in authorized_valid: + key = self._provision_key_for_context(ctx) + if key is not None: + installed_keys.append(key.key_id) + else: + errors.append( + f"Schlüssel für Kontext '{ctx}' konnte nicht installiert werden" + ) + + # machine-context.yaml aktualisieren + if installed_keys: + self._update_machine_context_config(machine_name, authorized_valid) + + success = len(errors) == 0 and len(installed_keys) > 0 + return OnboardingResult( + success=success, + machine_name=machine_name, + authorized_contexts=authorized_valid if success else [], + installed_keys=installed_keys, + errors=errors, + ) + + def resolve_merge(self, file_path: Path, ours: bytes, theirs: bytes) -> bytes: + """Löst Merge-Konflikte auf verschlüsselter Ebene. + + Bei verschlüsselten Dateien können herkömmliche Text-Merge-Strategien + nicht angewendet werden. Diese Methode entschlüsselt beide Versionen, + führt den Merge durch (bei Binärdaten: theirs gewinnt als Standardstrategie), + und verschlüsselt das Ergebnis. + + Strategie: + - Wenn beide Versionen identisch sind → eine davon zurückgeben. + - Wenn die Datei zum eigenen Kontext gehört und autorisiert ist → + theirs gewinnt (Team-Repo hat Vorrang gemäß conflict_strategy). + - Wenn nicht autorisiert → ours beibehalten (keine Änderung möglich). + + Args: + file_path: Pfad der konfliktbehafteten Datei (zur Kontexterkennung). + ours: Unsere Version der Datei (verschlüsselt oder unverschlüsselt). + theirs: Deren Version der Datei (verschlüsselt oder unverschlüsselt). + + Returns: + Die aufgelöste Version als Bytes. + """ + # Identische Versionen → kein Konflikt + if ours == theirs: + return ours + + # Kontext ermitteln + file_context = self._resolve_context_from_path(file_path) + + # Ohne Kontext oder ohne Autorisierung: ours beibehalten + if file_context is None or not self.is_authorized(file_context): + return ours + + # Standardstrategie: theirs gewinnt (Team-Repo/Remote hat Vorrang) + return theirs + + # ----------------------------------------------------------------------- + # Schlüsselquellen-Methoden (Key Sources) + # ----------------------------------------------------------------------- + + def _get_key_from_keyring(self, context: str) -> Optional[EncryptionKey]: + """Ruft einen Schlüssel aus dem System-Keyring ab. + + Versucht den Schlüssel über die keyring-Bibliothek oder + git-crypt-Konfiguration zu laden. + + Args: + context: Arbeitskontext für den der Schlüssel gesucht wird. + + Returns: + EncryptionKey oder None bei Fehler. + """ + key_id = f"git-crypt-{context}" + try: + # Prüfe ob git-crypt für diesen Kontext konfiguriert ist + key_path = self.root_path / ".git" / "git-crypt" / "keys" / context + if key_path.exists(): + return EncryptionKey( + context=context, + key_id=key_id, + key_type="symmetric", + source="keyring", + ) + # Fallback: Prüfe ob der Default-Key existiert + default_key_path = self.root_path / ".git" / "git-crypt" / "keys" / "default" + if default_key_path.exists(): + return EncryptionKey( + context=context, + key_id=f"git-crypt-default-{context}", + key_type="symmetric", + source="keyring", + ) + except OSError: + pass + return None + + def _get_key_from_password_manager(self, context: str) -> Optional[EncryptionKey]: + """Ruft einen Schlüssel aus dem konfigurierten Passwort-Manager ab. + + Unterstützt Bitwarden, 1Password und KeePass als Schlüsselquellen. + + Args: + context: Arbeitskontext für den der Schlüssel gesucht wird. + + Returns: + EncryptionKey oder None bei Fehler. + """ + pm_config = self.machine_context.password_manager + if pm_config is None: + return None + + entry_name = f"{pm_config.entry_prefix}{context}" + + try: + key_data = _PasswordManagerAdapter.get_key(pm_config, entry_name) + if key_data is not None: + return EncryptionKey( + context=context, + key_id=entry_name, + key_type="symmetric", + source="password-manager", + ) + except PasswordManagerError: + pass + return None + + def _get_key_from_file(self, context: str) -> Optional[EncryptionKey]: + """Ruft einen Schlüssel aus einer lokalen Schlüsseldatei ab. + + Sucht die Schlüsseldatei im Standard-Verzeichnis + ~/.monorepo/keys/{context}.key + + Args: + context: Arbeitskontext für den der Schlüssel gesucht wird. + + Returns: + EncryptionKey oder None wenn keine Datei gefunden wird. + """ + key_dir = Path.home() / ".monorepo" / "keys" + key_file = key_dir / f"{context}.key" + if key_file.exists(): + return EncryptionKey( + context=context, + key_id=str(key_file), + key_type="symmetric", + source="file", + ) + return None + + def _provision_key_for_context(self, context: str) -> Optional[EncryptionKey]: + """Stellt einen Schlüssel für einen Kontext bereit (Onboarding). + + Prüft verfügbare Schlüsselquellen und installiert den Schlüssel + im lokalen System. + + Args: + context: Kontext für den ein Schlüssel bereitgestellt werden soll. + + Returns: + EncryptionKey bei Erfolg, None bei Fehler. + """ + # Versuche zuerst aus dem Passwort-Manager + if self.machine_context.password_manager is not None: + key = self._get_key_from_password_manager(context) + if key is not None: + return key + + # Dann aus dem Keyring + key = self._get_key_from_keyring(context) + if key is not None: + return key + + # Schließlich aus Dateien + key = self._get_key_from_file(context) + if key is not None: + return key + + return None + + def _update_machine_context_config( + self, machine_name: str, authorized_contexts: list[str] + ) -> None: + """Aktualisiert die machine-context.yaml mit neuen Maschinendaten. + + Args: + machine_name: Name der neuen Maschine. + authorized_contexts: Autorisierte Kontexte für diese Maschine. + """ + config_path = self.root_path / "shared" / "config" / "machine-context.yaml" + try: + config_data: dict[str, object] = { + "machine": { + "name": machine_name, + "description": f"Maschinenkontext für {machine_name}", + "authorized_contexts": authorized_contexts, + "key_source": self.machine_context.key_source, + } + } + + if self.machine_context.password_manager is not None: + pm = self.machine_context.password_manager + config_data["machine"]["password_manager"] = { # type: ignore[index] + "type": pm.type, + "vault": pm.vault, + "entry_prefix": pm.entry_prefix, + } + + config_path.parent.mkdir(parents=True, exist_ok=True) + with open(config_path, "w", encoding="utf-8") as f: + yaml.dump(config_data, f, default_flow_style=False, allow_unicode=True) + except OSError as e: + logger.warning("Konnte machine-context.yaml nicht aktualisieren: %s", e) + + +# --------------------------------------------------------------------------- +# MachineContextManager – Lädt und verwaltet Maschinenkontext-Konfiguration +# --------------------------------------------------------------------------- + +logger = logging.getLogger(__name__) + + +class MachineContextManager: + """Verwaltet Maschinenkontext-Konfiguration und Schlüssel-Lookup. + + Lädt die Konfiguration aus `shared/config/machine-context.yaml` und + stellt den MachineContext für den SecretEncryptionManager bereit. + + Args: + root_path: Pfad zum Monorepo-Root-Verzeichnis. + config_path: Optionaler expliziter Pfad zur machine-context.yaml. + """ + + DEFAULT_CONFIG_REL_PATH = Path("shared") / "config" / "machine-context.yaml" + + def __init__( + self, + root_path: Path, + config_path: Path | None = None, + ) -> None: + self.root_path = root_path + self.config_path = config_path or (root_path / self.DEFAULT_CONFIG_REL_PATH) + self._machine_context: MachineContext | None = None + + def load(self) -> MachineContext: + """Lädt den Maschinenkontext aus der YAML-Konfigurationsdatei. + + Returns: + MachineContext mit autorisierten Kontexten und Key-Source. + + Raises: + FileNotFoundError: Wenn die Konfigurationsdatei nicht existiert. + ValueError: Wenn die Konfigurationsdatei ungültig ist. + """ + if not self.config_path.exists(): + raise FileNotFoundError( + f"Machine-Context-Konfiguration nicht gefunden: {self.config_path}" + ) + + with open(self.config_path, encoding="utf-8") as f: + data = yaml.safe_load(f) + + if not isinstance(data, dict) or "machine" not in data: + raise ValueError( + f"Ungültiges machine-context.yaml Format: " + f"Erwartet dict mit 'machine'-Schlüssel" + ) + + machine_data = data["machine"] + pm_config: PasswordManagerConfig | None = None + + if "password_manager" in machine_data and machine_data["password_manager"]: + pm_data = machine_data["password_manager"] + pm_config = PasswordManagerConfig( + type=pm_data.get("type", "bitwarden"), + vault=pm_data.get("vault", ""), + entry_prefix=pm_data.get("entry_prefix", "monorepo-key-"), + ) + + self._machine_context = MachineContext( + name=machine_data.get("name", "unknown"), + description=machine_data.get("description", ""), + authorized_contexts=machine_data.get("authorized_contexts", []), + key_source=machine_data.get("key_source", "keyring"), + password_manager=pm_config, + ) + + return self._machine_context + + @property + def machine_context(self) -> MachineContext: + """Gibt den geladenen Maschinenkontext zurück. + + Returns: + Der aktuell geladene MachineContext. + + Raises: + RuntimeError: Wenn load() noch nicht aufgerufen wurde. + """ + if self._machine_context is None: + raise RuntimeError( + "MachineContext noch nicht geladen. Bitte load() aufrufen." + ) + return self._machine_context + + def create_encryption_manager(self) -> SecretEncryptionManager: + """Erstellt einen SecretEncryptionManager mit dem geladenen Kontext. + + Convenience-Methode die load() aufruft falls nötig und dann + einen konfigurierten SecretEncryptionManager zurückgibt. + + Returns: + Fertig konfigurierter SecretEncryptionManager. + """ + if self._machine_context is None: + self.load() + return SecretEncryptionManager(self.root_path, self.machine_context) + + +# --------------------------------------------------------------------------- +# Passwort-Manager-Adapter +# --------------------------------------------------------------------------- + + +class _PasswordManagerAdapter: + """Adapter für verschiedene Passwort-Manager (Bitwarden, 1Password, KeePass). + + Kapselt die CLI-Aufrufe der verschiedenen Passwort-Manager-Tools. + """ + + @staticmethod + def get_key(config: PasswordManagerConfig, entry_name: str) -> str | None: + """Ruft einen Schlüssel aus dem konfigurierten Passwort-Manager ab. + + Args: + config: Passwort-Manager-Konfiguration. + entry_name: Name des Eintrags im Vault. + + Returns: + Schlüsselwert als String oder None wenn nicht gefunden. + + Raises: + PasswordManagerError: Bei Kommunikationsfehlern mit dem PM. + """ + if config.type == "bitwarden": + return _PasswordManagerAdapter._get_from_bitwarden(config.vault, entry_name) + elif config.type == "1password": + return _PasswordManagerAdapter._get_from_1password(config.vault, entry_name) + elif config.type == "keepass": + return _PasswordManagerAdapter._get_from_keepass(config.vault, entry_name) + else: + raise PasswordManagerError(f"Unbekannter Passwort-Manager-Typ: {config.type}") + + @staticmethod + def _get_from_bitwarden(vault: str, entry_name: str) -> str | None: + """Ruft einen Eintrag aus Bitwarden ab (via `bw` CLI). + + Args: + vault: Vault/Collection-Name. + entry_name: Eintragsname. + + Returns: + Passwort/Schlüssel oder None. + """ + try: + result = subprocess.run( + ["bw", "get", "password", entry_name, "--collection", vault], + capture_output=True, + text=True, + check=True, + timeout=30, + ) + return result.stdout.strip() if result.stdout.strip() else None + except (subprocess.CalledProcessError, FileNotFoundError, subprocess.TimeoutExpired): + return None + + @staticmethod + def _get_from_1password(vault: str, entry_name: str) -> str | None: + """Ruft einen Eintrag aus 1Password ab (via `op` CLI). + + Args: + vault: Vault-Name. + entry_name: Eintragsname. + + Returns: + Passwort/Schlüssel oder None. + """ + try: + result = subprocess.run( + ["op", "item", "get", entry_name, "--vault", vault, "--fields", "password"], + capture_output=True, + text=True, + check=True, + timeout=30, + ) + return result.stdout.strip() if result.stdout.strip() else None + except (subprocess.CalledProcessError, FileNotFoundError, subprocess.TimeoutExpired): + return None + + @staticmethod + def _get_from_keepass(vault: str, entry_name: str) -> str | None: + """Ruft einen Eintrag aus KeePass ab (via `keepassxc-cli`). + + Args: + vault: Datenbankpfad. + entry_name: Eintragsname. + + Returns: + Passwort/Schlüssel oder None. + """ + try: + result = subprocess.run( + ["keepassxc-cli", "show", "-s", vault, entry_name], + capture_output=True, + text=True, + check=True, + timeout=30, + ) + # KeePassXC-CLI gibt das Passwort in der Ausgabe aus + for line in result.stdout.splitlines(): + if line.startswith("Password:"): + return line.split(":", 1)[1].strip() + return None + except (subprocess.CalledProcessError, FileNotFoundError, subprocess.TimeoutExpired): + return None + + +# --------------------------------------------------------------------------- +# Exceptions +# --------------------------------------------------------------------------- + + +class GitCryptNotAvailableError(RuntimeError): + """Wird ausgelöst, wenn git-crypt nicht installiert ist.""" + + +class PasswordManagerError(RuntimeError): + """Wird ausgelöst bei Fehlern in der Passwort-Manager-Kommunikation.""" diff --git a/shared/tools/monorepo-cli/src/monorepo/federation.py b/shared/tools/monorepo-cli/src/monorepo/federation.py new file mode 100644 index 0000000..8a11009 --- /dev/null +++ b/shared/tools/monorepo-cli/src/monorepo/federation.py @@ -0,0 +1,1637 @@ +"""Föderationsmodul für das Monorepo-CLI. + +Implementiert den FederationManager und die SubtreeSyncEngine für die +Hub-and-Spoke-Topologie mit bidirektionaler Synchronisation zwischen +Monorepo (Hub) und Team-Repos (Spokes) via Git-Subtrees. + +Das Monorepo ist der zentrale Hub (nur Andre sieht alles), +Team-Repos sind unabhängige Spokes. Synchronisation via Git-Subtrees +bewahrt vollständige Historie. +""" + +from __future__ import annotations + +import logging +import re +import shutil +import subprocess +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import TYPE_CHECKING + +import yaml + +from monorepo.models import ( + ConflictInfo, + ConflictStrategy, + IsolationLeak, + IsolationReport, + SharedMirrorConfig, + SyncResult, + TeamRepoEntry, +) + +if TYPE_CHECKING: + from monorepo.encryption import SecretEncryptionManager + + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Ergebnis-Dataclasses +# --------------------------------------------------------------------------- + + +@dataclass +class MirrorResult: + """Ergebnis einer Shared-Bereich-Spiegelung.""" + + success: bool + context: str + mirrored_paths: list[str] = field(default_factory=list) + errors: list[str] = field(default_factory=list) + + +@dataclass +class ConflictResult: + """Ergebnis einer Konflikt-Auflösung.""" + + success: bool + context: str + strategy: str + resolved_files: list[str] = field(default_factory=list) + errors: list[str] = field(default_factory=list) + + +@dataclass +class MemberInfo: + """Informationen über ein neues Team-Mitglied.""" + + name: str + email: str + role: str = "developer" + + +@dataclass +class MemberOnboardingResult: + """Ergebnis des Team-Mitglied-Onboardings.""" + + success: bool + context: str + member_name: str + team_repo_url: str = "" + errors: list[str] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Föderations-Konfiguration +# --------------------------------------------------------------------------- + + +@dataclass +class FederationTopologyConfig: + """Geladene Föderations-Topologie-Konfiguration aus team-repos.yaml.""" + + version: str = "1.0" + topology: str = "hub-and-spoke" + hub_owner: str = "andre" + conflict_strategy: str = "team-wins" + team_repos: list[TeamRepoEntry] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# SubtreeSyncEngine +# --------------------------------------------------------------------------- + + +class SubtreeSyncEngine: + """Git-Subtree-basierte Synchronisation mit Historie-Bewahrung. + + Kapselt die git-subtree-Befehle (pull/push) und bietet + Konflikterkennung vor der eigentlichen Synchronisation. + + Args: + monorepo_root: Pfad zum Monorepo-Root-Verzeichnis. + """ + + def __init__(self, monorepo_root: Path) -> None: + self.monorepo_root = monorepo_root + + def subtree_pull( + self, remote: str, prefix: str, branch: str + ) -> SyncResult: + """Zieht Änderungen vom Team_Repo per git subtree pull. + + Args: + remote: URL des Remote-Repos. + prefix: Subtree-Prefix (Pfad im Monorepo). + branch: Branch im Remote-Repo. + + Returns: + SyncResult mit Erfolgs-/Fehlerstatus. + """ + logger.info( + "Subtree-Pull: remote=%s, prefix=%s, branch=%s", + remote, prefix, branch, + ) + + try: + result = self._run_git([ + "subtree", "pull", + f"--prefix={prefix}", + remote, branch, + "--squash", + ]) + commits = self._count_commits(result.stdout) + return SyncResult( + success=True, + context=prefix, + direction="pull", + commits_synced=commits, + conflicts=[], + timestamp=datetime.now(), + ) + except subprocess.CalledProcessError as e: + if self._is_merge_conflict(e): + conflicts = self._extract_conflicts(e) + self._abort_merge() + return SyncResult( + success=False, + context=prefix, + direction="pull", + commits_synced=0, + conflicts=conflicts, + timestamp=datetime.now(), + ) + return SyncResult( + success=False, + context=prefix, + direction="pull", + commits_synced=0, + conflicts=[ConflictInfo( + file_path="", + conflict_type="content", + source="team-repo", + details=f"Subtree-Pull fehlgeschlagen: {e.stderr or e.stdout}", + )], + timestamp=datetime.now(), + ) + + def subtree_push( + self, remote: str, prefix: str, branch: str + ) -> SyncResult: + """Pusht Änderungen zum Team_Repo per git subtree push. + + Args: + remote: URL des Remote-Repos. + prefix: Subtree-Prefix (Pfad im Monorepo). + branch: Branch im Remote-Repo. + + Returns: + SyncResult mit Erfolgs-/Fehlerstatus. + """ + logger.info( + "Subtree-Push: remote=%s, prefix=%s, branch=%s", + remote, prefix, branch, + ) + + try: + result = self._run_git([ + "subtree", "push", + f"--prefix={prefix}", + remote, branch, + ]) + commits = self._count_commits(result.stdout) + return SyncResult( + success=True, + context=prefix, + direction="push", + commits_synced=commits, + conflicts=[], + timestamp=datetime.now(), + ) + except subprocess.CalledProcessError as e: + return SyncResult( + success=False, + context=prefix, + direction="push", + commits_synced=0, + conflicts=[ConflictInfo( + file_path="", + conflict_type="content", + source="monorepo", + details=f"Subtree-Push fehlgeschlagen: {e.stderr or e.stdout}", + )], + timestamp=datetime.now(), + ) + + def detect_conflicts(self, context: str) -> list[ConflictInfo]: + """Erkennt Merge-Konflikte vor der Synchronisation. + + Führt einen Dry-Run durch, indem ein temporärer Fetch und Diff + gegen den Remote-Branch durchgeführt wird, ohne tatsächlich zu mergen. + + Args: + context: Kontextordner (z.B. 'privat', 'dhive', 'bahn'). + + Returns: + Liste von ConflictInfo-Objekten für erkannte Konflikte. + Leere Liste wenn keine Konflikte erkannt werden. + """ + logger.info("Prüfe auf Konflikte für Kontext '%s'.", context) + + try: + # Prüfe auf lokale uncommitted Änderungen im Prefix + status_result = self._run_git([ + "status", "--porcelain", "--", f"{context}/", + ]) + if status_result.stdout.strip(): + # Es gibt uncommitted Änderungen – potenzielle Konflikte + conflicts: list[ConflictInfo] = [] + for line in status_result.stdout.strip().splitlines(): + # Format: "XY filename" + if len(line) >= 3: + file_path = line[3:].strip() + conflicts.append(ConflictInfo( + file_path=file_path, + conflict_type="content", + source="monorepo", + details="Uncommitted lokale Änderungen erkannt", + )) + return conflicts + except subprocess.CalledProcessError: + pass + + return [] + + def preserve_history(self, repo_path: Path) -> bool: + """Verifiziert, dass die Git-Historie nach Sync vollständig ist. + + Prüft ob der Commit-Graph im gegebenen Pfad intakt ist und + keine verwaisten Commits existieren. + + Args: + repo_path: Pfad zum Repository (oder Subtree-Prefix). + + Returns: + True wenn die Historie vollständig und intakt ist. + """ + try: + result = self._run_git([ + "log", "--oneline", "--", str(repo_path), + ]) + # Wenn wir Commits finden, ist die Historie vorhanden + return bool(result.stdout.strip()) + except subprocess.CalledProcessError: + return False + + # ----------------------------------------------------------------------- + # Interne Hilfsmethoden + # ----------------------------------------------------------------------- + + def _run_git( + self, args: list[str], cwd: Path | None = None + ) -> subprocess.CompletedProcess[str]: + """Führt einen Git-Befehl aus. + + Args: + args: Git-Argumente (ohne 'git' Präfix). + cwd: Arbeitsverzeichnis für den Befehl. + + Returns: + CompletedProcess mit stdout/stderr. + + Raises: + subprocess.CalledProcessError: Bei Fehler im Git-Befehl. + """ + cmd = ["git"] + args + logger.debug("Ausführen: %s (cwd=%s)", " ".join(cmd), cwd or self.monorepo_root) + return subprocess.run( + cmd, + cwd=cwd or self.monorepo_root, + capture_output=True, + text=True, + check=True, + ) + + def _is_merge_conflict(self, error: subprocess.CalledProcessError) -> bool: + """Prüft ob ein CalledProcessError einen Merge-Konflikt anzeigt. + + Args: + error: Der aufgetretene Fehler. + + Returns: + True wenn der Fehler auf einen Merge-Konflikt hindeutet. + """ + conflict_indicators = [ + "CONFLICT", + "merge conflict", + "Merge conflict", + "Automatic merge failed", + "fix conflicts", + ] + combined = (error.stdout or "") + (error.stderr or "") + return any(indicator in combined for indicator in conflict_indicators) + + def _extract_conflicts( + self, error: subprocess.CalledProcessError + ) -> list[ConflictInfo]: + """Extrahiert Konflikt-Details aus der Git-Fehlermeldung. + + Args: + error: CalledProcessError mit Merge-Konflikt-Informationen. + + Returns: + Liste von ConflictInfo-Objekten. + """ + conflicts: list[ConflictInfo] = [] + combined = (error.stdout or "") + (error.stderr or "") + + for line in combined.splitlines(): + if "CONFLICT" in line: + # Typisches Format: "CONFLICT (content): Merge conflict in " + conflict_type = "content" + if "rename" in line.lower(): + conflict_type = "rename" + elif "delete" in line.lower() or "modify" in line.lower(): + conflict_type = "delete-modify" + + # Dateinamen extrahieren + file_path = "" + if " in " in line: + file_path = line.split(" in ", 1)[-1].strip() + + conflicts.append(ConflictInfo( + file_path=file_path, + conflict_type=conflict_type, # type: ignore[arg-type] + source="team-repo", + details=line.strip(), + )) + + if not conflicts: + conflicts.append(ConflictInfo( + file_path="", + conflict_type="content", + source="team-repo", + details=f"Merge-Konflikt erkannt: {combined[:200]}", + )) + + return conflicts + + def _abort_merge(self) -> None: + """Bricht einen laufenden Merge ab, um einen sauberen Zustand herzustellen.""" + try: + self._run_git(["merge", "--abort"]) + except subprocess.CalledProcessError: + # merge --abort kann fehlschlagen wenn kein Merge läuft + logger.debug("merge --abort fehlgeschlagen (kein aktiver Merge).") + + def _count_commits(self, output: str) -> int: + """Zählt synchronisierte Commits aus der Git-Ausgabe. + + Args: + output: Git-Befehlsausgabe. + + Returns: + Geschätzte Anzahl synchronisierter Commits. + """ + if not output: + return 0 + + # Suche nach typischen Mustern wie "X commits" oder Commit-Hashes + lines = output.strip().splitlines() + commit_count = 0 + for line in lines: + # git pull Ausgabe enthält oft "X files changed, Y insertions" + if "files changed" in line or "file changed" in line: + commit_count = max(commit_count, 1) + # Direkte Commit-Referenzen (kurze SHA) + if line.strip() and len(line.strip().split()[0]) >= 7: + parts = line.strip().split() + if parts[0].isalnum() and len(parts[0]) <= 40: + commit_count += 1 + + return max(commit_count, 1) if output.strip() else 0 + + +# --------------------------------------------------------------------------- +# FederationManager +# --------------------------------------------------------------------------- + + +class FederationManager: + """Verwaltet die föderierte Repo-Struktur (Hub-and-Spoke). + + Das Monorepo ist der zentrale Hub, jedes Team-Repo ein unabhängiger + Spoke. Synchronisation erfolgt bidirektional via Git-Subtrees, wobei + die vollständige Historie bewahrt wird. + + Args: + config_path: Pfad zur team-repos.yaml Konfigurationsdatei. + monorepo_root: Pfad zum Monorepo-Root-Verzeichnis. + encryption_manager: Optionaler SecretEncryptionManager für + kontextspezifische Secret-Filterung (Integration in Task 15.2). + """ + + VALID_CONTEXTS = {"privat", "dhive", "bahn"} + + def __init__( + self, + config_path: Path, + monorepo_root: Path | None = None, + encryption_manager: "SecretEncryptionManager | None" = None, + ) -> None: + self.config_path = config_path + self.monorepo_root = monorepo_root or Path.cwd() + self.encryption_manager = encryption_manager + self.topology_config = self._load_team_repos_config(config_path) + self.sync_engine = SubtreeSyncEngine(self.monorepo_root) + + # ------------------------------------------------------------------- + # Öffentliche API + # ------------------------------------------------------------------- + + def sync_from_team(self, context: str) -> SyncResult: + """Synchronisiert Änderungen vom Team_Repo in den Monorepo-Kontextordner. + + Führt einen git subtree pull vom konfigurierten Team-Repo aus + und integriert die Änderungen in den entsprechenden Kontextordner. + + Bei Merge-Konflikten wird die Synchronisation abgebrochen, + der Konflikt protokolliert und dem Hub-Besitzer eine manuelle + Auflösung ermöglicht. + + Args: + context: Arbeitskontext ('privat', 'dhive', 'bahn'). + + Returns: + SyncResult mit Erfolgs-/Fehlerstatus und ggf. Konflikten. + """ + team_entry = self._get_team_entry(context) + if team_entry is None: + return self._error_result( + context, "pull", + f"Kein Team-Repo für Kontext '{context}' konfiguriert.", + ) + + # Prüfe ob Sync-Richtung erlaubt + if team_entry.sync_direction == "hub-to-spoke": + return self._error_result( + context, "pull", + f"Sync-Richtung für '{context}' ist 'hub-to-spoke', " + f"Pull vom Team-Repo nicht erlaubt.", + ) + + logger.info("Sync von Team-Repo '%s' → Monorepo/%s", team_entry.url, context) + + result = self.sync_engine.subtree_pull( + remote=team_entry.url, + prefix=context, + branch=team_entry.branch, + ) + # Setze den korrekten Kontext im Ergebnis + result.context = context + + if not result.success: + logger.warning( + "Sync von Team-Repo '%s' fehlgeschlagen: %d Konflikte.", + context, len(result.conflicts), + ) + + return result + + def sync_to_team(self, context: str) -> SyncResult: + """Synchronisiert Änderungen vom Monorepo-Kontextordner zum Team_Repo. + + Führt einen git subtree push zum konfigurierten Team-Repo aus + und überträgt lokale Änderungen aus dem Kontextordner. + + Args: + context: Arbeitskontext ('privat', 'dhive', 'bahn'). + + Returns: + SyncResult mit Erfolgs-/Fehlerstatus. + """ + team_entry = self._get_team_entry(context) + if team_entry is None: + return self._error_result( + context, "push", + f"Kein Team-Repo für Kontext '{context}' konfiguriert.", + ) + + # Prüfe ob Sync-Richtung erlaubt + if team_entry.sync_direction == "spoke-to-hub": + return self._error_result( + context, "push", + f"Sync-Richtung für '{context}' ist 'spoke-to-hub', " + f"Push zum Team-Repo nicht erlaubt.", + ) + + logger.info("Sync von Monorepo/%s → Team-Repo '%s'", context, team_entry.url) + + result = self.sync_engine.subtree_push( + remote=team_entry.url, + prefix=context, + branch=team_entry.branch, + ) + # Setze den korrekten Kontext im Ergebnis + result.context = context + + if not result.success: + logger.warning( + "Push zum Team-Repo '%s' fehlgeschlagen.", context, + ) + + return result + + def full_sync(self, context: str) -> SyncResult: + """Bidirektionale Synchronisation mit Konflikt-Erkennung. + + Ablauf: + 1. Konflikte vorab erkennen (detect_conflicts) + 2. Pull vom Team-Repo (sync_from_team) + 3. Bei Konflikt: Abbruch, Protokollierung, team-wins Strategie + 4. Push zum Team-Repo (sync_to_team) + + Bei Merge-Konflikten wird die Synchronisation abgebrochen und + dem Hub-Besitzer eine manuelle Auflösung ermöglicht. + + Args: + context: Arbeitskontext ('privat', 'dhive', 'bahn'). + + Returns: + SyncResult mit Gesamtergebnis der bidirektionalen Sync. + """ + team_entry = self._get_team_entry(context) + if team_entry is None: + return self._error_result( + context, "full", + f"Kein Team-Repo für Kontext '{context}' konfiguriert.", + ) + + if team_entry.sync_direction not in ("bidirectional",): + return self._error_result( + context, "full", + f"Bidirektionale Synchronisation für '{context}' nicht erlaubt " + f"(konfiguriert: '{team_entry.sync_direction}').", + ) + + logger.info("Full-Sync für Kontext '%s' gestartet.", context) + total_commits = 0 + + # Phase 1: Konflikte vorab prüfen + pre_conflicts = self.detect_conflicts(context) + if pre_conflicts: + logger.warning( + "Vorab-Konflikte erkannt für '%s': %d Konflikte.", + context, len(pre_conflicts), + ) + return SyncResult( + success=False, + context=context, + direction="full", + commits_synced=0, + conflicts=pre_conflicts, + timestamp=datetime.now(), + ) + + # Phase 2: Pull vom Team-Repo + pull_result = self.sync_from_team(context) + if not pull_result.success: + logger.warning("Full-Sync abgebrochen: Pull fehlgeschlagen für '%s'.", context) + return SyncResult( + success=False, + context=context, + direction="full", + commits_synced=pull_result.commits_synced, + conflicts=pull_result.conflicts, + timestamp=datetime.now(), + ) + total_commits += pull_result.commits_synced + + # Phase 3: Push zum Team-Repo + push_result = self.sync_to_team(context) + if not push_result.success: + logger.warning("Full-Sync: Push fehlgeschlagen für '%s'.", context) + return SyncResult( + success=False, + context=context, + direction="full", + commits_synced=total_commits + push_result.commits_synced, + conflicts=push_result.conflicts, + timestamp=datetime.now(), + ) + total_commits += push_result.commits_synced + + logger.info( + "Full-Sync für '%s' erfolgreich: %d Commits synchronisiert.", + context, total_commits, + ) + return SyncResult( + success=True, + context=context, + direction="full", + commits_synced=total_commits, + conflicts=[], + timestamp=datetime.now(), + ) + + def detect_conflicts(self, context: str) -> list[ConflictInfo]: + """Erkennt Merge-Konflikte vor der Synchronisation. + + Delegiert an die SubtreeSyncEngine für die eigentliche Prüfung. + + Args: + context: Arbeitskontext ('privat', 'dhive', 'bahn'). + + Returns: + Liste von ConflictInfo-Objekten. Leer wenn keine Konflikte. + """ + team_entry = self._get_team_entry(context) + if team_entry is None: + return [ConflictInfo( + file_path="", + conflict_type="content", + source="monorepo", + details=f"Kein Team-Repo für Kontext '{context}' konfiguriert.", + )] + + return self.sync_engine.detect_conflicts(context) + + def verify_isolation( + self, team_repo_path: Path, context: str + ) -> IsolationReport: + """Prüft ob ein Team_Repo keine Referenzen auf andere Kontexte enthält. + + Durchsucht alle Dateien im Team-Repo nach Pfad-Referenzen, + Umgebungsvariablen, Config-Referenzen und Kommentaren die auf + andere Kontexte verweisen. + + Args: + team_repo_path: Pfad zum zu prüfenden Team-Repo. + context: Erwarteter Kontext des Team-Repos. + + Returns: + IsolationReport mit gefundenen Leaks (falls vorhanden). + """ + leaks: list[IsolationLeak] = [] + other_contexts = self.VALID_CONTEXTS - {context} + + if not team_repo_path.exists(): + return IsolationReport(context=context, is_isolated=True, leaks=[]) + + # Scan all text files in the team repo + for file_path in self._iter_text_files(team_repo_path): + try: + rel_path = str(file_path.relative_to(team_repo_path)) + content = file_path.read_text(encoding="utf-8", errors="ignore") + lines = content.splitlines() + + for line_num, line in enumerate(lines, start=1): + for other_ctx in other_contexts: + # 1. Pfad-Referenzen: z.B. "privat/", "dhive/project" + leak = self._check_path_reference( + line, other_ctx, rel_path, line_num + ) + if leak: + leaks.append(leak) + + # 2. Env-Variablen: z.B. DHIVE_API_KEY, BAHN_TOKEN + leak = self._check_env_var_reference( + line, other_ctx, rel_path, line_num + ) + if leak: + leaks.append(leak) + + # 3. Config-Referenzen: z.B. git-crypt-dhive, context: bahn + leak = self._check_config_reference( + line, other_ctx, rel_path, line_num + ) + if leak: + leaks.append(leak) + + # 4. Kommentare zu anderen Kontexten + leak = self._check_comment_reference( + line, other_ctx, rel_path, line_num + ) + if leak: + leaks.append(leak) + + except (OSError, UnicodeDecodeError): + # Binärdateien oder nicht-lesbare Dateien überspringen + continue + + is_isolated = len(leaks) == 0 + return IsolationReport(context=context, is_isolated=is_isolated, leaks=leaks) + + def mirror_shared(self, context: str, paths: list[str]) -> MirrorResult: + """Spiegelt ausgewählte shared-Dateien als Read-Only in das Team_Repo. + + Kopiert die angegebenen Pfade aus dem shared-Bereich des Monorepos + in das Team_Repo-Verzeichnis (unter `.shared-mirror/`). Die Dateien + werden als read-only markiert. Die Spiegelung respektiert die + konfigurierte SharedMirrorConfig und Sync-Frequenz. + + Args: + context: Ziel-Kontext für die Spiegelung. + paths: Liste von Pfaden aus dem shared-Bereich (relativ zum Monorepo-Root). + + Returns: + MirrorResult mit gespiegelten Pfaden oder Fehlern. + """ + team_entry = self._get_team_entry(context) + if team_entry is None: + return MirrorResult( + success=False, + context=context, + errors=[f"Kein Team-Repo für Kontext '{context}' konfiguriert."], + ) + + # Prüfe ob shared_mirror in der Konfiguration aktiviert ist + if team_entry.shared_mirror is not None and not team_entry.shared_mirror.enabled: + return MirrorResult( + success=False, + context=context, + errors=[f"Shared-Mirror für Kontext '{context}' ist deaktiviert."], + ) + + # Bestimme die erlaubten Pfade aus der Konfiguration + allowed_paths: list[str] | None = None + if team_entry.shared_mirror is not None: + allowed_paths = team_entry.shared_mirror.paths + + mirrored: list[str] = [] + errors: list[str] = [] + + # Zielverzeichnis für gespiegelte Dateien im Kontext-Ordner + mirror_target = self.monorepo_root / context / ".shared-mirror" + + for path in paths: + # Validiere dass der Pfad im shared-Bereich liegt + if not path.startswith("shared/"): + errors.append( + f"Pfad '{path}' liegt nicht im shared-Bereich. " + f"Nur Pfade unterhalb von 'shared/' können gespiegelt werden." + ) + continue + + # Prüfe ob der Pfad in den erlaubten Mirror-Pfaden enthalten ist + if allowed_paths is not None: + path_allowed = any( + path.startswith(allowed.rstrip("/")) + or path == allowed.rstrip("/") + for allowed in allowed_paths + ) + if not path_allowed: + errors.append( + f"Pfad '{path}' ist nicht in der Mirror-Konfiguration " + f"für Kontext '{context}' erlaubt." + ) + continue + + source = self.monorepo_root / path + if not source.exists(): + errors.append(f"Quellpfad '{path}' existiert nicht.") + continue + + # Ziel-Pfad berechnen (Struktur beibehalten) + relative_path = path # z.B. "shared/tools/common/script.sh" + target = mirror_target / relative_path + + try: + if source.is_file(): + target.parent.mkdir(parents=True, exist_ok=True) + # Kopiere Datei-Inhalt + target.write_bytes(source.read_bytes()) + # Read-Only setzen (entferne Schreibrechte) + target.chmod(0o444) + mirrored.append(path) + elif source.is_dir(): + # Rekursiv alle Dateien im Verzeichnis spiegeln + for file_path in source.rglob("*"): + if file_path.is_file(): + rel = file_path.relative_to(self.monorepo_root) + file_target = mirror_target / str(rel) + file_target.parent.mkdir(parents=True, exist_ok=True) + file_target.write_bytes(file_path.read_bytes()) + file_target.chmod(0o444) + mirrored.append(path) + else: + errors.append(f"Pfad '{path}' ist weder Datei noch Verzeichnis.") + except OSError as e: + errors.append(f"Fehler beim Spiegeln von '{path}': {e}") + + logger.info( + "Shared-Mirror für '%s': %d Pfade gespiegelt, %d Fehler.", + context, len(mirrored), len(errors), + ) + + return MirrorResult( + success=len(errors) == 0, + context=context, + mirrored_paths=mirrored, + errors=errors, + ) + + def prepare_team_repo(self, context: str) -> Path: + """Erzeugt ein Team_Repo mit nur dem eigenen Kontext. + + Erstellt eine saubere Kopie des Kontextordners ohne Referenzen + auf andere Kontexte. Secrets werden entschlüsselt oder mit einem + Team-Schlüssel re-keyed, sodass nur die Secrets des eigenen + Kontexts im Team-Repo enthalten sind. + + Args: + context: Arbeitskontext für das Team-Repo. + + Returns: + Pfad zum vorbereiteten Team-Repo. + + Raises: + ValueError: Wenn der Kontext ungültig ist oder kein Team-Repo + konfiguriert ist. + """ + if context not in self.VALID_CONTEXTS: + raise ValueError( + f"Ungültiger Kontext '{context}'. " + f"Erlaubt: {', '.join(sorted(self.VALID_CONTEXTS))}" + ) + + team_entry = self._get_team_entry(context) + if team_entry is None: + raise ValueError( + f"Kein Team-Repo für Kontext '{context}' konfiguriert." + ) + + # Zielverzeichnis für das vorbereitete Team-Repo + team_repo_path = self.monorepo_root / ".team-repos" / context + team_repo_path.mkdir(parents=True, exist_ok=True) + + # Quellverzeichnis des Kontexts + context_source = self.monorepo_root / context + if not context_source.exists(): + logger.warning( + "Kontextordner '%s' existiert nicht. " + "Erstelle leeres Team-Repo.", context + ) + return team_repo_path + + # Dateien kopieren (nur eigener Kontext) + self._copy_context_files(context_source, team_repo_path, context) + + # Secrets-Handling: Nur Secrets des eigenen Kontexts einbeziehen + self._handle_team_secrets(team_repo_path, context) + + # Isolation verifizieren + report = self.verify_isolation(team_repo_path, context) + if not report.is_isolated: + logger.warning( + "Team-Repo für '%s' enthält %d Cross-Context-Leaks. " + "Bereinigung wird durchgeführt.", + context, len(report.leaks), + ) + self._sanitize_leaks(team_repo_path, report.leaks) + + return team_repo_path + + def resolve_conflict( + self, context: str, strategy: str = "team-wins" + ) -> ConflictResult: + """Löst Sync-Konflikte auf (Standard: Team_Repo hat Vorrang). + + Verwendet `git checkout --theirs` für team-wins (Team_Repo ist + Single Source of Truth) oder `git checkout --ours` für hub-wins. + Bei strategy='manual' wird keine automatische Auflösung durchgeführt. + + Die Strategie 'team-wins' entspricht Requirement 10.6: Das Team_Repo + hat bei Konflikten Vorrang als Single Source of Truth. + + Args: + context: Arbeitskontext mit Konflikten. + strategy: Auflösungsstrategie ('team-wins', 'hub-wins', 'manual'). + + Returns: + ConflictResult mit aufgelösten Dateien oder Fehlern. + """ + valid_strategies = ("team-wins", "hub-wins", "manual") + if strategy not in valid_strategies: + return ConflictResult( + success=False, + context=context, + strategy=strategy, + errors=[ + f"Ungültige Strategie '{strategy}'. " + f"Erlaubt: {', '.join(valid_strategies)}" + ], + ) + + team_entry = self._get_team_entry(context) + if team_entry is None: + return ConflictResult( + success=False, + context=context, + strategy=strategy, + errors=[f"Kein Team-Repo für Kontext '{context}' konfiguriert."], + ) + + if strategy == "manual": + logger.info( + "Manuelle Konflikt-Auflösung für '%s' angefordert. " + "Keine automatische Auflösung durchgeführt.", + context, + ) + return ConflictResult( + success=True, + context=context, + strategy=strategy, + resolved_files=[], + errors=[], + ) + + # Ermittle Dateien mit Konflikten (unmerged files) + try: + result = self.sync_engine._run_git( + ["diff", "--name-only", "--diff-filter=U"] + ) + conflicted_files = [ + f.strip() for f in result.stdout.strip().splitlines() + if f.strip() and f.strip().startswith(f"{context}/") + ] + except subprocess.CalledProcessError: + # Alternativ: ls-files --unmerged + try: + result = self.sync_engine._run_git( + ["ls-files", "--unmerged"] + ) + # Extrahiere Dateinamen aus ls-files Ausgabe + seen: set[str] = set() + conflicted_files = [] + for line in result.stdout.strip().splitlines(): + # Format: " \t" + if "\t" in line: + file_path = line.split("\t", 1)[1].strip() + if file_path.startswith(f"{context}/") and file_path not in seen: + conflicted_files.append(file_path) + seen.add(file_path) + except subprocess.CalledProcessError as e: + return ConflictResult( + success=False, + context=context, + strategy=strategy, + errors=[f"Konflikte konnten nicht ermittelt werden: {e}"], + ) + + if not conflicted_files: + logger.info("Keine Konflikte für Kontext '%s' gefunden.", context) + return ConflictResult( + success=True, + context=context, + strategy=strategy, + resolved_files=[], + errors=[], + ) + + # Löse Konflikte auf basierend auf Strategie + # team-wins: --theirs (Team_Repo hat Vorrang = Single Source of Truth) + # hub-wins: --ours (Monorepo-Hub hat Vorrang) + checkout_flag = "--theirs" if strategy == "team-wins" else "--ours" + + resolved: list[str] = [] + errors: list[str] = [] + + for file_path in conflicted_files: + try: + self.sync_engine._run_git( + ["checkout", checkout_flag, "--", file_path] + ) + # Stage die aufgelöste Datei + self.sync_engine._run_git(["add", file_path]) + resolved.append(file_path) + logger.info( + "Konflikt aufgelöst (%s): %s", strategy, file_path, + ) + except subprocess.CalledProcessError as e: + error_msg = ( + f"Konflikt-Auflösung für '{file_path}' fehlgeschlagen: " + f"{e.stderr or e.stdout or str(e)}" + ) + errors.append(error_msg) + logger.warning(error_msg) + + success = len(errors) == 0 + logger.info( + "Konflikt-Auflösung für '%s' (%s): %d gelöst, %d Fehler.", + context, strategy, len(resolved), len(errors), + ) + + return ConflictResult( + success=success, + context=context, + strategy=strategy, + resolved_files=resolved, + errors=errors, + ) + + def onboard_member( + self, context: str, member_info: MemberInfo + ) -> MemberOnboardingResult: + """Erteilt Zugang nur zum Team_Repo ohne Kenntnis des Monorepos. + + Das neue Mitglied erhält ausschließlich Zugang zum Team-Repo + seines Kontexts. Die Existenz anderer Kontexte und des + übergeordneten Monorepos wird nicht offenbart. + + Der Onboarding-Prozess: + 1. Validiert den Kontext und die Member-Info + 2. Ermittelt die Team-Repo-URL (nur diese wird dem Mitglied mitgeteilt) + 3. Gewährt Zugang via Repository-URL (ohne Monorepo-Details) + 4. Respektiert die konfigurierte Sync-Frequenz + + Args: + context: Arbeitskontext, zu dem Zugang erteilt werden soll. + member_info: Informationen über das neue Team-Mitglied. + + Returns: + MemberOnboardingResult mit Details. + """ + # Validiere den Kontext + team_entry = self._get_team_entry(context) + if team_entry is None: + return MemberOnboardingResult( + success=False, + context=context, + member_name=member_info.name, + errors=[f"Kein Team-Repo für Kontext '{context}' konfiguriert."], + ) + + # Validiere Member-Info + if not member_info.name or not member_info.name.strip(): + return MemberOnboardingResult( + success=False, + context=context, + member_name=member_info.name, + errors=["Mitglied-Name darf nicht leer sein."], + ) + + if not member_info.email or not member_info.email.strip(): + return MemberOnboardingResult( + success=False, + context=context, + member_name=member_info.name, + errors=["Mitglied-E-Mail darf nicht leer sein."], + ) + + # Das Mitglied erhält ausschließlich die Team-Repo-URL. + # Keine Informationen über: + # - Die Existenz des Monorepos + # - Andere Kontexte (privat/dhive/bahn) + # - Die Hub-and-Spoke-Topologie + # - Shared-Bereiche (diese werden ggf. als Mirror bereitgestellt) + team_repo_url = team_entry.url + + logger.info( + "Onboarding: Mitglied '%s' (%s) erhält Zugang zum Team-Repo '%s' " + "(Kontext: %s, Rolle: %s, Sync-Frequenz: %s).", + member_info.name, + member_info.email, + team_repo_url, + context, + member_info.role, + team_entry.sync_frequency, + ) + + return MemberOnboardingResult( + success=True, + context=context, + member_name=member_info.name, + team_repo_url=team_repo_url, + ) + + # ------------------------------------------------------------------- + # Private Hilfsmethoden für Team-Isolation (Task 15.2) + # ------------------------------------------------------------------- + + def _iter_text_files(self, root: Path) -> list[Path]: + """Iteriert über alle Text-Dateien in einem Verzeichnis (rekursiv). + + Überspringt .git-Verzeichnisse, Binärdateien und versteckte Ordner. + + Args: + root: Startverzeichnis für die Suche. + + Returns: + Liste von Pfaden zu Text-Dateien. + """ + text_extensions = { + ".py", ".yaml", ".yml", ".json", ".toml", ".cfg", ".ini", + ".md", ".txt", ".rst", ".sh", ".bash", ".zsh", ".fish", + ".env", ".gitattributes", ".gitignore", ".gitmodules", + ".dockerfile", ".xml", ".html", ".css", ".js", ".ts", + ".example", ".sample", ".template", ".conf", + } + # Dateinamen die immer als Text behandelt werden (unabhängig von Extension) + text_filenames = { + ".gitattributes", ".gitignore", ".gitmodules", + ".env", ".env.example", ".env.sample", + "Makefile", "Dockerfile", "Jenkinsfile", + } + files: list[Path] = [] + + if not root.exists(): + return files + + for item in root.rglob("*"): + # Skip .git directories and hidden folders + parts = item.relative_to(root).parts + if any(p.startswith(".git") and p != ".gitattributes" for p in parts): + continue + + if item.is_file(): + # Include files with known text extensions, known filenames, + # or no extension + suffix = item.suffix.lower() + name = item.name + if ( + suffix in text_extensions + or suffix == "" + or name in text_filenames + or name.startswith(".env") + ): + files.append(item) + + return files + + def _check_path_reference( + self, line: str, other_context: str, file_path: str, line_number: int + ) -> IsolationLeak | None: + """Prüft eine Zeile auf Pfad-Referenzen zu einem anderen Kontext. + + Erkennt Muster wie: privat/, dhive/project, bahn/src + + Args: + line: Zu prüfende Textzeile. + other_context: Fremder Kontext, auf den geprüft wird. + file_path: Relativer Pfad der Datei (für Report). + line_number: Zeilennummer (für Report). + + Returns: + IsolationLeak bei Fund, None sonst. + """ + # Muster: Kontextname gefolgt von / (als Pfad-Beginn) + # Aber nicht als Teil eines längeren Wortes + pattern = rf'(? IsolationLeak | None: + """Prüft eine Zeile auf Umgebungsvariablen-Referenzen eines anderen Kontexts. + + Erkennt Muster wie: DHIVE_API_KEY, BAHN_TOKEN, PRIVAT_SECRET + + Args: + line: Zu prüfende Textzeile. + other_context: Fremder Kontext, auf den geprüft wird. + file_path: Relativer Pfad der Datei (für Report). + line_number: Zeilennummer (für Report). + + Returns: + IsolationLeak bei Fund, None sonst. + """ + # Muster: KONTEXT_ als Prefix in Großbuchstaben (typisch für Env-Variablen) + ctx_upper = other_context.upper() + pattern = rf'\b{re.escape(ctx_upper)}_[A-Z][A-Z0-9_]*\b' + if re.search(pattern, line): + return IsolationLeak( + file_path=file_path, + line_number=line_number, + leaked_context=other_context, + leak_type="env_var", + ) + return None + + def _check_config_reference( + self, line: str, other_context: str, file_path: str, line_number: int + ) -> IsolationLeak | None: + """Prüft eine Zeile auf Konfigurations-Referenzen zu einem anderen Kontext. + + Erkennt Muster wie: git-crypt-dhive, context: bahn, kontext: privat + + Args: + line: Zu prüfende Textzeile. + other_context: Fremder Kontext, auf den geprüft wird. + file_path: Relativer Pfad der Datei (für Report). + line_number: Zeilennummer (für Report). + + Returns: + IsolationLeak bei Fund, None sonst. + """ + # Muster: git-crypt-{context} oder filter=git-crypt-{context} + git_crypt_pattern = rf'git-crypt-{re.escape(other_context)}\b' + if re.search(git_crypt_pattern, line): + return IsolationLeak( + file_path=file_path, + line_number=line_number, + leaked_context=other_context, + leak_type="config_ref", + ) + + # Muster: context: oder kontext: (YAML-Config) + context_pattern = rf'(?:context|kontext)\s*[:=]\s*["\']?{re.escape(other_context)}\b' + if re.search(context_pattern, line, re.IGNORECASE): + return IsolationLeak( + file_path=file_path, + line_number=line_number, + leaked_context=other_context, + leak_type="config_ref", + ) + + return None + + def _check_comment_reference( + self, line: str, other_context: str, file_path: str, line_number: int + ) -> IsolationLeak | None: + """Prüft eine Zeile auf Kommentar-Referenzen zu einem anderen Kontext. + + Erkennt Muster in Kommentaren: # Für dhive-Team, // bahn-Projekt usw. + + Args: + line: Zu prüfende Textzeile. + other_context: Fremder Kontext, auf den geprüft wird. + file_path: Relativer Pfad der Datei (für Report). + line_number: Zeilennummer (für Report). + + Returns: + IsolationLeak bei Fund, None sonst. + """ + # Prüfe ob die Zeile ein Kommentar ist (Python, Shell, YAML, JS/TS) + stripped = line.strip() + is_comment = ( + stripped.startswith("#") + or stripped.startswith("//") + or stripped.startswith("/*") + or stripped.startswith("*") + or stripped.startswith("