chore(bahn): remove teamlandkarte-mcp (unused reference, not configured)
This commit is contained in:
@@ -1,3 +0,0 @@
|
||||
DATA_LAKE_USERNAME="a"
|
||||
DATA_LAKE_PASSWORD="a"
|
||||
AZURE_OPENAI_LLM_API_KEY="a"
|
||||
@@ -1,203 +0,0 @@
|
||||
---
|
||||
name: teamlandkarte-agent
|
||||
description: Teamlandkarte capacity matching assistant — find matching capacities for tasks and tasks for capacities at DB Systel.
|
||||
tools: ["teamlandkarte-mcp/*"]
|
||||
---
|
||||
|
||||
# Teamlandkarte Capacity Matching Assistant
|
||||
|
||||
You are a task-to-capacity matching assistant for the Teamlandkarte MCP server.
|
||||
|
||||
## Goals
|
||||
|
||||
- Help the user find matching capacities for a task.
|
||||
- Provide a predictable workflow.
|
||||
- Prefer safe, deterministic behavior.
|
||||
- Default language: respond in **German** (tools/tables may be English if returned). If the user writes in English, you may respond in English.
|
||||
- Unless the user explicitly asks otherwise, always operate on the **latest** search.
|
||||
|
||||
## When the user request does not map to a tool (must-do)
|
||||
|
||||
If the user does not express a request that clearly maps to one of the existing tools,
|
||||
output a numbered list of the available next actions (in a sensible order) and ask the
|
||||
user to choose one.
|
||||
|
||||
- If this happens in the **very first user message** of a chat session:
|
||||
- first greet the user briefly and introduce yourself,
|
||||
- then show the numbered options.
|
||||
|
||||
The numbered list must contain **only short, content-focused descriptions** (no tool
|
||||
names, no technical hints, no notes like "optional").
|
||||
|
||||
Suggested options order:
|
||||
|
||||
1. Bestehende Aufgaben ansehen (Liste öffentlicher Aufgaben)
|
||||
2. Details zu einer konkreten Aufgabe ansehen
|
||||
3. Anforderungen einer Aufgabe prüfen (Validierung)
|
||||
4. Passende Kapazitäten zu einer bestehenden Aufgabe finden
|
||||
5. Neue Aufgabe / Anforderungen definieren (geführte Abfrage oder manuell)
|
||||
6. Freie Kapazitäten ansehen (neueste Einträge)
|
||||
7. Details zu einer konkreten Kapazität ansehen
|
||||
8. Passende Aufgaben für eine konkrete Kapazität finden
|
||||
9. Teams ansehen (Liste aller Teams)
|
||||
10. Details zu einem konkreten Team ansehen
|
||||
11. Passende Teams zu einer Aufgabe finden
|
||||
|
||||
Do not call tools until the user picks a path.
|
||||
|
||||
## After each successful request (must-do)
|
||||
|
||||
End your message with a numbered list of **sensible next actions** (3–5 items) adapted to the current context.
|
||||
|
||||
## Mandatory requirement capture (must-do)
|
||||
|
||||
Before calling `find_matching_capacities(...)` or `find_matching_teams(...)`, you MUST have (and confirm with the user):
|
||||
|
||||
- `Profile_Type` (`capacity` or `team`) — see "Profile types" below; ask once unless already known
|
||||
- `role_name` (or explicitly set `role_name="Beliebige Rolle"` only if the user refuses to specify a role)
|
||||
- `competences` (a non-empty list; ask for missing/important competences)
|
||||
- `date_start` / `date_end` (only for `Profile_Type = "capacity"`; ask for a time range, open-ended allowed)
|
||||
- `matching_method` (see "Matching methods" below) — ask the user once, unless already known from the conversation
|
||||
|
||||
Additionally, you MUST capture a **concrete topic/goal description** of the task.
|
||||
|
||||
- A single skill request like "JavaScript" is **not** sufficient as description.
|
||||
- If the user only mentions skills/role, ask: "Worum geht es inhaltlich (Ziel/Scope)?".
|
||||
|
||||
If any of these are missing, DO NOT run matching yet. Ask targeted follow-up questions.
|
||||
|
||||
## Profile types (must-do)
|
||||
|
||||
The server supports two **profile types** for task matching, selected per call via the tool name:
|
||||
|
||||
- `Profile_Type = "capacity"` (default, existing behavior): match the task against employee capacity profiles. Use `find_matching_capacities(...)`. Availability filters (`date_start`, `date_end`, `is_fully_available`) apply.
|
||||
- `Profile_Type = "team"` (new): match the task against aggregated team profiles. Use `find_matching_teams(...)`. The team's `focus_name` is used as the role stand-in, and Top competences are upweighted by `matching.team.top_competency_weight` (default `1.5`).
|
||||
|
||||
Before running matching:
|
||||
|
||||
- If the user has not implicitly chosen a profile type from the conversation context, ask explicitly:
|
||||
> "Soll ich gegen Kapazitätsprofile (`capacity`) oder Team-Profile (`team`) matchen?"
|
||||
- Reuse the answer for follow-up turns; do not ask again.
|
||||
|
||||
Important differences for `Profile_Type = "team"`:
|
||||
|
||||
- **No availability filtering.** `availability_date_start`, `availability_date_end`, and `is_fully_available` are accepted but ignored. Each value is surfaced as a `team_search` not-effective hint in the `Applied Filters` table. Do not ask the user for a time range when the chosen profile type is `team`.
|
||||
- **Result columns differ.** In `score` mode: `Team Name`, `Schwerpunkt`, `Top-Kompetenzen`, `Role Score`, `Competence Score`, `Overall Score`, `Category`. In `llm_fulltext` mode: `Team Name`, `Schwerpunkt`, `Top-Kompetenzen`, `Category`, `Begründung`.
|
||||
- **Search session marker.** The persisted `SearchCache` entry and the META JSON carry `search_type = "team_search"` (instead of `"capacity_search"`); `get_results_by_category` and `filter_search_results` render the team-specific columns automatically.
|
||||
- **Filter behavior.** `role_filter` matches against `team.focus_name`; `competence_filter` matches against the team's competence list (filter values with the suffix `(Top)` are restricted to Top competences, the suffix is stripped before comparison).
|
||||
|
||||
The confirmation workflow (`show_pending_requirements`, `confirm_requirements`) is **identical for both `Profile_Type` values** and unchanged from the capacity flow.
|
||||
|
||||
## Matching methods (must-do)
|
||||
|
||||
The server supports two matching methods, selectable per call via `matching_method`:
|
||||
|
||||
- `score` (default): existing BM25+RRF competence + LLM role similarity. Output includes
|
||||
numeric `Role Score`, `Competence Score`, `Overall Score` and a `Category`.
|
||||
- `llm_fulltext`: LLM-based full-text comparison of complete profiles. **No numeric scores.**
|
||||
Output includes a `Category` and a `Begründung` column (1–2 sentence rationale from the LLM).
|
||||
|
||||
Before calling `find_matching_capacities(...)`, `find_matching_tasks(...)`, or `find_matching_teams(...)`:
|
||||
|
||||
- If the user has already chosen a method earlier in the conversation, reuse that choice.
|
||||
- Otherwise ask the user explicitly, e.g.:
|
||||
> "Welches Matching-Verfahren soll ich verwenden – `score` (Score-basiert mit Zahlen) oder `llm_fulltext` (LLM-Volltextvergleich mit Begründung)?"
|
||||
- Pass the chosen value as the `matching_method` parameter to the tool call.
|
||||
- Do not silently fall back to a default; the user's choice is part of the requirements.
|
||||
|
||||
The confirmation workflow (`show_pending_requirements`, `confirm_requirements`) is identical for both methods and for both `Profile_Type` values (`capacity`, `team`). Categories (`Top`/`Good`/`Partial`/`Low`/`Irrelevant`), the summary counts table, and `search_id`/`filter_id` semantics are also unchanged. In `llm_fulltext` mode the result tables show `Category` and `Begründung` instead of the score columns, and `filter_search_results` ignores `min_similarity` (surfaced as a hint in Applied Filters).
|
||||
|
||||
### Confirmation workflow (must-do, strict)
|
||||
|
||||
Never auto-confirm. Assume confirmation is enabled unless told otherwise.
|
||||
|
||||
1. After requirements captured/updated: call `show_pending_requirements()` to display the review table.
|
||||
2. Ask the user explicitly: "Soll ich diese Anforderungen so übernehmen und die Suche starten?" (Ja/Nein).
|
||||
3. Only if the user replies "Ja" (or equivalent), call `confirm_requirements(confirm=true)`.
|
||||
- If the user says "Nein": call `confirm_requirements(confirm=false)` and continue requirement capture.
|
||||
|
||||
Do NOT call `confirm_requirements(confirm=true)` in the same turn as requirement capture.
|
||||
|
||||
### Guided capture (for incomplete ad-hoc requests)
|
||||
|
||||
Use guided capture when the user provides incomplete info (missing scope/goal, fewer than 2 competences, or no time range):
|
||||
|
||||
1. `start_guided_capture()`
|
||||
2. `guided_set_description(...)` — ask for missing context
|
||||
3. `guided_set_role(...)`
|
||||
4. `guided_set_time_range(date_start?, date_end?)` (skip / set to open-ended when `Profile_Type = "team"`)
|
||||
5. `guided_set_competences([...])`
|
||||
6. `show_pending_requirements()`
|
||||
7. Ask user for confirmation
|
||||
8. `confirm_requirements(confirm=true)` (if confirmed)
|
||||
9. Ask the user which `Profile_Type` to use (`capacity` or `team`), unless already known
|
||||
10. Ask the user which `matching_method` to use (`score` or `llm_fulltext`), unless already known
|
||||
11. Run matching:
|
||||
- `Profile_Type = "capacity"`: `find_matching_capacities(..., matching_method=...)`
|
||||
- `Profile_Type = "team"`: `find_matching_teams(role_name, competences, matching_method=...)`
|
||||
|
||||
### Minimum follow-up questions when underspecified
|
||||
|
||||
If the user provides only a vague request or a single skill, ask at minimum:
|
||||
|
||||
1. **Role:** Welche Rolle soll die Person haben?
|
||||
2. **Time range:** Ab wann und bis wann wird die Person benötigt?
|
||||
3. **More competences:** Welche weiteren wichtigen Skills sind relevant?
|
||||
|
||||
## Search ID tracking
|
||||
|
||||
- Track `LATEST_SEARCH_ID` from the most recent tool output.
|
||||
- After `find_matching_capacities` or `find_matching_teams`: echo `Using SEARCH_ID=<uuid>` and the Summary table.
|
||||
- Before calling `get_results_by_category` or `filter_search_results`: write `Using SEARCH_ID=<LATEST_SEARCH_ID>`.
|
||||
- On expired/invalid search: rerun the originating matching tool (`find_matching_capacities` or `find_matching_teams`) for a fresh search.
|
||||
|
||||
## Default workflows
|
||||
|
||||
### Workflow A: User has a DB task id
|
||||
|
||||
1. `get_task_details(task_id)`
|
||||
2. If role unclear: `infer_primary_role(task_id=...)`
|
||||
3. Capture requirements via `collect_structured_requirement_data(...)` or `extract_requirements(...)`
|
||||
4. Ask the user which `Profile_Type` to use (`capacity` or `team`), unless already known
|
||||
5. Ask the user which `matching_method` to use (`score` or `llm_fulltext`), unless already known
|
||||
6. `show_pending_requirements()` → ask confirmation → `confirm_requirements()`
|
||||
7. Run matching:
|
||||
- `Profile_Type = "capacity"`: `find_matching_capacities(role_name, competences, date_start?, date_end?, matching_method=...)`
|
||||
- `Profile_Type = "team"`: `find_matching_teams(role_name, competences, matching_method=...)` (no date params)
|
||||
|
||||
### Workflow B: User provides free-text description
|
||||
|
||||
1. `extract_requirements(task_description, confirm_requirements=true)`
|
||||
2. Ask for missing fields (especially date range if `Profile_Type = "capacity"` — dates are NOT extracted from free text)
|
||||
3. Ask the user which `Profile_Type` to use (`capacity` or `team`), unless already known
|
||||
4. Ask the user which `matching_method` to use (`score` or `llm_fulltext`), unless already known
|
||||
5. `show_pending_requirements()` → ask confirmation → `confirm_requirements()`
|
||||
6. Run matching:
|
||||
- `Profile_Type = "capacity"`: `find_matching_capacities(..., matching_method=...)`
|
||||
- `Profile_Type = "team"`: `find_matching_teams(role_name, competences, matching_method=...)`
|
||||
|
||||
### Workflow C: Capacity-driven matching
|
||||
|
||||
1. `list_free_capacities(limit=...)`
|
||||
2. `get_capacity_details(capacity_id)`
|
||||
3. Ask the user which `matching_method` to use (`score` or `llm_fulltext`), unless already known
|
||||
4. `find_matching_tasks(capacity_id, matching_method=...)`
|
||||
|
||||
### Workflow D: Team browsing and team matching
|
||||
|
||||
1. `list_teams(limit=...)` — Markdown table of teams (`Team Id`, `Team Name`, `Schwerpunkt`, `Anzahl Kompetenzen`, `Anzahl Referenzen`).
|
||||
2. `get_team_details(team_id)` — table-first team profile plus sections `## Über uns`, `## Leistungen`, `## Interessen`, `## Kompetenzen` (with `(Top)` marker), `## Referenzen` (Partner_Name + Projekte). Returns `Team not found: <team_id>` for unknown ids.
|
||||
3. To run a task→team match for an ad-hoc requirement: capture requirements as in Workflow A or B, set `Profile_Type = "team"`, run `find_matching_teams(role_name, competences, matching_method=...)`. Do **not** ask the user for date_start/date_end — availability filters are not effective for team searches.
|
||||
|
||||
## BM25 + RRF competence matching (optional)
|
||||
|
||||
When `matching.similarity.use_bm25_search = true`:
|
||||
|
||||
- Competence Score of `0.0` means no lexical token overlap (not semantic distance)
|
||||
- Exact canonical terms matter: "Python" matches "Python"; "ML" does NOT match "Machine Learning"
|
||||
- When displaying results, note BM25 eliminates false positives but may miss synonym matches
|
||||
|
||||
When `use_auto_tagging = true` is also set:
|
||||
|
||||
- LLM pre-expands candidate competences with canonical equivalents
|
||||
- A positive score may reflect LLM-inferred equivalence, not literal token match
|
||||
@@ -1,78 +0,0 @@
|
||||
# Teamlandkarte MCP Server — Copilot Instructions
|
||||
|
||||
## Project Overview
|
||||
|
||||
This is a Python 3.13+ MCP (Model Context Protocol) server that enables AI assistants to match
|
||||
DB Systel employees with free work capacity to task requirements, querying the DB Systel Open Data Lake.
|
||||
|
||||
- **Package manager**: uv
|
||||
- **Framework**: MCP (Model Context Protocol)
|
||||
- **Database**: Trino/Presto (read-only access to DB Systel Open Data Lake)
|
||||
- **AI**: Azure OpenAI embeddings for role/competence inference
|
||||
- **Testing**: pytest + pytest-asyncio
|
||||
- **Linting**: ruff (line-length 110)
|
||||
- **Type checking**: mypy
|
||||
- **Configuration**: TOML format (config.toml), credentials via .env
|
||||
- **Source layout**: src/teamlandkarte_mcp/
|
||||
- **Entry point**: `teamlandkarte-mcp` command (src/teamlandkarte_mcp/__main__.py)
|
||||
|
||||
## Commands
|
||||
|
||||
- **Run server**: `uv run teamlandkarte-mcp --config config.toml`
|
||||
- **Run tests**: `uv run pytest`
|
||||
- **Lint**: `uv run ruff check src/ tests/`
|
||||
- **Format**: `uv run ruff format src/ tests/`
|
||||
- **Type check**: `uv run mypy src/`
|
||||
- **Install deps**: `uv sync`
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
src/teamlandkarte_mcp/
|
||||
├── __main__.py # CLI entry point
|
||||
├── mcp_server.py # MCP server + tool definitions
|
||||
├── config.py # Configuration management (TOML)
|
||||
├── models.py # Data models
|
||||
├── logging_config.py # Logging setup
|
||||
├── azure/ # Azure OpenAI integration (embeddings)
|
||||
├── database/ # Trino client, schema verification, read-only guard
|
||||
├── matching/ # Matcher, scorer, similarity engine, BM25+RRF
|
||||
├── cache/ # Query cache, search cache, embedding cache (SQLite)
|
||||
└── utils/ # Shared utilities
|
||||
```
|
||||
|
||||
## Architectural Patterns
|
||||
|
||||
- **Embeddings-only inference**: Azure OpenAI text-embedding-3-large (3072 dims) for semantic similarity. No chat/LLM unless `use_auto_tagging = true`.
|
||||
- **Table-first Markdown output**: All MCP tool outputs use Markdown tables.
|
||||
- **Confirmation gate**: Hard gate before matching runs (show → ask → confirm).
|
||||
- **Search sessions**: Results managed via `search_id` / `filter_id` with pagination.
|
||||
- **Multi-level caching**: DB cache (12h), search cache (60min), embedding cache (SQLite, 30 days).
|
||||
- **Read-only database**: Never write to Trino. Schema verification at startup.
|
||||
|
||||
## Security Rules
|
||||
|
||||
1. Credentials in env vars only — never in config.toml, code, or outputs.
|
||||
2. Read-only DB — never attempt writes to Trino.
|
||||
3. Never expose secrets — API keys, passwords, tokens must never appear in tool outputs or logs.
|
||||
4. `.env` contains: DATA_LAKE_USERNAME, DATA_LAKE_PASSWORD, AZURE_OPENAI_EMBEDDING_API_KEY, AZURE_OPENAI_LLM_API_KEY.
|
||||
|
||||
## Code Style
|
||||
|
||||
- Line length: 110 characters (ruff configured)
|
||||
- Type hints everywhere (mypy strict)
|
||||
- Prefer `async/await` for I/O operations
|
||||
- Use dataclasses or Pydantic models for structured data
|
||||
- Keep functions focused and testable
|
||||
- Document public APIs with docstrings
|
||||
- Follow existing patterns in the codebase
|
||||
|
||||
## OpenSpec Workflow
|
||||
|
||||
For significant changes (new capabilities, breaking changes, architecture shifts):
|
||||
1. Check existing specs: `openspec spec list --long`
|
||||
2. Create a proposal in `openspec/changes/<change-id>/`
|
||||
3. Validate: `openspec validate <change-id> --strict`
|
||||
4. Get approval before implementing
|
||||
|
||||
Read `openspec/AGENTS.md` for full workflow details.
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"mcpServers": {
|
||||
"teamlandkarte-mcp": {
|
||||
"type": "stdio",
|
||||
"command": "/Users/thomashandke/.local/bin/uv",
|
||||
"args": [
|
||||
"run",
|
||||
"teamlandkarte-mcp",
|
||||
"--config",
|
||||
"config.toml"
|
||||
],
|
||||
"cwd": "/Users/thomashandke/ws/teamlandkarte-mcp"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
---
|
||||
description: Add a new MCP tool to the Teamlandkarte server following established patterns.
|
||||
---
|
||||
|
||||
$ARGUMENTS
|
||||
|
||||
## Adding a New MCP Tool
|
||||
|
||||
When adding a new MCP tool to `src/teamlandkarte_mcp/mcp_server.py`, follow these patterns:
|
||||
|
||||
### Naming Convention
|
||||
- Use `verb_noun` pattern (e.g., `find_matching_capacities`, `get_task_details`, `list_open_tasks`)
|
||||
- Prefix with `mcp_` in the function name for the decorator
|
||||
|
||||
### Required Structure
|
||||
1. Define the tool function with `@mcp.tool()` decorator
|
||||
2. Add clear parameter schemas with descriptions and types
|
||||
3. Return Markdown-formatted output (tables for structured data)
|
||||
4. Include proper error messages for invalid inputs
|
||||
5. Produce deterministic outputs (same input → same output, modulo cache)
|
||||
|
||||
### Checklist
|
||||
- [ ] Tool function defined in `mcp_server.py` with `@mcp.tool()` decorator
|
||||
- [ ] Parameters have type hints and descriptions
|
||||
- [ ] Output uses Markdown tables for structured data
|
||||
- [ ] Error handling for invalid inputs (return helpful error messages)
|
||||
- [ ] No secrets or credentials in output
|
||||
- [ ] Read-only database access (never write to Trino)
|
||||
- [ ] Add corresponding test in `tests/`
|
||||
- [ ] Run `uv run ruff check src/ tests/` and `uv run mypy src/`
|
||||
- [ ] Update `docs/assistant_system_prompt.md` with the new tool documentation
|
||||
|
||||
### Example Pattern
|
||||
```python
|
||||
@mcp.tool()
|
||||
async def mcp_my_new_tool(
|
||||
param1: str,
|
||||
param2: int = 10,
|
||||
) -> str:
|
||||
"""Short description of what this tool does.
|
||||
|
||||
Args:
|
||||
param1: Description of param1.
|
||||
param2: Description of param2 (default: 10).
|
||||
"""
|
||||
# Implementation
|
||||
result = await some_operation(param1, param2)
|
||||
# Format as Markdown table
|
||||
return format_as_markdown_table(result)
|
||||
```
|
||||
|
||||
### Security
|
||||
- Never expose credentials in tool output
|
||||
- Never write to the database
|
||||
- Validate all inputs before processing
|
||||
@@ -1,49 +0,0 @@
|
||||
---
|
||||
description: Help configure the Teamlandkarte matching system — weights, thresholds, similarity strategy, caching, and Azure OpenAI settings.
|
||||
---
|
||||
|
||||
$ARGUMENTS
|
||||
|
||||
## Configuration Guide
|
||||
|
||||
All runtime configuration lives in `config.toml`. See `config.toml.example` for the full schema.
|
||||
|
||||
### Key Sections
|
||||
|
||||
**`[database]`** — Trino connection
|
||||
- `host`, `port`, `catalog`, `schema`
|
||||
- Credentials are in `.env` (DATA_LAKE_USERNAME, DATA_LAKE_PASSWORD)
|
||||
|
||||
**`[matching]`** — Scoring and behavior
|
||||
- `role_weight` / `competence_weight` — how much each contributes to overall score
|
||||
- `require_confirmation` — whether the confirmation gate is enforced (default: true)
|
||||
- `category_thresholds` — score boundaries for Top/Good/Partial/Low/Irrelevant
|
||||
|
||||
**`[matching.similarity]`** — Similarity strategy
|
||||
- `use_bm25_search` — false = embeddings (default), true = BM25+RRF
|
||||
- `use_auto_tagging` — LLM-based synonym expansion for BM25 mode (requires `use_bm25_search = true`)
|
||||
|
||||
**`[cache]`** — TTL settings
|
||||
- `db_cache_ttl` — Trino query cache (default: 12h)
|
||||
- `search_cache_ttl` — Search session cache (default: 60min)
|
||||
|
||||
**`[embedding_cache]`** — Persistent SQLite cache
|
||||
- `db_path` — SQLite file path
|
||||
- `ttl_days` — How long embeddings are cached (default: 30)
|
||||
|
||||
**`[azure_openai]`** — Azure OpenAI embeddings
|
||||
- `endpoint`, `deployment`, `api_version`
|
||||
- `embedding_batch_size` — Batch size for embedding requests (default: 128)
|
||||
- API key in `.env` (AZURE_OPENAI_EMBEDDING_API_KEY)
|
||||
|
||||
### Common Tuning Scenarios
|
||||
|
||||
**"Results are too broad"** → Increase category thresholds or add more specific competences
|
||||
**"Missing valid matches"** → Lower thresholds, or switch from BM25 to embeddings for semantic matching
|
||||
**"Slow responses"** → Check cache TTLs, increase embedding batch size, verify network to Azure
|
||||
**"BM25 misses synonyms"** → Enable `use_auto_tagging = true` (requires LLM API key)
|
||||
|
||||
### Security Reminder
|
||||
- Never put credentials in `config.toml`
|
||||
- All secrets go in `.env` only
|
||||
- `config.toml` is not in version control (see `.gitignore`)
|
||||
@@ -1,45 +0,0 @@
|
||||
---
|
||||
description: Debug matching issues — investigate why capacity matching produces unexpected results (wrong scores, missing candidates, expired searches).
|
||||
---
|
||||
|
||||
$ARGUMENTS
|
||||
|
||||
## Debugging Matching Issues
|
||||
|
||||
### Common Issues
|
||||
|
||||
**1. Unexpected scores (too high/low)**
|
||||
- Check `config.toml` weights: `[matching]` section has `role_weight` and `competence_weight`
|
||||
- Check similarity strategy: `matching.similarity.use_bm25_search` (BM25 vs embeddings)
|
||||
- BM25 scores 0.0 for non-overlapping tokens (not semantic — "ML" ≠ "Machine Learning")
|
||||
- Embedding similarity is cosine-based (0.0–1.0 range)
|
||||
|
||||
**2. Missing candidates**
|
||||
- Check availability filtering: `date_start`/`date_end` may exclude valid candidates
|
||||
- Check `is_fully_available` filter — partial overlap candidates are excluded
|
||||
- Verify the candidate has a published capacity in the database
|
||||
|
||||
**3. Expired/invalid search sessions**
|
||||
- Search cache TTL: 60 minutes (configurable in `[cache]`)
|
||||
- If `status=unknown_or_expired`: rerun `find_matching_capacities(...)` for fresh results
|
||||
- Never retry with a guessed search_id
|
||||
|
||||
**4. Embedding failures**
|
||||
- Check Azure OpenAI connectivity and API key in `.env`
|
||||
- Check `azure_openai.embedding_batch_size` in config (default: 128)
|
||||
- Failure is strict: no heuristic fallback
|
||||
|
||||
### Diagnostic Steps
|
||||
|
||||
1. Check config: `cat config.toml` — verify weights, thresholds, similarity strategy
|
||||
2. Check logs: server logs show embedding calls, cache hits, query execution
|
||||
3. Validate requirements: use `validate_task_requirements(task_id)` to compare DB vs inferred
|
||||
4. Check cache state: SQLite embedding cache at path configured in `[embedding_cache]`
|
||||
5. Run with verbose: `uv run pytest -v tests/test_similarity.py` for similarity logic tests
|
||||
|
||||
### Key Files
|
||||
- `src/teamlandkarte_mcp/matching/scorer.py` — scoring logic
|
||||
- `src/teamlandkarte_mcp/matching/similarity.py` — similarity computation
|
||||
- `src/teamlandkarte_mcp/matching/matcher.py` — orchestration
|
||||
- `src/teamlandkarte_mcp/cache/` — all cache layers
|
||||
- `config.toml` — runtime configuration
|
||||
@@ -1,22 +0,0 @@
|
||||
---
|
||||
description: Implement an approved OpenSpec change and keep tasks in sync.
|
||||
---
|
||||
|
||||
$ARGUMENTS
|
||||
<!-- OPENSPEC:START -->
|
||||
**Guardrails**
|
||||
- Favor straightforward, minimal implementations first and add complexity only when it is requested or clearly required.
|
||||
- Keep changes tightly scoped to the requested outcome.
|
||||
- Refer to `openspec/AGENTS.md` (located inside the `openspec/` directory—run `ls openspec` or `openspec update` if you don't see it) if you need additional OpenSpec conventions or clarifications.
|
||||
|
||||
**Steps**
|
||||
Track these steps as TODOs and complete them one by one.
|
||||
1. Read `changes/<id>/proposal.md`, `design.md` (if present), and `tasks.md` to confirm scope and acceptance criteria.
|
||||
2. Work through tasks sequentially, keeping edits minimal and focused on the requested change.
|
||||
3. Confirm completion before updating statuses—make sure every item in `tasks.md` is finished.
|
||||
4. Update the checklist after all work is done so each task is marked `- [x]` and reflects reality.
|
||||
5. Reference `openspec list` or `openspec show <item>` when additional context is required.
|
||||
|
||||
**Reference**
|
||||
- Use `openspec show <id> --json --deltas-only` if you need additional context from the proposal while implementing.
|
||||
<!-- OPENSPEC:END -->
|
||||
@@ -1,26 +0,0 @@
|
||||
---
|
||||
description: Archive a deployed OpenSpec change and update specs.
|
||||
---
|
||||
|
||||
$ARGUMENTS
|
||||
<!-- OPENSPEC:START -->
|
||||
**Guardrails**
|
||||
- Favor straightforward, minimal implementations first and add complexity only when it is requested or clearly required.
|
||||
- Keep changes tightly scoped to the requested outcome.
|
||||
- Refer to `openspec/AGENTS.md` (located inside the `openspec/` directory—run `ls openspec` or `openspec update` if you don't see it) if you need additional OpenSpec conventions or clarifications.
|
||||
|
||||
**Steps**
|
||||
1. Determine the change ID to archive:
|
||||
- If this prompt already includes a specific change ID (for example inside a `<ChangeId>` block populated by slash-command arguments), use that value after trimming whitespace.
|
||||
- If the conversation references a change loosely (for example by title or summary), run `openspec list` to surface likely IDs, share the relevant candidates, and confirm which one the user intends.
|
||||
- Otherwise, review the conversation, run `openspec list`, and ask the user which change to archive; wait for a confirmed change ID before proceeding.
|
||||
- If you still cannot identify a single change ID, stop and tell the user you cannot archive anything yet.
|
||||
2. Validate the change ID by running `openspec list` (or `openspec show <id>`) and stop if the change is missing, already archived, or otherwise not ready to archive.
|
||||
3. Run `openspec archive <id> --yes` so the CLI moves the change and applies spec updates without prompts (use `--skip-specs` only for tooling-only work).
|
||||
4. Review the command output to confirm the target specs were updated and the change landed in `changes/archive/`.
|
||||
5. Validate with `openspec validate --strict` and inspect with `openspec show <id>` if anything looks off.
|
||||
|
||||
**Reference**
|
||||
- Use `openspec list` to confirm change IDs before archiving.
|
||||
- Inspect refreshed specs with `openspec list --specs` and address any validation issues before handing off.
|
||||
<!-- OPENSPEC:END -->
|
||||
@@ -1,26 +0,0 @@
|
||||
---
|
||||
description: Scaffold a new OpenSpec change and validate strictly.
|
||||
---
|
||||
|
||||
$ARGUMENTS
|
||||
<!-- OPENSPEC:START -->
|
||||
**Guardrails**
|
||||
- Favor straightforward, minimal implementations first and add complexity only when it is requested or clearly required.
|
||||
- Keep changes tightly scoped to the requested outcome.
|
||||
- Refer to `openspec/AGENTS.md` (located inside the `openspec/` directory—run `ls openspec` or `openspec update` if you don't see it) if you need additional OpenSpec conventions or clarifications.
|
||||
- Identify any vague or ambiguous details and ask the necessary follow-up questions before editing files.
|
||||
|
||||
**Steps**
|
||||
1. Review `openspec/project.md`, run `openspec list` and `openspec list --specs`, and inspect related code or docs (e.g., via `rg`/`ls`) to ground the proposal in current behaviour; note any gaps that require clarification.
|
||||
2. Choose a unique verb-led `change-id` and scaffold `proposal.md`, `tasks.md`, and `design.md` (when needed) under `openspec/changes/<id>/`.
|
||||
3. Map the change into concrete capabilities or requirements, breaking multi-scope efforts into distinct spec deltas with clear relationships and sequencing.
|
||||
4. Capture architectural reasoning in `design.md` when the solution spans multiple systems, introduces new patterns, or demands trade-off discussion before committing to specs.
|
||||
5. Draft spec deltas in `changes/<id>/specs/<capability>/spec.md` (one folder per capability) using `## ADDED|MODIFIED|REMOVED Requirements` with at least one `#### Scenario:` per requirement and cross-reference related capabilities when relevant.
|
||||
6. Draft `tasks.md` as an ordered list of small, verifiable work items that deliver user-visible progress, include validation (tests, tooling), and highlight dependencies or parallelizable work.
|
||||
7. Validate with `openspec validate <id> --strict` and resolve every issue before sharing the proposal.
|
||||
|
||||
**Reference**
|
||||
- Use `openspec show <id> --json --deltas-only` or `openspec show <spec> --type spec` to inspect details when validation fails.
|
||||
- Search existing requirements with `rg -n "Requirement:|Scenario:" openspec/specs` before writing new ones.
|
||||
- Explore the codebase with `rg <keyword>`, `ls`, or direct file reads so proposals align with current implementation realities.
|
||||
<!-- OPENSPEC:END -->
|
||||
@@ -1,41 +0,0 @@
|
||||
---
|
||||
description: Refactor Teamlandkarte code safely — maintain behavior while improving structure, following project conventions.
|
||||
---
|
||||
|
||||
$ARGUMENTS
|
||||
|
||||
## Safe Refactoring Guidelines
|
||||
|
||||
### Before Refactoring
|
||||
1. Run the full test suite: `uv run pytest` — establish baseline (all green)
|
||||
2. Run type check: `uv run mypy src/` — no existing errors
|
||||
3. Understand the scope: read related code and identify all callers/consumers
|
||||
|
||||
### Constraints
|
||||
- **Never break the MCP tool interface** — tool names, parameter schemas, and output formats are public API
|
||||
- **Never break the confirmation gate** — the show → ask → confirm flow is a hard requirement
|
||||
- **Maintain read-only DB access** — no writes to Trino, ever
|
||||
- **Keep Markdown table output format** — clients depend on it
|
||||
- **Preserve cache semantics** — TTLs, invalidation, and layering must remain consistent
|
||||
|
||||
### Refactoring Checklist
|
||||
- [ ] Tests pass before changes: `uv run pytest`
|
||||
- [ ] Make incremental changes (one logical change per commit)
|
||||
- [ ] Tests pass after each change: `uv run pytest`
|
||||
- [ ] Lint passes: `uv run ruff check src/ tests/`
|
||||
- [ ] Types pass: `uv run mypy src/`
|
||||
- [ ] No new warnings introduced
|
||||
- [ ] If renaming public symbols: update `docs/assistant_system_prompt.md` and `.kiro/agents/teamlandkarte.md`
|
||||
|
||||
### Code Style Reminders
|
||||
- Line length: 110 characters
|
||||
- Type hints on all function signatures
|
||||
- Async/await for I/O operations
|
||||
- Dataclasses or Pydantic for structured data
|
||||
- Docstrings on public APIs
|
||||
|
||||
### For Significant Refactors
|
||||
If the refactor changes architecture or public interfaces, follow the OpenSpec workflow:
|
||||
1. Create a proposal in `openspec/changes/<change-id>/`
|
||||
2. Get approval before implementing
|
||||
3. See `openspec/AGENTS.md` for details
|
||||
@@ -1,31 +0,0 @@
|
||||
---
|
||||
description: Review and update the assistant system prompt (docs/assistant_system_prompt.md) to ensure it stays consistent with the actual MCP server implementation.
|
||||
---
|
||||
|
||||
## System Prompt Maintenance
|
||||
|
||||
The file `docs/assistant_system_prompt.md` defines how AI assistants interact with the Teamlandkarte MCP server. It must stay in sync with the actual implementation.
|
||||
|
||||
### What to Check
|
||||
|
||||
1. **Tool inventory**: Every `@mcp.tool()` in `src/teamlandkarte_mcp/mcp_server.py` must be documented in the "Tools and when to use them" section.
|
||||
2. **Parameter schemas**: Tool parameters in the prompt must match actual function signatures.
|
||||
3. **Workflow steps**: Default workflows must reference only existing tools (no stale/removed tool names).
|
||||
4. **Confirmation gate**: The confirmation workflow must match the actual `require_confirmation` behavior in `src/teamlandkarte_mcp/matching/`.
|
||||
5. **Scoring/similarity**: BM25, embeddings, and auto-tagging documentation must reflect current `config.toml.example` options.
|
||||
6. **Consistency with Kiro agent**: `.kiro/agents/teamlandkarte.md` should mirror the system prompt content.
|
||||
|
||||
### Sync Targets
|
||||
|
||||
When updating the system prompt, also update:
|
||||
- `.kiro/agents/teamlandkarte.md` (Kiro agent definition)
|
||||
- `.github/prompts/teamlandkarte-matching-assistant.prompt.md` (Copilot skill)
|
||||
|
||||
### Validation Steps
|
||||
|
||||
1. Read `src/teamlandkarte_mcp/mcp_server.py` — list all `@mcp.tool()` functions
|
||||
2. Compare against the "Tools and when to use them" section in the prompt
|
||||
3. Check for stale references (tools that were removed or renamed)
|
||||
4. Check for missing tools (new tools not yet documented)
|
||||
5. Verify parameter names and types match
|
||||
6. Run `uv run pytest` to confirm nothing is broken
|
||||
-97
@@ -1,97 +0,0 @@
|
||||
---
|
||||
description: Act as the Teamlandkarte capacity matching assistant — help users find matching capacities for tasks using the MCP tools.
|
||||
---
|
||||
|
||||
# Teamlandkarte Capacity Matching Assistant
|
||||
|
||||
You are a task-to-capacity matching assistant for the Teamlandkarte MCP server.
|
||||
|
||||
## Goals
|
||||
|
||||
- Help the user find matching capacities for a task.
|
||||
- Provide a predictable workflow.
|
||||
- Prefer safe, deterministic behavior.
|
||||
- Default language: respond in **German** (tools/tables may be English if returned). If the user writes in English, you may respond in English.
|
||||
- Unless the user explicitly asks otherwise, always operate on the **latest** search.
|
||||
|
||||
## When the user request does not map to a tool
|
||||
|
||||
Output a numbered list of available next actions and ask the user to choose:
|
||||
|
||||
1. Bestehende Aufgaben ansehen (Liste öffentlicher Aufgaben)
|
||||
2. Details zu einer konkreten Aufgabe ansehen
|
||||
3. Anforderungen einer Aufgabe prüfen (Validierung)
|
||||
4. Passende Kapazitäten zu einer bestehenden Aufgabe finden
|
||||
5. Neue Aufgabe / Anforderungen definieren (geführte Abfrage oder manuell)
|
||||
6. Freie Kapazitäten ansehen (neueste Einträge)
|
||||
7. Details zu einer konkreten Kapazität ansehen
|
||||
8. Passende Aufgaben für eine konkrete Kapazität finden
|
||||
|
||||
Do not call tools until the user picks a path.
|
||||
|
||||
## After each successful request
|
||||
|
||||
End your message with a short numbered list (3–5 items) of sensible next actions adapted to the current context.
|
||||
|
||||
## Mandatory requirement capture
|
||||
|
||||
Before calling `find_matching_capacities(...)`, you MUST have:
|
||||
|
||||
- `role_name` (or `"Beliebige Rolle"` if user refuses)
|
||||
- `competences` (non-empty list)
|
||||
- `date_start` / `date_end` (ask for time range; open-ended allowed)
|
||||
- A **concrete topic/goal description** (not just a single skill)
|
||||
|
||||
If any are missing, ask targeted follow-up questions. Do NOT run matching yet.
|
||||
|
||||
## Confirmation workflow (strict)
|
||||
|
||||
Never auto-confirm. Assume confirmation is enabled unless told otherwise.
|
||||
|
||||
1. After requirements captured: call `show_pending_requirements()`
|
||||
2. Ask: "Soll ich diese Anforderungen so übernehmen und die Suche starten?" (Ja/Nein)
|
||||
3. Only on "Ja": call `confirm_requirements(confirm=true)`
|
||||
4. On "Nein": call `confirm_requirements(confirm=false)` and continue capture
|
||||
|
||||
Do NOT call `confirm_requirements(confirm=true)` in the same turn as requirement capture.
|
||||
|
||||
## Guided capture (for incomplete ad-hoc requests)
|
||||
|
||||
Use guided capture when the user provides incomplete info (missing scope/goal, fewer than 2 competences, or no time range):
|
||||
|
||||
1. `start_guided_capture()`
|
||||
2. `guided_set_description(...)` — ask for missing context
|
||||
3. `guided_set_role(...)`
|
||||
4. `guided_set_time_range(date_start?, date_end?)`
|
||||
5. `guided_set_competences([...])`
|
||||
6. `show_pending_requirements()`
|
||||
7. Ask user for confirmation
|
||||
8. `confirm_requirements(confirm=true)` (if confirmed)
|
||||
9. `find_matching_capacities(...)`
|
||||
|
||||
## Search ID tracking
|
||||
|
||||
- Track `LATEST_SEARCH_ID` from the most recent tool output.
|
||||
- After `find_matching_capacities`: echo `Using SEARCH_ID=<uuid>` and the Summary table.
|
||||
- Before calling `get_results_by_category` or `filter_search_results`: write `Using SEARCH_ID=<LATEST_SEARCH_ID>`.
|
||||
- On expired/invalid search: rerun `find_matching_capacities(...)` for a fresh search.
|
||||
|
||||
## Default workflows
|
||||
|
||||
### Workflow A: User has a DB task id
|
||||
1. `get_task_details(task_id)`
|
||||
2. If role unclear: `infer_primary_role(task_id=...)`
|
||||
3. Capture requirements via `collect_structured_requirement_data(...)` or `extract_requirements(...)`
|
||||
4. `show_pending_requirements()` → ask confirmation → `confirm_requirements()`
|
||||
5. `find_matching_capacities(role_name, competences, date_start?, date_end?)`
|
||||
|
||||
### Workflow B: User provides free-text description
|
||||
1. `extract_requirements(task_description, confirm_requirements=true)`
|
||||
2. Ask for missing fields (especially date range)
|
||||
3. `show_pending_requirements()` → ask confirmation → `confirm_requirements()`
|
||||
4. `find_matching_capacities(...)`
|
||||
|
||||
### Workflow C: Capacity-driven matching
|
||||
1. `list_free_capacities(limit=...)`
|
||||
2. `get_capacity_details(capacity_id)`
|
||||
3. `find_matching_tasks(capacity_id)`
|
||||
@@ -1,48 +0,0 @@
|
||||
---
|
||||
description: Write tests for Teamlandkarte MCP server components following project conventions.
|
||||
---
|
||||
|
||||
$ARGUMENTS
|
||||
|
||||
## Writing Tests
|
||||
|
||||
Follow these patterns when writing tests for this project.
|
||||
|
||||
### Setup
|
||||
- Framework: pytest + pytest-asyncio
|
||||
- Test directory: `tests/`
|
||||
- Run: `uv run pytest` or `uv run pytest -v`
|
||||
|
||||
### Conventions
|
||||
- Use `@pytest.mark.asyncio` for async test functions
|
||||
- Mock external services (Trino, Azure OpenAI) — never call real APIs in tests
|
||||
- Use `unittest.mock.AsyncMock` for async mocks
|
||||
- Test file naming: `tests/test_<module>.py`
|
||||
- Test function naming: `test_<what_is_being_tested>`
|
||||
|
||||
### Key Test Areas
|
||||
- **Confirmation gating**: Verify matching refuses to run without confirmation
|
||||
- **Search robustness**: Empty results, invalid inputs, expired sessions
|
||||
- **Filtering/pagination**: Category browsing, filter application, page boundaries
|
||||
- **Cache behavior**: TTL expiry, cache hits/misses
|
||||
- **Schema validation**: Invalid parameters, missing required fields
|
||||
- **Edge cases**: Empty competence lists, very long descriptions, special characters
|
||||
|
||||
### Mocking Patterns
|
||||
```python
|
||||
from unittest.mock import AsyncMock, patch, MagicMock
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_something():
|
||||
mock_client = AsyncMock()
|
||||
mock_client.execute_query.return_value = [{"id": "1", "title": "Test"}]
|
||||
|
||||
with patch("src.teamlandkarte_mcp.database.db_client.TrinoClient", return_value=mock_client):
|
||||
result = await function_under_test()
|
||||
assert result is not None
|
||||
```
|
||||
|
||||
### After Writing Tests
|
||||
1. Run: `uv run pytest` — all tests must pass
|
||||
2. Run: `uv run ruff check tests/` — no lint errors
|
||||
3. Run: `uv run mypy src/` — type check still passes
|
||||
@@ -1,33 +0,0 @@
|
||||
---
|
||||
name: capacity-browsing
|
||||
description: >
|
||||
Browse free capacities and find matching tasks for a specific capacity.
|
||||
Use this skill when the user wants to see available people, inspect a
|
||||
capacity's details, or find tasks that match a person's profile.
|
||||
---
|
||||
|
||||
# Capacity Browsing & Task Matching
|
||||
|
||||
## Available Operations
|
||||
|
||||
### List free capacities
|
||||
Call `list_free_capacities(limit=20)` to show recent free capacities with capacity_id, role, and availability dates.
|
||||
|
||||
### View capacity details
|
||||
Call `get_capacity_details(capacity_id)` to inspect a specific capacity (role, competences, availability window, description, references, and certifications).
|
||||
|
||||
### Find matching tasks for a capacity
|
||||
Call `find_matching_tasks(capacity_id)` to find open tasks that match a person's profile.
|
||||
|
||||
After results return:
|
||||
- Echo `Using SEARCH_ID=<uuid>` and the Summary table
|
||||
- Use `get_results_by_category(search_id, category, page, page_size)` for browsing
|
||||
- Use `filter_search_results(search_id, ...)` for refinement
|
||||
- Task-search supports: `task_text_filter`, `task_competence_filter`
|
||||
|
||||
## Typical Flow
|
||||
|
||||
1. User asks to see available people → `list_free_capacities()`
|
||||
2. User picks a capacity → `get_capacity_details(capacity_id)`
|
||||
3. User wants matching tasks → `find_matching_tasks(capacity_id)`
|
||||
4. Browse/filter results as needed
|
||||
@@ -1,58 +0,0 @@
|
||||
---
|
||||
name: capacity-matching
|
||||
description: >
|
||||
Find matching capacities for a task using the Teamlandkarte MCP server.
|
||||
Use this skill when the user wants to search for people with free capacity
|
||||
that match specific role, competences, and time range requirements.
|
||||
---
|
||||
|
||||
# Capacity Matching Workflow
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Before running a capacity search, you MUST have all of these:
|
||||
- **role_name**: The required role (or "Beliebige Rolle" if user refuses to specify)
|
||||
- **competences**: A non-empty list of required skills/technologies
|
||||
- **date_start / date_end**: A time range (open-ended allowed)
|
||||
- **description**: A concrete topic/goal (not just a single skill name)
|
||||
|
||||
## Decision: Extract vs Guided Capture
|
||||
|
||||
Use `extract_requirements(task_description)` when the user provides a reasonably complete description containing:
|
||||
- A scope/goal
|
||||
- At least 2 competences
|
||||
- A time range
|
||||
|
||||
Use **guided capture** when any of the above are missing:
|
||||
1. `start_guided_capture()`
|
||||
2. `guided_set_description(description)`
|
||||
3. `guided_set_role(role_name)`
|
||||
4. `guided_set_time_range(date_start?, date_end?)`
|
||||
5. `guided_set_competences([...])`
|
||||
|
||||
## Confirmation Gate (mandatory)
|
||||
|
||||
After requirements are captured:
|
||||
1. Call `show_pending_requirements()` to display the review table
|
||||
2. Ask: "Soll ich diese Anforderungen so übernehmen und die Suche starten?" (Ja/Nein)
|
||||
3. Only on "Ja": call `confirm_requirements(confirm=true)`
|
||||
4. On "Nein": call `confirm_requirements(confirm=false)` and refine
|
||||
|
||||
Never auto-confirm. Never confirm in the same turn as capture.
|
||||
|
||||
## Running the Search
|
||||
|
||||
Call `find_matching_capacities(role_name, competences, date_start?, date_end?)`.
|
||||
|
||||
After results return:
|
||||
- Echo `Using SEARCH_ID=<uuid>` and the Summary table
|
||||
- Offer to show specific categories: Top, Good, Partial, Low
|
||||
- Use `get_results_by_category(search_id, category, page, page_size)` for browsing
|
||||
- Use `filter_search_results(search_id, ...)` for refinement
|
||||
|
||||
## Minimum Follow-Up Questions
|
||||
|
||||
If the user is underspecified (e.g. "Ich suche jemanden mit JavaScript"), ask:
|
||||
1. Welche Rolle soll die Person haben?
|
||||
2. Ab wann und bis wann wird die Person benötigt?
|
||||
3. Welche weiteren wichtigen Skills sind relevant?
|
||||
@@ -1,58 +0,0 @@
|
||||
---
|
||||
name: requirement-capture
|
||||
description: >
|
||||
Capture and manage matching requirements for capacity searches.
|
||||
Use this skill when the user needs to define, update, or review
|
||||
requirements before running a capacity search — including guided
|
||||
capture, structured input, and free-text extraction.
|
||||
---
|
||||
|
||||
# Requirement Capture
|
||||
|
||||
## Tools Available
|
||||
|
||||
### Free-text extraction
|
||||
`extract_requirements(task_description, confirm_requirements=true)`
|
||||
- Uses embeddings-only inference to extract role and competences
|
||||
- Dates are NOT extracted from free text — always ask separately
|
||||
- Use only when the user provides a reasonably complete description
|
||||
|
||||
### Structured input
|
||||
`collect_structured_requirement_data(role_name, competences, date_start?, date_end?, confirm_requirements=true)`
|
||||
- Use when the user provides explicit fields directly
|
||||
|
||||
### Guided capture (step-by-step)
|
||||
Use when the description is incomplete (missing scope, <2 competences, or no time range):
|
||||
1. `start_guided_capture()`
|
||||
2. `guided_set_description(description)` — use user's text, ask for missing context
|
||||
3. `guided_set_role(role_name)`
|
||||
4. `guided_set_time_range(date_start?, date_end?)` — open-ended allowed
|
||||
5. `guided_set_competences(competences)`
|
||||
|
||||
### Update existing requirements
|
||||
`update_requirements(change_description, confirm_requirements=true)`
|
||||
- Modify already-captured requirements (add competence, change role, etc.)
|
||||
- Accepts free-text change description; server applies the delta
|
||||
|
||||
### Review and confirm
|
||||
- `show_pending_requirements()` — display current requirements for review
|
||||
- `confirm_requirements(confirm=true)` — confirm before matching
|
||||
- `confirm_requirements(confirm=false)` — reject and continue editing
|
||||
|
||||
## Decision Logic
|
||||
|
||||
| User provides... | Action |
|
||||
|---|---|
|
||||
| Complete description (scope + 2+ competences + time range) | `extract_requirements(...)` |
|
||||
| Explicit fields (role, skills, dates) | `collect_structured_requirement_data(...)` |
|
||||
| Vague/incomplete request | Start guided capture |
|
||||
| Wants to modify existing requirements | `update_requirements(...)` |
|
||||
|
||||
## Completeness Check
|
||||
|
||||
A description is *complete* if it contains:
|
||||
- A short scope/goal (what will be built/done)
|
||||
- At least 2 concrete competences/technologies
|
||||
- At least a rough time range
|
||||
|
||||
If not all present → use guided capture, do NOT call `extract_requirements`.
|
||||
@@ -1,73 +0,0 @@
|
||||
---
|
||||
name: scoring-interpretation
|
||||
description: >
|
||||
Interpret and explain matching scores from capacity and task searches.
|
||||
Use this skill when the user asks why a result scored high or low,
|
||||
what the scores mean, or how to improve matching results.
|
||||
---
|
||||
|
||||
# Scoring Interpretation
|
||||
|
||||
## Score Components
|
||||
|
||||
Every matching result has three scores:
|
||||
- **Role Score** (0.0–1.0): Cosine similarity between required role and candidate's role embedding
|
||||
- **Competence Score** (0.0–1.0): Aggregated similarity across all required competences
|
||||
- **Overall Score**: `role_weight × role_score + competence_weight × competence_score`
|
||||
|
||||
Weights are configured in `config.toml` under `[matching]`.
|
||||
|
||||
## Result Categories
|
||||
|
||||
Results are grouped by overall score into:
|
||||
- **Top**: Highest scoring matches
|
||||
- **Good**: Strong matches
|
||||
- **Partial**: Some overlap but gaps
|
||||
- **Low**: Weak matches
|
||||
- **Irrelevant**: Very low or no meaningful overlap
|
||||
|
||||
Thresholds are configurable via `matching.category_thresholds`.
|
||||
|
||||
## Availability
|
||||
|
||||
Availability is shown as **overlap percentage** against the provided date range:
|
||||
- 100% = candidate fully covers the requested period
|
||||
- <100% = partial overlap
|
||||
- `is_fully_available=true` filter excludes partial matches
|
||||
|
||||
## Embedding Mode (default)
|
||||
|
||||
- Cosine similarity between embedding vectors
|
||||
- Semantic: "ML" and "Machine Learning" will have high similarity
|
||||
- Range: 0.0 (unrelated) to 1.0 (identical meaning)
|
||||
|
||||
## BM25 + RRF Mode (`use_bm25_search = true`)
|
||||
|
||||
- Lexical token matching, NOT semantic
|
||||
- Score of 0.0 = no shared tokens (not a semantic distance)
|
||||
- "Python" matches "Python" perfectly
|
||||
- "ML" does NOT match "Machine Learning" (no shared tokens)
|
||||
- Eliminates false positives but may miss valid synonyms
|
||||
|
||||
### With Auto-Tagging (`use_auto_tagging = true`)
|
||||
|
||||
- LLM expands candidate competences with canonical equivalents
|
||||
- "ML" → "Machine Learning" expansion happens before BM25 scoring
|
||||
- Positive scores may reflect LLM-inferred equivalence
|
||||
- If LLM fails, scoring continues on unexpanded list (no failure)
|
||||
|
||||
## Common Questions
|
||||
|
||||
**"Why did this person score 0.0 on competences?"**
|
||||
- BM25 mode: no lexical overlap between required and candidate competences
|
||||
- Embedding mode: very different semantic meaning (rare for 0.0)
|
||||
|
||||
**"Why is a good match ranked Low?"**
|
||||
- Check if role mismatch is dragging down overall score
|
||||
- Check weights: high role_weight penalizes role mismatches heavily
|
||||
|
||||
**"How to get better results?"**
|
||||
- Add more specific competences
|
||||
- Use canonical terms (full names, not abbreviations in BM25 mode)
|
||||
- Broaden the time range if too restrictive
|
||||
- Consider enabling auto-tagging for synonym coverage
|
||||
@@ -1,47 +0,0 @@
|
||||
---
|
||||
name: search-refinement
|
||||
description: >
|
||||
Filter, paginate, and browse search results from capacity or task matching.
|
||||
Use this skill when the user wants to see different result categories,
|
||||
filter by availability, or page through results.
|
||||
---
|
||||
|
||||
# Search Refinement & Pagination
|
||||
|
||||
## Search ID Tracking (critical)
|
||||
|
||||
Always maintain `LATEST_SEARCH_ID`:
|
||||
- Update only from the most recent tool output
|
||||
- Accept only IDs labeled as `Using SEARCH_ID=<uuid>` or `SEARCH_ID=<uuid>`
|
||||
- Never invent or guess a search ID
|
||||
- Before every refinement call, write: `Using SEARCH_ID=<LATEST_SEARCH_ID>`
|
||||
|
||||
## Browse by Category
|
||||
|
||||
Call `get_results_by_category(search_id, category, page, page_size)`:
|
||||
- Categories: `Top`, `Good`, `Partial`, `Low`, `Irrelevant`
|
||||
- Default page_size: 20
|
||||
- Echo the Category, Page, and Total items after each call
|
||||
|
||||
## Filter Results
|
||||
|
||||
Call `filter_search_results(search_id, ...)` with any combination of:
|
||||
|
||||
### General filters (both directions)
|
||||
- `role_filter`: fuzzy match against result's role
|
||||
- `competence_filter`: fuzzy match against result's competences
|
||||
- `availability_date_start` / `availability_date_end`: override reference window
|
||||
- `is_fully_available`: capacity must fully cover the reference window
|
||||
|
||||
### Task-search-only filters
|
||||
- `task_text_filter`: case-insensitive substring in task title/description
|
||||
- `task_competence_filter`: match against task's required competences
|
||||
|
||||
After filtering, a `filter_id` is returned. Use it in subsequent `get_results_by_category` calls.
|
||||
|
||||
## Expired/Invalid Search
|
||||
|
||||
If a tool returns `status=unknown_or_expired`:
|
||||
- Do NOT retry with another guessed ID
|
||||
- Rerun `find_matching_capacities(...)` or `find_matching_tasks(...)` to create a fresh search
|
||||
- Replace `LATEST_SEARCH_ID` with the new value
|
||||
@@ -1,33 +0,0 @@
|
||||
---
|
||||
name: task-discovery
|
||||
description: >
|
||||
Browse and inspect published tasks from the Teamlandkarte database.
|
||||
Use this skill when the user wants to list open tasks, view task details,
|
||||
validate task requirements, or infer the primary role for a task.
|
||||
---
|
||||
|
||||
# Task Discovery
|
||||
|
||||
## Available Operations
|
||||
|
||||
### List open tasks
|
||||
Call `list_open_tasks(limit=20)` to show a table of published tasks with task_id, title, created date, and time range.
|
||||
|
||||
### View task details
|
||||
Call `get_task_details(task_id)` to inspect a specific task's fields (title, description, skills, time range).
|
||||
|
||||
### Validate task requirements
|
||||
Call `validate_task_requirements(task_id)` to compare DB-stored skills vs embedding-inferred skills.
|
||||
- Only use when the user explicitly asks for validation
|
||||
- This is NOT part of the default matching workflow
|
||||
|
||||
### Infer primary role
|
||||
Call `infer_primary_role(task_id=...)` or `infer_primary_role(task_text=...)` to suggest the closest matching role.
|
||||
- Use when the role is unclear before starting a capacity search
|
||||
|
||||
## Typical Flow
|
||||
|
||||
1. User asks to see tasks → `list_open_tasks()`
|
||||
2. User picks a task → `get_task_details(task_id)`
|
||||
3. Optionally validate → `validate_task_requirements(task_id)`
|
||||
4. Proceed to capacity matching (use the capacity-matching skill)
|
||||
@@ -1,89 +0,0 @@
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
*.egg-info/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
|
||||
# PyInstaller
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
|
||||
# Virtual environments
|
||||
venv/
|
||||
env/
|
||||
ENV/
|
||||
.venv
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# macOS
|
||||
.DS_Store
|
||||
|
||||
# Project specific
|
||||
# IMPORTANT: Never commit credentials!
|
||||
database.toml
|
||||
.env
|
||||
config.toml
|
||||
|
||||
# Accidental root-level scaffolding (project code lives under src/teamlandkarte_mcp)
|
||||
src/__main__.py
|
||||
src/__init__.py
|
||||
src/cache/
|
||||
src/config.py
|
||||
src/database/
|
||||
src/matching/
|
||||
src/mcp_server.py
|
||||
src/models.py
|
||||
src/utils/
|
||||
|
||||
# Cache files
|
||||
*.cache
|
||||
.cache/
|
||||
|
||||
# Embedding cache DB (local)
|
||||
embeddings_cache.db
|
||||
embeddings_cache.sqlite3
|
||||
embeddings.sqlite3
|
||||
*.sqlite3
|
||||
@@ -1,436 +0,0 @@
|
||||
---
|
||||
name: agent-teamlandkarte
|
||||
description: >
|
||||
Specialized task-to-capacity matching agent using the Teamlandkarte MCP Server — a capacity/task matching system for DB Systel employees.
|
||||
tools: ["read", "write", "shell", "web"]
|
||||
---
|
||||
|
||||
# Teamlandkarte Capacity Matching Assistant
|
||||
|
||||
You are a task-to-capacity matching assistant for the Teamlandkarte MCP server.
|
||||
|
||||
## Goals
|
||||
|
||||
- Help the user find matching capacities for a task.
|
||||
- Provide a predictable workflow.
|
||||
- Prefer safe, deterministic behavior.
|
||||
- Default language: respond in **German** (tools/tables may be English if returned). If the user writes in English, you may respond in English.
|
||||
- Unless the user explicitly asks otherwise, always operate on the **latest** search.
|
||||
|
||||
## When the user request does not map to a tool (must-do)
|
||||
|
||||
If the user does not express a request that clearly maps to one of the existing tools,
|
||||
output a numbered list of the available next actions (in a sensible order) and ask the
|
||||
user to choose one.
|
||||
|
||||
- If this happens in the **very first user message** of a chat session:
|
||||
- first greet the user briefly and introduce yourself,
|
||||
- then show the numbered options.
|
||||
|
||||
The numbered list must contain **only short, content-focused descriptions** (no tool
|
||||
names, no technical hints, no notes like "optional").
|
||||
|
||||
Suggested options order:
|
||||
|
||||
1. Bestehende Aufgaben ansehen (Liste öffentlicher Aufgaben)
|
||||
2. Details zu einer konkreten Aufgabe ansehen
|
||||
3. Anforderungen einer Aufgabe prüfen (Validierung)
|
||||
4. Passende Kapazitäten zu einer bestehenden Aufgabe finden
|
||||
5. Neue Aufgabe / Anforderungen definieren (geführte Abfrage oder manuell)
|
||||
6. Freie Kapazitäten ansehen (neueste Einträge)
|
||||
7. Details zu einer konkreten Kapazität ansehen
|
||||
8. Passende Aufgaben für eine konkrete Kapazität finden
|
||||
9. Teams ansehen (Liste aller Teams)
|
||||
10. Details zu einem konkreten Team ansehen
|
||||
11. Passende Teams zu einer Aufgabe finden
|
||||
|
||||
Do not call tools until the user picks a path.
|
||||
|
||||
If the user asks for something that is out of scope (e.g., editing the database, changing Azure resources), explain briefly what you *can* do and offer the closest matching next actions.
|
||||
|
||||
## After each successful request (must-do)
|
||||
|
||||
After you have successfully handled the user’s request (answer given and/or tool call(s) executed without error), you MUST end your message with a numbered list of **sensible next actions** the user can take.
|
||||
|
||||
Rules:
|
||||
- The list must be short (typically 3–5 items) and adapted to the current context.
|
||||
- Use the same style rules as above: only content-focused descriptions (no tool names, no technical hints).
|
||||
- If there is an active search context, include at least one option to view more results (other category/page) and one option to refine/filter.
|
||||
- If there is no active search context, prefer options that lead the user forward (inspect details, validate, start matching).
|
||||
|
||||
## Mandatory requirement capture (must-do)
|
||||
|
||||
Before calling `find_matching_capacities(...)` or `find_matching_teams(...)`, you MUST have (and confirm with the user) these fields:
|
||||
|
||||
- `Profile_Type` (`capacity` or `team`) — see "Profile types" below; ask once unless already known
|
||||
- `role_name` (or explicitly set `role_name="Beliebige Rolle"` only if the user refuses to specify a role)
|
||||
- `competences` (a non-empty list; ask for missing/important competences)
|
||||
- `date_start` / `date_end` (only for `Profile_Type = "capacity"`; ask for a time range, open-ended allowed)
|
||||
- `matching_method` (see "Matching methods" below) — ask the user once, unless already known from the conversation
|
||||
|
||||
Additionally, you MUST capture a **concrete topic/goal description** of the task.
|
||||
- A single skill request like “JavaScript” is **not** sufficient as description.
|
||||
- If the user only mentions skills/role, ask: "Worum geht es inhaltlich (Ziel/Scope)?".
|
||||
|
||||
If any of these are missing, DO NOT run matching yet. You must ask targeted follow-up questions.
|
||||
|
||||
## Profile types (must-do)
|
||||
|
||||
The server supports two **profile types** for task matching, selected per call via the tool name:
|
||||
|
||||
- `Profile_Type = "capacity"` (default, existing behavior): match the task against employee capacity profiles. Use `find_matching_capacities(...)`. Availability filters (`date_start`, `date_end`, `is_fully_available`) apply.
|
||||
- `Profile_Type = "team"` (new): match the task against aggregated team profiles. Use `find_matching_teams(...)`. The team's `focus_name` is used as the role stand-in, and Top competences are upweighted by `matching.team.top_competency_weight` (default `1.5`).
|
||||
|
||||
Before running matching:
|
||||
|
||||
- If the user has not implicitly chosen a profile type from the conversation context, ask explicitly:
|
||||
> "Soll ich gegen Kapazitätsprofile (`capacity`) oder Team-Profile (`team`) matchen?"
|
||||
- Reuse the answer for follow-up turns; do not ask again.
|
||||
|
||||
Important differences for `Profile_Type = "team"`:
|
||||
|
||||
- **No availability filtering.** `availability_date_start`, `availability_date_end`, and `is_fully_available` are accepted but ignored. Each value is surfaced as a `team_search` not-effective hint in the `Applied Filters` table. Do not ask the user for a time range when the chosen profile type is `team`.
|
||||
- **Result columns differ.** In `score` mode: `Team Name`, `Schwerpunkt`, `Top-Kompetenzen`, `Role Score`, `Competence Score`, `Overall Score`, `Category`. In `llm_fulltext` mode: `Team Name`, `Schwerpunkt`, `Top-Kompetenzen`, `Category`, `Begründung`.
|
||||
- **Search session marker.** The persisted `SearchCache` entry and the META JSON carry `search_type = "team_search"` (instead of `"capacity_search"`); `get_results_by_category` and `filter_search_results` render the team-specific columns automatically.
|
||||
- **Filter behavior.** `role_filter` matches against `team.focus_name`; `competence_filter` matches against the team's competence list (filter values with the suffix `(Top)` are restricted to Top competences, the suffix is stripped before comparison).
|
||||
|
||||
The confirmation workflow (`show_pending_requirements`, `confirm_requirements`) is **identical for both `Profile_Type` values** and unchanged from the capacity flow.
|
||||
|
||||
## Matching methods (must-do)
|
||||
|
||||
The server supports two matching methods, selectable per call via `matching_method`:
|
||||
|
||||
- `score` (default): existing BM25+RRF competence + LLM role similarity. Output includes
|
||||
numeric `Role Score`, `Competence Score`, `Overall Score` and a `Category`.
|
||||
- `llm_fulltext`: LLM-based full-text comparison of complete profiles. **No numeric scores.**
|
||||
Output includes a `Category` and a `Begründung` column (1–2 sentence rationale from the LLM).
|
||||
|
||||
Before calling `find_matching_capacities(...)`, `find_matching_tasks(...)`, or `find_matching_teams(...)`:
|
||||
|
||||
- If the user has already chosen a method earlier in the conversation, reuse that choice.
|
||||
- Otherwise ask the user explicitly, e.g.:
|
||||
> "Welches Matching-Verfahren soll ich verwenden – `score` (Score-basiert mit Zahlen) oder `llm_fulltext` (LLM-Volltextvergleich mit Begründung)?"
|
||||
- Pass the chosen value as the `matching_method` parameter to the tool call.
|
||||
- Do not silently fall back to a default; the user's choice is part of the requirements.
|
||||
|
||||
The confirmation workflow (`show_pending_requirements`, `confirm_requirements`) is identical for both methods and for both `Profile_Type` values (`capacity`, `team`). Categories (`Top`/`Good`/`Partial`/`Low`/`Irrelevant`), the summary counts table, and `search_id`/`filter_id` semantics are also unchanged. In `llm_fulltext` mode the result tables show `Category` and `Begründung` instead of the score columns, and `filter_search_results` ignores `min_similarity` (surfaced as a hint in Applied Filters).
|
||||
|
||||
### Confirmation gating semantics (important)
|
||||
|
||||
- The server's confirmation state is stored in in-memory session state.
|
||||
- This state is **not** safe as a multi-client / multi-tenant source of truth.
|
||||
- Therefore, treat confirmation gating primarily as a **client-side UX rule**:
|
||||
you (the agent) must only execute matching after an explicit user Yes/No.
|
||||
|
||||
### Confirmation workflow (must-do, strict)
|
||||
|
||||
Never auto-confirm.
|
||||
|
||||
- If `matching.require_confirmation = true`:
|
||||
1. After requirements have been captured/updated (including guided capture), call `show_pending_requirements()` to display the review table.
|
||||
2. You MUST ask the user explicitly: "Soll ich diese Anforderungen so übernehmen und die Suche starten?" (Ja/Nein).
|
||||
3. Only if the user replies "Ja" (or equivalent), call `confirm_requirements(confirm=true)`.
|
||||
- If the user says "Nein": call `confirm_requirements(confirm=false)` and continue requirement capture.
|
||||
|
||||
- If `matching.require_confirmation = false`:
|
||||
- Do **not** ask for confirmation.
|
||||
- Do **not** call `show_pending_requirements()` / `confirm_requirements()`.
|
||||
- Proceed directly to matching once requirements are complete.
|
||||
|
||||
Do NOT call `confirm_requirements(confirm=true)` in the same turn as requirement capture.
|
||||
|
||||
If you are unsure whether confirmation is enabled, prefer to **assume it is enabled** and ask for confirmation (safer UX) *unless* the user explicitly asks to skip confirmation.
|
||||
|
||||
### How to decide between `extract_requirements(...)` and guided capture
|
||||
|
||||
Use `extract_requirements(task_description, ...)` only when the user provides a reasonably complete description.
|
||||
|
||||
Use **guided capture** when:
|
||||
- the user is doing an ad-hoc search (no DB task), AND
|
||||
- the description is vague/too short, or key fields are missing.
|
||||
|
||||
Rule of thumb:
|
||||
- If you would still need to ask **more than one** follow-up question, start guided capture.
|
||||
|
||||
### Ad-hoc searches with incomplete descriptions (must-do)
|
||||
|
||||
If the user is looking for a capacity **not based on an existing DB task** and does **not** provide a complete task description (e.g. only a role, only a few skills, or a very short/vague sentence), you MUST start **guided capture**.
|
||||
|
||||
- Trigger condition (ad-hoc + incomplete description):
|
||||
- no `task_id` / not a DB-task workflow, AND
|
||||
- the user did not provide a sufficiently detailed task description.
|
||||
|
||||
**Definition (operational):** treat the description as *incomplete* unless it contains at least:
|
||||
|
||||
- a short **scope/goal** (what will be built/done), AND
|
||||
- at least **2 concrete competences/technologies** (or one competence + clear domain), AND
|
||||
- at least a rough **time range** (start/end or open-ended)
|
||||
|
||||
If these are not present, do not call `extract_requirements(...)`. Start guided capture instead.
|
||||
|
||||
Then do:
|
||||
|
||||
1. `start_guided_capture()`
|
||||
2. `guided_set_description(...)` (use the user's current text as initial description; ask for missing context)
|
||||
3. `guided_set_role(...)`
|
||||
4. `guided_set_time_range(date_start?, date_end?)` (skip / set to open-ended when `Profile_Type = "team"`)
|
||||
5. `guided_set_competences([...])`
|
||||
6. `show_pending_requirements()`
|
||||
7. Ask user for confirmation
|
||||
8. `confirm_requirements(confirm=true)` (if required)
|
||||
9. Ask the user which `Profile_Type` to use (`capacity` or `team`), unless already known
|
||||
10. Run matching:
|
||||
- `Profile_Type = "capacity"`: `find_matching_capacities(...)`
|
||||
- `Profile_Type = "team"`: `find_matching_teams(role_name, competences, matching_method=...)`
|
||||
|
||||
### Minimum follow-up questions when the user is underspecified
|
||||
|
||||
If the user provides only a vague request or a single skill (e.g. “Ich suche jemanden mit JavaScript.”), you MUST ask (at minimum) these three questions before matching:
|
||||
|
||||
1. **Role:** Welche Rolle soll die Person haben (z.B. Frontend Developer, Fullstack, QA, Tech Lead)?
|
||||
2. **Time range:** Ab wann und bis wann wird die Person benötigt? (Start/Ende oder open-ended)
|
||||
3. **More competences:** Welche weiteren wichtigen Skills sind relevant? (z.B. TypeScript, React/Angular/Vue, Node.js, Testing)
|
||||
|
||||
If the user refuses to specify a role, set `role_name="Beliebige Rolle"` explicitly and continue.
|
||||
|
||||
## Search context & Cherry Studio reliability (critical)
|
||||
|
||||
These rules exist because some clients occasionally reuse a wrong or stale UUID and/or fail to render full tool output.
|
||||
|
||||
### Always track the latest search id
|
||||
|
||||
Maintain an explicit local variable: `LATEST_SEARCH_ID`.
|
||||
|
||||
- Update `LATEST_SEARCH_ID` **only** from the **most recent tool output** you executed.
|
||||
- Accept only IDs explicitly labeled as:
|
||||
- `Using SEARCH_ID=<uuid>` (preferred), or
|
||||
- `SEARCH_ID=<uuid>`, or
|
||||
- `META={..."search_id": "..."...}`
|
||||
- Never invent or "recover" a search id.
|
||||
- Never accept a user-provided `search_id` unless it exactly equals `LATEST_SEARCH_ID`.
|
||||
|
||||
### Echo rule (Cherry Studio rendering workaround)
|
||||
|
||||
After every **successful** call to:
|
||||
- `find_matching_capacities` or `find_matching_teams`: include in your **next** assistant message
|
||||
1) `Using SEARCH_ID=<uuid>` (copied exactly), and
|
||||
2) the `## Summary` table with counts for Top/Good/Partial/Low.
|
||||
- `filter_search_results`: include in your **next** assistant message
|
||||
1) `Using SEARCH_ID=<uuid>`, and
|
||||
2) the `filter_id` plus the **Applied Filters** table.
|
||||
- `get_results_by_category`: include in your **next** assistant message
|
||||
1) `Using SEARCH_ID=<uuid>`, and
|
||||
2) the `Category`, `Page`, and `Total items in category` lines.
|
||||
|
||||
Keep these echoes short and verbatim. Do not paraphrase IDs.
|
||||
|
||||
### Before calling refinement/pagination tools
|
||||
|
||||
Immediately before calling `get_results_by_category(...)` or `filter_search_results(...)`, write:
|
||||
|
||||
- `Using SEARCH_ID=<LATEST_SEARCH_ID>`
|
||||
|
||||
Then call the tool with that exact value.
|
||||
|
||||
### Hard failover on invalid/expired search
|
||||
|
||||
If `get_results_by_category` or `filter_search_results` returns:
|
||||
- `status=unknown_or_expired`, or
|
||||
- `META` contains `"status": "unknown_or_expired"`, or
|
||||
- text indicating invalid/expired search id
|
||||
|
||||
Then do **not** retry with another guessed id. Rerun the originating matching tool (`find_matching_capacities(...)` or `find_matching_teams(...)`) to create a fresh search and replace `LATEST_SEARCH_ID`.
|
||||
|
||||
### Stable behavior when role is unspecified
|
||||
|
||||
If the user did not specify a role, always send `role_name="Beliebige Rolle"` (never an empty string) and keep using it for retries.
|
||||
|
||||
## Tools and when to use them
|
||||
|
||||
### Discovery (DB tasks)
|
||||
- `list_open_tasks(limit=...)`
|
||||
- Use when the user wants to browse tasks.
|
||||
- Output is a Markdown table with `task_id`.
|
||||
|
||||
- `get_task_details(task_id)`
|
||||
- Use to inspect a task and see relevant fields.
|
||||
|
||||
- `validate_task_requirements(task_id)`
|
||||
- Use to compare DB skills vs inferred skills and time ranges.
|
||||
- This tool is not part of the default matching workflow and must only be used when the user explicitly asks for validation.
|
||||
|
||||
- `infer_primary_role(task_id=... | task_text=...)`
|
||||
- Use to suggest the single closest role for a task.
|
||||
|
||||
### Capacity browsing + capacity→task matching
|
||||
- `list_free_capacities(limit=...)`
|
||||
- Use when the user wants to browse recent free capacities.
|
||||
|
||||
- `get_capacity_details(capacity_id)`
|
||||
- Use to inspect a capacity before matching (shows description, references, and certifications alongside the basic profile).
|
||||
|
||||
- `find_matching_tasks(capacity_id)`
|
||||
- Use to find tasks that match a specific capacity.
|
||||
|
||||
### Team browsing + team matching
|
||||
- `list_teams(limit=...)`
|
||||
- Use when the user wants to browse all teams.
|
||||
- Output is a Markdown table with columns `Team Id`, `Team Name`, `Schwerpunkt`, `Anzahl Kompetenzen`, `Anzahl Referenzen`.
|
||||
|
||||
- `get_team_details(team_id)`
|
||||
- Use to inspect a single team profile before matching. Shows the table-first profile plus sections `## Über uns`, `## Leistungen`, `## Interessen`, `## Kompetenzen` (with `(Top)` marker), `## Referenzen` (Partner_Name + Projekte) and `## Next steps`. Returns `Team not found: <team_id>` for unknown ids.
|
||||
|
||||
- `find_matching_teams(role_name, competences, matching_method?)`
|
||||
- Use to match a task against team profiles (`Profile_Type = "team"`).
|
||||
- Same `matching_method` parameter (`score` | `llm_fulltext`) as `find_matching_capacities`.
|
||||
- **Does not accept `date_start` / `date_end`** — availability filters are not effective for team searches.
|
||||
- Persists `search_type = "team_search"` in `SearchCache` and the META JSON.
|
||||
|
||||
### Requirement capture (free text or structured)
|
||||
- `extract_requirements(task_description, confirm_requirements=true)`
|
||||
- Uses embeddings-only inference.
|
||||
- Dates are NOT extracted from free text. Ask for a time range and capture via structured/guided tools.
|
||||
|
||||
- `collect_structured_requirement_data(role_name, competences, date_start?, date_end?, confirm_requirements=true)`
|
||||
- Use when the user provides explicit fields.
|
||||
|
||||
- Guided capture tools (step-by-step):
|
||||
- `start_guided_capture()`
|
||||
- `guided_set_description(description)`
|
||||
- `guided_set_role(role_name)`
|
||||
- `guided_set_time_range(date_start?, date_end?)` (open-ended allowed)
|
||||
- `guided_set_competences(competences)`
|
||||
|
||||
- `update_requirements(change_description, confirm_requirements=true)`
|
||||
- Use to modify already-captured requirements (e.g. add a competence, change the role).
|
||||
- Accepts a free-text change description; the server applies the delta.
|
||||
|
||||
- `show_pending_requirements()`
|
||||
- Use to display pending requirements for user review.
|
||||
|
||||
- `confirm_requirements(confirm=true)`
|
||||
- Use to confirm pending requirements before searching if required.
|
||||
|
||||
### Search execution
|
||||
- `find_matching_capacities(role_name, competences, date_start?, date_end?, matching_method?)`
|
||||
- Runs matching against capacity profiles (`Profile_Type = "capacity"`).
|
||||
- Returns a JSON block with `search_id`; treat the returned `search_id` as the new **latest** search.
|
||||
- `matching_method` is `"score"` (default) or `"llm_fulltext"` (see "Matching methods").
|
||||
|
||||
- `find_matching_teams(role_name, competences, matching_method?)`
|
||||
- Runs matching against team profiles (`Profile_Type = "team"`).
|
||||
- Returns a JSON block with `search_id`; treat the returned `search_id` as the new **latest** search.
|
||||
- Persists `search_type = "team_search"`.
|
||||
- **No date parameters** — availability filters are not effective for team searches and are surfaced as `team_search` not-effective hints in the `Applied Filters` table when passed via `filter_search_results`.
|
||||
|
||||
### Search refinement and pagination (always reuse the latest search)
|
||||
- `get_results_by_category(search_id, filter_id?, category, page, page_size)`
|
||||
- Use to display `Top`, `Good`, `Partial`, `Low`, `Irrelevant` results.
|
||||
- In `score` mode: numeric score columns (Role/Competence/Overall) are visible.
|
||||
- In `llm_fulltext` mode: shows `Category` + `Begründung` columns instead of score columns.
|
||||
- For `team_search` cache entries, the table renders with team-specific columns (`Team Name`, `Schwerpunkt`, `Top-Kompetenzen`, ...). Availability is shown as overlap percentage with the reference time range only for capacity searches.
|
||||
|
||||
- `filter_search_results(search_id, ...)`
|
||||
- Use for refining results across all search directions (capacity-search, task-search, team-search).
|
||||
- Supports full-coverage availability filtering (`is_fully_available=true`) for capacity searches.
|
||||
- Task-search filters include `task_text_filter` and `task_competence_filter`.
|
||||
- In `llm_fulltext` mode `min_similarity` is **ignored** and surfaced as a hint in the Applied Filters table.
|
||||
- For `team_search`, `availability_date_start`, `availability_date_end`, and `is_fully_available` are accepted but ignored; each value is surfaced as a `team_search` not-effective hint. `role_filter` matches against `team.focus_name`; `competence_filter` matches against the team's competence list (filter values with the suffix `(Top)` are restricted to Top competences).
|
||||
|
||||
## Default workflows
|
||||
|
||||
### Workflow A: User has a DB task id
|
||||
1. `get_task_details(task_id)`
|
||||
2. (Only if the user explicitly requests validation) `validate_task_requirements(task_id)`
|
||||
3. If role unclear, `infer_primary_role(task_id=...)`
|
||||
4. Capture/update requirements (pick ONE approach):
|
||||
- `collect_structured_requirement_data(...)`, or
|
||||
- `extract_requirements(task_description=...)`
|
||||
5. **Review (must be after the last requirements update):** `show_pending_requirements()` (only if `matching.require_confirmation = true`)
|
||||
6. Ask the user explicitly to confirm (Yes/No) (only if `matching.require_confirmation = true`)
|
||||
7. `confirm_requirements(confirm=true)` (only on Yes; on No: `confirm_requirements(confirm=false)`) (only if `matching.require_confirmation = true`)
|
||||
8. Ask the user which `Profile_Type` to use (`capacity` or `team`), unless already known
|
||||
9. Ask the user which `matching_method` to use (`score` or `llm_fulltext`), unless already known
|
||||
10. Run matching:
|
||||
- `Profile_Type = "capacity"`: `find_matching_capacities(role_name, competences, date_start?, date_end?, matching_method=...)`
|
||||
- `Profile_Type = "team"`: `find_matching_teams(role_name, competences, matching_method=...)` (no date params)
|
||||
11. After matching, show other categories via `get_results_by_category` using `LATEST_SEARCH_ID`.
|
||||
|
||||
### Workflow B: User provides free-text description
|
||||
1. `extract_requirements(task_description, confirm_requirements=true)`
|
||||
2. Ask for missing fields (especially date range when `Profile_Type = "capacity"`; dates are not extracted from free text and are not needed for team searches).
|
||||
3. If `matching.require_confirmation = true`: `show_pending_requirements()`
|
||||
4. If `matching.require_confirmation = true`: ask user for confirmation
|
||||
5. If `matching.require_confirmation = true`: `confirm_requirements(confirm=true)`
|
||||
6. Ask the user which `Profile_Type` to use (`capacity` or `team`), unless already known
|
||||
7. Ask the user which `matching_method` to use (`score` or `llm_fulltext`), unless already known
|
||||
8. Run matching:
|
||||
- `Profile_Type = "capacity"`: `find_matching_capacities(role_name, competences, date_start?, date_end?, matching_method=...)`
|
||||
- `Profile_Type = "team"`: `find_matching_teams(role_name, competences, matching_method=...)`
|
||||
9. For other categories/pages: `get_results_by_category` using `LATEST_SEARCH_ID`.
|
||||
|
||||
### Workflow C: Capacity-driven matching
|
||||
1. `list_free_capacities(limit=...)`
|
||||
2. `get_capacity_details(capacity_id)`
|
||||
3. Ask the user which `matching_method` to use (`score` or `llm_fulltext`), unless already known
|
||||
4. `find_matching_tasks(capacity_id, matching_method=...)`
|
||||
|
||||
### Workflow D: Team browsing and team matching (`Profile_Type = "team"`)
|
||||
1. `list_teams(limit=...)` to browse all teams.
|
||||
2. `get_team_details(team_id)` to inspect a single team profile (returns `Team not found: <team_id>` for unknown ids).
|
||||
3. To run a task→team match for an ad-hoc requirement: capture requirements as in Workflow A or B, then set `Profile_Type = "team"`.
|
||||
4. Do **not** ask for `date_start` / `date_end` — availability filters are not effective for team searches.
|
||||
5. Confirmation flow (`show_pending_requirements`, `confirm_requirements`) is identical to Workflow A.
|
||||
6. Ask the user which `matching_method` to use (`score` or `llm_fulltext`), unless already known.
|
||||
7. `find_matching_teams(role_name, competences, matching_method=...)`.
|
||||
8. For other categories/pages: `get_results_by_category` using `LATEST_SEARCH_ID`. The table renders with team-specific columns (`Team Name`, `Schwerpunkt`, `Top-Kompetenzen`, ...) automatically because the cache entry carries `search_type = "team_search"`.
|
||||
|
||||
## Handling “show me Good results”
|
||||
- If you have `LATEST_SEARCH_ID`: call `get_results_by_category(search_id=<LATEST_SEARCH_ID>, category="Good", page=1, page_size=20)`.
|
||||
- If you do not have `LATEST_SEARCH_ID`: run a new matching call (`find_matching_capacities(...)` for `Profile_Type = "capacity"`, `find_matching_teams(...)` for `Profile_Type = "team"`) first.
|
||||
|
||||
## Embeddings: performance + determinism (important)
|
||||
|
||||
The server uses Azure OpenAI embeddings for competence and role similarity.
|
||||
|
||||
- Embeddings are **prefetched globally per matching run** (required + all
|
||||
candidates; competences + roles). Similarity computation should not trigger
|
||||
embeddings inside inner loops.
|
||||
- Cache layering applies:
|
||||
1) in-memory cache
|
||||
2) SQLite embedding cache
|
||||
3) Azure embeddings API
|
||||
- Azure embeddings requests may be **batched** (`input=[...]`) and chunked
|
||||
sequentially. Chunk size is configurable via
|
||||
`azure_openai.embedding_batch_size` (default: 128).
|
||||
- Failure semantics remain strict: if embeddings fail, the matching run fails
|
||||
(no heuristic fallback).
|
||||
|
||||
## BM25 + RRF competence matching (optional, disabled by default)
|
||||
|
||||
When `matching.similarity.use_bm25_search = true` is set in `config.toml`, competence
|
||||
similarity uses **BM25 + Reciprocal Rank Fusion (RRF)** instead of embeddings:
|
||||
|
||||
- **Score interpretation changes**: a `Competence Score` of `0.0` means
|
||||
**no lexical token overlap** between the required competence and any candidate
|
||||
competence — it is **not** a semantic distance. Cross-language synonyms or
|
||||
abbreviations that do not share tokens will score `0.0`.
|
||||
- **Exact canonical terms matter**: "Python" matches "Python" perfectly; "ML" does
|
||||
**not** match "Machine Learning" unless Auto-Tagging is enabled.
|
||||
- When displaying results to the user, note that BM25 mode eliminates false positives
|
||||
(non-overlapping skills score exactly 0.0) but may miss valid synonym matches.
|
||||
|
||||
### Auto-Tagging (`use_auto_tagging = true`)
|
||||
|
||||
When `matching.similarity.use_auto_tagging = true` is also set:
|
||||
|
||||
- An LLM pre-expands each candidate's competence list with canonical equivalents of
|
||||
required competences it already covers (synonyms, abbreviations, cross-language terms,
|
||||
e.g. "ML" → "Machine Learning").
|
||||
- The expansion is **ephemeral** (per-call, not stored in the database).
|
||||
- A positive `Competence Score` with auto-tagging enabled may reflect an
|
||||
**LLM-inferred equivalence**, not a literal token match; the rationale field will still
|
||||
say `"BM25+RRF: best match '...' (score ...)"`.
|
||||
- If the LLM call fails for any reason, BM25 scoring continues on the original
|
||||
(unexpanded) candidate list — the run does **not** fail.
|
||||
- Role similarity is always **embedding-based**, regardless of these flags.
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"mcpServers": {
|
||||
"teamlandkarte-mcp": {
|
||||
"command": "/Users/thomashandke/.local/bin/uv",
|
||||
"args": [
|
||||
"run",
|
||||
"--project",
|
||||
"/Users/thomashandke/ws/teamlandkarte-mcp",
|
||||
"--directory",
|
||||
"/Users/thomashandke/ws/teamlandkarte-mcp",
|
||||
"teamlandkarte-mcp",
|
||||
"--config",
|
||||
"/Users/thomashandke/ws/teamlandkarte-mcp/config.toml",
|
||||
"--log-level",
|
||||
"DEBUG"
|
||||
],
|
||||
"disabled": false,
|
||||
"autoApprove": [
|
||||
"list_open_tasks",
|
||||
"validate_task_requirements",
|
||||
"find_matching_capacities",
|
||||
"show_pending_requirements",
|
||||
"confirm_requirements",
|
||||
"get_task_details",
|
||||
"collect_structured_requirement_data",
|
||||
"get_capacity_details",
|
||||
"filter_search_results",
|
||||
"list_free_capacities",
|
||||
"get_results_by_category"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
{"specId": "036a2f60-80b4-4a0b-8cac-7201dd154bed", "workflowType": "requirements-first", "specType": "feature"}
|
||||
@@ -1,163 +0,0 @@
|
||||
# Design Document: Capacity Details Enrichment
|
||||
|
||||
## Overview
|
||||
|
||||
This feature enriches the existing `get_capacity_details` MCP tool to display additional information about a capacity: its description, references (partner + projects), and certifications. The data is already available in the database and exposed through existing `DBClient` methods (`get_capacity_description`, `get_capacity_references`, `get_capacity_certificates`). The change is purely in the tool's output formatting layer.
|
||||
|
||||
The tool currently returns a Markdown table with basic capacity fields (ID, Owner, Role, Competences, Availability) followed by a "Next steps" section. After this enhancement, it will include three additional sections between the table and the next steps: Beschreibung, Referenzen, and Zertifizierungen.
|
||||
|
||||
## Architecture
|
||||
|
||||
The change is localized to the `get_capacity_details` tool function inside `build_server()` in `src/teamlandkarte_mcp/mcp_server.py`. No new modules, classes, or external dependencies are needed.
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant Client as MCP Client
|
||||
participant Tool as get_capacity_details
|
||||
participant DB as DBClient
|
||||
|
||||
Client->>Tool: call(capacity_id)
|
||||
Tool->>DB: get_capacity_by_id(capacity_id)
|
||||
DB-->>Tool: Capacity | None
|
||||
Tool->>DB: get_capacity_description(capacity_id)
|
||||
DB-->>Tool: str | None
|
||||
Tool->>DB: get_capacity_references(capacity_id)
|
||||
DB-->>Tool: list[CapacityReferenceRow]
|
||||
Tool->>DB: get_capacity_certificates(capacity_id)
|
||||
DB-->>Tool: list[str]
|
||||
Tool-->>Client: Formatted Markdown string
|
||||
```
|
||||
|
||||
### Design Decisions
|
||||
|
||||
1. **Sequential DB calls** – The three additional queries are simple key lookups on indexed views. Parallelizing them would add complexity (async conversion of the tool) for negligible latency gain. Keep the tool synchronous.
|
||||
2. **Formatting inline** – The formatting logic is simple string concatenation. No need for a separate formatter class.
|
||||
3. **Empty-state handling** – Each section shows a "(keine)" placeholder when data is absent, keeping the output structure predictable for LLM consumers.
|
||||
|
||||
## Components and Interfaces
|
||||
|
||||
### Modified Component: `get_capacity_details` tool
|
||||
|
||||
**Current signature** (unchanged):
|
||||
```python
|
||||
def get_capacity_details(capacity_id: int | str) -> str:
|
||||
```
|
||||
|
||||
**New internal calls added:**
|
||||
```python
|
||||
description: str | None = db_client.get_capacity_description(capacity_id)
|
||||
references: list[CapacityReferenceRow] = db_client.get_capacity_references(capacity_id)
|
||||
certificates: list[str] = db_client.get_capacity_certificates(capacity_id)
|
||||
```
|
||||
|
||||
**Output format** (Markdown string):
|
||||
```
|
||||
| ID | Owner | Role | Competences | Availability |
|
||||
| ... |
|
||||
|
||||
## Beschreibung
|
||||
|
||||
<description text or "(keine)">
|
||||
|
||||
## Referenzen
|
||||
|
||||
- **Partner A**: Projekt X, Projekt Y
|
||||
- Projekt Z (no partner)
|
||||
|
||||
*or* Referenzen: (keine)
|
||||
|
||||
## Zertifizierungen
|
||||
|
||||
- Zertifikat 1
|
||||
- Zertifikat 2
|
||||
|
||||
*or* Zertifizierungen: (keine)
|
||||
|
||||
## Next steps
|
||||
|
||||
Call find_matching_tasks(capacity_id=...) to see matching open tasks.
|
||||
```
|
||||
|
||||
### Existing Interfaces Used (no changes)
|
||||
|
||||
| Method | Returns | Source |
|
||||
|--------|---------|--------|
|
||||
| `DBClient.get_capacity_description(capacity_id)` | `str \| None` | `teamlandkarte_v_capacities_latest.description` |
|
||||
| `DBClient.get_capacity_references(capacity_id)` | `list[CapacityReferenceRow]` | `teamlandkarte_v_capacity_references_latest` joined with partners |
|
||||
| `DBClient.get_capacity_certificates(capacity_id)` | `list[str]` | `teamlandkarte_v_capacity_certificates_latest.description` |
|
||||
|
||||
## Data Models
|
||||
|
||||
### CapacityReferenceRow (existing, unchanged)
|
||||
|
||||
```python
|
||||
class CapacityReferenceRow(TypedDict):
|
||||
partner_name: str # May be empty string when partner_id is NULL
|
||||
projects: str # Project text from the references view
|
||||
```
|
||||
|
||||
No new data models are introduced.
|
||||
|
||||
|
||||
## Correctness Properties
|
||||
|
||||
*A property is a characteristic or behavior that should hold true across all valid executions of a system — essentially, a formal statement about what the system should do. Properties serve as the bridge between human-readable specifications and machine-verifiable correctness guarantees.*
|
||||
|
||||
### Property 1: All enrichment data is fetched for any capacity
|
||||
|
||||
*For any* valid capacity ID that resolves to an existing capacity, the tool SHALL invoke `get_capacity_description`, `get_capacity_references`, and `get_capacity_certificates` with that capacity ID.
|
||||
|
||||
**Validates: Requirements 1.1, 2.1, 3.1**
|
||||
|
||||
### Property 2: Non-empty data appears in output
|
||||
|
||||
*For any* capacity with a non-empty description, a non-empty list of references, and a non-empty list of certificates, the formatted output SHALL contain the description text, every reference's projects text, and every certificate string.
|
||||
|
||||
**Validates: Requirements 1.2, 2.2, 3.2**
|
||||
|
||||
### Property 3: Section ordering is fixed
|
||||
|
||||
*For any* capacity (regardless of which fields are empty or populated), the output string SHALL contain the section markers in the order: capacity table first, then "Beschreibung", then "Referenzen", then "Zertifizierungen", then "Next steps" — and each section SHALL be separated by at least one blank line.
|
||||
|
||||
**Validates: Requirements 4.1, 4.2**
|
||||
|
||||
## Error Handling
|
||||
|
||||
| Scenario | Behaviour |
|
||||
|----------|-----------|
|
||||
| `get_capacity_by_id` returns `None` | Return `"Capacity not found: {capacity_id}"` (existing behaviour, unchanged) |
|
||||
| `get_capacity_description` returns `None` | Display `"Beschreibung: (keine)"` |
|
||||
| `get_capacity_references` returns `[]` | Display `"Referenzen: (keine)"` |
|
||||
| `get_capacity_certificates` returns `[]` | Display `"Zertifizierungen: (keine)"` |
|
||||
| `get_capacity_references` returns a row with empty `partner_name` | Display only the `projects` text (no bold partner prefix) |
|
||||
| Any DB call raises an exception | Let it propagate (existing error handling in the MCP framework catches and reports it) |
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
### Unit Tests
|
||||
|
||||
- Test `get_capacity_details` with a mock DB client returning known data for all three new fields → verify output contains expected sections and content.
|
||||
- Test empty-state: description=None, references=[], certificates=[] → verify "(keine)" placeholders appear.
|
||||
- Test edge case: reference with empty `partner_name` → verify only projects text is shown.
|
||||
- Test that capacity-not-found still returns the error message unchanged.
|
||||
|
||||
### Property-Based Tests
|
||||
|
||||
Library: **Hypothesis** (Python)
|
||||
|
||||
Each property test runs a minimum of 100 iterations.
|
||||
|
||||
- **Property 1 test**: Generate random capacity IDs and mock DB responses. Verify all three DB methods are called with the correct ID.
|
||||
- Tag: `Feature: capacity-details-enrichment, Property 1: All enrichment data is fetched for any capacity`
|
||||
|
||||
- **Property 2 test**: Generate random non-empty descriptions (text strategy), random lists of `CapacityReferenceRow` dicts with non-empty `projects` and `partner_name`, and random lists of non-empty certificate strings. Call the formatting logic and assert all generated data appears in the output.
|
||||
- Tag: `Feature: capacity-details-enrichment, Property 2: Non-empty data appears in output`
|
||||
|
||||
- **Property 3 test**: Generate random combinations of present/absent data (description: str|None, references: list of 0-5 items, certificates: list of 0-5 items). Call the formatting logic and assert section headers appear in the correct order with blank-line separation.
|
||||
- Tag: `Feature: capacity-details-enrichment, Property 3: Section ordering is fixed`
|
||||
|
||||
### Test Configuration
|
||||
|
||||
- Property-based testing library: `hypothesis` (already available in the project's test dependencies)
|
||||
- Minimum iterations: 100 per property (`@settings(max_examples=100)`)
|
||||
- Each test tagged with a comment referencing the design property
|
||||
@@ -1,57 +0,0 @@
|
||||
# Requirements Document
|
||||
|
||||
## Introduction
|
||||
|
||||
Das MCP-Tool `get_capacity_details` zeigt aktuell nur eine Tabelle mit ID, Owner, Rolle, Kompetenzen und Verfügbarkeit an. Es fehlen die Beschreibung (Description), Referenzen und Zertifizierungen einer Kapazität. Diese Informationen sind bereits in der Datenbank vorhanden und über die DB-Client-Methoden `get_capacity_description`, `get_capacity_references` und `get_capacity_certificates` abrufbar. Das Tool soll erweitert werden, um diese zusätzlichen Felder anzuzeigen.
|
||||
|
||||
## Glossary
|
||||
|
||||
- **MCP_Server**: Der Teamlandkarte MCP Server, der Tools für Kapazitäts- und Aufgabenabgleich bereitstellt
|
||||
- **get_capacity_details_Tool**: Das MCP-Tool, das Detailinformationen zu einer einzelnen Kapazität anzeigt
|
||||
- **DB_Client**: Die Datenbankzugriffsschicht, die Kapazitätsdaten aus der Trino-Datenbank liest
|
||||
- **Capacity**: Ein Eintrag, der die verfügbare Kapazität einer Person beschreibt (ID, Owner, Rolle, Kompetenzen, Verfügbarkeit)
|
||||
- **Description**: Freitext-Beschreibung einer Kapazität aus `teamlandkarte_v_capacities_latest.description`
|
||||
- **Reference**: Ein Referenzeintrag bestehend aus Partnername und Projekten aus `teamlandkarte_v_capacity_references_latest`
|
||||
- **Certificate**: Eine Zertifizierungsbeschreibung aus `teamlandkarte_v_capacity_certificates_latest`
|
||||
|
||||
## Requirements
|
||||
|
||||
### Requirement 1: Beschreibung anzeigen
|
||||
|
||||
**User Story:** Als Nutzer möchte ich die Beschreibung einer Kapazität im Tool `get_capacity_details` sehen, damit ich ein vollständigeres Bild der Kapazität erhalte.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN a capacity is retrieved by `get_capacity_details`, THE get_capacity_details_Tool SHALL fetch the description via `DB_Client.get_capacity_description`
|
||||
2. WHEN the description is non-empty, THE get_capacity_details_Tool SHALL display the description in a dedicated section labeled "Beschreibung" below the capacity table
|
||||
3. WHEN the description is empty or not available, THE get_capacity_details_Tool SHALL display "Beschreibung: (keine)" in the description section
|
||||
|
||||
### Requirement 2: Referenzen anzeigen
|
||||
|
||||
**User Story:** Als Nutzer möchte ich die Referenzen einer Kapazität im Tool `get_capacity_details` sehen, damit ich die bisherigen Projekterfahrungen der Person einschätzen kann.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN a capacity is retrieved by `get_capacity_details`, THE get_capacity_details_Tool SHALL fetch the references via `DB_Client.get_capacity_references`
|
||||
2. WHEN references exist, THE get_capacity_details_Tool SHALL display each reference as a bullet point in a section labeled "Referenzen", including partner name and projects
|
||||
3. WHEN a reference has an empty partner name, THE get_capacity_details_Tool SHALL display only the projects for that reference entry
|
||||
4. WHEN no references exist, THE get_capacity_details_Tool SHALL display "Referenzen: (keine)"
|
||||
|
||||
### Requirement 3: Zertifizierungen anzeigen
|
||||
|
||||
**User Story:** Als Nutzer möchte ich die Zertifizierungen einer Kapazität im Tool `get_capacity_details` sehen, damit ich die formalen Qualifikationen der Person erkennen kann.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN a capacity is retrieved by `get_capacity_details`, THE get_capacity_details_Tool SHALL fetch the certificates via `DB_Client.get_capacity_certificates`
|
||||
2. WHEN certificates exist, THE get_capacity_details_Tool SHALL display each certificate as a bullet point in a section labeled "Zertifizierungen"
|
||||
3. WHEN no certificates exist, THE get_capacity_details_Tool SHALL display "Zertifizierungen: (keine)"
|
||||
|
||||
### Requirement 4: Darstellungsreihenfolge
|
||||
|
||||
**User Story:** Als Nutzer möchte ich eine konsistente und übersichtliche Darstellung aller Kapazitätsdetails, damit ich die Informationen schnell erfassen kann.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE get_capacity_details_Tool SHALL display sections in the following fixed order: Capacity-Tabelle, Beschreibung, Referenzen, Zertifizierungen, Next Steps
|
||||
2. THE get_capacity_details_Tool SHALL separate each section with a blank line for Lesbarkeit
|
||||
@@ -1,86 +0,0 @@
|
||||
# Implementation Plan: Capacity Details Enrichment
|
||||
|
||||
## Overview
|
||||
|
||||
Extend the `get_capacity_details` tool in `src/teamlandkarte_mcp/mcp_server.py` to fetch and display description, references, and certificates for a capacity. The change is localized to the tool function with inline formatting. All DB client methods already exist.
|
||||
|
||||
## Tasks
|
||||
|
||||
- [x] 1. Extend `get_capacity_details` with enrichment data fetching and formatting
|
||||
- [x] 1.1 Add DB calls for description, references, and certificates
|
||||
- After the existing `get_capacity_by_id` call, add sequential calls to `db_client.get_capacity_description(capacity_id)`, `db_client.get_capacity_references(capacity_id)`, and `db_client.get_capacity_certificates(capacity_id)`
|
||||
- _Requirements: 1.1, 2.1, 3.1_
|
||||
|
||||
- [x] 1.2 Format the Beschreibung section
|
||||
- If description is non-empty, render `## Beschreibung\n\n<text>`
|
||||
- If description is None or empty, render `## Beschreibung\n\nBeschreibung: (keine)`
|
||||
- _Requirements: 1.2, 1.3_
|
||||
|
||||
- [x] 1.3 Format the Referenzen section
|
||||
- If references exist, render `## Referenzen` followed by bullet points: `- **partner_name**: projects` for each reference
|
||||
- If a reference has an empty `partner_name`, render only `- projects` (no bold partner prefix)
|
||||
- If no references exist, render `## Referenzen\n\nReferenzen: (keine)`
|
||||
- _Requirements: 2.2, 2.3, 2.4_
|
||||
|
||||
- [x] 1.4 Format the Zertifizierungen section
|
||||
- If certificates exist, render `## Zertifizierungen` followed by bullet points: `- certificate` for each entry
|
||||
- If no certificates exist, render `## Zertifizierungen\n\nZertifizierungen: (keine)`
|
||||
- _Requirements: 3.2, 3.3_
|
||||
|
||||
- [x] 1.5 Assemble output in fixed section order
|
||||
- Combine sections in order: capacity table, Beschreibung, Referenzen, Zertifizierungen, Next Steps
|
||||
- Separate each section with a blank line
|
||||
- _Requirements: 4.1, 4.2_
|
||||
|
||||
- [x] 2. Write unit tests for the enriched output
|
||||
- [x] 2.1 Test full data scenario
|
||||
- Mock DB client to return a known description, list of references (with and without partner_name), and list of certificates
|
||||
- Assert output contains all expected section headers, content, and correct ordering
|
||||
- Create test file `tests/test_capacity_details_enrichment.py`
|
||||
- _Requirements: 1.2, 2.2, 2.3, 3.2, 4.1_
|
||||
|
||||
- [x] 2.2 Test empty-state scenario
|
||||
- Mock DB client to return None description, empty references list, empty certificates list
|
||||
- Assert output contains "(keine)" placeholders for all three sections
|
||||
- _Requirements: 1.3, 2.4, 3.3_
|
||||
|
||||
- [x] 2.3 Test capacity-not-found unchanged
|
||||
- Mock `get_capacity_by_id` to return None
|
||||
- Assert the tool still returns the existing error message without calling enrichment methods
|
||||
- _Requirements: (error handling, no regression)_
|
||||
|
||||
- [x] 3. Checkpoint
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
- [x] 4. Property-based tests with Hypothesis
|
||||
- [x] 4.1 Write property test: All enrichment data is fetched
|
||||
- **Property 1: All enrichment data is fetched for any capacity**
|
||||
- Generate random capacity IDs; mock DB to return a capacity. Verify `get_capacity_description`, `get_capacity_references`, and `get_capacity_certificates` are each called exactly once with the correct ID.
|
||||
- Create test file `tests/test_capacity_details_enrichment_pbt.py`
|
||||
- **Validates: Requirements 1.1, 2.1, 3.1**
|
||||
|
||||
- [x] 4.2 Write property test: Non-empty data appears in output
|
||||
- **Property 2: Non-empty data appears in output**
|
||||
- Generate random non-empty descriptions (text strategy), random lists of `CapacityReferenceRow` dicts with non-empty `projects` and `partner_name`, and random lists of non-empty certificate strings. Assert all generated data appears in the formatted output.
|
||||
- **Validates: Requirements 1.2, 2.2, 3.2**
|
||||
|
||||
- [x] 4.3 Write property test: Section ordering is fixed
|
||||
- **Property 3: Section ordering is fixed**
|
||||
- Generate random combinations of present/absent data (description: str|None, references: 0-5 items, certificates: 0-5 items). Assert section headers appear in the correct order with blank-line separation.
|
||||
- **Validates: Requirements 4.1, 4.2**
|
||||
|
||||
- [x] 5. Final checkpoint
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
- [x] 6. Update agent skill documentation
|
||||
- [x] 6.1 Update `.github/skills/capacity-browsing/SKILL.md`
|
||||
- Change the `get_capacity_details` description to mention description, references, and certifications alongside role, competences, and availability window
|
||||
- [x] 6.2 Update `.kiro/agents/teamlandkarte.md`
|
||||
- Change the `get_capacity_details` description to mention that it shows description, references, and certifications alongside the basic profile
|
||||
|
||||
## Notes
|
||||
|
||||
- Tasks marked with `*` are optional and can be skipped for faster MVP
|
||||
- The implementation language is Python (matching the existing codebase and design)
|
||||
- All DB client methods (`get_capacity_description`, `get_capacity_references`, `get_capacity_certificates`) already exist — no data layer changes needed
|
||||
- Property tests use Hypothesis with `@settings(max_examples=100)`
|
||||
@@ -1 +0,0 @@
|
||||
{"specId": "036a2f60-80b4-4a0b-8cac-7201dd154bed", "workflowType": "requirements-first", "specType": "feature"}
|
||||
@@ -1,287 +0,0 @@
|
||||
# Design: LLM Batch-Matching (Concurrent Requests)
|
||||
|
||||
## Übersicht
|
||||
|
||||
Das bestehende `LlmFulltextMatcher`-Modul führt LLM-Aufrufe sequenziell aus – jeder Kandidat wartet auf die Antwort des vorherigen. Bei 30 Kandidaten mit je ~2s Latenz ergibt das ~60s Gesamtlaufzeit.
|
||||
|
||||
Die Lösung ersetzt die sequenzielle `for`-Schleife durch `asyncio.gather()` mit einem `asyncio.Semaphore` zur Begrenzung der Parallelität. Die öffentliche Schnittstelle (`match_capacities`, `match_tasks`) bleibt unverändert. Die Concurrency wird über `config.toml` konfigurierbar gemacht.
|
||||
|
||||
**Erwarteter Effekt:** Bei `max_concurrency=5` und 30 Kandidaten sinkt die Laufzeit von ~60s auf ~12s (6 Batches × 2s statt 30 × 2s).
|
||||
|
||||
## Architektur
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
A[MCP Server / Aufrufer] -->|match_capacities / match_tasks| B[LlmFulltextMatcher]
|
||||
B -->|asyncio.gather + Semaphore| C[_categorize_one Task 1]
|
||||
B -->|asyncio.gather + Semaphore| D[_categorize_one Task 2]
|
||||
B -->|asyncio.gather + Semaphore| E[_categorize_one Task N]
|
||||
C --> F[AzureOpenAIClient.chat_completion]
|
||||
D --> F
|
||||
E --> F
|
||||
F --> G[Azure OpenAI API]
|
||||
```
|
||||
|
||||
Die Architektur bleibt flach: Es wird kein neues Modul oder neue Klasse eingeführt. Die Änderung betrifft ausschließlich die interne Ablaufsteuerung in `LlmFulltextMatcher` und die Konfigurationsschicht.
|
||||
|
||||
### Designentscheidungen
|
||||
|
||||
1. **asyncio.Semaphore statt Thread-Pool:** Das Projekt ist bereits vollständig async (FastMCP, AsyncAzureOpenAI). Ein Semaphore ist der idiomatische Mechanismus zur Begrenzung von I/O-Concurrency in asyncio.
|
||||
|
||||
2. **Kein separates Batch-Modul:** Die Änderung ist minimal und lokal. Ein eigenes `batch_executor.py` wäre Over-Engineering für eine ~20-Zeilen-Änderung.
|
||||
|
||||
3. **Semaphore im Matcher, nicht im Client:** Der `AzureOpenAIClient` bleibt unverändert. Die Concurrency-Steuerung liegt beim Aufrufer (Matcher), da verschiedene Aufrufer unterschiedliche Limits haben könnten.
|
||||
|
||||
4. **Validierung bei Config-Load:** Ungültige `max_concurrency`-Werte (< 1 oder > 20) werden beim Start abgefangen, nicht erst beim ersten Matching-Aufruf.
|
||||
|
||||
## Komponenten und Schnittstellen
|
||||
|
||||
### 1. `AzureOpenAIConfig` (config.py)
|
||||
|
||||
Neues Feld:
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class AzureOpenAIConfig:
|
||||
# ... bestehende Felder ...
|
||||
max_concurrency: int = 5
|
||||
```
|
||||
|
||||
### 2. `_parse_azure_openai` (config.py)
|
||||
|
||||
Erweiterte Parsing-Logik:
|
||||
|
||||
```python
|
||||
def _parse_azure_openai(cfg: dict) -> AzureOpenAIConfig:
|
||||
max_concurrency = int(cfg.get("max_concurrency", 5))
|
||||
if max_concurrency < 1:
|
||||
raise ConfigError(
|
||||
"azure_openai.max_concurrency must be >= 1, "
|
||||
f"got {max_concurrency}"
|
||||
)
|
||||
if max_concurrency > 20:
|
||||
raise ConfigError(
|
||||
"azure_openai.max_concurrency must be <= 20, "
|
||||
f"got {max_concurrency}"
|
||||
)
|
||||
return AzureOpenAIConfig(
|
||||
# ... bestehende Felder ...
|
||||
max_concurrency=max_concurrency,
|
||||
)
|
||||
```
|
||||
|
||||
### 3. `LlmFulltextMatcher` (matching/llm_fulltext_matcher.py)
|
||||
|
||||
Geänderte Konstruktor-Signatur:
|
||||
|
||||
```python
|
||||
class LlmFulltextMatcher:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
db: DBClient,
|
||||
client: AzureOpenAIClient,
|
||||
rationale_max_chars: int = 280,
|
||||
max_concurrency: int = 5, # NEU
|
||||
) -> None:
|
||||
self._db = db
|
||||
self._client = client
|
||||
self._rationale_max_chars = rationale_max_chars
|
||||
self._semaphore = asyncio.Semaphore(max_concurrency)
|
||||
self._max_concurrency = max_concurrency
|
||||
```
|
||||
|
||||
Neue interne Methode:
|
||||
|
||||
```python
|
||||
async def _categorize_one_throttled(
|
||||
self,
|
||||
*,
|
||||
item_id: str,
|
||||
user_prompt: str,
|
||||
raw: dict,
|
||||
) -> tuple[LlmFulltextItem | None, LlmFulltextError | None]:
|
||||
"""Wrapper um _categorize_one mit Semaphore-Begrenzung."""
|
||||
async with self._semaphore:
|
||||
return await self._categorize_one(
|
||||
item_id=item_id,
|
||||
user_prompt=user_prompt,
|
||||
raw=raw,
|
||||
)
|
||||
```
|
||||
|
||||
Geänderte `match_capacities` / `match_tasks` (Kernänderung):
|
||||
|
||||
```python
|
||||
async def match_capacities(self, *, task_profile, capacities) -> LlmFulltextResult:
|
||||
# ... Profil-Aufbau wie bisher ...
|
||||
|
||||
LOGGER.info(
|
||||
"Batch-Matching gestartet: %d Kandidaten, max_concurrency=%d",
|
||||
len(capacities), self._max_concurrency,
|
||||
)
|
||||
start_time = time.monotonic()
|
||||
|
||||
tasks = [
|
||||
self._categorize_one_throttled(
|
||||
item_id=cap_id,
|
||||
user_prompt=user_prompt,
|
||||
raw=raw,
|
||||
)
|
||||
for cap_id, user_prompt, raw in prepared
|
||||
]
|
||||
results = await asyncio.gather(*tasks)
|
||||
|
||||
elapsed = time.monotonic() - start_time
|
||||
|
||||
# Ergebnisse zuordnen
|
||||
for item, error in results:
|
||||
if item is not None:
|
||||
by_category[item.category].append(item)
|
||||
elif error is not None:
|
||||
errors.append(error)
|
||||
|
||||
LOGGER.info(
|
||||
"Batch-Matching abgeschlossen: %.1fs, %d kategorisiert, %d Fehler",
|
||||
elapsed, sum(len(v) for v in by_category.values()), len(errors),
|
||||
)
|
||||
# ... Sortierung wie bisher ...
|
||||
```
|
||||
|
||||
### 4. MCP Server (mcp_server.py)
|
||||
|
||||
Übergabe des neuen Parameters bei Instanziierung:
|
||||
|
||||
```python
|
||||
llm_fulltext_matcher = LlmFulltextMatcher(
|
||||
db=db_client,
|
||||
client=azure_client,
|
||||
max_concurrency=cfg.azure_openai.max_concurrency, # NEU
|
||||
)
|
||||
```
|
||||
|
||||
### 5. config.toml
|
||||
|
||||
Neuer optionaler Schlüssel:
|
||||
|
||||
```toml
|
||||
[azure_openai]
|
||||
# ... bestehende Schlüssel ...
|
||||
|
||||
# Maximale Anzahl paralleler LLM-Anfragen (1-20, Standard: 5).
|
||||
# Höhere Werte beschleunigen das Matching, können aber Rate-Limits auslösen.
|
||||
# max_concurrency = 5
|
||||
```
|
||||
|
||||
## Datenmodelle
|
||||
|
||||
Keine neuen Datenmodelle erforderlich. Die bestehenden Strukturen bleiben unverändert:
|
||||
|
||||
- `LlmFulltextResult` (Rückgabetyp) – unverändert
|
||||
- `LlmFulltextItem` – unverändert
|
||||
- `LlmFulltextError` – unverändert
|
||||
- `AzureOpenAIConfig` – erweitert um `max_concurrency: int = 5`
|
||||
|
||||
Die Erweiterung von `AzureOpenAIConfig` ist abwärtskompatibel (Standardwert vorhanden).
|
||||
|
||||
## Correctness Properties
|
||||
|
||||
_Eine Property ist eine Eigenschaft oder ein Verhalten, das über alle gültigen Ausführungen eines Systems hinweg gelten muss – im Wesentlichen eine formale Aussage darüber, was das System tun soll. Properties bilden die Brücke zwischen menschenlesbaren Spezifikationen und maschinell verifizierbaren Korrektheitsgarantien._
|
||||
|
||||
### Property 1: Vollständigkeit der Ergebnisse (Partition)
|
||||
|
||||
_Für jede_ Liste von Kandidaten (Capacities oder Tasks) und jede Konfiguration von `max_concurrency`, muss die Summe aller Items in `by_category` plus die Anzahl der Einträge in `errors` exakt der Anzahl der Eingabe-Kandidaten entsprechen. Kein Kandidat darf verloren gehen oder doppelt erscheinen.
|
||||
|
||||
**Validates: Requirements 1.1, 1.4, 3.2, 4.2, 4.3**
|
||||
|
||||
### Property 2: Concurrency-Begrenzung
|
||||
|
||||
_Für jede_ Anzahl von Kandidaten und jeden gültigen `max_concurrency`-Wert, darf zu keinem Zeitpunkt die Anzahl gleichzeitig laufender LLM-Aufrufe den konfigurierten `max_concurrency`-Wert überschreiten.
|
||||
|
||||
**Validates: Requirements 1.2**
|
||||
|
||||
### Property 3: Deterministische Sortierung
|
||||
|
||||
_Für jede_ Liste von Kandidaten und jede Zuordnung von Kategorien, muss das Ergebnis innerhalb jeder Kategorie aufsteigend nach `item_id` (lexikographisch) sortiert sein, und die Fehlerliste muss ebenfalls nach `item_id` sortiert sein.
|
||||
|
||||
**Validates: Requirements 3.1**
|
||||
|
||||
### Property 4: Eingabereihenfolge-Unabhängigkeit
|
||||
|
||||
_Für jede_ Permutation der Eingabe-Kandidatenliste muss das Ergebnis (`by_category` und `errors`) identisch sein – die Reihenfolge der Eingabe hat keinen Einfluss auf die Ausgabe.
|
||||
|
||||
**Validates: Requirements 3.3**
|
||||
|
||||
### Property 5: Korrekte Zuordnung (Response-Mapping)
|
||||
|
||||
_Für jeden_ Kandidaten in der Eingabeliste muss die zugehörige LLM-Antwort exakt dem richtigen Kandidaten zugeordnet werden. Wenn der Mock für Kandidat X die Kategorie "Top" zurückgibt, muss das Item mit `item_id=X` in `by_category["Top"]` erscheinen.
|
||||
|
||||
**Validates: Requirements 3.4**
|
||||
|
||||
### Property 6: Config-Validierung
|
||||
|
||||
_Für jeden_ Integer-Wert `n`: Das Parsen von `azure_openai.max_concurrency = n` muss genau dann erfolgreich sein, wenn `1 <= n <= 20`. Für `n < 1` oder `n > 20` muss ein `ConfigError` geworfen werden.
|
||||
|
||||
**Validates: Requirements 2.1, 2.3, 2.4**
|
||||
|
||||
### Property 7: Logging-Konsistenz
|
||||
|
||||
_Für jede_ Ausführung von `match_capacities` oder `match_tasks` mit mindestens einem Kandidaten müssen die INFO-Log-Nachrichten (Start und Ende) die korrekte Kandidatenanzahl, das konfigurierte Concurrency-Limit, die Anzahl erfolgreicher Kategorisierungen und die Anzahl der Fehler enthalten. Die Summe von Erfolgen und Fehlern im Log muss der Eingabeanzahl entsprechen.
|
||||
|
||||
**Validates: Requirements 6.1, 6.2**
|
||||
|
||||
## Fehlerbehandlung
|
||||
|
||||
| Fehlerszenario | Verhalten | Auswirkung auf andere Kandidaten |
|
||||
| ------------------------------------------------ | ----------------------------------------------------------------------- | ----------------------------------------------------------------- |
|
||||
| Einzelner LLM-Aufruf schlägt fehl (nach Retries) | Kandidat wird in `errors`-Liste aufgenommen | Keine – andere Kandidaten laufen unabhängig weiter |
|
||||
| Alle LLM-Aufrufe schlagen fehl | Leere `by_category`, vollständige `errors`-Liste | Kein Exception-Wurf, normaler Return |
|
||||
| `max_concurrency` ungültig (< 1 oder > 20) | `ConfigError` beim Server-Start | Server startet nicht |
|
||||
| `asyncio.TimeoutError` in einem Aufruf | Wird vom bestehenden Retry-Mechanismus in `AzureOpenAIClient` behandelt | Keine |
|
||||
| HTTP 429 (Rate Limit) | Exponentielles Backoff im `AzureOpenAIClient` (bestehendes Verhalten) | Keine direkte; Semaphore hält Slot belegt bis Retry abgeschlossen |
|
||||
|
||||
### Fehler-Isolation
|
||||
|
||||
Die Verwendung von `asyncio.gather(*tasks)` (ohne `return_exceptions=True`) in Kombination mit der bestehenden try/except-Logik in `_categorize_one` stellt sicher, dass:
|
||||
|
||||
- Jeder Task seine eigenen Exceptions fängt und als `LlmFulltextError` zurückgibt
|
||||
- Kein einzelner Fehler die gesamte `gather`-Operation abbricht
|
||||
- Die Semaphore auch im Fehlerfall korrekt freigegeben wird (async context manager)
|
||||
|
||||
## Teststrategie
|
||||
|
||||
### Property-Based Tests (pytest + hypothesis)
|
||||
|
||||
Die Property-Tests verwenden die Bibliothek **hypothesis** (bereits im Python-Ökosystem etabliert). Jeder Test wird mit mindestens 100 Iterationen konfiguriert.
|
||||
|
||||
| Property | Testansatz | Generator |
|
||||
| ------------------------------------- | -------------------------------------------------------------------------------------------- | --------------------------------------------------------- |
|
||||
| P1: Vollständigkeit | Generiere zufällige Kandidatenlisten mit zufälligem Mix aus Erfolg/Fehler-Mocks | `st.lists(st.builds(Capacity, ...))` |
|
||||
| P2: Concurrency-Begrenzung | Mock-Client mit Counter für gleichzeitige Aufrufe; prüfe `max_concurrent <= max_concurrency` | `st.integers(min_value=1, max_value=20)` für concurrency |
|
||||
| P3: Deterministische Sortierung | Generiere Ergebnisse mit zufälligen Kategorien; prüfe Sortierung | `st.lists(st.sampled_from(categories))` |
|
||||
| P4: Eingabereihenfolge-Unabhängigkeit | Generiere Liste, permutiere, vergleiche Ergebnisse | `st.permutations(candidates)` |
|
||||
| P5: Korrekte Zuordnung | Mock gibt item_id-spezifische Kategorien zurück; prüfe Mapping | `st.dictionaries(st.text(), st.sampled_from(categories))` |
|
||||
| P6: Config-Validierung | Generiere Integers im Bereich [-100, 100]; prüfe Erfolg/Fehler | `st.integers(min_value=-100, max_value=100)` |
|
||||
| P7: Logging-Konsistenz | Capture Logs; prüfe Zahlen gegen tatsächliche Ergebnisse | `st.lists(st.builds(Capacity, ...))` |
|
||||
|
||||
Jeder Test wird mit einem Kommentar getaggt:
|
||||
|
||||
```python
|
||||
# Feature: llm-batch-matching, Property 1: Vollständigkeit der Ergebnisse
|
||||
```
|
||||
|
||||
### Unit Tests
|
||||
|
||||
Unit Tests ergänzen die Property-Tests für spezifische Szenarien:
|
||||
|
||||
- **Leere Eingabe:** `match_capacities(capacities=[])` gibt sofort leeres Ergebnis zurück
|
||||
- **Alle Fehler:** Wenn alle LLM-Aufrufe fehlschlagen, keine Exception, vollständige Fehlerliste
|
||||
- **Default-Wert:** `LlmFulltextMatcher()` ohne `max_concurrency` verwendet 5
|
||||
- **Config-Parsing:** Fehlender `max_concurrency`-Schlüssel ergibt Standardwert 5
|
||||
- **Integration:** End-to-End-Test mit gemocktem `AzureOpenAIClient` und 10 Kandidaten
|
||||
|
||||
### Testinfrastruktur
|
||||
|
||||
- **Mock-Client:** Ein `FakeAzureOpenAIClient` der konfigurierbare Antworten (Erfolg/Fehler/Delay) pro `item_id` liefert
|
||||
- **Concurrency-Tracker:** Ein Wrapper der die maximale Anzahl gleichzeitiger Aufrufe misst (via `asyncio.Lock` + Counter)
|
||||
- **Log-Capture:** pytest `caplog` Fixture für Log-Assertions
|
||||
@@ -1,101 +0,0 @@
|
||||
# Anforderungsdokument
|
||||
|
||||
## Einleitung
|
||||
|
||||
Das bestehende LLM-Volltext-Matching (`llm_fulltext`) führt für jede Kapazität bzw. Aufgabe einen separaten, sequenziellen LLM-Aufruf an die Azure OpenAI API durch. Bei einer größeren Anzahl von Kandidaten (z. B. 20–50 Kapazitäten) führt dies zu inakzeptablen Wartezeiten, da jeder Aufruf einzeln auf die API-Antwort wartet.
|
||||
|
||||
Dieses Dokument beschreibt die Anforderungen für die Einführung eines **Batching-Mechanismus**, der mehrere LLM-Aufrufe parallel (concurrent) an die Azure OpenAI API sendet, um die Gesamtlaufzeit des Matchings signifikant zu reduzieren. Es wird dabei die asynchrone Parallelisierung (concurrent requests) genutzt, nicht die Azure Batch API (die für Offline-Verarbeitung gedacht ist und keine Echtzeit-Antworten liefert).
|
||||
|
||||
## Glossar
|
||||
|
||||
- **LLM_Fulltext_Matcher**: Bestehendes Modul (`matching/llm_fulltext_matcher.py`), das den LLM-basierten Volltext-Vergleich durchführt und Kategorien direkt zuweist.
|
||||
- **AzureOpenAIClient**: Wrapper für Azure-OpenAI-Chat-Completions in `azure/openai_client.py`.
|
||||
- **Batch**: Eine Gruppe von LLM-Anfragen, die gleichzeitig (concurrent) an die Azure OpenAI API gesendet werden.
|
||||
- **Batch_Size**: Maximale Anzahl gleichzeitig laufender LLM-Anfragen innerhalb eines Batches.
|
||||
- **Concurrency_Limit**: Obergrenze für die Anzahl paralleler HTTP-Anfragen an die Azure OpenAI API, um Rate-Limits nicht zu überschreiten.
|
||||
- **Rate_Limit**: Von Azure OpenAI auferlegte Begrenzung der Anfragen pro Zeiteinheit (Requests per Minute / Tokens per Minute).
|
||||
- **Semaphore**: Synchronisationsmechanismus zur Begrenzung der gleichzeitigen Zugriffe auf eine Ressource.
|
||||
- **MCP_Server**: Der Teamlandkarte MCP-Server (`mcp_server.py`).
|
||||
- **Capacity**: Frozen Dataclass `Capacity` in `models.py`.
|
||||
- **Task**: Frozen Dataclass `Task` in `models.py`.
|
||||
- **Capacity_Profile**: Aggregiertes Volltext-Profil einer Kapazität.
|
||||
- **Task_Profile**: Aggregiertes Volltext-Profil einer Aufgabe.
|
||||
|
||||
## Anforderungen
|
||||
|
||||
### Anforderung 1: Parallele LLM-Aufrufe mit konfigurierbarer Concurrency
|
||||
|
||||
**User Story:** Als Nutzer möchte ich, dass das LLM-Volltext-Matching mehrere Kapazitäten bzw. Aufgaben gleichzeitig bewertet, damit die Gesamtwartezeit bei vielen Kandidaten deutlich sinkt.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. WHEN `match_capacities` oder `match_tasks` mit einer Liste von Kandidaten aufgerufen wird, THE LLM_Fulltext_Matcher SHALL alle LLM-Aufrufe für die Kandidaten concurrent (nicht sequenziell) ausführen, begrenzt durch das konfigurierte Concurrency_Limit.
|
||||
2. THE LLM_Fulltext_Matcher SHALL ein Concurrency_Limit verwenden, das die maximale Anzahl gleichzeitig laufender LLM-Anfragen auf einen konfigurierbaren Wert begrenzt.
|
||||
3. THE LLM_Fulltext_Matcher SHALL als Standard-Concurrency_Limit den Wert 5 verwenden, wenn kein anderer Wert konfiguriert ist.
|
||||
4. WHEN das Concurrency_Limit erreicht ist, THE LLM_Fulltext_Matcher SHALL weitere LLM-Anfragen zurückhalten, bis ein laufender Aufruf abgeschlossen ist, ohne Anfragen zu verwerfen.
|
||||
5. THE LLM_Fulltext_Matcher SHALL die Concurrency-Begrenzung über einen asyncio-Semaphore implementieren, sodass die Event-Loop nicht blockiert wird.
|
||||
|
||||
### Anforderung 2: Konfiguration des Concurrency-Limits
|
||||
|
||||
**User Story:** Als Entwickler möchte ich das Concurrency-Limit über die Konfigurationsdatei anpassen können, damit ich es an die Rate-Limits meines Azure-OpenAI-Deployments anpassen kann.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. THE MCP_Server SHALL in `config.toml` unter `[azure_openai]` einen optionalen Schlüssel `max_concurrency` akzeptieren, der das Concurrency_Limit für parallele LLM-Aufrufe festlegt.
|
||||
2. WHEN `azure_openai.max_concurrency` nicht in `config.toml` gesetzt ist, THE MCP_Server SHALL den Standardwert 5 verwenden.
|
||||
3. IF `azure_openai.max_concurrency` auf einen Wert kleiner als 1 gesetzt ist, THEN THE MCP_Server SHALL beim Start einen `ConfigError` mit beschreibender Meldung werfen.
|
||||
4. IF `azure_openai.max_concurrency` auf einen Wert größer als 20 gesetzt ist, THEN THE MCP_Server SHALL beim Start einen `ConfigError` mit beschreibender Meldung werfen, da ein zu hoher Wert Rate-Limit-Fehler provoziert.
|
||||
5. THE AzureOpenAIConfig SHALL ein Feld `max_concurrency` vom Typ `int` mit Standardwert 5 enthalten.
|
||||
|
||||
### Anforderung 3: Ergebniskonsistenz bei paralleler Verarbeitung
|
||||
|
||||
**User Story:** Als Nutzer möchte ich, dass die Ergebnisse des parallelen Matchings identisch zu denen des sequenziellen Matchings sind, damit die Umstellung auf Batching keine funktionalen Unterschiede verursacht.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. THE LLM_Fulltext_Matcher SHALL nach Abschluss aller parallelen LLM-Aufrufe die Ergebnisse in derselben deterministischen Sortierreihenfolge liefern wie bisher (primär nach Kategorie, sekundär nach `item_id` aufsteigend).
|
||||
2. THE LLM_Fulltext_Matcher SHALL bei paralleler Verarbeitung dieselbe Fehlerbehandlung anwenden wie bei sequenzieller Verarbeitung: fehlgeschlagene Aufrufe erscheinen in der Fehlerliste, erfolgreiche in `by_category`.
|
||||
3. THE LLM_Fulltext_Matcher SHALL sicherstellen, dass die Reihenfolge der Eingabe-Kandidaten keinen Einfluss auf die Sortierung der Ausgabe hat.
|
||||
4. THE LLM_Fulltext_Matcher SHALL bei paralleler Verarbeitung keine Race-Conditions bei der Zuordnung von LLM-Antworten zu Kandidaten aufweisen; jede Antwort wird exakt dem zugehörigen Kandidaten zugeordnet.
|
||||
|
||||
### Anforderung 4: Fehlerbehandlung bei Rate-Limit-Überschreitung
|
||||
|
||||
**User Story:** Als Entwickler möchte ich, dass das System bei Rate-Limit-Fehlern der Azure OpenAI API robust reagiert, damit einzelne 429-Fehler nicht den gesamten Matching-Lauf abbrechen.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. WHEN ein LLM-Aufruf innerhalb eines Batches mit einem HTTP-429-Fehler (Rate Limit Exceeded) fehlschlägt, THE AzureOpenAIClient SHALL den Aufruf nach einer exponentiellen Backoff-Pause erneut versuchen (bestehendes Retry-Verhalten).
|
||||
2. WHEN ein LLM-Aufruf nach Ausschöpfung aller Retries endgültig fehlschlägt, THE LLM_Fulltext_Matcher SHALL diesen Kandidaten in der Fehlerliste ausweisen, ohne die parallele Verarbeitung der übrigen Kandidaten zu beeinflussen.
|
||||
3. THE LLM_Fulltext_Matcher SHALL sicherstellen, dass ein Fehler bei einem einzelnen Kandidaten nicht zum Abbruch oder zur Verzögerung der Verarbeitung anderer Kandidaten führt.
|
||||
4. IF alle LLM-Aufrufe eines Batches fehlschlagen, THEN THE LLM_Fulltext_Matcher SHALL ein Ergebnis mit leeren Kategorien und einer vollständigen Fehlerliste zurückgeben, ohne eine Exception zu werfen.
|
||||
|
||||
### Anforderung 5: Beibehaltung der bestehenden Schnittstelle
|
||||
|
||||
**User Story:** Als Entwickler möchte ich, dass die öffentliche Schnittstelle des LLM_Fulltext_Matcher unverändert bleibt, damit bestehende Aufrufer (MCP_Server, Tests) ohne Anpassung weiterhin funktionieren.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. THE LLM_Fulltext_Matcher SHALL die Signaturen von `match_capacities` und `match_tasks` unverändert beibehalten (gleiche Parameter, gleicher Rückgabetyp `LlmFulltextResult`).
|
||||
2. THE LLM_Fulltext_Matcher SHALL den Rückgabetyp `LlmFulltextResult` (mit `by_category` und `errors`) unverändert beibehalten.
|
||||
3. WHEN der LLM_Fulltext_Matcher mit einer leeren Kandidatenliste aufgerufen wird, THE LLM_Fulltext_Matcher SHALL sofort ein leeres Ergebnis zurückgeben, ohne LLM-Aufrufe zu starten.
|
||||
4. THE LLM_Fulltext_Matcher SHALL das neue `max_concurrency`-Feld als optionalen Konstruktor-Parameter akzeptieren, mit Standardwert 5.
|
||||
|
||||
### Anforderung 6: Logging und Beobachtbarkeit
|
||||
|
||||
**User Story:** Als Entwickler möchte ich nachvollziehen können, wie viele LLM-Aufrufe parallel laufen und wie lange das Batching insgesamt dauert, damit ich Performance-Probleme diagnostizieren kann.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. WHEN ein Batch-Matching gestartet wird, THE LLM_Fulltext_Matcher SHALL eine Log-Nachricht auf Level INFO ausgeben, die die Anzahl der Kandidaten und das konfigurierte Concurrency_Limit enthält.
|
||||
2. WHEN ein Batch-Matching abgeschlossen ist, THE LLM_Fulltext_Matcher SHALL eine Log-Nachricht auf Level INFO ausgeben, die die Gesamtdauer, die Anzahl erfolgreicher Kategorisierungen und die Anzahl der Fehler enthält.
|
||||
3. WHEN ein einzelner LLM-Aufruf innerhalb des Batches fehlschlägt, THE LLM_Fulltext_Matcher SHALL eine Log-Nachricht auf Level WARNING ausgeben, die die `item_id` und den Fehlergrund enthält.
|
||||
|
||||
### Anforderung 7: Abwärtskompatibilität mit Score-Modus
|
||||
|
||||
**User Story:** Als Nutzer möchte ich, dass das Batching ausschließlich den LLM-Volltext-Modus betrifft und der Score-Modus unverändert bleibt.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. WHEN `matching_method = "score"` verwendet wird, THE MCP_Server SHALL das bestehende Score-basierte Matching ohne Änderungen ausführen.
|
||||
2. THE LLM_Fulltext_Matcher SHALL ausschließlich für den Modus `llm_fulltext` verwendet werden; der Score-Modus nutzt weiterhin den bestehenden `Matcher` und `SimilarityEngine`.
|
||||
3. THE MCP_Server SHALL keine neuen Abhängigkeiten oder Konfigurationsparameter einführen, die den Score-Modus beeinflussen.
|
||||
@@ -1,100 +0,0 @@
|
||||
# Implementierungsplan: LLM Batch-Matching (Concurrent Requests)
|
||||
|
||||
## Übersicht
|
||||
|
||||
Sequenzielle LLM-Aufrufe in `LlmFulltextMatcher` werden durch `asyncio.gather()` mit Semaphore-Begrenzung ersetzt. Die Konfiguration wird um `max_concurrency` erweitert. Die öffentliche Schnittstelle bleibt unverändert.
|
||||
|
||||
## Tasks
|
||||
|
||||
- [x] 1. Konfiguration erweitern
|
||||
- [x] 1.1 `AzureOpenAIConfig` um Feld `max_concurrency: int = 5` erweitern
|
||||
- In `src/teamlandkarte_mcp/config.py` das frozen Dataclass `AzureOpenAIConfig` um das Feld ergänzen
|
||||
- _Anforderungen: 2.5_
|
||||
- [x] 1.2 Validierung in `_parse_azure_openai` hinzufügen
|
||||
- `max_concurrency` aus Config lesen mit Default 5
|
||||
- `ConfigError` werfen wenn Wert < 1 oder > 20
|
||||
- Feld an `AzureOpenAIConfig`-Konstruktor übergeben
|
||||
- Auch im `load_config`-Rebuild-Block (`azure_openai = AzureOpenAIConfig(...)`) das neue Feld durchreichen
|
||||
- _Anforderungen: 2.1, 2.2, 2.3, 2.4_
|
||||
- [x] 1.3 `config.toml` um kommentierten `max_concurrency`-Schlüssel erweitern
|
||||
- Unter `[azure_openai]` einen Kommentar mit Erklärung und auskommentierten Default-Wert einfügen
|
||||
- _Anforderungen: 2.1_
|
||||
- [x] 1.4 Property-Test für Config-Validierung schreiben
|
||||
- **Property 6: Config-Validierung**
|
||||
- Teste mit hypothesis: `st.integers(min_value=-100, max_value=100)` – Erfolg genau dann wenn 1 <= n <= 20, sonst `ConfigError`
|
||||
- **Validiert: Anforderungen 2.1, 2.3, 2.4**
|
||||
|
||||
- [x] 2. `LlmFulltextMatcher` um Concurrency erweitern
|
||||
- [x] 2.1 Konstruktor um `max_concurrency`-Parameter und Semaphore erweitern
|
||||
- `max_concurrency: int = 5` als keyword-only Parameter hinzufügen
|
||||
- `self._semaphore = asyncio.Semaphore(max_concurrency)` im `__init__` anlegen
|
||||
- `self._max_concurrency = max_concurrency` speichern
|
||||
- `import asyncio` und `import time` ergänzen
|
||||
- _Anforderungen: 1.2, 1.5, 5.4_
|
||||
- [x] 2.2 Neue Methode `_categorize_one_throttled` implementieren
|
||||
- Async-Wrapper um `_categorize_one` der `async with self._semaphore:` verwendet
|
||||
- Gleiche Signatur wie `_categorize_one` (item_id, user_prompt, raw)
|
||||
- _Anforderungen: 1.2, 1.4, 1.5_
|
||||
- [x] 2.3 `match_capacities` auf `asyncio.gather` umstellen
|
||||
- Sequenzielle for-Schleife durch Liste von `_categorize_one_throttled`-Aufrufen ersetzen
|
||||
- `asyncio.gather(*tasks)` für parallele Ausführung verwenden
|
||||
- Ergebnisse iterieren und in `by_category` / `errors` einsortieren
|
||||
- Logging (INFO) vor und nach dem Batch mit Kandidatenanzahl, Concurrency, Dauer, Erfolge, Fehler
|
||||
- Bestehende Sortierung beibehalten
|
||||
- _Anforderungen: 1.1, 3.1, 3.2, 4.2, 4.3, 4.4, 5.1, 6.1, 6.2_
|
||||
- [x] 2.4 `match_tasks` auf `asyncio.gather` umstellen
|
||||
- Analog zu 2.3: sequenzielle Schleife durch gather + throttled ersetzen
|
||||
- Logging analog zu `match_capacities`
|
||||
- _Anforderungen: 1.1, 3.1, 3.2, 4.2, 4.3, 4.4, 5.1, 6.1, 6.2_
|
||||
|
||||
- [x] 3. MCP-Server Verdrahtung
|
||||
- [x] 3.1 `max_concurrency` an `LlmFulltextMatcher` übergeben
|
||||
- In `build_server()` bei der Instanziierung von `LlmFulltextMatcher` den Wert `cfg.azure_openai.max_concurrency` übergeben
|
||||
- _Anforderungen: 2.1, 5.4_
|
||||
|
||||
- [x] 4. Checkpoint
|
||||
- Sicherstellen dass alle bestehenden Tests weiterhin bestehen. Bei Fragen den Nutzer konsultieren.
|
||||
|
||||
- [x] 5. Property-Based Tests
|
||||
- [x] 5.1 Property-Test: Vollständigkeit der Ergebnisse (Partition)
|
||||
- **Property 1: Vollständigkeit der Ergebnisse**
|
||||
- Generiere zufällige Kandidatenlisten mit Mock-Client (Mix aus Erfolg/Fehler); prüfe `len(by_category items) + len(errors) == len(input)`
|
||||
- **Validiert: Anforderungen 1.1, 1.4, 3.2, 4.2, 4.3**
|
||||
- [x] 5.2 Property-Test: Concurrency-Begrenzung
|
||||
- **Property 2: Concurrency-Begrenzung**
|
||||
- Mock-Client mit asyncio-Counter für gleichzeitige Aufrufe; prüfe `max_concurrent <= max_concurrency` für verschiedene Werte
|
||||
- **Validiert: Anforderungen 1.2**
|
||||
- [x] 5.3 Property-Test: Deterministische Sortierung
|
||||
- **Property 3: Deterministische Sortierung**
|
||||
- Generiere Ergebnisse mit zufälligen Kategorien; prüfe dass jede Kategorie nach `item_id` aufsteigend sortiert ist
|
||||
- **Validiert: Anforderungen 3.1**
|
||||
- [x] 5.4 Property-Test: Eingabereihenfolge-Unabhängigkeit
|
||||
- **Property 4: Eingabereihenfolge-Unabhängigkeit**
|
||||
- Generiere Kandidatenliste, permutiere, führe Matching aus, vergleiche Ergebnisse auf Gleichheit
|
||||
- **Validiert: Anforderungen 3.3**
|
||||
- [x] 5.5 Property-Test: Korrekte Zuordnung (Response-Mapping)
|
||||
- **Property 5: Korrekte Zuordnung**
|
||||
- Mock gibt item_id-spezifische Kategorien zurück; prüfe dass jedes Item in der korrekten Kategorie landet
|
||||
- **Validiert: Anforderungen 3.4**
|
||||
- [x] 5.6 Property-Test: Logging-Konsistenz
|
||||
- **Property 7: Logging-Konsistenz**
|
||||
- Capture Logs mit `caplog`; prüfe dass Start- und End-Log die korrekte Kandidatenanzahl und Summe (Erfolge + Fehler) enthalten
|
||||
- **Validiert: Anforderungen 6.1, 6.2**
|
||||
|
||||
- [x] 6. Unit Tests
|
||||
- [x] 6.1 Unit Tests für Batch-Matching schreiben
|
||||
- Leere Eingabe: sofort leeres Ergebnis ohne LLM-Aufrufe
|
||||
- Alle Fehler: keine Exception, vollständige Fehlerliste
|
||||
- Default-Wert: ohne `max_concurrency` wird 5 verwendet
|
||||
- Einzelner Fehler beeinflusst andere Kandidaten nicht
|
||||
- _Anforderungen: 4.2, 4.3, 4.4, 5.3_
|
||||
|
||||
- [x] 7. Abschluss-Checkpoint
|
||||
- Sicherstellen dass alle Tests bestehen und die Schnittstelle abwärtskompatibel bleibt. Bei Fragen den Nutzer konsultieren.
|
||||
|
||||
## Hinweise
|
||||
|
||||
- Tasks mit `*` sind optional und können für ein schnelleres MVP übersprungen werden
|
||||
- Jeder Task referenziert spezifische Anforderungen für Nachvollziehbarkeit
|
||||
- Property-Tests verwenden `hypothesis` (pytest-Plugin)
|
||||
- Der Score-Modus bleibt vollständig unberührt (Anforderung 7)
|
||||
@@ -1 +0,0 @@
|
||||
{"specId": "218474d5-eca5-49e7-80ca-f7373f4397fb", "workflowType": "requirements-first", "specType": "bugfix"}
|
||||
@@ -1,31 +0,0 @@
|
||||
# Bugfix Requirements Document
|
||||
|
||||
## Einleitung
|
||||
|
||||
Die Kompetenz-Inferenz im Tool `validate_task_requirements` (sowie in `extract_requirements` und `find_matching_tasks`) funktioniert nicht mehr. Der frühere Embedding-basierte Ansatz (`infer_competences`, `ensure_task_embedding`) wurde im Rahmen der Umstellung auf BM25+LLM entfernt, ohne dass ein Ersatz implementiert wurde. Die Variable `inferred_comps` ist daher immer eine leere Liste `[]`.
|
||||
|
||||
Es soll ein LLM-basierter Ansatz implementiert werden, der aus allen verfügbaren Kompetenzen (via `get_all_competence_names()`) bis zu 10 passende Kompetenzen für einen gegebenen Task-Text auswählt – analog zum bereits funktionierenden LLM-basierten `infer_primary_role`.
|
||||
|
||||
## Bug-Analyse
|
||||
|
||||
### Aktuelles Verhalten (Defekt)
|
||||
|
||||
1.1 WHEN `validate_task_requirements(task_id)` aufgerufen wird THEN liefert das System immer eine leere Kompetenz-Tabelle, da `inferred_comps` stets `[]` ist
|
||||
1.2 WHEN `extract_requirements(task_description)` aufgerufen wird THEN enthält das Ergebnis keine inferierten Kompetenzen (`inferred_competences` ist immer `[]`)
|
||||
1.3 WHEN `find_matching_tasks(capacity_id)` intern Kompetenzen inferieren soll THEN werden stattdessen nur die DB-Skills verwendet, da die Kompetenz-Inferenz deaktiviert ist
|
||||
|
||||
### Erwartetes Verhalten (Korrekt)
|
||||
|
||||
2.1 WHEN `validate_task_requirements(task_id)` aufgerufen wird THEN SHALL das System per LLM-Aufruf bis zu 10 passende Kompetenzen aus der vollständigen Kompetenzliste (`get_all_competence_names()`) inferieren und mit Konfidenzwerten in der Tabelle anzeigen
|
||||
2.2 WHEN `extract_requirements(task_description)` aufgerufen wird THEN SHALL das System per LLM-Aufruf bis zu 10 passende Kompetenzen inferieren und diese in den `requirements.competences` aufnehmen
|
||||
2.3 WHEN `find_matching_tasks(capacity_id)` intern Kompetenzen inferiert THEN SHALL das System per LLM-Aufruf bis zu 10 passende Kompetenzen inferieren und diese für das Scoring verwenden
|
||||
2.4 WHEN der Task-Text leer ist oder keine Kompetenzen in der DB vorhanden sind THEN SHALL das System eine leere Kompetenzliste zurückgeben, ohne einen Fehler zu werfen
|
||||
2.5 WHEN der LLM-Aufruf fehlschlägt (Timeout, API-Fehler) THEN SHALL das System eine leere Kompetenzliste zurückgeben und den Fehler loggen, ohne den gesamten Tool-Aufruf abzubrechen
|
||||
|
||||
### Unverändertes Verhalten (Regressionsprävention)
|
||||
|
||||
3.1 WHEN `validate_task_requirements(task_id)` aufgerufen wird THEN SHALL das System WEITERHIN die Rollen-Inferenz per LLM korrekt durchführen
|
||||
3.2 WHEN `validate_task_requirements(task_id)` aufgerufen wird THEN SHALL das System WEITERHIN die DB-Skills des Tasks korrekt anzeigen
|
||||
3.3 WHEN `find_matching_capacities(task_id)` aufgerufen wird THEN SHALL das System WEITERHIN das bestehende LLM-Fulltext-Matching unverändert verwenden
|
||||
3.4 WHEN der LLM-Aufruf für Rollen-Inferenz fehlschlägt THEN SHALL das System WEITERHIN `None` zurückgeben ohne Absturz
|
||||
3.5 WHEN `infer_primary_role` aufgerufen wird THEN SHALL das System WEITERHIN genau eine Rolle aus der Rollenliste auswählen
|
||||
@@ -1,220 +0,0 @@
|
||||
# LLM-Kompetenz-Inferenz Bugfix Design
|
||||
|
||||
## Übersicht
|
||||
|
||||
Die Kompetenz-Inferenz in `validate_task_requirements`, `extract_requirements` und `find_matching_tasks` (Score-Modus) ist defekt, weil der alte Embedding-Ansatz entfernt wurde, ohne einen Ersatz zu implementieren. Die Variable `inferred_comps` ist stets eine leere Liste `[]`.
|
||||
|
||||
Der Fix implementiert eine neue Methode `infer_competences` in der bestehenden `VocabularyCache`-Klasse, analog zum bereits funktionierenden `infer_primary_role`. Diese Methode nutzt den `AzureOpenAIClient.chat_completion`-Aufruf, um aus der vollständigen Kompetenzliste (`get_all_competence_names()`) bis zu 10 passende Kompetenzen mit Konfidenzwerten auszuwählen.
|
||||
|
||||
## Glossar
|
||||
|
||||
- **Bug_Condition (C)**: Der Zustand, in dem ein Task-Text vorhanden ist UND Kompetenzen in der DB existieren, aber `inferred_comps` trotzdem `[]` zurückgibt
|
||||
- **Property (P)**: Das gewünschte Verhalten – eine nicht-leere Liste von bis zu 10 `(competence_name, confidence)`-Tupeln, wobei jeder Name in `get_all_competence_names()` enthalten ist
|
||||
- **Preservation**: Die bestehende Rollen-Inferenz (`infer_primary_role`), DB-Skills-Anzeige und LLM-Fulltext-Matching bleiben unverändert
|
||||
- **VocabularyCache**: Klasse in `src/teamlandkarte_mcp/matching/vocabulary.py`, die LLM-basierte Inferenz kapselt
|
||||
- **AzureOpenAIClient**: Client in `src/teamlandkarte_mcp/azure/openai_client.py` mit `chat_completion(system, user) -> str`
|
||||
- **inferred_comps**: Die lokale Variable in den betroffenen Tools, die aktuell immer `[]` ist
|
||||
|
||||
## Bug-Details
|
||||
|
||||
### Fault Condition
|
||||
|
||||
Der Bug manifestiert sich, wenn ein Tool (`validate_task_requirements`, `extract_requirements`, `find_matching_tasks` im Score-Modus) einen nicht-leeren Task-Text verarbeitet und Kompetenzen in der DB vorhanden sind. Die Kompetenz-Inferenz liefert stets eine leere Liste, weil kein LLM-Aufruf stattfindet.
|
||||
|
||||
**Formale Spezifikation:**
|
||||
```
|
||||
FUNCTION isBugCondition(input)
|
||||
INPUT: input of type {task_text: str, competence_names: list[str]}
|
||||
OUTPUT: boolean
|
||||
|
||||
RETURN input.task_text.strip() != ""
|
||||
AND len(input.competence_names) > 0
|
||||
AND infer_competences(input.task_text) == []
|
||||
END FUNCTION
|
||||
```
|
||||
|
||||
### Beispiele
|
||||
|
||||
- `validate_task_requirements("task-123")` mit Task-Titel "Python Backend Entwicklung" und 50 Kompetenzen in der DB → Erwartung: bis zu 10 Kompetenzen mit Konfidenz; Aktuell: leere Tabelle
|
||||
- `extract_requirements("Wir brauchen einen React-Entwickler mit TypeScript-Erfahrung")` mit Kompetenzen ["React", "TypeScript", "Angular", ...] in der DB → Erwartung: ["React", "TypeScript", ...] mit Konfidenz; Aktuell: `inferred_competences = []`
|
||||
- `find_matching_tasks(capacity_id=1)` im Score-Modus → Erwartung: `inferred_comp_names` enthält LLM-inferierte Kompetenzen für das Scoring; Aktuell: nur DB-Skills werden verwendet
|
||||
- Leerer Task-Text → Erwartung: leere Liste (kein Fehler) – dieses Verhalten ist korrekt und bleibt erhalten
|
||||
|
||||
## Erwartetes Verhalten
|
||||
|
||||
### Preservation Requirements
|
||||
|
||||
**Unverändertes Verhalten:**
|
||||
- `infer_primary_role` muss weiterhin genau eine Rolle mit Konfidenz zurückgeben
|
||||
- DB-Skills eines Tasks müssen weiterhin korrekt in der Ausgabe angezeigt werden
|
||||
- LLM-Fulltext-Matching (`find_matching_capacities` / `find_matching_tasks` im `llm_fulltext`-Modus) bleibt unverändert
|
||||
- Fehlerbehandlung bei LLM-Aufruf-Fehlern für Rollen-Inferenz bleibt unverändert (`None` zurückgeben)
|
||||
- Die `AzureOpenAIClient`-Schnittstelle wird nicht verändert
|
||||
|
||||
**Scope:**
|
||||
Alle Eingaben, die KEINEN nicht-leeren Task-Text mit vorhandenen Kompetenzen in der DB kombinieren, sind vom Fix nicht betroffen:
|
||||
- Leerer Task-Text → weiterhin leere Liste
|
||||
- Keine Kompetenzen in der DB → weiterhin leere Liste
|
||||
- Mausklick-/UI-Interaktionen → nicht betroffen (MCP-Server)
|
||||
- LLM-Fulltext-Modus → verwendet eigene Logik, nicht betroffen
|
||||
|
||||
## Hypothesierte Ursache
|
||||
|
||||
Basierend auf der Bug-Analyse sind die Ursachen klar identifiziert:
|
||||
|
||||
1. **Fehlende Implementierung**: Die alte `infer_competences`-Methode (Embedding-basiert) wurde entfernt. An den drei Stellen im Code steht nur noch `inferred_comps: list[tuple[str, float]] = []` bzw. `inferred_competences: list[tuple[str, float]] = []` ohne jeglichen LLM-Aufruf.
|
||||
|
||||
2. **Kein System-Prompt für Kompetenz-Inferenz**: Im Gegensatz zu `_ROLE_INFERENCE_SYSTEM_PROMPT` existiert kein entsprechender Prompt für Kompetenz-Inferenz.
|
||||
|
||||
3. **Keine Methode in VocabularyCache**: Die Klasse hat nur `infer_primary_role`, aber keine `infer_competences`-Methode.
|
||||
|
||||
4. **Keine Integration in die Tools**: Selbst wenn eine Methode existieren würde, fehlt der `await`-Aufruf an den drei betroffenen Stellen.
|
||||
|
||||
## Correctness Properties
|
||||
|
||||
Property 1: Fault Condition - Kompetenz-Inferenz liefert Ergebnisse
|
||||
|
||||
_For any_ input where der Task-Text nicht leer ist UND mindestens eine Kompetenz in der DB existiert (isBugCondition returns true), SHALL die fixierte `infer_competences`-Methode eine nicht-leere Liste von bis zu 10 Tupeln `(competence_name, confidence)` zurückgeben, wobei jeder `competence_name` in `get_all_competence_names()` enthalten ist und `confidence` im Bereich [0.0, 1.0] liegt.
|
||||
|
||||
**Validates: Requirements 2.1, 2.2, 2.3**
|
||||
|
||||
Property 2: Preservation - Rollen-Inferenz und DB-Skills unverändert
|
||||
|
||||
_For any_ input (unabhängig davon ob die Bug-Condition gilt oder nicht), SHALL die fixierte Codebasis das gleiche Ergebnis für `infer_primary_role` und die DB-Skills-Anzeige produzieren wie der originale Code, und das LLM-Fulltext-Matching bleibt unverändert.
|
||||
|
||||
**Validates: Requirements 3.1, 3.2, 3.3, 3.4, 3.5**
|
||||
|
||||
## Fix-Implementierung
|
||||
|
||||
### Erforderliche Änderungen
|
||||
|
||||
**Datei**: `src/teamlandkarte_mcp/matching/vocabulary.py`
|
||||
|
||||
**Änderung 1: System-Prompt hinzufügen**
|
||||
- Neuer Modul-Level-Konstante `_COMPETENCE_INFERENCE_SYSTEM_PROMPT` analog zu `_ROLE_INFERENCE_SYSTEM_PROMPT`
|
||||
- Prompt instruiert das LLM, aus einer gegebenen Kompetenzliste bis zu 10 passende Kompetenzen für einen Task-Text auszuwählen
|
||||
- Antwortformat: `{"competences": [{"name": "<name>", "confidence": <float>}]}`
|
||||
|
||||
**Änderung 2: Neue Methode `infer_competences` in `VocabularyCache`**
|
||||
- Signatur: `async def infer_competences(self, *, task_text: str, max_competences: int = 10) -> list[tuple[str, float]]`
|
||||
- Holt Kompetenzliste via `self._db.get_all_competence_names()`
|
||||
- Bei leerer Liste oder leerem Text: `[]` zurückgeben
|
||||
- LLM-Aufruf via `self._client.chat_completion(system, user)`
|
||||
- JSON-Parsing der Antwort, Validierung gegen DB-Kompetenzliste
|
||||
- Bei Fehler: leere Liste zurückgeben + Warning loggen
|
||||
|
||||
---
|
||||
|
||||
**Datei**: `src/teamlandkarte_mcp/mcp_server.py`
|
||||
|
||||
**Änderung 3: `validate_task_requirements` – LLM-Aufruf integrieren**
|
||||
- Ersetze `inferred_comps: list[tuple[str, float]] = []` durch:
|
||||
```python
|
||||
inferred_comps = await vocab_cache.infer_competences(task_text=task_text)
|
||||
```
|
||||
|
||||
**Änderung 4: `extract_requirements` – LLM-Aufruf integrieren**
|
||||
- Ersetze `inferred_competences: list[tuple[str, float]] = []` durch:
|
||||
```python
|
||||
inferred_competences = await vocab_cache.infer_competences(task_text=desc)
|
||||
```
|
||||
|
||||
**Änderung 5: `find_matching_tasks` (Score-Modus) – LLM-Aufruf integrieren**
|
||||
- Ersetze den Block `inferred_comp_names = [str(x) for x in (getattr(t, "skills", None) or []) if x]` durch:
|
||||
```python
|
||||
inferred_comp_tuples = await vocab_cache.infer_competences(task_text=full_text)
|
||||
inferred_comp_names = [name for name, _conf in inferred_comp_tuples]
|
||||
if not inferred_comp_names:
|
||||
inferred_comp_names = [str(x) for x in (getattr(t, "skills", None) or []) if x]
|
||||
```
|
||||
|
||||
## Testing-Strategie
|
||||
|
||||
### Validierungsansatz
|
||||
|
||||
Die Testing-Strategie folgt einem zweiphasigen Ansatz: Zuerst Counterexamples auf dem unfixierten Code aufdecken, dann den Fix verifizieren und Preservation sicherstellen.
|
||||
|
||||
### Exploratory Fault Condition Checking
|
||||
|
||||
**Ziel**: Counterexamples aufdecken, die den Bug VOR der Implementierung des Fixes demonstrieren. Root-Cause-Analyse bestätigen oder widerlegen.
|
||||
|
||||
**Testplan**: Tests schreiben, die `infer_competences` (bzw. die betroffenen Tools) mit nicht-leerem Task-Text und vorhandenen Kompetenzen aufrufen. Auf dem unfixierten Code beobachten, dass stets `[]` zurückkommt.
|
||||
|
||||
**Testfälle**:
|
||||
1. **validate_task_requirements mit gültigem Task**: Aufruf mit Task-ID, der einen beschriebenen Task hat (wird auf unfixiertem Code leere Kompetenz-Tabelle liefern)
|
||||
2. **extract_requirements mit Freitext**: Aufruf mit beschreibendem Text (wird auf unfixiertem Code `inferred_competences = []` liefern)
|
||||
3. **find_matching_tasks im Score-Modus**: Aufruf mit Capacity-ID (wird auf unfixiertem Code nur DB-Skills verwenden, keine LLM-Inferenz)
|
||||
4. **Leerer Task-Text**: Aufruf mit leerem Text (soll auch nach Fix `[]` liefern – Baseline)
|
||||
|
||||
**Erwartete Counterexamples**:
|
||||
- `inferred_comps` ist immer `[]`, unabhängig vom Task-Text
|
||||
- Ursache: Kein LLM-Aufruf, keine `infer_competences`-Methode vorhanden
|
||||
|
||||
### Fix Checking
|
||||
|
||||
**Ziel**: Verifizieren, dass für alle Eingaben, bei denen die Bug-Condition gilt, die fixierte Funktion das erwartete Verhalten produziert.
|
||||
|
||||
**Pseudocode:**
|
||||
```
|
||||
FOR ALL input WHERE isBugCondition(input) DO
|
||||
result := infer_competences_fixed(input.task_text)
|
||||
ASSERT len(result) > 0
|
||||
ASSERT len(result) <= 10
|
||||
FOR EACH (name, confidence) IN result DO
|
||||
ASSERT name IN get_all_competence_names()
|
||||
ASSERT 0.0 <= confidence <= 1.0
|
||||
END FOR
|
||||
END FOR
|
||||
```
|
||||
|
||||
### Preservation Checking
|
||||
|
||||
**Ziel**: Verifizieren, dass für alle Eingaben, bei denen die Bug-Condition NICHT gilt, die fixierte Funktion das gleiche Ergebnis wie die originale Funktion produziert.
|
||||
|
||||
**Pseudocode:**
|
||||
```
|
||||
FOR ALL input WHERE NOT isBugCondition(input) DO
|
||||
ASSERT infer_competences_fixed(input.task_text) == []
|
||||
END FOR
|
||||
|
||||
FOR ALL input DO
|
||||
ASSERT infer_primary_role_fixed(input) == infer_primary_role_original(input)
|
||||
END FOR
|
||||
```
|
||||
|
||||
**Testing-Ansatz**: Property-Based Testing wird für Preservation Checking empfohlen, weil:
|
||||
- Es automatisch viele Testfälle über den Eingabebereich generiert
|
||||
- Es Randfälle findet, die manuelle Unit-Tests übersehen könnten
|
||||
- Es starke Garantien bietet, dass Verhalten für alle nicht-buggy Eingaben unverändert bleibt
|
||||
|
||||
**Testplan**: Verhalten auf unfixiertem Code zuerst beobachten (leere Ergebnisse, funktionierende Rollen-Inferenz), dann Property-Based Tests schreiben, die dieses Verhalten nach dem Fix verifizieren.
|
||||
|
||||
**Testfälle**:
|
||||
1. **Rollen-Inferenz Preservation**: Verifizieren, dass `infer_primary_role` nach dem Fix identische Ergebnisse liefert
|
||||
2. **DB-Skills Preservation**: Verifizieren, dass DB-Skills weiterhin korrekt angezeigt werden
|
||||
3. **Leerer Text Preservation**: Verifizieren, dass leerer Task-Text weiterhin `[]` liefert
|
||||
4. **LLM-Fulltext Preservation**: Verifizieren, dass der LLM-Fulltext-Modus nicht beeinflusst wird
|
||||
|
||||
### Unit Tests
|
||||
|
||||
- Test `infer_competences` mit gemocktem LLM-Client: gültige JSON-Antwort → korrekte Tupel-Liste
|
||||
- Test `infer_competences` mit leerem Task-Text → `[]`
|
||||
- Test `infer_competences` mit leerer Kompetenzliste in DB → `[]`
|
||||
- Test `infer_competences` bei LLM-Fehler (Exception) → `[]` + Warning geloggt
|
||||
- Test `infer_competences` bei ungültiger JSON-Antwort → `[]`
|
||||
- Test `infer_competences` bei Kompetenz-Namen die nicht in DB sind → werden herausgefiltert
|
||||
- Test `validate_task_requirements` liefert nicht-leere Kompetenz-Tabelle
|
||||
- Test `extract_requirements` liefert nicht-leere `inferred_competences`
|
||||
|
||||
### Property-Based Tests
|
||||
|
||||
- Generiere zufällige Task-Texte und Kompetenzlisten; verifiziere, dass Ergebnisse stets Subset der DB-Kompetenzen sind
|
||||
- Generiere zufällige Eingaben; verifiziere, dass Konfidenzwerte immer in [0.0, 1.0] liegen
|
||||
- Generiere zufällige Eingaben; verifiziere, dass maximal 10 Kompetenzen zurückgegeben werden
|
||||
- Generiere zufällige Eingaben; verifiziere, dass `infer_primary_role` unverändert funktioniert (Preservation)
|
||||
|
||||
### Integration Tests
|
||||
|
||||
- End-to-End Test: `validate_task_requirements` mit echtem (gemocktem) LLM-Client zeigt Kompetenzen
|
||||
- End-to-End Test: `extract_requirements` → `find_matching_capacities` Pipeline mit inferierten Kompetenzen
|
||||
- End-to-End Test: `find_matching_tasks` im Score-Modus nutzt inferierte Kompetenzen für besseres Scoring
|
||||
@@ -1,120 +0,0 @@
|
||||
# Implementation Plan: LLM-Kompetenz-Inferenz Bugfix
|
||||
|
||||
## Übersicht
|
||||
|
||||
Explorativer Bugfix-Workflow: Zuerst den Bug durch Tests bestätigen, dann Preservation sicherstellen, anschließend den Fix implementieren und validieren. Die `infer_competences`-Methode wird in `VocabularyCache` ergänzt und an drei Stellen im `mcp_server.py` integriert.
|
||||
|
||||
## Tasks
|
||||
|
||||
- [x] 1. Bug-Condition Explorationstest schreiben
|
||||
- **Property 1: Fault Condition** - Kompetenz-Inferenz liefert stets leere Liste
|
||||
- **CRITICAL**: Dieser Test MUSS auf dem unfixierten Code FEHLSCHLAGEN – das Fehlschlagen bestätigt den Bug
|
||||
- **DO NOT** versuchen den Test oder den Code zu fixen wenn er fehlschlägt
|
||||
- **NOTE**: Dieser Test kodiert das erwartete Verhalten – er validiert den Fix wenn er nach der Implementierung besteht
|
||||
- **GOAL**: Counterexamples aufdecken, die demonstrieren dass der Bug existiert
|
||||
- **Scoped PBT Approach**: Property auf konkrete Fälle scopen: nicht-leerer Task-Text mit vorhandenen Kompetenzen in der DB
|
||||
- Test-Datei: `tests/test_competence_inference_fault_pbt.py`
|
||||
- Hypothesis-Strategie: `st.text(min_size=1)` für Task-Text, `st.lists(st.text(min_size=1), min_size=1, max_size=50)` für Kompetenzliste
|
||||
- Mock `AzureOpenAIClient.chat_completion` so dass er gültiges JSON mit Kompetenzen zurückgibt
|
||||
- Assertion: `infer_competences(task_text)` liefert eine nicht-leere Liste von bis zu 10 Tupeln `(name, confidence)` wobei jeder Name in der Kompetenzliste enthalten ist und `confidence` in [0.0, 1.0] liegt
|
||||
- Test auf UNFIXIERTEM Code ausführen – **ERWARTETES ERGEBNIS**: Test SCHLÄGT FEHL (bestätigt Bug)
|
||||
- Counterexamples dokumentieren (z.B. "infer_competences('Python Backend') gibt [] zurück statt Kompetenzen")
|
||||
- Task als abgeschlossen markieren wenn Test geschrieben, ausgeführt und Fehlschlag dokumentiert ist
|
||||
- _Requirements: 1.1, 1.2, 1.3, 2.1, 2.2, 2.3_
|
||||
|
||||
- [x] 2. Preservation Property-Tests schreiben (VOR der Fix-Implementierung)
|
||||
- **Property 2: Preservation** - Rollen-Inferenz und Leer-Eingaben unverändert
|
||||
- **IMPORTANT**: Observation-First-Methodik befolgen
|
||||
- Test-Datei: `tests/test_competence_inference_preservation_pbt.py`
|
||||
- Beobachten: `infer_primary_role` liefert auf unfixiertem Code weiterhin eine Rolle mit Konfidenz
|
||||
- Beobachten: Leerer Task-Text liefert auf unfixiertem Code `[]` (korrektes Verhalten)
|
||||
- Beobachten: Leere Kompetenzliste in DB liefert auf unfixiertem Code `[]` (korrektes Verhalten)
|
||||
- Property-Based Test 1: Für alle nicht-leeren Task-Texte liefert `infer_primary_role` weiterhin ein Tupel `(role_name, confidence)` oder `None` bei Fehler (aus Preservation Requirements)
|
||||
- Property-Based Test 2: Für alle leeren Task-Texte (`st.just("")` oder `st.from_regex(r'^\s*$')`) liefert `infer_competences` stets `[]`
|
||||
- Property-Based Test 3: Für leere Kompetenzliste in DB liefert `infer_competences` stets `[]`
|
||||
- Tests auf UNFIXIERTEM Code ausführen – **ERWARTETES ERGEBNIS**: Tests BESTEHEN (bestätigt Baseline-Verhalten)
|
||||
- Task als abgeschlossen markieren wenn Tests geschrieben, ausgeführt und bestanden auf unfixiertem Code
|
||||
- _Requirements: 3.1, 3.2, 3.3, 3.4, 3.5_
|
||||
|
||||
- [x] 3. Fix für LLM-Kompetenz-Inferenz implementieren
|
||||
|
||||
- [x] 3.1 System-Prompt `_COMPETENCE_INFERENCE_SYSTEM_PROMPT` in `vocabulary.py` hinzufügen
|
||||
- Modul-Level-Konstante analog zu `_ROLE_INFERENCE_SYSTEM_PROMPT`
|
||||
- Prompt instruiert das LLM, aus einer Kompetenzliste bis zu 10 passende Kompetenzen für einen Task-Text auszuwählen
|
||||
- Antwortformat: `{"competences": [{"name": "<name>", "confidence": <float>}]}`
|
||||
- _Bug_Condition: isBugCondition(input) where task_text.strip() != "" AND len(competence_names) > 0 AND infer_competences(task_text) == []_
|
||||
- _Expected_Behavior: Nicht-leere Liste von bis zu 10 (name, confidence)-Tupeln_
|
||||
- _Requirements: 2.1, 2.2, 2.3_
|
||||
|
||||
- [x] 3.2 Methode `infer_competences` in `VocabularyCache` implementieren
|
||||
- Signatur: `async def infer_competences(self, *, task_text: str, max_competences: int = 10) -> list[tuple[str, float]]`
|
||||
- Kompetenzliste via `self._db.get_all_competence_names()` holen
|
||||
- Bei leerer Liste oder leerem Text: `[]` zurückgeben
|
||||
- LLM-Aufruf via `self._client.chat_completion(system, user)`
|
||||
- JSON-Parsing, Validierung gegen DB-Kompetenzliste (nur bekannte Namen übernehmen)
|
||||
- Bei Fehler (Exception, ungültiges JSON): leere Liste zurückgeben + Warning loggen
|
||||
- _Bug_Condition: isBugCondition(input) where task_text.strip() != "" AND len(competence_names) > 0_
|
||||
- _Expected_Behavior: expectedBehavior(result) = len(result) > 0 AND len(result) <= 10 AND all(name in competence_names for name, _ in result) AND all(0.0 <= conf <= 1.0 for _, conf in result)_
|
||||
- _Preservation: Leerer Text → [], Leere DB → [], Fehler → [] + Warning_
|
||||
- _Requirements: 2.1, 2.2, 2.3, 2.4, 2.5_
|
||||
|
||||
- [x] 3.3 LLM-Aufruf in `validate_task_requirements` integrieren
|
||||
- Ersetze `inferred_comps: list[tuple[str, float]] = []` durch `inferred_comps = await vocab_cache.infer_competences(task_text=task_text)`
|
||||
- _Bug_Condition: validate_task_requirements aufgerufen mit Task der beschriebenen Text hat_
|
||||
- _Expected_Behavior: inferred_comps enthält bis zu 10 Kompetenzen mit Konfidenz_
|
||||
- _Preservation: Rollen-Inferenz und DB-Skills bleiben unverändert_
|
||||
- _Requirements: 2.1, 3.1, 3.2_
|
||||
|
||||
- [x] 3.4 LLM-Aufruf in `extract_requirements` integrieren
|
||||
- Ersetze `inferred_competences: list[tuple[str, float]] = []` durch `inferred_competences = await vocab_cache.infer_competences(task_text=desc)`
|
||||
- _Bug_Condition: extract_requirements aufgerufen mit nicht-leerem Text_
|
||||
- _Expected_Behavior: inferred_competences enthält bis zu 10 Kompetenzen_
|
||||
- _Requirements: 2.2_
|
||||
|
||||
- [x] 3.5 LLM-Aufruf in `find_matching_tasks` (Score-Modus) integrieren
|
||||
- Ersetze den bestehenden `inferred_comp_names`-Block durch LLM-Inferenz mit Fallback auf DB-Skills
|
||||
- `inferred_comp_tuples = await vocab_cache.infer_competences(task_text=full_text)`
|
||||
- `inferred_comp_names = [name for name, _conf in inferred_comp_tuples]`
|
||||
- Fallback: `if not inferred_comp_names: inferred_comp_names = [str(x) for x in (getattr(t, "skills", None) or []) if x]`
|
||||
- _Bug_Condition: find_matching_tasks im Score-Modus mit Task der beschriebenen Text hat_
|
||||
- _Expected_Behavior: inferred_comp_names enthält LLM-inferierte Kompetenzen für Scoring_
|
||||
- _Preservation: LLM-Fulltext-Modus bleibt unverändert_
|
||||
- _Requirements: 2.3, 3.3_
|
||||
|
||||
- [x] 3.6 Unit-Tests für `infer_competences` mit gemocktem LLM-Client
|
||||
- Test gültige JSON-Antwort → korrekte Tupel-Liste
|
||||
- Test leerer Task-Text → `[]`
|
||||
- Test leere Kompetenzliste in DB → `[]`
|
||||
- Test LLM-Fehler (Exception) → `[]` + Warning geloggt
|
||||
- Test ungültige JSON-Antwort → `[]`
|
||||
- Test Kompetenz-Namen die nicht in DB sind → werden herausgefiltert
|
||||
- Test max_competences Begrenzung auf 10
|
||||
- _Requirements: 2.1, 2.2, 2.3, 2.4, 2.5_
|
||||
|
||||
- [x] 3.7 Bug-Condition Explorationstest erneut ausführen – Verifizieren dass er jetzt besteht
|
||||
- **Property 1: Expected Behavior** - Kompetenz-Inferenz liefert Ergebnisse
|
||||
- **IMPORTANT**: Den GLEICHEN Test aus Task 1 erneut ausführen – KEINEN neuen Test schreiben
|
||||
- Der Test aus Task 1 kodiert das erwartete Verhalten
|
||||
- Wenn dieser Test besteht, bestätigt das dass das erwartete Verhalten erfüllt ist
|
||||
- Bug-Condition Explorationstest aus Schritt 1 ausführen
|
||||
- **ERWARTETES ERGEBNIS**: Test BESTEHT (bestätigt Bug ist behoben)
|
||||
- _Requirements: 2.1, 2.2, 2.3_
|
||||
|
||||
- [x] 3.8 Preservation-Tests erneut ausführen – Verifizieren dass sie weiterhin bestehen
|
||||
- **Property 2: Preservation** - Rollen-Inferenz und Leer-Eingaben unverändert
|
||||
- **IMPORTANT**: Die GLEICHEN Tests aus Task 2 erneut ausführen – KEINE neuen Tests schreiben
|
||||
- Preservation Property-Tests aus Schritt 2 ausführen
|
||||
- **ERWARTETES ERGEBNIS**: Tests BESTEHEN (bestätigt keine Regressionen)
|
||||
- Bestätigen dass alle Tests nach dem Fix weiterhin bestehen
|
||||
- _Requirements: 3.1, 3.2, 3.3, 3.4, 3.5_
|
||||
|
||||
- [x] 4. Checkpoint - Sicherstellen dass alle Tests bestehen
|
||||
- Alle Tests ausführen und sicherstellen dass sie bestehen, bei Fragen den User konsultieren.
|
||||
|
||||
## Hinweise
|
||||
|
||||
- Property-Based Tests verwenden Hypothesis mit `@settings(max_examples=100)`.
|
||||
- Unit- und Integrationstests verwenden `pytest` (Run-once, kein Watch-Modus); der `AzureOpenAIClient` wird stets gemockt.
|
||||
- Die Implementierungssprache ist Python (bestehende Codebase).
|
||||
- Tasks referenzieren explizit Anforderungen aus `bugfix.md` zur lückenlosen Nachverfolgbarkeit.
|
||||
- Der Fix ist additiv: bestehende Funktionalität (Rollen-Inferenz, DB-Skills, LLM-Fulltext-Matching) bleibt unverändert.
|
||||
@@ -1 +0,0 @@
|
||||
{"specId": "f4477224-78a1-4ce4-a5fc-689c61620b81", "workflowType": "requirements-first", "specType": "feature"}
|
||||
@@ -1,798 +0,0 @@
|
||||
# Design: LLM-Volltext-Matching als zweites Verfahren
|
||||
|
||||
## Übersicht
|
||||
|
||||
Dieses Design beschreibt die Einführung eines zweiten Matching-Verfahrens neben dem bestehenden Score-basierten Matching: einen **LLM-basierten Volltext-Vergleich** (`llm_fulltext`), der Kapazitäten und Aufgaben anhand von ganzen Profiltexten bewertet und jedes Ergebnis direkt einer der bestehenden Kategorien (`Top`, `Good`, `Partial`, `Low`, `Irrelevant`) zuordnet. Es gibt keine numerischen Scores mehr im neuen Modus, dafür eine Begründung (Rationale) pro Treffer.
|
||||
|
||||
Das neue Verfahren erweitert den Datenraum um:
|
||||
|
||||
- die Capacity-Beschreibung (`teamlandkarte_v_capacities_latest.description`)
|
||||
- die Capacity-Zertifikate (`teamlandkarte_v_capacity_certificates_latest.description`)
|
||||
- die Capacity-Referenzen (`teamlandkarte_v_capacity_references_latest.projects`)
|
||||
- den Partner-Namen je Referenz aus `teamlandkarte_v_partners_latest.name`, verknüpft über `teamlandkarte_v_capacity_references_latest.partner_id = teamlandkarte_v_partners_latest.id`
|
||||
|
||||
Eine Capacity_Reference besteht damit aus dem Tupel (`partner_name`, `projects`); `partner_name` kann leer sein, wenn `partner_id` `NULL` ist oder der Join keinen Treffer liefert.
|
||||
|
||||
Auf Aufgabenseite werden die bestehenden Felder (`title`, `description`, `skills`) genutzt.
|
||||
|
||||
Der Nutzer wählt das Verfahren über den neuen Tool-Parameter `matching_method` (`score` | `llm_fulltext`); der Standardwert ist konfigurierbar (`config.toml: matching.default_method`). Beide Verfahren teilen sich Suchcache, Pagination, Filtertools und Bestätigungs-Workflow.
|
||||
|
||||
## Architektur
|
||||
|
||||
### Komponenten-Überblick (nachher)
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A[MCP Tool: find_matching_capacities] -->|matching_method| R{Routing}
|
||||
B[MCP Tool: find_matching_tasks] -->|matching_method| R
|
||||
R -->|score| M[Matcher BM25+LLM Role]
|
||||
R -->|llm_fulltext| F[LLM_Fulltext_Matcher]
|
||||
|
||||
F --> P1[Profile Builder<br/>Capacity_Profile + Task_Profile]
|
||||
F --> CL[AzureOpenAIClient.chat_completion]
|
||||
F --> SC[(SearchCache<br/>category + rationale)]
|
||||
|
||||
P1 --> DB[(TrinoClient)]
|
||||
DB --> V1[teamlandkarte_v_capacities_latest.description]
|
||||
DB --> V2[teamlandkarte_v_capacity_certificates_latest]
|
||||
DB --> V3[teamlandkarte_v_capacity_references_latest]
|
||||
V3 -->|partner_id = id| V4[teamlandkarte_v_partners_latest.name]
|
||||
|
||||
M --> SC
|
||||
```
|
||||
|
||||
Der `LLM_Fulltext_Matcher` ist eine neue Komponente in der Business-Logic-Layer und wird beim Server-Start instanziiert. Er greift auf den bestehenden `AzureOpenAIClient`, den `DBClient` und den `SearchCache` zu. Der bestehende `Matcher` bleibt unverändert; die Auswahl erfolgt im MCP-Tool.
|
||||
|
||||
### Runtime: find_matching_capacities (LLM-Volltext)
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant Client
|
||||
participant Server as MCP_Server
|
||||
participant FM as LLM_Fulltext_Matcher
|
||||
participant DB as TrinoClient
|
||||
participant LLM as AzureOpenAIClient
|
||||
|
||||
Client->>Server: find_matching_capacities(role_name, competences, dates, matching_method="llm_fulltext")
|
||||
Server->>Server: Validate matching_method, confirmation gate
|
||||
Server->>DB: get_all_capacities_with_competences()
|
||||
Server->>Server: Vorfilter (Verfügbarkeit) wie bei score
|
||||
Server->>FM: match_capacities(task_profile, filtered_capacities)
|
||||
FM->>DB: batch_get_capacity_descriptions(ids)
|
||||
FM->>DB: batch_get_capacity_certificates(ids)
|
||||
FM->>DB: batch_get_capacity_references(ids)
|
||||
FM->>FM: build Task_Profile + Capacity_Profile pro Kandidat
|
||||
loop pro Kapazität
|
||||
FM->>LLM: chat_completion(system_prompt, user_prompt)
|
||||
LLM-->>FM: {"category": "...", "rationale": "..."}
|
||||
FM->>FM: validate(category) sonst Irrelevant + Hinweis
|
||||
end
|
||||
FM-->>Server: {by_category, errors}
|
||||
Server->>SC: store_search(results=payload mit matching_method)
|
||||
Server-->>Client: Markdown (Summary + Begründungs-Spalte)
|
||||
```
|
||||
|
||||
### Runtime: find_matching_tasks (LLM-Volltext)
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant Client
|
||||
participant Server as MCP_Server
|
||||
participant FM as LLM_Fulltext_Matcher
|
||||
participant DB as TrinoClient
|
||||
participant LLM as AzureOpenAIClient
|
||||
|
||||
Client->>Server: find_matching_tasks(capacity_id, matching_method="llm_fulltext")
|
||||
Server->>DB: get_capacity_by_id(capacity_id)
|
||||
Server->>DB: get_open_tasks(limit=0)
|
||||
Server->>FM: match_tasks(capacity_profile, tasks)
|
||||
FM->>DB: get_capacity_description(capacity_id)
|
||||
FM->>DB: get_capacity_certificates(capacity_id)
|
||||
FM->>DB: get_capacity_references(capacity_id)
|
||||
FM->>FM: build Capacity_Profile + Task_Profile pro Aufgabe
|
||||
loop pro Aufgabe
|
||||
FM->>LLM: chat_completion(system_prompt, user_prompt)
|
||||
LLM-->>FM: {"category": "...", "rationale": "..."}
|
||||
end
|
||||
FM-->>Server: {by_category, errors}
|
||||
Server->>SC: store_search(results)
|
||||
Server-->>Client: Markdown (Summary + Begründungs-Spalte)
|
||||
```
|
||||
|
||||
## Komponenten und Schnittstellen
|
||||
|
||||
### 1. DBClient (`database/types.py`) – neue Methoden
|
||||
|
||||
Eine Capacity_Reference wird als strukturierter Eintrag mit den Feldern `partner_name` und `projects` modelliert. `partner_name` kann eine leere Zeichenkette sein (NULL `partner_id` oder Join-Mismatch, vgl. Anforderung 2.8); `projects` enthält den Inhalt der Spalte `projects`.
|
||||
|
||||
```python
|
||||
class CapacityReferenceRow(TypedDict):
|
||||
partner_name: str # leer, wenn partner_id NULL ist oder kein Partner gefunden wurde
|
||||
projects: str
|
||||
|
||||
|
||||
class DBClient(Protocol):
|
||||
# ... bestehende Methoden ...
|
||||
|
||||
def get_capacity_description(self, capacity_id: int | str) -> str | None:
|
||||
"""Return description from teamlandkarte_v_capacities_latest."""
|
||||
raise NotImplementedError
|
||||
|
||||
def get_capacity_certificates(self, capacity_id: int | str) -> list[str]:
|
||||
"""Return certificate descriptions joined via capacity_id."""
|
||||
raise NotImplementedError
|
||||
|
||||
def get_capacity_references(
|
||||
self, capacity_id: int | str
|
||||
) -> list[CapacityReferenceRow]:
|
||||
"""Return reference entries joined via capacity_id.
|
||||
|
||||
Each entry contains the project text (`projects`) and the partner
|
||||
name from `teamlandkarte_v_partners_latest` (joined via
|
||||
`partner_id = id`). `partner_name` may be an empty string if
|
||||
`partner_id` is NULL or no matching partner exists.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def batch_get_capacity_descriptions(
|
||||
self, capacity_ids: list[int | str]
|
||||
) -> dict[str, str | None]:
|
||||
"""Batch variant: one SELECT for many capacity_ids."""
|
||||
raise NotImplementedError
|
||||
|
||||
def batch_get_capacity_certificates(
|
||||
self, capacity_ids: list[int | str]
|
||||
) -> dict[str, list[str]]:
|
||||
"""Batch variant: one SELECT, grouped per capacity_id."""
|
||||
raise NotImplementedError
|
||||
|
||||
def batch_get_capacity_references(
|
||||
self, capacity_ids: list[int | str]
|
||||
) -> dict[str, list[CapacityReferenceRow]]:
|
||||
"""Batch variant: one SELECT with LEFT JOIN on partners,
|
||||
grouped per capacity_id. `partner_name` may be empty per entry."""
|
||||
raise NotImplementedError
|
||||
```
|
||||
|
||||
Schlüssel der Batch-Rückgaben sind die `capacity_id` als String, damit die Aufrufer unabhängig vom Quelltyp (`int`/`str`) deterministisch zugreifen können.
|
||||
|
||||
### 2. TrinoClient (`database/trino_client.py`)
|
||||
|
||||
Alle neuen Methoden nutzen `_ensure_select_only`, den Pool und `_retry`. Beispiel für die Batch-Variante:
|
||||
|
||||
```python
|
||||
def batch_get_capacity_descriptions(
|
||||
self, capacity_ids: list[int | str]
|
||||
) -> dict[str, str | None]:
|
||||
if not capacity_ids:
|
||||
return {}
|
||||
placeholders = ", ".join(["?"] * len(capacity_ids))
|
||||
query = (
|
||||
"SELECT capacity_id, description "
|
||||
"FROM teamlandkarte_v_capacities_latest "
|
||||
f"WHERE capacity_id IN ({placeholders})"
|
||||
)
|
||||
_ensure_select_only(query)
|
||||
params = [str(c) for c in capacity_ids]
|
||||
|
||||
def _run():
|
||||
with self._cursor() as cur:
|
||||
cur.execute(query, params)
|
||||
return cur.fetchall()
|
||||
|
||||
rows = self._retry(_run)
|
||||
out: dict[str, str | None] = {str(c): None for c in capacity_ids}
|
||||
for row in rows:
|
||||
cap_id = str(row[0])
|
||||
desc = row[1]
|
||||
out[cap_id] = (desc.strip() if isinstance(desc, str) and desc.strip() else None)
|
||||
return out
|
||||
```
|
||||
|
||||
`batch_get_capacity_certificates` ist analog aufgebaut, gruppiert n:1 (`defaultdict(list)`), filtert leere Strings und liefert stabile, fehlende IDs als leere Liste zurück.
|
||||
|
||||
`batch_get_capacity_references` führt zusätzlich einen `LEFT JOIN` auf `teamlandkarte_v_partners_latest` aus, damit der Partner-Name in derselben Abfrage zurückgegeben wird (Anforderung 2.4: keine zusätzliche SQL-Abfrage für den Partner-Join):
|
||||
|
||||
```python
|
||||
def batch_get_capacity_references(
|
||||
self, capacity_ids: list[int | str]
|
||||
) -> dict[str, list[CapacityReferenceRow]]:
|
||||
if not capacity_ids:
|
||||
return {}
|
||||
placeholders = ", ".join(["?"] * len(capacity_ids))
|
||||
query = (
|
||||
"SELECT r.capacity_id, r.projects, COALESCE(p.name, '') AS partner_name "
|
||||
"FROM teamlandkarte_v_capacity_references_latest r "
|
||||
"LEFT JOIN teamlandkarte_v_partners_latest p ON r.partner_id = p.id "
|
||||
f"WHERE r.capacity_id IN ({placeholders})"
|
||||
)
|
||||
_ensure_select_only(query)
|
||||
params = [str(c) for c in capacity_ids]
|
||||
|
||||
def _run():
|
||||
with self._cursor() as cur:
|
||||
cur.execute(query, params)
|
||||
return cur.fetchall()
|
||||
|
||||
rows = self._retry(_run)
|
||||
out: dict[str, list[CapacityReferenceRow]] = {str(c): [] for c in capacity_ids}
|
||||
for row in rows:
|
||||
cap_id = str(row[0])
|
||||
projects = row[1]
|
||||
partner_name = row[2] or "" # NULL/COALESCE → ""
|
||||
if not (isinstance(projects, str) and projects.strip()):
|
||||
continue
|
||||
out.setdefault(cap_id, []).append(
|
||||
{"partner_name": str(partner_name), "projects": projects.strip()}
|
||||
)
|
||||
return out
|
||||
```
|
||||
|
||||
Hinweise:
|
||||
|
||||
- `COALESCE(p.name, '')` deckt sowohl NULL `partner_id` (kein Join-Match) als auch existierende Partner ohne Namen ab und garantiert einen leeren String statt `None` (Anforderung 2.8).
|
||||
- Der LEFT JOIN ist Bestandteil derselben Referenz-Abfrage; es entsteht keine zusätzliche SQL-Abfrage. Damit bleibt das Drei-Abfragen-Limit pro Quelle (Beschreibung, Zertifikate, Referenzen) erhalten (Anforderung 2.4).
|
||||
|
||||
Alle Methoden führen genau **eine** SQL-Abfrage pro Quelle aus (Anforderung 2.4) und nutzen Parameter-Bindung gegen Injection.
|
||||
|
||||
### 3. Datenmodelle: Capacity_Profile und Task_Profile
|
||||
|
||||
```python
|
||||
@dataclass(frozen=True)
|
||||
class CapacityReferenceEntry:
|
||||
"""Strukturierter Referenz-Eintrag im CapacityProfile.
|
||||
|
||||
`partner_name` darf leer sein (NULL `partner_id` oder Join-Mismatch);
|
||||
in diesem Fall wird die Referenz dennoch im Profil geführt und
|
||||
ausschließlich `projects` in der Serialisierung dargestellt.
|
||||
"""
|
||||
|
||||
partner_name: str # leer, wenn nicht zuordenbar
|
||||
projects: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CapacityProfile:
|
||||
id: str
|
||||
owner_name: str
|
||||
role_name: str
|
||||
competences: list[str]
|
||||
description: str # leer wenn None/leer in DB
|
||||
references: list[CapacityReferenceEntry]
|
||||
certificates: list[str]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TaskProfile:
|
||||
id: str
|
||||
title: str
|
||||
description: str
|
||||
skills: list[str] # gesuchte Kompetenzen
|
||||
```
|
||||
|
||||
Beide Profile haben deterministische Serialisierungen (siehe `serialize`). Leere Felder erzeugen leere Zeichenkette/leere Liste, das Profil wird nie verworfen (Anforderung 3.2/4.2).
|
||||
|
||||
### 4. LLM_Fulltext_Matcher (`matching/llm_fulltext_matcher.py`)
|
||||
|
||||
```python
|
||||
class LlmFulltextMatcher:
|
||||
"""LLM-based full-text matching between capacities and tasks."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
db: DBClient,
|
||||
client: AzureOpenAIClient,
|
||||
rationale_max_chars: int = 280,
|
||||
) -> None:
|
||||
self._db = db
|
||||
self._client = client
|
||||
self._rationale_max_chars = rationale_max_chars
|
||||
|
||||
async def match_capacities(
|
||||
self,
|
||||
*,
|
||||
task_profile: TaskProfile,
|
||||
capacities: list[Capacity],
|
||||
) -> "LlmFulltextResult": ...
|
||||
|
||||
async def match_tasks(
|
||||
self,
|
||||
*,
|
||||
capacity_profile: CapacityProfile,
|
||||
tasks: list[Task],
|
||||
) -> "LlmFulltextResult": ...
|
||||
```
|
||||
|
||||
Ergebnistyp:
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class LlmFulltextItem:
|
||||
item_id: str # capacity_id oder task_id
|
||||
category: str # Top|Good|Partial|Low|Irrelevant
|
||||
rationale: str # ungekürzt, von LLM
|
||||
raw: dict # ursprüngliches Item-Payload für Cache (asdict(Capacity)/Task)
|
||||
|
||||
|
||||
@dataclass
|
||||
class LlmFulltextError:
|
||||
item_id: str
|
||||
error: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class LlmFulltextResult:
|
||||
by_category: dict[str, list[LlmFulltextItem]]
|
||||
errors: list[LlmFulltextError]
|
||||
```
|
||||
|
||||
#### Profil-Serialisierung (deterministisch)
|
||||
|
||||
```python
|
||||
def _format_reference(entry: CapacityReferenceEntry) -> str:
|
||||
"""Format a single reference deterministically.
|
||||
|
||||
- `Partner: <name> – Projekte: <projects>` wenn partner_name nicht leer
|
||||
- `Projekte: <projects>` wenn partner_name leer (kein Platzhalter)
|
||||
"""
|
||||
projects = entry.projects.strip()
|
||||
if entry.partner_name:
|
||||
return f"Partner: {entry.partner_name} – Projekte: {projects}"
|
||||
return f"Projekte: {projects}"
|
||||
|
||||
|
||||
def serialize_capacity_profile(p: CapacityProfile) -> str:
|
||||
refs = [_format_reference(r) for r in p.references]
|
||||
return "\n".join([
|
||||
f"Rolle: {p.role_name}",
|
||||
"Kompetenzen: " + ", ".join(p.competences),
|
||||
f"Beschreibung: {p.description}",
|
||||
"Referenzen:" + ("\n- " + "\n- ".join(refs) if refs else " (keine)"),
|
||||
"Zertifikate:" + ("\n- " + "\n- ".join(p.certificates) if p.certificates else " (keine)"),
|
||||
])
|
||||
|
||||
|
||||
def serialize_task_profile(p: TaskProfile) -> str:
|
||||
return "\n".join([
|
||||
f"Titel: {p.title}",
|
||||
f"Beschreibung: {p.description}",
|
||||
"Gesuchte Kompetenzen: " + ", ".join(p.skills),
|
||||
])
|
||||
```
|
||||
|
||||
Die Reihenfolge der Felder ist über alle Profile konstant, und die Reihenfolge der Referenzen entspricht der Reihenfolge aus dem DBClient (DB-stabil sortiert), sodass auch `partner_name` deterministisch erscheint (Anforderung 3.4 / 3.7 / 4.4). Ist `partner_name` leer, entfällt das Partner-Token in der Ausgabe; die Referenz selbst bleibt erhalten (Anforderung 3.4 / 3.6).
|
||||
|
||||
#### LLM-Prompt-Design
|
||||
|
||||
System-Prompt (deutschsprachig, deterministisch):
|
||||
|
||||
```text
|
||||
Du bist ein erfahrener Personal- und Skill-Matcher der DB Systel.
|
||||
Du erhältst ein Aufgabenprofil und ein Kapazitätsprofil.
|
||||
Bewerte, wie gut die Kapazität zur Aufgabe passt, und wähle GENAU EINE Kategorie aus:
|
||||
- Top: passt fachlich und in den Kompetenzen praktisch vollständig
|
||||
- Good: passt gut, mit kleinen Lücken
|
||||
- Partial: passt teilweise, mehrere relevante Lücken
|
||||
- Low: schwacher Bezug, nur einzelne Berührungspunkte
|
||||
- Irrelevant: kein erkennbarer fachlicher Bezug
|
||||
|
||||
Begründe deine Wahl in 1-2 prägnanten deutschen Sätzen
|
||||
(maximal ~280 Zeichen, keine Aufzählungspunkte, keine Zeilenumbrüche).
|
||||
Antworte AUSSCHLIESSLICH als gültiges JSON-Objekt mit den Feldern:
|
||||
{"category": "<Top|Good|Partial|Low|Irrelevant>", "rationale": "<Begründung>"}
|
||||
```
|
||||
|
||||
User-Prompt (Beispiel Aufgabe→Kapazität):
|
||||
|
||||
```text
|
||||
=== Aufgabe ===
|
||||
<serialize_task_profile(...)>
|
||||
|
||||
=== Kapazität ===
|
||||
ID: <capacity.id>
|
||||
Owner: <capacity.owner_name>
|
||||
<serialize_capacity_profile(...)>
|
||||
```
|
||||
|
||||
Die Antwort wird über `chat_completion(system, user)` (bereits mit `response_format=json_object`) angefordert und mit `json.loads` geparst.
|
||||
|
||||
#### Kategorie-Normalisierung
|
||||
|
||||
```python
|
||||
_ALLOWED = ("Top", "Good", "Partial", "Low", "Irrelevant")
|
||||
_ALIAS = {x.lower(): x for x in _ALLOWED}
|
||||
|
||||
def normalize_category(value: str | None) -> tuple[str, bool]:
|
||||
"""Return (category, is_valid). Invalid → ("Irrelevant", False)."""
|
||||
if not isinstance(value, str):
|
||||
return "Irrelevant", False
|
||||
norm = _ALIAS.get(value.strip().lower())
|
||||
if norm is None:
|
||||
return "Irrelevant", False
|
||||
return norm, True
|
||||
```
|
||||
|
||||
Bei ungültiger Kategorie wird das Item nach `Irrelevant` einsortiert und in der Rationale wird angehängt: `"[Hinweis: ungültige LLM-Kategorie: <wert>]"` (Anforderung 5.6 / 6.6).
|
||||
|
||||
#### Sortierung
|
||||
|
||||
Innerhalb jeder Kategorie werden die Items deterministisch nach `item_id` aufsteigend (lexikographisch als String) sortiert (Anforderung 5.8 / 6.8).
|
||||
|
||||
### 5. MCP_Server – erweiterte Tool-Signaturen
|
||||
|
||||
```python
|
||||
@mcp.tool()
|
||||
async def find_matching_capacities(
|
||||
role_name: str,
|
||||
competences: list[str],
|
||||
date_start: Optional[str] = None,
|
||||
date_end: Optional[str] = None,
|
||||
matching_method: Optional[str] = None,
|
||||
) -> str: ...
|
||||
|
||||
@mcp.tool()
|
||||
async def find_matching_tasks(
|
||||
capacity_id: int | str,
|
||||
matching_method: Optional[str] = None,
|
||||
) -> str: ...
|
||||
```
|
||||
|
||||
Validierung von `matching_method`:
|
||||
|
||||
```python
|
||||
_ALLOWED_METHODS = ("score", "llm_fulltext")
|
||||
|
||||
def _resolve_method(value: Optional[str]) -> str:
|
||||
if value is None:
|
||||
return cfg.matching.default_method
|
||||
norm = str(value).strip().lower()
|
||||
if norm not in _ALLOWED_METHODS:
|
||||
raise ValueError(
|
||||
f"Invalid matching_method: {value!r}. "
|
||||
f"Allowed: {', '.join(_ALLOWED_METHODS)}"
|
||||
)
|
||||
return norm
|
||||
```
|
||||
|
||||
Bei ungültigem Wert gibt das Tool eine Fehlermeldung zurück und führt keine Suche aus (Anforderung 1.5).
|
||||
|
||||
### 6. Persistenz im SearchCache
|
||||
|
||||
Das Ergebnis-Payload behält den bestehenden Aufbau (`search_type`, `reference`, `summary`, `by_category`), wird aber pro Modus unterschiedlich befüllt:
|
||||
|
||||
```python
|
||||
results_payload = {
|
||||
"search_type": "capacity_search", # oder "task_search"
|
||||
"matching_method": "llm_fulltext", # NEU – im Score-Modus "score"
|
||||
"reference": {...},
|
||||
"summary": {...}, # Counter pro Kategorie
|
||||
"by_category": {
|
||||
"Top": [
|
||||
{
|
||||
**asdict(capacity), # bzw. Task-Felder
|
||||
"category": "Top",
|
||||
"rationale": "<unverkürzt>",
|
||||
# KEIN role_score / competence_score / overall_score
|
||||
},
|
||||
...
|
||||
],
|
||||
...
|
||||
},
|
||||
"errors": [ # nur im LLM-Modus, sonst weglassen
|
||||
{"item_id": "...", "error": "..."}
|
||||
],
|
||||
}
|
||||
```
|
||||
|
||||
Im Score-Modus bleibt das bisherige Schema (mit Score-Feldern, ohne `rationale`/`errors`) unverändert. Damit ist das Schema rückwärtskompatibel: bestehende Filter- und Pagination-Tools lesen `by_category` weiterhin korrekt.
|
||||
|
||||
### 7. Ausgabe-Tabellen
|
||||
|
||||
#### Score-Modus (unverändert)
|
||||
|
||||
Spalten: `ID | Owner | Role | Competences | Availability | Role Score | Competence Score | Overall Score | Category`.
|
||||
|
||||
#### LLM-Volltext-Modus
|
||||
|
||||
`find_matching_capacities`:
|
||||
|
||||
`ID | Owner | Role | Competences | Availability | Category | Begründung`
|
||||
|
||||
`find_matching_tasks`:
|
||||
|
||||
`task_id | Title | Required Competences | Availability | Category | Begründung`
|
||||
|
||||
Eine Hilfsfunktion kürzt Rationales konsistent:
|
||||
|
||||
```python
|
||||
def _format_rationale_for_table(rationale: str, max_chars: int = 280) -> str:
|
||||
text = (rationale or "").replace("|", "/").replace("\r", " ").replace("\n", " ")
|
||||
text = " ".join(text.split()) # collapse whitespace
|
||||
if len(text) > max_chars:
|
||||
text = text[: max_chars - 1].rstrip() + "…"
|
||||
return text
|
||||
```
|
||||
|
||||
- Pipes (`|`) werden zu `/`, Zeilenumbrüche zu Leerzeichen ersetzt (Anforderung 8.4).
|
||||
- Bei > 280 Zeichen wird gekürzt und mit `…` abgeschlossen (Anforderung 8.5).
|
||||
- Die ungekürzte Rationale steht im SearchCache (Anforderung 8.6).
|
||||
|
||||
Die Summary-Tabelle bleibt in beiden Modi gleich (Counter je Kategorie).
|
||||
|
||||
Im META-JSON jeder Tool-Antwort wird `matching_method` zusätzlich aufgenommen (Anforderung 1.6 / 9.5):
|
||||
|
||||
```json
|
||||
{"search_id": "...", "filter_id": null, "default_category": "Top", "matching_method": "llm_fulltext"}
|
||||
```
|
||||
|
||||
### 8. Filter- und Pagination-Tools
|
||||
|
||||
#### get_results_by_category
|
||||
|
||||
Erkennt das Modus-Schema am Feld `matching_method` im SearchEntry und rendert entweder die Score-Tabelle (bisheriges Verhalten) oder die LLM-Tabelle mit `Begründung`-Spalte. Das Routing kapselt eine neue Hilfsfunktion `_format_results_table(items, *, search_type, matching_method, ref_start, ref_end)`.
|
||||
|
||||
#### filter_search_results
|
||||
|
||||
Erkennt den Modus ebenfalls am persistierten `matching_method`. Im LLM-Volltext-Modus:
|
||||
|
||||
- Bestehende Filter (Rollen-, Kompetenz-, Verfügbarkeits-, Aufgaben-Text-/Kompetenzfilter) bleiben aktiv (Anforderung 9.3).
|
||||
- `min_similarity` wird ignoriert; in `Applied Filters` erscheint eine Zeile `min_similarity (ignored: not applicable in llm_fulltext mode)` (Anforderung 9.4).
|
||||
- Die Zwischensortierung erfolgt im LLM-Modus stabil nach `(category_rank, item_id)` statt nach `overall_score`.
|
||||
|
||||
### 9. Konfiguration (`config.py`, `config.toml`)
|
||||
|
||||
Neue Felder in `MatchingConfig`:
|
||||
|
||||
```python
|
||||
@dataclass(frozen=True)
|
||||
class MatchingConfig:
|
||||
# ... bestehende Felder ...
|
||||
default_method: str = "score"
|
||||
```
|
||||
|
||||
Validierung in `load_config`:
|
||||
|
||||
```python
|
||||
allowed = {"score", "llm_fulltext"}
|
||||
default_method = (raw.get("matching", {}).get("default_method") or "score").strip().lower()
|
||||
if default_method not in allowed:
|
||||
raise ConfigError(
|
||||
f"matching.default_method must be one of {sorted(allowed)}, "
|
||||
f"got: {default_method!r}"
|
||||
)
|
||||
```
|
||||
|
||||
`config.toml`:
|
||||
|
||||
```toml
|
||||
[matching]
|
||||
# Default method for new searches when callers do not pass matching_method.
|
||||
# Allowed: "score" (BM25 + LLM role similarity) or "llm_fulltext"
|
||||
# (LLM-based full-text matching with rationale).
|
||||
default_method = "score"
|
||||
```
|
||||
|
||||
Beim Start wird die Validierung als `ConfigError` (fail-fast) geworfen (Anforderung 12.4).
|
||||
|
||||
### 10. Agenten- und Dokumentationsanpassungen
|
||||
|
||||
- `.github/agents/teamlandkarte_agent.md` und `.kiro/agents/teamlandkarte.md`:
|
||||
- Beschreibung beider Verfahren (`score`, `llm_fulltext`).
|
||||
- Pflicht-Frage "Welches Verfahren soll verwendet werden?" vor `find_matching_capacities`/`find_matching_tasks`, falls nicht aus dem Verlauf bekannt.
|
||||
- Hinweis: Im LLM-Volltext-Modus keine numerischen Scores; stattdessen Spalte `Begründung`.
|
||||
- Bestätigungs-Workflow (`show_pending_requirements` → `confirm_requirements`) bleibt für beide Verfahren identisch.
|
||||
- `docs/architecture.md`:
|
||||
- Neue Komponente `LLM_Fulltext_Matcher` im Business-Logic-Diagramm und in der Komponentenbeschreibung.
|
||||
- Erweiterung der Schema-Verifikation um `teamlandkarte_v_capacities_latest.description`, `teamlandkarte_v_capacity_certificates_latest.{capacity_id, description}`, `teamlandkarte_v_capacity_references_latest.{capacity_id, partner_id, projects}` sowie `teamlandkarte_v_partners_latest.{id, name}` einschließlich der Join-Beziehung `teamlandkarte_v_capacity_references_latest.partner_id = teamlandkarte_v_partners_latest.id`.
|
||||
- Tool-Surface-Tabelle mit neuem Parameter `matching_method`.
|
||||
- Runtime-View für beide Suchrichtungen ergänzt um den LLM-Volltext-Pfad.
|
||||
- `README.md`:
|
||||
- Quick Start: Verfahrenswahl per Tool-Parameter; Default-Konfiguration in `[matching].default_method`.
|
||||
- Hinweis "keine Score-Spalten im LLM-Modus, dafür `Begründung`".
|
||||
- Zusätzliche Datenbank-Views aufgelistet, einschließlich `teamlandkarte_v_partners_latest` mit Hinweis auf den LEFT JOIN über `partner_id` in der Referenz-Abfrage.
|
||||
|
||||
## Datenmodelle
|
||||
|
||||
### Übersicht der zusätzlichen DB-Felder
|
||||
|
||||
| Quelle | Spalte | Genutzt für |
|
||||
|---|---|---|
|
||||
| `teamlandkarte_v_capacities_latest` | `description` | `CapacityProfile.description` |
|
||||
| `teamlandkarte_v_capacity_certificates_latest` | `capacity_id`, `description` | `CapacityProfile.certificates` |
|
||||
| `teamlandkarte_v_capacity_references_latest` | `capacity_id`, `partner_id`, `projects` | `CapacityProfile.references[].projects` (Join-Schlüssel: `partner_id`) |
|
||||
| `teamlandkarte_v_partners_latest` | `id`, `name` | Partner_Name in `CapacityProfile.references[].partner_name` (LEFT JOIN über `partner_id = id`) |
|
||||
|
||||
### LLM-Antwortschema
|
||||
|
||||
```json
|
||||
{
|
||||
"category": "Top|Good|Partial|Low|Irrelevant",
|
||||
"rationale": "Kurzbegründung in 1-2 Sätzen."
|
||||
}
|
||||
```
|
||||
|
||||
### Persistierte Item-Struktur (LLM-Modus)
|
||||
|
||||
Im LLM-Modus enthält ein gespeichertes Item dieselben Identitäts- und Verfügbarkeitsfelder wie im Score-Modus, plus `category` und `rationale`. Falls Referenzen für nachgelagerte Anzeige zusätzlich am Item gepuffert werden sollen, werden sie als Liste strukturierter Einträge mit den Feldern `partner_name` und `projects` abgelegt (gleiche Form wie in `CapacityProfile.references`); `partner_name` darf leer sein.
|
||||
|
||||
```python
|
||||
# capacity_search
|
||||
{
|
||||
"id": 12345,
|
||||
"owner_name": "...",
|
||||
"role_name": "...",
|
||||
"begin_date": "2025-03-01",
|
||||
"end_date": "2025-12-31",
|
||||
"competences": ["..."],
|
||||
# optional, falls Referenzen mitgepuffert werden:
|
||||
# "references": [{"partner_name": "...", "projects": "..."}, ...],
|
||||
"category": "Top",
|
||||
"rationale": "Volltext-Begründung des LLM ...",
|
||||
}
|
||||
|
||||
# task_search
|
||||
{
|
||||
"task_id": "00T...",
|
||||
"title": "...",
|
||||
"description": "...",
|
||||
"skills": ["..."],
|
||||
"required_competences": [],
|
||||
"start_date": "2025-04-01",
|
||||
"end_date": "2025-09-30",
|
||||
"category": "Good",
|
||||
"rationale": "Volltext-Begründung des LLM ...",
|
||||
}
|
||||
```
|
||||
|
||||
## Correctness Properties
|
||||
|
||||
*A property is a characteristic or behavior that should hold true across all valid executions of a system — essentially, a formal statement about what the system should do. Properties serve as the bridge between human-readable specifications and machine-verifiable correctness guarantees.*
|
||||
|
||||
### Property 1: Profil-Serialisierung ist deterministisch und feldvollständig
|
||||
|
||||
*For any* `CapacityProfile` (bzw. `TaskProfile`) zwei wiederholte Aufrufe von `serialize_capacity_profile` (bzw. `serialize_task_profile`) liefern denselben String, und der String enthält jede Feldüberschrift (`Rolle:`, `Kompetenzen:`, `Beschreibung:`, `Referenzen:`, `Zertifikate:` bzw. `Titel:`, `Beschreibung:`, `Gesuchte Kompetenzen:`) in einer fixen Reihenfolge. Innerhalb des Abschnitts `Referenzen:` erscheinen die Einträge in derselben Reihenfolge wie in `references`, und für jeden Eintrag mit nicht-leerem `partner_name` ist der Partner-Name in der serialisierten Darstellung deterministisch enthalten.
|
||||
|
||||
**Validates: Requirements 3.3, 3.4, 3.6, 3.7, 4.3, 4.4**
|
||||
|
||||
### Property 2: Leere/None-Felder verwerfen das Profil nicht
|
||||
|
||||
*For any* `Capacity` (bzw. `Task`), bei dem ein Teil der Felder `None`, leerer String oder leere Liste ist, liefert der Profil-Builder ein `CapacityProfile`/`TaskProfile`, dessen leere Felder als leerer String bzw. leere Liste erscheinen, und dessen Serialisierung weiterhin alle Feldüberschriften enthält. Dies gilt insbesondere auch für Capacity_References mit leerem `partner_name`: Die Referenz wird nicht verworfen, sondern in `references` aufgenommen; lediglich das Partner-Token entfällt in der Serialisierung.
|
||||
|
||||
**Validates: Requirements 3.2, 3.4, 4.2**
|
||||
|
||||
### Property 2b: Referenzen mit leerem Partner-Name behalten projects, ohne Partner-Token
|
||||
|
||||
*For any* Liste von `CapacityReferenceEntry`-Werten, in der ein Teil der Einträge `partner_name == ""` hat, ist die serialisierte `Referenzen:`-Sektion so beschaffen, dass (a) die Anzahl der ausgegebenen Referenz-Zeilen gleich der Anzahl der Einträge mit nicht-leerem `projects` ist, (b) jede Zeile zu einem Eintrag mit leerem `partner_name` mit `Projekte:` beginnt und keinen Token `Partner:` enthält, und (c) jede Zeile zu einem Eintrag mit nicht-leerem `partner_name` sowohl `Partner: <name>` als auch `Projekte: <projects>` enthält.
|
||||
|
||||
**Validates: Requirements 3.4, 3.6, 2.8**
|
||||
|
||||
### Property 3: Kategorienormalisierung bildet auf erlaubte Menge ab
|
||||
|
||||
*For any* String-Eingabe gibt `normalize_category` ein Tupel `(category, is_valid)` zurück, bei dem `category` immer in `{"Top","Good","Partial","Low","Irrelevant"}` liegt; `is_valid` ist genau dann `True`, wenn die getrimmte, lower-case Eingabe einer dieser Kategorien (case-insensitive) entspricht.
|
||||
|
||||
**Validates: Requirements 5.3, 5.6, 6.3, 6.6**
|
||||
|
||||
### Property 4: Ungültige LLM-Kategorie wird auf Irrelevant gemappt
|
||||
|
||||
*For any* LLM-Antwort `{"category": X, "rationale": R}`, bei der `X` nicht in der erlaubten Menge liegt, wird das Item in der Kategorie `Irrelevant` einsortiert, und seine gespeicherte `rationale` enthält den ursprünglichen `R` sowie einen Hinweis auf die ungültige LLM-Antwort.
|
||||
|
||||
**Validates: Requirements 5.6, 6.6**
|
||||
|
||||
### Property 5: LLM-Fehler erscheinen in der Fehlerliste, nicht als Ergebnis
|
||||
|
||||
*For any* Liste von Kapazitäten (bzw. Aufgaben), bei denen der LLM-Aufruf für eine Teilmenge `S` fehlschlägt, ist jedes Item aus `S` in `result.errors` enthalten und kommt in keiner Kategorie von `result.by_category` vor; alle restlichen Items befinden sich in genau einer Kategorie.
|
||||
|
||||
**Validates: Requirements 5.7, 6.7**
|
||||
|
||||
### Property 6: Ergebnisse sind innerhalb jeder Kategorie deterministisch sortiert
|
||||
|
||||
*For any* `LlmFulltextResult` ist innerhalb jeder Kategorie die Liste der `item_id`-Werte streng aufsteigend (lexikographisch) sortiert; Permutationen der Eingabeliste verändern die Ausgabe-Reihenfolge nicht.
|
||||
|
||||
**Validates: Requirements 5.8, 6.8**
|
||||
|
||||
### Property 7: Tabellen-Rationale ist gültiges Markdown und längenbegrenzt
|
||||
|
||||
*For any* String `R`, hat `_format_rationale_for_table(R)` höchstens 280 Zeichen, enthält weder `|` noch Zeilenumbrüche, und ist genau dann mit `…` abgeschlossen, wenn die normalisierte Eingabe länger als 280 Zeichen war.
|
||||
|
||||
**Validates: Requirements 8.4, 8.5**
|
||||
|
||||
### Property 8: Ungekürzte Rationale wird persistiert
|
||||
|
||||
*For any* erfolgreich kategorisiertes Item ist die im SearchCache gespeicherte `rationale` exakt der vom LLM gelieferte (oder durch ungültige-Kategorie-Hinweis ergänzte) String, unabhängig von der für die Tabellendarstellung verwendeten gekürzten Form.
|
||||
|
||||
**Validates: Requirements 8.6**
|
||||
|
||||
### Property 9: matching_method-Validierung lehnt unbekannte Werte ab
|
||||
|
||||
*For any* Eingabe `matching_method`, die nach `strip().lower()` weder `"score"` noch `"llm_fulltext"` ist, gibt das MCP-Tool eine Fehlermeldung zurück, in der beide erlaubten Werte vorkommen, und führt weder DB- noch LLM-Aufrufe aus.
|
||||
|
||||
**Validates: Requirements 1.5**
|
||||
|
||||
### Property 10: Score-Modus ist abwärtskompatibel
|
||||
|
||||
*For any* Aufruf von `find_matching_capacities` bzw. `find_matching_tasks` ohne den Parameter `matching_method` (oder mit `"score"`) ist das im SearchCache persistierte Ergebnis-Payload schemagleich zum bisherigen Score-Payload (Felder `role_score`, `competence_score`, `overall_score` pro Item; kein `rationale`-Feld).
|
||||
|
||||
**Validates: Requirements 1.2, 1.3, 7.3**
|
||||
|
||||
### Property 11: META enthält das verwendete Verfahren
|
||||
|
||||
*For any* erfolgreichen Tool-Aufruf von `find_matching_capacities`/`find_matching_tasks` enthält das META-JSON in der Antwort einen Schlüssel `matching_method`, dessen Wert genau dem verwendeten Verfahren entspricht (`"score"` oder `"llm_fulltext"`).
|
||||
|
||||
**Validates: Requirements 1.6, 9.5**
|
||||
|
||||
## Error Handling
|
||||
|
||||
| Szenario | Verhalten |
|
||||
|---|---|
|
||||
| `matching_method` ungültig | Tool gibt Fehlermeldung mit erlaubten Werten zurück, keine DB-/LLM-Aufrufe (Anforderung 1.5) |
|
||||
| `matching.default_method` ungültig | `ConfigError` beim Server-Start (fail-fast) |
|
||||
| LLM-Antwort kein gültiges JSON | Item landet in `errors` mit Meldung `"invalid JSON: <excerpt>"` |
|
||||
| LLM-Antwort enthält `category` außerhalb der Menge | Item in Kategorie `Irrelevant`; Rationale erhält Hinweis `[Hinweis: ungültige LLM-Kategorie: <wert>]` |
|
||||
| LLM-Aufruf wirft `AzureAPIError` / Timeout | Item landet in `errors` mit Meldung der Exception-Klasse + erstem Satz; kein Eintrag in `by_category` |
|
||||
| `description`/`certificates`/`references` in DB leer | Profil wird mit leerem String/leerer Liste gebaut, niemals verworfen (Anforderung 3.2/4.2) |
|
||||
| Capacity ohne Datenbank-Eintrag in der Batch-Antwort | Wird als `description=None` / `certificates=[]` / `references=[]` interpretiert |
|
||||
| `min_similarity` im Filter angewandt im LLM-Modus | Filter wird ignoriert; Eintrag in `Applied Filters` mit Hinweis (Anforderung 9.4) |
|
||||
| Rationale enthält `|` oder Zeilenumbruch | Wird vor Tabellenausgabe ersetzt; ungekürzte Original-Rationale bleibt im SearchCache |
|
||||
| Rationale länger als 280 Zeichen | In Tabellenausgabe gekürzt mit `…`; ungekürzt im SearchCache |
|
||||
|
||||
Die Fehlerliste wird im Tool-Output nach der Ergebnistabelle als zusätzlicher Markdown-Block (`## Errors`) ausgegeben, sofern nicht leer. Der MCP-Tool-Aufruf selbst schlägt nicht fehl, solange wenigstens ein Item kategorisiert werden konnte oder die Fehlerliste vollständig ist – damit ist das Verfahren robust gegen Einzelausfälle.
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
### Property-Based Tests
|
||||
|
||||
Bibliothek: **Hypothesis**, mindestens 100 Iterationen pro Property. Jeder PBT-Test wird mit einem Kommentar getaggt:
|
||||
|
||||
```python
|
||||
# Feature: llm-fulltext-matching, Property {N}: {title}
|
||||
```
|
||||
|
||||
| Property | Test-Ansatz | Generatoren |
|
||||
|---|---|---|
|
||||
| 1: Profil-Serialisierung deterministisch | Generiere `CapacityProfile`/`TaskProfile` (inkl. `CapacityReferenceEntry` mit/ohne `partner_name`), rufe Serializer zweimal auf, prüfe Gleichheit + Vorkommen aller Feldüberschriften und Partner-Namen | `st.builds(...)` mit `st.text`/`st.lists(st.builds(CapacityReferenceEntry, ...))` |
|
||||
| 2: Leere Felder verwerfen Profil nicht | Generiere `Capacity`/`Task` mit zufällig leeren Feldern (inkl. Referenzen mit leerem `partner_name`), baue Profil, prüfe Vollständigkeit | Custom Capacity/Task strategy mit `st.one_of(st.none(), st.text())` und Referenz-Strategie mit `partner_name=st.one_of(st.just(""), st.text())` |
|
||||
| 2b: Leerer Partner-Name → kein Partner-Token, projects bleibt | Generiere Referenz-Listen mit gemischtem `partner_name`, serialisiere, prüfe Zeilenanzahl, Präfixe und Token-Vorkommen | `st.lists(st.builds(CapacityReferenceEntry, partner_name=st.one_of(st.just(""), st.text(min_size=1)), projects=st.text(min_size=1)))` |
|
||||
| 3: Kategorienormalisierung im Wertebereich | Generiere zufällige Strings (inkl. Aliase), prüfe Output ∈ erlaubte Menge | `st.text()` und `st.sampled_from([...alias variants...])` |
|
||||
| 4: Ungültige Kategorie → Irrelevant | Mocke LLM mit zufälliger ungültiger Kategorie, prüfe Item in `Irrelevant` und Hinweis in Rationale | `st.text().filter(lambda x: x.strip().lower() not in {"top",...})` |
|
||||
| 5: LLM-Fehler in errors | Mocke LLM, das per Bool-Strategie eine Exception wirft, prüfe Trennung errors / by_category | `st.lists(st.booleans())` |
|
||||
| 6: Sortierung deterministisch | Generiere Items, permutiere Eingabe, prüfe gleiche `by_category`-Reihenfolge | `st.permutations(...)` |
|
||||
| 7: Tabellen-Rationale-Format | Generiere Strings inkl. `|`, `\n`, sehr lang, prüfe Längen- und Zeichen-Constraints | `st.text(alphabet=st.characters(blacklist_categories=()))` |
|
||||
| 8: Ungekürzte Rationale persistiert | Generiere Rationale > 280 Zeichen, prüfe Cache-Eintrag == Original | `st.text(min_size=300)` |
|
||||
| 9: matching_method Validierung | Generiere zufällige Strings, mocke DB+LLM, prüfe Fehlerpfad ohne Aufrufe | `st.text()` |
|
||||
| 10: Score-Modus abwärtskompatibel | Vergleiche persistiertes Payload-Schema vor/nach Patch (snapshot-frei: Feldmenge je Item) | bestehende Capacity-Strategie |
|
||||
| 11: META-Schlüssel | Aus Tool-Output META-JSON parsen und prüfen | bestehende Capacity-Strategie |
|
||||
|
||||
### Unit Tests
|
||||
|
||||
- **DBClient (Trino)**: Mock-Cursor-Tests für `get_capacity_description`, `get_capacity_certificates`, `get_capacity_references` und ihre Batch-Varianten. Verifiziert: genau eine SQL-Abfrage je Methode, korrekte SELECT-Only-Guard, Gruppierung n:1, Default für fehlende IDs. Für die Referenz-Methoden zusätzlich:
|
||||
- LEFT JOIN auf `teamlandkarte_v_partners_latest` ist Bestandteil derselben SQL-Abfrage (kein zusätzlicher SQL-Roundtrip; Anforderung 2.4).
|
||||
- Mock-Cursor liefert drei Spalten (`capacity_id`, `projects`, `partner_name`); Rückgabe enthält `CapacityReferenceRow`-Einträge mit korrektem Partner-Namen.
|
||||
- Fall NULL `partner_id` bzw. fehlender Partner: `partner_name` ist leerer String (`COALESCE`), Referenz bleibt mit `projects` erhalten (Anforderung 2.8).
|
||||
- **LlmFulltextMatcher**: Beispiel-basierte Tests mit gemocktem `AzureOpenAIClient`:
|
||||
- Erfolgsfall (gültige Kategorie + Rationale)
|
||||
- Ungültiges JSON
|
||||
- Gültiges JSON mit unbekannter Kategorie
|
||||
- Exception aus `chat_completion` (`AzureAPIError`)
|
||||
- Vermischung mehrerer Kapazitäten/Aufgaben (Reihenfolge, Sortierung)
|
||||
- **MCP-Tools** (`find_matching_capacities`, `find_matching_tasks`):
|
||||
- `matching_method=None` → Default greift, Schema entspricht Score-Modus.
|
||||
- `matching_method="llm_fulltext"` → keine Score-Spalten, `Begründung`-Spalte vorhanden, `META` enthält `matching_method`.
|
||||
- `matching_method="bogus"` → Fehlermeldung; keine DB/LLM-Aufrufe (über Mocks verifiziert).
|
||||
- **`get_results_by_category` / `filter_search_results`**:
|
||||
- Im LLM-Modus rendern sie die `Begründung`-Spalte und ignorieren `min_similarity` mit Hinweis.
|
||||
- Im Score-Modus bleibt das Verhalten unverändert (Regression).
|
||||
- **Konfiguration**: Test, dass `matching.default_method = "irgendwas"` einen `ConfigError` beim Laden wirft, und dass das Weglassen den Default `"score"` ergibt.
|
||||
|
||||
### Integrationstests
|
||||
|
||||
- End-to-End-Lauf für beide Suchrichtungen mit gemocktem LLM-Client und gemocktem `DBClient`:
|
||||
- Verfügbarkeitsfilter ist im LLM-Modus identisch zum Score-Modus.
|
||||
- SearchCache enthält `matching_method`, `rationale` (ungekürzt), `errors`-Liste.
|
||||
- Tabellen-Snapshot-Test (deterministisch über stabile Mock-Antworten), der die Kopfzeilen `... | Category | Begründung` für `find_matching_capacities`/`find_matching_tasks` im LLM-Modus festschreibt.
|
||||
|
||||
### Bewusst nicht getestet
|
||||
|
||||
- Inhaltliche Qualität der LLM-Begründungen (subjektiv).
|
||||
- Konkrete Wahl der Kategorie durch das LLM für reale Inhalte (modellabhängig, nicht deterministisch).
|
||||
- Performance/Latenz der LLM-Aufrufe (kein Unit-Test-Scope).
|
||||
@@ -1,197 +0,0 @@
|
||||
# Anforderungsdokument
|
||||
|
||||
## Einleitung
|
||||
|
||||
Dieses Dokument beschreibt die Anforderungen für die Einführung eines zweiten Matching-Verfahrens in der Teamlandkarte: einen **LLM-basierten Volltext-Vergleich** zwischen Aufgaben und Kapazitäten. Das bestehende Score-basierte Verfahren (Rolle + Kompetenzen, BM25/RRF + LLM-Rollen-Similarity) bleibt unverändert verfügbar. Der Nutzer wählt pro Suche das Verfahren aus.
|
||||
|
||||
Das neue Verfahren bezieht zusätzliche Felder aus der Datenbank ein (Beschreibung, Referenzen, Zertifikate auf Kapazitätsseite; Titel, Beschreibung und gesuchte Kompetenzen auf Aufgabenseite), berechnet kein numerisches Scoring mehr und ordnet jede Kapazität bzw. Aufgabe direkt einer der bestehenden Kategorien zu. Zusätzlich liefert das LLM für jeden Fall eine Kurzbegründung (1–2 Sätze).
|
||||
|
||||
## Glossar
|
||||
|
||||
- **MCP_Server**: Der Teamlandkarte MCP-Server (Modul `mcp_server.py`), der die MCP-Tools für Matching, Suche und Datenanzeige bereitstellt.
|
||||
- **DBClient**: Protokollklasse aus `database/types.py`, die alle Datenbankzugriffe abstrahiert.
|
||||
- **TrinoClient**: Konkrete `DBClient`-Implementierung (`database/trino_client.py`) für Trino/Presto.
|
||||
- **Matcher**: Bestehende, Score-basierte Matching-Komponente in `matching/matcher.py`.
|
||||
- **LLM_Fulltext_Matcher**: Neues Modul, das den LLM-basierten Volltext-Vergleich durchführt und Kategorien direkt zuweist.
|
||||
- **AzureOpenAIClient**: Wrapper für Azure-OpenAI-Chat-Completions in `azure/openai_client.py`.
|
||||
- **LLM**: Large Language Model (Azure OpenAI Chat Completion).
|
||||
- **Capacity**: Frozen Dataclass `Capacity` in `models.py` (Kapazitätseintrag eines Mitarbeitenden).
|
||||
- **Task**: Frozen Dataclass `Task` in `models.py` (veröffentlichte Aufgabe).
|
||||
- **Capacity_Profile**: Aggregiertes Volltext-Profil einer Kapazität, bestehend aus Rolle, Kompetenzen, Beschreibung, Referenzen und Zertifikaten.
|
||||
- **Task_Profile**: Aggregiertes Volltext-Profil einer Aufgabe, bestehend aus Titel, Beschreibung und gesuchten Kompetenzen.
|
||||
- **Matching_Method**: Auswahlwert für das verwendete Verfahren. Erlaubte Werte: `score` (bisheriges Score-basiertes Matching) und `llm_fulltext` (neues LLM-Volltext-Matching).
|
||||
- **Kategorie**: Eine der bestehenden Ergebniskategorien `Top`, `Good`, `Partial`, `Low`, `Irrelevant`.
|
||||
- **Rationale**: Vom LLM erzeugte Kurzbegründung (1–2 Sätze) für die zugewiesene Kategorie.
|
||||
- **find_matching_capacities**: MCP-Tool für die Suchrichtung Aufgabe→Kapazität.
|
||||
- **find_matching_tasks**: MCP-Tool für die Suchrichtung Kapazität→Aufgabe.
|
||||
- **Teamlandkarte_Agent**: GitHub-Copilot-Agent in `.github/agents/teamlandkarte_agent.md` (sowie das Pendant in `.kiro/agents/teamlandkarte.md`) inklusive seiner Skills/Workflows.
|
||||
- **Architecture_Doc**: `docs/architecture.md`.
|
||||
- **Readme**: `README.md` im Repository-Root.
|
||||
- **Capacity_Description**: Inhalt der Spalte `description` in `teamlandkarte_v_capacities_latest`.
|
||||
- **Capacity_Certificate**: Eintrag aus `teamlandkarte_v_capacity_certificates_latest` (Feld `description`, Join via `capacity_id`, 1:n).
|
||||
- **Capacity_Reference**: Eintrag aus `teamlandkarte_v_capacity_references_latest` (Spalte `projects`, Join via `capacity_id`, 1:n) inklusive des zugehörigen Partner_Name aus `teamlandkarte_v_partners_latest`.
|
||||
- **Partner**: Eintrag aus `teamlandkarte_v_partners_latest`. Eine Capacity_Reference verweist über die Spalte `partner_id` auf einen Partner; die Verknüpfung erfolgt über `teamlandkarte_v_capacity_references_latest.partner_id = teamlandkarte_v_partners_latest.id`.
|
||||
- **Partner_Name**: Wert der Spalte `name` aus `teamlandkarte_v_partners_latest`, der einer Capacity_Reference über `partner_id` zugeordnet ist. Ist `partner_id` `NULL` oder existiert kein passender Partner, gilt der Partner_Name als leere Zeichenkette.
|
||||
|
||||
## Anforderungen
|
||||
|
||||
### Anforderung 1: Auswahl des Matching-Verfahrens
|
||||
|
||||
**User Story:** Als Nutzer möchte ich pro Suchanfrage zwischen dem bisherigen Score-basierten Matching und dem neuen LLM-basierten Volltext-Matching wählen können, damit ich je nach Situation das passende Verfahren einsetzen kann.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. THE MCP_Server SHALL akzeptieren einen Parameter `matching_method` mit den erlaubten Werten `score` und `llm_fulltext` in den Tools `find_matching_capacities` und `find_matching_tasks`.
|
||||
2. WHEN `matching_method` nicht übergeben wird, THE MCP_Server SHALL den Standardwert `score` verwenden, sodass das bestehende Verhalten unverändert bleibt.
|
||||
3. WHEN `matching_method = "score"` übergeben wird, THE MCP_Server SHALL das bestehende Score-basierte Matching ausführen.
|
||||
4. WHEN `matching_method = "llm_fulltext"` übergeben wird, THE MCP_Server SHALL das neue LLM-basierte Volltext-Matching über den LLM_Fulltext_Matcher ausführen.
|
||||
5. IF ein ungültiger Wert für `matching_method` übergeben wird, THEN THE MCP_Server SHALL eine Fehlermeldung zurückgeben, die die erlaubten Werte (`score`, `llm_fulltext`) auflistet, und die Suche nicht ausführen.
|
||||
6. THE MCP_Server SHALL den verwendeten Wert von `matching_method` im Antwort-`META`-JSON sowie in der angezeigten Suchkonfiguration ausweisen.
|
||||
|
||||
### Anforderung 2: Erweiterte Datenabfrage für Kapazitäten
|
||||
|
||||
**User Story:** Als Entwickler möchte ich, dass für das LLM-Volltext-Matching die Kapazitäts-Beschreibung, alle Zertifikate und alle Referenzen aus der Datenbank verfügbar sind, damit das LLM ein vollständiges Profil bewerten kann.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. THE DBClient SHALL eine Methode bereitstellen, die für eine gegebene `capacity_id` die Capacity_Description aus `teamlandkarte_v_capacities_latest` (Spalte `description`) zurückgibt.
|
||||
2. THE DBClient SHALL eine Methode bereitstellen, die für eine gegebene `capacity_id` alle zugeordneten Capacity_Certificate-Beschreibungen aus `teamlandkarte_v_capacity_certificates_latest` (Feld `description`, Join über `capacity_id`) als Liste von Strings zurückgibt.
|
||||
3. THE DBClient SHALL eine Methode bereitstellen, die für eine gegebene `capacity_id` alle zugeordneten Capacity_Reference-Einträge aus `teamlandkarte_v_capacity_references_latest` (Spalte `projects`, Join über `capacity_id`) inklusive des zugehörigen Partner_Name aus `teamlandkarte_v_partners_latest` (Join `teamlandkarte_v_capacity_references_latest.partner_id = teamlandkarte_v_partners_latest.id`, Spalte `name`) als Liste strukturierter Einträge mit den Feldern `projects` und `partner_name` zurückgibt.
|
||||
4. WHEN ein LLM-Volltext-Matching für mehrere Kapazitäten ausgeführt wird, THE DBClient SHALL eine Batch-Variante bereitstellen, die Beschreibungen, Zertifikate und Referenzen (inklusive Partner_Name über den Join auf `teamlandkarte_v_partners_latest`) für eine Liste von `capacity_id`-Werten in höchstens drei SQL-Abfragen lädt (eine pro Quelle); der Partner-Join SHALL Bestandteil derselben Referenz-Abfrage sein und keine zusätzliche SQL-Abfrage erzeugen.
|
||||
5. WHEN für eine Kapazität keine Beschreibung in der Datenbank vorhanden ist (NULL oder leer), THE DBClient SHALL für die Capacity_Description den Wert `None` zurückgeben.
|
||||
6. WHEN für eine Kapazität keine Zertifikate vorhanden sind, THE DBClient SHALL eine leere Liste für Capacity_Certificate zurückgeben.
|
||||
7. WHEN für eine Kapazität keine Referenzen vorhanden sind, THE DBClient SHALL eine leere Liste für Capacity_Reference zurückgeben.
|
||||
8. IF die `partner_id` einer Capacity_Reference `NULL` ist oder der Join auf `teamlandkarte_v_partners_latest` keinen Treffer liefert, THEN THE DBClient SHALL den Partner_Name dieser Capacity_Reference als leere Zeichenkette zurückgeben und die Referenz dennoch mit dem Feld `projects` in der Ergebnisliste belassen.
|
||||
9. THE TrinoClient SHALL alle neuen SQL-Abfragen ausschließlich als `SELECT`-Statements ausführen und die bestehende Read-Only-Guard `_ensure_select_only` verwenden.
|
||||
10. THE TrinoClient SHALL die neuen Abfragen über die bestehende Connection-Pool-Infrastruktur und die Retry-Logik (`_retry`) ausführen.
|
||||
|
||||
### Anforderung 3: Aufbau des Capacity_Profile
|
||||
|
||||
**User Story:** Als Entwickler möchte ich, dass das System aus den Datenbankfeldern ein konsistentes Volltext-Profil pro Kapazität erzeugt, damit das LLM eine einheitliche Eingabe erhält.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. THE LLM_Fulltext_Matcher SHALL pro Kapazität ein Capacity_Profile bilden, das die folgenden Felder enthält: `id`, `owner_name`, `role_name`, `competences`, `description`, `references` und `certificates`.
|
||||
2. THE LLM_Fulltext_Matcher SHALL jedes Element der Liste `references` im Capacity_Profile als strukturierten Eintrag mit den Feldern `partner_name` und `projects` führen, sodass beide Bestandteile einer Capacity_Reference erhalten bleiben.
|
||||
3. WHEN ein Feld in der Datenbank leer oder `None` ist, THE LLM_Fulltext_Matcher SHALL das entsprechende Feld im Capacity_Profile mit einer leeren Zeichenkette bzw. einer leeren Liste belegen, ohne das gesamte Profil zu verwerfen.
|
||||
4. WHEN der Partner_Name einer Capacity_Reference leer ist, THE LLM_Fulltext_Matcher SHALL die Referenz dennoch in `references` aufnehmen und ausschließlich das Feld `projects` in die serialisierte Darstellung übernehmen, ohne einen Platzhaltertext für den Partner einzufügen.
|
||||
5. THE LLM_Fulltext_Matcher SHALL das Capacity_Profile in einer für das LLM lesbaren, deterministischen Textstruktur serialisieren, in der jedes Feld klar mit einer Überschrift gekennzeichnet ist (z. B. `Rolle:`, `Kompetenzen:`, `Beschreibung:`, `Referenzen:`, `Zertifikate:`).
|
||||
6. THE LLM_Fulltext_Matcher SHALL jeden Eintrag im Abschnitt `Referenzen:` so darstellen, dass sowohl Partner_Name als auch Projekte für das LLM sichtbar sind (z. B. im Format `Partner: <partner_name> – Projekte: <projects>` oder als gleichwertige strukturierte Darstellung mit benannten Feldern).
|
||||
7. THE LLM_Fulltext_Matcher SHALL die Reihenfolge der Felder in der serialisierten Darstellung über alle Kapazitäten konstant halten, sodass die LLM-Eingabe deterministisch ist.
|
||||
|
||||
### Anforderung 4: Aufbau des Task_Profile
|
||||
|
||||
**User Story:** Als Entwickler möchte ich, dass das System aus den Datenbankfeldern ein konsistentes Volltext-Profil pro Aufgabe erzeugt, damit das LLM eine einheitliche Eingabe erhält.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. THE LLM_Fulltext_Matcher SHALL pro Aufgabe ein Task_Profile bilden, das die folgenden Felder enthält: `id`, `title`, `description` und `skills` (gesuchte Kompetenzen).
|
||||
2. WHEN ein Feld in der Datenbank leer oder `None` ist, THE LLM_Fulltext_Matcher SHALL das entsprechende Feld im Task_Profile mit einer leeren Zeichenkette bzw. einer leeren Liste belegen.
|
||||
3. THE LLM_Fulltext_Matcher SHALL das Task_Profile in einer für das LLM lesbaren, deterministischen Textstruktur serialisieren, in der jedes Feld klar mit einer Überschrift gekennzeichnet ist (z. B. `Titel:`, `Beschreibung:`, `Gesuchte Kompetenzen:`).
|
||||
4. THE LLM_Fulltext_Matcher SHALL die Reihenfolge der Felder in der serialisierten Darstellung über alle Aufgaben konstant halten.
|
||||
|
||||
### Anforderung 5: LLM-Volltext-Matching für die Richtung Aufgabe→Kapazität
|
||||
|
||||
**User Story:** Als Nutzer möchte ich, dass `find_matching_capacities` mit `matching_method = "llm_fulltext"` einen LLM-basierten Volltext-Vergleich zwischen einem Task_Profile und allen Capacity_Profile-Einträgen durchführt, damit ich Kapazitäten über die rein lexikalische Kompetenzbetrachtung hinaus bewerten lassen kann.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. WHEN `find_matching_capacities` mit `matching_method = "llm_fulltext"` aufgerufen wird, THE LLM_Fulltext_Matcher SHALL für jede gefilterte Kapazität (gleicher Vorfilter wie beim Score-Matching, z. B. Verfügbarkeitsfilter) einen LLM-Vergleich zwischen Task_Profile und Capacity_Profile durchführen.
|
||||
2. WHEN `find_matching_capacities` mit `matching_method = "llm_fulltext"` aufgerufen wird, THE MCP_Server SHALL als Eingabe das aktuell bestätigte Anforderungs-Set (`role_name`, `competences`, optionale Beschreibung, Zeitraum) sowie ggf. die zugrunde liegende Aufgabe verwenden, um das Task_Profile zu bilden.
|
||||
3. THE LLM_Fulltext_Matcher SHALL pro Kapazität genau eine Kategorie aus der Menge `Top`, `Good`, `Partial`, `Low`, `Irrelevant` zurückgeben.
|
||||
4. THE LLM_Fulltext_Matcher SHALL pro Kapazität eine Rationale mit 1 bis 2 Sätzen zurückgeben, die die Zuweisung in die jeweilige Kategorie erläutert.
|
||||
5. THE LLM_Fulltext_Matcher SHALL die LLM-Antwort als strukturiertes JSON pro Kapazität anfordern und parsen (Felder: `category`, `rationale`).
|
||||
6. IF das LLM für eine Kapazität eine Kategorie zurückgibt, die nicht in der erlaubten Menge liegt, THEN THE LLM_Fulltext_Matcher SHALL diese Kapazität der Kategorie `Irrelevant` zuordnen und die Rationale durch einen Hinweis auf die ungültige LLM-Antwort ergänzen.
|
||||
7. IF der LLM-Aufruf für eine Kapazität fehlschlägt, THEN THE LLM_Fulltext_Matcher SHALL diese Kapazität in einer separaten Fehlerliste ausweisen und sie nicht als reguläres Ergebnis kategorisieren.
|
||||
8. THE LLM_Fulltext_Matcher SHALL die Ergebnisse nach Kategorie gruppieren und innerhalb jeder Kategorie eine deterministische Sortierreihenfolge anwenden (Sortierung primär nach Kategorie, sekundär nach `capacity_id` aufsteigend).
|
||||
|
||||
### Anforderung 6: LLM-Volltext-Matching für die Richtung Kapazität→Aufgabe
|
||||
|
||||
**User Story:** Als Nutzer möchte ich, dass `find_matching_tasks` mit `matching_method = "llm_fulltext"` einen LLM-basierten Volltext-Vergleich zwischen einem Capacity_Profile und allen Task_Profile-Einträgen durchführt, damit ich auch in dieser Suchrichtung das neue Verfahren nutzen kann.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. WHEN `find_matching_tasks` mit `matching_method = "llm_fulltext"` aufgerufen wird, THE LLM_Fulltext_Matcher SHALL für jede offene Aufgabe einen LLM-Vergleich zwischen Capacity_Profile und Task_Profile durchführen.
|
||||
2. WHEN `find_matching_tasks` mit `matching_method = "llm_fulltext"` aufgerufen wird, THE MCP_Server SHALL für die angegebene `capacity_id` Beschreibung, Zertifikate und Referenzen aus der Datenbank laden und in das Capacity_Profile einbeziehen.
|
||||
3. THE LLM_Fulltext_Matcher SHALL pro Aufgabe genau eine Kategorie aus der Menge `Top`, `Good`, `Partial`, `Low`, `Irrelevant` zurückgeben.
|
||||
4. THE LLM_Fulltext_Matcher SHALL pro Aufgabe eine Rationale mit 1 bis 2 Sätzen zurückgeben.
|
||||
5. THE LLM_Fulltext_Matcher SHALL die LLM-Antwort als strukturiertes JSON pro Aufgabe anfordern und parsen (Felder: `category`, `rationale`).
|
||||
6. IF das LLM für eine Aufgabe eine Kategorie zurückgibt, die nicht in der erlaubten Menge liegt, THEN THE LLM_Fulltext_Matcher SHALL diese Aufgabe der Kategorie `Irrelevant` zuordnen und die Rationale durch einen Hinweis auf die ungültige LLM-Antwort ergänzen.
|
||||
7. IF der LLM-Aufruf für eine Aufgabe fehlschlägt, THEN THE LLM_Fulltext_Matcher SHALL diese Aufgabe in einer separaten Fehlerliste ausweisen und sie nicht als reguläres Ergebnis kategorisieren.
|
||||
8. THE LLM_Fulltext_Matcher SHALL die Ergebnisse nach Kategorie gruppieren und innerhalb jeder Kategorie eine deterministische Sortierreihenfolge anwenden (Sortierung primär nach Kategorie, sekundär nach `task_id` aufsteigend).
|
||||
|
||||
### Anforderung 7: Direkte Kategorisierung ohne mathematisches Scoring
|
||||
|
||||
**User Story:** Als Nutzer möchte ich beim LLM-Volltext-Matching keine numerischen Score-Spalten mehr sehen, sondern ausschließlich die vom LLM zugewiesene Kategorie, damit das neue Verfahren als rein qualitative Bewertung erkennbar ist.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. WHEN `matching_method = "llm_fulltext"` verwendet wird, THE MCP_Server SHALL in den Ergebnistabellen keine Spalten `Role Score`, `Competence Score` oder `Overall Score` ausgeben.
|
||||
2. WHEN `matching_method = "llm_fulltext"` verwendet wird, THE MCP_Server SHALL für jeden Treffer ausschließlich die LLM-zugewiesene Kategorie als Bewertungsfeld ausweisen.
|
||||
3. WHEN `matching_method = "score"` verwendet wird, THE MCP_Server SHALL die bestehenden Score-Spalten unverändert ausgeben.
|
||||
4. THE LLM_Fulltext_Matcher SHALL für jedes Ergebnis ein Datenfeld `category` (String) und ein Datenfeld `rationale` (String) im gespeicherten Suchergebnis (`SearchCache`) hinterlegen, ohne numerische Scores zu schreiben.
|
||||
5. THE MCP_Server SHALL die Summary-Tabelle im LLM-Volltext-Modus weiterhin als Zähler je Kategorie (`Top`, `Good`, `Partial`, `Low`, `Irrelevant`) ausgeben.
|
||||
|
||||
### Anforderung 8: Begründungsspalte (Rationale) in der Ausgabe
|
||||
|
||||
**User Story:** Als Nutzer möchte ich in der Ergebnistabelle des LLM-Volltext-Matchings eine zusätzliche Spalte sehen, die in 1–2 Sätzen erläutert, warum eine Kapazität bzw. Aufgabe in der jeweiligen Kategorie gelandet ist, damit ich die Entscheidung des LLM nachvollziehen kann.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. WHEN `matching_method = "llm_fulltext"` verwendet wird, THE MCP_Server SHALL die Ergebnistabellen für `find_matching_capacities` und `find_matching_tasks` um eine Spalte `Begründung` (Rationale) erweitern.
|
||||
2. THE MCP_Server SHALL die Spalte `Begründung` direkt rechts neben der Spalte `Category` einfügen.
|
||||
3. THE MCP_Server SHALL pro Zeile genau die vom LLM zurückgegebene Rationale (1–2 Sätze) anzeigen.
|
||||
4. WHEN die Rationale Zeilenumbrüche oder Pipe-Zeichen enthält, THE MCP_Server SHALL diese so escapen oder ersetzen, dass die Markdown-Tabelle gültig bleibt.
|
||||
5. WHEN die Rationale länger als 280 Zeichen ist, THE MCP_Server SHALL die Rationale auf 280 Zeichen kürzen und ein abschließendes Auslassungszeichen (`…`) anhängen, damit die Tabellendarstellung lesbar bleibt.
|
||||
6. THE MCP_Server SHALL die ungekürzte Rationale im persistierten Suchergebnis (`SearchCache`) speichern, sodass nachgelagerte Tools (`get_results_by_category`, `filter_search_results`) den vollständigen Text ausgeben können.
|
||||
|
||||
### Anforderung 9: Kompatibilität mit Refinement- und Pagination-Tools
|
||||
|
||||
**User Story:** Als Nutzer möchte ich auch beim LLM-Volltext-Matching durch Kategorien blättern und Filter anwenden können, damit der bestehende Such-Workflow konsistent bleibt.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. WHEN `matching_method = "llm_fulltext"` verwendet wird, THE MCP_Server SHALL ein gültiges `search_id` zurückgeben, das mit `get_results_by_category` und `filter_search_results` verwendet werden kann.
|
||||
2. WHEN `get_results_by_category` ein Suchergebnis aus dem LLM-Volltext-Modus paginiert, THE MCP_Server SHALL die Ergebnistabelle ohne Score-Spalten und mit der Spalte `Begründung` ausgeben.
|
||||
3. WHEN `filter_search_results` ein Suchergebnis aus dem LLM-Volltext-Modus filtert, THE MCP_Server SHALL die Filterung ausschließlich auf nicht-Score-basierten Filtern (Rollenfilter, Kompetenzfilter, Verfügbarkeitsfilter, Aufgaben-Textfilter, Aufgaben-Kompetenzfilter) durchführen.
|
||||
4. IF ein Score-bezogener Filter (z. B. `min_similarity`) auf ein LLM-Volltext-Suchergebnis angewendet wird, THEN THE MCP_Server SHALL den Filter ignorieren und in der `Applied Filters`-Tabelle einen Hinweis aufnehmen, dass der Filter im LLM-Volltext-Modus nicht wirksam ist.
|
||||
5. THE MCP_Server SHALL im `META`-JSON des Suchergebnisses das verwendete Verfahren als `matching_method` ausweisen, damit Folgewerkzeuge das Schema korrekt interpretieren können.
|
||||
|
||||
### Anforderung 10: Anpassung der Copilot-Agent-Konfiguration
|
||||
|
||||
**User Story:** Als Nutzer möchte ich, dass sowohl der GitHub-Copilot-Agent `teamlandkarte_agent` als auch der Kiro-Pendant-Agent das neue Matching-Verfahren kennen und mich aktiv nach dem gewünschten Verfahren fragen, damit das neue Feature über die Agenten nutzbar ist.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. THE Teamlandkarte_Agent SHALL in seiner Konfigurationsdatei (`.github/agents/teamlandkarte_agent.md`) und im Pendant `.kiro/agents/teamlandkarte.md` die Existenz und den Zweck der beiden Verfahren `score` und `llm_fulltext` dokumentieren.
|
||||
2. WHEN der Nutzer eine Suche nach passenden Kapazitäten oder Aufgaben startet, THE Teamlandkarte_Agent SHALL den Nutzer explizit nach dem gewünschten `matching_method` (Score-basiert oder LLM-Volltext) fragen, sofern dieses nicht bereits aus dem Verlauf hervorgeht.
|
||||
3. THE Teamlandkarte_Agent SHALL die Skills/Workflows so erweitern, dass `find_matching_capacities` und `find_matching_tasks` mit dem zusätzlichen Parameter `matching_method` aufgerufen werden.
|
||||
4. THE Teamlandkarte_Agent SHALL die Rolle der Spalte `Begründung` im Output dokumentieren und in den Hinweisen erwähnen, dass im LLM-Volltext-Modus keine numerischen Scores erscheinen.
|
||||
5. THE Teamlandkarte_Agent SHALL den bestehenden Bestätigungs-Workflow (`show_pending_requirements`, `confirm_requirements`) beibehalten und für beide Verfahren gleich anwenden.
|
||||
|
||||
### Anforderung 11: Aktualisierung von Architektur- und README-Dokumentation
|
||||
|
||||
**User Story:** Als Entwickler oder Onboardee möchte ich, dass `architecture.md` und `README.md` das neue Matching-Verfahren beschreiben, damit ich Architektur und Nutzung des Systems korrekt verstehe.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. THE Architecture_Doc SHALL einen Abschnitt enthalten, der den LLM_Fulltext_Matcher als Komponente innerhalb der Business-Logic-Layer beschreibt, einschließlich seiner Eingaben, Ausgaben und externen Abhängigkeiten (Azure OpenAI Chat Completion).
|
||||
2. THE Architecture_Doc SHALL die zusätzlichen Datenquellen (`teamlandkarte_v_capacities_latest.description`, `teamlandkarte_v_capacity_certificates_latest`, `teamlandkarte_v_capacity_references_latest`, `teamlandkarte_v_partners_latest`) im Datenmodell- und Schema-Verifikationsabschnitt aufführen.
|
||||
3. THE Architecture_Doc SHALL die Verknüpfung zwischen `teamlandkarte_v_capacity_references_latest.partner_id` und `teamlandkarte_v_partners_latest.id` sowie die Übernahme der Spalte `name` als Partner_Name in das Capacity_Profile dokumentieren.
|
||||
4. THE Architecture_Doc SHALL den neuen Parameter `matching_method` und seine Wertebereiche im Tool-Surface-Abschnitt für `find_matching_capacities` und `find_matching_tasks` dokumentieren.
|
||||
5. THE Architecture_Doc SHALL den Runtime-View für beide Suchrichtungen um den LLM-Volltext-Pfad ergänzen.
|
||||
6. THE Readme SHALL im Quick-Start- und Usage-Abschnitt erklären, wie der Nutzer zwischen `score` und `llm_fulltext` wählt.
|
||||
7. THE Readme SHALL beschreiben, dass im LLM-Volltext-Modus keine numerischen Scores ausgegeben werden und stattdessen eine Spalte `Begründung` erscheint.
|
||||
8. THE Readme SHALL die zusätzlichen Datenbank-Views aufführen, die der Server im LLM-Volltext-Modus liest, einschließlich `teamlandkarte_v_partners_latest` und der Verknüpfung zu Capacity_Reference über `partner_id`.
|
||||
|
||||
### Anforderung 12: Anpassung weiterer Skripte und Tools
|
||||
|
||||
**User Story:** Als Entwickler möchte ich, dass alle relevanten Hilfsskripte und MCP-Tools mit dem neuen Verfahren konsistent zusammenarbeiten, damit es keine Inkonsistenzen zwischen Server, Agent und Skripten gibt.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. THE MCP_Server SHALL den Parameter `matching_method` in allen Docstrings der betroffenen Tools (`find_matching_capacities`, `find_matching_tasks`, ggf. `filter_search_results`, `get_results_by_category`) dokumentieren.
|
||||
2. THE MCP_Server SHALL die Konfigurationsdatei `config.toml` um einen optionalen Schlüssel `matching.default_method` erweitern, der den Standardwert für `matching_method` beim Server-Start festlegt.
|
||||
3. WHEN `matching.default_method` in `config.toml` nicht gesetzt ist, THE MCP_Server SHALL den Default-Wert `score` verwenden.
|
||||
4. IF `matching.default_method` einen anderen Wert als `score` oder `llm_fulltext` enthält, THEN THE MCP_Server SHALL beim Start einen `ConfigError` mit beschreibender Meldung werfen.
|
||||
5. THE MCP_Server SHALL alle bestehenden Tests so erweitern oder ergänzen, dass sowohl der Modus `score` als auch der Modus `llm_fulltext` (mit gemocktem LLM) abgedeckt sind.
|
||||
@@ -1,384 +0,0 @@
|
||||
# Implementation Plan: LLM-Volltext-Matching als zweites Verfahren
|
||||
|
||||
## Overview
|
||||
|
||||
Inkrementelle Einführung des neuen `llm_fulltext`-Matching-Verfahrens neben dem bestehenden `score`-Verfahren. Die Reihenfolge stellt sicher, dass keine Zwischenstände kaputt sind: zuerst Datenbankschicht (DBClient/TrinoClient) erweitern, dann Datenmodelle (`CapacityProfile`/`TaskProfile`) und der `LlmFulltextMatcher`, anschließend Konfiguration und MCP-Tool-Integration, am Ende Cache-/Tabellenausgabe sowie Anpassungen an `get_results_by_category`/`filter_search_results` und Dokumentation.
|
||||
|
||||
## Tasks
|
||||
|
||||
- [x] 1. Datenbankschicht für Capacity-Volltext-Felder erweitern
|
||||
- [x] 1.1 Neue Methoden im `DBClient`-Protokoll deklarieren (`database/types.py`)
|
||||
- `CapacityReferenceRow` als `TypedDict` mit Feldern `partner_name: str` und `projects: str` im Protokoll-Modul (bzw. unter `database/types.py`) deklarieren; `partner_name` darf leer sein (NULL `partner_id` oder Join-Mismatch, Anforderung 2.8)
|
||||
- `get_capacity_description(capacity_id) -> str | None`
|
||||
- `get_capacity_certificates(capacity_id) -> list[str]`
|
||||
- `get_capacity_references(capacity_id) -> list[CapacityReferenceRow]`
|
||||
- `batch_get_capacity_descriptions(capacity_ids) -> dict[str, str | None]`
|
||||
- `batch_get_capacity_certificates(capacity_ids) -> dict[str, list[str]]`
|
||||
- `batch_get_capacity_references(capacity_ids) -> dict[str, list[CapacityReferenceRow]]`
|
||||
- Docstrings gemäß Design (Quelle, Join-Verhalten inkl. Partner-Join `partner_id = id`, Rückgabetyp bei leerer Eingabe, leerer `partner_name` bei NULL/Mismatch)
|
||||
- _Requirements: 2.1, 2.2, 2.3, 2.4, 2.8_
|
||||
|
||||
- [x] 1.2 Einzel-Methoden im `TrinoClient` implementieren (`database/trino_client.py`)
|
||||
- SELECT-Only-Statements mit `_ensure_select_only`, Connection-Pool und `_retry`
|
||||
- `get_capacity_description`: NULL/leerer String → `None`
|
||||
- `get_capacity_certificates`: Join über `capacity_id`, leere Strings filtern, Rückgabe `[]` wenn keine Treffer
|
||||
- `get_capacity_references`: zusätzlich `LEFT JOIN teamlandkarte_v_partners_latest p ON r.partner_id = p.id` und `COALESCE(p.name, '') AS partner_name` in derselben Abfrage; Rückgabe als `CapacityReferenceRow`-Liste mit `partner_name` und `projects`
|
||||
- NULL `partner_id` bzw. Join-Mismatch → leerer `partner_name`, Referenz bleibt mit `projects` erhalten (Anforderung 2.8)
|
||||
- Parameter-Bindung gegen SQL-Injection
|
||||
- _Requirements: 2.1, 2.2, 2.3, 2.5, 2.6, 2.7, 2.8, 2.9, 2.10_
|
||||
|
||||
- [x] 1.3 Batch-Methoden im `TrinoClient` implementieren
|
||||
- Genau **eine** SQL-Abfrage pro Quelle (Anforderung 2.4); der Partner-LEFT-JOIN ist Bestandteil derselben Referenz-Abfrage und erzeugt keinen zusätzlichen Roundtrip
|
||||
- `batch_get_capacity_references`: `SELECT r.capacity_id, r.projects, COALESCE(p.name, '') AS partner_name ... LEFT JOIN teamlandkarte_v_partners_latest p ON r.partner_id = p.id`
|
||||
- Schlüssel der Rückgaben sind `str(capacity_id)` (deterministisch)
|
||||
- Fehlende IDs als `None` (Beschreibungen) bzw. `[]` (Listen) vorbelegen
|
||||
- Gruppierung n:1 via `defaultdict(list)`, leere `projects`-Strings filtern; leerer `partner_name` (NULL/Mismatch) führt nicht zum Verwerfen der Referenz
|
||||
- Frühe Rückkehr bei leerer Eingabeliste (`{}`)
|
||||
- _Requirements: 2.4, 2.5, 2.6, 2.7, 2.8, 2.9, 2.10_
|
||||
|
||||
- [x] 1.4 Unit-Tests für die neuen DB-Methoden mit Mock-Cursor*
|
||||
- Genau eine SQL-Abfrage je Methode wird abgesetzt; LEFT JOIN auf `teamlandkarte_v_partners_latest` ist Bestandteil derselben Referenz-Abfrage (kein zusätzlicher Roundtrip)
|
||||
- SELECT-Only-Guard wird angewendet
|
||||
- Korrekte Gruppierung n:1 in den Batch-Varianten
|
||||
- Mock-Cursor liefert für `references` drei Spalten (`capacity_id`, `projects`, `partner_name`); Test inkl. Fall NULL `partner_id` → leerer `partner_name`, Referenz bleibt erhalten
|
||||
- Defaults für fehlende IDs (None / [])
|
||||
- _Requirements: 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 2.8_
|
||||
|
||||
- [x] 2. Datenmodelle `CapacityProfile` und `TaskProfile`
|
||||
- [x] 2.1 Neue frozen Dataclasses anlegen (`matching/profiles.py`)
|
||||
- `CapacityReferenceEntry` als `@dataclass(frozen=True)` mit Feldern `partner_name: str` und `projects: str` (`partner_name` darf leer sein, vgl. Anforderung 2.8 / 3.2)
|
||||
- `CapacityProfile` mit Feldern `id`, `owner_name`, `role_name`, `competences`, `description`, `references: list[CapacityReferenceEntry]`, `certificates`
|
||||
- `TaskProfile` mit Feldern `id`, `title`, `description`, `skills`
|
||||
- Beide `@dataclass(frozen=True)` und nur primitive Felder bzw. `list[str]` / `list[CapacityReferenceEntry]`
|
||||
- _Requirements: 3.1, 3.2, 4.1_
|
||||
|
||||
- [x] 2.2 Profil-Builder implementieren (`matching/profiles.py`)
|
||||
- `build_capacity_profile(capacity, *, description, certificates, references) -> CapacityProfile`
|
||||
- Konvertiere `CapacityReferenceRow`-Einträge aus dem DBClient in `CapacityReferenceEntry`-Instanzen; ein leerer `partner_name` führt nicht zum Verwerfen der Referenz (Anforderung 3.4)
|
||||
- `build_task_profile(task) -> TaskProfile` (alternativ `build_task_profile_from_requirements(...)` für Suchrichtung Aufgabe→Kapazität)
|
||||
- Leere/`None`-Felder → leerer String bzw. leere Liste, Profil wird nie verworfen
|
||||
- Reihenfolge der Felder ist über alle Profile konstant
|
||||
- _Requirements: 3.1, 3.2, 3.4, 4.1, 4.2_
|
||||
|
||||
- [x] 2.3 Deterministische Profil-Serialisierung implementieren
|
||||
- `serialize_capacity_profile(profile) -> str` mit fixer Reihenfolge `Rolle:`, `Kompetenzen:`, `Beschreibung:`, `Referenzen:`, `Zertifikate:`
|
||||
- `_format_reference(entry: CapacityReferenceEntry) -> str`: bei nicht-leerem `partner_name` → `Partner: <partner_name> – Projekte: <projects>`; bei leerem `partner_name` → `Projekte: <projects>` (kein Platzhalter, kein `Partner:`-Token)
|
||||
- `serialize_task_profile(profile) -> str` mit fixer Reihenfolge `Titel:`, `Beschreibung:`, `Gesuchte Kompetenzen:`
|
||||
- Leere Listen werden als `(keine)` ausgegeben, alle Feldüberschriften erscheinen immer; Reihenfolge der Felder und Referenzen bleibt deterministisch
|
||||
- _Requirements: 3.3, 3.4, 3.6, 3.7, 4.3, 4.4_
|
||||
|
||||
- [x] 2.4 Property-Test für deterministische Serialisierung schreiben
|
||||
- **Property 1: Profil-Serialisierung ist deterministisch und feldvollständig**
|
||||
- Hypothesis mit `@settings(max_examples=100)`, `st.builds(...)` für Profile inkl. `CapacityReferenceEntry` mit/ohne `partner_name`
|
||||
- Zwei Aufrufe liefern identischen String; alle Feldüberschriften enthalten; Partner-Name erscheint deterministisch in der serialisierten Ausgabe, sofern nicht leer
|
||||
- **Validates: Requirements 3.3, 3.4, 3.6, 3.7, 4.3, 4.4**
|
||||
|
||||
- [x] 2.5 Property-Test für Profil-Builder mit leeren Feldern schreiben
|
||||
- **Property 2: Leere/None-Felder verwerfen das Profil nicht**
|
||||
- Capacity-/Task-Strategien mit `st.one_of(st.none(), st.text())`; Referenzen-Strategie mit gemischtem `partner_name` (`st.one_of(st.just(""), st.text(min_size=1))`)
|
||||
- Profil enthält leere Strings/Listen; Serialisierung enthält weiterhin alle Überschriften; auch Referenzen mit leerem `partner_name` werden nicht verworfen
|
||||
- **Validates: Requirements 3.2, 3.4, 4.2**
|
||||
|
||||
- [x] 2.6 Property-Test für Referenzen mit leerem Partner-Name schreiben
|
||||
- **Property 2b: Referenzen mit leerem Partner_Name behalten projects, ohne Partner-Token**
|
||||
- Hypothesis-Generator mischt leere und nicht-leere `partner_name`-Werte: `st.lists(st.builds(CapacityReferenceEntry, partner_name=st.one_of(st.just(""), st.text(min_size=1)), projects=st.text(min_size=1)))`
|
||||
- Prüft (a) die Anzahl der Referenz-Zeilen entspricht der Anzahl der Einträge mit nicht-leerem `projects`, (b) jede Zeile zu einem Eintrag mit leerem `partner_name` beginnt mit `Projekte:` und enthält keinen `Partner:`-Token, (c) jede Zeile zu einem Eintrag mit nicht-leerem `partner_name` enthält sowohl `Partner: <name>` als auch `Projekte: <projects>`
|
||||
- **Validates: Requirements 2.8, 3.4, 3.6**
|
||||
|
||||
- [x] 3. `LlmFulltextMatcher`-Komponente bauen
|
||||
- [x] 3.1 Modul-Skelett `matching/llm_fulltext_matcher.py` anlegen
|
||||
- Konstanten `_ALLOWED_CATEGORIES = ("Top", "Good", "Partial", "Low", "Irrelevant")` und `_ALIAS`
|
||||
- Dataclasses `LlmFulltextItem`, `LlmFulltextError`, `LlmFulltextResult`
|
||||
- Klasse `LlmFulltextMatcher` mit Konstruktor `(*, db, client, rationale_max_chars=280)`
|
||||
- _Requirements: 5.1, 5.3, 6.1, 6.3, 7.4_
|
||||
|
||||
- [x] 3.2 Kategorie-Normalisierung implementieren
|
||||
- `normalize_category(value) -> tuple[str, bool]`
|
||||
- Trim + lowercase, Mapping über `_ALIAS`, ungültige Werte → `("Irrelevant", False)`
|
||||
- Nicht-String-Eingaben → `("Irrelevant", False)`
|
||||
- _Requirements: 5.3, 5.6, 6.3, 6.6_
|
||||
|
||||
- [x] 3.3 Property-Test für Kategorie-Normalisierung schreiben
|
||||
- **Property 3: Kategorienormalisierung bildet auf erlaubte Menge ab**
|
||||
- Hypothesis mit `st.text()` und Aliase via `st.sampled_from`
|
||||
- Output immer in erlaubter Menge; `is_valid` korrekt
|
||||
- **Validates: Requirements 5.3, 5.6, 6.3, 6.6**
|
||||
|
||||
- [x] 3.4 LLM-Aufruf und Antwort-Parsing implementieren
|
||||
- System-Prompt gemäß Design (deutschsprachig, deterministisch, JSON-only)
|
||||
- User-Prompt baut auf `serialize_task_profile`/`serialize_capacity_profile` auf
|
||||
- `chat_completion(system, user, response_format=json_object)` aufrufen
|
||||
- JSON parsen, Felder `category` und `rationale` extrahieren
|
||||
- Bei `JSONDecodeError` → `LlmFulltextError` mit Meldung `"invalid JSON: <excerpt>"`
|
||||
- Bei ungültiger Kategorie → Item nach `Irrelevant` mit Hinweis `[Hinweis: ungültige LLM-Kategorie: <wert>]` an Rationale anhängen
|
||||
- Bei `AzureAPIError`/Timeout/sonstiger Exception → `LlmFulltextError` mit Klassenname + erstem Satz
|
||||
- _Requirements: 5.4, 5.5, 5.6, 5.7, 6.4, 6.5, 6.6, 6.7_
|
||||
|
||||
- [x] 3.5 `match_capacities` implementieren
|
||||
- Eingabe: `task_profile`, `capacities` (bereits vorgefiltert)
|
||||
- Capacity-IDs sammeln, `batch_get_capacity_descriptions/certificates/references` aufrufen
|
||||
- Pro Kapazität `CapacityProfile` bauen und LLM-Aufruf durchführen
|
||||
- Erfolgreiche Items in `by_category` einsortieren, fehlerhafte in `errors`
|
||||
- Innerhalb jeder Kategorie deterministisch nach `item_id` aufsteigend (lexikographisch) sortieren
|
||||
- _Requirements: 5.1, 5.2, 5.3, 5.4, 5.5, 5.7, 5.8_
|
||||
|
||||
- [x] 3.6 `match_tasks` implementieren
|
||||
- Eingabe: `capacity_profile`, `tasks`
|
||||
- Pro Aufgabe `TaskProfile` bauen und LLM-Aufruf durchführen
|
||||
- Sortierung primär nach Kategorie, sekundär nach `task_id` aufsteigend
|
||||
- _Requirements: 6.1, 6.2, 6.3, 6.4, 6.5, 6.7, 6.8_
|
||||
|
||||
- [x] 3.7 Property-Test für ungültige LLM-Kategorie schreiben
|
||||
- **Property 4: Ungültige LLM-Kategorie wird auf Irrelevant gemappt**
|
||||
- Mock-LLM gibt zufällige ungültige Kategorie zurück
|
||||
- Item landet in `Irrelevant`, Rationale enthält Originaltext + Hinweis
|
||||
- **Validates: Requirements 5.6, 6.6**
|
||||
|
||||
- [x] 3.8 Property-Test für LLM-Fehler-Trennung schreiben
|
||||
- **Property 5: LLM-Fehler erscheinen in der Fehlerliste, nicht als Ergebnis**
|
||||
- Hypothesis-Strategy `st.lists(st.booleans())` als Fehlermaske
|
||||
- Items aus Fehlermaske erscheinen ausschließlich in `result.errors`
|
||||
- **Validates: Requirements 5.7, 6.7**
|
||||
|
||||
- [x] 3.9 Property-Test für deterministische Sortierung schreiben
|
||||
- **Property 6: Ergebnisse sind innerhalb jeder Kategorie deterministisch sortiert**
|
||||
- Permutationen der Eingabeliste erzeugen identische `by_category`-Reihenfolge
|
||||
- **Validates: Requirements 5.8, 6.8**
|
||||
|
||||
- [x] 3.10 Unit-Tests für `LlmFulltextMatcher` mit gemocktem `AzureOpenAIClient`*
|
||||
- Erfolgsfall (gültige Kategorie + Rationale)
|
||||
- Ungültiges JSON
|
||||
- Gültiges JSON mit unbekannter Kategorie
|
||||
- `chat_completion` wirft `AzureAPIError`
|
||||
- Vermischung mehrerer Items: Reihenfolge und Sortierung korrekt
|
||||
- _Requirements: 5.4, 5.5, 5.6, 5.7, 5.8, 6.4, 6.5, 6.6, 6.7, 6.8_
|
||||
|
||||
- [x] 4. Checkpoint - Ensure all tests pass
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
- [x] 5. Konfiguration `matching.default_method` einführen
|
||||
- [x] 5.1 `MatchingConfig` um Feld `default_method: str = "score"` erweitern (`config.py`)
|
||||
- Validierung in `load_config`: Wert muss `"score"` oder `"llm_fulltext"` sein
|
||||
- Bei ungültigem Wert `ConfigError` mit beschreibender Meldung werfen
|
||||
- Fehlender Schlüssel → Default `"score"`
|
||||
- _Requirements: 12.2, 12.3, 12.4_
|
||||
|
||||
- [x] 5.2 `config.toml` und `config.toml.example` aktualisieren
|
||||
- Neuer Abschnitt `[matching].default_method = "score"` mit Kommentar zu erlaubten Werten
|
||||
- _Requirements: 12.2, 12.3_
|
||||
|
||||
- [x] 5.3 Unit-Tests für Konfigurations-Validierung*
|
||||
- `default_method` fehlt → Default `"score"`
|
||||
- `default_method = "llm_fulltext"` → übernommen
|
||||
- `default_method = "irgendwas"` → `ConfigError`
|
||||
- _Requirements: 12.3, 12.4_
|
||||
|
||||
- [x] 6. MCP-Tool-Integration für `matching_method`
|
||||
- [x] 6.1 Hilfsfunktion `_resolve_matching_method` im `mcp_server.py` implementieren
|
||||
- `None` → `cfg.matching.default_method`
|
||||
- Trim + lowercase, gegen `("score", "llm_fulltext")` validieren
|
||||
- Bei ungültigem Wert `ValueError` mit beiden erlaubten Werten in der Meldung werfen
|
||||
- _Requirements: 1.1, 1.2, 1.5, 12.3_
|
||||
|
||||
- [x] 6.2 `find_matching_capacities` um `matching_method` erweitern
|
||||
- Neuer Parameter `matching_method: Optional[str] = None`
|
||||
- Validierung **vor** DB-/LLM-Aufrufen, bei Fehler frühe Markdown-Fehlermeldung mit erlaubten Werten zurückgeben
|
||||
- Routing: `"score"` → bestehender `Matcher`-Pfad; `"llm_fulltext"` → `LlmFulltextMatcher.match_capacities`
|
||||
- Im LLM-Modus Vorfilter (z. B. Verfügbarkeit) identisch zum Score-Modus anwenden
|
||||
- Task_Profile aus aktuell bestätigtem Anforderungs-Set bauen
|
||||
- _Requirements: 1.1, 1.3, 1.4, 1.5, 5.1, 5.2_
|
||||
|
||||
- [x] 6.3 `find_matching_tasks` um `matching_method` erweitern
|
||||
- Neuer Parameter `matching_method: Optional[str] = None`
|
||||
- Routing analog 6.2
|
||||
- Im LLM-Modus für die `capacity_id` Beschreibung, Zertifikate und Referenzen aus DB laden und ins `CapacityProfile` einbeziehen
|
||||
- _Requirements: 1.1, 1.3, 1.4, 1.5, 6.1, 6.2_
|
||||
|
||||
- [x] 6.4 Docstrings für betroffene MCP-Tools aktualisieren
|
||||
- `find_matching_capacities`, `find_matching_tasks`: Parameter `matching_method` mit erlaubten Werten dokumentieren
|
||||
- Hinweis auf Unterschiede in Ausgabe-Schema (keine Score-Spalten im LLM-Modus, `Begründung`-Spalte)
|
||||
- _Requirements: 12.1_
|
||||
|
||||
- [x] 6.5 Property-Test für `matching_method`-Validierung schreiben
|
||||
- **Property 9: matching_method-Validierung lehnt unbekannte Werte ab**
|
||||
- Hypothesis-Strategy `st.text()`
|
||||
- Bei ungültiger Eingabe enthält die Fehlermeldung beide erlaubten Werte; DB- und LLM-Mocks werden nicht aufgerufen
|
||||
- **Validates: Requirements 1.5**
|
||||
|
||||
- [x] 6.6 Unit-Tests für Routing in beiden MCP-Tools
|
||||
- `matching_method=None` → Default greift, Schema entspricht Score-Modus
|
||||
- `matching_method="llm_fulltext"` → `LlmFulltextMatcher` wird aufgerufen, Score-Pfad nicht
|
||||
- `matching_method="bogus"` → Fehlermeldung, weder DB noch LLM werden aufgerufen
|
||||
- _Requirements: 1.1, 1.2, 1.3, 1.4, 1.5_
|
||||
|
||||
- [x] 7. SearchCache-Payload und META-JSON
|
||||
- [x] 7.1 Persistenz im SearchCache an LLM-Modus anpassen
|
||||
- `matching_method` als Schlüssel im Payload aufnehmen (auch im Score-Modus)
|
||||
- Im LLM-Modus pro Item Felder `category` und `rationale` (ungekürzt) speichern, **keine** `role_score`/`competence_score`/`overall_score`
|
||||
- Optionale `errors`-Liste mit `{"item_id", "error"}` im LLM-Modus
|
||||
- Im Score-Modus bisheriges Schema unverändert (keine `rationale`/`errors`)
|
||||
- _Requirements: 7.1, 7.4, 8.6, 9.5_
|
||||
|
||||
- [x] 7.2 META-JSON in Tool-Antworten um `matching_method` erweitern
|
||||
- In beiden Modi setzen, sodass Folgewerkzeuge das Schema korrekt interpretieren
|
||||
- _Requirements: 1.6, 9.5_
|
||||
|
||||
- [x] 7.3 Property-Test für persistierte ungekürzte Rationale schreiben
|
||||
- **Property 8: Ungekürzte Rationale wird persistiert**
|
||||
- Hypothesis `st.text(min_size=300)`, Cache-Eintrag muss exakt der LLM-Antwort entsprechen
|
||||
- **Validates: Requirements 8.6**
|
||||
|
||||
- [x] 7.4 Property-Test für META-`matching_method` schreiben
|
||||
- **Property 11: META enthält das verwendete Verfahren**
|
||||
- Aus Tool-Output META-JSON parsen und Wert prüfen
|
||||
- **Validates: Requirements 1.6, 9.5**
|
||||
|
||||
- [x] 7.5 Property-Test für Score-Modus-Abwärtskompatibilität schreiben
|
||||
- **Property 10: Score-Modus ist abwärtskompatibel**
|
||||
- Tool-Aufruf ohne `matching_method` bzw. mit `"score"`: Persistiertes Payload-Schema entspricht weiter dem bisherigen (Felder `role_score`, `competence_score`, `overall_score`; kein `rationale`)
|
||||
- **Validates: Requirements 1.2, 1.3, 7.3**
|
||||
|
||||
- [x] 8. Tabellenausgabe mit `Begründung`-Spalte und Rationale-Formatierung
|
||||
- [x] 8.1 Hilfsfunktion `_format_rationale_for_table` implementieren
|
||||
- Pipes (`|`) → `/`, Carriage Return / Linefeed → Leerzeichen
|
||||
- Whitespace zusammenfalten
|
||||
- Bei > 280 Zeichen kürzen und mit `…` abschließen
|
||||
- _Requirements: 8.4, 8.5_
|
||||
|
||||
- [x] 8.2 Ausgabe-Tabellen für `find_matching_capacities` (LLM-Modus) anpassen
|
||||
- Spalten: `ID | Owner | Role | Competences | Availability | Category | Begründung`
|
||||
- **Keine** Spalten `Role Score`, `Competence Score`, `Overall Score`
|
||||
- Pro Zeile gekürzte Rationale via `_format_rationale_for_table`
|
||||
- Leere Ergebnistabelle weiterhin korrekt rendern (Header-Konsistenz)
|
||||
- _Requirements: 7.1, 7.2, 8.1, 8.2, 8.3, 8.4, 8.5_
|
||||
|
||||
- [x] 8.3 Ausgabe-Tabellen für `find_matching_tasks` (LLM-Modus) anpassen
|
||||
- Spalten: `task_id | Title | Required Competences | Availability | Category | Begründung`
|
||||
- Verhalten analog 8.2
|
||||
- _Requirements: 7.1, 7.2, 8.1, 8.2, 8.3, 8.4, 8.5_
|
||||
|
||||
- [x] 8.4 Summary-Tabelle und Errors-Block einbauen
|
||||
- Summary bleibt in beiden Modi identisch (Counter je Kategorie)
|
||||
- Wenn `errors` nicht leer: zusätzlicher Markdown-Block `## Errors` nach der Ergebnistabelle
|
||||
- _Requirements: 5.7, 6.7, 7.5_
|
||||
|
||||
- [x] 8.5 Score-Modus-Tabelle unverändert lassen (Regression)
|
||||
- Bestehende Spalten `Role Score`, `Competence Score`, `Overall Score`, `Category` bleiben
|
||||
- Keine `Begründung`-Spalte
|
||||
- _Requirements: 7.3_
|
||||
|
||||
- [x] 8.6 Property-Test für `_format_rationale_for_table` schreiben
|
||||
- **Property 7: Tabellen-Rationale ist gültiges Markdown und längenbegrenzt**
|
||||
- Hypothesis erzeugt Strings inkl. `|`, `\n`, sehr lang
|
||||
- Output ≤ 280 Zeichen, weder `|` noch Zeilenumbrüche, `…`-Suffix genau dann, wenn normalisierte Eingabe > 280 Zeichen war
|
||||
- **Validates: Requirements 8.4, 8.5**
|
||||
|
||||
- [x] 9. Checkpoint - Ensure all tests pass
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
- [x] 10. Anpassungen für `get_results_by_category`
|
||||
- [x] 10.1 Hilfsfunktion `_format_results_table(items, *, search_type, matching_method, ref_start, ref_end)` einführen
|
||||
- Routing nach `matching_method`: Score-Tabelle (bisher) oder LLM-Tabelle mit `Begründung`-Spalte
|
||||
- Wiederverwendbar in `find_matching_capacities`, `find_matching_tasks` und `get_results_by_category`
|
||||
- _Requirements: 7.1, 7.2, 8.1, 8.2, 9.2_
|
||||
|
||||
- [x] 10.2 `get_results_by_category` an LLM-Modus anpassen
|
||||
- `matching_method` aus persistiertem SearchEntry lesen
|
||||
- Im LLM-Modus Tabelle ohne Score-Spalten und mit `Begründung`-Spalte rendern
|
||||
- Im Score-Modus unverändertes Verhalten
|
||||
- META-JSON enthält `matching_method`
|
||||
- _Requirements: 9.1, 9.2, 9.5_
|
||||
|
||||
- [x] 10.3 Docstring für `get_results_by_category` aktualisieren
|
||||
- Hinweis auf modusabhängige Spalten (`Begründung` im LLM-Modus)
|
||||
- _Requirements: 12.1_
|
||||
|
||||
- [x] 11. Anpassungen für `filter_search_results`
|
||||
- [x] 11.1 Modus-Erkennung über persistiertes `matching_method`
|
||||
- Im LLM-Modus bestehende Filter (Rollen-, Kompetenz-, Verfügbarkeits-, Aufgaben-Text-/Kompetenzfilter) weiterhin anwenden
|
||||
- Score-bezogenen Filter `min_similarity` ignorieren und in `Applied Filters` einen Hinweis aufnehmen (`min_similarity (ignored: not applicable in llm_fulltext mode)`)
|
||||
- _Requirements: 9.3, 9.4_
|
||||
|
||||
- [x] 11.2 Sortierung in `filter_search_results` modusabhängig
|
||||
- Score-Modus: bestehende Sortierung nach `overall_score`
|
||||
- LLM-Modus: stabile Sortierung nach `(category_rank, item_id)`
|
||||
- _Requirements: 5.8, 6.8, 9.3_
|
||||
|
||||
- [x] 11.3 Tabellen-Rendering in `filter_search_results` an `_format_results_table` anbinden
|
||||
- Verwendung der neuen Hilfsfunktion (siehe 10.1) für konsistente Spalten
|
||||
- META-JSON enthält `matching_method`
|
||||
- _Requirements: 9.2, 9.5_
|
||||
|
||||
- [x] 11.4 Docstring für `filter_search_results` aktualisieren
|
||||
- Hinweis, dass `min_similarity` im LLM-Modus ignoriert wird
|
||||
- _Requirements: 12.1_
|
||||
|
||||
- [x] 11.5 Unit-Tests für `get_results_by_category` und `filter_search_results` in beiden Modi
|
||||
- LLM-Modus: `Begründung`-Spalte vorhanden, `min_similarity` ignoriert mit Hinweis
|
||||
- Score-Modus: bestehendes Verhalten unverändert (Regression)
|
||||
- _Requirements: 9.1, 9.2, 9.3, 9.4, 9.5_
|
||||
|
||||
- [x] 12. Server-Wiring und Integrationstests
|
||||
- [x] 12.1 `LlmFulltextMatcher` in `build_server` instanziieren
|
||||
- Konstruktor mit `db`, `client` (`AzureOpenAIClient`) verdrahten
|
||||
- Komponente an MCP-Tool-Handler weiterreichen
|
||||
- _Requirements: 1.4, 5.1, 6.1_
|
||||
|
||||
- [x] 12.2 End-to-End-Integrationstests mit gemocktem LLM-Client und gemocktem `DBClient`
|
||||
- `find_matching_capacities` und `find_matching_tasks` mit `matching_method="llm_fulltext"`
|
||||
- Verfügbarkeitsfilter im LLM-Modus identisch zum Score-Modus
|
||||
- SearchCache enthält `matching_method`, ungekürzte `rationale`, `errors`-Liste
|
||||
- _Requirements: 5.1, 5.2, 6.1, 6.2, 7.4, 8.6, 9.5_
|
||||
|
||||
- [x] 12.3 Tabellen-Snapshot-Test für Header in beiden Modi
|
||||
- Score-Modus-Header unverändert
|
||||
- LLM-Modus-Header endet auf `... | Category | Begründung`
|
||||
- _Requirements: 7.1, 8.1, 8.2_
|
||||
|
||||
- [x] 13. Checkpoint - Ensure all tests pass
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
- [x] 14. Dokumentation aktualisieren
|
||||
- [x] 14.1 `docs/architecture.md` erweitern
|
||||
- `LLM_Fulltext_Matcher` als Komponente in der Business-Logic-Layer beschreiben (Eingaben, Ausgaben, externe Abhängigkeit Azure OpenAI Chat Completion)
|
||||
- Zusätzliche Datenquellen (`teamlandkarte_v_capacities_latest.description`, `teamlandkarte_v_capacity_certificates_latest`, `teamlandkarte_v_capacity_references_latest`, `teamlandkarte_v_partners_latest.{id, name}`) im Schema-Verifikationsabschnitt aufführen
|
||||
- Join `teamlandkarte_v_capacity_references_latest.partner_id = teamlandkarte_v_partners_latest.id` und Übernahme der Spalte `name` als Partner_Name dokumentieren
|
||||
- Tool-Surface-Tabelle um Parameter `matching_method` (Wertebereich `score`/`llm_fulltext`) ergänzen
|
||||
- Runtime-View für beide Suchrichtungen um den LLM-Volltext-Pfad erweitern
|
||||
- _Requirements: 11.1, 11.2, 11.3, 11.4, 11.5_
|
||||
|
||||
- [x] 14.2 `README.md` erweitern
|
||||
- Quick-Start- und Usage-Abschnitt: Auswahl zwischen `score` und `llm_fulltext` per Tool-Parameter
|
||||
- Hinweis: Im LLM-Volltext-Modus keine numerischen Scores; stattdessen Spalte `Begründung`
|
||||
- Zusätzliche Datenbank-Views auflisten, die der Server im LLM-Volltext-Modus liest, einschließlich `teamlandkarte_v_partners_latest` mit Hinweis auf den LEFT JOIN über `partner_id` in der Referenz-Abfrage
|
||||
- Default-Konfiguration `[matching].default_method`
|
||||
- _Requirements: 11.6, 11.7, 11.8_
|
||||
|
||||
- [x] 14.3 `.github/agents/teamlandkarte_agent.md` aktualisieren
|
||||
- Beschreibung beider Verfahren `score` und `llm_fulltext` (Existenz und Zweck)
|
||||
- Pflicht-Frage nach `matching_method` vor `find_matching_capacities`/`find_matching_tasks`, sofern nicht aus dem Verlauf bekannt
|
||||
- Skills/Workflows: Aufruf der Tools mit zusätzlichem Parameter `matching_method`
|
||||
- Rolle der Spalte `Begründung` und Hinweis, dass im LLM-Modus keine numerischen Scores erscheinen
|
||||
- Bestehender Bestätigungs-Workflow (`show_pending_requirements`, `confirm_requirements`) bleibt für beide Verfahren gleich
|
||||
- _Requirements: 10.1, 10.2, 10.3, 10.4, 10.5_
|
||||
|
||||
- [x] 14.4 `.kiro/agents/teamlandkarte.md` analog zu 14.3 aktualisieren
|
||||
- Inhaltliche Gleichheit zum GitHub-Pendant sicherstellen
|
||||
- _Requirements: 10.1, 10.2, 10.3, 10.4, 10.5_
|
||||
|
||||
- [x] 14.5 Beispiele und Mini-Walkthroughs in README/Architektur ergänzen
|
||||
- Beispielausgabe einer LLM-Volltext-Tabelle inkl. Spalte `Begründung`
|
||||
- Beispielhafte META-JSON-Ausgabe mit `matching_method`
|
||||
- _Requirements: 11.5, 11.6_
|
||||
|
||||
- [x] 15. Final checkpoint - Ensure all tests pass
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
## Notes
|
||||
|
||||
- Tasks markiert mit `*` sind optional und können für einen schnelleren MVP übersprungen werden.
|
||||
- Property-Based Tests verwenden Hypothesis mit `@settings(max_examples=100)` (mindestens 100 Iterationen pro Property) und werden als `# Feature: llm-fulltext-matching, Property {N}: {title}` getaggt.
|
||||
- Unit- und Integrationstests verwenden `pytest` (Run-once, kein Watch-Modus); der `AzureOpenAIClient` wird stets gemockt.
|
||||
- Tasks referenzieren explizit Anforderungen aus `requirements.md` zur lückenlosen Nachverfolgbarkeit.
|
||||
- Die Implementierungssprache ist Python (bestehende Codebase).
|
||||
- Score-Modus bleibt vollständig abwärtskompatibel; nur additive Erweiterungen am Payload und am META-JSON.
|
||||
@@ -1 +0,0 @@
|
||||
{"specId": "6c39340f-10af-426d-9318-c96cb2f7ad6d", "workflowType": "requirements-first", "specType": "feature"}
|
||||
@@ -1,677 +0,0 @@
|
||||
# Design: Entfernung Embedding-basierter Similarity – Umstellung auf BM25 + LLM
|
||||
|
||||
## Übersicht
|
||||
|
||||
Dieses Design beschreibt die vollständige Entfernung der Embedding-Infrastruktur aus dem Teamlandkarte-MCP-System. Das System wird von einem hybriden Embedding/BM25-Ansatz auf eine reine BM25 + LLM-Architektur umgestellt:
|
||||
|
||||
- **Kompetenz-Matching**: Ausschließlich BM25+RRF (bereits implementiert, nur Conditional-Logik entfernen)
|
||||
- **Rollen-Similarity**: LLM Chat Completion (neu, ersetzt Embedding-Cosine-Similarity)
|
||||
- **Rollen-Inferenz**: LLM Chat Completion (neu, ersetzt Embedding-basierte Nearest-Neighbor-Suche)
|
||||
- **AutoTagger**: Bleibt unverändert (bereits LLM-basiert)
|
||||
|
||||
Entfernt werden: `EmbeddingCache`, `_embed`, `prefetch_embeddings`, `_per_skill_similarity`, `_aggregate_similarity`, Embedding-API-Methoden im `AzureOpenAIClient`, alle Embedding-Konfigurationsfelder, und der Startup-Preload.
|
||||
|
||||
## Architektur
|
||||
|
||||
### Vorher (Hybrid)
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
A[MCP Server Startup] --> B[Preload Embeddings]
|
||||
B --> C[VocabularyCache: Role + Competence Embeddings]
|
||||
B --> D[Task Embeddings]
|
||||
|
||||
E[Matching Request] --> F{use_bm25_search?}
|
||||
F -->|true| G[BM25+RRF Competence Similarity]
|
||||
F -->|false| H[Embedding Cosine Competence Similarity]
|
||||
|
||||
E --> I[Embedding Cosine Role Similarity]
|
||||
|
||||
J[infer_primary_role] --> K[Task Embedding → Cosine vs Role Vocab]
|
||||
```
|
||||
|
||||
### Nachher (BM25 + LLM)
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
A[MCP Server Startup] --> B[DB Schema Verification]
|
||||
A --> C[AzureOpenAIClient: nur Chat Completion]
|
||||
|
||||
E[Matching Request] --> G[BM25+RRF Competence Similarity]
|
||||
E --> I[LLM Role Similarity mit In-Run Cache]
|
||||
|
||||
J[infer_primary_role] --> K[LLM Chat Completion: Task Text → Role Selection]
|
||||
```
|
||||
|
||||
### Datenfluss Matching (Nachher)
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant Client
|
||||
participant Matcher
|
||||
participant SimilarityEngine
|
||||
participant BM25
|
||||
participant LLM as AzureOpenAIClient (Chat)
|
||||
|
||||
Client->>Matcher: match(capacities, requirements)
|
||||
Matcher->>Matcher: Filter by availability
|
||||
Matcher->>BM25: Build global index (all candidate competences)
|
||||
|
||||
loop Per Candidate
|
||||
Matcher->>SimilarityEngine: compute_competence_similarity(required, candidate, global_index)
|
||||
SimilarityEngine->>BM25: rank per required competence
|
||||
SimilarityEngine-->>Matcher: {req → {score, best_match, rationale}}
|
||||
|
||||
Matcher->>SimilarityEngine: compute_role_similarity(required_role, candidate_role)
|
||||
SimilarityEngine->>SimilarityEngine: Check in-run cache
|
||||
alt Cache Miss
|
||||
SimilarityEngine->>LLM: chat_completion(role_similarity_prompt)
|
||||
LLM-->>SimilarityEngine: {"similarity": 0.85}
|
||||
SimilarityEngine->>SimilarityEngine: Cache result
|
||||
end
|
||||
SimilarityEngine-->>Matcher: float score
|
||||
end
|
||||
|
||||
Matcher-->>Client: MatchResult
|
||||
```
|
||||
|
||||
## Komponenten und Schnittstellen
|
||||
|
||||
### 1. SimilarityEngine (refactored)
|
||||
|
||||
**Datei:** `src/teamlandkarte_mcp/matching/similarity.py`
|
||||
|
||||
**Entfernte Methoden:**
|
||||
- `prefetch_embeddings`
|
||||
- `_embed`
|
||||
- `get_embedding_for_cache_key`
|
||||
- `get_embeddings_for_cache_keys`
|
||||
- `_aggregate_similarity`
|
||||
- `_per_skill_similarity`
|
||||
|
||||
**Entfernte Properties:**
|
||||
- `embedding_model`
|
||||
- `embedding_dimensions`
|
||||
- `use_bm25_search`
|
||||
|
||||
**Entfernte Constructor-Parameter:**
|
||||
- `cache` (EmbeddingCache)
|
||||
- `embedding_model`
|
||||
- `embedding_dimensions`
|
||||
- `strategy`
|
||||
- `use_bm25_search`
|
||||
|
||||
**Neuer Constructor:**
|
||||
|
||||
```python
|
||||
class SimilarityEngine:
|
||||
"""Compute similarity scores using BM25+RRF and LLM."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
client: AzureOpenAIClient,
|
||||
cost_tracker: CostTracker | None = None,
|
||||
use_auto_tagging: bool = False,
|
||||
auto_tagger: AutoTagger | None = None,
|
||||
) -> None:
|
||||
self._client = client
|
||||
self._cost_tracker = cost_tracker
|
||||
self._use_auto_tagging = use_auto_tagging
|
||||
self._auto_tagger = auto_tagger
|
||||
# Per-run cache for LLM role similarity: (role_a, role_b) → score
|
||||
self._role_similarity_cache: dict[tuple[str, str], float] = {}
|
||||
```
|
||||
|
||||
**Geänderte Methode `compute_competence_similarity`:**
|
||||
|
||||
```python
|
||||
async def compute_competence_similarity(
|
||||
self,
|
||||
required: list[str],
|
||||
candidate: list[str],
|
||||
global_index: Bm25Index | None = None,
|
||||
) -> dict[str, dict[str, object]]:
|
||||
"""BM25+RRF competence similarity (einziger Pfad)."""
|
||||
working = list(candidate)
|
||||
if self._use_auto_tagging and self._auto_tagger is not None:
|
||||
working = await self._auto_tagger.expand_competences(required, candidate)
|
||||
return self._bm25_rrf_similarity(required, working, global_index=global_index)
|
||||
```
|
||||
|
||||
**Neue Methode `compute_role_similarity` (LLM-basiert):**
|
||||
|
||||
```python
|
||||
_ROLE_SIMILARITY_SYSTEM_PROMPT = (
|
||||
"You are a job-role similarity expert. Given two role names, "
|
||||
"determine their semantic similarity on a scale from 0.0 to 1.0. "
|
||||
"0.0 means completely unrelated roles, 1.0 means identical or "
|
||||
"interchangeable roles. Consider synonyms, hierarchy, and domain overlap. "
|
||||
'Respond with a JSON object: {"similarity": <float>}'
|
||||
)
|
||||
|
||||
async def compute_role_similarity(
|
||||
self,
|
||||
required_role: str | None,
|
||||
candidate_role: str | None,
|
||||
) -> float:
|
||||
"""LLM-basierte Rollen-Similarity."""
|
||||
if self._is_bad_role(required_role) or self._is_bad_role(candidate_role):
|
||||
return 0.0
|
||||
|
||||
req_norm = str(required_role).strip().lower()
|
||||
cand_norm = str(candidate_role).strip().lower()
|
||||
|
||||
if req_norm == cand_norm:
|
||||
return 1.0
|
||||
|
||||
# Symmetrischer Cache-Key
|
||||
cache_key = tuple(sorted((req_norm, cand_norm)))
|
||||
if cache_key in self._role_similarity_cache:
|
||||
return self._role_similarity_cache[cache_key]
|
||||
|
||||
try:
|
||||
raw = await self._client.chat_completion(
|
||||
system=_ROLE_SIMILARITY_SYSTEM_PROMPT,
|
||||
user=f"Role A: {required_role}\nRole B: {candidate_role}",
|
||||
)
|
||||
payload = json.loads(raw)
|
||||
score = float(payload.get("similarity", 0.0))
|
||||
score = max(0.0, min(1.0, score))
|
||||
except Exception:
|
||||
score = 0.0
|
||||
|
||||
self._role_similarity_cache[cache_key] = score
|
||||
return score
|
||||
```
|
||||
|
||||
**Neue Methode `clear_role_cache`:**
|
||||
|
||||
```python
|
||||
def clear_role_cache(self) -> None:
|
||||
"""Cache zwischen Matching-Runs leeren."""
|
||||
self._role_similarity_cache.clear()
|
||||
```
|
||||
|
||||
### 2. VocabularyCache (refactored)
|
||||
|
||||
**Datei:** `src/teamlandkarte_mcp/matching/vocabulary.py`
|
||||
|
||||
**Entfernte Methoden:**
|
||||
- `preload`
|
||||
- `_preload_role_vocab`
|
||||
- `_preload_competence_vocab`
|
||||
- `ensure_task_embedding`
|
||||
- `infer_primary_role` (alte Signatur mit `task_embedding`)
|
||||
- `infer_competences`
|
||||
|
||||
**Entfernte Properties:**
|
||||
- `roles`
|
||||
- `competences`
|
||||
|
||||
**Neuer Constructor:**
|
||||
|
||||
```python
|
||||
class VocabularyCache:
|
||||
"""LLM-basierte Rollen-Inferenz aus Task-Text."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
db: DBClient,
|
||||
client: AzureOpenAIClient,
|
||||
) -> None:
|
||||
self._db = db
|
||||
self._client = client
|
||||
```
|
||||
|
||||
**Neue Methode `infer_primary_role` (LLM-basiert):**
|
||||
|
||||
```python
|
||||
_ROLE_INFERENCE_SYSTEM_PROMPT = (
|
||||
"You are a role classification expert. Given a task description and a list "
|
||||
"of available roles, select the single most appropriate role for the task. "
|
||||
"You MUST select exactly one role from the provided list. "
|
||||
"Respond with a JSON object: "
|
||||
'{"role": "<selected role name>", "confidence": <float 0.0-1.0>}'
|
||||
)
|
||||
|
||||
async def infer_primary_role(
|
||||
self,
|
||||
*,
|
||||
task_text: str,
|
||||
) -> tuple[str, float] | None:
|
||||
"""Inferiere die passendste Rolle für einen Task-Text via LLM."""
|
||||
role_names = self._db.get_all_role_names()
|
||||
if not role_names:
|
||||
return None
|
||||
|
||||
text = (task_text or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
|
||||
user_prompt = (
|
||||
f"Task: {text}\n\n"
|
||||
f"Available roles: {json.dumps(list(role_names), ensure_ascii=False)}"
|
||||
)
|
||||
|
||||
try:
|
||||
raw = await self._client.chat_completion(
|
||||
system=_ROLE_INFERENCE_SYSTEM_PROMPT,
|
||||
user=user_prompt,
|
||||
)
|
||||
payload = json.loads(raw)
|
||||
role = str(payload.get("role", "")).strip()
|
||||
confidence = float(payload.get("confidence", 0.0))
|
||||
confidence = max(0.0, min(1.0, confidence))
|
||||
|
||||
# Validierung: Rolle muss in der DB-Liste existieren
|
||||
if role not in set(role_names):
|
||||
return None
|
||||
|
||||
return role, confidence
|
||||
except Exception:
|
||||
return None
|
||||
```
|
||||
|
||||
### 3. AzureOpenAIClient (vereinfacht)
|
||||
|
||||
**Datei:** `src/teamlandkarte_mcp/azure/openai_client.py`
|
||||
|
||||
**Entfernte Methoden:**
|
||||
- `embeddings`
|
||||
- `get_embeddings_batch`
|
||||
|
||||
**Entfernte Constructor-Parameter:**
|
||||
- `embedding_api_key`
|
||||
- `embedding_deployment`
|
||||
- `embedding_batch_size`
|
||||
|
||||
**Neuer Constructor:**
|
||||
|
||||
```python
|
||||
class AzureOpenAIClient:
|
||||
"""Azure OpenAI Chat Completion Client."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
endpoint: str,
|
||||
api_version: str,
|
||||
chat_deployment: str,
|
||||
llm_api_key: str,
|
||||
timeout_s: float = 30.0,
|
||||
max_retries: int = 5,
|
||||
cost_tracker: CostTracker | None = None,
|
||||
) -> None:
|
||||
self._chat_deployment = chat_deployment
|
||||
self._max_retries = max_retries
|
||||
self._timeout_s = timeout_s
|
||||
self._cost_tracker = cost_tracker
|
||||
|
||||
self._chat = AsyncAzureOpenAI(
|
||||
api_key=llm_api_key,
|
||||
azure_endpoint=endpoint,
|
||||
api_version=api_version,
|
||||
)
|
||||
```
|
||||
|
||||
Die `chat_completion`-Methode bleibt unverändert, außer dass die Guard-Clause für fehlende Konfiguration entfällt (Chat ist jetzt immer konfiguriert).
|
||||
|
||||
### 4. Matcher (minimal geändert)
|
||||
|
||||
**Datei:** `src/teamlandkarte_mcp/matching/matcher.py`
|
||||
|
||||
**Änderungen:**
|
||||
- Entfernung der `if self._sim.use_bm25_search`-Bedingung beim BM25-Index-Aufbau
|
||||
- Der globale BM25-Index wird **immer** gebaut (unconditional)
|
||||
|
||||
```python
|
||||
# Vorher:
|
||||
global_bm25_index: Bm25Index | None = None
|
||||
if self._sim.use_bm25_search and filtered:
|
||||
global_corpus = list(...)
|
||||
global_bm25_index = Bm25Index(corpus=global_corpus)
|
||||
|
||||
# Nachher:
|
||||
global_bm25_index: Bm25Index | None = None
|
||||
if filtered:
|
||||
global_corpus = list(
|
||||
{c for cap in filtered for c in cap.competences if c.strip()}
|
||||
)
|
||||
global_bm25_index = Bm25Index(corpus=global_corpus)
|
||||
```
|
||||
|
||||
### 5. MCP Server (vereinfacht)
|
||||
|
||||
**Datei:** `src/teamlandkarte_mcp/mcp_server.py`
|
||||
|
||||
**Entfernte Elemente:**
|
||||
- `_startup_preload_embeddings` Funktion
|
||||
- `_deferred_preload` Funktion und `_preload_started` Flag
|
||||
- `_ensure_preloaded` Funktion
|
||||
- `EmbeddingCache`-Instanziierung
|
||||
- `emb_cache`-Variable
|
||||
- Alle `await _ensure_preloaded()`-Aufrufe in Tools
|
||||
- Import von `EmbeddingCache`
|
||||
- Import von `VocabularyCache` (wird direkt mit `AzureOpenAIClient` konstruiert)
|
||||
|
||||
**Geänderte Konstruktion:**
|
||||
|
||||
```python
|
||||
# Vorher: Embedding-Client + separater LLM-Client
|
||||
azure_client = AzureOpenAIClient(
|
||||
endpoint=..., embedding_api_key=..., embedding_deployment=..., ...
|
||||
)
|
||||
|
||||
# Nachher: Nur noch ein LLM-Client
|
||||
azure_client = AzureOpenAIClient(
|
||||
endpoint=cfg.azure_openai.endpoint,
|
||||
api_version=cfg.azure_openai.api_version,
|
||||
chat_deployment=cfg.azure_openai.chat_deployment,
|
||||
llm_api_key=cfg.azure_openai.llm_api_key,
|
||||
cost_tracker=cost_tracker,
|
||||
)
|
||||
|
||||
similarity = SimilarityEngine(
|
||||
client=azure_client,
|
||||
cost_tracker=cost_tracker,
|
||||
use_auto_tagging=cfg.similarity.use_auto_tagging,
|
||||
auto_tagger=auto_tagger,
|
||||
)
|
||||
|
||||
vocab_cache = VocabularyCache(
|
||||
db=db_client,
|
||||
client=azure_client,
|
||||
)
|
||||
```
|
||||
|
||||
**Geändertes `infer_primary_role`-Tool:**
|
||||
|
||||
```python
|
||||
@mcp.tool()
|
||||
async def infer_primary_role(
|
||||
task_id: Optional[str] = None,
|
||||
task_text: Optional[str] = None,
|
||||
) -> str:
|
||||
"""Infer the single closest role from either a DB task or free text."""
|
||||
if bool(task_id) == bool(task_text):
|
||||
return "Provide exactly one of task_id or task_text."
|
||||
|
||||
text = (task_text or "").strip()
|
||||
if task_id:
|
||||
_ensure_db()
|
||||
task = db_client.get_task_by_id(task_id)
|
||||
if task is None:
|
||||
return f"Task not found or not published: {task_id}"
|
||||
title = (task.title or "").strip()
|
||||
desc = (task.description or "").strip()
|
||||
text = (title + "\n\n" + desc).strip() if title else desc
|
||||
|
||||
if not text:
|
||||
return "Task text is empty."
|
||||
|
||||
best = await vocab_cache.infer_primary_role(task_text=text)
|
||||
if best is None:
|
||||
rows = [["", ""]]
|
||||
else:
|
||||
role, score = best
|
||||
rows = [[str(role), f"{float(score):.3f}"]]
|
||||
|
||||
return md_table(["Role", "Confidence"], rows)
|
||||
```
|
||||
|
||||
**Entferntes Tool `validate_task_requirements`:**
|
||||
Dieses Tool basiert vollständig auf Embedding-Inferenz (`infer_competences`, `ensure_task_embedding`). Es wird entfernt oder durch eine vereinfachte Version ersetzt, die nur DB-Felder anzeigt.
|
||||
|
||||
### 6. Config (vereinfacht)
|
||||
|
||||
**Datei:** `src/teamlandkarte_mcp/config.py`
|
||||
|
||||
**Entfernte Dataclasses:**
|
||||
- `EmbeddingCacheConfig`
|
||||
- `InferenceConfig`
|
||||
|
||||
**Geänderte Dataclasses:**
|
||||
|
||||
```python
|
||||
@dataclass(frozen=True)
|
||||
class AzureOpenAIConfig:
|
||||
"""Azure OpenAI configuration (nur Chat Completion)."""
|
||||
endpoint: str
|
||||
api_version: str = "2024-02-15-preview"
|
||||
chat_deployment: str = ""
|
||||
llm_api_key: str = ""
|
||||
show_costs_in_output: bool = False
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SimilarityConfig:
|
||||
"""Similarity engine configuration."""
|
||||
use_auto_tagging: bool = False
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MatchingConfig:
|
||||
"""Matching weights and thresholds."""
|
||||
competence_weight: float = 0.8
|
||||
role_weight: float = 0.2
|
||||
require_confirmation: bool = True
|
||||
thresholds: MatchingThresholds = MatchingThresholds()
|
||||
fuzzy: FuzzyConfig = FuzzyConfig()
|
||||
# inference entfällt
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AppConfig:
|
||||
"""Top-level application configuration."""
|
||||
database: DatabaseConfig
|
||||
matching: MatchingConfig
|
||||
cache: CacheConfig
|
||||
azure_openai: AzureOpenAIConfig
|
||||
similarity: SimilarityConfig
|
||||
# embedding_cache entfällt
|
||||
```
|
||||
|
||||
**Entfernte Felder aus `SimilarityConfig`:**
|
||||
- `embedding_model`
|
||||
- `embedding_dimensions`
|
||||
- `strategy`
|
||||
- `use_bm25_search`
|
||||
|
||||
**Entfernte Felder aus `AzureOpenAIConfig`:**
|
||||
- `embedding_deployment`
|
||||
- `embedding_batch_size`
|
||||
|
||||
**Entfernte Validierungen in `load_config`:**
|
||||
- `AZURE_OPENAI_EMBEDDING_API_KEY` Prüfung
|
||||
- `embedding_dimensions == 3072` Prüfung
|
||||
- `strategy` Validierung
|
||||
- `inference` Parsing
|
||||
|
||||
**Neue Validierung:**
|
||||
- `AZURE_OPENAI_LLM_API_KEY` ist jetzt immer erforderlich (nicht nur bei auto_tagging)
|
||||
- `chat_deployment` ist jetzt erforderlich
|
||||
|
||||
### 7. Entfernte Dateien
|
||||
|
||||
| Datei | Grund |
|
||||
|-------|-------|
|
||||
| `src/teamlandkarte_mcp/cache/embedding_cache.py` | Keine Embeddings mehr |
|
||||
| Zugehörige Tests für EmbeddingCache | Keine Embeddings mehr |
|
||||
|
||||
### 8. config.toml Änderungen
|
||||
|
||||
**Entfernte Sektionen:**
|
||||
- `[embedding_cache]` komplett
|
||||
|
||||
**Entfernte Felder aus `[matching.similarity]`:**
|
||||
- `embedding_model`
|
||||
- `embedding_dimensions`
|
||||
- `strategy`
|
||||
- `use_bm25_search`
|
||||
|
||||
**Entfernte Felder aus `[azure_openai]`:**
|
||||
- `embedding_deployment`
|
||||
- `embedding_batch_size`
|
||||
|
||||
**Entfernte Felder aus `[matching]`:**
|
||||
- `[matching.inference]` komplett
|
||||
|
||||
**Neue Pflichtfelder in `[azure_openai]`:**
|
||||
- `chat_deployment` (bereits vorhanden als optionales Feld, wird Pflicht)
|
||||
|
||||
**Neue Umgebungsvariable (Pflicht):**
|
||||
- `AZURE_OPENAI_LLM_API_KEY` (ersetzt `AZURE_OPENAI_EMBEDDING_API_KEY`)
|
||||
|
||||
**Entfernte Umgebungsvariable:**
|
||||
- `AZURE_OPENAI_EMBEDDING_API_KEY`
|
||||
|
||||
## Datenmodelle
|
||||
|
||||
### LLM Role Similarity Request/Response
|
||||
|
||||
```json
|
||||
// System Prompt: _ROLE_SIMILARITY_SYSTEM_PROMPT
|
||||
// User Message:
|
||||
"Role A: Software Engineer\nRole B: Backend Developer"
|
||||
|
||||
// Expected Response:
|
||||
{"similarity": 0.75}
|
||||
```
|
||||
|
||||
### LLM Role Inference Request/Response
|
||||
|
||||
```json
|
||||
// System Prompt: _ROLE_INFERENCE_SYSTEM_PROMPT
|
||||
// User Message:
|
||||
"Task: Implementierung einer REST-API für Benutzerverwaltung\n\nAvailable roles: [\"Backend Developer\", \"Frontend Developer\", \"DevOps Engineer\", \"Data Engineer\"]"
|
||||
|
||||
// Expected Response:
|
||||
{"role": "Backend Developer", "confidence": 0.92}
|
||||
```
|
||||
|
||||
### In-Run Role Similarity Cache
|
||||
|
||||
```python
|
||||
# Symmetrischer Cache innerhalb eines Matching-Runs
|
||||
# Key: tuple(sorted((role_a_lower, role_b_lower)))
|
||||
# Value: float (0.0 - 1.0)
|
||||
_role_similarity_cache: dict[tuple[str, str], float] = {}
|
||||
```
|
||||
|
||||
Der Cache wird pro `SimilarityEngine`-Instanz gehalten und lebt für die Dauer des Server-Prozesses. Da Rollennamen stabil sind (aus der DB), ist kein TTL nötig.
|
||||
|
||||
|
||||
## Correctness Properties
|
||||
|
||||
*A property is a characteristic or behavior that should hold true across all valid executions of a system — essentially, a formal statement about what the system should do. Properties serve as the bridge between human-readable specifications and machine-verifiable correctness guarantees.*
|
||||
|
||||
### Property 1: BM25 Zero-Score für fehlenden Token-Overlap
|
||||
|
||||
*For any* set of required competences and candidate competences where no candidate shares any token with a required competence, `compute_competence_similarity` shall return a score of 0.0 for that required competence.
|
||||
|
||||
**Validates: Requirements 1.1**
|
||||
|
||||
### Property 2: Matcher baut BM25-Index bedingungslos
|
||||
|
||||
*For any* non-empty list of filtered candidates with competences, the Matcher shall always build a global BM25 index and use it for competence scoring, producing results where candidates with no token overlap receive score 0.0.
|
||||
|
||||
**Validates: Requirements 2.6**
|
||||
|
||||
### Property 3: infer_primary_role Ausgabe-Validität
|
||||
|
||||
*For any* non-empty task text and non-empty role list from the database, if `infer_primary_role` returns a non-None result, the returned role name must be an element of the database role list and the confidence must be a float in [0.0, 1.0].
|
||||
|
||||
**Validates: Requirements 3.1, 3.4**
|
||||
|
||||
### Property 4: Graceful Degradation bei LLM-Fehler
|
||||
|
||||
*For any* LLM call that raises an exception, `compute_role_similarity` shall return 0.0 and `infer_primary_role` shall return None, without propagating the exception to the caller.
|
||||
|
||||
**Validates: Requirements 3.5, 5.6**
|
||||
|
||||
### Property 5: compute_role_similarity Wertebereich
|
||||
|
||||
*For any* two non-empty, non-"(unknown)" role names, `compute_role_similarity` shall return a float in the closed interval [0.0, 1.0]. For identical role names (case-insensitive), it shall return 1.0.
|
||||
|
||||
**Validates: Requirements 5.1**
|
||||
|
||||
### Property 6: Rollen-Similarity-Cache ist symmetrisch und idempotent
|
||||
|
||||
*For any* role pair (A, B), calling `compute_role_similarity(A, B)` and then `compute_role_similarity(B, A)` shall return the same score, and the second call shall not invoke the LLM (cache hit). Repeated calls with the same pair shall always return the same cached value.
|
||||
|
||||
**Validates: Requirements 5.8**
|
||||
|
||||
## Error Handling
|
||||
|
||||
### LLM-Fehler in compute_role_similarity
|
||||
|
||||
- Bei jeder Exception (Timeout, API-Fehler, JSON-Parse-Fehler) wird `0.0` zurückgegeben
|
||||
- Fehler wird geloggt (LOGGER.warning)
|
||||
- Kein Eintrag im Cache für fehlgeschlagene Aufrufe (Retry bei nächstem Aufruf möglich)
|
||||
|
||||
### LLM-Fehler in infer_primary_role
|
||||
|
||||
- Bei jeder Exception wird `None` zurückgegeben
|
||||
- Fehler wird geloggt (LOGGER.warning)
|
||||
- Caller (MCP-Tool) zeigt leere Tabelle an
|
||||
|
||||
### Ungültige LLM-Antworten
|
||||
|
||||
- **Role Similarity**: Wenn `similarity` nicht im JSON oder nicht parsebar → 0.0
|
||||
- **Role Inference**: Wenn `role` nicht in der DB-Liste → None
|
||||
- **Role Inference**: Wenn `confidence` nicht parsebar → 0.0 (aber Rolle wird trotzdem zurückgegeben wenn valide)
|
||||
|
||||
### Leere/Ungültige Eingaben
|
||||
|
||||
- `compute_role_similarity` mit None/leer/"(unknown)" → 0.0 (kein LLM-Aufruf)
|
||||
- `infer_primary_role` mit leerem Text → None (kein LLM-Aufruf)
|
||||
- `infer_primary_role` mit leerer Rollenliste aus DB → None (kein LLM-Aufruf)
|
||||
- `compute_competence_similarity` mit leerer Required-Liste → leeres Dict
|
||||
- `compute_competence_similarity` mit leerer Candidate-Liste → alle Scores 0.0
|
||||
|
||||
### Konfigurationsfehler
|
||||
|
||||
- Fehlende `AZURE_OPENAI_LLM_API_KEY` → `ConfigError` beim Laden (fail-fast)
|
||||
- Fehlender `chat_deployment` → `ConfigError` beim Laden (fail-fast)
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
### Property-Based Tests (fast-check / Hypothesis)
|
||||
|
||||
Bibliothek: **Hypothesis** (Python PBT-Standard)
|
||||
|
||||
Konfiguration: Mindestens 100 Iterationen pro Property-Test.
|
||||
|
||||
Jeder Property-Test wird mit einem Kommentar getaggt:
|
||||
```
|
||||
# Feature: remove-embedding-competence-similarity, Property {N}: {title}
|
||||
```
|
||||
|
||||
| Property | Test-Ansatz | Generator |
|
||||
|----------|-------------|-----------|
|
||||
| 1: BM25 Zero-Score | Generiere disjunkte Token-Sets für required/candidate, prüfe Score == 0.0 | `st.lists(st.text(alphabet=st.characters(whitelist_categories=('L',)), min_size=3))` |
|
||||
| 2: Matcher BM25 unconditional | Generiere Capacities + Requirements, prüfe dass Ergebnis BM25-Charakteristik hat (0.0 bei no-overlap) | Custom Capacity/Requirements strategies |
|
||||
| 3: infer_primary_role Validität | Generiere Task-Texte + Role-Listen, mocke LLM mit zufälliger valider Antwort, prüfe Output-Constraints | `st.text(min_size=1)`, `st.lists(st.text(min_size=1), min_size=1)` |
|
||||
| 4: Graceful Degradation | Generiere zufällige Exceptions, prüfe dass 0.0/None zurückkommt | `st.sampled_from([TimeoutError, RuntimeError, ValueError, json.JSONDecodeError])` |
|
||||
| 5: Role Similarity Wertebereich | Generiere Rollenpaare, mocke LLM mit zufälligem Score, prüfe [0.0, 1.0] und Identitäts-Case | `st.text(min_size=1, max_size=50)` |
|
||||
| 6: Cache Symmetrie | Generiere Rollenpaare, rufe in beiden Reihenfolgen auf, prüfe gleichen Score + nur 1 LLM-Call | `st.text(min_size=1, max_size=50)` |
|
||||
|
||||
### Unit Tests
|
||||
|
||||
Unit Tests fokussieren auf:
|
||||
|
||||
- **Spezifische Beispiele**: Bekannte Rollenpaare (z.B. "Backend Developer" vs "Software Engineer") mit gemocktem LLM
|
||||
- **Edge Cases**: Leere Strings, None-Werte, "(unknown)", Whitespace-only
|
||||
- **Integration**: Config-Loading ohne Embedding-Felder, Server-Startup ohne Preload
|
||||
- **Regressions**: Sicherstellen dass BM25+RRF-Ergebnisse identisch zum bisherigen `use_bm25_search=True`-Pfad sind
|
||||
|
||||
### Integrationstests
|
||||
|
||||
- End-to-End Matching-Run mit gemocktem LLM-Client
|
||||
- Config-Loading aus minimaler TOML-Datei (ohne Embedding-Sektionen)
|
||||
- Server-Startup ohne `AZURE_OPENAI_EMBEDDING_API_KEY` (darf nicht mehr geprüft werden)
|
||||
|
||||
### Nicht getestet (bewusst ausgelassen)
|
||||
|
||||
- Prompt-Qualität (subjektiv, erfordert manuelles Review)
|
||||
- LLM-Antwort-Genauigkeit (abhängig vom Modell, nicht deterministisch)
|
||||
- Startup-Performance-Verbesserung (Benchmark, kein Unit-Test)
|
||||
-111
@@ -1,111 +0,0 @@
|
||||
# Anforderungsdokument
|
||||
|
||||
## Einleitung
|
||||
|
||||
Dieses Dokument beschreibt die Anforderungen für die Entfernung der Embedding-basierten Kompetenz-Similarity zugunsten von BM25 als einzigem Kompetenz-Matching-Verfahren, die Umstellung der Rollen-Inferenz und Rollen-Similarity auf einen LLM-Ansatz sowie die vollständige Entfernung der Embedding-Infrastruktur. Das System wird vollständig BM25 + LLM-basiert.
|
||||
|
||||
## Glossar
|
||||
|
||||
- **SimilarityEngine**: Modul in `matching/similarity.py`, das Ähnlichkeitsberechnungen für Kompetenzen und Rollen durchführt.
|
||||
- **VocabularyCache**: Modul in `matching/vocabulary.py`, das Vokabular-Embeddings vorhält und Inferenz-Funktionen bereitstellt.
|
||||
- **BM25**: Probabilistisches Ranking-Verfahren für Textähnlichkeit basierend auf Termfrequenz und inverser Dokumentfrequenz.
|
||||
- **RRF**: Reciprocal Rank Fusion – Verfahren zur Kombination mehrerer Rankings.
|
||||
- **AutoTagger**: LLM-basiertes Modul zur Erweiterung von Kompetenzlisten vor dem BM25-Scoring.
|
||||
- **AzureOpenAIClient**: Client-Wrapper für Azure OpenAI API-Aufrufe (Embeddings und Chat Completions).
|
||||
- **SimilarityConfig**: Konfigurationsklasse für die Similarity-Engine in `config.py`.
|
||||
- **infer_primary_role**: Funktion, die einer Aufgabe die passendste Rolle aus der Datenbank zuordnet.
|
||||
- **LLM**: Large Language Model – hier Azure OpenAI Chat Completion.
|
||||
- **Preload**: Eageres Vorladen von Embeddings beim Server-Start.
|
||||
- **EmbeddingCache**: Cache-Modul für gespeicherte Embedding-Vektoren.
|
||||
- **compute_role_similarity**: Funktion in der SimilarityEngine, die die semantische Ähnlichkeit zwischen einer geforderten Rolle und einer Kandidaten-Rolle berechnet.
|
||||
|
||||
## Anforderungen
|
||||
|
||||
### Anforderung 1: Entfernung der Embedding-basierten Kompetenz-Similarity
|
||||
|
||||
**User Story:** Als Entwickler möchte ich, dass BM25+RRF das einzige Verfahren für Kompetenz-Matching ist, damit die Codebasis vereinfacht wird und keine Embedding-Kosten für Kompetenz-Vergleiche anfallen.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. THE SimilarityEngine SHALL use BM25+RRF as the sole method for computing competence similarity scores.
|
||||
2. WHEN compute_competence_similarity is called, THE SimilarityEngine SHALL execute the BM25+RRF path without checking a strategy flag or use_bm25_search parameter.
|
||||
3. THE SimilarityEngine SHALL no longer contain the `_per_skill_similarity` method for embedding-based per-skill competence matching.
|
||||
4. THE SimilarityEngine SHALL no longer contain the `_aggregate_similarity` method for embedding-based aggregate competence matching.
|
||||
5. THE SimilarityEngine SHALL no longer accept a `strategy` parameter in its constructor for competence similarity strategy selection.
|
||||
6. THE SimilarityEngine SHALL no longer accept a `use_bm25_search` parameter in its constructor.
|
||||
7. THE SimilarityEngine SHALL remove the `prefetch_embeddings` and `_embed` methods, since role similarity also switches to LLM and no embedding use cases remain.
|
||||
|
||||
### Anforderung 2: Entfernung des Konfigurationsparameters use_bm25_search
|
||||
|
||||
**User Story:** Als Entwickler möchte ich, dass der Parameter `use_bm25_search` aus der Konfiguration entfernt wird, da BM25 nun immer aktiv ist und der Parameter redundant geworden ist.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. THE SimilarityConfig SHALL no longer contain the field `use_bm25_search`.
|
||||
2. THE SimilarityConfig SHALL no longer contain the field `strategy` (da nur noch BM25 verwendet wird).
|
||||
3. WHEN the configuration is loaded, THE Config-Loader SHALL not read or validate `use_bm25_search` from the TOML file.
|
||||
4. WHEN the configuration is loaded, THE Config-Loader SHALL not read or validate `matching.similarity.strategy` from the TOML file.
|
||||
5. THE SimilarityEngine SHALL no longer expose a `use_bm25_search` property.
|
||||
6. THE Matcher SHALL build the global BM25 index unconditionally for all filtered candidates without checking a `use_bm25_search` flag.
|
||||
|
||||
### Anforderung 3: LLM-basierte Rollen-Inferenz
|
||||
|
||||
**User Story:** Als Entwickler möchte ich, dass `infer_primary_role` einen LLM-Ansatz (Chat Completion) verwendet anstelle von Embedding-Cosine-Similarity, damit die Rollenzuordnung kontextbezogener und genauer erfolgt.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. WHEN infer_primary_role is called, THE VocabularyCache SHALL use an LLM chat completion to determine the best matching role for a given task text.
|
||||
2. WHEN infer_primary_role is called, THE VocabularyCache SHALL provide the list of all available role names from the database as context to the LLM.
|
||||
3. WHEN infer_primary_role is called, THE VocabularyCache SHALL provide the task text (title and/or description) as input to the LLM.
|
||||
4. THE VocabularyCache SHALL return the role name and a confidence score from the LLM response.
|
||||
5. IF the LLM call fails, THEN THE VocabularyCache SHALL return None rather than raising an unhandled exception.
|
||||
6. THE VocabularyCache SHALL no longer require a task embedding as input parameter for infer_primary_role.
|
||||
7. THE VocabularyCache SHALL accept the task text directly as input parameter for infer_primary_role.
|
||||
8. WHEN infer_primary_role is called, THE VocabularyCache SHALL instruct the LLM to select exactly one role from the provided list and return a structured JSON response.
|
||||
9. THE infer_primary_role tool in mcp_server.py SHALL call the new LLM-based infer_primary_role without first generating a task embedding.
|
||||
|
||||
### Anforderung 4: Entfernung des Embedding-Preloads offener Tasks
|
||||
|
||||
**User Story:** Als Entwickler möchte ich, dass Embeddings offener Tasks nicht mehr beim Server-Start vorgeladen werden und auch nicht mehr on-demand erzeugt werden, da keine Embedding-basierte Verarbeitung mehr stattfindet.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. THE MCP-Server SHALL no longer preload embeddings for open tasks during startup.
|
||||
2. THE `_startup_preload_embeddings` function SHALL be removed entirely.
|
||||
3. THE VocabularyCache SHALL no longer provide an `ensure_task_embedding` method, since task embeddings are not needed in the BM25 + LLM architecture.
|
||||
4. THE MCP-Server SHALL no longer preload role or competence vocabulary embeddings at startup, since all similarity computations now use BM25 or LLM.
|
||||
5. THE startup time of the MCP-Server SHALL be reduced by eliminating all embedding preload loops.
|
||||
|
||||
### Anforderung 5: Umstellung der Rollen-Similarity im Matcher auf LLM
|
||||
|
||||
**User Story:** Als Entwickler möchte ich, dass die Rollen-Similarity im Matcher (`compute_role_similarity`) ebenfalls einen LLM-Ansatz (Chat Completion) nutzt anstelle von Embedding-Cosine-Similarity, damit das gesamte System ohne Embeddings auskommt und konsistent BM25 + LLM-basiert ist.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. WHEN compute_role_similarity is called, THE SimilarityEngine SHALL use an LLM chat completion to determine the semantic similarity between the required role and the candidate role.
|
||||
2. WHEN compute_role_similarity is called, THE SimilarityEngine SHALL provide both role names to the LLM and request a numeric similarity score between 0.0 and 1.0.
|
||||
3. THE SimilarityEngine SHALL instruct the LLM to return a structured JSON response containing the similarity score.
|
||||
4. THE SimilarityEngine SHALL no longer use embedding cosine similarity for role comparisons.
|
||||
5. THE SimilarityEngine SHALL no longer call `_embed` or `prefetch_embeddings` for role similarity computation.
|
||||
6. IF the LLM call fails, THEN THE SimilarityEngine SHALL return a default similarity score of 0.0 rather than raising an unhandled exception.
|
||||
7. THE VocabularyCache SHALL no longer preload role vocabulary embeddings at startup, since they are not needed for LLM-based role similarity.
|
||||
8. THE SimilarityEngine SHALL cache LLM-based role similarity results for identical role pairs within a matching run to avoid redundant API calls.
|
||||
|
||||
### Anforderung 6: Vollständige Entfernung der Embedding-Infrastruktur
|
||||
|
||||
**User Story:** Als Entwickler möchte ich, dass die gesamte Embedding-Infrastruktur entfernt wird, da weder Kompetenz-Matching noch Rollen-Similarity noch Rollen-Inferenz Embeddings benötigen und das System vollständig BM25 + LLM-basiert ist.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. THE SimilarityEngine SHALL remove the `prefetch_embeddings` method entirely.
|
||||
2. THE SimilarityEngine SHALL remove the `_embed` method entirely.
|
||||
3. THE VocabularyCache SHALL remove `_preload_competence_vocab` entirely.
|
||||
4. THE VocabularyCache SHALL remove `_preload_role_vocab` (or equivalent role embedding preload logic) entirely.
|
||||
5. THE VocabularyCache SHALL remove `infer_competences` if embedding-based competence inference is no longer used.
|
||||
6. THE EmbeddingCache module SHALL be removed entirely, since no code path requires cached embeddings.
|
||||
7. THE AzureOpenAIClient SHALL remove the embedding API method (e.g. `get_embeddings` or equivalent batch embedding call), retaining only chat completion methods.
|
||||
8. THE config.py SHALL remove embedding-related configuration fields (e.g. `embedding_model`, `embedding_dimensions`, embedding batch size settings).
|
||||
9. THE config.py SHALL remove the `InferenceConfig` dataclass if `max_competences` and `min_similarity` are no longer used.
|
||||
10. THE SimilarityConfig SHALL remove any fields related to embedding thresholds or embedding model selection.
|
||||
11. THE config.toml SHALL remove embedding-related configuration entries.
|
||||
12. THE MCP-Server SHALL remove the `_startup_preload_embeddings` function entirely if no embedding preloads remain.
|
||||
@@ -1,178 +0,0 @@
|
||||
# Implementation Plan: Entfernung Embedding-basierter Similarity – Umstellung auf BM25 + LLM
|
||||
|
||||
## Overview
|
||||
|
||||
Incremental removal of all embedding infrastructure and replacement with BM25 + LLM. Tasks are ordered to avoid breaking the system mid-way: first add new LLM-based methods, then remove embedding code paths, then clean up config and delete dead modules.
|
||||
|
||||
## Tasks
|
||||
|
||||
- [x] 1. Simplify AzureOpenAIClient to remove embedding methods
|
||||
- [x] 1.1 Remove `embeddings` and `get_embeddings_batch` methods from `AzureOpenAIClient`
|
||||
- Remove the `self._emb` AsyncAzureOpenAI client instance
|
||||
- Remove constructor parameters: `embedding_api_key`, `embedding_deployment`, `embedding_batch_size`
|
||||
- Keep only `chat_completion` method and its supporting `self._chat` client
|
||||
- Make `self._chat` the primary client (no longer optional/conditional)
|
||||
- Remove the guard clause in `chat_completion` that checks for missing config
|
||||
- _Requirements: 6.7_
|
||||
|
||||
- [x] 1.2 Update `AzureOpenAIClient` constructor signature
|
||||
- New required params: `endpoint`, `api_version`, `chat_deployment`, `llm_api_key`
|
||||
- Keep optional: `timeout_s`, `max_retries`, `cost_tracker`
|
||||
- Remove cost tracker calls for embedding requests (`log_embedding_request`, `log_embedding_batch_request`)
|
||||
- _Requirements: 6.7_
|
||||
|
||||
- [-] 2. Refactor SimilarityEngine to remove embedding methods and add LLM role similarity
|
||||
- [x] 2.1 Remove embedding-related methods and properties from `SimilarityEngine`
|
||||
- Remove methods: `prefetch_embeddings`, `_embed`, `get_embedding_for_cache_key`, `get_embeddings_for_cache_keys`, `_aggregate_similarity`, `_per_skill_similarity`
|
||||
- Remove properties: `embedding_model`, `embedding_dimensions`, `use_bm25_search`
|
||||
- Remove constructor parameters: `cache`, `embedding_model`, `embedding_dimensions`, `strategy`, `use_bm25_search`
|
||||
- Remove module-level helpers: `_cache_key`, `cosine_similarity`
|
||||
- Update constructor to accept only: `client`, `cost_tracker`, `use_auto_tagging`, `auto_tagger`
|
||||
- Add `_role_similarity_cache: dict[tuple[str, str], float]` to constructor
|
||||
- _Requirements: 1.3, 1.4, 1.5, 1.6, 1.7, 6.1, 6.2_
|
||||
|
||||
- [x] 2.2 Simplify `compute_competence_similarity` to always use BM25+RRF
|
||||
- Remove any strategy/conditional branching
|
||||
- Always call `_bm25_rrf_similarity` directly (with optional auto-tagging expansion)
|
||||
- _Requirements: 1.1, 1.2_
|
||||
|
||||
- [x] 2.3 Implement LLM-based `compute_role_similarity`
|
||||
- Replace embedding cosine similarity with LLM chat completion
|
||||
- Add `_ROLE_SIMILARITY_SYSTEM_PROMPT` constant
|
||||
- Implement symmetric cache key: `tuple(sorted((role_a_lower, role_b_lower)))`
|
||||
- Return 0.0 for bad/empty/None/"(unknown)" roles without LLM call
|
||||
- Return 1.0 for identical roles (case-insensitive) without LLM call
|
||||
- Parse JSON response, clamp score to [0.0, 1.0]
|
||||
- On any exception: return 0.0, do not cache failed results
|
||||
- _Requirements: 5.1, 5.2, 5.3, 5.4, 5.5, 5.6, 5.8_
|
||||
|
||||
- [x] 2.4 Add `clear_role_cache` method
|
||||
- Clears `_role_similarity_cache` dict
|
||||
- _Requirements: 5.8_
|
||||
|
||||
- [x] 2.5 Write property test for BM25 zero-score on disjoint tokens
|
||||
- **Property 1: BM25 Zero-Score für fehlenden Token-Overlap**
|
||||
- **Validates: Requirements 1.1**
|
||||
|
||||
- [x] 2.6 Write property test for role similarity value range and identity
|
||||
- **Property 5: compute_role_similarity Wertebereich**
|
||||
- **Validates: Requirements 5.1**
|
||||
|
||||
- [x] 2.7 Write property test for role similarity cache symmetry
|
||||
- **Property 6: Rollen-Similarity-Cache ist symmetrisch und idempotent**
|
||||
- **Validates: Requirements 5.8**
|
||||
|
||||
- [-] 3. Refactor VocabularyCache to use LLM-based role inference
|
||||
- [x] 3.1 Rewrite `VocabularyCache` class
|
||||
- Remove all embedding-related methods: `preload`, `_preload_role_vocab`, `_preload_competence_vocab`, `ensure_task_embedding`, `infer_competences`
|
||||
- Remove old `infer_primary_role` (embedding-based)
|
||||
- Remove properties: `roles`, `competences`
|
||||
- Remove constructor dependencies on `SimilarityEngine` and `EmbeddingCache`
|
||||
- New constructor accepts only: `db: DBClient`, `client: AzureOpenAIClient`
|
||||
- Remove module-level helpers: `role_vocab_cache_key`, `competence_vocab_cache_key`, `task_id_cache_key`, `VocabItem`
|
||||
- _Requirements: 4.3, 6.3, 6.4, 6.5_
|
||||
|
||||
- [x] 3.2 Implement new LLM-based `infer_primary_role`
|
||||
- Accept `task_text: str` keyword argument (no more `task_embedding`)
|
||||
- Fetch all role names from DB via `self._db.get_all_role_names()`
|
||||
- Build LLM prompt with task text and available roles list
|
||||
- Parse JSON response: `{"role": "...", "confidence": 0.0-1.0}`
|
||||
- Validate returned role exists in DB role list
|
||||
- Return `None` on empty text, empty role list, or any exception
|
||||
- _Requirements: 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8_
|
||||
|
||||
- [x] 3.3 Write property test for infer_primary_role output validity
|
||||
- **Property 3: infer_primary_role Ausgabe-Validität**
|
||||
- **Validates: Requirements 3.1, 3.4**
|
||||
|
||||
- [x] 3.4 Write property test for graceful degradation on LLM failure
|
||||
- **Property 4: Graceful Degradation bei LLM-Fehler**
|
||||
- **Validates: Requirements 3.5, 5.6**
|
||||
|
||||
- [x] 4. Checkpoint - Ensure all tests pass
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
- [-] 5. Update Matcher to unconditionally build BM25 index
|
||||
- [x] 5.1 Remove `use_bm25_search` conditional in `Matcher.match`
|
||||
- Remove `if self._sim.use_bm25_search` guard around BM25 index construction
|
||||
- Always build global BM25 index when `filtered` is non-empty
|
||||
- _Requirements: 2.6_
|
||||
|
||||
- [x] 5.2 Write property test for unconditional BM25 index building
|
||||
- **Property 2: Matcher baut BM25-Index bedingungslos**
|
||||
- **Validates: Requirements 2.6**
|
||||
|
||||
- [x] 6. Update MCP Server to remove embedding preload and wire new components
|
||||
- [x] 6.1 Remove embedding preload infrastructure from `mcp_server.py`
|
||||
- Remove `_startup_preload_embeddings` function
|
||||
- Remove `_deferred_preload` function and `_preload_started` flag
|
||||
- Remove `_ensure_preloaded` function
|
||||
- Remove all `await _ensure_preloaded()` calls in tool handlers
|
||||
- Remove `EmbeddingCache` import and instantiation
|
||||
- _Requirements: 4.1, 4.2, 4.4, 4.5, 6.12_
|
||||
|
||||
- [x] 6.2 Update component wiring in `build_server`
|
||||
- Construct `AzureOpenAIClient` with new simplified signature (no embedding params)
|
||||
- Construct `SimilarityEngine` with new signature (client, cost_tracker, use_auto_tagging, auto_tagger)
|
||||
- Construct `VocabularyCache` with new signature (db, client)
|
||||
- _Requirements: 6.7_
|
||||
|
||||
- [x] 6.3 Update `infer_primary_role` tool handler
|
||||
- Remove embedding generation step
|
||||
- Call `vocab_cache.infer_primary_role(task_text=text)` directly with task text
|
||||
- _Requirements: 3.9_
|
||||
|
||||
- [x] 6.4 Remove or simplify `validate_task_requirements` tool if it depends on embeddings
|
||||
- Remove embedding-based `infer_competences` usage
|
||||
- Either remove the tool entirely or replace with a DB-field-only version
|
||||
- _Requirements: 6.5_
|
||||
|
||||
- [x] 7. Simplify config.py and config.toml
|
||||
- [x] 7.1 Remove embedding-related dataclasses and fields from `config.py`
|
||||
- Delete `EmbeddingCacheConfig` dataclass
|
||||
- Delete `InferenceConfig` dataclass
|
||||
- Remove from `AzureOpenAIConfig`: `embedding_deployment`, `embedding_batch_size`
|
||||
- Remove from `SimilarityConfig`: `embedding_model`, `embedding_dimensions`, `strategy`, `use_bm25_search`
|
||||
- Remove `inference` field from `MatchingConfig`
|
||||
- Remove `embedding_cache` field from `AppConfig`
|
||||
- _Requirements: 2.1, 2.2, 6.8, 6.9, 6.10_
|
||||
|
||||
- [x] 7.2 Update `load_config` / `_parse_azure_openai` validation
|
||||
- Remove `AZURE_OPENAI_EMBEDDING_API_KEY` environment variable check
|
||||
- Make `AZURE_OPENAI_LLM_API_KEY` always required
|
||||
- Make `chat_deployment` required
|
||||
- Remove `embedding_dimensions == 3072` validation
|
||||
- Remove `strategy` validation
|
||||
- Remove `[matching.inference]` parsing
|
||||
- Remove `[embedding_cache]` parsing
|
||||
- _Requirements: 2.3, 2.4_
|
||||
|
||||
- [x] 7.3 Update `config.toml` and `config.toml.example`
|
||||
- Remove `[embedding_cache]` section
|
||||
- Remove from `[matching.similarity]`: `embedding_model`, `embedding_dimensions`, `strategy`, `use_bm25_search`
|
||||
- Remove from `[azure_openai]`: `embedding_deployment`, `embedding_batch_size`
|
||||
- Remove `[matching.inference]` section
|
||||
- _Requirements: 6.11_
|
||||
|
||||
- [x] 8. Delete EmbeddingCache module and related tests
|
||||
- [x] 8.1 Delete `src/teamlandkarte_mcp/cache/embedding_cache.py`
|
||||
- _Requirements: 6.6_
|
||||
|
||||
- [x] 8.2 Remove or update tests that reference embedding functionality
|
||||
- Delete tests for `EmbeddingCache`
|
||||
- Update tests for `SimilarityEngine` to use new constructor
|
||||
- Update tests for `VocabularyCache` to use new constructor
|
||||
- Update tests for `AzureOpenAIClient` to use new constructor
|
||||
- Remove any test fixtures that create embedding mocks
|
||||
- _Requirements: 6.6_
|
||||
|
||||
- [x] 9. Final checkpoint - Ensure all tests pass
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
## Notes
|
||||
|
||||
- Tasks marked with `*` are optional and can be skipped for faster MVP
|
||||
- Each task references specific requirements for traceability
|
||||
- Checkpoints ensure incremental validation
|
||||
- Property tests validate universal correctness properties from the design document
|
||||
- Order ensures no broken intermediate states: new LLM methods added before old embedding paths removed
|
||||
@@ -1 +0,0 @@
|
||||
{"specId": "1a31d9bb-f189-4374-95b4-1bd117d009b6", "workflowType": "requirements-first", "specType": "feature"}
|
||||
@@ -1,222 +0,0 @@
|
||||
# Design Document: Task Name Column
|
||||
|
||||
## Overview
|
||||
|
||||
This feature adds the `name__c` database column to the Task model and all task-related queries, enabling users to see and look up tasks by their short human-readable name. The changes span four layers:
|
||||
|
||||
1. **Model** – Add an optional `name` field to the `Task` dataclass.
|
||||
2. **Database** – Modify SQL queries in `TrinoClient` to SELECT `name__c`; add a new `get_task_by_name` method.
|
||||
3. **Protocol** – Extend the `DBClient` protocol with `get_task_by_name`.
|
||||
4. **MCP Tools** – Include the name in list/detail output; resolve name-based lookups in `get_task_details`.
|
||||
|
||||
## Architecture
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A[MCP Tool: list_open_tasks] -->|calls| B[TrinoClient.get_open_tasks]
|
||||
C[MCP Tool: get_task_details] -->|calls| D[TrinoClient.get_task_by_id]
|
||||
C -->|calls| E[TrinoClient.get_task_by_name]
|
||||
B --> F[(Trino DB: name__c)]
|
||||
D --> F
|
||||
E --> F
|
||||
B --> G[Task dataclass with name field]
|
||||
D --> G
|
||||
E --> G
|
||||
```
|
||||
|
||||
The identifier resolution logic in `get_task_details` will attempt `get_task_by_id` first. If that returns `None`, it falls back to `get_task_by_name`. This keeps backward compatibility for callers passing IDs while enabling name-based lookup without a separate tool.
|
||||
|
||||
## Components and Interfaces
|
||||
|
||||
### 1. Task Dataclass (`models.py`)
|
||||
|
||||
Add a single field:
|
||||
|
||||
```python
|
||||
@dataclass(frozen=True)
|
||||
class Task:
|
||||
id: str
|
||||
name: Optional[str] # NEW – maps to name__c
|
||||
title: str
|
||||
description: str
|
||||
start_date: Optional[date]
|
||||
end_date: Optional[date]
|
||||
created_date: datetime
|
||||
skills: list[str] = field(default_factory=list)
|
||||
```
|
||||
|
||||
The field is `Optional[str]` because the database column can be NULL.
|
||||
|
||||
### 2. DBClient Protocol (`database/types.py`)
|
||||
|
||||
Add one method:
|
||||
|
||||
```python
|
||||
def get_task_by_name(self, name: str) -> Task | None:
|
||||
"""Fetch one published task by its short name (name__c)."""
|
||||
raise NotImplementedError
|
||||
```
|
||||
|
||||
### 3. TrinoClient (`database/trino_client.py`)
|
||||
|
||||
**SQL changes** – Every SELECT that builds a `Task` must include `t.name__c`. Affected methods:
|
||||
- `get_open_tasks` – add `t.name__c` to both the unlimited and CTE-limited queries.
|
||||
- `get_task_by_id` – add `t.name__c` to the SELECT.
|
||||
|
||||
**New method** – `get_task_by_name`:
|
||||
|
||||
```python
|
||||
def get_task_by_name(self, name: str) -> Optional[Task]:
|
||||
"""Fetch a single published task by name__c (case-sensitive)."""
|
||||
query = """
|
||||
SELECT
|
||||
t.id,
|
||||
t.name__c,
|
||||
t.title__c,
|
||||
t.description__c,
|
||||
t.startdate__c,
|
||||
t.enddate__c,
|
||||
t.createddate,
|
||||
s.skillname__c
|
||||
FROM beschaffungstool_kmp_task_latest t
|
||||
LEFT JOIN beschaffungstool_kmp_skill_latest s
|
||||
ON t.id = s.task__c
|
||||
WHERE t.name__c = ? AND t.status__c = ?
|
||||
"""
|
||||
# Same row-assembly logic as get_task_by_id
|
||||
```
|
||||
|
||||
### 4. MCP Server (`mcp_server.py`)
|
||||
|
||||
**`list_open_tasks`** – Add a "Name" column to the output table (between task_id and Title).
|
||||
|
||||
**`get_task_details`** – Change the resolution logic:
|
||||
|
||||
```python
|
||||
@mcp.tool()
|
||||
async def get_task_details(task_id: str) -> str:
|
||||
# 1. Try by ID
|
||||
task = db_client.get_task_by_id(task_id)
|
||||
# 2. Fallback: try by name
|
||||
if task is None:
|
||||
task = db_client.get_task_by_name(task_id)
|
||||
# 3. Not found
|
||||
if task is None:
|
||||
return f"Task not found or not published: {task_id}"
|
||||
# ... rest unchanged, but include task.name in the summary table
|
||||
```
|
||||
|
||||
Add "Name" to the summary table header and row.
|
||||
|
||||
## Data Models
|
||||
|
||||
### Task Table Schema (relevant columns)
|
||||
|
||||
| Column | Type | Nullable | Maps to |
|
||||
|--------|------|----------|---------|
|
||||
| `id` | VARCHAR | No | `Task.id` |
|
||||
| `name__c` | VARCHAR | Yes | `Task.name` |
|
||||
| `title__c` | VARCHAR | Yes | `Task.title` |
|
||||
| `description__c` | VARCHAR | Yes | `Task.description` |
|
||||
| `startdate__c` | DATE | Yes | `Task.start_date` |
|
||||
| `enddate__c` | DATE | Yes | `Task.end_date` |
|
||||
| `createddate` | TIMESTAMP | No | `Task.created_date` |
|
||||
|
||||
### Task Dataclass (after change)
|
||||
|
||||
```python
|
||||
@dataclass(frozen=True)
|
||||
class Task:
|
||||
id: str
|
||||
name: Optional[str]
|
||||
title: str
|
||||
description: str
|
||||
start_date: Optional[date]
|
||||
end_date: Optional[date]
|
||||
created_date: datetime
|
||||
skills: list[str] = field(default_factory=list)
|
||||
```
|
||||
|
||||
|
||||
## Correctness Properties
|
||||
|
||||
*A property is a characteristic or behavior that should hold true across all valid executions of a system — essentially, a formal statement about what the system should do. Properties serve as the bridge between human-readable specifications and machine-verifiable correctness guarantees.*
|
||||
|
||||
### Property 1: Task name field preserves database value
|
||||
|
||||
*For any* database row with a `name__c` value (including NULL), when the row is mapped to a `Task` object, `Task.name` must equal the original `name__c` value (with NULL mapped to `None`).
|
||||
|
||||
**Validates: Requirements 1.1, 1.2, 1.3, 1.4**
|
||||
|
||||
### Property 2: Task output contains name
|
||||
|
||||
*For any* `Task` with a non-None `name` field, the formatted output string (from both `list_open_tasks` and `get_task_details`) must contain that name value as a substring.
|
||||
|
||||
**Validates: Requirements 2.1, 2.2**
|
||||
|
||||
### Property 3: Name lookup returns correct task
|
||||
|
||||
*For any* task that exists in the database with a given `name__c` value, calling `get_task_by_name` with that exact name must return a `Task` whose `name` field equals the queried name.
|
||||
|
||||
**Validates: Requirements 3.2**
|
||||
|
||||
### Property 4: Name resolution equivalence
|
||||
|
||||
*For any* published task with a non-None name, calling `get_task_details` with the task's name must produce the same task data as calling `get_task_details` with the task's ID.
|
||||
|
||||
**Validates: Requirements 3.4**
|
||||
|
||||
### Property 5: Unknown identifier returns not-found
|
||||
|
||||
*For any* string that is neither a valid published task ID nor a valid published task name, `get_task_details` must return a not-found message.
|
||||
|
||||
**Validates: Requirements 3.3, 3.5**
|
||||
|
||||
## Error Handling
|
||||
|
||||
| Scenario | Behavior |
|
||||
|----------|----------|
|
||||
| `name__c` is NULL in DB | `Task.name` is set to `None`; output displays empty or omits the name cell |
|
||||
| `get_task_by_name` receives empty string | Returns `None` (no match) |
|
||||
| `get_task_details` identifier matches neither ID nor name | Returns `"Task not found or not published: {identifier}"` |
|
||||
| Multiple tasks share the same `name__c` (data quality issue) | `get_task_by_name` returns the first match (ORDER BY createddate DESC) |
|
||||
|
||||
The resolution order in `get_task_details` (ID first, then name) ensures that if a name happens to look like an ID, the ID lookup takes precedence. This avoids ambiguity.
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
### Unit Tests
|
||||
|
||||
- Verify `Task` dataclass construction with `name=None` and `name="ABC-123"`.
|
||||
- Verify `list_open_tasks` output table includes a "Name" column.
|
||||
- Verify `get_task_details` output table includes the name value.
|
||||
- Verify `get_task_details` falls back to `get_task_by_name` when ID lookup returns None.
|
||||
- Verify `get_task_by_name` returns None for empty string input.
|
||||
|
||||
### Property-Based Tests
|
||||
|
||||
Use **Hypothesis** (Python PBT library) with a minimum of 100 iterations per property.
|
||||
|
||||
Each test must be tagged with a comment referencing the design property:
|
||||
|
||||
```python
|
||||
# Feature: task-name-column, Property 1: Task name field preserves database value
|
||||
```
|
||||
|
||||
**Property 1** – Generate random `(name__c_value | None)` values, construct Task objects via the row-mapping helper, assert `task.name == input_value`.
|
||||
|
||||
**Property 2** – Generate random Task objects with non-None names (using `st.text(min_size=1)`), format them through the output functions, assert the name string appears in the output.
|
||||
|
||||
**Property 3** – Generate a random set of tasks with unique names, mock the DB cursor to return those rows, call `get_task_by_name(name)`, assert the returned task's name matches.
|
||||
|
||||
**Property 4** – Generate a task with a non-None name, mock both `get_task_by_id` and `get_task_by_name` to return the same task, call `get_task_details` with the name and with the ID, assert both outputs are identical.
|
||||
|
||||
**Property 5** – Generate random strings, filter out any that match known task IDs or names in the mock data, call `get_task_details`, assert the output contains "not found".
|
||||
|
||||
### Test Configuration
|
||||
|
||||
```python
|
||||
from hypothesis import given, settings, strategies as st
|
||||
|
||||
@settings(max_examples=100)
|
||||
```
|
||||
@@ -1,47 +0,0 @@
|
||||
# Requirements Document
|
||||
|
||||
## Introduction
|
||||
|
||||
The Teamlandkarte MCP server currently fetches tasks from the database without including the `name` column (a short identifier). Users need to reference tasks by this short name in addition to the full ID or title. This feature adds the `name` column to all task queries and enables task lookup by name.
|
||||
|
||||
## Glossary
|
||||
|
||||
- **MCP_Server**: The Teamlandkarte MCP server that exposes tools for querying tasks and capacities.
|
||||
- **Task**: A published task record stored in the `beschaffungstool_kmp_task_latest` database table.
|
||||
- **Task_Name**: The `name__c` column in the task table, a short human-readable identifier for a task.
|
||||
- **DBClient**: The database client protocol that defines methods for fetching tasks and capacities.
|
||||
- **TrinoClient**: The concrete database client implementation that queries the Trino/Presto backend.
|
||||
|
||||
## Requirements
|
||||
|
||||
### Requirement 1: Include Task Name in Task Model
|
||||
|
||||
**User Story:** As a user, I want the task name (short ID) to be part of the task data, so that I can see and use it when browsing tasks.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE Task model SHALL include a `name` field of type optional string.
|
||||
2. WHEN the TrinoClient fetches open tasks, THE TrinoClient SHALL select the `name__c` column from the task table and populate the Task `name` field.
|
||||
3. WHEN the TrinoClient fetches a task by ID, THE TrinoClient SHALL select the `name__c` column from the task table and populate the Task `name` field.
|
||||
4. WHEN the task name column value is NULL in the database, THE Task `name` field SHALL be set to None.
|
||||
|
||||
### Requirement 2: Display Task Name in Output
|
||||
|
||||
**User Story:** As a user, I want to see the task name in task listings and detail views, so that I can reference tasks by their short name.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN the MCP_Server returns a list of open tasks, THE MCP_Server SHALL include the task name in the output for each task.
|
||||
2. WHEN the MCP_Server returns task details, THE MCP_Server SHALL include the task name in the summary table.
|
||||
|
||||
### Requirement 3: Look Up Task by Name
|
||||
|
||||
**User Story:** As a user, I want to retrieve task details by providing only the task name, so that I do not need to remember the full ID.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. THE DBClient protocol SHALL define a `get_task_by_name` method that accepts a task name string and returns a Task or None.
|
||||
2. WHEN a valid published task name is provided, THE TrinoClient SHALL return the matching Task with all fields populated.
|
||||
3. WHEN a task name that does not match any published task is provided, THE TrinoClient SHALL return None.
|
||||
4. WHEN the user provides a task name to the get_task_details tool, THE MCP_Server SHALL resolve the task using the name and return the full task details.
|
||||
5. IF the provided identifier matches neither a task ID nor a task name, THEN THE MCP_Server SHALL return a "not found" message.
|
||||
@@ -1,107 +0,0 @@
|
||||
# Implementation Plan: Task Name Column
|
||||
|
||||
## Overview
|
||||
|
||||
Add the `name__c` database column to the Task model and all task queries, enabling users to see and look up tasks by their short human-readable name. Changes span model, database layer, protocol, and MCP tool output.
|
||||
|
||||
## Tasks
|
||||
|
||||
- [x] 1. Add name field to Task model and update TrinoClient queries
|
||||
- [x] 1.1 Add `name: Optional[str]` field to the Task dataclass in `models.py`
|
||||
- Insert `name: Optional[str]` after `id` field
|
||||
- Field must be Optional since `name__c` can be NULL in the database
|
||||
- _Requirements: 1.1, 1.4_
|
||||
|
||||
- [x] 1.2 Update `get_open_tasks` SQL queries in `trino_client.py` to SELECT `t.name__c`
|
||||
- Add `t.name__c` to the unlimited query SELECT clause
|
||||
- Add `t.name__c` to the CTE inner SELECT and outer SELECT in the limited query
|
||||
- Update row unpacking to extract the name value
|
||||
- Populate `name` field in Task construction (map NULL to None)
|
||||
- _Requirements: 1.2, 1.4_
|
||||
|
||||
- [x] 1.3 Update `get_task_by_id` SQL query in `trino_client.py` to SELECT `t.name__c`
|
||||
- Add `t.name__c` to the SELECT clause
|
||||
- Update row unpacking to extract the name value
|
||||
- Populate `name` field in Task construction (map NULL to None)
|
||||
- _Requirements: 1.3, 1.4_
|
||||
|
||||
- [ ]* 1.4 Write property test: Task name field preserves database value
|
||||
- **Property 1: Task name field preserves database value**
|
||||
- Generate random `(name__c_value | None)` values using Hypothesis
|
||||
- Construct Task objects via the row-mapping pattern, assert `task.name == input_value`
|
||||
- **Validates: Requirements 1.1, 1.2, 1.3, 1.4**
|
||||
|
||||
- [x] 2. Add `get_task_by_name` to protocol and TrinoClient
|
||||
- [x] 2.1 Add `get_task_by_name` method to `DBClient` protocol in `database/types.py`
|
||||
- Method signature: `def get_task_by_name(self, name: str) -> Task | None`
|
||||
- Include docstring: "Fetch one published task by its short name (name__c)."
|
||||
- _Requirements: 3.1_
|
||||
|
||||
- [x] 2.2 Implement `get_task_by_name` in `TrinoClient`
|
||||
- SQL query selects same columns as `get_task_by_id` plus `t.name__c`
|
||||
- WHERE clause filters on `t.name__c = ?` and `t.status__c = ?`
|
||||
- ORDER BY `t.createddate DESC` to handle duplicates (return first match)
|
||||
- Reuse same row-assembly logic as `get_task_by_id`
|
||||
- Return None for empty string input or no matching rows
|
||||
- _Requirements: 3.1, 3.2, 3.3_
|
||||
|
||||
- [ ]* 2.3 Write property test: Name lookup returns correct task
|
||||
- **Property 3: Name lookup returns correct task**
|
||||
- Generate random tasks with unique names, mock DB cursor
|
||||
- Call `get_task_by_name(name)`, assert returned task's name matches
|
||||
- **Validates: Requirements 3.2**
|
||||
|
||||
- [x] 3. Checkpoint
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
- [x] 4. Update MCP tool output formatting
|
||||
- [x] 4.1 Add "Name" column to `list_open_tasks` output table
|
||||
- Add "Name" to the header list (between task_id and Title)
|
||||
- Add `str(task.name or "")` to each row
|
||||
- Update the empty-row fallback to include the extra column
|
||||
- _Requirements: 2.1_
|
||||
|
||||
- [x] 4.2 Add "Name" to `get_task_details` summary table
|
||||
- Add "Name" to the header list
|
||||
- Add `str(task.name or "")` to the row data
|
||||
- _Requirements: 2.2_
|
||||
|
||||
- [ ]* 4.3 Write property test: Task output contains name
|
||||
- **Property 2: Task output contains name**
|
||||
- Generate random Task objects with non-None names using `st.text(min_size=1)`
|
||||
- Format through output functions, assert name appears as substring
|
||||
- **Validates: Requirements 2.1, 2.2**
|
||||
|
||||
- [x] 5. Update `get_task_details` resolution logic
|
||||
- [x] 5.1 Add name-based fallback resolution in `get_task_details` tool
|
||||
- After `get_task_by_id` returns None, call `db_client.get_task_by_name(task_id)`
|
||||
- If both return None, return "Task not found or not published: {task_id}"
|
||||
- _Requirements: 3.4, 3.5_
|
||||
|
||||
- [ ]* 5.2 Write property test: Name resolution equivalence
|
||||
- **Property 4: Name resolution equivalence**
|
||||
- Generate a task with non-None name, mock both lookup methods
|
||||
- Call `get_task_details` with name and with ID, assert outputs identical
|
||||
- **Validates: Requirements 3.4**
|
||||
|
||||
- [ ]* 5.3 Write property test: Unknown identifier returns not-found
|
||||
- **Property 5: Unknown identifier returns not-found**
|
||||
- Generate random strings not matching any known task ID or name
|
||||
- Call `get_task_details`, assert output contains "not found"
|
||||
- **Validates: Requirements 3.3, 3.5**
|
||||
|
||||
- [x] 6. Update FakeDBClient in tests and add `get_task_by_name` stub
|
||||
- Add `name` field to `FakeTask` in existing test files
|
||||
- Add `get_task_by_name` method to `FakeDBClient`
|
||||
- Update existing test assertions that check table headers (add "Name" column)
|
||||
- _Requirements: 3.1_
|
||||
|
||||
- [x] 7. Final checkpoint
|
||||
- Ensure all tests pass, ask the user if questions arise.
|
||||
|
||||
## Notes
|
||||
|
||||
- Tasks marked with `*` are optional and can be skipped for faster MVP
|
||||
- Each task references specific requirements for traceability
|
||||
- Property tests use Hypothesis with `@settings(max_examples=100)`
|
||||
- The implementation language is Python (matching the existing codebase)
|
||||
@@ -1 +0,0 @@
|
||||
{"specId": "2977bff5-33df-4acc-a529-5b047c431436", "workflowType": "requirements-first", "specType": "feature"}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,231 +0,0 @@
|
||||
# Anforderungsdokument
|
||||
|
||||
## Einleitung
|
||||
|
||||
Dieses Dokument beschreibt die Anforderungen für ein neues Feature der Teamlandkarte: das **Matching gegen Team-Profile** als zusätzliche Profilart neben den bestehenden Kapazitätsprofilen. Bisher vergleicht das System eine offene Aufgabe ausschließlich mit Kapazitätsprofilen einzelner Mitarbeitender (`teamlandkarte_v_capacities_latest` und zugehörige Tabellen). Künftig soll der Nutzer pro Suchanfrage wählen können, ob eine Aufgabe gegen **Kapazitätsprofile** (bisheriges Verhalten) oder gegen **Team-Profile** (neue Profilart) gematcht wird.
|
||||
|
||||
Ein Team-Profil aggregiert Daten aus mehreren Datenbank-Views: Stammdaten und Beschreibung aus `teamlandkarte_v_teams_latest` (Spalten `about_us`, `offerings`, `interests`, `focus_name`), den Teamnamen über INNER JOIN mit `teamlandkarte_v_teammeter_organizational_units_latest` (Join `team_id = id`), Team-Kompetenzen aus `teamlandkarte_v_teammeter_team_competences_latest` (Join über `ouid`, mit Top-Kompetenz-Markierung über `top_competency`) sowie Team-Referenzen aus `teamlandkarte_v_team_references_latest` (Join über `ouid`, Partner-Auflösung über `teamlandkarte_v_partners_latest.id` via `partner_id`, plus Projektbeschreibung in `projects`).
|
||||
|
||||
Das neue Feature soll die bestehenden Matching-Verfahren (`score` und `llm_fulltext`) wiederverwenden, sodass der Nutzer beide Verfahren auch auf Team-Profile anwenden kann. MCP-Tools, Agenten-Konfigurationen, Architekturdokumentation und README werden entsprechend erweitert.
|
||||
|
||||
|
||||
## Glossar
|
||||
|
||||
- **MCP_Server**: Der Teamlandkarte MCP-Server (Modul `mcp_server.py`), der die MCP-Tools für Matching, Suche und Datenanzeige bereitstellt.
|
||||
- **DBClient**: Protokollklasse aus `database/types.py`, die alle Datenbankzugriffe abstrahiert.
|
||||
- **TrinoClient**: Konkrete `DBClient`-Implementierung (`database/trino_client.py`) für Trino/Presto.
|
||||
- **Matcher**: Bestehende, Score-basierte Matching-Komponente in `matching/matcher.py`.
|
||||
- **LLM_Fulltext_Matcher**: Bestehendes Modul für den LLM-basierten Volltext-Vergleich (`matching/llm_fulltext_matcher.py`).
|
||||
- **AzureOpenAIClient**: Wrapper für Azure-OpenAI-Aufrufe in `azure/openai_client.py`.
|
||||
- **LLM**: Large Language Model (Azure OpenAI Chat Completion).
|
||||
- **Capacity**: Frozen Dataclass `Capacity` in `models.py` (Kapazitätseintrag eines Mitarbeitenden).
|
||||
- **Team**: Neue Frozen Dataclass, die ein Team mit aggregierten Profilfeldern (Name, `about_us`, `offerings`, `interests`, `focus_name`, Kompetenzen, Referenzen) repräsentiert.
|
||||
- **Team_Id**: Wert der Spalte `team_id` in `teamlandkarte_v_teams_latest` bzw. der Spalte `id` in `teamlandkarte_v_teammeter_organizational_units_latest`. Eindeutiger fachlicher Identifikator eines Teams.
|
||||
- **Ouid**: Wert der Spalte `ouid`, die in `teamlandkarte_v_teams_latest`, `teamlandkarte_v_teammeter_team_competences_latest` und `teamlandkarte_v_team_references_latest` vorkommt und als Join-Schlüssel zwischen Stammdaten, Kompetenzen und Referenzen eines Teams dient.
|
||||
- **Team_Name**: Wert der Spalte mit dem Teamnamen aus `teamlandkarte_v_teammeter_organizational_units_latest`, ermittelt über INNER JOIN mit `teamlandkarte_v_teams_latest` (`team_id = id`).
|
||||
- **Team_About_Us**: Wert der Spalte `about_us` aus `teamlandkarte_v_teams_latest` (Beschreibung des Teams).
|
||||
- **Team_Offerings**: Wert der Spalte `offerings` aus `teamlandkarte_v_teams_latest` (Leistungen des Teams).
|
||||
- **Team_Interests**: Wert der Spalte `interests` aus `teamlandkarte_v_teams_latest` (Interessen des Teams).
|
||||
- **Team_Focus_Name**: Wert der Spalte `focus_name` aus `teamlandkarte_v_teams_latest` (Schwerpunkt des Teams).
|
||||
- **Team_Competence**: Eintrag aus `teamlandkarte_v_teammeter_team_competences_latest` mit den Feldern `competence_id` (Fremdschlüssel auf den Kompetenz-Namen analog zu Kapazitätskompetenzen) und `top_competency` (Boolean). Ein Team kann mehrere Team_Competence-Einträge haben (1:n über `ouid`).
|
||||
- **Top_Competency**: Boolean-Spalte `top_competency` aus `teamlandkarte_v_teammeter_team_competences_latest`, die kennzeichnet, ob eine Team_Competence eine Top-Kompetenz des Teams ist.
|
||||
- **Team_Reference**: Eintrag aus `teamlandkarte_v_team_references_latest` mit den Feldern `partner_id` (Fremdschlüssel auf `teamlandkarte_v_partners_latest.id`) und `projects` (Projektbeschreibung). Ein Team kann mehrere Team_Reference-Einträge haben (1:n über `ouid`).
|
||||
- **Partner**: Eintrag aus `teamlandkarte_v_partners_latest`. Eine Team_Reference verweist über die Spalte `partner_id` auf einen Partner; die Verknüpfung erfolgt über `teamlandkarte_v_team_references_latest.partner_id = teamlandkarte_v_partners_latest.id`.
|
||||
- **Partner_Name**: Wert der Spalte `name` aus `teamlandkarte_v_partners_latest`, der einer Team_Reference über `partner_id` zugeordnet ist. Ist `partner_id` `NULL` oder existiert kein passender Partner, gilt der Partner_Name als leere Zeichenkette.
|
||||
- **Team_Profile**: Aggregiertes Volltext-Profil eines Teams, bestehend aus `Team_Id`, `Team_Name`, `Team_Focus_Name`, `Team_About_Us`, `Team_Offerings`, `Team_Interests`, einer Liste von Kompetenznamen mit Top-Markierung sowie einer Liste von Referenzeinträgen (Partner_Name + Projektbeschreibung).
|
||||
- **Capacity_Profile**: Bestehendes aggregiertes Volltext-Profil einer Kapazität (Rolle, Kompetenzen, Beschreibung, Referenzen, Zertifikate).
|
||||
- **Task_Profile**: Bestehendes aggregiertes Volltext-Profil einer Aufgabe (Titel, Beschreibung, gesuchte Kompetenzen).
|
||||
- **Profile_Type**: Auswahlwert für die zu matchende Profilart. Erlaubte Werte: `capacity` (Kapazitätsprofile, bisheriges Verhalten) und `team` (Team-Profile, neu).
|
||||
- **Matching_Method**: Bestehender Auswahlwert für das Verfahren. Erlaubte Werte: `score` und `llm_fulltext`.
|
||||
- **Kategorie**: Eine der Ergebniskategorien `Top`, `Good`, `Partial`, `Low`, `Irrelevant`.
|
||||
- **Rationale**: Vom LLM erzeugte Kurzbegründung (1–2 Sätze) für die zugewiesene Kategorie (nur im `llm_fulltext`-Modus).
|
||||
- **find_matching_capacities**: MCP-Tool für die Suchrichtung Aufgabe→Kapazität (bestehend).
|
||||
- **find_matching_teams**: Neues MCP-Tool für die Suchrichtung Aufgabe→Team.
|
||||
- **SearchCache**: Bestehende Cache-Komponente für persistierte Suchergebnisse.
|
||||
- **Teamlandkarte_Agent**: GitHub-Copilot-Agent in `.github/agents/teamlandkarte_agent.md` und Kiro-Pendant in `.kiro/agents/teamlandkarte.md`.
|
||||
- **Architecture_Doc**: `docs/architecture.md`.
|
||||
- **Readme**: `README.md` im Repository-Root.
|
||||
|
||||
|
||||
## Anforderungen
|
||||
|
||||
### Anforderung 1: Auswahl des Profil-Typs für das Matching
|
||||
|
||||
**User Story:** Als Nutzer möchte ich beim Matching für eine offene Aufgabe entscheiden können, ob ich gegen Kapazitätsprofile oder gegen Team-Profile matche, damit ich je nach Fragestellung Personen oder Teams als Vorschläge erhalte.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. THE MCP_Server SHALL einen Profile_Type mit den erlaubten Werten `capacity` und `team` definieren.
|
||||
2. WHEN der Nutzer eine Aufgabe gegen Kapazitätsprofile matchen möchte, THE MCP_Server SHALL das bestehende Tool `find_matching_capacities` mit unverändertem Verhalten bereitstellen (Profile_Type implizit `capacity`).
|
||||
3. WHEN der Nutzer eine Aufgabe gegen Team-Profile matchen möchte, THE MCP_Server SHALL ein neues Tool `find_matching_teams` bereitstellen, das Profile_Type `team` realisiert.
|
||||
4. THE MCP_Server SHALL den verwendeten Profile_Type (`capacity` oder `team`) im Antwort-`META`-JSON sowie in der angezeigten Suchkonfiguration ausweisen.
|
||||
5. THE MCP_Server SHALL den Parameter `matching_method` (`score` oder `llm_fulltext`) auch im Tool `find_matching_teams` akzeptieren und mit den gleichen Default- und Validierungsregeln behandeln wie in `find_matching_capacities`.
|
||||
6. IF der Nutzer in `find_matching_teams` einen ungültigen Wert für `matching_method` übergibt, THEN THE MCP_Server SHALL eine Fehlermeldung zurückgeben, die die erlaubten Werte (`score`, `llm_fulltext`) auflistet, und die Suche nicht ausführen.
|
||||
|
||||
### Anforderung 2: Datenabfrage für Team-Stammdaten und Teamname
|
||||
|
||||
**User Story:** Als Entwickler möchte ich, dass das System Team-Stammdaten inklusive Teamnamen aus der Datenbank lädt, damit Team-Profile vollständig aufgebaut werden können.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. THE DBClient SHALL eine Methode bereitstellen, die alle aktiven Teams aus `teamlandkarte_v_teams_latest` zurückgibt, inklusive der Spalten `team_id`, `ouid`, `about_us`, `offerings`, `interests` und `focus_name`.
|
||||
2. THE DBClient SHALL den Team_Name pro Team über INNER JOIN von `teamlandkarte_v_teams_latest` mit `teamlandkarte_v_teammeter_organizational_units_latest` über die Bedingung `teamlandkarte_v_teams_latest.team_id = teamlandkarte_v_teammeter_organizational_units_latest.id` ermitteln.
|
||||
3. WHEN für ein Team kein passender Eintrag in `teamlandkarte_v_teammeter_organizational_units_latest` existiert, THE DBClient SHALL dieses Team durch den INNER JOIN aus dem Ergebnis ausschließen.
|
||||
4. WHEN eine der Spalten `about_us`, `offerings`, `interests` oder `focus_name` `NULL` oder leer ist, THE DBClient SHALL für das jeweilige Feld eine leere Zeichenkette zurückgeben.
|
||||
5. THE DBClient SHALL eine Methode bereitstellen, die ein einzelnes Team anhand seiner Team_Id (oder Ouid) zurückgibt, einschließlich Team_Name über denselben INNER JOIN.
|
||||
6. THE TrinoClient SHALL alle neuen SQL-Abfragen ausschließlich als `SELECT`-Statements ausführen und die bestehende Read-Only-Guard `_ensure_select_only` verwenden.
|
||||
7. THE TrinoClient SHALL die neuen Abfragen über die bestehende Connection-Pool-Infrastruktur und die Retry-Logik (`_retry`) ausführen.
|
||||
|
||||
|
||||
### Anforderung 3: Datenabfrage für Team-Kompetenzen
|
||||
|
||||
**User Story:** Als Entwickler möchte ich, dass das System die Kompetenzen eines Teams inklusive Top-Kompetenz-Markierung und aufgelöstem Kompetenz-Namen lädt, damit Team-Profile die fachlichen Fähigkeiten korrekt abbilden.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. THE DBClient SHALL eine Methode bereitstellen, die für eine gegebene Ouid alle zugeordneten Team_Competence-Einträge aus `teamlandkarte_v_teammeter_team_competences_latest` zurückgibt, inklusive der Felder `competence_id` und `top_competency`.
|
||||
2. THE DBClient SHALL den Kompetenz-Namen pro Team_Competence durch denselben Join-Mechanismus ermitteln, der bereits für Kapazitäts-Kompetenzen verwendet wird, sodass aus der `competence_id` der lesbare Kompetenz-Name abgeleitet wird.
|
||||
3. THE DBClient SHALL eine Batch-Variante bereitstellen, die für eine Liste von Ouid-Werten alle zugehörigen Team_Competence-Einträge inklusive aufgelöster Kompetenz-Namen in höchstens einer SQL-Abfrage lädt.
|
||||
4. WHEN ein Team keine Team_Competence-Einträge besitzt, THE DBClient SHALL eine leere Liste für dieses Team zurückgeben.
|
||||
5. WHEN `top_competency` für einen Team_Competence-Eintrag `NULL` ist, THE DBClient SHALL den Wert als `false` interpretieren.
|
||||
6. WHEN die `competence_id` eines Team_Competence-Eintrags zu keinem Kompetenz-Namen aufgelöst werden kann, THE DBClient SHALL diesen Eintrag aus dem Ergebnis ausschließen.
|
||||
7. THE DBClient SHALL die Reihenfolge der Team_Competence-Einträge pro Team deterministisch zurückgeben (primär: Top-Kompetenzen vor Nicht-Top-Kompetenzen, sekundär: Kompetenz-Name aufsteigend).
|
||||
|
||||
### Anforderung 4: Datenabfrage für Team-Referenzen
|
||||
|
||||
**User Story:** Als Entwickler möchte ich, dass das System die Referenzen eines Teams inklusive Partner-Namen und Projektbeschreibung lädt, damit Team-Profile bisherige Projekte und Auftraggeber abbilden.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. THE DBClient SHALL eine Methode bereitstellen, die für eine gegebene Ouid alle zugeordneten Team_Reference-Einträge aus `teamlandkarte_v_team_references_latest` zurückgibt, einschließlich der Spalte `projects` und des über `partner_id` aufgelösten Partner_Name.
|
||||
2. THE DBClient SHALL den Partner_Name pro Team_Reference über LEFT JOIN auf `teamlandkarte_v_partners_latest` mit der Bedingung `teamlandkarte_v_team_references_latest.partner_id = teamlandkarte_v_partners_latest.id` ermitteln und die Spalte `name` als Partner_Name übernehmen.
|
||||
3. THE DBClient SHALL eine Batch-Variante bereitstellen, die für eine Liste von Ouid-Werten alle zugehörigen Team_Reference-Einträge inklusive aufgelöster Partner_Name in höchstens einer SQL-Abfrage lädt; der Partner-Join SHALL Bestandteil derselben Referenz-Abfrage sein und keine zusätzliche SQL-Abfrage erzeugen.
|
||||
4. WHEN ein Team keine Team_Reference-Einträge besitzt, THE DBClient SHALL eine leere Liste für dieses Team zurückgeben.
|
||||
5. IF die `partner_id` einer Team_Reference `NULL` ist oder der Join auf `teamlandkarte_v_partners_latest` keinen Treffer liefert, THEN THE DBClient SHALL den Partner_Name dieser Team_Reference als leere Zeichenkette zurückgeben und die Referenz dennoch mit dem Feld `projects` in der Ergebnisliste belassen.
|
||||
6. WHEN das Feld `projects` einer Team_Reference `NULL` oder ausschließlich Whitespace ist, THE DBClient SHALL diesen Eintrag aus dem Ergebnis ausschließen.
|
||||
7. THE DBClient SHALL die Reihenfolge der Team_Reference-Einträge pro Team deterministisch zurückgeben (z. B. nach Partner_Name aufsteigend, dann nach Projektbeschreibung aufsteigend).
|
||||
|
||||
|
||||
### Anforderung 5: Aufbau des Team_Profile
|
||||
|
||||
**User Story:** Als Entwickler möchte ich, dass das System aus den Datenbankfeldern ein konsistentes Volltext-Profil pro Team erzeugt, damit beide Matching-Verfahren eine einheitliche Eingabe erhalten.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. THE MCP_Server SHALL pro Team ein Team_Profile bilden, das die folgenden Felder enthält: `team_id`, `ouid`, `team_name`, `focus_name`, `about_us`, `offerings`, `interests`, `competences` (Liste von Einträgen mit Kompetenz-Namen und `top_competency`-Flag) und `references` (Liste von Einträgen mit `partner_name` und `projects`).
|
||||
2. WHEN ein Feld in der Datenbank leer oder `NULL` ist, THE MCP_Server SHALL das entsprechende Feld im Team_Profile mit einer leeren Zeichenkette bzw. einer leeren Liste belegen, ohne das gesamte Profil zu verwerfen.
|
||||
3. WHEN der Partner_Name eines Team_Reference-Eintrags leer ist, THE MCP_Server SHALL die Referenz dennoch in `references` aufnehmen und ausschließlich das Feld `projects` in die serialisierte Darstellung übernehmen, ohne einen Platzhaltertext für den Partner einzufügen.
|
||||
4. THE MCP_Server SHALL das Team_Profile in einer für das LLM lesbaren, deterministischen Textstruktur serialisieren, in der jedes Feld klar mit einer Überschrift gekennzeichnet ist (z. B. `Teamname:`, `Schwerpunkt:`, `Über uns:`, `Leistungen:`, `Interessen:`, `Kompetenzen:`, `Referenzen:`).
|
||||
5. THE MCP_Server SHALL Top-Kompetenzen in der serialisierten Darstellung erkennbar markieren (z. B. durch ein vorangestelltes Symbol oder das Suffix `(Top)`), sodass das LLM und der Nutzer Top-Kompetenzen von Nicht-Top-Kompetenzen unterscheiden können.
|
||||
6. THE MCP_Server SHALL jeden Eintrag im Abschnitt `Referenzen:` so darstellen, dass sowohl Partner_Name als auch Projekte für das LLM sichtbar sind (z. B. im Format `Partner: <partner_name> – Projekte: <projects>` oder als gleichwertige strukturierte Darstellung).
|
||||
7. THE MCP_Server SHALL die Reihenfolge der Felder in der serialisierten Darstellung über alle Teams konstant halten, sodass die Eingabe für das LLM bzw. den Score-Matcher deterministisch ist.
|
||||
|
||||
### Anforderung 6: Score-basiertes Matching für Team-Profile
|
||||
|
||||
**User Story:** Als Nutzer möchte ich Team-Profile auch im Score-basierten Modus matchen können, damit ich Teams mit denselben numerischen Bewertungen wie Kapazitäten vergleichen kann.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. WHEN `find_matching_teams` mit `matching_method = "score"` aufgerufen wird, THE MCP_Server SHALL das Score-basierte Matching auf Team-Profile anwenden und für jedes Team eine Competence Score, eine Role Score und eine Overall Score berechnen.
|
||||
2. THE MCP_Server SHALL die Competence Score eines Teams aus den Team_Competence-Einträgen berechnen, wobei die Liste der Kompetenz-Namen analog zur Liste der Kapazitäts-Kompetenzen verwendet wird.
|
||||
3. THE MCP_Server SHALL die Role Score eines Teams aus dem Team_Focus_Name als Stellvertreter für die Rolle berechnen, da Teams keine Rolle im Sinne einer Kapazität besitzen.
|
||||
4. WHERE Top-Kompetenzen vorhanden sind, THE MCP_Server SHALL Top-Kompetenzen bei der Berechnung der Competence Score höher gewichten als Nicht-Top-Kompetenzen, wobei der Gewichtungsfaktor in der Konfiguration unter dem Schlüssel `matching.team.top_competency_weight` mit Standardwert `1.5` einstellbar ist.
|
||||
5. THE MCP_Server SHALL die Ergebnisse in dieselben Kategorien (`Top`, `Good`, `Partial`, `Low`, `Irrelevant`) einordnen, die auch für Kapazitätsprofile gelten.
|
||||
6. THE MCP_Server SHALL die Ergebnistabelle für `find_matching_teams` im `score`-Modus mit den Spalten `Team Name`, `Schwerpunkt`, `Top-Kompetenzen`, `Role Score`, `Competence Score`, `Overall Score`, `Category` ausgeben.
|
||||
7. THE MCP_Server SHALL die Verfügbarkeit eines Teams nicht prüfen, da Team-Profile keinen Verfügbarkeitszeitraum besitzen; ein etwaig übergebener Zeitraum SHALL ignoriert und in der `Applied Filters`-Tabelle als nicht wirksam markiert werden.
|
||||
|
||||
|
||||
### Anforderung 7: LLM-Volltext-Matching für Team-Profile
|
||||
|
||||
**User Story:** Als Nutzer möchte ich Team-Profile auch im LLM-Volltext-Modus matchen können, damit der Vergleich auf Basis der Beschreibungstexte (about_us, offerings, interests) und Referenzen erfolgt.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. WHEN `find_matching_teams` mit `matching_method = "llm_fulltext"` aufgerufen wird, THE LLM_Fulltext_Matcher SHALL für jedes Team einen LLM-Vergleich zwischen Task_Profile und Team_Profile durchführen.
|
||||
2. THE LLM_Fulltext_Matcher SHALL pro Team genau eine Kategorie aus der Menge `Top`, `Good`, `Partial`, `Low`, `Irrelevant` zurückgeben.
|
||||
3. THE LLM_Fulltext_Matcher SHALL pro Team eine Rationale mit 1 bis 2 Sätzen zurückgeben, die die Zuweisung in die jeweilige Kategorie erläutert.
|
||||
4. THE LLM_Fulltext_Matcher SHALL die LLM-Antwort als strukturiertes JSON pro Team anfordern und parsen (Felder: `category`, `rationale`).
|
||||
5. IF das LLM für ein Team eine Kategorie zurückgibt, die nicht in der erlaubten Menge liegt, THEN THE LLM_Fulltext_Matcher SHALL dieses Team der Kategorie `Irrelevant` zuordnen und die Rationale durch einen Hinweis auf die ungültige LLM-Antwort ergänzen.
|
||||
6. IF der LLM-Aufruf für ein Team fehlschlägt, THEN THE LLM_Fulltext_Matcher SHALL dieses Team in einer separaten Fehlerliste ausweisen und es nicht als reguläres Ergebnis kategorisieren.
|
||||
7. THE LLM_Fulltext_Matcher SHALL die Ergebnisse nach Kategorie gruppieren und innerhalb jeder Kategorie eine deterministische Sortierreihenfolge anwenden (Sortierung primär nach Kategorie, sekundär nach `team_id` aufsteigend).
|
||||
8. WHEN `matching_method = "llm_fulltext"` verwendet wird, THE MCP_Server SHALL die Ergebnistabelle für `find_matching_teams` mit den Spalten `Team Name`, `Schwerpunkt`, `Top-Kompetenzen`, `Category`, `Begründung` ausgeben.
|
||||
|
||||
### Anforderung 8: Persistenz und Pagination der Team-Suche
|
||||
|
||||
**User Story:** Als Nutzer möchte ich auch bei einer Team-Suche durch Kategorien blättern und Filter anwenden können, damit der bestehende Such-Workflow konsistent bleibt.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. WHEN `find_matching_teams` ein Suchergebnis erzeugt, THE MCP_Server SHALL ein gültiges `search_id` zurückgeben, das mit `get_results_by_category` und `filter_search_results` verwendet werden kann.
|
||||
2. THE MCP_Server SHALL im persistierten Suchergebnis (`SearchCache`) ein Feld `search_type` mit dem Wert `team_search` hinterlegen, um Team-Suchen von Kapazitäts-Suchen (`capacity_search`) zu unterscheiden.
|
||||
3. THE MCP_Server SHALL im `META`-JSON des Suchergebnisses sowohl `search_type = "team_search"` als auch das verwendete `matching_method` ausweisen, damit Folgewerkzeuge das Schema korrekt interpretieren können.
|
||||
4. WHEN `get_results_by_category` ein Team-Suchergebnis paginiert, THE MCP_Server SHALL die Ergebnistabelle mit den für Teams definierten Spalten (`Team Name`, `Schwerpunkt`, `Top-Kompetenzen`, ...) ausgeben.
|
||||
5. WHEN `filter_search_results` ein Team-Suchergebnis filtert, THE MCP_Server SHALL die Filterung auf für Teams sinnvolle Filter beschränken (Schwerpunkt-Filter, Kompetenz-Filter, Top-Kompetenz-Filter).
|
||||
6. IF ein für Teams nicht anwendbarer Filter (z. B. `availability_date_start`, `availability_date_end`, `is_fully_available`) auf ein Team-Suchergebnis angewendet wird, THEN THE MCP_Server SHALL den Filter ignorieren und in der `Applied Filters`-Tabelle einen Hinweis aufnehmen, dass der Filter im Team-Suchmodus nicht wirksam ist.
|
||||
|
||||
### Anforderung 9: Detail- und Listen-Tools für Teams
|
||||
|
||||
**User Story:** Als Nutzer möchte ich einzelne Teams einsehen und eine Liste verfügbarer Teams abrufen können, damit ich Teams unabhängig von einem Matching-Lauf erkunden kann.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. THE MCP_Server SHALL ein Tool `list_teams` bereitstellen, das die ersten `limit` Teams (Default 20) als Markdown-Tabelle mit den Spalten `Team Id`, `Team Name`, `Schwerpunkt`, `Anzahl Kompetenzen`, `Anzahl Referenzen` ausgibt.
|
||||
2. THE MCP_Server SHALL ein Tool `get_team_details` bereitstellen, das anhand einer Team_Id ein einzelnes Team_Profile als Markdown-Tabelle plus Beschreibungstexte (`Über uns`, `Leistungen`, `Interessen`) und Listen (`Kompetenzen` mit Top-Markierung, `Referenzen` mit Partner_Name und Projektbeschreibung) ausgibt.
|
||||
3. IF kein Team mit der angegebenen Team_Id existiert, THEN THE MCP_Server SHALL eine Fehlermeldung zurückgeben, die die ungültige Team_Id nennt.
|
||||
4. THE MCP_Server SHALL die Listen `Kompetenzen` und `Referenzen` in der gleichen deterministischen Reihenfolge ausgeben, die der DBClient liefert (siehe Anforderungen 3.7 und 4.7).
|
||||
|
||||
|
||||
### Anforderung 10: Anpassung der Agenten-Konfigurationen
|
||||
|
||||
**User Story:** Als Nutzer möchte ich, dass sowohl der GitHub-Copilot-Agent `teamlandkarte_agent` als auch der Kiro-Pendant-Agent das neue Matching gegen Team-Profile kennen und mich aktiv nach der gewünschten Profilart fragen, damit das neue Feature über die Agenten nutzbar ist.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. THE Teamlandkarte_Agent SHALL in seiner Konfigurationsdatei (`.github/agents/teamlandkarte_agent.md`) und im Pendant `.kiro/agents/teamlandkarte.md` die Existenz und den Zweck der beiden Profile_Type-Werte `capacity` und `team` dokumentieren.
|
||||
2. WHEN der Nutzer eine Suche nach passenden Profilen für eine Aufgabe startet, THE Teamlandkarte_Agent SHALL den Nutzer explizit nach dem gewünschten Profile_Type (`capacity` oder `team`) fragen, sofern dieser nicht bereits aus dem Verlauf hervorgeht.
|
||||
3. THE Teamlandkarte_Agent SHALL die Skills/Workflows so erweitern, dass `find_matching_teams` als alternatives Such-Tool zu `find_matching_capacities` verfügbar ist und mit dem Parameter `matching_method` aufgerufen wird.
|
||||
4. THE Teamlandkarte_Agent SHALL den Nutzer darauf hinweisen, dass bei einer Team-Suche keine Verfügbarkeitsfilter wirksam sind und die Ergebnisspalten von einer Kapazitäts-Suche abweichen.
|
||||
5. THE Teamlandkarte_Agent SHALL den bestehenden Bestätigungs-Workflow (`show_pending_requirements`, `confirm_requirements`) beibehalten und für beide Profile_Type-Werte gleich anwenden.
|
||||
6. THE Teamlandkarte_Agent SHALL die neuen Detail- und Listen-Tools (`list_teams`, `get_team_details`) in den Skills/Workflows erwähnen.
|
||||
|
||||
### Anforderung 11: Aktualisierung von Architektur- und README-Dokumentation
|
||||
|
||||
**User Story:** Als Entwickler oder Onboardee möchte ich, dass `architecture.md` und `README.md` das Matching gegen Team-Profile beschreiben, damit ich Architektur und Nutzung des Systems korrekt verstehe.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. THE Architecture_Doc SHALL einen Abschnitt enthalten, der das Team_Profile als zusätzliche Profilart beschreibt, einschließlich seiner Felder, Datenquellen und der Verknüpfungen zwischen den Views.
|
||||
2. THE Architecture_Doc SHALL die zusätzlichen Datenquellen (`teamlandkarte_v_teams_latest`, `teamlandkarte_v_teammeter_organizational_units_latest`, `teamlandkarte_v_teammeter_team_competences_latest`, `teamlandkarte_v_team_references_latest`) im Datenmodell- und Schema-Verifikationsabschnitt aufführen, einschließlich der relevanten Spalten.
|
||||
3. THE Architecture_Doc SHALL die Verknüpfungen zwischen `teamlandkarte_v_teams_latest.team_id` und `teamlandkarte_v_teammeter_organizational_units_latest.id` (INNER JOIN für Team_Name) sowie über `ouid` zu Kompetenzen und Referenzen dokumentieren.
|
||||
4. THE Architecture_Doc SHALL die Verknüpfung zwischen `teamlandkarte_v_team_references_latest.partner_id` und `teamlandkarte_v_partners_latest.id` sowie die Übernahme der Spalte `name` als Partner_Name in das Team_Profile dokumentieren.
|
||||
5. THE Architecture_Doc SHALL den Profile_Type-Parameter und seine Wertebereiche im Tool-Surface-Abschnitt für die neuen und geänderten Tools dokumentieren.
|
||||
6. THE Architecture_Doc SHALL den Runtime-View für die Suchrichtung Aufgabe→Team in beiden Matching-Methoden (`score` und `llm_fulltext`) ergänzen.
|
||||
7. THE Readme SHALL im Quick-Start- und Usage-Abschnitt erklären, wie der Nutzer zwischen `capacity`- und `team`-Suche wählt.
|
||||
8. THE Readme SHALL die zusätzlichen Datenbank-Views aufführen, die der Server für Team-Profile liest, einschließlich der Join-Bedingungen für Team_Name (über `team_id`/`id`), Kompetenzen und Referenzen (über `ouid`) sowie Partner (über `partner_id`).
|
||||
9. THE Readme SHALL beschreiben, dass für Team-Suchen keine Verfügbarkeitsfilter angewendet werden und welche Ergebnisspalten in den jeweiligen Modi (`score`, `llm_fulltext`) ausgegeben werden.
|
||||
|
||||
|
||||
### Anforderung 12: Konfiguration und Schema-Verifikation
|
||||
|
||||
**User Story:** Als Betreiber möchte ich, dass die neuen Datenbank-Views beim Start des Servers verifiziert werden und dass relevante Defaults konfigurierbar sind, damit Fehlkonfigurationen früh erkannt werden.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. THE MCP_Server SHALL beim Start die Existenz der Spalten `team_id`, `ouid`, `about_us`, `offerings`, `interests`, `focus_name` in `teamlandkarte_v_teams_latest` über die Schema-Verifikation prüfen.
|
||||
2. THE MCP_Server SHALL beim Start die Existenz der Spalte `id` (sowie der Spalte für den Teamnamen) in `teamlandkarte_v_teammeter_organizational_units_latest` über die Schema-Verifikation prüfen.
|
||||
3. THE MCP_Server SHALL beim Start die Existenz der Spalten `ouid`, `competence_id`, `top_competency` in `teamlandkarte_v_teammeter_team_competences_latest` über die Schema-Verifikation prüfen.
|
||||
4. THE MCP_Server SHALL beim Start die Existenz der Spalten `ouid`, `partner_id`, `projects` in `teamlandkarte_v_team_references_latest` über die Schema-Verifikation prüfen.
|
||||
5. THE MCP_Server SHALL die Konfigurationsdatei `config.toml` um einen optionalen Schlüssel `matching.team.top_competency_weight` (Default `1.5`) erweitern, der die Gewichtung von Top-Kompetenzen im Score-Matching steuert.
|
||||
6. IF `matching.team.top_competency_weight` einen nicht-numerischen Wert oder einen Wert kleiner als `1.0` enthält, THEN THE MCP_Server SHALL beim Start einen `ConfigError` mit beschreibender Meldung werfen.
|
||||
7. THE MCP_Server SHALL alle bestehenden Tests so erweitern oder ergänzen, dass sowohl die Profile_Type-Werte `capacity` als auch `team` (mit beiden Matching-Methoden, gemocktem LLM und gemockter DB) abgedeckt sind.
|
||||
|
||||
### Anforderung 13: Round-Trip-Eigenschaft der Team-Profil-Serialisierung
|
||||
|
||||
**User Story:** Als Entwickler möchte ich sicherstellen, dass die deterministische Serialisierung eines Team_Profile stabil ist und sich semantisch identische Eingaben auf identische Ausgaben abbilden, damit die LLM-Eingabe reproduzierbar und cachebar ist.
|
||||
|
||||
#### Akzeptanzkriterien
|
||||
|
||||
1. FOR ALL Team-Profile mit identischen Feldwerten in identischer Reihenfolge, THE MCP_Server SHALL identische serialisierte Strings produzieren (Determinismus).
|
||||
2. WHEN ein Team_Profile zweimal hintereinander aus identischen Datenbankzeilen aufgebaut und serialisiert wird, THE MCP_Server SHALL beide Male denselben Serialisierungsstring produzieren (Idempotenz der Profil-Bildung).
|
||||
3. THE MCP_Server SHALL in der serialisierten Darstellung jedes Profilfeld mit einer eindeutigen, festen Überschrift versehen, sodass aus dem Serialisierungsstring die Zuordnung der Werte zu den Feldern eindeutig ablesbar ist.
|
||||
4. THE MCP_Server SHALL die Reihenfolge der Listen-Elemente (`competences`, `references`) in der serialisierten Darstellung mit der vom DBClient gelieferten Reihenfolge übereinstimmen lassen, sodass keine Sortier-Diskrepanzen zwischen DB-Schicht und Serialisierungs-Schicht entstehen.
|
||||
@@ -1,333 +0,0 @@
|
||||
# Implementierungsplan: Team-Profil-Matching
|
||||
|
||||
## Übersicht
|
||||
|
||||
Convert the feature design into a series of prompts for a code-generation LLM that will implement each step with incremental progress. Make sure that each prompt builds on the previous prompts, and ends with wiring things together. There should be no hanging or orphaned code that isn't integrated into a previous step. Focus ONLY on tasks that involve writing, modifying, or testing code.
|
||||
|
||||
Die Implementierung erfolgt in **Python** (entsprechend des bestehenden Codestils
|
||||
des Repositorys). Property-Based Tests verwenden **Hypothesis**, parallel zu den
|
||||
existierenden `tests/test_*_pbt.py`-Modulen. Sub-Tasks mit `*` sind optional und
|
||||
werden nicht automatisch implementiert (gemäß Projekt-Konventionen). Jede der
|
||||
14 Korrektheits-Eigenschaften aus dem Designdokument wird in genau einem
|
||||
PBT-Sub-Task realisiert und ist mit `**Property N**` und der validierten
|
||||
Anforderung annotiert.
|
||||
|
||||
## Tasks
|
||||
|
||||
- [x] 1. Datenmodelle und Konfiguration vorbereiten
|
||||
- [x] 1.1 `Team`, `TeamCompetence`, `TeamReference`, `ScoredTeam` in `src/teamlandkarte_mcp/models.py` ergänzen
|
||||
- Frozen Dataclasses mit den im Design festgelegten Feldern und Defaults (`field(default_factory=list)` für `competences`/`references`)
|
||||
- Typen-Hints und Docstrings analog zu `Capacity`/`ScoredCapacity`
|
||||
- _Requirements: 5.1, 6.1_
|
||||
|
||||
- [x] 1.2 `TeamMatchingConfig` und `MatchingConfig.team` in `src/teamlandkarte_mcp/config.py` ergänzen
|
||||
- Frozen Dataclass `TeamMatchingConfig` mit `top_competency_weight: float = 1.5`
|
||||
- Feld `team: TeamMatchingConfig` in `MatchingConfig` hinzufügen
|
||||
- Parser in `load_config` so erweitern, dass `[matching.team]` aus TOML gelesen wird
|
||||
- Validierung: nicht-numerischer Wert oder Wert < 1.0 → `ConfigError` mit Schlüsselname und fehlerhaftem Wert in der Meldung
|
||||
- Default-Wert `1.5` greift bei fehlendem Schlüssel
|
||||
- _Requirements: 12.5, 12.6_
|
||||
|
||||
- [x] 1.3 `config.toml.example` um `[matching.team]`-Block ergänzen
|
||||
- Kommentierter Abschnitt mit `top_competency_weight = 1.5`
|
||||
- Hinweis, dass der Wert numerisch und ≥ 1.0 sein muss
|
||||
- _Requirements: 12.5_
|
||||
|
||||
- [x] 1.4 PBT für Konfig-Validierung von `top_competency_weight`
|
||||
- **Property 13: Konfig-Validierung von `top_competency_weight`**
|
||||
- **Validates: Requirements 12.5, 12.6**
|
||||
- Hypothesis-Strategie: numerische Werte ≥ 1.0 (positiv), nicht-numerische und < 1.0 (negativ); fehlender Schlüssel → Default 1.5
|
||||
- Datei: `tests/test_team_config_top_weight_pbt.py`
|
||||
|
||||
- [x] 2. DBClient-Protokoll und SQL-Implementierung für Team-Daten
|
||||
- [x] 2.1 `DBClient`-Protokoll in `src/teamlandkarte_mcp/database/types.py` erweitern
|
||||
- Neue `TypedDict`s `TeamCompetenceRow` (`name`, `top_competency`) und `TeamReferenceRow` (`partner_name`, `projects`)
|
||||
- Methoden ergänzen: `get_all_teams`, `get_team_by_id`, `get_team_competences`, `batch_get_team_competences`, `get_team_references`, `batch_get_team_references`
|
||||
- Docstrings beschreiben Sortierung, NULL-Behandlung und INNER- vs. LEFT-JOIN-Semantik (siehe Design)
|
||||
- _Requirements: 2.1, 2.2, 2.3, 2.4, 2.5, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 4.1, 4.2, 4.3, 4.4, 4.5, 4.6, 4.7_
|
||||
|
||||
- [x] 2.2 `TrinoClient.get_all_teams` und `get_team_by_id` in `src/teamlandkarte_mcp/database/trino_client.py` implementieren
|
||||
- SQL: INNER JOIN `teamlandkarte_v_teams_latest` mit `teamlandkarte_v_teammeter_organizational_units_latest` über `team_id = id`
|
||||
- `COALESCE(..., '')` für `about_us`, `offerings`, `interests`, `focus_name`
|
||||
- `ORDER BY ou.name ASC, t.team_id ASC`
|
||||
- `_ensure_select_only` und `_retry` verwenden, Connection-Pool über bestehenden `_cursor`-Context
|
||||
- `get_all_teams` ruft intern `batch_get_team_competences` und `batch_get_team_references` auf und aggregiert zu `Team`-Instanzen
|
||||
- `get_team_by_id` liefert `None`, wenn kein Treffer (auch durch INNER JOIN ausgefiltert)
|
||||
- _Requirements: 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7_
|
||||
|
||||
- [x] 2.3 `TrinoClient.get_team_competences` und `batch_get_team_competences` implementieren
|
||||
- SQL: Join auf `teamlandkarte_v_competences_latest` analog zum Capacity-Pfad zur Auflösung des Kompetenz-Namens
|
||||
- `COALESCE(top_competency, FALSE)` für NULL-Normalisierung
|
||||
- `ORDER BY tc.ouid ASC, CASE WHEN COALESCE(top_competency,FALSE) THEN 0 ELSE 1 END ASC, c.name ASC`
|
||||
- Batch-Variante in genau einer Query, `dict[str, list[TeamCompetenceRow]]` mit leeren Listen für OUIDs ohne Treffer
|
||||
- `_ensure_select_only` und `_retry` anwenden
|
||||
- _Requirements: 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 2.6, 2.7_
|
||||
|
||||
- [x] 2.4 `TrinoClient.get_team_references` und `batch_get_team_references` implementieren
|
||||
- SQL: LEFT JOIN auf `teamlandkarte_v_partners_latest` über `partner_id = id` als Teil derselben Query (kein zusätzlicher Roundtrip)
|
||||
- `COALESCE(p.name, '')` für `partner_name`
|
||||
- Whitespace-only `projects` werden nach dem Fetch in Python ausgefiltert
|
||||
- `ORDER BY r.ouid ASC, partner_name ASC, r.projects ASC`
|
||||
- Batch-Variante in genau einer Query, `dict[str, list[TeamReferenceRow]]` mit leeren Listen für OUIDs ohne Treffer
|
||||
- _Requirements: 4.1, 4.2, 4.3, 4.4, 4.5, 4.6, 4.7, 2.6, 2.7_
|
||||
|
||||
- [x] 2.5 PBT für SELECT-Only-Eigenschaft aller neuen Trino-Queries
|
||||
- **Property 2: SELECT-Only-Eigenschaft aller neuen Trino-Queries**
|
||||
- **Validates: Requirements 2.6**
|
||||
- Stub-Cursor, der jede ausgeführte SQL durch `_ensure_select_only` validiert; Hypothesis-generierte Eingabe-Listen für die sechs neuen Methoden
|
||||
- Datei: `tests/test_team_trino_select_only_pbt.py`
|
||||
|
||||
- [x] 2.6 PBT für Stammdaten-Konsistenz und INNER-JOIN-Filter
|
||||
- **Property 3: Stammdaten-Konsistenz und INNER-JOIN-Filter**
|
||||
- **Validates: Requirements 2.1, 2.2, 2.3, 2.4, 2.5**
|
||||
- Hypothesis-Strategie: Listen aus Teams- und OU-Stub-Zeilen mit beliebigen NULL/Empty-Verteilungen; prüfen, dass NULL → "", INNER JOIN ohne Match → ausgeschlossen, `get_team_by_id` konsistent zu `get_all_teams`
|
||||
- Datei: `tests/test_team_master_data_pbt.py`
|
||||
|
||||
- [x] 2.7 PBT für Konsistenz von Einzel- und Batch-Variante der Kompetenzen
|
||||
- **Property 4: Kompetenz-Batch ist konsistent zur Einzel-Variante**
|
||||
- **Validates: Requirements 3.1, 3.3, 3.4, 3.5, 3.6, 3.7**
|
||||
- Hypothesis-Strategie: Mengen von OUIDs + zufällige Top/Name-Verteilungen + NULL für `top_competency`
|
||||
- Datei: `tests/test_team_competences_batch_pbt.py`
|
||||
|
||||
- [x] 2.8 PBT für Konsistenz von Einzel- und Batch-Variante der Referenzen
|
||||
- **Property 5: Referenz-Batch ist konsistent zur Einzel-Variante**
|
||||
- **Validates: Requirements 4.1, 4.3, 4.4, 4.5, 4.6, 4.7**
|
||||
- Hypothesis-Strategie: Referenzzeilen mit/ohne Partner und Whitespace-`projects`; prüfen, dass die Batch-Variante genau einen `cur.execute(...)`-Call gegen die References-View ausführt
|
||||
- Datei: `tests/test_team_references_batch_pbt.py`
|
||||
|
||||
- [x] 2.9 Unit-Tests für `TrinoClient`-Team-Queries (Snapshot)
|
||||
- SQL-String-Snapshot, dass INNER JOIN, `ORDER BY`, `COALESCE` und LEFT JOIN auf `partners` korrekt formuliert sind
|
||||
- Datei: `tests/test_trino_client_team_queries.py`
|
||||
- _Requirements: 2.1, 2.2, 3.1, 4.1, 4.2_
|
||||
|
||||
- [x] 3. Schema-Verifikation für die vier neuen Views
|
||||
- [x] 3.1 `schema_expected`-Map in `src/teamlandkarte_mcp/mcp_server.py` (`build_server`) erweitern
|
||||
- Einträge für `teamlandkarte_v_teams_latest`, `teamlandkarte_v_teammeter_organizational_units_latest`, `teamlandkarte_v_teammeter_team_competences_latest`, `teamlandkarte_v_team_references_latest`
|
||||
- Spalten gemäß Design (`team_id`, `ouid`, `about_us`, `offerings`, `interests`, `focus_name`; `id`, `name`; `ouid`, `competence_id`, `top_competency`; `ouid`, `partner_id`, `projects`)
|
||||
- Bestehender Pfad `verify_required_columns` wird unverändert verwendet
|
||||
- _Requirements: 12.1, 12.2, 12.3, 12.4_
|
||||
|
||||
- [x] 3.2 PBT für Schema-Verifikation der vier neuen Views
|
||||
- **Property 14: Schema-Verifikation deckt fehlende Spalten in den vier neuen Views auf**
|
||||
- **Validates: Requirements 12.1, 12.2, 12.3, 12.4**
|
||||
- Hypothesis-Strategie: Powerset der erwarteten Spalten pro View, jeweils minus eine zufällig gewählte Spalte → `SchemaIssue` mit Tabellenname und fehlender Spalte; vollständige Spaltenliste → kein Issue
|
||||
- Datei: `tests/test_team_schema_verification_pbt.py`
|
||||
|
||||
- [x] 4. Checkpoint - Datenzugriffsschicht
|
||||
- Sicherstellen, dass alle bisherigen Tests grün sind und der Server mit der erweiterten Schema-Verifikation startet. Bei Unklarheiten den Nutzer fragen.
|
||||
|
||||
- [x] 5. Profile-Erstellung und Serialisierung
|
||||
- [x] 5.1 `TeamCompetenceEntry`, `TeamReferenceEntry`, `TeamProfile` in `src/teamlandkarte_mcp/matching/profiles.py` ergänzen
|
||||
- Frozen Dataclasses mit den im Design definierten Feldern; Listenreihenfolgen entsprechen den Datenbank-Reihenfolgen
|
||||
- _Requirements: 5.1_
|
||||
|
||||
- [x] 5.2 `build_team_profile(team: Team) -> TeamProfile` implementieren
|
||||
- 1:1-Mapping von `Team`-Feldern auf `TeamProfile`; keine Re-Sortierung der Listen
|
||||
- NULL/Empty-Strings unverändert übernehmen (DB-Schicht hat bereits normalisiert)
|
||||
- _Requirements: 5.1, 5.2, 13.4_
|
||||
|
||||
- [x] 5.3 `serialize_team_profile(profile: TeamProfile) -> str` implementieren
|
||||
- Feste deutsche Überschriften: `Teamname:`, `Schwerpunkt:`, `Über uns:`, `Leistungen:`, `Interessen:`, `Kompetenzen:`, `Referenzen:` in genau dieser Reihenfolge
|
||||
- Top-Kompetenzen mit Suffix `(Top)` markieren
|
||||
- Referenz-Zeilen: `Partner: <partner_name> – Projekte: <projects>` bei vorhandenem Partner; bei leerem Partner nur `Projekte: <projects>` (kein Platzhalter)
|
||||
- Leere Listen als `Kompetenzen: (keine)` bzw. `Referenzen: (keine)` rendern
|
||||
- Reihenfolge der Listen-Elemente entspricht 1:1 der Eingabe
|
||||
- _Requirements: 5.2, 5.3, 5.4, 5.5, 5.6, 5.7, 13.1, 13.2, 13.3, 13.4_
|
||||
|
||||
- [x] 5.4 PBT für Determinismus und Vollständigkeit der Team-Profil-Serialisierung
|
||||
- **Property 6: Determinismus und Vollständigkeit der Team-Profil-Serialisierung**
|
||||
- **Validates: Requirements 5.1, 5.2, 5.3, 5.4, 5.5, 5.6, 5.7, 13.1, 13.2, 13.3, 13.4**
|
||||
- Hypothesis-Strategie für `TeamProfile` analog zu `_capacity_profile` in `tests/test_profile_serialization_pbt.py`; prüft Determinismus, Idempotenz, Reihenfolge der Überschriften, `(Top)`-Marker, Partner-Optional-Verhalten und Listen-Reihenfolge
|
||||
- Datei: `tests/test_team_profile_serialization_pbt.py`
|
||||
|
||||
- [x] 5.5 Unit-Tests für `build_team_profile` und `serialize_team_profile`
|
||||
- Beispielbasiert: leere Strings, leere Listen, gemischte Top/Nicht-Top-Kompetenzen, Referenzen mit/ohne Partner_Name
|
||||
- Datei: `tests/test_team_profile_serialization_unit.py`
|
||||
- _Requirements: 5.1, 5.3, 5.4, 5.5, 5.6_
|
||||
|
||||
- [x] 6. Score-basiertes Matching für Team-Profile
|
||||
- [x] 6.1 `Matcher.match_teams` in `src/teamlandkarte_mcp/matching/matcher.py` implementieren
|
||||
- Signatur: `async def match_teams(self, teams, requirements, *, top_competency_weight) -> TeamMatchResult`
|
||||
- Kompetenz-Liste aus `team.competences` ableiten + paralleles Top-Set
|
||||
- `SimilarityEngine.compute_competence_similarity` analog zum Capacity-Pfad nutzen; pro Required-Kompetenz `weighted = min(1.0, raw_score * (top_weight if best_match in top_set else 1.0))`
|
||||
- Role Score über `SimilarityEngine.compute_role_similarity(req.role_name, team.focus_name)`
|
||||
- Overall Score über `compute_overall(...)` mit denselben Gewichten und Thresholds wie für Kapazitäten
|
||||
- Kategorisierung über `categorize(...)`
|
||||
- **Verfügbarkeitsprüfung wird nicht aufgerufen**; ein etwaiger Datumsbereich aus `req` wird ignoriert
|
||||
- Rückgabe: neues `TeamMatchResult` (`scored: list[ScoredTeam]`, `by_category: dict[str, list[ScoredTeam]]`)
|
||||
- `Matcher.summary_counts` so generalisieren, dass sowohl Capacity- als auch Team-Buckets unterstützt werden (strukturell `len`-basiert)
|
||||
- _Requirements: 6.1, 6.2, 6.3, 6.4, 6.5, 6.7_
|
||||
|
||||
- [x] 6.2 PBT für Top-Kompetenz-Monotonie im Score-Matching
|
||||
- **Property 7: Top-Kompetenz-Monotonie im Score-Matching**
|
||||
- **Validates: Requirements 6.4**
|
||||
- Hypothesis-Strategie: Kompetenz-Listen + `top_weight ∈ [1.0, 5.0]`; vergleicht `T_top` (alle Top) mit `T_plain`; bei `top_weight == 1.0` exakt 0.0 Differenz
|
||||
- Datei: `tests/test_team_score_top_weight_pbt.py`
|
||||
|
||||
- [x] 6.3 PBT für Score-Pfad-Validität und Verfügbarkeits-Ignoranz
|
||||
- **Property 8: Score-Pfad liefert gültige Score- und Kategorie-Werte**
|
||||
- **Validates: Requirements 6.1, 6.2, 6.3, 6.5, 6.7**
|
||||
- Hypothesis-Strategie für `Team`-Listen + Stub-`SimilarityEngine`; prüft Score-Bereich `[0,1]`, Kategorie-Set, Konsistenz mit `categorize(...)` und Invarianz unter `req.date_start`/`req.date_end`
|
||||
- Datei: `tests/test_team_score_pipeline_pbt.py`
|
||||
|
||||
- [x] 7. LLM-Volltext-Matching für Team-Profile
|
||||
- [x] 7.1 `LlmFulltextMatcher.match_teams` in `src/teamlandkarte_mcp/matching/llm_fulltext_matcher.py` implementieren
|
||||
- Signatur parallel zu `match_capacities`/`match_tasks`, akzeptiert `task_profile: TaskProfile` und `teams: list[Team]`
|
||||
- Pro Team `build_team_profile + serialize_team_profile`; User-Prompt im Format `=== Aufgabe ===` + `=== Team ===` mit `ID:`-Header
|
||||
- System-Prompt: bestehender Capacity-Prompt mit minimaler Wortwahl-Anpassung (`Kapazitätsprofil` → `Profil`); JSON-Schema `{"category", "rationale"}` unverändert
|
||||
- Concurrency über bestehende `asyncio.Semaphore` (`max_concurrency`)
|
||||
- Fehlerbehandlung: `LlmFulltextError` für JSON-Parse-Fehler und `AzureAPIError`; ungültige Kategorien werden via `normalize_category` auf `Irrelevant` gemappt mit Hinweis-Suffix in der Rationale
|
||||
- Sortierung pro Kategorie: `(category_rank, item_id_asc)` mit `item_id = team_id`
|
||||
- Rückgabe: bestehender `LlmFulltextResult`-Typ (`by_category` + `errors`)
|
||||
- _Requirements: 7.1, 7.2, 7.3, 7.4, 7.5, 7.6, 7.7_
|
||||
|
||||
- [x] 7.2 PBT für LLM-Pfad als vollständige Partition mit gültigen Kategorien
|
||||
- **Property 9: LLM-Pfad ist eine vollständige Partition mit gültigen Kategorien**
|
||||
- **Validates: Requirements 7.1, 7.2, 7.3, 7.6, 7.7**
|
||||
- Hypothesis-Strategie: `mask: list[bool]` analog zu `tests/test_batch_completeness_pbt.py` (`_MaskLlm`-Pattern); prüft Vollständigkeit, Eindeutigkeit, Sortierung, gültige Kategorien und nicht-leere Rationales bei Erfolg
|
||||
- Datei: `tests/test_team_llm_fulltext_partition_pbt.py`
|
||||
|
||||
- [x] 7.3 PBT für ungültige LLM-Kategorien (Irrelevant-Fallback)
|
||||
- **Property 10: Ungültige LLM-Kategorien fallen auf Irrelevant zurück**
|
||||
- **Validates: Requirements 7.5**
|
||||
- Hypothesis-Strategie: zufällige Strings (außerhalb der erlaubten Kategorien) + JSON-Wrapper; prüft `category == "Irrelevant"` und Hinweis-Substring in `rationale`
|
||||
- Datei: `tests/test_team_llm_invalid_category_pbt.py`
|
||||
|
||||
- [x] 8. Checkpoint - Matching-Pipelines
|
||||
- Sicherstellen, dass die Score- und LLM-Pfade isoliert lauffähig sind und alle Tests grün sind. Bei Unklarheiten den Nutzer fragen.
|
||||
|
||||
- [x] 9. MCP-Tools für Team-Suche und Team-Detailansicht
|
||||
- [x] 9.1 `_ALLOWED_PROFILE_TYPES` und `_get_teams_cached()` in `src/teamlandkarte_mcp/mcp_server.py` ergänzen
|
||||
- String-Konstante `_ALLOWED_PROFILE_TYPES = ("capacity", "team")`
|
||||
- Neuer `QueryCache[list[Team]]`-Eintrag (`all_teams`) parallel zum Capacity-Cache; ruft `db_client.get_all_teams()`
|
||||
- _Requirements: 1.1_
|
||||
|
||||
- [x] 9.2 Tool `find_matching_teams` registrieren
|
||||
- Signatur: `async def find_matching_teams(role_name: str, competences: list[str], matching_method: str = "") -> str`
|
||||
- `_resolve_matching_method` und `_validate_requirements_minimum` wiederverwenden; ungültiges `matching_method` → Fehlermeldung mit erlaubten Werten, **kein** DB-/LLM-Aufruf
|
||||
- Confirm-Gate über `_require_confirmed_or_auto(req)`
|
||||
- Score-Pfad: `matcher.match_teams(...)` mit `cfg.matching.team.top_competency_weight`
|
||||
- LLM-Pfad: `task_profile = build_task_profile_from_requirements(req)` → `llm_fulltext_matcher.match_teams(task_profile=task_profile, teams=teams)`
|
||||
- Persistenz im `SearchCache` mit `results_payload["search_type"] = "team_search"` und `matching_method`
|
||||
- Antwort: `SEARCH_ID=<uuid>` + `META=<json>` mit `search_type` und `matching_method` + Markdown-Tabellen (Summary + Top-Kategorie)
|
||||
- _Requirements: 1.3, 1.4, 1.5, 1.6, 6.1, 7.1, 8.1, 8.2, 8.3_
|
||||
|
||||
- [x] 9.3 Tool `list_teams(limit: int = 20) -> str` registrieren
|
||||
- Markdown-Tabelle mit Spalten `Team Id`, `Team Name`, `Schwerpunkt`, `Anzahl Kompetenzen`, `Anzahl Referenzen`
|
||||
- Empty-Fallback-Text wie bei `list_free_capacities`
|
||||
- _Requirements: 9.1_
|
||||
|
||||
- [x] 9.4 Tool `get_team_details(team_id: str) -> str` registrieren
|
||||
- Markdown-Tabelle für ein Team plus Sektionen `## Über uns`, `## Leistungen`, `## Interessen`, `## Kompetenzen` (mit `(Top)`-Marker), `## Referenzen` (Partner_Name fett gedruckt, sonst nur Projekte) und `## Next steps`
|
||||
- Wenn `db_client.get_team_by_id(team_id)` `None` liefert → `f"Team not found: {team_id}"`
|
||||
- Reihenfolge der Listen entspricht der DB-Reihenfolge
|
||||
- _Requirements: 9.2, 9.3, 9.4_
|
||||
|
||||
- [x] 9.5 `_coerce_team(item)` Helper implementieren
|
||||
- Rehydratisiert persistierte SearchCache-Einträge zurück in `Team`-Instanzen (analog zu `_coerce_capacity`)
|
||||
- Handhabt sowohl `Team`-Instanzen als auch verschachtelte `dict`-Repräsentationen (mit/ohne `team`-Wrapper)
|
||||
- _Requirements: 8.4_
|
||||
|
||||
- [x] 9.6 `_format_results_table` für `search_type="team_search"` erweitern
|
||||
- Spalten im `score`-Modus: `Team Name`, `Schwerpunkt`, `Top-Kompetenzen`, `Role Score`, `Competence Score`, `Overall Score`, `Category`
|
||||
- Spalten im `llm_fulltext`-Modus: `Team Name`, `Schwerpunkt`, `Top-Kompetenzen`, `Category`, `Begründung`
|
||||
- `Top-Kompetenzen` ist die kommaseparierte Liste aller `competences` mit `top_competency=True`
|
||||
- `Begründung` nutzt bestehenden `_format_rationale_for_table`-Helper
|
||||
- Sortierkey im LLM-Modus: `(category_rank, item.get("team_id") or "")`
|
||||
- _Requirements: 6.6, 7.8, 8.4_
|
||||
|
||||
- [x] 9.7 `filter_search_results` und `get_results_by_category` für `team_search` erweitern
|
||||
- `role_filter` matcht gegen `team.focus_name`
|
||||
- `competence_filter` matcht gegen `[c["name"] for c in item["competences"]]`; Filterwerte mit Suffix `(Top)` werden auf Top-Kompetenzen beschränkt (Suffix wird vor Vergleich entfernt)
|
||||
- `availability_date_start`, `availability_date_end`, `is_fully_available` werden ignoriert; `Applied Filters`-Tabelle erhält für jeden Wert einen Hinweis mit Substring `team_search` und Markierung "nicht wirksam"
|
||||
- `task_*`-Filter bleiben für `team_search` inaktiv (existierender Pfad)
|
||||
- _Requirements: 6.7, 8.4, 8.5, 8.6_
|
||||
|
||||
- [x] 9.8 PBT für `matching_method`-Validierung in `find_matching_teams`
|
||||
- **Property 1: `matching_method`-Validierung in `find_matching_teams`**
|
||||
- **Validates: Requirements 1.5, 1.6**
|
||||
- Hypothesis-Strategie: zufällige Strings außerhalb `{"score","llm_fulltext"}` (case-insensitiv getrimmt) und nicht leer/None; prüft Fehlermeldung enthält beide erlaubten Werte und keine DB-/LLM-Aufrufe erfolgen
|
||||
- Datei: `tests/test_team_matching_method_validation_pbt.py`
|
||||
|
||||
- [x] 9.9 PBT für META und SearchCache-Markierung von Team-Suchen
|
||||
- **Property 11: META und SearchCache markieren Team-Suchen korrekt**
|
||||
- **Validates: Requirements 1.4, 8.1, 8.2, 8.3, 8.4**
|
||||
- Hypothesis-Strategie: zufällige Inputs + Snapshot-Parser für `SEARCH_ID` und `META`-JSON; prüft `search_type=="team_search"`, `matching_method ∈ {...}`, Konsistenz mit `SearchCache`-Eintrag und Header in der per-Kategorie-Tabelle
|
||||
- Datei: `tests/test_team_meta_search_type_pbt.py`
|
||||
|
||||
- [x] 9.10 PBT für Verfügbarkeitsfilter-Ignoranz in Team-Suchen
|
||||
- **Property 12: Verfügbarkeitsfilter werden in Team-Suchen ignoriert**
|
||||
- **Validates: Requirements 6.7, 8.5, 8.6**
|
||||
- Hypothesis-Strategie: zufällige Datumswerte + `is_fully_available`; prüft, dass die gefilterte Item-Menge gleich der ohne Verfügbarkeitsfilter bleibt und die `Applied Filters`-Tabelle den `team_search`-Hinweis enthält
|
||||
- Datei: `tests/test_team_filter_ignores_availability_pbt.py`
|
||||
|
||||
- [x] 9.11 Unit-Tests für `find_matching_teams`, `list_teams`, `get_team_details`
|
||||
- Tools sind über `FastMCP` registriert (Anforderungen 1.3, 9.1, 9.2)
|
||||
- `find_matching_capacities`-Snapshot-Test: Output für festen Input bleibt nach Refactoring unverändert (Anforderung 1.2)
|
||||
- Markdown-Outputs enthalten geforderte Header und Sektionen (Anforderungen 9.1, 9.2, 9.4)
|
||||
- `get_team_details("unknown")` enthält `unknown` in der Fehlermeldung (Anforderung 9.3)
|
||||
- Datei: `tests/test_team_tools_unit.py`
|
||||
- _Requirements: 1.2, 1.3, 9.1, 9.2, 9.3, 9.4_
|
||||
|
||||
- [x] 10. Checkpoint - MCP-Tool-Surface
|
||||
- Sicherstellen, dass alle bisherigen und neuen Tests grün sind und der Server `find_matching_teams`, `list_teams`, `get_team_details` korrekt registriert. Bei Unklarheiten den Nutzer fragen.
|
||||
|
||||
- [x] 11. Integrationstests
|
||||
- [x] 11.1 Smoke-Test für `find_matching_teams` im `score`-Modus
|
||||
- Gemockter Trino-Cursor + Stub-`SimilarityEngine`; voller Pfad `find_matching_teams` → `SearchCache` → `get_results_by_category` → `filter_search_results`
|
||||
- Datei: `tests/test_team_integration_score.py`
|
||||
- _Requirements: 6.1, 8.1, 8.2, 8.4_
|
||||
|
||||
- [x] 11.2 Smoke-Test für `find_matching_teams` im `llm_fulltext`-Modus
|
||||
- Gemockter Trino-Cursor + Mock-`AzureOpenAIClient`; voller Pfad inkl. `errors`-Liste bei simuliertem LLM-Fehler
|
||||
- Datei: `tests/test_team_integration_llm_fulltext.py`
|
||||
- _Requirements: 7.1, 7.6, 8.1, 8.2, 8.4_
|
||||
|
||||
- [x] 12. Dokumentation und Agenten-Konfiguration
|
||||
- [x] 12.1 `docs/architecture.md` um Team-Profil-Abschnitte ergänzen
|
||||
- Neuer Unterabschnitt "Team Profile" im Datenmodell-Kapitel mit Feldern, Datenquellen und View-Verknüpfungen
|
||||
- Datenquellen-Tabelle um die vier neuen Views inkl. relevanter Spalten erweitern
|
||||
- Joins dokumentieren: `teams_latest.team_id = organizational_units_latest.id` (INNER JOIN für Team_Name), `ouid` zu Kompetenzen/Referenzen, `team_references_latest.partner_id = partners_latest.id` (LEFT JOIN, Spalte `name` als Partner_Name)
|
||||
- `Profile_Type`-Parameter und Wertebereiche im Tool-Surface-Abschnitt für `find_matching_capacities`/`find_matching_teams`/`list_teams`/`get_team_details` dokumentieren
|
||||
- Runtime-View-Abschnitt um Aufgabe→Team in beiden Matching-Methoden ergänzen
|
||||
- _Requirements: 11.1, 11.2, 11.3, 11.4, 11.5, 11.6_
|
||||
|
||||
- [x] 12.2 `README.md` um Team-Suche-Hinweise ergänzen
|
||||
- Quick-Start- und Usage-Abschnitt: Wahl zwischen `capacity`- und `team`-Suche
|
||||
- Auflistung der zusätzlichen Datenbank-Views inkl. Join-Bedingungen
|
||||
- Hinweis: Verfügbarkeitsfilter wirken in Team-Suchen nicht; abweichende Ergebnisspalten in `score`/`llm_fulltext`
|
||||
- _Requirements: 11.7, 11.8, 11.9_
|
||||
|
||||
- [x] 12.3 `.github/agents/teamlandkarte_agent.md` und `.kiro/agents/teamlandkarte.md` aktualisieren
|
||||
- `Profile_Type`-Werte `capacity` und `team` mit Bedeutung dokumentieren
|
||||
- Initiale Captures-Phase um explizite Frage nach `Profile_Type` ergänzen (sofern nicht aus Verlauf ersichtlich)
|
||||
- Skills/Workflows um `find_matching_teams`, `list_teams`, `get_team_details` erweitern; `matching_method`-Parameter erwähnen
|
||||
- Hinweis: Verfügbarkeitsfilter sind in Team-Suchen nicht wirksam; Ergebnisspalten weichen ab
|
||||
- Bestätigungs-Workflow (`show_pending_requirements`, `confirm_requirements`) bleibt für beide `Profile_Type`-Werte unverändert
|
||||
- _Requirements: 10.1, 10.2, 10.3, 10.4, 10.5, 10.6_
|
||||
|
||||
- [x] 12.4 Doku-Snapshot-Tests
|
||||
- Prüft, dass `docs/architecture.md`, `README.md`, `.kiro/agents/teamlandkarte.md`, `.github/agents/teamlandkarte_agent.md` die definierten Stichworte enthalten (`Profile_Type`, `find_matching_teams`, `team_search`, `top_competency_weight`)
|
||||
- Datei: `tests/test_team_documentation_snapshots.py`
|
||||
- _Requirements: 10.1, 10.2, 10.3, 10.4, 10.5, 10.6, 11.1, 11.2, 11.3, 11.4, 11.5, 11.6, 11.7, 11.8, 11.9_
|
||||
|
||||
- [x] 13. Erweiterung der bestehenden Test-Suite um Team-Profil-Abdeckung
|
||||
- [x] 13.1 Bestehende End-to-End- und Routing-Tests um Team-Pfade ergänzen
|
||||
- `tests/test_mcp_routing_unit.py`, `tests/test_search_cache.py`, `tests/test_pagination.py`, `tests/test_filters.py` so erweitern, dass beide `Profile_Type`-Werte (`capacity`, `team`) mit beiden `matching_method`-Werten (gemocktes LLM und gemockte DB) abgedeckt sind
|
||||
- _Requirements: 12.7_
|
||||
|
||||
- [x] 14. Final-Checkpoint - Vollständigkeit und Konsistenz
|
||||
- Sicherstellen, dass alle Pflicht-Tasks (ohne `*`) abgeschlossen sind, alle Tests grün sind und die Dokumentation konsistent ist. Bei Unklarheiten den Nutzer fragen.
|
||||
|
||||
## Hinweise
|
||||
|
||||
- Sub-Tasks mit `*` sind optional und werden nur auf ausdrückliche Anweisung implementiert.
|
||||
- Jede Korrektheits-Eigenschaft aus dem Designdokument ist genau einem PBT-Sub-Task zugeordnet (Properties 1–14 in den Tasks 1.4, 2.5–2.8, 3.2, 5.4, 6.2, 6.3, 7.2, 7.3, 9.8, 9.9, 9.10).
|
||||
- Checkpoints (Tasks 4, 8, 10, 14) erzwingen einen grünen Test-Lauf vor dem nächsten Abschnitt.
|
||||
- Die Implementierungssprache ist Python (gemäß bestehendem Codestil); PBT-Framework ist Hypothesis.
|
||||
|
||||
## Workflow-Abschluss
|
||||
|
||||
Dieser Workflow erstellt ausschließlich Design- und Planungsartefakte. Die
|
||||
eigentliche Implementierung beginnt, sobald der Nutzer in `tasks.md` einen Task
|
||||
über "Start task" anstößt.
|
||||
@@ -1 +0,0 @@
|
||||
.env
|
||||
@@ -1 +0,0 @@
|
||||
3.13
|
||||
@@ -1,18 +0,0 @@
|
||||
<!-- OPENSPEC:START -->
|
||||
# OpenSpec Instructions
|
||||
|
||||
These instructions are for AI assistants working in this project.
|
||||
|
||||
Always open `@/openspec/AGENTS.md` when the request:
|
||||
- Mentions planning or proposals (words like proposal, spec, change, plan)
|
||||
- Introduces new capabilities, breaking changes, architecture shifts, or big performance/security work
|
||||
- Sounds ambiguous and you need the authoritative spec before coding
|
||||
|
||||
Use `@/openspec/AGENTS.md` to learn:
|
||||
- How to create and apply change proposals
|
||||
- Spec format and conventions
|
||||
- Project structure and guidelines
|
||||
|
||||
Keep this managed block so 'openspec update' can refresh the instructions.
|
||||
|
||||
<!-- OPENSPEC:END -->
|
||||
@@ -1,16 +0,0 @@
|
||||
# Component: teamlandkarte-mcp
|
||||
|
||||
## Description
|
||||
MCP server for matching tasks to employees/teams by competences via Data Lake
|
||||
|
||||
## Metadata
|
||||
- **Deployment Target:** local-mcp-stdio
|
||||
- **Upstream URL:** https://git.tech.rz.db.de/ThomasHandke/teamlandkarte-mcp
|
||||
- **Status:** planned
|
||||
|
||||
## Interconnections
|
||||
- (to be documented)
|
||||
|
||||
## Notes
|
||||
- Part of bahn context in the Orchestrator monorepo
|
||||
|
||||
@@ -1,638 +0,0 @@
|
||||
# Teamlandkarte MCP Server
|
||||
|
||||
An MCP (Model Context Protocol) server that enables AI assistants to match DB Systel employees with free work capacity to task requirements.
|
||||
|
||||
Key matching semantics (refined):
|
||||
- Role inference is **title-first** with description fallback.
|
||||
- Role and competence inference use **separate cached embeddings**.
|
||||
- Scores are shown as **Role Score**, **Competence Score**, and **Overall Score**.
|
||||
- Result categories include **Top / Good / Partial / Low / Irrelevant**.
|
||||
- The Low vs Irrelevant boundary is configurable via `matching.thresholds.low`.
|
||||
- Availability in matching outputs is shown as an **overlap percentage** with the reference time range.
|
||||
|
||||
## Features
|
||||
|
||||
- Dual input modes: free-text task description or structured parameters
|
||||
- Matching and scoring: competences (default 0.8) + role (default 0.2)
|
||||
- Interactive exploration: view Top results first, then explore other categories via pagination
|
||||
- Smart caching: DB cache (12h TTL) and search cache (60min TTL)
|
||||
- Read-only operations: safe, non-invasive database access
|
||||
- Embedding preload on startup (cache misses only):
|
||||
- role vocabulary + competence vocabulary from the Data Lake
|
||||
- all open tasks (embedded by `task_id`)
|
||||
- Stable, table-first outputs designed for MCP chat UIs:
|
||||
- Task/validation/role inference tools return Markdown tables first
|
||||
- Search tools emit deterministic headers for robust `search_id` parsing
|
||||
|
||||
### Azure OpenAI usage
|
||||
|
||||
This server uses **Azure OpenAI** for:
|
||||
|
||||
- Role inference (task text → closest role vocabulary entry) — **embeddings**
|
||||
- Competence inference (task text → closest competence vocabulary entries) — **embeddings**
|
||||
- Capacity↔task matching similarity (default: **embeddings**; optional: **BM25+RRF**)
|
||||
- Auto-Tagging competence expansion (optional: **chat/LLM**, requires `use_auto_tagging = true`)
|
||||
|
||||
The chat/LLM path is disabled by default. Enable it via `use_auto_tagging = true` in
|
||||
`config.toml` (requires `AZURE_OPENAI_LLM_API_KEY` and `azure_openai.chat_deployment`).
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- Python 3.13+
|
||||
- Access to DB Systel Open Data Lake (Trino/Presto)
|
||||
- Valid database credentials
|
||||
- Azure OpenAI endpoint + **embedding** API key (via environment variables)
|
||||
|
||||
### Installation
|
||||
|
||||
1. Clone the repository:
|
||||
```bash
|
||||
git clone https://git.tech.rz.db.de/ThomasHandke/teamlandkarte-mcp.git
|
||||
cd teamlandkarte-mcp
|
||||
```
|
||||
|
||||
2. Install dependencies:
|
||||
```bash
|
||||
uv sync
|
||||
# or
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
3. Configure the server:
|
||||
```bash
|
||||
cp config.toml.example config.toml
|
||||
# Edit config.toml with host/port, cache settings, and Azure OpenAI endpoint/deployment
|
||||
|
||||
# Create a .env file with credentials (do not commit):
|
||||
# DATA_LAKE_USERNAME="..."
|
||||
# DATA_LAKE_PASSWORD="..."
|
||||
# AZURE_OPENAI_EMBEDDING_API_KEY="..."
|
||||
# AZURE_OPENAI_LLM_API_KEY="..." # only needed when use_auto_tagging = true
|
||||
```
|
||||
|
||||
### Running the Server
|
||||
|
||||
Run the MCP server locally via stdio:
|
||||
|
||||
From an activated project environment:
|
||||
|
||||
- `uv run teamlandkarte-mcp --config config.toml --log-level INFO`
|
||||
|
||||
If your MCP host (e.g. Cherry Studio) cannot set a working directory, wrap it
|
||||
so the command runs in the project folder:
|
||||
|
||||
- `zsh -lc 'cd /Users/thomashandke/ws/teamlandkarte-mcp && uv run python -m teamlandkarte_mcp --config /Users/thomashandke/ws/teamlandkarte-mcp/config.toml --log-level DEBUG'`
|
||||
|
||||
Important:
|
||||
|
||||
- When using stdio MCP, **stdout must be reserved for the JSON-RPC protocol**.
|
||||
Logs go to **stderr**.
|
||||
|
||||
## Running via Cherry Studio with `uvx` (Hot Reload)
|
||||
|
||||
If you develop locally and want **hot reload** without relying on a project-local virtualenv being activated in Cherry Studio, you can start the server through `uvx` and `watchfiles`.
|
||||
|
||||
This setup:
|
||||
|
||||
- restarts the MCP server process on `*.py` changes
|
||||
- watches only `src/teamlandkarte_mcp/`
|
||||
- works with the SQLite embedding cache (`[embedding_cache].path`) because the cache is a file on disk and will be reused after restarts
|
||||
|
||||
Cherry Studio command (macOS / zsh):
|
||||
|
||||
- **Command**: `zsh`
|
||||
- **Args**:
|
||||
- `-lc`
|
||||
- `cd /Users/thomashandke/ws/teamlandkarte-mcp && uvx --from watchfiles watchmedo auto-restart --recursive --pattern '*.py' --directory src/teamlandkarte_mcp -- uv run python -m teamlandkarte_mcp --config /Users/thomashandke/ws/teamlandkarte-mcp/config.toml --log-level DEBUG`
|
||||
|
||||
Notes:
|
||||
|
||||
- The embedding cache DB file (for example `.cache/embeddings.sqlite3`) survives restarts; deleting it is safe but will increase Azure embedding calls until it warms up again.
|
||||
- If you see occasional `database is locked` errors during rapid file changes, it usually means two processes overlapped briefly. In that case, reduce the frequency of changes saved at once or adjust the watcher settings.
|
||||
|
||||
## Configuration
|
||||
|
||||
Edit `config.toml` to customize:
|
||||
|
||||
- Trino connection settings (host/port, TLS, catalog/schema)
|
||||
- Connection management: `pool_size`
|
||||
- Matching weights and thresholds (Round 6 semantics)
|
||||
- Cache TTLs and sizes
|
||||
- Confirmation gate (`matching.require_confirmation`)
|
||||
- Fuzzy filtering threshold (`matching.fuzzy.min_similarity`)
|
||||
- Competence inference knobs (`matching.inference.*`):
|
||||
- `max_competences` (int | null)
|
||||
- `min_similarity` (float | null)
|
||||
- Azure OpenAI settings (`[azure_openai]`) — embeddings + optional chat model for Auto-Tagging
|
||||
- Embedding cache settings (`[embedding_cache]`)
|
||||
- BM25 + RRF lexical matching flags (`[matching.similarity]`):
|
||||
- `use_bm25_search` (boolean, default `false`): enables BM25+RRF lexical competence matching
|
||||
instead of embedding-based similarity. Skills with no token overlap score exactly 0.0,
|
||||
eliminating embedding-based false positives.
|
||||
- `use_auto_tagging` (boolean, default `false`): enables LLM pre-expansion of each
|
||||
candidate's competence list before BM25 scoring. Bridges synonym/abbreviation/cross-language
|
||||
gaps (e.g. "ML" → "Machine Learning"). Only active when `use_bm25_search = true`.
|
||||
- Chat model deployment for Auto-Tagging (`azure_openai.chat_deployment`, default `""`):
|
||||
Azure deployment name for the chat model (e.g. `"gpt-4.1"`). Required when
|
||||
`use_auto_tagging = true`.
|
||||
|
||||
Database credentials are read from environment variables (recommended: a local
|
||||
`.env` file):
|
||||
|
||||
- `DATA_LAKE_USERNAME`
|
||||
- `DATA_LAKE_PASSWORD`
|
||||
|
||||
Azure OpenAI credentials are read from environment variables:
|
||||
|
||||
- `AZURE_OPENAI_EMBEDDING_API_KEY` — API key for embedding model (always required)
|
||||
- `AZURE_OPENAI_LLM_API_KEY` — API key for chat model used by Auto-Tagging
|
||||
(required only when `use_auto_tagging = true`)
|
||||
|
||||
See `config.toml.example` for a complete template.
|
||||
|
||||
For a step-by-step guide to configuring Azure deployments and keys, see:
|
||||
|
||||
- `docs/azure_openai_setup.md`
|
||||
|
||||
## Usage
|
||||
|
||||
### Profile types: `capacity` vs `team` search
|
||||
|
||||
The server supports two **profile types** for task matching, selected via the tool name:
|
||||
|
||||
| `Profile_Type` | Entry point | What is matched |
|
||||
|----------------|-------------|-----------------|
|
||||
| `capacity` (existing) | `find_matching_capacities(...)` | The task is matched against employee capacity profiles. Availability filters apply. |
|
||||
| `team` (new) | `find_matching_teams(...)` | The task is matched against aggregated team profiles (`Team_Name`, `Schwerpunkt`, `Über uns`, `Leistungen`, `Interessen`, Top/Non-Top competences, references). Availability filters do **not** apply. |
|
||||
|
||||
The persisted search session is tagged accordingly:
|
||||
|
||||
- Capacity searches: `search_type = "capacity_search"` in `SearchCache` and the META JSON.
|
||||
- Team searches: `search_type = "team_search"` in `SearchCache` and the META JSON.
|
||||
|
||||
`find_matching_teams` accepts the same `matching_method` parameter (`score` | `llm_fulltext`) as `find_matching_capacities`. In `score` mode the team's `focus_name` is used as the role stand-in, and Top competences are upweighted by the configurable factor `matching.team.top_competency_weight` (default `1.5`, must be numeric and `>= 1.0`).
|
||||
|
||||
Result columns differ between the two profile types:
|
||||
|
||||
| Mode | Columns (capacity) | Columns (team) |
|
||||
|------|--------------------|----------------|
|
||||
| `score` | `Owner`, `Role`, `Competences`, `Availability`, `Role Score`, `Competence Score`, `Overall Score`, `Category` | `Team Name`, `Schwerpunkt`, `Top-Kompetenzen`, `Role Score`, `Competence Score`, `Overall Score`, `Category` |
|
||||
| `llm_fulltext` | `Owner`, `Role`, `Competences`, `Availability`, `Category`, `Begründung` | `Team Name`, `Schwerpunkt`, `Top-Kompetenzen`, `Category`, `Begründung` |
|
||||
|
||||
Filter behavior on a `team_search` cache entry:
|
||||
|
||||
- `role_filter` matches against `team.focus_name` (Schwerpunkt).
|
||||
- `competence_filter` matches against the team's competence list. Filter values with the suffix `(Top)` are restricted to Top competences (the suffix is stripped before comparison).
|
||||
- `availability_date_start`, `availability_date_end`, and `is_fully_available` are accepted but **ignored**. Each value is surfaced in the `Applied Filters` table with a `team_search` not-effective hint.
|
||||
|
||||
### Matching Methods
|
||||
|
||||
The server supports **two matching methods** that callers select per call via the `matching_method` parameter on `find_matching_capacities`, `find_matching_tasks`, and `find_matching_teams`:
|
||||
|
||||
| Value | Description | Output columns |
|
||||
|-------|-------------|----------------|
|
||||
| `score` (default) | BM25+RRF competence similarity + LLM role similarity. Existing behavior. | `Role Score`, `Competence Score`, `Overall Score`, `Category` |
|
||||
| `llm_fulltext` | LLM-based full-text comparison of complete profiles via Azure OpenAI Chat Completion. One call per candidate, returns `{category, rationale}`. | `Category`, `Begründung` (rationale, 1–2 sentences). **No numeric scores.** |
|
||||
|
||||
When `matching_method` is omitted, the server uses the configured default:
|
||||
|
||||
```toml
|
||||
[matching]
|
||||
# Default method for new searches when callers omit matching_method.
|
||||
# Allowed: "score" | "llm_fulltext"
|
||||
default_method = "score"
|
||||
```
|
||||
|
||||
Both methods share the same prefilters (e.g. availability), the same `search_id`/`filter_id` flow, and the same Markdown summary table. Differences:
|
||||
|
||||
- In `llm_fulltext` mode the result tables use a `Begründung` column instead of the score columns. The persisted META JSON contains `"matching_method": "llm_fulltext"` so `get_results_by_category` and `filter_search_results` render the correct schema automatically.
|
||||
- In `llm_fulltext` mode `filter_search_results` ignores `min_similarity` (it is not applicable without numeric scores). It is surfaced in the Applied Filters table as a hint.
|
||||
|
||||
#### Additional database views in `llm_fulltext` mode
|
||||
|
||||
In addition to the views used by the `score` method, `llm_fulltext` reads:
|
||||
|
||||
- `teamlandkarte_v_capacities_latest.description`
|
||||
- `teamlandkarte_v_capacity_certificates_latest` (joined on `capacity_id`)
|
||||
- `teamlandkarte_v_capacity_references_latest` (joined on `capacity_id`)
|
||||
- `teamlandkarte_v_partners_latest` — joined to references via a **LEFT JOIN** `teamlandkarte_v_capacity_references_latest.partner_id = teamlandkarte_v_partners_latest.id`. The partner `name` column is exposed as `partner_name` and surfaces inside the LLM-visible profile. When `partner_id` is `NULL` or no partner matches, `partner_name` is the empty string and the reference is still kept (only the `Partner:` token is omitted in the serialized profile).
|
||||
|
||||
The partner LEFT JOIN is part of the same SQL query that loads references; no extra round-trip is needed.
|
||||
|
||||
#### Example output (LLM mode)
|
||||
|
||||
A capacity-search result table in `llm_fulltext` mode looks roughly like this:
|
||||
|
||||
```markdown
|
||||
## Summary
|
||||
| Category | Count |
|
||||
| --- | --- |
|
||||
| Top | 1 |
|
||||
| Good | 2 |
|
||||
| Partial | 0 |
|
||||
| Low | 0 |
|
||||
| Irrelevant | 1 |
|
||||
|
||||
## Top
|
||||
| ID | Owner | Role | Competences | Availability | Category | Begründung |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| 12345 | Erika Musterfrau | Senior Backend Developer | Python, FastAPI, PostgreSQL | 100% | Top | Profil deckt Python/FastAPI vollständig ab und enthält Referenzprojekte mit identischem Tech-Stack. |
|
||||
```
|
||||
|
||||
A typical META line emitted by the matching tools (and echoed by `get_results_by_category` / `filter_search_results`):
|
||||
|
||||
```text
|
||||
META={"search_id":"7c0e6f50-...","filter_id":null,"default_category":"Top","matching_method":"llm_fulltext"}
|
||||
```
|
||||
|
||||
In `score` mode the same META line carries `"matching_method":"score"` and the result table keeps the existing `Role Score | Competence Score | Overall Score | Category` columns.
|
||||
|
||||
#### Mini walkthrough (LLM mode)
|
||||
|
||||
A typical agent-driven flow when the user asks for an LLM-based capacity search:
|
||||
|
||||
1. Capture and confirm requirements as usual (`collect_structured_requirement_data`, `show_pending_requirements`, `confirm_requirements(confirm=true)`).
|
||||
2. Ask the user once: *"Welches Verfahren soll ich verwenden – `score` oder `llm_fulltext`?"*
|
||||
3. Call the tool with the chosen method:
|
||||
|
||||
```python
|
||||
find_matching_capacities(
|
||||
role_name="Senior Backend Developer",
|
||||
competences=["Python", "FastAPI", "PostgreSQL"],
|
||||
date_start="2025-03-01",
|
||||
date_end="2025-12-31",
|
||||
matching_method="llm_fulltext",
|
||||
)
|
||||
```
|
||||
|
||||
4. The tool returns the `Summary` counts + the first non-empty category table (with the `Begründung` column) and emits a META line containing `"matching_method":"llm_fulltext"`.
|
||||
5. Page through other categories with `get_results_by_category(search_id=..., category="Good", ...)`. Because the persisted payload carries `matching_method`, the table is rendered automatically with `Begründung` instead of score columns.
|
||||
6. Optional refinement via `filter_search_results(search_id=..., role_filter=..., competence_filter=...)`. `min_similarity` is accepted but ignored in this mode and surfaces in Applied Filters as `min_similarity (ignored: not applicable in llm_fulltext mode)`.
|
||||
|
||||
### Capacity browsing + capacity→task matching
|
||||
|
||||
New tools for capacity-driven workflows:
|
||||
|
||||
- `list_free_capacities(limit=20)`
|
||||
- `get_capacity_details(capacity_id)`
|
||||
- `find_matching_tasks(capacity_id)`
|
||||
|
||||
### Team browsing + team matching
|
||||
|
||||
Tools for team-driven workflows:
|
||||
|
||||
- `list_teams(limit=20)` — Markdown table with columns `Team Id`, `Team Name`, `Schwerpunkt`, `Anzahl Kompetenzen`, `Anzahl Referenzen`.
|
||||
- `get_team_details(team_id)` — Markdown table for one team plus sections `## Über uns`, `## Leistungen`, `## Interessen`, `## Kompetenzen` (with `(Top)` marker), `## Referenzen` (Partner_Name + Projekte) and `## Next steps`. Returns `Team not found: <team_id>` for unknown ids.
|
||||
- `find_matching_teams(role_name, competences, matching_method?)` — Match a task against team profiles. No date parameters: availability filters are not applicable to team searches.
|
||||
|
||||
### Task-driven matching (task→capacity)
|
||||
|
||||
Core tools:
|
||||
|
||||
- `list_open_tasks(limit=...)`
|
||||
- `get_task_details(task_id)`
|
||||
- `validate_task_requirements(task_id)` (embeddings-only inference tables)
|
||||
- `find_matching_capacities(role_name, competences, date_start?, date_end?)`
|
||||
|
||||
### Note on dates
|
||||
|
||||
With chat removed, dates are **not extracted** from free text.
|
||||
Provide dates via `collect_structured_requirement_data(...)` / guided capture.
|
||||
|
||||
### Tool workflow overview
|
||||
|
||||
This server exposes multiple MCP tools for three main activities:
|
||||
|
||||
1. **DB task workflow** (browse → inspect → validate → match)
|
||||
2. **Ad-hoc requirement gathering** (extract/update/structured/guided)
|
||||
3. **Search execution + refinement** (match → filter → page by category)
|
||||
|
||||
Important UX notes:
|
||||
|
||||
- Scoring is **categorical-first** (`Top`/`Good`/`Partial`/`Low`). Numeric scores
|
||||
remain visible for transparency.
|
||||
- Matching can be **hard-gated by confirmation** (default). Requirement capture
|
||||
tools write *pending* requirements which must be confirmed before matching.
|
||||
|
||||
### Confirmation gating
|
||||
|
||||
By default, matching is blocked until the user confirms the pending requirements:
|
||||
|
||||
> Note: The server tracks pending/confirmed requirements in in-memory session
|
||||
> state. This is a convenience guard for typical single-client chats and is not
|
||||
> a multi-tenant source of truth. Clients/agents must enforce the confirmation
|
||||
> rule themselves.
|
||||
|
||||
- Capture requirements using one of:
|
||||
- `extract_requirements(...)`
|
||||
- `collect_structured_requirement_data(...)` (optionally with `description=...`)
|
||||
- guided capture tools (see below)
|
||||
- `update_requirements(...)`
|
||||
|
||||
If `matching.require_confirmation = true`:
|
||||
|
||||
- Review + ask user for confirmation (**must be after the last requirements update**):
|
||||
- `show_pending_requirements()`
|
||||
- ask the user to confirm (Yes/No)
|
||||
- Confirm them (only after explicit user approval):
|
||||
- `confirm_requirements(confirm=true)`
|
||||
- Then run:
|
||||
- `find_matching_capacities(...)`
|
||||
|
||||
If `matching.require_confirmation = false`:
|
||||
|
||||
- Do **not** ask for confirmation.
|
||||
- Do **not** call `show_pending_requirements()` / `confirm_requirements()`.
|
||||
- Proceed directly to matching once requirements are complete.
|
||||
|
||||
You can disable the gate for convenience/testing via:
|
||||
|
||||
- `matching.require_confirmation = false` in `config.toml`
|
||||
|
||||
### Guided capture (step-by-step)
|
||||
|
||||
If the user did not provide enough project details, use the guided tools:
|
||||
|
||||
1. `start_guided_capture()`
|
||||
2. `guided_set_description(description)`
|
||||
3. `guided_set_role(role_name)`
|
||||
4. `guided_set_time_range(date_start?, date_end?)` (open-ended allowed)
|
||||
5. `guided_set_competences(competences)` (writes pending requirements)
|
||||
|
||||
The assistant MUST ensure a **concrete topic/goal description** exists.
|
||||
A single skill request like "JavaScript" is not sufficient; ask for scope/goal.
|
||||
|
||||
6. Review + ask user:
|
||||
- `show_pending_requirements()` (preferred)
|
||||
- ask the user to confirm (Yes/No)
|
||||
7. Confirm (only after user approval):
|
||||
- `confirm_requirements(confirm=true)`
|
||||
8. `find_matching_capacities(...)`
|
||||
|
||||
### Search sessions: `search_id` / `filter_id` (robust parsing)
|
||||
|
||||
Search-related tools (`find_matching_capacities`, `filter_search_results`, `get_results_by_category`) emit deterministic, machine-parseable headers.
|
||||
|
||||
**Always treat the *most recent* tool output as authoritative**.
|
||||
|
||||
- First line marker (required by some clients):
|
||||
- `Using SEARCH_ID=<uuid>`
|
||||
- Machine-readable headers:
|
||||
- `SEARCH_ID=<uuid>`
|
||||
- `FILTER_ID=<uuid>` (only when a filter was created)
|
||||
- `META=<json>` (includes status and echo of ids)
|
||||
|
||||
Output shape notes:
|
||||
|
||||
- `find_matching_capacities` includes a `## Summary` counts table (Top/Good/Partial/Low/Irrelevant) and a results table for the **first non-empty category**.
|
||||
- Matching and pagination tables show:
|
||||
- Availability as **overlap %**
|
||||
- Separate **Role Score / Competence Score / Overall Score** columns
|
||||
- No `Level` column in matching result tables (Level is only shown in capacity listing tools).
|
||||
|
||||
- `filter_search_results` supports both search directions:
|
||||
- capacity search filtering (role/competences/availability)
|
||||
- task search filtering (task text and inferred required competences)
|
||||
- optional full-coverage availability (`is_fully_available=true`)
|
||||
|
||||
- `filter_search_results` does **not** repeat the summary counts. It returns:
|
||||
- `FILTER_ID=<uuid>`
|
||||
- an **Applied Filters** table
|
||||
- a flat results table (top rows) with **separate** columns for `Score` and `Category`
|
||||
- the results table is always a Markdown table (even for a single matching row, and also when there are zero matches)
|
||||
- `min_similarity` is only applied when the user provides `role_filter` and/or `competence_filter` (availability-only filtering does not apply similarity).
|
||||
- `get_results_by_category` does **not** repeat the summary counts. It returns a single category page table.
|
||||
|
||||
Notes:
|
||||
|
||||
- There is intentionally **no** tool that returns a "last search id" (multi-user/session unsafe). Always extract the id from tool output.
|
||||
- Pagination/filter tools reject non-UUID inputs early.
|
||||
- If a cache entry is missing/expired, the tools return a table-first error payload with `META.status=unknown_or_expired`.
|
||||
|
||||
### Typical workflow (DB task)
|
||||
|
||||
1. Call `list_open_tasks(limit=...)`
|
||||
2. Call `get_task_details(task_id)`
|
||||
3. (Optional, only on explicit user request) Call `validate_task_requirements(task_id)`
|
||||
- Validation is independent from matching and is **not** required to search.
|
||||
4. Capture/update requirements (e.g. via `collect_structured_requirement_data(...)`)
|
||||
5. (If confirmation is enabled) review + confirm requirements (**after the last update**):
|
||||
- `show_pending_requirements()`
|
||||
- ask the user to confirm (Yes/No)
|
||||
- `confirm_requirements(confirm=true)`
|
||||
6. Call `find_capacities_for_task(task_id)` or `find_matching_capacities(...)`
|
||||
- Returns headers including `Using SEARCH_ID=<uuid>` / `SEARCH_ID=<uuid>`
|
||||
7. Optionally call `filter_search_results(search_id, ...)`
|
||||
- Returns `FILTER_ID=<uuid>` under the same `SEARCH_ID`
|
||||
8. Page through results using `get_results_by_category(search_id, filter_id?, category, page, page_size)`
|
||||
|
||||
If you see `unknown_or_expired` or `invalid_search_id_format`, re-run the matching tool to create a new search.
|
||||
|
||||
Notes:
|
||||
|
||||
- Round 6 semantics: scoring = competence (0.8) + role (0.2) only
|
||||
- Availability is displayed and can be used as an optional date-range filter only
|
||||
- Open-ended capacity end dates (`end_date = null`) are treated as "available without limit" for filtering
|
||||
|
||||
## Global rules
|
||||
|
||||
1. **Always prefer the latest search**
|
||||
- When a tool returns a JSON block containing `search_id`, treat it as the current/latest search.
|
||||
- Replace any previously stored search context with this latest `search_id`.
|
||||
- If the user says “show Good/Partial/Low” without a `search_id`, use the latest `search_id`.
|
||||
- If the tool output contains a line `Using SEARCH_ID=<uuid>`, treat **that exact value** as authoritative.
|
||||
- Never guess or substitute a UUID from other context. Only use search IDs that were returned by tools.
|
||||
- **Never “make up” a UUID** and never copy a UUID from user prose/logs—only from tool output headers.
|
||||
|
||||
2. **Do not rerun a search unless necessary**
|
||||
- If the user asks to show another category/page for an existing search, do not call `find_matching_capacities` again.
|
||||
- Only rerun matching if:
|
||||
- no prior `search_id` exists, or
|
||||
- the user explicitly requests a new search, or
|
||||
- `get_results_by_category` / `filter_search_results` returns “unknown_or_expired”. In that case: **do not try another ID**; rerun `find_matching_capacities`.
|
||||
|
||||
3. **If the user’s `search_id` is invalid**
|
||||
- If the server returns “Invalid search_id format (expected UUID)”, instruct the user to copy the UUID from the tool output header (`SEARCH_ID=<uuid>` or `META={...}`).
|
||||
|
||||
4. **Confirmation gate**
|
||||
- If `find_matching_capacities` returns a confirmation error (requirements must be confirmed), do **not** auto-confirm.
|
||||
- Call `show_pending_requirements()` (preferred) to display the review table.
|
||||
- Ask the user explicitly to confirm (Yes/No).
|
||||
- Only after user approval call `confirm_requirements(confirm=true)`.
|
||||
- Then rerun the matching call.
|
||||
- If the user says No: call `confirm_requirements(confirm=false)` and continue requirement capture.
|
||||
- (Alternative marker: if requirements were already shown to the user in chat, call `request_requirements_confirmation()` before `confirm_requirements(...)`.)
|
||||
- If confirmation is disabled by config, proceed directly.
|
||||
|
||||
5. **Be table-first**
|
||||
- Prefer tool outputs that are Markdown tables.
|
||||
|
||||
## Tools and when to use them
|
||||
|
||||
## Database Schema
|
||||
|
||||
The server queries these views from the Open Data Lake (Trino):
|
||||
|
||||
Capacity profiles:
|
||||
|
||||
- `teamlandkarte_v_capacities_latest`
|
||||
- `teamlandkarte_v_capacity_competences_latest`
|
||||
- `teamlandkarte_v_competences_latest`
|
||||
|
||||
Capacity LLM-fulltext profiles additionally read:
|
||||
|
||||
- `teamlandkarte_v_capacities_latest.description`
|
||||
- `teamlandkarte_v_capacity_certificates_latest` (joined on `capacity_id`)
|
||||
- `teamlandkarte_v_capacity_references_latest` (joined on `capacity_id`)
|
||||
- `teamlandkarte_v_partners_latest` (LEFT JOIN on `capacity_references.partner_id = partners.id`)
|
||||
|
||||
Team profiles read four additional views (used by `find_matching_teams`, `list_teams`, and `get_team_details`):
|
||||
|
||||
- `teamlandkarte_v_teams_latest` — team master data (`team_id`, `ouid`, `about_us`, `offerings`, `interests`, `focus_name`).
|
||||
- `teamlandkarte_v_teammeter_organizational_units_latest` — joined to teams via **INNER JOIN** `teams_latest.team_id = organizational_units_latest.id`. The column `name` is exposed as the team name. Teams without a matching organizational unit row are excluded by the inner join.
|
||||
- `teamlandkarte_v_teammeter_team_competences_latest` — joined to teams via `ouid`. Each entry carries a `top_competency` flag (NULL is normalized to `false`). The competence name is resolved via `teamlandkarte_v_competences_latest` analogous to capacity competences. Entries without a resolvable competence name are filtered out.
|
||||
- `teamlandkarte_v_team_references_latest` — joined to teams via `ouid`, with a **LEFT JOIN** `team_references.partner_id = partners_latest.id` to resolve the partner name. References with `NULL` or whitespace-only `projects` are filtered out; references without a matching partner keep `partner_name = ""` and are still returned.
|
||||
|
||||
The partner LEFT JOIN is part of the same SQL query that loads the references, so no extra round-trip is needed.
|
||||
|
||||
Tasks/skills are read from:
|
||||
|
||||
- `beschaffungstool_kmp_task_latest`
|
||||
- `beschaffungstool_kmp_skill_latest`
|
||||
|
||||
Notes:
|
||||
|
||||
- The task "required role name" field is currently **not available** in the DB
|
||||
and is therefore not shown.
|
||||
|
||||
## Embeddings performance: global prefetch + batching
|
||||
|
||||
Matching uses Azure OpenAI embeddings for competence/role similarity.
|
||||
|
||||
To reduce latency and cost, the server:
|
||||
|
||||
- deduplicates and **prefetches embeddings globally per matching run** (required +
|
||||
all candidate competences/roles)
|
||||
- resolves embeddings via in-memory cache → SQLite cache → Azure API
|
||||
- uses **batched** Azure embeddings calls (`input=[...]`) with sequential
|
||||
chunking
|
||||
|
||||
### Tuning batch size
|
||||
|
||||
You can adjust the number of inputs per Azure embeddings request via:
|
||||
|
||||
- `azure_openai.embedding_batch_size` (default: `128`)
|
||||
|
||||
See `config.toml.example` for details.
|
||||
|
||||
## Development
|
||||
|
||||
### Project Structure
|
||||
|
||||
```
|
||||
teamlandkarte-mcp/
|
||||
├── src/
|
||||
│ └── teamlandkarte_mcp/
|
||||
│ ├── mcp_server.py # MCP server implementation
|
||||
│ ├── config.py # Configuration management
|
||||
│ ├── database/
|
||||
│ │ └── trino_client.py # Trino database client
|
||||
│ ├── matching/
|
||||
│ │ ├── matcher.py # Matching algorithm
|
||||
│ │ └── scorer.py # Scoring logic
|
||||
│ └── cache/
|
||||
│ └── query_cache.py # Result caching
|
||||
├── openspec/ # Specifications & changes
|
||||
├── config.toml # Configuration (not in VCS)
|
||||
└── main.py # Entry point
|
||||
```
|
||||
|
||||
### OpenSpec Workflow
|
||||
|
||||
This project uses OpenSpec for spec-driven development:
|
||||
|
||||
```bash
|
||||
# View active changes
|
||||
openspec list
|
||||
|
||||
# View current change
|
||||
openspec show add-capacity-matching-mcp-server
|
||||
|
||||
# Validate specifications
|
||||
openspec validate --strict
|
||||
```
|
||||
|
||||
## Local Setup (uv)
|
||||
|
||||
This project uses `uv` to manage dependencies and a virtual environment under `.venv`.
|
||||
|
||||
1. Create/sync the environment:
|
||||
- `uv venv .venv`
|
||||
- `uv pip sync pyproject.toml`
|
||||
|
||||
2. Install the project as editable (creates the `teamlandkarte-mcp` console script):
|
||||
- `uv pip install -e .`
|
||||
|
||||
3. Configure credentials:
|
||||
- Copy `config.toml.example` to `config.toml` and fill in your credentials.
|
||||
- `config.toml` is ignored via `.gitignore` and must stay local-only.
|
||||
|
||||
## Running (stdio MCP)
|
||||
|
||||
Run the MCP server locally via stdio:
|
||||
|
||||
- `.venv/bin/teamlandkarte-mcp`
|
||||
|
||||
If your shell does not automatically find the console script, always use the explicit path above.
|
||||
|
||||
## Security
|
||||
|
||||
- `config.toml` contains credentials and must not be committed.
|
||||
- If credentials were pasted into chat logs or ever committed to git history, rotate them.
|
||||
|
||||
See `docs/security.md`.
|
||||
|
||||
## Documentation
|
||||
|
||||
- Configuration reference: `config.toml.example`
|
||||
- Architecture and ADRs: `openspec/changes/add-capacity-matching-mcp-server/architecture.md`
|
||||
- Design notes: `openspec/changes/add-capacity-matching-mcp-server/design.md`
|
||||
- Capacity matching specification (Round 6): `openspec/changes/add-capacity-matching-mcp-server/specs/capacity-matching/spec.md`
|
||||
- Security: `docs/security.md`
|
||||
|
||||
## Roadmap
|
||||
|
||||
**Stage 1** (Current): Basic capacity matching
|
||||
- Free-text and structured input modes
|
||||
- LLM-based competence matching
|
||||
- Categorical scoring
|
||||
- Result caching
|
||||
|
||||
**Future Stages**:
|
||||
- Advanced filtering options
|
||||
- Capacity booking workflow
|
||||
- Historical trend analysis
|
||||
- Integration with other HR systems
|
||||
|
||||
## Contributing
|
||||
|
||||
1. Review the [OpenSpec workflow](openspec/AGENTS.md)
|
||||
2. Create change proposals for significant features
|
||||
3. Follow the implementation tasks
|
||||
4. Submit merge requests on GitLab
|
||||
|
||||
## License
|
||||
|
||||
[Add license information]
|
||||
|
||||
## Authors
|
||||
|
||||
Thomas Handke - Deutsche Bahn Systel GmbH
|
||||
|
||||
## Links
|
||||
|
||||
- [GitLab Repository](https://git.tech.rz.db.de/ThomasHandke/teamlandkarte-mcp)
|
||||
- [Model Context Protocol](https://modelcontextprotocol.io/)
|
||||
- [DB Systel Open Data Lake Documentation](https://docs.example.com) <!-- Update with actual link -->
|
||||
@@ -1,85 +0,0 @@
|
||||
# Teamlandkarte MCP configuration
|
||||
# Copy this file to `config.toml` and adjust values.
|
||||
#
|
||||
# Credentials are never stored in this file. Provide them via environment
|
||||
# variables (e.g. via `.env`):
|
||||
# - DATA_LAKE_USERNAME
|
||||
# - DATA_LAKE_PASSWORD
|
||||
# - AZURE_OPENAI_LLM_API_KEY
|
||||
|
||||
[database]
|
||||
backend = "trino"
|
||||
host = "trino.example.local"
|
||||
port = 443
|
||||
http_scheme = "https"
|
||||
verify_ssl = true
|
||||
catalog = "hive"
|
||||
schema = "tier1_open_lake"
|
||||
connect_timeout = 10
|
||||
pool_size = 4
|
||||
|
||||
[matching]
|
||||
competence_weight = 0.8
|
||||
role_weight = 0.2
|
||||
require_confirmation = true
|
||||
# Default matching method used when callers do not pass `matching_method`.
|
||||
# Allowed values:
|
||||
# - "score" : BM25 + LLM role similarity (numeric scores).
|
||||
# - "llm_fulltext" : LLM-based full-text matching with rationale.
|
||||
default_method = "score"
|
||||
|
||||
[matching.thresholds]
|
||||
top = 0.8
|
||||
good = 0.65
|
||||
partial = 0.5
|
||||
low = 0.3
|
||||
|
||||
[matching.fuzzy]
|
||||
min_similarity = 0.7
|
||||
|
||||
[matching.similarity]
|
||||
# When true, an LLM pre-expands each candidate's competence list with
|
||||
# canonical equivalents of required competences it already covers
|
||||
# (synonyms, abbreviations, cross-language). The expansion is ephemeral
|
||||
# and never persisted. Requires AZURE_OPENAI_LLM_API_KEY env var.
|
||||
# Default: false.
|
||||
use_auto_tagging = false
|
||||
|
||||
[matching.team]
|
||||
# Gewichtungsfaktor für Top-Kompetenzen im Score-basierten Team-Matching.
|
||||
# Wird nur von `find_matching_teams` (Profile_Type = "team") verwendet und
|
||||
# beeinflusst die Capacity-Suche nicht.
|
||||
# Der Wert muss numerisch und >= 1.0 sein. Default: 1.5.
|
||||
top_competency_weight = 1.5
|
||||
|
||||
[cache]
|
||||
db_ttl_hours = 12
|
||||
search_ttl_minutes = 60
|
||||
max_size = 100
|
||||
|
||||
[azure_openai]
|
||||
# Azure OpenAI endpoint configuration.
|
||||
#
|
||||
# IMPORTANT:
|
||||
# - `endpoint` must be your Azure OpenAI resource endpoint.
|
||||
# - `chat_deployment` must be set to a valid Azure deployment name.
|
||||
#
|
||||
# Credentials are read from:
|
||||
# - AZURE_OPENAI_LLM_API_KEY
|
||||
endpoint = "https://YOUR-RESOURCE.openai.azure.com"
|
||||
|
||||
# Azure OpenAI API version (must be supported by your resource).
|
||||
api_version = "2024-02-15-preview"
|
||||
|
||||
# Chat/LLM deployment name (e.g. "gpt-4.1"). Required.
|
||||
# API key must be provided via AZURE_OPENAI_LLM_API_KEY environment variable.
|
||||
chat_deployment = ""
|
||||
|
||||
# Optional: include a cost summary block in certain tool outputs.
|
||||
# This always remains approximate unless you configure prices in CostTracker.
|
||||
# Defaults to false.
|
||||
show_costs_in_output = false
|
||||
|
||||
# Maximale Anzahl paralleler LLM-Anfragen (1-50, Standard: 5).
|
||||
# Höhere Werte beschleunigen das Matching, können aber Rate-Limits auslösen.
|
||||
max_concurrency = 25
|
||||
@@ -1,720 +0,0 @@
|
||||
# Architecture Documentation (Arc42)
|
||||
|
||||
## 1. Introduction and Goals
|
||||
|
||||
### 1.1 Requirements Overview
|
||||
|
||||
The Teamlandkarte MCP Server enables AI assistants to match **tasks** to DB Systel employees with **free work capacity**, and to match a **specific capacity** to relevant **tasks**.
|
||||
|
||||
The server:
|
||||
- queries the DB Systel Open Data Lake (Trino) read-only
|
||||
- performs **BM25+RRF lexical matching** for competence similarity
|
||||
- uses **LLM chat completions** (Azure OpenAI) for role similarity and role inference
|
||||
- returns deterministic, table-first Markdown outputs suitable for chat UIs
|
||||
|
||||
Availability is displayed and may be applied as an optional date-range filter. It does **not** influence similarity scoring.
|
||||
|
||||
**Key Features:**
|
||||
- MCP workflow for DB tasks and ad-hoc matching
|
||||
- LLM-based role inference (tool: `infer_primary_role`) using Data Lake role vocabulary
|
||||
- BM25+RRF competence matching with optional LLM-based auto-tagging pre-expansion
|
||||
- **Two matching methods** selectable per call via `matching_method`:
|
||||
- `score` (default): BM25+RRF competence + LLM role similarity → numeric scores + category
|
||||
- `llm_fulltext`: LLM-based full-text comparison of complete profiles → category + rationale (no numeric scores)
|
||||
- Capacity browsing + capacity-to-task matching tools:
|
||||
- `list_free_capacities`, `get_capacity_details`, `find_matching_tasks`
|
||||
- **Hard confirmation gate** before matching runs by default (`matching.require_confirmation = true`)
|
||||
- Interactive result exploration with search session management (search_id + filter_id)
|
||||
- Multi-level caching (DB query cache + search result cache)
|
||||
- **Strict DB schema verification at startup** (fail-fast) against required view columns
|
||||
- Fuzzy filtering on search results (role, competence) via `fuzzywuzzy`
|
||||
|
||||
### 1.2 Quality Goals
|
||||
|
||||
| Priority | Quality Goal | Motivation |
|
||||
|----------|-------------|------------|
|
||||
| 1 | **Security** | Read-only database access, secure credential management, no data leakage |
|
||||
| 2 | **Performance** | Fast response times through caching, connection pooling |
|
||||
| 3 | **Usability** | Simple MCP interface, clear result presentation, deterministic tool outputs |
|
||||
| 4 | **Maintainability** | Modular architecture, clear separation of concerns, testable components |
|
||||
| 5 | **Reliability** | Fail-fast schema checks, graceful degradation on LLM failures |
|
||||
|
||||
#### Reliability
|
||||
|
||||
- The server uses Azure OpenAI for **chat completions** (role similarity, role inference, auto-tagging).
|
||||
- There is **no embedding dependency** -- competence matching is purely lexical (BM25+RRF).
|
||||
- LLM failures in role similarity return 0.0 (graceful degradation).
|
||||
- LLM failures in auto-tagging fall back to the unmodified candidate list.
|
||||
- LLM failures in role inference return `None` (surfaced to the user).
|
||||
|
||||
### 1.3 Stakeholders
|
||||
|
||||
| Role | Contact | Expectations |
|
||||
|------|---------|--------------|
|
||||
| Product Owner | Thomas Handke | Feature delivery, quality, timeline |
|
||||
| End Users | DB Systel Employees | Fast, accurate matching |
|
||||
| Database Team | DB Systel IT | Minimal database load, read-only access |
|
||||
| Security Team | DB Systel Security | Credential protection, audit logging |
|
||||
|
||||
---
|
||||
|
||||
## 2. Architecture Constraints
|
||||
|
||||
### 2.1 Technical Constraints
|
||||
|
||||
| Constraint | Background |
|
||||
|------------|------------|
|
||||
| Python 3.13+ | Required for latest language features and type hints |
|
||||
| Trino client | Required for Trino/Presto connectivity to the Open Data Lake |
|
||||
| MCP Protocol | Must comply with Model Context Protocol specification (FastMCP SDK) |
|
||||
| Azure OpenAI chat completions | Required for role similarity, role inference, and optional auto-tagging |
|
||||
| Read-only database | No write operations allowed on production database |
|
||||
| Strict schema verification | Server fails fast if required view columns are missing |
|
||||
|
||||
### 2.2 Organizational Constraints
|
||||
|
||||
| Constraint | Background |
|
||||
|------------|------------|
|
||||
| Internal DB Infrastructure | Hosted on Deutsche Bahn Systel GitLab and data lake |
|
||||
| Credential Management | Stored in environment variables / `.env`, not in version control |
|
||||
| OpenSpec Workflow | All significant changes require spec proposals |
|
||||
|
||||
### 2.3 Conventions
|
||||
|
||||
| Convention | Details |
|
||||
|------------|---------|
|
||||
| Code Style | PEP 8, type hints, dataclasses for DTOs, ruff for linting/formatting |
|
||||
| Configuration | TOML format (`config.toml`) for all configuration |
|
||||
| API Design | MCP tools with clear parameter schemas, Markdown table outputs |
|
||||
| Documentation | Inline docstrings, OpenSpec for architecture decisions |
|
||||
| Testing | pytest + pytest-asyncio + hypothesis |
|
||||
|
||||
---
|
||||
|
||||
## 3. System Scope and Context
|
||||
|
||||
### 3.1 Business Context
|
||||
|
||||
```text
|
||||
End User (DB Systel Employee)
|
||||
| natural language
|
||||
AI Assistant (MCP client)
|
||||
| MCP (tool calls over stdio)
|
||||
Teamlandkarte MCP Server
|
||||
| SQL (read-only)
|
||||
DB Systel Open Data Lake (Trino)
|
||||
|
||||
Teamlandkarte MCP Server
|
||||
| HTTPS (chat completions)
|
||||
Azure OpenAI
|
||||
```
|
||||
|
||||
**External Entities:**
|
||||
|
||||
| Entity | Role | Interface |
|
||||
|--------|------|-----------|
|
||||
| AI Assistant (Claude, GPT, etc.) | Interprets user requests and calls MCP tools | MCP Protocol over stdio |
|
||||
| DB Systel Open Data Lake | Stores employee capacity, competence, task, and role data | Trino/Presto SQL |
|
||||
| Azure OpenAI | Provides chat completions for role similarity + inference + auto-tagging | HTTPS (Azure OpenAI REST API) |
|
||||
| End User | Requests matching through natural conversation | Natural language via AI assistant |
|
||||
|
||||
### 3.2 Technical Context
|
||||
|
||||
```text
|
||||
MCP Client (assistant)
|
||||
<-> MCP protocol (tool calls, stdio JSON-RPC)
|
||||
Teamlandkarte MCP Server (Python)
|
||||
<-> Trino (read-only SQL, connection pool)
|
||||
<-> Azure OpenAI (chat completions)
|
||||
<-> Local caches (DB query cache + search session cache)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. Solution Strategy
|
||||
|
||||
### 4.1 Architecture Approach
|
||||
|
||||
**Layered Architecture** with clear separation:
|
||||
1. **MCP Interface Layer** (`mcp_server.py`): Exposes tools to client, formats Markdown output
|
||||
2. **Business Logic Layer** (`matching/`): Matcher, scorer, SimilarityEngine, VocabularyCache, BM25, RRF, AutoTagger, **LlmFulltextMatcher** (LLM-based full-text matching with rationale)
|
||||
3. **Data Access Layer** (`database/`): TrinoClient, connection pool, query logger, schema verification, read-only guard
|
||||
4. **External Integration Layer** (`azure/`): AzureOpenAIClient, CostTracker
|
||||
5. **Infrastructure** (`cache/`, `utils/`, `config.py`): QueryCache, SearchCache, date/markdown utilities, configuration loading
|
||||
|
||||
### 4.2 Key Design Decisions
|
||||
|
||||
| Decision | Rationale |
|
||||
|----------|-----------|
|
||||
| **BM25+RRF competence matching** | Lexical matching eliminates embedding false-positives; zero-out rule ensures no token overlap = score 0.0 |
|
||||
| **LLM-based role similarity** | Semantic role comparison requires understanding synonyms and hierarchy; cached per-run |
|
||||
| **LLM-based role inference** | Selects from DB vocabulary; constrained to known roles only |
|
||||
| **LLM-based full-text matching (alternative method)** | `llm_fulltext` mode compares complete capacity/task profiles (description, references, certificates, skills) via Azure OpenAI Chat Completion and returns a category + 1–2 sentence rationale per item; no numeric scores |
|
||||
| **Optional LLM auto-tagging** | Bridges lexical gap for BM25 by expanding candidate competences with covered synonyms |
|
||||
| **Global BM25 index** | One index over all candidates' competences ensures stable IDF weights across the pool |
|
||||
| **Hard confirmation gating (default)** | Prevents accidental matching runs; user must confirm requirements |
|
||||
| **Markdown table output** | Better rendering in LLM clients; table-first outputs for robust parsing |
|
||||
| **Connection pooling** | Bounded pool (default 4) for Trino connections; thread-safe acquire/release |
|
||||
| **Strict startup schema verification** | Fail-fast when upstream views change columns |
|
||||
| **Two-tier caching** | DB query cache (TTL) + search session cache (TTL) for interactive exploration |
|
||||
| **Fuzzy filtering** | `fuzzywuzzy` for post-hoc filtering of search results by role/competence text |
|
||||
|
||||
### 4.3 Technology Stack
|
||||
|
||||
| Layer | Technology | Justification |
|
||||
|-------|------------|---------------|
|
||||
| Runtime | Python 3.13 | Modern features, strong typing |
|
||||
| MCP SDK | `mcp` (FastMCP) | MCP server runtime, stdio transport |
|
||||
| Database Client | `trino` | Trino/Presto SQL connectivity |
|
||||
| Caching | `cachetools` (TTLCache) | In-memory TTL caches for DB queries and search sessions |
|
||||
| Configuration | TOML (`tomllib`) | Human-readable, validated at startup |
|
||||
| Competence Matching | `rank-bm25` (BM25Okapi) | Lexical ranking, no native extensions |
|
||||
| Fuzzy Matching | `fuzzywuzzy` + `python-Levenshtein` | Fast fuzzy string matching for filters |
|
||||
| LLM Integration | `openai` (AsyncAzureOpenAI) | Azure OpenAI chat completions |
|
||||
| Environment | `python-dotenv` | Load credentials from `.env` |
|
||||
|
||||
---
|
||||
|
||||
## 5. Building Block View
|
||||
|
||||
### 5.1 Level 0: System Context
|
||||
|
||||
```
|
||||
+-------------------------------------------+
|
||||
| Teamlandkarte MCP Server |
|
||||
| (Task <-> Capacity Matching) |
|
||||
+-------------------------------------------+
|
||||
```
|
||||
|
||||
### 5.2 Level 1: Container View
|
||||
|
||||
```
|
||||
+------------------------------------------------------------+
|
||||
| Teamlandkarte MCP Server |
|
||||
| |
|
||||
| +-------------------------------------------------------+ |
|
||||
| | MCP Interface Layer (mcp_server.py) | |
|
||||
| | - task browsing + details | |
|
||||
| | - requirement capture + confirmation gate | |
|
||||
| | - task->capacity matching + search refinement | |
|
||||
| | - capacity browsing + capacity->task matching | |
|
||||
| +----------------------------+---------------------------+ |
|
||||
| | |
|
||||
| +----------------------------v---------------------------+ |
|
||||
| | Business Logic Layer (matching/) | |
|
||||
| | - Matcher (orchestrates matching runs) | |
|
||||
| | - SimilarityEngine (BM25+RRF competence, LLM role) | |
|
||||
| | - VocabularyCache (LLM role inference from DB vocab) | |
|
||||
| | - Scorer (weighted overall score + categorization) | |
|
||||
| | - Bm25Index + bm25_rank_competences | |
|
||||
| | - reciprocal_rank_fusion (RRF normalization) | |
|
||||
| | - AutoTagger (optional LLM competence expansion) | |
|
||||
| | - LlmFulltextMatcher (LLM full-text profile matching) | |
|
||||
| +----------------------------+---------------------------+ |
|
||||
| | |
|
||||
| +----------------------------v---------------------------+ |
|
||||
| | Data Access Layer (database/) | |
|
||||
| | - TrinoClient (SQL queries, implements DBClient) | |
|
||||
| | - ConnectionPool (bounded, thread-safe) | |
|
||||
| | - SchemaVerifier (startup fail-fast) | |
|
||||
| | - ReadOnly guard (SELECT/WITH only) | |
|
||||
| | - QueryLogger (redacted SQL logging) | |
|
||||
| +----------------------------+---------------------------+ |
|
||||
| | |
|
||||
| +----------------------------v---------------------------+ |
|
||||
| | External Integration (azure/) | |
|
||||
| | - AzureOpenAIClient (chat completions, retry/backoff) | |
|
||||
| | - CostTracker (session-level token accounting) | |
|
||||
| +--------------------------------------------------------+ |
|
||||
| |
|
||||
| +--------------------------------------------------------+ |
|
||||
| | Infrastructure (cache/, utils/, config.py) | |
|
||||
| | - QueryCache (TTL, DB results) | |
|
||||
| | - SearchCache (TTL, search sessions + filters) | |
|
||||
| | - dates.py (ISO parsing, overlap checks) | |
|
||||
| | - markdown.py (table rendering) | |
|
||||
| | - config.py (TOML loading + validation) | |
|
||||
| +--------------------------------------------------------+ |
|
||||
+------------------------------------------------------------+
|
||||
| |
|
||||
v v
|
||||
+-----------------+ +-----------------+
|
||||
| Open Data Lake | | Azure OpenAI |
|
||||
| (Trino) | | (Chat API) |
|
||||
+-----------------+ +-----------------+
|
||||
```
|
||||
|
||||
### 5.3 Level 2: Source Module Map
|
||||
|
||||
```
|
||||
src/teamlandkarte_mcp/
|
||||
├── __init__.py
|
||||
├── __main__.py # CLI entry point, arg parsing, logging setup
|
||||
├── config.py # TOML config loading + validation (AppConfig)
|
||||
├── logging_config.py # Logging to stderr (MCP stdio safety)
|
||||
├── mcp_server.py # FastMCP server, all tool definitions, SessionState
|
||||
├── models.py # Frozen dataclasses: Task, Capacity, Requirements, ScoredCapacity
|
||||
├── azure/
|
||||
│ ├── openai_client.py # AsyncAzureOpenAI wrapper (chat completions, retry)
|
||||
│ └── cost_tracker.py # Session-level token/cost accounting
|
||||
├── cache/
|
||||
│ ├── query_cache.py # Generic TTLCache wrapper for DB queries
|
||||
│ └── search_cache.py # Search session store (search_id, filter_id, TTL)
|
||||
├── database/
|
||||
│ ├── types.py # DBClient Protocol definition
|
||||
│ ├── db_client.py # Factory: create_db_client() -> TrinoClient
|
||||
│ ├── trino_client.py # TrinoClient (implements DBClient, uses pool)
|
||||
│ ├── pool.py # ConnectionPool (bounded, thread-safe, LIFO)
|
||||
│ ├── schema_verifier.py # Startup column verification
|
||||
│ ├── read_only.py # SQL guard (SELECT/WITH only)
|
||||
│ ├── query_logger.py # Redacted query logging
|
||||
│ └── hive_client.py # (empty, reserved)
|
||||
├── matching/
|
||||
│ ├── matcher.py # Matcher: orchestrates matching runs
|
||||
│ ├── scorer.py # compute_overall(), categorize()
|
||||
│ ├── similarity.py # SimilarityEngine (BM25+RRF competence, LLM role)
|
||||
│ ├── vocabulary.py # VocabularyCache (LLM role inference)
|
||||
│ ├── bm25.py # Bm25Index, bm25_rank_competences, _tokenize
|
||||
│ ├── rrf.py # reciprocal_rank_fusion (zero-out rule)
|
||||
│ ├── auto_tagger.py # AutoTagger (LLM competence expansion)
|
||||
│ ├── llm_fulltext_matcher.py # LlmFulltextMatcher (LLM full-text profile matching)
|
||||
│ ├── profiles.py # CapacityProfile, TaskProfile + serializers
|
||||
│ ├── task_analyzer.py # (deprecated stub)
|
||||
│ └── task_helpers.py # (deprecated stub)
|
||||
└── utils/
|
||||
├── dates.py # parse_iso_date, availability_overlaps
|
||||
└── markdown.py # md_table (GFM table rendering)
|
||||
```
|
||||
|
||||
### 5.4 Tool Surface (MCP Tools)
|
||||
|
||||
#### Discovery (DB tasks)
|
||||
|
||||
| Tool | Responsibility | Dependencies |
|
||||
|------|----------------|--------------|
|
||||
| `list_open_tasks` | List newest published tasks | DB |
|
||||
| `get_task_details` | Table-first task fields + inferred role | DB, LLM (role inference) |
|
||||
| `validate_task_requirements` | Compare DB fields vs LLM-inferred role + competences | DB, LLM |
|
||||
| `infer_primary_role` | Infer closest role from task id or free text | DB vocab, LLM |
|
||||
|
||||
#### Requirement capture + confirmation gate
|
||||
|
||||
| Tool | Responsibility | Dependencies |
|
||||
|------|----------------|--------------|
|
||||
| `extract_requirements` | LLM-based inference for role + competences from free text | DB vocab, LLM |
|
||||
| `collect_structured_requirement_data` | Accept user-supplied fields and stage requirements | session state |
|
||||
| `update_requirements` | (deprecated stub, directs to `collect_structured_requirement_data`) | -- |
|
||||
| `start_guided_capture` | Start step-by-step guided capture | session state |
|
||||
| `guided_set_description` | Set description (step 1/4) | session state |
|
||||
| `guided_set_role` | Set role (step 2/4) | session state |
|
||||
| `guided_set_time_range` | Set date range (step 3/4) | session state |
|
||||
| `guided_set_competences` | Set competences (step 4/4) | session state |
|
||||
| `show_pending_requirements` | Show review table for pending requirements | session state |
|
||||
| `request_requirements_confirmation` | Mark that assistant asked user to confirm | session state |
|
||||
| `confirm_requirements` | Confirm or reject staged requirements | session state |
|
||||
|
||||
#### Task -> capacity matching
|
||||
|
||||
| Tool | Responsibility | Dependencies |
|
||||
|------|----------------|--------------|
|
||||
| `find_matching_capacities` | Compute matches and store a search session (`search_id`). Accepts `matching_method` (`"score"` \| `"llm_fulltext"`). | DB, BM25, LLM (role) or LlmFulltextMatcher, caches |
|
||||
| `filter_search_results` | Refine stored results (fuzzy role/competence, availability, min_similarity). `min_similarity` is ignored in `llm_fulltext` mode and surfaced as a hint in the Applied Filters table. | SearchCache |
|
||||
| `get_results_by_category` | Page through stored results by category. Renders `Begründung` column instead of score columns when the search was run in `llm_fulltext` mode. | SearchCache |
|
||||
|
||||
#### Capacity browsing + capacity -> task matching
|
||||
|
||||
| Tool | Responsibility | Dependencies |
|
||||
|------|----------------|--------------|
|
||||
| `list_free_capacities` | List recently created free capacities | DB |
|
||||
| `get_capacity_details` | Table-first capacity fields + next steps | DB |
|
||||
| `find_matching_tasks` | Find tasks that match a specific capacity. Accepts `matching_method` (`"score"` \| `"llm_fulltext"`). | DB, BM25, LLM (role) or LlmFulltextMatcher, caches |
|
||||
|
||||
#### Team browsing + task -> team matching
|
||||
|
||||
| Tool | Responsibility | Dependencies |
|
||||
|------|----------------|--------------|
|
||||
| `list_teams` | List the first *N* teams (default 20) as a Markdown table (`Team Id`, `Team Name`, `Schwerpunkt`, `Anzahl Kompetenzen`, `Anzahl Referenzen`). | DB |
|
||||
| `get_team_details` | Table-first team fields plus sections `## Über uns`, `## Leistungen`, `## Interessen`, `## Kompetenzen` (with `(Top)` marker), `## Referenzen` (Partner_Name + Projekte) and `## Next steps`. | DB |
|
||||
| `find_matching_teams` | Match a task against team profiles and store a search session (`search_id`). Accepts `matching_method` (`"score"` \| `"llm_fulltext"`). Persists `search_type = "team_search"` and the chosen `matching_method` in `SearchCache` and the META JSON. | DB, BM25, LLM (role) or LlmFulltextMatcher, caches |
|
||||
|
||||
#### Profile_Type parameter
|
||||
|
||||
| Parameter | Allowed values | Default | Selection mechanism | Affected tools |
|
||||
|-----------|---------------|---------|---------------------|----------------|
|
||||
| `Profile_Type` | `"capacity"`, `"team"` | implicit per tool name | Tool selection (`find_matching_capacities` ⇒ `capacity`, `find_matching_teams` ⇒ `team`); persisted as `search_type` (`"capacity_search"` / `"team_search"`) in `SearchCache` and META JSON | `find_matching_capacities`, `find_matching_teams`, `list_teams`, `get_team_details` |
|
||||
|
||||
- `Profile_Type = "capacity"` (existing behavior): match a task against employee capacity profiles. `find_matching_capacities` is the entry point; availability filters apply.
|
||||
- `Profile_Type = "team"` (new): match a task against aggregated team profiles. `find_matching_teams` is the entry point; `list_teams` and `get_team_details` browse and inspect team profiles independently of a matching run. Availability filters are **not** applied for team searches and are surfaced as not-effective hints in the `Applied Filters` table.
|
||||
|
||||
#### Matching method parameter
|
||||
|
||||
| Parameter | Allowed values | Default | Affected tools |
|
||||
|-----------|---------------|---------|----------------|
|
||||
| `matching_method` | `"score"`, `"llm_fulltext"` | `[matching].default_method` (TOML, default `"score"`) | `find_matching_capacities`, `find_matching_tasks`, `find_matching_teams` |
|
||||
|
||||
- `"score"`: existing BM25+RRF competence + LLM role similarity. Output includes `Role Score`, `Competence Score`, `Overall Score`, `Category`. For team searches, the team's `focus_name` is used as the role stand-in and top competences are weighted by `matching.team.top_competency_weight` (default `1.5`).
|
||||
- `"llm_fulltext"`: LLM-based full-text comparison via `LlmFulltextMatcher`. Output replaces all score columns with a single `Begründung` column (1–2 sentence rationale from the LLM). The persisted META JSON contains `matching_method` so downstream tools (`get_results_by_category`, `filter_search_results`) know which schema to render.
|
||||
|
||||
For team searches, the result table columns are:
|
||||
|
||||
| Mode | Columns |
|
||||
|------|---------|
|
||||
| `team_search` × `score` | `Team Name`, `Schwerpunkt`, `Top-Kompetenzen`, `Role Score`, `Competence Score`, `Overall Score`, `Category` |
|
||||
| `team_search` × `llm_fulltext` | `Team Name`, `Schwerpunkt`, `Top-Kompetenzen`, `Category`, `Begründung` |
|
||||
|
||||
### 5.5 LlmFulltextMatcher (component)
|
||||
|
||||
The `LlmFulltextMatcher` (in `matching/llm_fulltext_matcher.py`) is the business-logic component behind `matching_method = "llm_fulltext"`.
|
||||
|
||||
| Aspect | Details |
|
||||
|--------|---------|
|
||||
| **Inputs** | A `TaskProfile` (id, title, description, skills) for `match_capacities`; a `CapacityProfile` (id, owner_name, role_name, competences, description, references with `partner_name`/`projects`, certificates) for `match_tasks`. The candidate list (`capacities` or `tasks`) is already prefiltered by the MCP tool (same prefilter as in `score` mode, e.g. availability). |
|
||||
| **Profile sources** | Built from in-memory `Capacity`/`Task` objects plus extras loaded from the DB via `DBClient.batch_get_capacity_descriptions`, `batch_get_capacity_certificates`, and `batch_get_capacity_references` (with the partner LEFT JOIN inside the same SQL query). |
|
||||
| **Outputs** | `LlmFulltextResult` with `by_category: dict[str, list[LlmFulltextItem]]` (categories `Top`/`Good`/`Partial`/`Low`/`Irrelevant`; each item carries `category`, ungekürzte `rationale`, raw fields) and `errors: list[LlmFulltextError]` for items where the LLM call/parse failed. |
|
||||
| **External dependency** | Azure OpenAI Chat Completion via `AzureOpenAIClient.chat_completion(...)` with `response_format=json_object`, deterministic German system prompt, JSON schema `{"category": "...", "rationale": "..."}`. One LLM call per candidate. |
|
||||
| **Determinism** | Profile serialization is field-stable; results within a category are sorted lexicographically by `item_id`; invalid LLM categories map to `Irrelevant` with a hint appended to the rationale. |
|
||||
| **Persistence** | Items are stored in `SearchCache` with `category` + ungekürzte `rationale` (no numeric score fields); `errors` are persisted alongside; META JSON carries `matching_method`. |
|
||||
|
||||
---
|
||||
|
||||
## 6. Runtime View (Key Flows)
|
||||
|
||||
### 6.1 Server startup (fail-fast)
|
||||
|
||||
1. Parse CLI arguments (`--config`, `--log-level`)
|
||||
2. Configure logging (stderr only, MCP stdio safety)
|
||||
3. Load and validate `config.toml` + environment variables
|
||||
4. Create DB client (TrinoClient with connection pool)
|
||||
5. Verify required DB view columns (schema verification -> fail-fast on mismatch)
|
||||
6. Initialize caches (QueryCache, SearchCache)
|
||||
7. Initialize Azure OpenAI client + CostTracker
|
||||
8. Optionally construct AutoTagger (if `use_auto_tagging = true`)
|
||||
9. Initialize SimilarityEngine, VocabularyCache, Matcher
|
||||
10. Register all MCP tools
|
||||
11. Start MCP server (stdio transport via `mcp.run()`)
|
||||
|
||||
### 6.2 Task -> capacity matching
|
||||
|
||||
1. Capture requirements (guided capture or `extract_requirements` or `collect_structured_requirement_data`)
|
||||
2. (If `require_confirmation = true`) confirmation gate:
|
||||
- `show_pending_requirements` -> user reviews
|
||||
- `request_requirements_confirmation` -> assistant marks confirmation requested
|
||||
- `confirm_requirements(true)` -> user confirms
|
||||
3. Run `find_matching_capacities(role_name, competences, date_start?, date_end?, matching_method?)`:
|
||||
- Resolve `matching_method` (parameter > `[matching].default_method` > `"score"`)
|
||||
- Query all capacities + competences from DB (cached)
|
||||
- Filter by date overlap (same prefilter for both methods)
|
||||
- Branch by `matching_method`:
|
||||
- `"score"`: build global BM25 index, run BM25+RRF competence similarity + LLM role similarity, compute weighted overall score + categorize
|
||||
- `"llm_fulltext"`: batch-load `description`, `certificates`, `references` (with partner LEFT JOIN) for the filtered capacities; build `TaskProfile` + `CapacityProfile`; one Azure OpenAI Chat Completion per capacity (`response_format=json_object`); LLM returns `{category, rationale}`; invalid categories → `Irrelevant` with hint; LLM/JSON failures recorded in a separate `errors` list
|
||||
- Store results in SearchCache (payload includes `matching_method`; LLM mode stores `category` + ungekürzte `rationale` per item, no score fields) -> return `search_id`
|
||||
4. Browse/filter results:
|
||||
- `get_results_by_category(search_id, category, page, page_size)` (renders `Begründung` column in LLM mode)
|
||||
- `filter_search_results(search_id, ...)` -> returns `filter_id`. In LLM mode, `min_similarity` is ignored and surfaced as a hint; sorting falls back to `(category_rank, item_id)`.
|
||||
|
||||
### 6.3 Capacity -> task matching
|
||||
|
||||
1. Browse capacities (`list_free_capacities`)
|
||||
2. Inspect (`get_capacity_details`)
|
||||
3. Run `find_matching_tasks(capacity_id, matching_method?)`:
|
||||
- Resolve `matching_method` (parameter > `[matching].default_method` > `"score"`)
|
||||
- Query capacity details + competences from DB
|
||||
- Query open tasks from DB (same prefilter for both methods)
|
||||
- Branch by `matching_method`:
|
||||
- `"score"`: for each task infer role + competences, compute BM25+RRF + LLM role similarity
|
||||
- `"llm_fulltext"`: load `description`, `certificates`, `references` (with partner LEFT JOIN) for the capacity; build `CapacityProfile` once; for each task build a `TaskProfile` and run one Azure OpenAI Chat Completion; same error/category-normalization handling as in 6.2
|
||||
- Store results as a search session (`matching_method` persisted) -> return `search_id`
|
||||
4. Browse/filter results (same tools as task->capacity)
|
||||
|
||||
### 6.4 Task -> team matching
|
||||
|
||||
1. Capture requirements as in 6.2 (guided capture, `extract_requirements`, or `collect_structured_requirement_data`)
|
||||
2. Optional confirmation gate (same as in 6.2): `show_pending_requirements` -> `request_requirements_confirmation` -> `confirm_requirements(true)`
|
||||
3. Run `find_matching_teams(role_name, competences, matching_method?)`:
|
||||
- Resolve `matching_method` (parameter > `[matching].default_method` > `"score"`); reject unknown values with an error listing the allowed values, without any DB or LLM call
|
||||
- Validate minimum requirements (`competences` non-empty)
|
||||
- Load all teams from the DB-backed cache `_get_teams_cached()`. The DB layer joins `teamlandkarte_v_teams_latest` with `teamlandkarte_v_teammeter_organizational_units_latest` (INNER JOIN over `team_id = id` for the team name), and aggregates competences (`teamlandkarte_v_teammeter_team_competences_latest` joined to `teamlandkarte_v_competences_latest` over `competence_id = id`) and references (`teamlandkarte_v_team_references_latest` LEFT JOIN `teamlandkarte_v_partners_latest` over `partner_id = id`, exposing `name` as `Partner_Name`). Joins between team master data, competences, and references run over `ouid`.
|
||||
- **No availability prefilter**: team profiles do not carry an availability range; any provided date range is ignored and surfaced as a not-effective hint in the `Applied Filters` table.
|
||||
- Branch by `matching_method`:
|
||||
- `"score"`: build the requirements competence list, derive each team's competence list and parallel top-competence set from `Team.competences`, and compute competence similarity via the existing `SimilarityEngine`. Top competences are upweighted by `matching.team.top_competency_weight` (default `1.5`); the per-required score is clamped to `[0, 1]`. Role similarity uses `Team.focus_name` as the role stand-in (`compute_role_similarity(req.role_name, team.focus_name)`). Overall score and category use the same `compute_overall(...)` / `categorize(...)` thresholds as for capacities.
|
||||
- `"llm_fulltext"`: build a `TaskProfile` from the requirements and a `TeamProfile` per team via `build_team_profile` + `serialize_team_profile`. The user prompt is `=== Aufgabe ===` + `=== Team ===` (with an `ID:` header). The system prompt and JSON schema (`{"category": ..., "rationale": ...}`) are reused from the capacity LLM path; invalid categories are normalized to `Irrelevant` with a hint suffix in the rationale; LLM/JSON errors land in a separate `errors` list.
|
||||
- Store results in `SearchCache` with `search_type = "team_search"` and `matching_method = ...` -> return `search_id`. The META JSON exposes both `search_type` and `matching_method` so `get_results_by_category` / `filter_search_results` render the team-specific columns.
|
||||
4. Browse/filter results:
|
||||
- `get_results_by_category(search_id, category, page, page_size)` (renders `Begründung` column in `llm_fulltext` mode)
|
||||
- `filter_search_results(search_id, ...)`. `role_filter` matches against `Team_Focus_Name`; `competence_filter` matches the competence names of the team and supports a `(Top)` suffix that restricts the filter to top competences. `availability_date_start`, `availability_date_end`, and `is_fully_available` are ignored for `team_search` and surfaced as `team_search` not-effective hints in the `Applied Filters` table.
|
||||
|
||||
### 6.5 Role inference flow
|
||||
|
||||
1. `infer_primary_role(task_id=... | task_text=...)`
|
||||
2. Fetch all role names from DB vocabulary view
|
||||
3. Send task text + role list to LLM (JSON response format)
|
||||
4. Validate returned role exists in DB vocabulary
|
||||
5. Return `(role_name, confidence)` or `None` on failure
|
||||
|
||||
---
|
||||
|
||||
## 7. Deployment View
|
||||
|
||||
```
|
||||
+-------------------------------------------------+
|
||||
| Developer Machine / CI |
|
||||
| |
|
||||
| +--------------------------------------------+ |
|
||||
| | MCP Client (IDE / AI Assistant) | |
|
||||
| | <-> stdio (JSON-RPC) | |
|
||||
| | teamlandkarte-mcp process | |
|
||||
| | (Python 3.13, single process) | |
|
||||
| +--------------------------------------------+ |
|
||||
| |
|
||||
| config.toml + .env (credentials) |
|
||||
+-------------------------------------------------+
|
||||
| |
|
||||
v v
|
||||
+-----------------+ +-----------------+
|
||||
| Trino cluster | | Azure OpenAI |
|
||||
| (Data Lake) | | (Chat API) |
|
||||
+-----------------+ +-----------------+
|
||||
```
|
||||
|
||||
**Requirements:**
|
||||
- Python 3.13+ with project dependencies installed
|
||||
- Trino connectivity (host/port via `config.toml`, credentials via env vars)
|
||||
- Azure OpenAI endpoint + chat deployment + API key (env var `AZURE_OPENAI_LLM_API_KEY`)
|
||||
- Environment variables: `DATA_LAKE_USERNAME`, `DATA_LAKE_PASSWORD`, `AZURE_OPENAI_LLM_API_KEY`
|
||||
|
||||
---
|
||||
|
||||
## 8. Cross-cutting Concepts
|
||||
|
||||
### 8.1 Deterministic outputs
|
||||
|
||||
- Tools produce **table-first Markdown output** (GFM tables via `md_table()`).
|
||||
- Search tools emit deterministic headers with `SEARCH_ID=...` / `FILTER_ID=...` for robust client parsing.
|
||||
- Scoring formula is deterministic: `overall = competence_weight * competence_score + role_weight * role_score`.
|
||||
|
||||
### 8.2 Caching strategy
|
||||
|
||||
| Cache | Implementation | TTL | Purpose |
|
||||
|-------|---------------|-----|---------|
|
||||
| DB query cache | `QueryCache` (cachetools TTLCache) | Configurable (`db_ttl_hours`, default 12h) | Avoid repeated expensive Trino queries |
|
||||
| Search session cache | `SearchCache` (cachetools TTLCache) | Configurable (`search_ttl_minutes`, default 60min) | Enable interactive exploration of results |
|
||||
| LLM role similarity cache | In-memory dict on `SimilarityEngine` | Per matching run (cleared between runs) | Avoid duplicate LLM calls for same role pair |
|
||||
|
||||
### 8.3 Security
|
||||
|
||||
- **Read-only DB access**: `ensure_select_only()` guard rejects non-SELECT/WITH queries
|
||||
- **No write operations**: Server never modifies the Data Lake
|
||||
- **Credentials via environment**: `DATA_LAKE_USERNAME`, `DATA_LAKE_PASSWORD`, `AZURE_OPENAI_LLM_API_KEY` -- never in config files
|
||||
- **Query logging**: Parameters are redacted before logging (`query_logger.py`)
|
||||
- **Logging to stderr**: MCP stdio transport requires stdout to remain clean JSON-RPC
|
||||
|
||||
### 8.4 Error handling
|
||||
|
||||
- **Schema verification failure**: Server refuses to start (fail-fast)
|
||||
- **DB connectivity**: Lazy validation on first DB-backed tool call
|
||||
- **LLM failures (role similarity)**: Return 0.0, no caching of failed result
|
||||
- **LLM failures (auto-tagging)**: Return unmodified candidate list (graceful degradation)
|
||||
- **LLM failures (role inference)**: Return `None`, logged as warning
|
||||
- **Connection pool exhaustion**: `PoolExhaustedError` raised immediately (no blocking wait)
|
||||
- **Config errors**: `ConfigError` raised at startup with descriptive message
|
||||
|
||||
### 8.5 Connection pooling
|
||||
|
||||
- `ConnectionPool` (generic, thread-safe, LIFO queue)
|
||||
- Bounded at `pool_size` (default 4) connections
|
||||
- Connections created lazily on demand
|
||||
- `closeall()` for graceful shutdown
|
||||
|
||||
---
|
||||
|
||||
## 9. Scoring and Matching Algorithm
|
||||
|
||||
### 9.1 Overall scoring formula
|
||||
|
||||
```
|
||||
overall_score = competence_weight * competence_score + role_weight * role_score
|
||||
```
|
||||
|
||||
Default weights: `competence_weight = 0.8`, `role_weight = 0.2` (must sum to 1.0).
|
||||
|
||||
### 9.2 Competence scoring (BM25 + RRF)
|
||||
|
||||
For each required competence:
|
||||
1. Query the global BM25 index (built over all filtered candidates' competences)
|
||||
2. Filter ranked results to the current candidate's competence set
|
||||
3. Apply Reciprocal Rank Fusion (k=60) to normalize scores to [0.0, 1.0]
|
||||
4. **Zero-out rule**: No token overlap -> score 0.0 (eliminates false positives)
|
||||
|
||||
Aggregate: `competence_score = mean(per_required_competence_scores)`
|
||||
|
||||
Threshold for "matched": score >= 0.5.
|
||||
|
||||
### 9.3 Role scoring (LLM)
|
||||
|
||||
- LLM chat completion compares two role names semantically -> similarity in [0.0, 1.0]
|
||||
- Identical roles (case-insensitive) -> 1.0 (short-circuit, no LLM call)
|
||||
- Empty/unknown roles -> 0.0 (short-circuit)
|
||||
- Results cached symmetrically per matching run
|
||||
|
||||
### 9.4 Categorization
|
||||
|
||||
| Category | Threshold |
|
||||
|----------|-----------|
|
||||
| Top | overall >= configured `top` (default 0.8) |
|
||||
| Good | overall >= configured `good` (default 0.65) |
|
||||
| Partial | overall >= configured `partial` (default 0.5) |
|
||||
| Low | overall >= configured `low` (default 0.3) |
|
||||
| Irrelevant | overall < configured `low` |
|
||||
|
||||
(Thresholds are configurable via `[matching.thresholds]`.)
|
||||
|
||||
### 9.5 Optional: Auto-Tagging (LLM competence expansion)
|
||||
|
||||
When `matching.similarity.use_auto_tagging = true`:
|
||||
1. Before BM25 scoring, the AutoTagger asks the LLM which required competences are already covered by the candidate's existing competences (via synonym, abbreviation, cross-language equivalence)
|
||||
2. Covered required-competence names are appended to the candidate's working list
|
||||
3. Expansion is **ephemeral** -- never persisted to DB
|
||||
4. On LLM failure, scoring proceeds with the unmodified list
|
||||
|
||||
---
|
||||
|
||||
## 10. Configuration Reference
|
||||
|
||||
### 10.1 `config.toml` structure
|
||||
|
||||
```toml
|
||||
[database]
|
||||
backend = "trino" # Only "trino" supported
|
||||
host = "..."
|
||||
port = 443
|
||||
http_scheme = "https"
|
||||
verify_ssl = true
|
||||
catalog = "hive"
|
||||
schema = "tier1_open_lake"
|
||||
connect_timeout = 10
|
||||
pool_size = 4
|
||||
|
||||
[matching]
|
||||
competence_weight = 0.8 # Must sum to 1.0 with role_weight
|
||||
role_weight = 0.2
|
||||
require_confirmation = true # Hard gate before matching
|
||||
default_method = "score" # Default for matching_method when callers omit it.
|
||||
# Allowed: "score" | "llm_fulltext".
|
||||
|
||||
[matching.thresholds]
|
||||
top = 0.8
|
||||
good = 0.65
|
||||
partial = 0.5
|
||||
low = 0.3
|
||||
|
||||
[matching.fuzzy]
|
||||
min_similarity = 0.7 # Fuzzy filter threshold
|
||||
|
||||
[matching.similarity]
|
||||
use_auto_tagging = false # LLM pre-expansion for BM25
|
||||
|
||||
[cache]
|
||||
db_ttl_hours = 12
|
||||
search_ttl_minutes = 60
|
||||
max_size = 100
|
||||
|
||||
[azure_openai]
|
||||
endpoint = "https://..."
|
||||
api_version = "2024-02-15-preview"
|
||||
chat_deployment = "gpt-4.1"
|
||||
show_costs_in_output = false
|
||||
```
|
||||
|
||||
### 10.2 Required environment variables
|
||||
|
||||
| Variable | Purpose |
|
||||
|----------|---------|
|
||||
| `DATA_LAKE_USERNAME` | Trino login username |
|
||||
| `DATA_LAKE_PASSWORD` | Trino login password |
|
||||
| `AZURE_OPENAI_LLM_API_KEY` | Azure OpenAI API key for chat completions |
|
||||
|
||||
---
|
||||
|
||||
## 11. Data Model
|
||||
|
||||
### 11.1 Core domain objects (frozen dataclasses)
|
||||
|
||||
| Class | Fields | Purpose |
|
||||
|-------|--------|---------|
|
||||
| `Task` | id, name, title, description, start_date, end_date, created_date, skills | Published task from DB |
|
||||
| `Capacity` | id, owner_name, role_name, role_level, begin_date, end_date, competences | Employee capacity entry |
|
||||
| `Requirements` | role_name, competences, date_start, date_end, description | Structured matching input |
|
||||
| `ScoredCapacity` | capacity, competence_score, role_score, overall_score, category, matched_competences, missing_competences | Matching result |
|
||||
| `Team` | team_id, ouid, team_name, focus_name, about_us, offerings, interests, competences, references | Aggregated team master data |
|
||||
| `TeamCompetence` | name, top_competency | Single team competence with top marker |
|
||||
| `TeamReference` | partner_name, projects | Single team reference; `partner_name` may be empty |
|
||||
| `ScoredTeam` | team, competence_score, role_score, overall_score, category, matched_competences, missing_competences | Team matching result (score mode) |
|
||||
|
||||
### 11.2 Team Profile
|
||||
|
||||
A **Team Profile** is the second profile type next to the existing capacity profile. It aggregates fields from four read-only views into a single immutable `Team` instance and is used both for browsing (`list_teams`, `get_team_details`) and for matching (`find_matching_teams` in both `score` and `llm_fulltext` modes).
|
||||
|
||||
| Field | Type | Source | Notes |
|
||||
|-------|------|--------|-------|
|
||||
| `team_id` | `str` | `teamlandkarte_v_teams_latest.team_id` | Stable identifier; persisted in `SearchCache`. |
|
||||
| `ouid` | `str` | `teamlandkarte_v_teams_latest.ouid` | Join key for competences and references. |
|
||||
| `team_name` | `str` | `teamlandkarte_v_teammeter_organizational_units_latest.name` | Resolved via INNER JOIN `teams_latest.team_id = organizational_units_latest.id`. Teams without an OU match are excluded. |
|
||||
| `focus_name` | `str` | `teamlandkarte_v_teams_latest.focus_name` | NULL → `""`. Used as role stand-in in `score` mode. |
|
||||
| `about_us` | `str` | `teamlandkarte_v_teams_latest.about_us` | NULL → `""`. |
|
||||
| `offerings` | `str` | `teamlandkarte_v_teams_latest.offerings` | NULL → `""`. |
|
||||
| `interests` | `str` | `teamlandkarte_v_teams_latest.interests` | NULL → `""`. |
|
||||
| `competences` | `list[TeamCompetence]` | `teamlandkarte_v_teammeter_team_competences_latest` joined to `teamlandkarte_v_competences_latest` over `competence_id = id` | Joined on `ouid`. `top_competency = COALESCE(top_competency, FALSE)`. Order: top competences first, then name ascending. |
|
||||
| `references` | `list[TeamReference]` | `teamlandkarte_v_team_references_latest` LEFT JOIN `teamlandkarte_v_partners_latest` over `partner_id = id` | Joined on `ouid`. Partner_Name from `p.name` (`COALESCE(..., '')`). Whitespace-only `projects` are filtered. Order: partner_name asc, projects asc. |
|
||||
|
||||
**Profile serialization** (`build_team_profile` + `serialize_team_profile` in `matching/profiles.py`) emits a deterministic, German-headed text block in fixed order: `Teamname:`, `Schwerpunkt:`, `Über uns:`, `Leistungen:`, `Interessen:`, `Kompetenzen:`, `Referenzen:`. Top competences are suffixed with `(Top)`. Reference rows render as `Partner: <partner_name> – Projekte: <projects>` when the partner is known, and as `Projekte: <projects>` (no placeholder) when the partner is empty. Empty lists render as `Kompetenzen: (keine)` / `Referenzen: (keine)`. List ordering matches the DB-provided order.
|
||||
|
||||
### 11.3 Database views (Trino)
|
||||
|
||||
The server queries read-only views in the `tier1_open_lake` schema. Schema verification at startup checks:
|
||||
- `teamlandkarte_v_capacity_roles_latest`: requires columns `name`, `active`, `staffing_board_relevant`
|
||||
- `teamlandkarte_v_capacities_latest`: requires column `creation_date`
|
||||
- `teamlandkarte_v_teams_latest`: requires columns `team_id`, `ouid`, `about_us`, `offerings`, `interests`, `focus_name`
|
||||
- `teamlandkarte_v_teammeter_organizational_units_latest`: requires columns `id`, `name`
|
||||
- `teamlandkarte_v_teammeter_team_competences_latest`: requires columns `ouid`, `competence_id`, `top_competency`
|
||||
- `teamlandkarte_v_team_references_latest`: requires columns `ouid`, `partner_id`, `projects`
|
||||
|
||||
#### Additional views read in `llm_fulltext` mode
|
||||
|
||||
The following columns/views are not part of the startup schema verification but are read at runtime when `matching_method = "llm_fulltext"`:
|
||||
|
||||
| View | Columns used | Used for |
|
||||
|------|--------------|----------|
|
||||
| `teamlandkarte_v_capacities_latest` | `description` | `CapacityProfile.description` |
|
||||
| `teamlandkarte_v_capacity_certificates_latest` | `capacity_id`, `description` | `CapacityProfile.certificates` (1:n via `capacity_id`) |
|
||||
| `teamlandkarte_v_capacity_references_latest` | `capacity_id`, `partner_id`, `projects` | `CapacityProfile.references[].projects` (1:n via `capacity_id`) |
|
||||
| `teamlandkarte_v_partners_latest` | `id`, `name` | Partner name per reference (`CapacityProfile.references[].partner_name`) |
|
||||
|
||||
**Reference ↔ Partner join:** `teamlandkarte_v_capacity_references_latest.partner_id = teamlandkarte_v_partners_latest.id`. The partner `name` column is exposed as `partner_name` (via `COALESCE(p.name, '')`) inside the same SQL query that loads the references, so no extra round-trip is required. When `partner_id` is `NULL` or no partner matches, `partner_name` is the empty string and the reference is still returned with its `projects` text. The LEFT JOIN keeps the reference list complete even for unknown partners.
|
||||
|
||||
#### Views read for team profiles
|
||||
|
||||
The following views back the `Team` data type and are loaded by `find_matching_teams`, `list_teams`, and `get_team_details`:
|
||||
|
||||
| View | Columns used | Used for |
|
||||
|------|--------------|----------|
|
||||
| `teamlandkarte_v_teams_latest` | `team_id`, `ouid`, `about_us`, `offerings`, `interests`, `focus_name` | Team master data and free-text fields |
|
||||
| `teamlandkarte_v_teammeter_organizational_units_latest` | `id`, `name` | `Team.team_name` via INNER JOIN over `teams_latest.team_id = organizational_units_latest.id` |
|
||||
| `teamlandkarte_v_teammeter_team_competences_latest` | `ouid`, `competence_id`, `top_competency` | Team competences with top markers (joined to `teamlandkarte_v_competences_latest` over `competence_id = id` to resolve the competence name) |
|
||||
| `teamlandkarte_v_team_references_latest` | `ouid`, `partner_id`, `projects` | Team references with partner names |
|
||||
| `teamlandkarte_v_partners_latest` | `id`, `name` | `Partner_Name` per team reference |
|
||||
|
||||
**Team_Name join (INNER JOIN):** `teamlandkarte_v_teams_latest.team_id = teamlandkarte_v_teammeter_organizational_units_latest.id`. Teams without a matching OU row are excluded from `get_all_teams` and `get_team_by_id`.
|
||||
|
||||
**Competence and reference joins:** Both `teamlandkarte_v_teammeter_team_competences_latest` and `teamlandkarte_v_team_references_latest` are joined to the team master data over the shared `ouid` column.
|
||||
|
||||
**Reference ↔ Partner join (LEFT JOIN):** `teamlandkarte_v_team_references_latest.partner_id = teamlandkarte_v_partners_latest.id`. The `name` column from `teamlandkarte_v_partners_latest` is exposed as `Partner_Name` via `COALESCE(p.name, '')` inside the same SQL query that loads the references. When `partner_id` is `NULL` or no partner matches, `Partner_Name` is the empty string and the reference is still returned with its `projects` text. Whitespace-only `projects` are filtered out in the DB layer.
|
||||
|
||||
---
|
||||
|
||||
## 12. Testing
|
||||
|
||||
- **Unit tests**: pytest + pytest-asyncio
|
||||
- **Property-based tests**: hypothesis (for scoring, BM25, RRF invariants)
|
||||
- **Type checking**: mypy (strict)
|
||||
- **Linting/formatting**: ruff
|
||||
- **No integration test infrastructure**: DB and Azure OpenAI are mocked in tests
|
||||
@@ -1,338 +0,0 @@
|
||||
# System prompt: Teamlandkarte Capacity Matching Assistant
|
||||
|
||||
You are a task-to-capacity matching assistant for the Teamlandkarte MCP server.
|
||||
|
||||
## Goals
|
||||
|
||||
- Help the user find matching capacities for a task.
|
||||
- Provide a predictable workflow.
|
||||
- Prefer safe, deterministic behavior.
|
||||
- Default language: respond in **German** (tools/tables may be English if returned). If the user writes in English, you may respond in English.
|
||||
- Unless the user explicitly asks otherwise, always operate on the **latest** search.
|
||||
|
||||
## When the user request does not map to a tool (must-do)
|
||||
|
||||
If the user does not express a request that clearly maps to one of the existing tools,
|
||||
output a numbered list of the available next actions (in a sensible order) and ask the
|
||||
user to choose one.
|
||||
|
||||
- If this happens in the **very first user message** of a chat session:
|
||||
- first greet the user briefly and introduce yourself,
|
||||
- then show the numbered options.
|
||||
|
||||
The numbered list must contain **only short, content-focused descriptions** (no tool
|
||||
names, no technical hints, no notes like "optional").
|
||||
|
||||
Suggested options order:
|
||||
|
||||
1. Bestehende Aufgaben ansehen (Liste öffentlicher Aufgaben)
|
||||
2. Details zu einer konkreten Aufgabe ansehen
|
||||
3. Anforderungen einer Aufgabe prüfen (Validierung)
|
||||
4. Passende Kapazitäten zu einer bestehenden Aufgabe finden
|
||||
5. Neue Aufgabe / Anforderungen definieren (geführte Abfrage oder manuell)
|
||||
6. Freie Kapazitäten ansehen (neueste Einträge)
|
||||
7. Details zu einer konkreten Kapazität ansehen
|
||||
8. Passende Aufgaben für eine konkrete Kapazität finden
|
||||
|
||||
Do not call tools until the user picks a path.
|
||||
|
||||
If the user asks for something that is out of scope (e.g., editing the database, changing Azure resources), explain briefly what you *can* do and offer the closest matching next actions.
|
||||
|
||||
## After each successful request (must-do)
|
||||
|
||||
After you have successfully handled the user’s request (answer given and/or tool call(s) executed without error), you MUST end your message with a numbered list of **sensible next actions** the user can take.
|
||||
|
||||
Rules:
|
||||
- The list must be short (typically 3–5 items) and adapted to the current context.
|
||||
- Use the same style rules as above: only content-focused descriptions (no tool names, no technical hints).
|
||||
- If there is an active search context, include at least one option to view more results (other category/page) and one option to refine/filter.
|
||||
- If there is no active search context, prefer options that lead the user forward (inspect details, validate, start matching).
|
||||
|
||||
## Mandatory requirement capture (must-do)
|
||||
|
||||
Before calling `find_matching_capacities(...)`, you MUST have (and confirm with the user) these fields:
|
||||
|
||||
- `role_name` (or explicitly set `role_name="Beliebige Rolle"` only if the user refuses to specify a role)
|
||||
- `competences` (a non-empty list; ask for missing/important competences)
|
||||
- `date_start` / `date_end` (ask for a time range; open-ended allowed)
|
||||
|
||||
Additionally, you MUST capture a **concrete topic/goal description** of the task.
|
||||
- A single skill request like “JavaScript” is **not** sufficient as description.
|
||||
- If the user only mentions skills/role, ask: "Worum geht es inhaltlich (Ziel/Scope)?".
|
||||
|
||||
If any of these are missing, DO NOT run matching yet. You must ask targeted follow-up questions.
|
||||
|
||||
### Confirmation gating semantics (important)
|
||||
|
||||
- The server's confirmation state is stored in in-memory session state.
|
||||
- This state is **not** safe as a multi-client / multi-tenant source of truth.
|
||||
- Therefore, treat confirmation gating primarily as a **client-side UX rule**:
|
||||
you (the agent) must only execute matching after an explicit user Yes/No.
|
||||
|
||||
### Confirmation workflow (must-do, strict)
|
||||
|
||||
Never auto-confirm.
|
||||
|
||||
- If `matching.require_confirmation = true`:
|
||||
1. After requirements have been captured/updated (including guided capture), call `show_pending_requirements()` to display the review table.
|
||||
2. You MUST ask the user explicitly: "Soll ich diese Anforderungen so übernehmen und die Suche starten?" (Ja/Nein).
|
||||
3. Only if the user replies "Ja" (or equivalent), call `confirm_requirements(confirm=true)`.
|
||||
- If the user says "Nein": call `confirm_requirements(confirm=false)` and continue requirement capture.
|
||||
|
||||
- If `matching.require_confirmation = false`:
|
||||
- Do **not** ask for confirmation.
|
||||
- Do **not** call `show_pending_requirements()` / `confirm_requirements()`.
|
||||
- Proceed directly to matching once requirements are complete.
|
||||
|
||||
Do NOT call `confirm_requirements(confirm=true)` in the same turn as requirement capture.
|
||||
|
||||
If you are unsure whether confirmation is enabled, prefer to **assume it is enabled** and ask for confirmation (safer UX) *unless* the user explicitly asks to skip confirmation.
|
||||
|
||||
### How to decide between `extract_requirements(...)` and guided capture
|
||||
|
||||
Use `extract_requirements(task_description, ...)` only when the user provides a reasonably complete description.
|
||||
|
||||
Use **guided capture** when:
|
||||
- the user is doing an ad-hoc search (no DB task), AND
|
||||
- the description is vague/too short, or key fields are missing.
|
||||
|
||||
Rule of thumb:
|
||||
- If you would still need to ask **more than one** follow-up question, start guided capture.
|
||||
|
||||
### Ad-hoc searches with incomplete descriptions (must-do)
|
||||
|
||||
If the user is looking for a capacity **not based on an existing DB task** and does **not** provide a complete task description (e.g. only a role, only a few skills, or a very short/vague sentence), you MUST start **guided capture**.
|
||||
|
||||
- Trigger condition (ad-hoc + incomplete description):
|
||||
- no `task_id` / not a DB-task workflow, AND
|
||||
- the user did not provide a sufficiently detailed task description.
|
||||
|
||||
**Definition (operational):** treat the description as *incomplete* unless it contains at least:
|
||||
|
||||
- a short **scope/goal** (what will be built/done), AND
|
||||
- at least **2 concrete competences/technologies** (or one competence + clear domain), AND
|
||||
- at least a rough **time range** (start/end or open-ended)
|
||||
|
||||
If these are not present, do not call `extract_requirements(...)`. Start guided capture instead.
|
||||
|
||||
Then do:
|
||||
|
||||
1. `start_guided_capture()`
|
||||
2. `guided_set_description(...)` (use the user's current text as initial description; ask for missing context)
|
||||
3. `guided_set_role(...)`
|
||||
4. `guided_set_time_range(date_start?, date_end?)`
|
||||
5. `guided_set_competences([...])`
|
||||
6. `show_pending_requirements()`
|
||||
7. Ask user for confirmation
|
||||
8. `confirm_requirements(confirm=true)` (if required)
|
||||
9. `find_matching_capacities(...)`
|
||||
|
||||
### Minimum follow-up questions when the user is underspecified
|
||||
|
||||
If the user provides only a vague request or a single skill (e.g. “Ich suche jemanden mit JavaScript.”), you MUST ask (at minimum) these three questions before matching:
|
||||
|
||||
1. **Role:** Welche Rolle soll die Person haben (z.B. Frontend Developer, Fullstack, QA, Tech Lead)?
|
||||
2. **Time range:** Ab wann und bis wann wird die Person benötigt? (Start/Ende oder open-ended)
|
||||
3. **More competences:** Welche weiteren wichtigen Skills sind relevant? (z.B. TypeScript, React/Angular/Vue, Node.js, Testing)
|
||||
|
||||
If the user refuses to specify a role, set `role_name="Beliebige Rolle"` explicitly and continue.
|
||||
|
||||
## Search context & Cherry Studio reliability (critical)
|
||||
|
||||
These rules exist because some clients occasionally reuse a wrong or stale UUID and/or fail to render full tool output.
|
||||
|
||||
### Always track the latest search id
|
||||
|
||||
Maintain an explicit local variable: `LATEST_SEARCH_ID`.
|
||||
|
||||
- Update `LATEST_SEARCH_ID` **only** from the **most recent tool output** you executed.
|
||||
- Accept only IDs explicitly labeled as:
|
||||
- `Using SEARCH_ID=<uuid>` (preferred), or
|
||||
- `SEARCH_ID=<uuid>`, or
|
||||
- `META={..."search_id": "..."...}`
|
||||
- Never invent or "recover" a search id.
|
||||
- Never accept a user-provided `search_id` unless it exactly equals `LATEST_SEARCH_ID`.
|
||||
|
||||
### Echo rule (Cherry Studio rendering workaround)
|
||||
|
||||
After every **successful** call to:
|
||||
- `find_matching_capacities`: include in your **next** assistant message
|
||||
1) `Using SEARCH_ID=<uuid>` (copied exactly), and
|
||||
2) the `## Summary` table with counts for Top/Good/Partial/Low.
|
||||
- `filter_search_results`: include in your **next** assistant message
|
||||
1) `Using SEARCH_ID=<uuid>`, and
|
||||
2) the `filter_id` plus the **Applied Filters** table.
|
||||
- `get_results_by_category`: include in your **next** assistant message
|
||||
1) `Using SEARCH_ID=<uuid>`, and
|
||||
2) the `Category`, `Page`, and `Total items in category` lines.
|
||||
|
||||
Keep these echoes short and verbatim. Do not paraphrase IDs.
|
||||
|
||||
### Before calling refinement/pagination tools
|
||||
|
||||
Immediately before calling `get_results_by_category(...)` or `filter_search_results(...)`, write:
|
||||
|
||||
- `Using SEARCH_ID=<LATEST_SEARCH_ID>`
|
||||
|
||||
Then call the tool with that exact value.
|
||||
|
||||
### Hard failover on invalid/expired search
|
||||
|
||||
If `get_results_by_category` or `filter_search_results` returns:
|
||||
- `status=unknown_or_expired`, or
|
||||
- `META` contains `"status": "unknown_or_expired"`, or
|
||||
- text indicating invalid/expired search id
|
||||
|
||||
Then do **not** retry with another guessed id. Rerun `find_matching_capacities(...)` to create a fresh search and replace `LATEST_SEARCH_ID`.
|
||||
|
||||
### Stable behavior when role is unspecified
|
||||
|
||||
If the user did not specify a role, always send `role_name="Beliebige Rolle"` (never an empty string) and keep using it for retries.
|
||||
|
||||
## Tools and when to use them
|
||||
|
||||
### Discovery (DB tasks)
|
||||
- `list_open_tasks(limit=...)`
|
||||
- Use when the user wants to browse tasks.
|
||||
- Output is a Markdown table with `task_id`.
|
||||
|
||||
- `get_task_details(task_id)`
|
||||
- Use to inspect a task and see relevant fields.
|
||||
|
||||
- `validate_task_requirements(task_id)`
|
||||
- Use to compare DB skills vs inferred skills and time ranges.
|
||||
- This tool is not part of the default matching workflow and must only be used when the user explicitly asks for validation.
|
||||
|
||||
- `infer_primary_role(task_id=... | task_text=...)`
|
||||
- Use to suggest the single closest role for a task.
|
||||
|
||||
### Capacity browsing + capacity→task matching
|
||||
- `list_free_capacities(limit=...)`
|
||||
- Use when the user wants to browse recent free capacities.
|
||||
|
||||
- `get_capacity_details(capacity_id)`
|
||||
- Use to inspect a capacity before matching.
|
||||
|
||||
- `find_matching_tasks(capacity_id)`
|
||||
- Use to find tasks that match a specific capacity.
|
||||
|
||||
### Requirement capture (free text or structured)
|
||||
- `extract_requirements(task_description, confirm_requirements=true)`
|
||||
- Uses embeddings-only inference.
|
||||
- Dates are NOT extracted from free text. Ask for a time range and capture via structured/guided tools.
|
||||
|
||||
- `collect_structured_requirement_data(role_name, competences, date_start?, date_end?, confirm_requirements=true)`
|
||||
- Use when the user provides explicit fields.
|
||||
|
||||
- Guided capture tools (step-by-step):
|
||||
- `start_guided_capture()`
|
||||
- `guided_set_description(description)`
|
||||
- `guided_set_role(role_name)`
|
||||
- `guided_set_time_range(date_start?, date_end?)` (open-ended allowed)
|
||||
- `guided_set_competences(competences)`
|
||||
|
||||
- `update_requirements(change_description, confirm_requirements=true)`
|
||||
- Use to modify already-captured requirements (e.g. add a competence, change the role).
|
||||
- Accepts a free-text change description; the server applies the delta.
|
||||
|
||||
- `show_pending_requirements()`
|
||||
- Use to display pending requirements for user review.
|
||||
|
||||
- `confirm_requirements(confirm=true)`
|
||||
- Use to confirm pending requirements before searching if required.
|
||||
|
||||
### Search execution
|
||||
- `find_matching_capacities(role_name, competences, date_start?, date_end?)`
|
||||
- Runs matching and returns a JSON block with `search_id`.
|
||||
- Treat the returned `search_id` as the new **latest** search.
|
||||
|
||||
### Search refinement and pagination (always reuse the latest search)
|
||||
- `get_results_by_category(search_id, filter_id?, category, page, page_size)`
|
||||
- Use to display `Top`, `Good`, `Partial`, `Low`, `Irrelevant` results.
|
||||
- Numeric score columns are always visible (Role/Competence/Overall).
|
||||
- Availability is shown as overlap percentage with the reference time range.
|
||||
|
||||
- `filter_search_results(search_id, ...)`
|
||||
- Use for refining results for both directions (capacity-search and task-search).
|
||||
- Supports full-coverage availability filtering (`is_fully_available=true`).
|
||||
- Task-search filters include `task_text_filter` and `task_competence_filter`.
|
||||
|
||||
## Default workflows
|
||||
|
||||
### Workflow A: User has a DB task id
|
||||
1. `get_task_details(task_id)`
|
||||
2. (Only if the user explicitly requests validation) `validate_task_requirements(task_id)`
|
||||
3. If role unclear, `infer_primary_role(task_id=...)`
|
||||
4. Capture/update requirements (pick ONE approach):
|
||||
- `collect_structured_requirement_data(...)`, or
|
||||
- `extract_requirements(task_description=...)`
|
||||
5. **Review (must be after the last requirements update):** `show_pending_requirements()` (only if `matching.require_confirmation = true`)
|
||||
6. Ask the user explicitly to confirm (Yes/No) (only if `matching.require_confirmation = true`)
|
||||
7. `confirm_requirements(confirm=true)` (only on Yes; on No: `confirm_requirements(confirm=false)`) (only if `matching.require_confirmation = true`)
|
||||
8. Run matching:
|
||||
- `find_matching_capacities(role_name, competences, date_start?, date_end?)`
|
||||
9. After matching, show other categories via `get_results_by_category` using `LATEST_SEARCH_ID`.
|
||||
|
||||
### Workflow B: User provides free-text description
|
||||
1. `extract_requirements(task_description, confirm_requirements=true)`
|
||||
2. Ask for missing fields, especially date range (dates are not extracted).
|
||||
3. If `matching.require_confirmation = true`: `show_pending_requirements()`
|
||||
4. If `matching.require_confirmation = true`: ask user for confirmation
|
||||
5. If `matching.require_confirmation = true`: `confirm_requirements(confirm=true)`
|
||||
6. `find_matching_capacities(role_name, competences, date_start?, date_end?)`
|
||||
7. For other categories/pages: `get_results_by_category` using `LATEST_SEARCH_ID`.
|
||||
|
||||
### Workflow C: Capacity-driven matching
|
||||
1. `list_free_capacities(limit=...)`
|
||||
2. `get_capacity_details(capacity_id)`
|
||||
3. `find_matching_tasks(capacity_id)`
|
||||
|
||||
## Handling “show me Good results”
|
||||
- If you have `LATEST_SEARCH_ID`: call `get_results_by_category(search_id=<LATEST_SEARCH_ID>, category="Good", page=1, page_size=20)`.
|
||||
- If you do not have `LATEST_SEARCH_ID`: run a new `find_matching_capacities(...)` first.
|
||||
|
||||
## Embeddings: performance + determinism (important)
|
||||
|
||||
The server uses Azure OpenAI embeddings for competence and role similarity.
|
||||
|
||||
- Embeddings are **prefetched globally per matching run** (required + all
|
||||
candidates; competences + roles). Similarity computation should not trigger
|
||||
embeddings inside inner loops.
|
||||
- Cache layering applies:
|
||||
1) in-memory cache
|
||||
2) SQLite embedding cache
|
||||
3) Azure embeddings API
|
||||
- Azure embeddings requests may be **batched** (`input=[...]`) and chunked
|
||||
sequentially. Chunk size is configurable via
|
||||
`azure_openai.embedding_batch_size` (default: 128).
|
||||
- Failure semantics remain strict: if embeddings fail, the matching run fails
|
||||
(no heuristic fallback).
|
||||
|
||||
## BM25 + RRF competence matching (optional, disabled by default)
|
||||
|
||||
When `matching.similarity.use_bm25_search = true` is set in `config.toml`, competence
|
||||
similarity uses **BM25 + Reciprocal Rank Fusion (RRF)** instead of embeddings:
|
||||
|
||||
- **Score interpretation changes**: a `Competence Score` of `0.0` means
|
||||
**no lexical token overlap** between the required competence and any candidate
|
||||
competence — it is **not** a semantic distance. Cross-language synonyms or
|
||||
abbreviations that do not share tokens will score `0.0`.
|
||||
- **Exact canonical terms matter**: "Python" matches "Python" perfectly; "ML" does
|
||||
**not** match "Machine Learning" unless Auto-Tagging is enabled.
|
||||
- When displaying results to the user, note that BM25 mode eliminates false positives
|
||||
(non-overlapping skills score exactly 0.0) but may miss valid synonym matches.
|
||||
|
||||
### Auto-Tagging (`use_auto_tagging = true`)
|
||||
|
||||
When `matching.similarity.use_auto_tagging = true` is also set:
|
||||
|
||||
- An LLM pre-expands each candidate's competence list with canonical equivalents of
|
||||
required competences it already covers (synonyms, abbreviations, cross-language terms,
|
||||
e.g. "ML" → "Machine Learning").
|
||||
- The expansion is **ephemeral** (per-call, not stored in the database).
|
||||
- A positive `Competence Score` with auto-tagging enabled may reflect an
|
||||
**LLM-inferred equivalence**, not a literal token match; the rationale field will still
|
||||
say `"BM25+RRF: best match '...' (score ...)"`.
|
||||
- If the LLM call fails for any reason, BM25 scoring continues on the original
|
||||
(unexpanded) candidate list — the run does **not** fail.
|
||||
- Role similarity is always **embedding-based**, regardless of these flags.
|
||||
@@ -1,82 +0,0 @@
|
||||
# Azure OpenAI setup
|
||||
|
||||
This project requires Azure OpenAI for **embeddings only**:
|
||||
|
||||
- **Embeddings** (`text-embedding-3-large`, **3072 dims**):
|
||||
- role inference (against Data Lake role vocabulary)
|
||||
- competence inference (against Data Lake competence vocabulary)
|
||||
- similarity scoring for matching
|
||||
|
||||
There are **no chat/LLM features** anymore.
|
||||
|
||||
## 1) Create an embedding deployment in Azure
|
||||
|
||||
In your Azure OpenAI resource, create an embedding deployment for:
|
||||
|
||||
- `text-embedding-3-large`
|
||||
|
||||
Notes:
|
||||
|
||||
- The deployment **name** must match what you set in `config.toml`.
|
||||
- This project uses 3072 embedding dimensions.
|
||||
|
||||
## 2) Configure `config.toml`
|
||||
|
||||
Copy the template:
|
||||
|
||||
- `cp config.toml.example config.toml`
|
||||
|
||||
Set the Azure section:
|
||||
|
||||
```toml
|
||||
[azure_openai]
|
||||
endpoint = "https://<your-resource>.openai.azure.com"
|
||||
api_version = "2024-02-15-preview"
|
||||
embedding_deployment = "text-embedding-3-large"
|
||||
|
||||
# Embeddings batching (number of inputs per embeddings API request)
|
||||
embedding_batch_size = 128
|
||||
|
||||
# Optional
|
||||
show_costs_in_output = false
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- Embeddings requests may be batched (`input=[...]`) and chunked sequentially.
|
||||
Increase `embedding_batch_size` to reduce round-trips, or decrease it if you
|
||||
suspect request-size related failures.
|
||||
|
||||
## 3) Configure credentials via environment variables
|
||||
|
||||
The server reads the API key from:
|
||||
|
||||
- `AZURE_OPENAI_EMBEDDING_API_KEY`
|
||||
|
||||
Recommended: keep secrets in a local `.env` file (do not commit):
|
||||
|
||||
```bash
|
||||
AZURE_OPENAI_EMBEDDING_API_KEY="..."
|
||||
```
|
||||
|
||||
The server also needs database credentials:
|
||||
|
||||
- `DATA_LAKE_USERNAME`
|
||||
- `DATA_LAKE_PASSWORD`
|
||||
|
||||
## 4) Verify your setup
|
||||
|
||||
### 4.1 Quick import/run smoke check
|
||||
|
||||
Start the server with debug logs and confirm it loads your config:
|
||||
|
||||
- `uv run teamlandkarte-mcp --config config.toml --log-level DEBUG`
|
||||
|
||||
### 4.2 Typical Azure errors
|
||||
|
||||
- HTTP 401/403: wrong key or wrong resource
|
||||
- HTTP 404: deployment name mismatch
|
||||
- HTTP 429: throttling; warm the embedding cache and/or reduce concurrency
|
||||
- network/TLS: verify corporate proxy/TLS setup
|
||||
|
||||
See `docs/troubleshooting.md` for additional details.
|
||||
@@ -1,19 +0,0 @@
|
||||
# Security notes
|
||||
|
||||
## Credentials
|
||||
|
||||
- `config.toml` is intentionally ignored via `.gitignore`.
|
||||
- If `config.toml` (or any file containing credentials) was ever committed to git history, treat those credentials as compromised.
|
||||
|
||||
## Credential rotation checklist
|
||||
|
||||
1. Rotate the database password / token.
|
||||
2. Invalidate any exposed API tokens.
|
||||
3. Re-issue credentials with least-privilege read-only access.
|
||||
4. Audit logs for suspicious access.
|
||||
|
||||
## Git history cleanup (if needed)
|
||||
|
||||
If a secret accidentally entered git history, remove it from history using secure tools (for example `git filter-repo`) and force-push.
|
||||
|
||||
Do **not** rely on a simple revert; secrets remain accessible in history.
|
||||
@@ -1,314 +0,0 @@
|
||||
# Semantic Similarity & False Positives
|
||||
|
||||
## Problem
|
||||
|
||||
When searching for **Python** and **Machine Learning**, a candidate with only **JavaScript, TypeScript, Node.js, Vue.js, PWA, CI/CD, Web Architectures** receives a score of **0.392** (39%).
|
||||
|
||||
This seems too high for someone who has **none of the searched competences**.
|
||||
|
||||
## Root Cause: Semantic Embeddings
|
||||
|
||||
The `per_skill` strategy works as follows:
|
||||
|
||||
1. For each **required** competence, find the **best match** from candidate competences
|
||||
2. Average those best-match scores
|
||||
|
||||
The Azure OpenAI embedding model (`text-embedding-3-large`) finds **semantic similarity** between concepts, not just exact matches:
|
||||
|
||||
### Example Similarities
|
||||
|
||||
| Required | Best Candidate Match | Why Similar? | Approx. Score |
|
||||
|----------|---------------------|--------------|---------------|
|
||||
| Python | JavaScript / TypeScript / Node.js | All programming languages | ~0.50-0.60 |
|
||||
| Machine Learning | Progressive Web App / general software dev | Broader technical domain | ~0.20-0.30 |
|
||||
|
||||
**Average:** `(0.55 + 0.25) / 2 = 0.40` → explains the 0.392 score!
|
||||
|
||||
## Why This Happens
|
||||
|
||||
Embedding models are trained to understand:
|
||||
- **Python** and **JavaScript** are both programming languages
|
||||
- They share many concepts (variables, functions, loops, OOP)
|
||||
- Semantically, they're closer to each other than to "accounting" or "project management"
|
||||
|
||||
This is **by design** - embeddings capture semantic relationships, not just exact string matches.
|
||||
|
||||
## Solution Options
|
||||
|
||||
### Option 1: Raise the `partial` Threshold ⭐ RECOMMENDED
|
||||
|
||||
**Change in `config.toml`:**
|
||||
```toml
|
||||
[matching.thresholds]
|
||||
top = 0.8
|
||||
good = 0.65
|
||||
partial = 0.5 # ← was 0.4, now 0.5
|
||||
low = 0.3 # ← was 0.2, now 0.3
|
||||
```
|
||||
|
||||
**Effect:**
|
||||
- Score 0.392 → category "Low" (instead of "Partial")
|
||||
- Clearer distinction between "somewhat related" and "actually matching"
|
||||
- More balanced category distribution
|
||||
|
||||
**Pros:**
|
||||
- Simple one-line config change
|
||||
- Makes categories more meaningful
|
||||
- Doesn't affect matching, only categorization
|
||||
|
||||
**Cons:**
|
||||
- Might push some legitimate partial matches into "Low"
|
||||
|
||||
---
|
||||
|
||||
### Option 2: Stricter Competence Inference
|
||||
|
||||
**Change in `config.toml`:**
|
||||
```toml
|
||||
[matching.inference]
|
||||
max_competences = 16
|
||||
min_similarity = 0.6 # ← was 0.4, raise to 0.6 or 0.7
|
||||
```
|
||||
|
||||
**Effect:**
|
||||
- When inferring competences from descriptions, only closer semantic matches count
|
||||
- Doesn't affect manual competence lists
|
||||
|
||||
**Pros:**
|
||||
- Reduces noise in inferred competences
|
||||
- More precise matching
|
||||
|
||||
**Cons:**
|
||||
- Only affects inference, not manual searches
|
||||
- Might miss some valid related competences
|
||||
|
||||
---
|
||||
|
||||
### Option 3: More Specific Competences
|
||||
|
||||
**Instead of:**
|
||||
```
|
||||
Required: ["Python", "Machine Learning"]
|
||||
```
|
||||
|
||||
**Use:**
|
||||
```
|
||||
Required: ["Python", "scikit-learn", "TensorFlow", "PyTorch", "pandas", "NumPy"]
|
||||
```
|
||||
|
||||
**Effect:**
|
||||
- More specific competences are harder to match accidentally
|
||||
- "scikit-learn" is semantically very different from "Vue.js"
|
||||
|
||||
**Pros:**
|
||||
- More precise matching
|
||||
- Better reflects actual skill requirements
|
||||
|
||||
**Cons:**
|
||||
- Requires more detailed competence lists
|
||||
- Might miss candidates who have Python but use different ML libraries
|
||||
|
||||
---
|
||||
|
||||
### Option 4: Switch to `aggregate` Strategy
|
||||
|
||||
**Change in `config.toml`:**
|
||||
```toml
|
||||
[matching.similarity]
|
||||
strategy = "aggregate" # ← was "per_skill"
|
||||
```
|
||||
|
||||
**How it works:**
|
||||
- Computes **mean embedding** of all required competences
|
||||
- Computes **mean embedding** of all candidate competences
|
||||
- Returns cosine similarity between the two means
|
||||
|
||||
**Effect with your example:**
|
||||
- Required mean: average of [Python, Machine Learning]
|
||||
- Candidate mean: average of [JavaScript, TypeScript, Node.js, Vue.js, PWA, CI/CD, Web Arch]
|
||||
- The diluted candidate mean has **lower similarity** to Python/ML than per-skill
|
||||
|
||||
**Pros:**
|
||||
- Better for "overall technical fit" vs "specific skill coverage"
|
||||
- Reduces false positives from cross-domain matches
|
||||
|
||||
**Cons:**
|
||||
- Extra candidate competences dilute the score (see SCORE_BUG_FIX.md)
|
||||
- All required competences get the same score
|
||||
- Less granular than per-skill
|
||||
|
||||
---
|
||||
|
||||
### Option 5: Filter by Matched Competences
|
||||
|
||||
**Use MCP tools:**
|
||||
```python
|
||||
# After search, filter to only show results where specific competences matched
|
||||
filter_search_results(
|
||||
search_id="...",
|
||||
filter_id="python-ml-only",
|
||||
min_similarity=0.65, # High threshold for "matched"
|
||||
)
|
||||
```
|
||||
|
||||
**Effect:**
|
||||
- Only shows results where similarity >= 0.65 for Python or Machine Learning
|
||||
- Filters out the JavaScript/TypeScript candidates
|
||||
|
||||
**Pros:**
|
||||
- Most precise control
|
||||
- Can experiment with different thresholds interactively
|
||||
|
||||
**Cons:**
|
||||
- Requires post-search filtering
|
||||
- Adds an extra step to workflow
|
||||
|
||||
---
|
||||
|
||||
## Recommended Approach
|
||||
|
||||
**Combination of Option 1 + Option 5:**
|
||||
|
||||
1. **Raise `partial` threshold to 0.5:**
|
||||
```toml
|
||||
[matching.thresholds]
|
||||
partial = 0.5
|
||||
```
|
||||
|
||||
2. **Use interactive filtering when needed:**
|
||||
- For broad exploration: accept lower scores
|
||||
- For precise matching: apply `filter_search_results()` with higher thresholds
|
||||
|
||||
This gives you:
|
||||
- ✅ Clearer category boundaries (0.392 → Low, not Partial)
|
||||
- ✅ Flexibility to tighten results when needed
|
||||
- ✅ No loss of recall (candidates still appear, just in correct category)
|
||||
|
||||
---
|
||||
|
||||
### Option 6: Switch to BM25 + RRF (lexical matching) ⭐ BEST for exact skill names
|
||||
|
||||
**Change in `config.toml`:**
|
||||
```toml
|
||||
[matching.similarity]
|
||||
use_bm25_search = true
|
||||
```
|
||||
|
||||
**How it works:**
|
||||
- For each required competence, ranks candidate competences using BM25 (term-frequency/
|
||||
inverse-document-frequency lexical scoring) and normalizes via Reciprocal Rank Fusion.
|
||||
- Candidates with **no shared token** with the required competence receive score **0.0**
|
||||
— the false-positive problem is completely eliminated.
|
||||
|
||||
**Effect with your example:**
|
||||
- Required: `["Python", "Machine Learning"]`
|
||||
- Candidate: `["JavaScript", "TypeScript", "Node.js", "Vue.js"]`
|
||||
- Score: **0.0** (no token overlap at all)
|
||||
|
||||
**Pros:**
|
||||
- ✅ Eliminates false positives for lexically disjoint skills
|
||||
- ✅ No Azure embedding calls for competence matching (faster, lower cost)
|
||||
- ✅ Deterministic — same query always returns same ranked order
|
||||
|
||||
**Cons:**
|
||||
- ❌ Misses synonyms and cross-language pairs:
|
||||
- "ML" ≠ "Machine Learning" (no token overlap)
|
||||
- "Softwarearchitektur" ≠ "Software Architecture"
|
||||
- "Data Science" ≠ "Machine Learning"
|
||||
- ❌ Role similarity still uses embeddings (unchanged)
|
||||
|
||||
**Mitigation:** combine with `use_auto_tagging = true` (see Option 7 below).
|
||||
|
||||
---
|
||||
|
||||
### Option 7: BM25 + RRF + Auto-Tagging ⭐ BEST for mixed-language / abbreviation-heavy datasets
|
||||
|
||||
**Change in `config.toml`:**
|
||||
```toml
|
||||
[matching.similarity]
|
||||
use_bm25_search = true
|
||||
use_auto_tagging = true
|
||||
|
||||
[azure_openai]
|
||||
chat_deployment = "gpt-4.1" # your chat deployment name
|
||||
```
|
||||
|
||||
**And set the env variable:**
|
||||
```
|
||||
AZURE_OPENAI_LLM_API_KEY=<your-chat-api-key>
|
||||
```
|
||||
|
||||
**How it works:**
|
||||
- Before BM25 scoring, the LLM is asked: "which of these *required* competences are
|
||||
already covered by the candidate's *existing* competences (via synonym, abbreviation,
|
||||
or cross-language equivalence)?"
|
||||
- The canonical required-competence names identified by the LLM are appended to the
|
||||
candidate's working list for this BM25 call only (result is **never persisted**).
|
||||
- BM25 then finds token matches including the LLM-expanded terms.
|
||||
|
||||
**Effect:**
|
||||
- "ML" → LLM identifies "Machine Learning" as covered → added → BM25 matches ✅
|
||||
- "CI" → LLM identifies "CI/CD" as covered → BM25 matches ✅
|
||||
- "Softwarearchitektur" → LLM identifies "Software Architecture" → BM25 matches ✅
|
||||
|
||||
**Pros:**
|
||||
- ✅ Eliminates false positives (BM25 zero-out rule)
|
||||
- ✅ Bridges synonyms, abbreviations, and cross-language pairs (LLM)
|
||||
- ✅ Expansion is ephemeral — no database pollution
|
||||
|
||||
**Cons:**
|
||||
- ❌ One additional LLM call per candidate per matching run (latency + cost)
|
||||
- ❌ Requires `AZURE_OPENAI_LLM_API_KEY` environment variable
|
||||
|
||||
---
|
||||
|
||||
## Strategy Comparison
|
||||
|
||||
| Strategy | False positives | Synonyms/Cross-language | Cost | Speed |
|
||||
|----------|----------------|------------------------|------|-------|
|
||||
| `per_skill` (default) | ⚠️ Moderate | ✅ Good | Azure embeddings | Fast (cached) |
|
||||
| `aggregate` | ⚠️ Moderate | ✅ Good | Azure embeddings | Fast (cached) |
|
||||
| `use_bm25_search` | ✅ Eliminated | ❌ Misses | No embedding calls | Very fast |
|
||||
| `use_bm25_search` + `use_auto_tagging` | ✅ Eliminated | ✅ Good | + 1 LLM call/candidate | Moderate |
|
||||
|
||||
---
|
||||
|
||||
## Technical Background
|
||||
|
||||
### Why Embeddings Find Semantic Similarity
|
||||
|
||||
The `text-embedding-3-large` model is trained on massive amounts of text including:
|
||||
- Programming language documentation
|
||||
- Technical tutorials and courses
|
||||
- Stack Overflow discussions
|
||||
- GitHub repositories
|
||||
|
||||
It learns that:
|
||||
- **Python** and **JavaScript** both appear in programming contexts
|
||||
- They share similar syntax concepts
|
||||
- Developers often know both
|
||||
- They're used for similar tasks (web development, data processing)
|
||||
|
||||
This makes the embeddings **semantically aware**, which is powerful for fuzzy matching but can cause false positives when you need exact skill matches.
|
||||
|
||||
### Per-Skill vs Aggregate
|
||||
|
||||
**Per-Skill:**
|
||||
- Finds best match for each required competence
|
||||
- Good for: "Must have Python AND Machine Learning"
|
||||
- Risk: Cross-domain false positives (Python ↔ JavaScript)
|
||||
|
||||
**Aggregate:**
|
||||
- Computes overall semantic fit
|
||||
- Good for: "General technical background in this domain"
|
||||
- Risk: Diluted by extra competences
|
||||
|
||||
Choose based on your use case!
|
||||
|
||||
---
|
||||
|
||||
## See Also
|
||||
|
||||
- `SCORE_BUG_FIX.md` - Detailed test coverage for scoring behavior
|
||||
- `docs/architecture.md` - Similarity strategy documentation
|
||||
- `config.toml.example` - Configuration options
|
||||
@@ -1,161 +0,0 @@
|
||||
---
|
||||
name: test-agent
|
||||
description: >
|
||||
Specialized agent for developing and maintaining the Teamlandkarte MCP Server —
|
||||
a capacity/task matching system for DB Systel employees. Use this agent when working
|
||||
on the Teamlandkarte codebase: adding features, fixing bugs, writing tests, refactoring,
|
||||
or understanding the matching/capacity system.
|
||||
tools: ["read", "write", "shell", "web"]
|
||||
---
|
||||
|
||||
You are a specialized development assistant for the **Teamlandkarte MCP Server** project.
|
||||
This is a Python 3.13+ MCP (Model Context Protocol) server that enables AI assistants to match
|
||||
DB Systel employees with free work capacity to task requirements, querying the DB Systel Open Data Lake.
|
||||
|
||||
## Project Overview
|
||||
|
||||
- **Language**: Python 3.13+
|
||||
- **Package manager**: uv
|
||||
- **Framework**: MCP (Model Context Protocol)
|
||||
- **Database**: Trino/Presto (read-only access to DB Systel Open Data Lake)
|
||||
- **AI**: Azure OpenAI embeddings for role/competence inference
|
||||
- **Testing**: pytest + pytest-asyncio
|
||||
- **Linting**: ruff (line-length 110)
|
||||
- **Type checking**: mypy
|
||||
- **Configuration**: TOML format (config.toml), credentials via .env
|
||||
- **Source layout**: src/teamlandkarte_mcp/
|
||||
- **Entry point**: `teamlandkarte-mcp` command (src/teamlandkarte_mcp/__main__.py)
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
src/teamlandkarte_mcp/
|
||||
├── __main__.py # CLI entry point
|
||||
├── mcp_server.py # MCP server + tool definitions
|
||||
├── config.py # Configuration management (TOML)
|
||||
├── models.py # Data models
|
||||
├── logging_config.py # Logging setup
|
||||
├── azure/ # Azure OpenAI integration (embeddings)
|
||||
├── database/ # Trino client, schema verification, read-only guard
|
||||
├── matching/ # Matcher, scorer, similarity engine, BM25+RRF
|
||||
├── cache/ # Query cache, search cache, embedding cache (SQLite)
|
||||
└── utils/ # Shared utilities
|
||||
```
|
||||
|
||||
## Commands
|
||||
|
||||
- **Run server**: `uv run teamlandkarte-mcp --config config.toml`
|
||||
- **Run tests**: `uv run pytest`
|
||||
- **Lint**: `uv run ruff check src/ tests/`
|
||||
- **Format**: `uv run ruff format src/ tests/`
|
||||
- **Type check**: `uv run mypy src/`
|
||||
- **Install deps**: `uv sync`
|
||||
|
||||
## Architectural Patterns
|
||||
|
||||
### Embeddings-Only Inference
|
||||
The system uses Azure OpenAI embeddings (text-embedding-3-large, 3072 dimensions) for semantic
|
||||
similarity. There is no chat/LLM by default — matching is purely embedding-based unless
|
||||
`use_auto_tagging = true` is configured.
|
||||
|
||||
### Table-First Markdown Output
|
||||
All MCP tool outputs use Markdown tables as the primary format for structured data. This ensures
|
||||
clean rendering in chat UIs. Always maintain this pattern when adding or modifying tool outputs.
|
||||
|
||||
### Confirmation Gate
|
||||
Before any matching run executes, the system enforces a hard confirmation gate:
|
||||
1. `show_pending_requirements` → displays what will be matched
|
||||
2. `request_requirements_confirmation` → asks user to confirm
|
||||
3. `confirm_requirements` → user explicitly confirms
|
||||
Only after confirmation does matching proceed. Never bypass this pattern.
|
||||
|
||||
### Search Sessions
|
||||
Search results are managed via `search_id` and `filter_id` identifiers. Results can be
|
||||
filtered and paginated after the initial search. Maintain session state correctly.
|
||||
|
||||
### Multi-Level Caching
|
||||
- **DB cache**: 12h TTL for Trino query results (cachetools)
|
||||
- **Search cache**: 60min TTL for search results
|
||||
- **Embedding cache**: Persistent SQLite with configurable TTL (default 30 days)
|
||||
|
||||
### Read-Only Database Access
|
||||
The Trino connection is strictly read-only. Schema verification runs at startup (fail-fast).
|
||||
Never write to the database. Never expose connection credentials in outputs.
|
||||
|
||||
## MCP Tools Categories
|
||||
|
||||
- **Discovery**: list_open_tasks, get_task_details, validate_task_requirements, infer_primary_role
|
||||
- **Requirement Capture**: extract_requirements, collect_structured_requirement_data, guided capture flow
|
||||
- **Confirmation Gate**: show_pending_requirements, request_requirements_confirmation, confirm_requirements
|
||||
- **Matching**: find_matching_capacities, find_matching_tasks
|
||||
- **Exploration**: filter_search_results, get_results_by_category
|
||||
- **Capacity Browsing**: list_free_capacities, get_capacity_details
|
||||
|
||||
When designing new MCP tools:
|
||||
- Produce deterministic outputs (same input → same output, modulo cache state)
|
||||
- Use Markdown tables for structured data
|
||||
- Define clear parameter schemas with descriptions
|
||||
- Follow the existing naming convention (verb_noun pattern)
|
||||
- Include proper error messages for invalid inputs
|
||||
|
||||
## Security Rules
|
||||
|
||||
1. **Credentials in env vars only** — never in config.toml, never in code, never in outputs
|
||||
2. **Read-only DB** — never attempt writes to Trino
|
||||
3. **Never expose secrets** — API keys, passwords, tokens must never appear in tool outputs or logs
|
||||
4. **.env file** — contains DATA_LAKE_USERNAME, DATA_LAKE_PASSWORD, AZURE_OPENAI_EMBEDDING_API_KEY, AZURE_OPENAI_LLM_API_KEY
|
||||
5. **config.toml** — not in VCS, contains endpoint URLs and tuning parameters only
|
||||
|
||||
## Configuration
|
||||
|
||||
Runtime configuration lives in `config.toml` (TOML format). See `config.toml.example` for the
|
||||
full schema. Key sections:
|
||||
- `[database]` — Trino connection (host, port, catalog, schema)
|
||||
- `[matching]` — weights, thresholds, fuzzy settings, inference, similarity strategy
|
||||
- `[cache]` — TTL settings
|
||||
- `[embedding_cache]` — SQLite path and TTL
|
||||
- `[azure_openai]` — endpoint, deployment, API version, batch size
|
||||
|
||||
## Testing Patterns
|
||||
|
||||
- Use **pytest** with **pytest-asyncio** for async tests
|
||||
- Focus areas: confirmation gating, search robustness, filtering/pagination, cache behavior
|
||||
- Mock external services (Trino, Azure OpenAI) in unit tests
|
||||
- Test edge cases: empty results, invalid inputs, cache expiry, schema mismatches
|
||||
- Run tests with: `uv run pytest` (or `uv run pytest -v` for verbose)
|
||||
- Test files live in `tests/` directory
|
||||
|
||||
## OpenSpec Workflow
|
||||
|
||||
This project uses **OpenSpec** for spec-driven development. For significant changes
|
||||
(new capabilities, breaking changes, architecture shifts):
|
||||
|
||||
1. Check existing specs: `openspec spec list --long`
|
||||
2. Check active changes: `openspec list`
|
||||
3. Create a proposal in `openspec/changes/<change-id>/`
|
||||
4. Include: proposal.md, tasks.md, optional design.md, and spec deltas
|
||||
5. Validate: `openspec validate <change-id> --strict`
|
||||
6. Get approval before implementing
|
||||
|
||||
Read `openspec/AGENTS.md` for full workflow details.
|
||||
|
||||
## Code Style
|
||||
|
||||
- Line length: 110 characters (ruff configured)
|
||||
- Use type hints everywhere (mypy strict)
|
||||
- Follow existing patterns in the codebase
|
||||
- Prefer `async/await` for I/O operations
|
||||
- Use dataclasses or Pydantic models for structured data
|
||||
- Keep functions focused and testable
|
||||
- Document public APIs with docstrings
|
||||
|
||||
## When Working on This Project
|
||||
|
||||
1. Read relevant existing code before making changes
|
||||
2. Match existing patterns and conventions
|
||||
3. Run `uv run ruff check src/ tests/` after changes
|
||||
4. Run `uv run mypy src/` for type safety
|
||||
5. Run `uv run pytest` to verify nothing breaks
|
||||
6. For significant changes, follow the OpenSpec workflow
|
||||
7. Never commit credentials or secrets
|
||||
8. Keep Markdown table outputs clean and consistent
|
||||
@@ -1,227 +0,0 @@
|
||||
# Troubleshooting
|
||||
|
||||
## The server fails at startup with `Missing config.toml`
|
||||
|
||||
- Copy the template and fill in non-secret settings:
|
||||
- `cp config.toml.example config.toml`
|
||||
- Ensure `--config` points to the correct file when starting the server.
|
||||
|
||||
## Database connectivity problems
|
||||
|
||||
Symptoms:
|
||||
|
||||
- `DatabaseError: Trino error: ...`
|
||||
|
||||
Checklist:
|
||||
|
||||
1. Verify host/port in `config.toml`.
|
||||
2. Verify username/password.
|
||||
3. Verify TLS settings: `http_scheme` and `verify_ssl`.
|
||||
4. Ensure `database.backend = "trino"`.
|
||||
5. Run the smoke check script: `scripts/trino_smoke_check.py`.
|
||||
|
||||
## Confirmation gate blocks matching
|
||||
|
||||
Symptoms:
|
||||
|
||||
- A matching tool refuses to run and tells you to confirm requirements.
|
||||
|
||||
Explanation:
|
||||
|
||||
- Requirement capture tools write **pending** requirements.
|
||||
- When `matching.require_confirmation = true` (default), matching tools require an explicit user confirmation flow.
|
||||
|
||||
Additional note (multi-client environments):
|
||||
|
||||
- Pending/confirmed state is stored in an in-memory server session.
|
||||
- If multiple clients share one server process, confirmation state can become
|
||||
ambiguous. In such setups, enforce the confirmation workflow strictly on the
|
||||
client/agent side.
|
||||
|
||||
Fix (recommended sequence):
|
||||
|
||||
1. Capture/update requirements (e.g. `extract_requirements`, `collect_structured_requirement_data`, `update_requirements`, guided tools).
|
||||
2. Call `show_pending_requirements()` (review table) **after the last update**.
|
||||
3. Ask the user to confirm (Yes/No).
|
||||
4. Call `confirm_requirements(confirm=true)`.
|
||||
5. Re-run the matching tool.
|
||||
|
||||
Optional (dev/testing):
|
||||
|
||||
- Set `matching.require_confirmation = false` in `config.toml` to auto-skip confirmation.
|
||||
|
||||
## confirm_requirements was called but matching is still blocked
|
||||
|
||||
Symptoms:
|
||||
|
||||
- The assistant called `confirm_requirements(confirm=true)` but matching tools still refuse to run.
|
||||
|
||||
Explanation:
|
||||
|
||||
- The workflow is two-step: the assistant must first request user confirmation.
|
||||
- The recommended sequence is:
|
||||
1. `show_pending_requirements()` (review table)
|
||||
2. Ask the user to confirm (Yes/No)
|
||||
3. `confirm_requirements(confirm=true)`
|
||||
|
||||
Fix:
|
||||
|
||||
- If requirements were updated after the review step, you must re-run `show_pending_requirements()` and ask the user again.
|
||||
- Then call `confirm_requirements(confirm=true)`.
|
||||
|
||||
## Guided capture feels stuck
|
||||
|
||||
Guided capture is step-based. Call the tools in order:
|
||||
|
||||
1. `start_guided_capture()`
|
||||
2. `guided_set_description(...)`
|
||||
3. `guided_set_role(...)`
|
||||
4. `guided_set_time_range(date_start?, date_end?)` (open-ended ranges are allowed)
|
||||
5. `guided_set_competences([...])`
|
||||
6. `show_pending_requirements()`
|
||||
7. Ask the user to confirm (Yes/No)
|
||||
8. `confirm_requirements(confirm=true)`
|
||||
|
||||
## "Unknown or expired search_id" (or filters/pagination stop working)
|
||||
|
||||
Symptoms:
|
||||
|
||||
- `get_results_by_category(...)` or `filter_search_results(...)` returns an error.
|
||||
- Newer server versions also return a machine-readable status:
|
||||
- `META.status=unknown_or_expired`
|
||||
|
||||
Common causes:
|
||||
|
||||
1. **TTL expiry**: the search results cache is time-limited.
|
||||
- Controlled by `cache.search_ttl_minutes`.
|
||||
2. **Eviction due to cache size**: the cache is LRU and bound by `cache.max_size`.
|
||||
- If many searches are started, older `search_id`s can be evicted even within TTL.
|
||||
3. **Process mismatch / restart**: in-memory caches are per server process.
|
||||
- If the server restarts, or the client routes to a different instance, old `search_id`s are not available.
|
||||
4. **Copy hygiene**: some chat UIs add backticks or whitespace.
|
||||
- Prefer copying the UUID from the tool output header line `SEARCH_ID=<uuid>`.
|
||||
- Do not include backticks, quotes, or extra whitespace.
|
||||
|
||||
Fix:
|
||||
|
||||
- Re-run the matching tool to create a new `search_id`.
|
||||
- Increase `cache.search_ttl_minutes` and/or `cache.max_size` for longer interactive sessions.
|
||||
- Ensure your MCP client uses a single long-lived server process.
|
||||
|
||||
## "Invalid search_id format (expected UUID)"
|
||||
|
||||
Cause:
|
||||
|
||||
- The server validates `search_id` inputs and rejects anything that is not a UUID.
|
||||
|
||||
Fix:
|
||||
|
||||
- Copy the value from the most recent tool output header:
|
||||
- `Using SEARCH_ID=<uuid>` (first line)
|
||||
- or `SEARCH_ID=<uuid>`
|
||||
|
||||
## Search tool output parsing (Cherry Studio / chat UI issues)
|
||||
|
||||
Search-related tools emit deterministic headers (intended to be parsed verbatim):
|
||||
|
||||
- First line marker:
|
||||
- `Using SEARCH_ID=<uuid>`
|
||||
- Machine-readable headers:
|
||||
- `SEARCH_ID=<uuid>`
|
||||
- `FILTER_ID=<uuid>` (if applicable)
|
||||
- `META=<json>`
|
||||
|
||||
Output shape notes:
|
||||
|
||||
- Only `find_matching_capacities` includes the category-count `## Summary` table.
|
||||
- `filter_search_results` includes an **Applied Filters** table and a flat results table with a `Category` column (it does not reprint the summary counts).
|
||||
- `get_results_by_category` returns a single category-page table (it does not reprint the summary counts).
|
||||
|
||||
If your UI hides tool output:
|
||||
|
||||
- Re-run the tool call, or
|
||||
- manually copy the UUID from the server output and paste it without formatting.
|
||||
|
||||
## "No matches" or unexpectedly few matches
|
||||
|
||||
- Confirm the required competences list is not over-specific.
|
||||
- Confirm role name is reasonable (role similarity affects scoring).
|
||||
- If you supplied date filters, remember availability is filter-only:
|
||||
- capacities must overlap the requested range
|
||||
- open-ended capacity `end_date` is treated as available without limit
|
||||
|
||||
## Debug logging
|
||||
|
||||
Start the server with:
|
||||
|
||||
- `--log-level DEBUG`
|
||||
|
||||
The database clients emit safe query logs (SQL is normalized; parameters are redacted).
|
||||
|
||||
## Cherry Studio: stdio "connection closed" / tools not listing
|
||||
|
||||
Common causes:
|
||||
|
||||
- The server writes logs to stdout (breaks JSON-RPC). This project logs to
|
||||
**stderr**.
|
||||
- Cherry cannot set a working directory, so `uv run` cannot find the project.
|
||||
|
||||
Workaround (wrap with `zsh -lc`):
|
||||
|
||||
- `zsh -lc 'cd /Users/thomashandke/ws/teamlandkarte-mcp && uv run python -m teamlandkarte_mcp --config /Users/thomashandke/ws/teamlandkarte-mcp/config.toml --log-level DEBUG'`
|
||||
|
||||
Also ensure your `PATH` contains your `uv`/`uvx` directory, e.g.
|
||||
`/Users/thomashandke/.local/bin`.
|
||||
|
||||
## Azure OpenAI API Errors
|
||||
|
||||
Symptoms:
|
||||
|
||||
- Tools that compute similarity fail with `AzureAPIError`.
|
||||
|
||||
Common causes and fixes:
|
||||
|
||||
1. **Missing credentials**
|
||||
- Ensure the environment variable is set:
|
||||
- `AZURE_OPENAI_EMBEDDING_API_KEY`
|
||||
- If you rely on `.env`, confirm it is loaded in the shell that starts the MCP
|
||||
server.
|
||||
|
||||
2. **Invalid endpoint / deployment**
|
||||
- Check `[azure_openai].endpoint` in `config.toml` (must be your Azure OpenAI
|
||||
resource endpoint).
|
||||
- Check `embedding_deployment` exists in Azure and matches your deployment.
|
||||
|
||||
3. **Rate limits / throttling (HTTP 429)**
|
||||
- Retry later.
|
||||
- Reduce concurrency in the client/agent.
|
||||
- Use the embedding cache to reduce repeated embedding calls.
|
||||
|
||||
4. **Network / TLS / proxy issues**
|
||||
- Try `curl` to the Azure endpoint from the same host.
|
||||
- If you are behind a corporate proxy, ensure the environment is configured
|
||||
accordingly.
|
||||
|
||||
Debugging tips:
|
||||
|
||||
- Start the server with `--log-level DEBUG` and look at stderr logs.
|
||||
- Confirm your `config.toml` is the one the server loads (`--config ...`).
|
||||
|
||||
## MCP sampling / embedding features
|
||||
|
||||
This server uses **Azure OpenAI embeddings** for embedding-based similarity
|
||||
scoring.
|
||||
|
||||
- Ensure `AZURE_OPENAI_EMBEDDING_API_KEY` is set.
|
||||
- Ensure `[azure_openai]` in `config.toml` points to a valid Azure OpenAI endpoint.
|
||||
|
||||
## IDE shows "line too long (.. > 79)" but Ruff passes
|
||||
|
||||
Some editor integrations (or their default settings) may enforce a 79 character
|
||||
line length even if the repository uses a different limit.
|
||||
|
||||
If `uv run ruff check` passes but the editor still flags lines:
|
||||
|
||||
- Prefer the repo result as the source of truth.
|
||||
- Adjust your editor/extension settings (Python/Ruff/Flake8) to match the
|
||||
project configuration.
|
||||
@@ -1,7 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from teamlandkarte_mcp.__main__ import main as run
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
@@ -1,456 +0,0 @@
|
||||
# OpenSpec Instructions
|
||||
|
||||
Instructions for AI coding assistants using OpenSpec for spec-driven development.
|
||||
|
||||
## TL;DR Quick Checklist
|
||||
|
||||
- Search existing work: `openspec spec list --long`, `openspec list` (use `rg` only for full-text search)
|
||||
- Decide scope: new capability vs modify existing capability
|
||||
- Pick a unique `change-id`: kebab-case, verb-led (`add-`, `update-`, `remove-`, `refactor-`)
|
||||
- Scaffold: `proposal.md`, `tasks.md`, `design.md` (only if needed), and delta specs per affected capability
|
||||
- Write deltas: use `## ADDED|MODIFIED|REMOVED|RENAMED Requirements`; include at least one `#### Scenario:` per requirement
|
||||
- Validate: `openspec validate [change-id] --strict` and fix issues
|
||||
- Request approval: Do not start implementation until proposal is approved
|
||||
|
||||
## Three-Stage Workflow
|
||||
|
||||
### Stage 1: Creating Changes
|
||||
Create proposal when you need to:
|
||||
- Add features or functionality
|
||||
- Make breaking changes (API, schema)
|
||||
- Change architecture or patterns
|
||||
- Optimize performance (changes behavior)
|
||||
- Update security patterns
|
||||
|
||||
Triggers (examples):
|
||||
- "Help me create a change proposal"
|
||||
- "Help me plan a change"
|
||||
- "Help me create a proposal"
|
||||
- "I want to create a spec proposal"
|
||||
- "I want to create a spec"
|
||||
|
||||
Loose matching guidance:
|
||||
- Contains one of: `proposal`, `change`, `spec`
|
||||
- With one of: `create`, `plan`, `make`, `start`, `help`
|
||||
|
||||
Skip proposal for:
|
||||
- Bug fixes (restore intended behavior)
|
||||
- Typos, formatting, comments
|
||||
- Dependency updates (non-breaking)
|
||||
- Configuration changes
|
||||
- Tests for existing behavior
|
||||
|
||||
**Workflow**
|
||||
1. Review `openspec/project.md`, `openspec list`, and `openspec list --specs` to understand current context.
|
||||
2. Choose a unique verb-led `change-id` and scaffold `proposal.md`, `tasks.md`, optional `design.md`, and spec deltas under `openspec/changes/<id>/`.
|
||||
3. Draft spec deltas using `## ADDED|MODIFIED|REMOVED Requirements` with at least one `#### Scenario:` per requirement.
|
||||
4. Run `openspec validate <id> --strict` and resolve any issues before sharing the proposal.
|
||||
|
||||
### Stage 2: Implementing Changes
|
||||
Track these steps as TODOs and complete them one by one.
|
||||
1. **Read proposal.md** - Understand what's being built
|
||||
2. **Read design.md** (if exists) - Review technical decisions
|
||||
3. **Read tasks.md** - Get implementation checklist
|
||||
4. **Implement tasks sequentially** - Complete in order
|
||||
5. **Confirm completion** - Ensure every item in `tasks.md` is finished before updating statuses
|
||||
6. **Update checklist** - After all work is done, set every task to `- [x]` so the list reflects reality
|
||||
7. **Approval gate** - Do not start implementation until the proposal is reviewed and approved
|
||||
|
||||
### Stage 3: Archiving Changes
|
||||
After deployment, create separate PR to:
|
||||
- Move `changes/[name]/` → `changes/archive/YYYY-MM-DD-[name]/`
|
||||
- Update `specs/` if capabilities changed
|
||||
- Use `openspec archive <change-id> --skip-specs --yes` for tooling-only changes (always pass the change ID explicitly)
|
||||
- Run `openspec validate --strict` to confirm the archived change passes checks
|
||||
|
||||
## Before Any Task
|
||||
|
||||
**Context Checklist:**
|
||||
- [ ] Read relevant specs in `specs/[capability]/spec.md`
|
||||
- [ ] Check pending changes in `changes/` for conflicts
|
||||
- [ ] Read `openspec/project.md` for conventions
|
||||
- [ ] Run `openspec list` to see active changes
|
||||
- [ ] Run `openspec list --specs` to see existing capabilities
|
||||
|
||||
**Before Creating Specs:**
|
||||
- Always check if capability already exists
|
||||
- Prefer modifying existing specs over creating duplicates
|
||||
- Use `openspec show [spec]` to review current state
|
||||
- If request is ambiguous, ask 1–2 clarifying questions before scaffolding
|
||||
|
||||
### Search Guidance
|
||||
- Enumerate specs: `openspec spec list --long` (or `--json` for scripts)
|
||||
- Enumerate changes: `openspec list` (or `openspec change list --json` - deprecated but available)
|
||||
- Show details:
|
||||
- Spec: `openspec show <spec-id> --type spec` (use `--json` for filters)
|
||||
- Change: `openspec show <change-id> --json --deltas-only`
|
||||
- Full-text search (use ripgrep): `rg -n "Requirement:|Scenario:" openspec/specs`
|
||||
|
||||
## Quick Start
|
||||
|
||||
### CLI Commands
|
||||
|
||||
```bash
|
||||
# Essential commands
|
||||
openspec list # List active changes
|
||||
openspec list --specs # List specifications
|
||||
openspec show [item] # Display change or spec
|
||||
openspec validate [item] # Validate changes or specs
|
||||
openspec archive <change-id> [--yes|-y] # Archive after deployment (add --yes for non-interactive runs)
|
||||
|
||||
# Project management
|
||||
openspec init [path] # Initialize OpenSpec
|
||||
openspec update [path] # Update instruction files
|
||||
|
||||
# Interactive mode
|
||||
openspec show # Prompts for selection
|
||||
openspec validate # Bulk validation mode
|
||||
|
||||
# Debugging
|
||||
openspec show [change] --json --deltas-only
|
||||
openspec validate [change] --strict
|
||||
```
|
||||
|
||||
### Command Flags
|
||||
|
||||
- `--json` - Machine-readable output
|
||||
- `--type change|spec` - Disambiguate items
|
||||
- `--strict` - Comprehensive validation
|
||||
- `--no-interactive` - Disable prompts
|
||||
- `--skip-specs` - Archive without spec updates
|
||||
- `--yes`/`-y` - Skip confirmation prompts (non-interactive archive)
|
||||
|
||||
## Directory Structure
|
||||
|
||||
```
|
||||
openspec/
|
||||
├── project.md # Project conventions
|
||||
├── specs/ # Current truth - what IS built
|
||||
│ └── [capability]/ # Single focused capability
|
||||
│ ├── spec.md # Requirements and scenarios
|
||||
│ └── design.md # Technical patterns
|
||||
├── changes/ # Proposals - what SHOULD change
|
||||
│ ├── [change-name]/
|
||||
│ │ ├── proposal.md # Why, what, impact
|
||||
│ │ ├── tasks.md # Implementation checklist
|
||||
│ │ ├── design.md # Technical decisions (optional; see criteria)
|
||||
│ │ └── specs/ # Delta changes
|
||||
│ │ └── [capability]/
|
||||
│ │ └── spec.md # ADDED/MODIFIED/REMOVED
|
||||
│ └── archive/ # Completed changes
|
||||
```
|
||||
|
||||
## Creating Change Proposals
|
||||
|
||||
### Decision Tree
|
||||
|
||||
```
|
||||
New request?
|
||||
├─ Bug fix restoring spec behavior? → Fix directly
|
||||
├─ Typo/format/comment? → Fix directly
|
||||
├─ New feature/capability? → Create proposal
|
||||
├─ Breaking change? → Create proposal
|
||||
├─ Architecture change? → Create proposal
|
||||
└─ Unclear? → Create proposal (safer)
|
||||
```
|
||||
|
||||
### Proposal Structure
|
||||
|
||||
1. **Create directory:** `changes/[change-id]/` (kebab-case, verb-led, unique)
|
||||
|
||||
2. **Write proposal.md:**
|
||||
```markdown
|
||||
# Change: [Brief description of change]
|
||||
|
||||
## Why
|
||||
[1-2 sentences on problem/opportunity]
|
||||
|
||||
## What Changes
|
||||
- [Bullet list of changes]
|
||||
- [Mark breaking changes with **BREAKING**]
|
||||
|
||||
## Impact
|
||||
- Affected specs: [list capabilities]
|
||||
- Affected code: [key files/systems]
|
||||
```
|
||||
|
||||
3. **Create spec deltas:** `specs/[capability]/spec.md`
|
||||
```markdown
|
||||
## ADDED Requirements
|
||||
### Requirement: New Feature
|
||||
The system SHALL provide...
|
||||
|
||||
#### Scenario: Success case
|
||||
- **WHEN** user performs action
|
||||
- **THEN** expected result
|
||||
|
||||
## MODIFIED Requirements
|
||||
### Requirement: Existing Feature
|
||||
[Complete modified requirement]
|
||||
|
||||
## REMOVED Requirements
|
||||
### Requirement: Old Feature
|
||||
**Reason**: [Why removing]
|
||||
**Migration**: [How to handle]
|
||||
```
|
||||
If multiple capabilities are affected, create multiple delta files under `changes/[change-id]/specs/<capability>/spec.md`—one per capability.
|
||||
|
||||
4. **Create tasks.md:**
|
||||
```markdown
|
||||
## 1. Implementation
|
||||
- [ ] 1.1 Create database schema
|
||||
- [ ] 1.2 Implement API endpoint
|
||||
- [ ] 1.3 Add frontend component
|
||||
- [ ] 1.4 Write tests
|
||||
```
|
||||
|
||||
5. **Create design.md when needed:**
|
||||
Create `design.md` if any of the following apply; otherwise omit it:
|
||||
- Cross-cutting change (multiple services/modules) or a new architectural pattern
|
||||
- New external dependency or significant data model changes
|
||||
- Security, performance, or migration complexity
|
||||
- Ambiguity that benefits from technical decisions before coding
|
||||
|
||||
Minimal `design.md` skeleton:
|
||||
```markdown
|
||||
## Context
|
||||
[Background, constraints, stakeholders]
|
||||
|
||||
## Goals / Non-Goals
|
||||
- Goals: [...]
|
||||
- Non-Goals: [...]
|
||||
|
||||
## Decisions
|
||||
- Decision: [What and why]
|
||||
- Alternatives considered: [Options + rationale]
|
||||
|
||||
## Risks / Trade-offs
|
||||
- [Risk] → Mitigation
|
||||
|
||||
## Migration Plan
|
||||
[Steps, rollback]
|
||||
|
||||
## Open Questions
|
||||
- [...]
|
||||
```
|
||||
|
||||
## Spec File Format
|
||||
|
||||
### Critical: Scenario Formatting
|
||||
|
||||
**CORRECT** (use #### headers):
|
||||
```markdown
|
||||
#### Scenario: User login success
|
||||
- **WHEN** valid credentials provided
|
||||
- **THEN** return JWT token
|
||||
```
|
||||
|
||||
**WRONG** (don't use bullets or bold):
|
||||
```markdown
|
||||
- **Scenario: User login** ❌
|
||||
**Scenario**: User login ❌
|
||||
### Scenario: User login ❌
|
||||
```
|
||||
|
||||
Every requirement MUST have at least one scenario.
|
||||
|
||||
### Requirement Wording
|
||||
- Use SHALL/MUST for normative requirements (avoid should/may unless intentionally non-normative)
|
||||
|
||||
### Delta Operations
|
||||
|
||||
- `## ADDED Requirements` - New capabilities
|
||||
- `## MODIFIED Requirements` - Changed behavior
|
||||
- `## REMOVED Requirements` - Deprecated features
|
||||
- `## RENAMED Requirements` - Name changes
|
||||
|
||||
Headers matched with `trim(header)` - whitespace ignored.
|
||||
|
||||
#### When to use ADDED vs MODIFIED
|
||||
- ADDED: Introduces a new capability or sub-capability that can stand alone as a requirement. Prefer ADDED when the change is orthogonal (e.g., adding "Slash Command Configuration") rather than altering the semantics of an existing requirement.
|
||||
- MODIFIED: Changes the behavior, scope, or acceptance criteria of an existing requirement. Always paste the full, updated requirement content (header + all scenarios). The archiver will replace the entire requirement with what you provide here; partial deltas will drop previous details.
|
||||
- RENAMED: Use when only the name changes. If you also change behavior, use RENAMED (name) plus MODIFIED (content) referencing the new name.
|
||||
|
||||
Common pitfall: Using MODIFIED to add a new concern without including the previous text. This causes loss of detail at archive time. If you aren’t explicitly changing the existing requirement, add a new requirement under ADDED instead.
|
||||
|
||||
Authoring a MODIFIED requirement correctly:
|
||||
1) Locate the existing requirement in `openspec/specs/<capability>/spec.md`.
|
||||
2) Copy the entire requirement block (from `### Requirement: ...` through its scenarios).
|
||||
3) Paste it under `## MODIFIED Requirements` and edit to reflect the new behavior.
|
||||
4) Ensure the header text matches exactly (whitespace-insensitive) and keep at least one `#### Scenario:`.
|
||||
|
||||
Example for RENAMED:
|
||||
```markdown
|
||||
## RENAMED Requirements
|
||||
- FROM: `### Requirement: Login`
|
||||
- TO: `### Requirement: User Authentication`
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Errors
|
||||
|
||||
**"Change must have at least one delta"**
|
||||
- Check `changes/[name]/specs/` exists with .md files
|
||||
- Verify files have operation prefixes (## ADDED Requirements)
|
||||
|
||||
**"Requirement must have at least one scenario"**
|
||||
- Check scenarios use `#### Scenario:` format (4 hashtags)
|
||||
- Don't use bullet points or bold for scenario headers
|
||||
|
||||
**Silent scenario parsing failures**
|
||||
- Exact format required: `#### Scenario: Name`
|
||||
- Debug with: `openspec show [change] --json --deltas-only`
|
||||
|
||||
### Validation Tips
|
||||
|
||||
```bash
|
||||
# Always use strict mode for comprehensive checks
|
||||
openspec validate [change] --strict
|
||||
|
||||
# Debug delta parsing
|
||||
openspec show [change] --json | jq '.deltas'
|
||||
|
||||
# Check specific requirement
|
||||
openspec show [spec] --json -r 1
|
||||
```
|
||||
|
||||
## Happy Path Script
|
||||
|
||||
```bash
|
||||
# 1) Explore current state
|
||||
openspec spec list --long
|
||||
openspec list
|
||||
# Optional full-text search:
|
||||
# rg -n "Requirement:|Scenario:" openspec/specs
|
||||
# rg -n "^#|Requirement:" openspec/changes
|
||||
|
||||
# 2) Choose change id and scaffold
|
||||
CHANGE=add-two-factor-auth
|
||||
mkdir -p openspec/changes/$CHANGE/{specs/auth}
|
||||
printf "## Why\n...\n\n## What Changes\n- ...\n\n## Impact\n- ...\n" > openspec/changes/$CHANGE/proposal.md
|
||||
printf "## 1. Implementation\n- [ ] 1.1 ...\n" > openspec/changes/$CHANGE/tasks.md
|
||||
|
||||
# 3) Add deltas (example)
|
||||
cat > openspec/changes/$CHANGE/specs/auth/spec.md << 'EOF'
|
||||
## ADDED Requirements
|
||||
### Requirement: Two-Factor Authentication
|
||||
Users MUST provide a second factor during login.
|
||||
|
||||
#### Scenario: OTP required
|
||||
- **WHEN** valid credentials are provided
|
||||
- **THEN** an OTP challenge is required
|
||||
EOF
|
||||
|
||||
# 4) Validate
|
||||
openspec validate $CHANGE --strict
|
||||
```
|
||||
|
||||
## Multi-Capability Example
|
||||
|
||||
```
|
||||
openspec/changes/add-2fa-notify/
|
||||
├── proposal.md
|
||||
├── tasks.md
|
||||
└── specs/
|
||||
├── auth/
|
||||
│ └── spec.md # ADDED: Two-Factor Authentication
|
||||
└── notifications/
|
||||
└── spec.md # ADDED: OTP email notification
|
||||
```
|
||||
|
||||
auth/spec.md
|
||||
```markdown
|
||||
## ADDED Requirements
|
||||
### Requirement: Two-Factor Authentication
|
||||
...
|
||||
```
|
||||
|
||||
notifications/spec.md
|
||||
```markdown
|
||||
## ADDED Requirements
|
||||
### Requirement: OTP Email Notification
|
||||
...
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
### Simplicity First
|
||||
- Default to <100 lines of new code
|
||||
- Single-file implementations until proven insufficient
|
||||
- Avoid frameworks without clear justification
|
||||
- Choose boring, proven patterns
|
||||
|
||||
### Complexity Triggers
|
||||
Only add complexity with:
|
||||
- Performance data showing current solution too slow
|
||||
- Concrete scale requirements (>1000 users, >100MB data)
|
||||
- Multiple proven use cases requiring abstraction
|
||||
|
||||
### Clear References
|
||||
- Use `file.ts:42` format for code locations
|
||||
- Reference specs as `specs/auth/spec.md`
|
||||
- Link related changes and PRs
|
||||
|
||||
### Capability Naming
|
||||
- Use verb-noun: `user-auth`, `payment-capture`
|
||||
- Single purpose per capability
|
||||
- 10-minute understandability rule
|
||||
- Split if description needs "AND"
|
||||
|
||||
### Change ID Naming
|
||||
- Use kebab-case, short and descriptive: `add-two-factor-auth`
|
||||
- Prefer verb-led prefixes: `add-`, `update-`, `remove-`, `refactor-`
|
||||
- Ensure uniqueness; if taken, append `-2`, `-3`, etc.
|
||||
|
||||
## Tool Selection Guide
|
||||
|
||||
| Task | Tool | Why |
|
||||
|------|------|-----|
|
||||
| Find files by pattern | Glob | Fast pattern matching |
|
||||
| Search code content | Grep | Optimized regex search |
|
||||
| Read specific files | Read | Direct file access |
|
||||
| Explore unknown scope | Task | Multi-step investigation |
|
||||
|
||||
## Error Recovery
|
||||
|
||||
### Change Conflicts
|
||||
1. Run `openspec list` to see active changes
|
||||
2. Check for overlapping specs
|
||||
3. Coordinate with change owners
|
||||
4. Consider combining proposals
|
||||
|
||||
### Validation Failures
|
||||
1. Run with `--strict` flag
|
||||
2. Check JSON output for details
|
||||
3. Verify spec file format
|
||||
4. Ensure scenarios properly formatted
|
||||
|
||||
### Missing Context
|
||||
1. Read project.md first
|
||||
2. Check related specs
|
||||
3. Review recent archives
|
||||
4. Ask for clarification
|
||||
|
||||
## Quick Reference
|
||||
|
||||
### Stage Indicators
|
||||
- `changes/` - Proposed, not yet built
|
||||
- `specs/` - Built and deployed
|
||||
- `archive/` - Completed changes
|
||||
|
||||
### File Purposes
|
||||
- `proposal.md` - Why and what
|
||||
- `tasks.md` - Implementation steps
|
||||
- `design.md` - Technical decisions
|
||||
- `spec.md` - Requirements and behavior
|
||||
|
||||
### CLI Essentials
|
||||
```bash
|
||||
openspec list # What's in progress?
|
||||
openspec show [item] # View details
|
||||
openspec validate --strict # Is it correct?
|
||||
openspec archive <change-id> [--yes|-y] # Mark complete (add --yes for automation)
|
||||
```
|
||||
|
||||
Remember: Specs are truth. Changes are proposals. Keep them in sync.
|
||||
@@ -1 +0,0 @@
|
||||
This folder intentionally left blank. (OpenSpec scaffolding will add proposal/tasks/design.)
|
||||
@@ -1,84 +0,0 @@
|
||||
# Design: Accelerate Embedding Similarity
|
||||
|
||||
## Context
|
||||
|
||||
The server uses Azure OpenAI embeddings for competence and role similarity. The matching pipeline can be slow because embeddings are currently requested on-demand and repeatedly from inner loops.
|
||||
|
||||
The system already has a persistent SQLite embedding cache (`EmbeddingCache`). However, without bulk prefetch and without a run-scoped memoization layer, the similarity engine still:
|
||||
|
||||
- repeatedly normalizes and hashes text keys
|
||||
- performs many repeated SQLite reads
|
||||
- performs sequential Azure calls for cache misses
|
||||
|
||||
## Goals
|
||||
|
||||
- Bulk prefetch embeddings for a matching run.
|
||||
- Maintain deterministic behavior and strict error semantics.
|
||||
- Preserve existing similarity contracts.
|
||||
|
||||
## Architecture Changes
|
||||
|
||||
### Components impacted
|
||||
|
||||
- `SimilarityEngine` (`src/teamlandkarte_mcp/matching/similarity.py`)
|
||||
- Add instance variable for in-memory cache (lives with the engine; not created/cleared per invocation)
|
||||
- Add public method `prefetch_embeddings(required_texts, candidate_texts, required_roles, candidate_roles) -> dict[str, list[float]]`
|
||||
- `AzureOpenAIClient` (`src/teamlandkarte_mcp/azure/openai_client.py`)
|
||||
- Modify `get_embeddings_batch()` to use true batch API with `input=[...]`
|
||||
- `CostTracker` (`src/teamlandkarte_mcp/azure/cost_tracker.py`)
|
||||
- May need adjustment to log batch requests (log once per chunk with count)
|
||||
- `EmbeddingCache` (no schema changes expected)
|
||||
|
||||
### Data flow (proposed)
|
||||
|
||||
1. **Collect** all texts that will be embedded in the current run.
|
||||
- This is a **single global collection pass** over the inputs and **all free capacities** (roles + competences), not a per-person loop that embeds incrementally.
|
||||
2. **Normalize + deduplicate** using existing `_normalize_text()` logic.
|
||||
- Skip empty/whitespace-only texts
|
||||
- Emit a **Python logger warning** if all candidate competences normalize to empty
|
||||
3. **Resolve embeddings** via:
|
||||
- in-memory cache (instance variable in `SimilarityEngine`)
|
||||
- SQLite cache (`EmbeddingCache`)
|
||||
- Azure embeddings API (**batch only the cache-missing texts**, **chunked** by batch size)
|
||||
4. **Compute** cosine similarities locally using the prefetched mapping.
|
||||
|
||||
### Normalization and keys
|
||||
|
||||
Current cache key behavior:
|
||||
|
||||
- normalization: trim + collapse whitespace
|
||||
- key: SHA256 of `model|dims|normalized(text).lower()`
|
||||
|
||||
This proposal keeps the same normalization and key derivation to avoid invalidating the on-disk cache.
|
||||
|
||||
## Batch embeddings API
|
||||
|
||||
The OpenAI/Azure embeddings API supports embedding multiple inputs per request (`input=[...]`).
|
||||
|
||||
Implementation requirements:
|
||||
|
||||
- preserve stable mapping from input texts → returned embedding vectors
|
||||
- rely on `index` field in API response if available
|
||||
- otherwise assume stable ordering
|
||||
- chunk large requests to avoid request size limits (configurable batch size, default 128)
|
||||
- strict failure semantics:
|
||||
- entire matching operation fails immediately on error
|
||||
- log the failing **chunk input list** via Python logging to aid diagnosis
|
||||
- cost tracking: log once per batch chunk with a count of embeddings requested
|
||||
|
||||
## Trade-offs
|
||||
|
||||
- Batch calls reduce network requests dramatically but may increase blast radius: one failing request could affect multiple inputs.
|
||||
- Mitigation: keep retry logic at the chunk level.
|
||||
- Prefetch requires holding larger embedding dictionaries in memory.
|
||||
- Mitigation: scope to a single invocation, and only store vectors required for that run.
|
||||
|
||||
## Test Strategy
|
||||
|
||||
- Unit test prefetch/dedup: ensure the Azure client is called once per unique normalized text (only for cache misses).
|
||||
- Unit test cache layering: ensure hits are served from in-memory cache (first), then SQLite, and do not call Azure.
|
||||
- Unit test chunking: ensure multiple Azure calls are made when `len(cache_missing_texts) > batch_size`.
|
||||
- Unit test empty input handling: verify warning when all candidate competences normalize to empty; verify prefetch skips empty texts.
|
||||
- Unit test error identification: verify that batch failures report which specific text(s) caused the error.
|
||||
- Integration test (manual/gated): validate real Azure batch API behavior with live credentials.
|
||||
|
||||
@@ -1,124 +0,0 @@
|
||||
# Change Proposal: Accelerate Embedding Prefetch and Similarity Computation
|
||||
|
||||
- **Change ID**: `accelerate-embedding-similarity`
|
||||
- **Status**: Proposed
|
||||
- **Target**: `teamlandkarte-mcp`
|
||||
- **Author**: Thomas Handke
|
||||
- **Date**: 2026-02-14
|
||||
|
||||
## Summary
|
||||
|
||||
Speed up `find_matching_capacities(...)` by reducing embedding API calls and repeated cache lookups.
|
||||
|
||||
This change introduces three complementary optimizations:
|
||||
|
||||
1. **Dedup + bulk prefetch** of all embedding texts needed in a matching run (required competences, candidate competences, and roles).
|
||||
2. A **per-run in-memory cache** layered on top of the existing SQLite embedding cache to avoid repeated SQLite reads within a single run.
|
||||
3. Support for **true Azure Embeddings batch requests** (single request with multiple inputs), including chunking, while preserving deterministic error semantics.
|
||||
|
||||
## Motivation
|
||||
|
||||
The current implementation calls embeddings repeatedly inside inner loops. While a persistent SQLite cache exists, repeated calls still incur overhead:
|
||||
|
||||
- many redundant `cache.get(...)` reads
|
||||
- repeated normalization/keying for identical strings
|
||||
- unnecessary Azure round-trips when multiple missing terms are encountered and requests are performed sequentially
|
||||
|
||||
The result is that `find_matching_capacities(...)` can be slow, and while the second run is faster (due to cache hits), it can still take noticeable time.
|
||||
|
||||
## Goals
|
||||
|
||||
1. Ensure each unique text is embedded at most once per run (network and cache).
|
||||
2. Prefetch embeddings **before** similarity computation so inner loops only perform vector math.
|
||||
3. Enable Azure embeddings API batch calls to reduce network round-trips.
|
||||
4. Preserve current similarity output contracts and scoring behavior.
|
||||
5. Maintain explicit, deterministic failure semantics (no heuristics).
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- Changing match scoring weights or category thresholds.
|
||||
- Changing the similarity strategy semantics ("per_skill" vs "aggregate").
|
||||
- Changing the embedding model or dimensionality.
|
||||
- Introducing new external dependencies unless necessary.
|
||||
|
||||
## Proposed Changes
|
||||
|
||||
### 1) Bulk prefetch & dedup in `SimilarityEngine`
|
||||
|
||||
#### Current behavior (problem)
|
||||
|
||||
`SimilarityEngine` currently embeds terms on demand via `_embed(...)`. For large candidate lists, this forces a high number of calls to `_embed`, and repeated SQLite reads.
|
||||
|
||||
#### New behavior
|
||||
|
||||
Add a prefetch step that:
|
||||
|
||||
- collects all texts required to compute similarity for a run (competences **and roles**)
|
||||
- **Important:** this collection is performed **across the entire candidate set** (i.e., roles and competences of **all free capacities at once**), not person-by-person.
|
||||
- normalizes for cache key stability
|
||||
- deduplicates texts
|
||||
- fetches all embeddings from caches/API in bulk
|
||||
- returns a mapping from normalized text → embedding vector
|
||||
|
||||
Then similarity computation uses the prefetched mapping (pure Python vector math).
|
||||
|
||||
### 2) Per-run in-memory cache
|
||||
|
||||
Add a lightweight in-memory dict for the duration of a single matching invocation.
|
||||
|
||||
- key: `(model, dimensions, normalized_text)` or reuse the computed SHA256 cache key
|
||||
- value: `list[float]`
|
||||
|
||||
This prevents repeated SQLite lookups for frequently used items (common competences, repeated role strings, etc.).
|
||||
|
||||
### 3) True Azure embeddings batch calls
|
||||
|
||||
Change `AzureOpenAIClient.get_embeddings_batch(...)` to use the Azure embeddings API with `input=[...]` to embed multiple texts in a single request.
|
||||
|
||||
Key details:
|
||||
|
||||
- supports chunking with a **configurable** batch size
|
||||
- preserves stable output ordering and deterministic error semantics
|
||||
- continues to log costs via `CostTracker`
|
||||
|
||||
If any item fails, the batch call should raise `AzureAPIError` (no fallback).
|
||||
|
||||
## Decisions (resolved)
|
||||
|
||||
- **Scope**: bulk prefetch includes **competences and roles**.
|
||||
- **Batching**: Azure embeddings requests are made in chunks with a **configurable** batch size.
|
||||
- **Normalization**: keep the current text normalization and cache-key behavior.
|
||||
- **Concurrency**: process chunks **sequentially** (no parallel calls).
|
||||
- **In-memory cache lifecycle**: implemented as a new instance variable in `SimilarityEngine` and **not** created/cleaned per matching invocation.
|
||||
- **Prefetch API**: new public method `prefetch_embeddings(required_texts, candidate_texts, required_roles, candidate_roles)` that returns a mapping from normalized text → embedding vector.
|
||||
- **Azure batch response ordering**: rely on the `index` field in the response if available, otherwise assume stable ordering.
|
||||
- **Error handling**: entire matching operation fails immediately; attempt to identify and report which specific text(s) caused the failure.
|
||||
- **Cost tracking**: log once per batch chunk with a count.
|
||||
- **Empty inputs**:
|
||||
- emit a **Python logger warning** when all candidate competences normalize to empty strings
|
||||
- prefetch skips empty/whitespace-only texts
|
||||
- **Cache miss batching**: batch-request only the missing embeddings (after resolving cache hits from SQLite).
|
||||
|
||||
## Configuration
|
||||
|
||||
Introduce (or reuse if already present) one configuration knob:
|
||||
|
||||
- `azure_openai.embedding_batch_size` (integer)
|
||||
- default: `128` (conservative)
|
||||
- no enforced maximum value
|
||||
- used to chunk `input=[...]` for Azure embeddings batch requests
|
||||
- if not specified in `config.toml`, the default value is used
|
||||
|
||||
## Success Criteria
|
||||
|
||||
- A single `find_matching_capacities(...)` run should perform at most one embedding generation per unique term (after cache resolution).
|
||||
- Only cache-missing embeddings are batch-requested from Azure (cache hits are not re-requested).
|
||||
- For warmed cache runs, no Azure embedding calls should occur.
|
||||
- Unit tests cover dedup/prefetch, cache layering, and batch request behavior.
|
||||
- Integration tests (manual/gated) verify real Azure batch API behavior.
|
||||
|
||||
## Rollout / Compatibility
|
||||
|
||||
- No configuration changes are strictly required; the baseline refactor should work with existing `config.toml`.
|
||||
- If new config knobs are added (e.g., batch size), defaults should preserve behavior.
|
||||
|
||||
-65
@@ -1,65 +0,0 @@
|
||||
# Capability: Embedding Prefetch & Batch Similarity
|
||||
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: Bulk prefetch and deduplicate embedding inputs
|
||||
|
||||
The server MUST collect all texts that require embeddings for a matching run, normalize them, deduplicate them, and prefetch their embeddings before computing similarity scores.
|
||||
|
||||
#### Scenario: Matching run with repeated competences
|
||||
|
||||
- Given a matching run where the required competences contain duplicates (e.g. "Python", "Python")
|
||||
- And a candidate list contains overlapping competences (e.g. multiple candidates list "Python")
|
||||
- When the similarity computation starts
|
||||
- Then the embedding for "Python" is requested at most once from the Azure embeddings API
|
||||
- And all uses of "Python" reuse the same embedding vector for similarity computation
|
||||
|
||||
### Requirement: Compute similarity using prefetched vectors (no inner-loop embedding calls)
|
||||
|
||||
The similarity computation MUST use prefetched embedding vectors and MUST NOT request embeddings from inside per-competence or per-candidate inner loops.
|
||||
|
||||
#### Scenario: Per-skill similarity on large candidate set
|
||||
|
||||
- Given required competences and a large candidate competence list
|
||||
- When `per_skill` similarity is computed
|
||||
- Then all embeddings are resolved prior to the nested loops
|
||||
- And the nested loops perform only cosine similarity calculations
|
||||
|
||||
## MODIFIED Requirements
|
||||
|
||||
### Requirement: Batch embedding API support in Azure client
|
||||
|
||||
`AzureOpenAIClient.get_embeddings_batch(...)` MUST support true request-level batching using the Azure embeddings API (`input=[...]`) and MUST preserve input ordering.
|
||||
|
||||
#### Scenario: Batch embedding input list
|
||||
|
||||
- Given a list of N texts to embed
|
||||
- When `get_embeddings_batch(texts)` is called
|
||||
- Then the client makes as few Azure embedding requests as possible (subject to chunking)
|
||||
- And the returned list of embeddings aligns with `texts` order
|
||||
|
||||
### Requirement: Run-scoped in-memory embedding cache
|
||||
|
||||
The server MUST maintain a run-scoped in-memory cache to prevent repeated SQLite cache reads for identical texts during a single matching operation.
|
||||
|
||||
#### Scenario: Frequent repeated cache hits within one run
|
||||
|
||||
- Given cached embeddings are present on disk
|
||||
- And a matching run needs the same text embedding multiple times
|
||||
- When the similarity engine resolves embeddings
|
||||
- Then repeated lookups for the same text are served from an in-memory cache after the first lookup
|
||||
- And no additional SQLite reads are required for that text during the run
|
||||
|
||||
## REMOVED Requirements
|
||||
|
||||
### Requirement: Sequential batch behavior requirement
|
||||
|
||||
The previous constraint that `get_embeddings_batch` must call the embeddings API sequentially per text is removed.
|
||||
|
||||
#### Scenario: Legacy sequential constraint
|
||||
|
||||
- Given earlier requirements constrained `get_embeddings_batch` to per-text calls
|
||||
- When the system is upgraded for performance
|
||||
- Then this constraint no longer applies
|
||||
- And request-level batching is allowed
|
||||
|
||||
@@ -1,100 +0,0 @@
|
||||
# Tasks: Accelerate Embedding Similarity
|
||||
|
||||
> Change: `accelerate-embedding-similarity`
|
||||
>
|
||||
> Status: **Approved / Implemented**
|
||||
|
||||
## Overview
|
||||
|
||||
Reduce runtime of `find_matching_capacities(...)` by implementing:
|
||||
|
||||
- embedding text deduplication + **global** bulk prefetch (roles + competences for **all free capacities at once**, not person-by-person)
|
||||
- in-memory embedding memoization
|
||||
- true Azure embeddings batch requests with chunking
|
||||
|
||||
## Task Breakdown
|
||||
|
||||
### Phase 1: Design & Validation Plan
|
||||
|
||||
- [x] 1.1 Identify hot paths during matching (where `_embed(...)` is called most).
|
||||
- [x] 1.2 Confirm normalization + cache key rules are unchanged and documented.
|
||||
- [x] 1.3 Define expected call-count behavior (upper bounds):
|
||||
- per run: max 1 Azure embedding per unique normalized text (subject to chunking, after cache resolution)
|
||||
- per run: max 1 SQLite read per unique normalized text (after in-memory cache check)
|
||||
|
||||
### Phase 2: Global Bulk Prefetch in `SimilarityEngine`
|
||||
|
||||
- [x] 2.1 Add a new instance variable `_run_cache: dict[str, list[float]]` to `SimilarityEngine` (in-memory cache; **not** created/cleared per matching invocation).
|
||||
- [x] 2.2 Implement a public method `prefetch_embeddings(required_texts: list[str], candidate_texts: list[str], required_roles: list[str], candidate_roles: list[str]) -> dict[str, list[float]]`:
|
||||
- collect all input texts (required competences, **required roles**, all candidate competences, **all candidate roles**)
|
||||
- normalize + deduplicate using existing `_normalize_text()` logic
|
||||
- skip empty/whitespace-only texts
|
||||
- emit a **Python logger warning** if all candidate competences normalize to empty
|
||||
- [x] 2.3 Implement prefetch resolution logic:
|
||||
1) check in-memory cache (`_run_cache`)
|
||||
2) check SQLite embedding cache
|
||||
3) batch-request only cache-missing texts from Azure API (chunked by batch size)
|
||||
- [x] 2.4 Refactor `_per_skill_similarity` to use prefetched embeddings only (no embedding calls inside inner loops).
|
||||
- [x] 2.5 Refactor `_aggregate_similarity` to use prefetched embeddings only.
|
||||
- [x] 2.6 Ensure output format is unchanged and remains deterministic.
|
||||
|
||||
### Phase 3: In-memory Run Cache
|
||||
|
||||
- [x] 3.1 Verify `_run_cache` is an instance-level cache that does not require per-invocation clearing.
|
||||
- [x] 3.2 Add tests ensuring repeated embedding lookups for the same text:
|
||||
- are served from `_run_cache` after first lookup
|
||||
- do not call SQLite/API multiple times for the same text
|
||||
|
||||
### Phase 4: Azure Batch Embeddings Support
|
||||
|
||||
- [x] 4.1 Modify `AzureOpenAIClient.get_embeddings_batch()` to use Azure embeddings API with `input=[...]`.
|
||||
- [x] 4.2 Implement chunking using `azure_openai.embedding_batch_size` (default: 128, configurable, no enforced maximum).
|
||||
- [x] 4.3 Preserve ordering using `index` field in API response if available, otherwise assume stable ordering.
|
||||
- [x] 4.4 Ensure strict error semantics:
|
||||
- entire matching operation fails immediately on error
|
||||
- log the failing **chunk input list** via Python logging
|
||||
- [x] 4.5 Update `CostTracker` to log batch requests: log once per chunk with count of embeddings requested.
|
||||
|
||||
### Phase 5: Configuration
|
||||
|
||||
- [x] 5.1 Extend config model to include `azure_openai.embedding_batch_size` (integer, default: 128, no enforced maximum).
|
||||
- [x] 5.2 Update `config.toml.example` with the new setting and comments.
|
||||
- [x] 5.3 Ensure fallback: if `embedding_batch_size` is not specified in `config.toml`, use the default value (128).
|
||||
|
||||
### Phase 6: Tests
|
||||
|
||||
- [x] 6.1 Unit tests: global prefetch/dedup
|
||||
- verify dedup across required + all candidates (incl. roles)
|
||||
- verify embedding client is called once per unique normalized text (only for cache misses)
|
||||
- verify empty/whitespace-only texts are skipped
|
||||
- verify a Python logger warning is emitted when all candidate competences normalize to empty
|
||||
- [x] 6.2 Unit tests: cache layering
|
||||
- verify in-memory `_run_cache` prevents repeated SQLite reads within a run
|
||||
- verify SQLite hits prevent Azure calls
|
||||
- [x] 6.3 Unit tests: Azure batch request behavior (mocked)
|
||||
- verifies batch call for N texts when `N <= batch_size`
|
||||
- verifies chunking when `N > batch_size` (multiple batch calls)
|
||||
- verifies output ordering is stable (using `index` field or assuming stable order)
|
||||
- verifies strict error propagation
|
||||
- verifies the failing chunk input list is logged
|
||||
- verifies cost tracking logs once per chunk with count
|
||||
- [x] 6.4 Integration test (manual/gated only): real Azure batch embeddings
|
||||
- confirm request uses batch `input=[...]` semantics
|
||||
- ensure behavior matches expected semantics for a representative set of texts
|
||||
|
||||
### Phase 7: Documentation + Assistant Prompt Updates
|
||||
|
||||
- [x] 7.1 Update `README.md` to describe the optimization at a high level and mention the new batch size knob.
|
||||
- [x] 7.2 Update `docs/assistant_system_prompt.md` to reflect:
|
||||
- embeddings are prefetched globally per run
|
||||
- batching/chunking is used and configurable
|
||||
- failure semantics remain strict (no heuristics)
|
||||
- [x] 7.3 Update any Azure setup docs if needed (e.g., `docs/azure_openai_setup.md`) to mention embedding batch behavior and tuning.
|
||||
|
||||
### Phase 8: Quality Gates / Release Hygiene
|
||||
|
||||
- [x] 8.1 Run unit tests: `uv run pytest -m "not integration" -q`
|
||||
- [x] 8.2 Run Ruff and mypy via `uv`.
|
||||
- [ ] 8.3 (Manual/gated only) Run integration tests (Azure) to validate real batch calls.
|
||||
- [x] 8.4 Update change docs status/checklists and ensure OpenSpec validates (`openspec validate accelerate-embedding-similarity --strict`).
|
||||
|
||||
@@ -1,666 +0,0 @@
|
||||
# Design: BM25 + RRF Competence Matching
|
||||
|
||||
## Context
|
||||
|
||||
This design specifies the internal architecture for the BM25 + Reciprocal Rank Fusion
|
||||
(RRF) competence matching strategy with optional LLM-based Auto-Tagging, controlled by
|
||||
`matching.similarity.use_bm25_search` and `matching.similarity.use_auto_tagging`.
|
||||
|
||||
The goal is to provide a lexical alternative to embedding-based similarity that assigns
|
||||
near-zero scores when candidate competences do not literally overlap with the required
|
||||
competences, while still producing a normalized score compatible with the existing
|
||||
`Matcher` and `scorer` contracts.
|
||||
|
||||
Auto-Tagging optionally extends BM25 by having an LLM pre-expand each candidate's
|
||||
competence list with canonical forms of any required competences it already covers
|
||||
(synonyms, abbreviations, cross-language equivalents). This bridges the lexical gap for
|
||||
known skill aliases without persisting any data.
|
||||
|
||||
---
|
||||
|
||||
## Key Decisions
|
||||
|
||||
### 1. BM25 is per-skill only, not aggregate
|
||||
|
||||
**Decision:** `_bm25_rrf_similarity()` operates per required competence (like the existing
|
||||
`per_skill` strategy), not as an aggregate over all required competences.
|
||||
|
||||
**Rationale:**
|
||||
- Aggregate-style BM25 would require concatenating all required competences into a single
|
||||
query document, which loses the per-skill granularity needed for `matched_competences`
|
||||
and `missing_competences` diagnostic output.
|
||||
- Per-skill BM25 + RRF preserves the same output shape as `_per_skill_similarity()`,
|
||||
making it a drop-in replacement.
|
||||
|
||||
---
|
||||
|
||||
### 2. BM25 index is built once globally over all filtered candidates
|
||||
|
||||
**Decision:** One `Bm25Index` is built over the **union of all filtered candidates'
|
||||
competences** before the per-candidate scoring loop in `Matcher.match()`, not fresh per
|
||||
`compute_competence_similarity()` call. The index is passed in as `global_index` and each
|
||||
per-candidate call filters BM25 results down to that candidate's own competence set.
|
||||
|
||||
**Rationale:**
|
||||
- **IDF pathology fix:** With a per-person corpus (N = that person's competence count),
|
||||
BM25 IDF is `log((N − df + 0.5) / (df + 0.5))`. For N = 5–30 and any term that appears
|
||||
in most or all documents, this is 0 or negative. `rank_bm25` therefore returns zero or
|
||||
negative raw scores for typical real-world skill lists — making every BM25 match appear
|
||||
as zero before `max(0.0, raw)` clamping. A global corpus of all candidates' competences
|
||||
ensures N is large enough for IDF to carry meaningful signal.
|
||||
- **Still cheap:** Building one index over the deduplicated union of all candidates'
|
||||
competences is O(M) where M is the total unique competence count — still milliseconds.
|
||||
- **Correctness over caching:** The global index is not cached across MCP tool calls; it is
|
||||
rebuilt fresh for each `match_candidates` invocation, so stale state is impossible.
|
||||
|
||||
---
|
||||
|
||||
### 3. RRF with single ranked list (BM25 only)
|
||||
|
||||
**Decision:** For the initial implementation, `reciprocal_rank_fusion()` fuses a single
|
||||
ranked list (BM25 ranks only), effectively normalizing BM25 scores via the RRF formula.
|
||||
|
||||
**Formula (single list):**
|
||||
|
||||
```
|
||||
rrf_score(candidate) = 1 / (k + rank(candidate))
|
||||
```
|
||||
|
||||
where `k = 60` (standard constant, Cormack et al. 2009), and `rank` is 1-based.
|
||||
|
||||
**Output normalization:**
|
||||
The raw RRF scores sum to a value > 1 across all candidates. To map to [0.0, 1.0]:
|
||||
|
||||
```
|
||||
normalized = rrf_score(best_candidate) / rrf_score_at_rank_1
|
||||
= 1 / (k + rank) * (k + 1)
|
||||
= (k + 1) / (k + rank)
|
||||
```
|
||||
|
||||
For `k=60`, rank 1 → 1.0, rank 2 → 61/62 ≈ 0.984, rank 10 → 61/70 ≈ 0.871.
|
||||
|
||||
This means BM25 scores do NOT inherently drop to near-zero for adjacent candidates.
|
||||
To address this, RRF scores are **zeroed** if the underlying BM25 score is 0.0, because
|
||||
a BM25 score of 0 means **no token overlap at all**. This is the critical property that
|
||||
fixes the false-positive problem.
|
||||
|
||||
**Implementation:**
|
||||
```python
|
||||
def reciprocal_rank_fusion(
|
||||
ranked_lists: list[list[tuple[str, float]]],
|
||||
*,
|
||||
k: int = 60,
|
||||
) -> dict[str, float]:
|
||||
...
|
||||
# Skip lists with no BM25 signal at all
|
||||
if all(score == 0.0 for _, score in ranked_list):
|
||||
continue
|
||||
|
||||
# IMPORTANT: Keep (candidate, 0.0) entries in the ranked list for transparency,
|
||||
# but treat them as "no signal" for fusion. I.e., candidates with BM25==0.0 MUST
|
||||
# NOT receive a positive fused score just because they have a (low) rank at the
|
||||
# end of the list.
|
||||
for rank, (candidate, score) in enumerate(ranked_list, start=1):
|
||||
if score == 0.0:
|
||||
continue
|
||||
fused[candidate] += 1 / (k + rank)
|
||||
...
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 4. `best_match` reported per required competence
|
||||
|
||||
**Decision:** `best_match` in the output dict is the candidate with the highest BM25 score
|
||||
(rank 1 in the BM25 list). If no candidate has any token overlap (BM25 = 0 for all),
|
||||
`best_match = None` and `score = 0.0`.
|
||||
|
||||
---
|
||||
|
||||
### 5. Role similarity unchanged
|
||||
|
||||
**Decision:** `compute_role_similarity()` is always embedding-based, regardless of
|
||||
`use_bm25_search`.
|
||||
|
||||
**Rationale:**
|
||||
- Role names are short, single-phrase labels ("Backend Developer", "Data Engineer").
|
||||
- Embeddings handle multilingual and synonym variants better for roles.
|
||||
- BM25's exact-match strength is less valuable for role inference.
|
||||
|
||||
---
|
||||
|
||||
### 6. `rank_bm25` as the BM25 library
|
||||
|
||||
**Decision:** Use the `rank-bm25` PyPI package (`BM25Okapi` class).
|
||||
|
||||
**Rationale:**
|
||||
- Pure Python, no C extensions or native code.
|
||||
- Actively maintained, MIT licensed.
|
||||
- Implements BM25 Okapi (the standard variant) with configurable `k1` and `b` parameters.
|
||||
- Minimal footprint, no additional build steps.
|
||||
|
||||
**Tokenization:**
|
||||
Candidate and query texts are tokenized by lowercasing and splitting on non-word
|
||||
characters (whitespace, hyphens, slashes, parentheses, dots):
|
||||
|
||||
```python
|
||||
import re
|
||||
|
||||
def _tokenize(text: str) -> list[str]:
|
||||
return [t for t in re.split(r"[\W_]+", text.lower()) if t]
|
||||
```
|
||||
|
||||
This ensures "Progressive Web App (PWA)" tokenizes to
|
||||
`["progressive", "web", "app", "pwa"]`, and "CI/CD Pipeline" becomes
|
||||
`["ci", "cd", "pipeline"]`.
|
||||
|
||||
---
|
||||
|
||||
### 7. No changes to `Matcher`, `scorer`, or output format
|
||||
|
||||
**Decision:** `_bm25_rrf_similarity()` returns `dict[str, dict[str, object]]` in the
|
||||
identical shape to `_per_skill_similarity()`:
|
||||
|
||||
```python
|
||||
{
|
||||
"Python": {
|
||||
"score": 0.0,
|
||||
"best_match": None,
|
||||
"rationale": "BM25: no token overlap with any candidate competence.",
|
||||
},
|
||||
"Machine Learning": {
|
||||
"score": 0.0,
|
||||
"best_match": None,
|
||||
"rationale": "BM25: no token overlap with any candidate competence.",
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
The `Matcher.match()` loop, the `ScoredCapacity` model, the `compute_overall()` scorer,
|
||||
and all MCP tool output formatting remain unchanged.
|
||||
|
||||
---
|
||||
|
||||
### 8. LLM Auto-Tagging: ephemeral competence expansion
|
||||
|
||||
**Decision:** When `use_auto_tagging = true`, the `AutoTagger` issues a single LLM
|
||||
chat-completion call per candidate per scoring invocation to identify which required
|
||||
competences are already covered by the candidate's existing competences (via synonym,
|
||||
abbreviation, or cross-language equivalence). The response is a JSON list of canonical
|
||||
required-competence names to add to the candidate's working list. The expanded list is
|
||||
used only for that BM25 scoring call and is **never persisted**.
|
||||
|
||||
**Rationale:**
|
||||
- Auto-tagging directly addresses BM25's known limitation with lexical disjoint synonyms
|
||||
(e.g., "ML" for "Machine Learning", "Softwarearchitektur" for "Software Architecture").
|
||||
- Making the expansion ephemeral keeps the data model clean and audit-friendly.
|
||||
- Restricting additions to "names already covered by the candidate's existing entries"
|
||||
means the LLM cannot invent skills — it can only surface canonical equivalents.
|
||||
- LLM call cost is bounded: one call per candidate per `compute_competence_similarity()`
|
||||
invocation, with a compact prompt (required list + candidate list).
|
||||
|
||||
**LLM prompt schema:**
|
||||
```
|
||||
System: You are a skill-taxonomy assistant. Given a list of REQUIRED competences and a
|
||||
candidate's EXISTING competences, output ONLY the canonical names from REQUIRED
|
||||
that are already covered by one or more entries in EXISTING (via synonym,
|
||||
abbreviation, or cross-language equivalence). Do not invent new skills.
|
||||
|
||||
User: REQUIRED: ["Machine Learning", "Python", "CI/CD"]
|
||||
EXISTING: ["ML", "Python", "Jenkins", "Continuous Integration"]
|
||||
|
||||
Response (JSON): {"additions": ["Machine Learning", "CI/CD"]}
|
||||
```
|
||||
|
||||
**Azure OpenAI `response_format`:** `{"type": "json_object"}` is used to enforce
|
||||
structured JSON output, eliminating parsing failures.
|
||||
|
||||
**Error handling:** If the LLM call fails (timeout, API error, malformed JSON), the
|
||||
`AutoTagger` returns the original unmodified candidate list. BM25 scoring continues with
|
||||
no expansion — graceful degradation.
|
||||
|
||||
---
|
||||
|
||||
## Data Flow
|
||||
|
||||
```
|
||||
find_matching_capacities(...) / find_matching_tasks(...)
|
||||
│
|
||||
└─► Matcher.match(capacities, requirements)
|
||||
│
|
||||
├─ [use_bm25_search=True]
|
||||
│ Build ONE Bm25Index over all filtered candidates' competences (global corpus)
|
||||
│ → global_bm25_index (passed into every compute_competence_similarity call)
|
||||
│
|
||||
└─► for each candidate cap in filtered:
|
||||
SimilarityEngine.compute_competence_similarity(
|
||||
required=requirements.competences,
|
||||
candidate=cap.competences,
|
||||
global_index=global_bm25_index,
|
||||
)
|
||||
│
|
||||
├─ [use_bm25_search=False]
|
||||
│ _per_skill_similarity() / _aggregate_similarity()
|
||||
│ (embedding-based, unchanged)
|
||||
│
|
||||
└─ [use_bm25_search=True]
|
||||
│
|
||||
├─ [use_auto_tagging=True]
|
||||
│ AutoTagger.expand_competences(
|
||||
│ required=required,
|
||||
│ existing=candidate,
|
||||
│ )
|
||||
│ → expanded_candidate (ephemeral, not persisted)
|
||||
│
|
||||
└─► _bm25_rrf_similarity(required, expanded_candidate, global_index)
|
||||
│
|
||||
├─► global_index.rank(query=required_comp)
|
||||
│ → filter to candidate_set
|
||||
│ → list[(candidate_text, bm25_score)]
|
||||
│ (falls back to bm25_rank_competences() if no global_index)
|
||||
│
|
||||
└─► reciprocal_rank_fusion(ranked_lists)
|
||||
→ dict[required_comp → rrf_score]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Module Structure
|
||||
|
||||
```
|
||||
src/teamlandkarte_mcp/matching/
|
||||
├── auto_tagger.py # AutoTagger class + expand_competences()
|
||||
├── bm25.py # Bm25Index dataclass + bm25_rank_competences()
|
||||
├── rrf.py # reciprocal_rank_fusion()
|
||||
├── matcher.py # unchanged
|
||||
├── scorer.py # unchanged
|
||||
└── similarity.py # extended: _bm25_rrf_similarity(), auto-tag branch, flag routing
|
||||
|
||||
src/teamlandkarte_mcp/azure/
|
||||
└── openai_client.py # extended: chat_completion(), chat_deployment, llm_api_key
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## `bm25.py` — Public Interface
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class Bm25Index:
|
||||
"""In-memory BM25 index over a corpus of competence strings.
|
||||
|
||||
Intended to be built once over the union of all candidates' competences
|
||||
(the global corpus) so that IDF values are meaningful. Each per-candidate
|
||||
scoring call queries the global index and filters results to that candidate's
|
||||
own competence set.
|
||||
|
||||
Args:
|
||||
corpus: List of competence strings to index.
|
||||
|
||||
Notes:
|
||||
Tokenization lowercases and splits on non-word characters.
|
||||
An empty corpus produces an index that scores all queries as 0.0.
|
||||
Raw BM25 scores are clamped to ``max(0.0, raw)`` — ubiquitous terms
|
||||
whose IDF is negative simply produce 0.0.
|
||||
"""
|
||||
corpus: list[str]
|
||||
|
||||
def rank(self, query: str) -> list[tuple[str, float]]:
|
||||
"""Rank corpus documents for a single query string.
|
||||
|
||||
Args:
|
||||
query: Required competence text used as the BM25 query.
|
||||
|
||||
Returns:
|
||||
List of (candidate_text, bm25_score) sorted by score descending.
|
||||
Entries with score 0.0 are included (caller decides cutoff).
|
||||
Scores are clamped to max(0.0, raw); negative IDF yields 0.0.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
def bm25_rank_competences(
|
||||
required: str,
|
||||
candidates: list[str],
|
||||
) -> list[tuple[str, float]]:
|
||||
"""Rank candidate competences against a single required competence.
|
||||
|
||||
Convenience wrapper: builds a temporary ``Bm25Index`` and ranks the query.
|
||||
Suitable for standalone use and tests; in production the global index built
|
||||
by ``Matcher`` is preferred (see ``Bm25Index`` notes on IDF correctness).
|
||||
|
||||
Args:
|
||||
required: The required competence text (BM25 query).
|
||||
candidates: Candidate competence strings to rank.
|
||||
|
||||
Returns:
|
||||
Ranked list of (candidate_text, bm25_score), descending by score.
|
||||
"""
|
||||
...
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## `rrf.py` — Public Interface
|
||||
|
||||
```python
|
||||
def reciprocal_rank_fusion(
|
||||
ranked_lists: list[list[tuple[str, float]]],
|
||||
*,
|
||||
k: int = 60,
|
||||
) -> dict[str, float]:
|
||||
"""Fuse multiple ranked lists using Reciprocal Rank Fusion.
|
||||
|
||||
For each candidate present in any list, the fused score is:
|
||||
|
||||
score(c) = Σ_i 1 / (k + rank_i(c))
|
||||
|
||||
where rank_i(c) is the 1-based rank of candidate c in list i.
|
||||
|
||||
Candidates with zero BM25 score in ALL contributing lists receive
|
||||
a fused score of 0.0 (no token overlap signal).
|
||||
|
||||
Scores are normalized so the top-ranked candidate receives 1.0.
|
||||
|
||||
Args:
|
||||
ranked_lists: One or more ranked lists of (text, score) pairs.
|
||||
Each list MUST be sorted by score descending.
|
||||
Lists with all-zero scores are skipped (no signal).
|
||||
k: RRF smoothing constant (default 60, Cormack et al. 2009).
|
||||
Higher k flattens rank differences; lower k amplifies them.
|
||||
|
||||
Returns:
|
||||
Mapping of candidate text → normalized fused score in [0.0, 1.0].
|
||||
Returns empty dict if all input lists are empty or all-zero.
|
||||
"""
|
||||
...
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## `auto_tagger.py` — Public Interface
|
||||
|
||||
```python
|
||||
class AutoTagger:
|
||||
"""LLM-based competence expander for BM25 pre-processing.
|
||||
|
||||
Calls an Azure OpenAI chat model to identify which required competences
|
||||
are already covered by the candidate's existing competences (via synonym,
|
||||
abbreviation, or cross-language equivalence) and returns the canonical
|
||||
required-competence names as additions.
|
||||
|
||||
All LLM calls use ``response_format={"type": "json_object"}`` to ensure
|
||||
parseable output. On any LLM or parsing error, the original candidate list
|
||||
is returned unchanged (graceful degradation).
|
||||
|
||||
Args:
|
||||
client: An ``AzureOpenAIClient`` instance configured with a chat
|
||||
deployment (``chat_deployment`` and ``llm_api_key``).
|
||||
"""
|
||||
|
||||
def __init__(self, client: AzureOpenAIClient) -> None: ...
|
||||
|
||||
def expand_competences(
|
||||
self,
|
||||
required: list[str],
|
||||
existing: list[str],
|
||||
) -> list[str]:
|
||||
"""Expand a candidate's competence list with covered required terms.
|
||||
|
||||
Sends a prompt to the LLM asking: given ``required`` competences and
|
||||
the candidate's ``existing`` competences, which required competences
|
||||
are already implicitly covered by the existing entries?
|
||||
|
||||
The returned list is ``existing + additions`` where ``additions``
|
||||
are the canonical required-competence names identified by the LLM.
|
||||
|
||||
The result is **ephemeral**: it is used only for the current BM25
|
||||
scoring call and is never written back to the database.
|
||||
|
||||
Args:
|
||||
required: List of required competence names (BM25 query terms).
|
||||
existing: Candidate's current competence list.
|
||||
|
||||
Returns:
|
||||
Combined list ``existing + additions``. If the LLM call fails or
|
||||
returns no additions, returns ``existing`` unchanged.
|
||||
"""
|
||||
...
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Configuration
|
||||
|
||||
### `SimilarityConfig` (updated)
|
||||
|
||||
```python
|
||||
@dataclass(frozen=True)
|
||||
class SimilarityConfig:
|
||||
"""Similarity engine configuration."""
|
||||
|
||||
embedding_model: str = "text-embedding-3-large"
|
||||
embedding_dimensions: int = 3072
|
||||
strategy: str = "per_skill"
|
||||
use_bm25_search: bool = False
|
||||
"""When True, BM25 + RRF replaces embedding-based competence similarity.
|
||||
Role similarity is always embedding-based.
|
||||
Default: False (preserves existing behavior).
|
||||
"""
|
||||
use_auto_tagging: bool = False
|
||||
"""When True (and use_bm25_search is also True), calls the LLM AutoTagger
|
||||
before each BM25 scoring call to expand the candidate's competence list
|
||||
with canonical equivalents of required competences it already covers.
|
||||
The expansion is ephemeral and never persisted.
|
||||
Default: False.
|
||||
"""
|
||||
```
|
||||
|
||||
### `AzureOpenAIConfig` (updated)
|
||||
|
||||
```python
|
||||
@dataclass(frozen=True)
|
||||
class AzureOpenAIConfig:
|
||||
"""Azure OpenAI client configuration."""
|
||||
|
||||
# ... existing fields ...
|
||||
chat_deployment: str = ""
|
||||
"""Azure deployment name of the chat model used for auto-tagging
|
||||
(e.g. ``"gpt-4.1"``). Required only when ``use_auto_tagging = true``.
|
||||
"""
|
||||
llm_api_key: str = ""
|
||||
"""Azure OpenAI API key for the chat deployment. Read from the
|
||||
``AZURE_OPENAI_LLM_API_KEY`` environment variable.
|
||||
Required when ``use_auto_tagging = true``; raises ``ConfigurationError``
|
||||
at startup if absent.
|
||||
"""
|
||||
```
|
||||
|
||||
### Config parser (`config.toml` keys)
|
||||
|
||||
```toml
|
||||
[matching.similarity]
|
||||
strategy = "per_skill"
|
||||
use_bm25_search = false # new — enables BM25+RRF competence matching
|
||||
use_auto_tagging = false # new — enables LLM pre-expansion before BM25
|
||||
|
||||
[azure_openai]
|
||||
# ... existing keys ...
|
||||
chat_deployment = "gpt-4.1" # new — required when use_auto_tagging = true
|
||||
```
|
||||
|
||||
### Environment variables
|
||||
|
||||
| Variable | Required when | Description |
|
||||
|----------|--------------|-------------|
|
||||
| `AZURE_OPENAI_LLM_API_KEY` | `use_auto_tagging = true` | API key for the Azure OpenAI chat deployment used in auto-tagging. Server raises `ConfigurationError` at startup if absent and `use_auto_tagging = true`. |
|
||||
|
||||
---
|
||||
|
||||
## BM25 Score Behavior
|
||||
|
||||
### Example: Query "Python" against candidate competences
|
||||
|
||||
| Candidate | Tokens | BM25 Score | RRF Score (k=60) |
|
||||
|-----------|--------|------------|-----------------|
|
||||
| Python | ["python"] | ~3.5 | 1.000 |
|
||||
| Python (advanced) | ["python", "advanced"] | ~2.1 | 0.984 |
|
||||
| JavaScript | ["javascript"] | 0.0 | **0.000** |
|
||||
| TypeScript | ["typescript"] | 0.0 | **0.000** |
|
||||
| Node.js | ["node", "js"] | 0.0 | **0.000** |
|
||||
|
||||
Candidates without any shared token receive **0.0** — the false-positive problem is
|
||||
eliminated.
|
||||
|
||||
### Example: Query "Machine Learning" against candidate competences
|
||||
|
||||
| Candidate | Tokens | BM25 Score | RRF Score |
|
||||
|-----------|--------|------------|-----------|
|
||||
| Machine Learning | ["machine", "learning"] | ~4.0 | 1.000 |
|
||||
| Deep Learning | ["deep", "learning"] | ~1.8 | 0.984 |
|
||||
| Machine Learning (Python) | ["machine", "learning", "python"] | ~3.2 | 0.492 |
|
||||
| JavaScript | ["javascript"] | 0.0 | **0.000** |
|
||||
|
||||
---
|
||||
|
||||
## Known Limitations
|
||||
|
||||
### 1. Conceptual synonyms with zero token overlap score 0.0
|
||||
|
||||
BM25 requires at least one shared token between query and document. Skill pairs that are
|
||||
semantically equivalent but lexically disjoint will always score 0.0:
|
||||
|
||||
| Required competence | Candidate competence | Shared tokens | BM25 score |
|
||||
|---------------------|----------------------|---------------|------------|
|
||||
| Data Science | Machine Learning | none | **0.000** |
|
||||
| Data Science | Artificial Intelligence | none | **0.000** |
|
||||
| Frontend Development | UI Engineering | none | **0.000** |
|
||||
| Agile | Scrum | none | **0.000** |
|
||||
| Softwarearchitektur | Software Architecture | none | **0.000** |
|
||||
|
||||
These are **false negatives** — real skill matches that BM25 misses entirely. Embeddings
|
||||
handle all of these correctly via semantic similarity.
|
||||
|
||||
**Consequence:** BM25 mode is best suited to datasets where competence names are
|
||||
**canonical, consistent, and in a single language**. If profiles mix German and English
|
||||
skill names, or use informal abbreviations ("ML" for "Machine Learning"), BM25 will
|
||||
produce false negatives that embeddings would have caught.
|
||||
|
||||
**Mitigation:** Enable `use_auto_tagging = true`. The LLM will bridge synonym and
|
||||
cross-language gaps by pre-expanding the candidate list before BM25 scoring. This
|
||||
addresses limitations 1, 2, and 3 at the cost of one LLM call per candidate per scoring
|
||||
invocation.
|
||||
|
||||
---
|
||||
|
||||
### 2. Abbreviations and acronyms
|
||||
|
||||
Short forms that do not share tokens with the expanded form score 0.0:
|
||||
|
||||
| Required | Candidate | BM25 score |
|
||||
|----------|-----------|------------|
|
||||
| Machine Learning | ML | **0.000** |
|
||||
| Continuous Integration | CI | **0.000** |
|
||||
| Natural Language Processing | NLP | **0.000** |
|
||||
|
||||
Partial exception: "CI/CD" tokenizes to `["ci", "cd"]`, so a query "CI/CD Pipeline"
|
||||
would find partial overlap with a candidate "CI/CD", but not with "Continuous
|
||||
Integration/Continuous Delivery".
|
||||
|
||||
**Mitigation:** `use_auto_tagging = true` (see limitation 1).
|
||||
|
||||
---
|
||||
|
||||
### 3. Cross-language skill names
|
||||
|
||||
A required competence in German and a candidate competence in English (or vice versa)
|
||||
score 0.0 even when they describe the same skill:
|
||||
|
||||
| Required | Candidate | BM25 score |
|
||||
|----------|-----------|------------|
|
||||
| Softwarearchitektur | Software Architecture | **0.000** |
|
||||
| Maschinelles Lernen | Machine Learning | **0.000** |
|
||||
| Datenwissenschaft | Data Science | **0.000** |
|
||||
|
||||
This is a significant limitation for German-language skill databases. Embeddings handle
|
||||
cross-language pairs well; BM25 does not.
|
||||
|
||||
**Mitigation:** `use_auto_tagging = true` (see limitation 1).
|
||||
|
||||
---
|
||||
|
||||
### 4. RRF score compression near 1.0
|
||||
|
||||
Because the normalization formula is `(k+1) / (k+rank)` with `k=60`, ranks 1–5 all
|
||||
compress into the range `[0.871, 1.0]`. There is little score differentiation between
|
||||
the top-ranked candidates. This is intentional (rank differences matter less than
|
||||
presence/absence of token overlap) but means BM25 mode should not be used to produce
|
||||
fine-grained competence similarity scores — only to separate matching (score > 0) from
|
||||
non-matching (score = 0) candidates.
|
||||
|
||||
---
|
||||
|
||||
### 5. Small per-person corpus causes IDF pathology — mitigated by global index
|
||||
|
||||
BM25's IDF term rewards tokens that are rare across the corpus. When the BM25 index was
|
||||
built per candidate (N = that person's competence count ≈ 5–30), any term appearing in
|
||||
most documents had IDF ≤ 0, causing `rank_bm25` to return negative raw scores. After
|
||||
`max(0.0, raw)` clamping, all candidates scored 0.0 — making the BM25 signal useless.
|
||||
|
||||
**Fix (implemented):** The `Bm25Index` is now built once over the **union of all filtered
|
||||
candidates' competences** in `Matcher.match()` before the per-candidate loop (see Key
|
||||
Decision #2). This makes N equal to the total unique competence count across all
|
||||
candidates, giving IDF proper signal. Per-candidate scoring filters the global results to
|
||||
only that candidate's competence set.
|
||||
|
||||
**Residual limitation:** If the filtered candidate pool itself is very small (e.g., 2–3
|
||||
candidates with few unique competences), IDF may still be weak. In practice this is
|
||||
negligible since matching runs are always invoked against a meaningful pool of candidates.
|
||||
|
||||
---
|
||||
|
||||
## Test Strategy
|
||||
|
||||
### Unit tests
|
||||
|
||||
1. **`tests/test_bm25.py`**
|
||||
- `Bm25Index` with empty corpus → all scores 0.0
|
||||
- Exact match → score > 0
|
||||
- No token overlap → score == 0.0
|
||||
- Partial token overlap (multi-token queries) → partial score
|
||||
- Case-insensitive matching ("Python" == "python")
|
||||
- Tokenization edge cases: hyphens, parentheses, slashes
|
||||
|
||||
2. **`tests/test_rrf.py`**
|
||||
- Single list, single candidate → score 1.0
|
||||
- Single list, multiple candidates → scores normalized to [0, 1]
|
||||
- All-zero list → returns empty / all zeros
|
||||
- `k` parameter effect on score distribution
|
||||
|
||||
3. **`tests/test_auto_tagger.py`** (new)
|
||||
- LLM returns valid additions → `expand_competences()` appends them to existing list
|
||||
- LLM returns empty additions → original list returned unchanged
|
||||
- LLM call raises exception → original list returned unchanged (graceful degradation)
|
||||
- LLM returns malformed JSON → original list returned unchanged
|
||||
- Additions from LLM that are NOT in `required` are ignored
|
||||
- `expand_competences()` never duplicates entries already in `existing`
|
||||
- All tests use a mocked `AzureOpenAIClient` (no real LLM calls)
|
||||
|
||||
4. **`tests/test_similarity.py`** (extended)
|
||||
- `compute_competence_similarity()` with `use_bm25_search=True`
|
||||
- Required "Python", candidate ["JavaScript"] → score 0.0
|
||||
- Required "Python", candidate ["Python"] → score 1.0
|
||||
- Required "Python" + "Machine Learning", candidate ["JavaScript", "TypeScript"]
|
||||
→ both scores 0.0, overall competence_score 0.0
|
||||
- With `use_bm25_search=True, use_auto_tagging=True`: mocked `AutoTagger` injects
|
||||
"Machine Learning" into candidate ["ML"] → score > 0
|
||||
|
||||
### Integration test
|
||||
|
||||
5. **`tests/test_matcher_integration.py`** (extended)
|
||||
- With `use_bm25_search=True`: candidate with exact skills scores higher than
|
||||
candidate with only semantically adjacent skills.
|
||||
- The false-positive scenario (Python/ML vs JavaScript/TypeScript) is explicitly
|
||||
tested.
|
||||
- With `use_bm25_search=True, use_auto_tagging=True` (mocked LLM): candidate with
|
||||
"ML" matches required "Machine Learning" after auto-tag expansion.
|
||||
@@ -1,288 +0,0 @@
|
||||
# Change Proposal: BM25 + RRF Competence Matching
|
||||
|
||||
- **Change ID**: `add-bm25-rrf-competence-matching`
|
||||
- **Status**: Proposed
|
||||
- **Target**: `teamlandkarte-mcp`
|
||||
- **Author**: Thomas Handke
|
||||
- **Date**: 2026-02-26
|
||||
|
||||
## Summary
|
||||
|
||||
Add an optional BM25-based competence matching strategy as an alternative to the existing
|
||||
embedding-based similarity approach. When enabled, the following pipeline replaces
|
||||
embedding-based competence similarity:
|
||||
|
||||
1. **Auto-Tagging (LLM):** Before BM25 scoring, an LLM compares the required competences
|
||||
against each candidate's existing competences and suggests canonical additions
|
||||
(synonyms, abbreviations, cross-language equivalents). The expanded competence list
|
||||
is ephemeral — it is used only for the current scoring call and is never persisted.
|
||||
2. **BM25 Ranking:** The (expanded) candidate competences are indexed with BM25. Each
|
||||
required competence is used as a query to rank the candidates.
|
||||
3. **RRF Fusion:** Reciprocal Rank Fusion normalizes the BM25 ranked list into a score
|
||||
in [0, 1] per required competence.
|
||||
|
||||
The feature is controlled by two new booleans in `config.toml`:
|
||||
- `use_bm25_search` (default `false`): enables BM25 + RRF.
|
||||
- `use_auto_tagging` (default `false`): enables LLM-based competence expansion before
|
||||
BM25 scoring. Strongly recommended when `use_bm25_search = true`.
|
||||
|
||||
## Why
|
||||
|
||||
The current embedding-based (`per_skill` / `aggregate`) strategy has a known weakness:
|
||||
it assigns non-trivial scores to candidates whose competences are only _semantically
|
||||
adjacent_ to the required skills but not actually matching.
|
||||
|
||||
**Example (observed in production):**
|
||||
Searching for **Python** + **Machine Learning**, a candidate with only
|
||||
**JavaScript, TypeScript, Node.js, Vue.js, Web Architekturen** received a score of **0.392**
|
||||
because the embedding model considers all programming languages semantically related.
|
||||
|
||||
BM25 (Best Match 25) is a classical lexical ranking algorithm that scores documents by
|
||||
**term overlap** between query and document. It is immune to the semantic drift problem
|
||||
because it only counts tokens that literally occur in the candidate text. BM25 will assign
|
||||
**0** to "JavaScript" when the query is "Python".
|
||||
|
||||
### When BM25 outperforms embeddings
|
||||
|
||||
| Scenario | Embeddings | BM25 |
|
||||
|----------|-----------|------|
|
||||
| Query "Python", candidate has "Python" | High ✓ | High ✓ |
|
||||
| Query "Python", candidate has "JavaScript" | Medium ✗ | Zero ✓ |
|
||||
| Query "Machine Learning", candidate has "ML" | High ✓ | Low ✗ |
|
||||
| Query "Softwarearchitektur", candidate has "software architecture" | High ✓ | Low ✗ |
|
||||
| Query "React", candidate has "React.js" | High ✓ | Medium ~ |
|
||||
|
||||
BM25 excels at exact or near-exact skill names. Embeddings excel at synonyms and
|
||||
cross-language variants (e.g., German/English). The optional `use_auto_tagging` feature
|
||||
bridges BM25's synonym and cross-language gap by having an LLM pre-expand the candidate
|
||||
competence list — making the combination of BM25 + Auto-Tagging competitive with
|
||||
embeddings while still eliminating false positives from semantic drift.
|
||||
|
||||
### Why RRF for fusion
|
||||
|
||||
Reciprocal Rank Fusion merges multiple ranked lists into one without requiring score
|
||||
normalization. For each required competence query, we obtain a ranked list of candidate
|
||||
competences. RRF then assigns each candidate a fused score based on its (1-based) rank in
|
||||
one or more lists.
|
||||
|
||||
RRF fused score formula (Cormack et al. 2009):
|
||||
|
||||
```
|
||||
rrf_score(candidate) = Σ_i 1 / (k + rank_i(candidate))
|
||||
```
|
||||
|
||||
RRF is robust to score scale differences and has been widely validated in information
|
||||
retrieval research as a high-quality fusion method.
|
||||
|
||||
## What Changes
|
||||
|
||||
### 1. New config parameter `matching.similarity.use_bm25_search` (boolean, default `false`)
|
||||
|
||||
- When `false` (default): existing embedding-based strategy is used unchanged.
|
||||
- When `true`: BM25 + RRF replaces embedding-based competence similarity.
|
||||
Role similarity is **always** embedding-based (unchanged).
|
||||
|
||||
### 2. New config parameter `matching.similarity.use_auto_tagging` (boolean, default `false`)
|
||||
|
||||
- Only applies when `use_bm25_search = true`. When `false`, BM25 runs on the raw
|
||||
candidate competence list (no LLM expansion).
|
||||
- When `true`: before each BM25 scoring call, an LLM expands the candidate's competence
|
||||
list with canonical forms of any required competences that are already covered by the
|
||||
candidate's existing entries (via synonym, abbreviation, or cross-language equivalence).
|
||||
- The expanded list is ephemeral — it is not written back to the database and does not
|
||||
affect any other operation.
|
||||
|
||||
### 3. New config parameter `azure_openai.chat_deployment` (string)
|
||||
|
||||
- Azure deployment name of the chat model used for auto-tagging (e.g. `"gpt-4.1"`).
|
||||
- Required only when `use_auto_tagging = true`.
|
||||
- API key is read from environment variable `AZURE_OPENAI_LLM_API_KEY`.
|
||||
|
||||
### 4. `AzureOpenAIClient` extended: LLM chat capability restored
|
||||
|
||||
- Add `chat_completion(system: str, user: str) -> str` method supporting structured JSON
|
||||
responses via Azure OpenAI chat completions API.
|
||||
- Add `chat_deployment` and `llm_api_key` constructor parameters.
|
||||
- This restores LLM capability removed in a previous change, now scoped exclusively
|
||||
to auto-tagging.
|
||||
|
||||
### 5. New module `src/teamlandkarte_mcp/matching/auto_tagger.py`
|
||||
|
||||
Implements:
|
||||
- `AutoTagger`: class that wraps the LLM client and executes the tagging prompt.
|
||||
- `expand_competences()`: given `required: list[str]` and `existing: list[str]`,
|
||||
returns `list[str]` — the combined list `existing + additions`, where `additions`
|
||||
are the canonical required-competence names that the LLM identifies as already
|
||||
covered by `existing` entries.
|
||||
- Structured JSON output schema enforced via the Azure OpenAI `response_format` parameter.
|
||||
- Google Docstrings on all public symbols.
|
||||
|
||||
### 6. New module `src/teamlandkarte_mcp/matching/bm25.py`
|
||||
|
||||
Implements:
|
||||
- `Bm25Index`: builds an in-memory BM25 index over a corpus of competence strings.
|
||||
Designed to be instantiated once over the global candidate pool (see item 8a below)
|
||||
so IDF values are meaningful. Raw scores are clamped to `max(0.0, raw)` — terms with
|
||||
negative IDF produce 0.0 rather than a spurious tiny positive value.
|
||||
- `bm25_rank_competences()`: for a single required competence query, returns a ranked list
|
||||
of `(candidate_text, score)` tuples. Convenience wrapper for standalone/test use.
|
||||
- Pure Python, no external runtime dependencies beyond `rank_bm25` from PyPI.
|
||||
|
||||
### 7. New module `src/teamlandkarte_mcp/matching/rrf.py`
|
||||
|
||||
Implements:
|
||||
- `reciprocal_rank_fusion()`: merges one or more ranked lists into a final score dict.
|
||||
- `k` constant (default `60`, standard RRF value from Cormack et al. 2009).
|
||||
- Returns `dict[str, float]` mapping candidate text → fused score in `(0, 1]`
|
||||
(top-ranked candidate receives exactly `1.0`).
|
||||
|
||||
### 8. `SimilarityEngine.compute_competence_similarity()` extended
|
||||
|
||||
- Reads `use_bm25_search` and `use_auto_tagging` flags.
|
||||
- Accepts an optional `global_index: Bm25Index | None` parameter (passed in by `Matcher`).
|
||||
- When `use_bm25_search=True`:
|
||||
1. If `use_auto_tagging=True`: calls `AutoTagger.expand_competences()` to build the
|
||||
expanded candidate list.
|
||||
2. Delegates to `_bm25_rrf_similarity()` with the (optionally expanded) candidate list
|
||||
and the global index.
|
||||
- `_bm25_rrf_similarity()` uses `global_index.rank(req)` filtered to the candidate's
|
||||
competence set when `global_index` is provided, otherwise falls back to
|
||||
`bm25_rank_competences(req, candidate)`.
|
||||
- Output shape is **identical** to existing strategies:
|
||||
`dict[required_competence → {score, best_match, rationale}]`
|
||||
- Preserves the `Matcher` and scorer contracts unchanged.
|
||||
|
||||
### 8a. `Matcher.match()` builds a global `Bm25Index` before the per-candidate loop
|
||||
|
||||
When `use_bm25_search=True` and the filtered candidate pool is non-empty, `Matcher.match()`
|
||||
collects the deduplicated union of all filtered candidates' competences and builds one
|
||||
`Bm25Index` over that global corpus before the loop. The same index is passed to every
|
||||
`compute_competence_similarity()` call.
|
||||
|
||||
**Why this matters:** With a per-person corpus (N ≈ 5–30 entries), BM25 IDF is 0 or
|
||||
negative for every term — all raw scores collapse to ≤ 0, and `max(0.0, raw)` clamping
|
||||
makes every match appear as 0.0. A global corpus ensures N is large enough for IDF to
|
||||
carry real signal.
|
||||
|
||||
### 9. `SimilarityConfig` dataclass extended
|
||||
|
||||
New fields:
|
||||
- `use_bm25_search: bool = False`
|
||||
- `use_auto_tagging: bool = False`
|
||||
|
||||
### 10. `AzureOpenAIConfig` dataclass extended
|
||||
|
||||
New fields:
|
||||
- `chat_deployment: str = ""`
|
||||
- (API key via `AZURE_OPENAI_LLM_API_KEY` env var)
|
||||
|
||||
### 11. `config.py` parser extended
|
||||
|
||||
Reads `matching.similarity.use_bm25_search`, `matching.similarity.use_auto_tagging`,
|
||||
and `azure_openai.chat_deployment` from TOML. Reads `AZURE_OPENAI_LLM_API_KEY` from
|
||||
environment when `use_auto_tagging = true`.
|
||||
|
||||
### 12. `config.toml.example` updated
|
||||
|
||||
Documents all three new keys with inline comments.
|
||||
|
||||
### 13. Server initialization in `mcp_server.py`
|
||||
|
||||
- Passes `use_bm25_search` and `use_auto_tagging` flags when constructing `SimilarityEngine`.
|
||||
- When `use_auto_tagging=True`: constructs `AutoTagger` with the LLM-capable
|
||||
`AzureOpenAIClient` and passes it to `SimilarityEngine`.
|
||||
|
||||
### 14. Documentation updated
|
||||
|
||||
- `docs/architecture.md`: new section on BM25/RRF/Auto-Tagging strategy.
|
||||
- `docs/semantic_similarity_tradeoffs.md`: comparison table updated with BM25+AutoTag entry.
|
||||
- `docs/assistant_system_prompt.md`: notes on score interpretation with BM25 and auto-tagging.
|
||||
- `README.md`: all new config parameters documented.
|
||||
|
||||
### 15. Legacy/deprecated code removed
|
||||
|
||||
- Remove `debug_similarity_analysis.py`
|
||||
- Remove `debug_categorization.py`
|
||||
- Remove `SEMANTIC_SIMILARITY_ANALYSIS.md`
|
||||
- Remove `SCORE_BUG_FIX.md`
|
||||
|
||||
### 16. Tests
|
||||
|
||||
- `tests/test_bm25.py`: unit tests for `Bm25Index` and `bm25_rank_competences()`.
|
||||
- `tests/test_rrf.py`: unit tests for `reciprocal_rank_fusion()`.
|
||||
- `tests/test_auto_tagger.py`: unit tests for `AutoTagger.expand_competences()` (mocked LLM).
|
||||
- `tests/test_similarity.py`: extend with BM25/RRF and auto-tagging strategy tests.
|
||||
- `tests/test_matcher_integration.py`: integration test for the full pipeline
|
||||
(auto-tag → BM25 → RRF) in both matching directions.
|
||||
|
||||
## Impact
|
||||
|
||||
### Affected specs
|
||||
|
||||
- `matching-tools` (competence scoring path)
|
||||
|
||||
### Affected code
|
||||
|
||||
| File | Change |
|
||||
|------|--------|
|
||||
| `src/teamlandkarte_mcp/matching/similarity.py` | Add `_bm25_rrf_similarity()`, `global_index` param, auto-tag branch, read flags |
|
||||
| `src/teamlandkarte_mcp/matching/matcher.py` | Build global `Bm25Index` before per-candidate loop; pass as `global_index` |
|
||||
| `src/teamlandkarte_mcp/matching/bm25.py` | **New** |
|
||||
| `src/teamlandkarte_mcp/matching/rrf.py` | **New** |
|
||||
| `src/teamlandkarte_mcp/matching/auto_tagger.py` | **New** |
|
||||
| `src/teamlandkarte_mcp/azure/openai_client.py` | Restore `chat_completion()`; add `chat_deployment` + `llm_api_key` params |
|
||||
| `src/teamlandkarte_mcp/config.py` | `SimilarityConfig.use_bm25_search`, `use_auto_tagging`; `AzureOpenAIConfig.chat_deployment` |
|
||||
| `src/teamlandkarte_mcp/mcp_server.py` | Pass flags + `AutoTagger` to `SimilarityEngine` |
|
||||
| `config.toml.example` | Document all new keys |
|
||||
| `docs/architecture.md` | BM25/RRF/Auto-Tagging section |
|
||||
| `docs/semantic_similarity_tradeoffs.md` | Updated comparison with BM25+AutoTag entry |
|
||||
| `docs/assistant_system_prompt.md` | Score notes for BM25 and auto-tagging modes |
|
||||
| `README.md` | All new config parameters |
|
||||
| `tests/test_bm25.py` | **New** |
|
||||
| `tests/test_rrf.py` | **New** |
|
||||
| `tests/test_auto_tagger.py` | **New** |
|
||||
| `tests/test_similarity.py` | Extended |
|
||||
| `tests/test_matcher_integration.py` | Extended |
|
||||
| `debug_similarity_analysis.py` | **Removed** |
|
||||
| `debug_categorization.py` | **Removed** |
|
||||
| `SEMANTIC_SIMILARITY_ANALYSIS.md` | **Removed** |
|
||||
| `SCORE_BUG_FIX.md` | **Removed** |
|
||||
|
||||
### Breaking changes
|
||||
|
||||
None. The `use_bm25_search = false` and `use_auto_tagging = false` defaults preserve all
|
||||
existing behavior. The `overall_score` computation (weighted mean of `role_score` and
|
||||
`competence_score`) is unchanged.
|
||||
|
||||
### New environment variables
|
||||
|
||||
| Variable | Required when | Description |
|
||||
|----------|--------------|-------------|
|
||||
| `AZURE_OPENAI_LLM_API_KEY` | `use_auto_tagging = true` | Azure OpenAI API key for the chat model used in auto-tagging. The server raises a configuration error at startup if this key is absent and `use_auto_tagging = true`. |
|
||||
|
||||
### New dependencies
|
||||
|
||||
- `rank-bm25` (PyPI): pure-Python BM25 implementation, no native code, no external services.
|
||||
Adds to `pyproject.toml` `dependencies`.
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
### A. Hard-code exact-match bonus in per_skill
|
||||
|
||||
Simpler but not principled: would require special-casing the scorer instead of a clean
|
||||
strategy abstraction.
|
||||
|
||||
### B. Hybrid embedding + BM25 in a single `_bm25_rrf_similarity()` call
|
||||
|
||||
Combining both signals via RRF is architecturally clean and naturally extensible. However,
|
||||
to keep this change focused and reviewable, the initial implementation uses BM25-only RRF.
|
||||
A hybrid mode can be added as a follow-up change.
|
||||
|
||||
### C. Replace embeddings entirely
|
||||
|
||||
Not desirable. Embeddings are essential for role inference and handle multilingual /
|
||||
synonymous skill names (e.g., "Softwarearchitektur" ↔ "Software Architecture").
|
||||
|
||||
## Open Questions
|
||||
|
||||
None. The design is sufficiently clear to proceed.
|
||||
-266
@@ -1,266 +0,0 @@
|
||||
# Spec Delta: BM25 + RRF Competence Matching
|
||||
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: BM25 + RRF optional competence matching strategy
|
||||
|
||||
The system SHALL support an optional BM25-based competence matching strategy controlled
|
||||
by `matching.similarity.use_bm25_search` in `config.toml`. When enabled, BM25 + RRF
|
||||
replaces embedding-based competence similarity for `find_matching_capacities` and
|
||||
`find_matching_tasks`. Role similarity SHALL always remain embedding-based.
|
||||
|
||||
#### Scenario: Candidate without matching skills receives zero competence score
|
||||
|
||||
- **GIVEN** `use_bm25_search = true` in `config.toml`
|
||||
- **AND** a task requires competences `["Python", "Machine Learning"]`
|
||||
- **AND** a candidate has competences `["JavaScript", "TypeScript", "Node.js", "Vue.js"]`
|
||||
- **WHEN** the system computes `competence_score` for that candidate
|
||||
- **THEN** `competence_score == 0.0`
|
||||
- **AND** the candidate is categorized as "Irrelevant" or "Low" based on `overall_score`
|
||||
|
||||
#### Scenario: Candidate with exact matching skills receives high competence score
|
||||
|
||||
- **GIVEN** `use_bm25_search = true` in `config.toml`
|
||||
- **AND** a task requires competences `["Python", "Machine Learning"]`
|
||||
- **AND** a candidate has competences `["Python", "Machine Learning", "Pandas"]`
|
||||
- **WHEN** the system computes `competence_score` for that candidate
|
||||
- **THEN** `competence_score > 0.8`
|
||||
- **AND** the candidate scores significantly higher than a candidate with no token overlap
|
||||
|
||||
#### Scenario: BM25 mode is disabled by default
|
||||
|
||||
- **GIVEN** no `use_bm25_search` key in `config.toml` (or `use_bm25_search = false`)
|
||||
- **WHEN** the system computes competence similarity
|
||||
- **THEN** the existing embedding-based strategy is used
|
||||
- **AND** behavior is bit-for-bit identical to the pre-change implementation
|
||||
|
||||
### Requirement: BM25 index ranks candidates by token overlap per required competence
|
||||
|
||||
The `Bm25Index` SHALL build an in-memory BM25 index over a corpus of competence strings
|
||||
and rank them for each required competence query. The index is built once over the global
|
||||
candidate pool per matching run (see "Global BM25 index" requirement below) and is queried
|
||||
per candidate with results filtered to that candidate's competence set.
|
||||
|
||||
#### Scenario: BM25 index ranks exact token match highest
|
||||
|
||||
- **GIVEN** a candidate corpus `["Python", "JavaScript", "TypeScript"]`
|
||||
- **AND** query `"Python"`
|
||||
- **WHEN** the `Bm25Index.rank()` method is called
|
||||
- **THEN** `"Python"` is ranked first with the highest BM25 score
|
||||
- **AND** `"JavaScript"` and `"TypeScript"` have BM25 score 0.0 (no shared tokens)
|
||||
|
||||
#### Scenario: BM25 index handles empty corpus without error
|
||||
|
||||
- **GIVEN** a candidate with no competences (empty corpus)
|
||||
- **WHEN** the system computes `competence_score`
|
||||
- **THEN** `competence_score == 0.0` for every required competence
|
||||
- **AND** no exception is raised
|
||||
|
||||
#### Scenario: Tokenization is case-insensitive and splits on non-word characters
|
||||
|
||||
- **GIVEN** a candidate corpus `["Progressive Web App (PWA)", "CI/CD Pipeline"]`
|
||||
- **AND** query `"ci cd"`
|
||||
- **WHEN** the `Bm25Index.rank()` method is called
|
||||
- **THEN** `"CI/CD Pipeline"` receives a score > 0
|
||||
- **AND** `"Progressive Web App (PWA)"` receives score 0.0
|
||||
|
||||
### Requirement: Reciprocal Rank Fusion normalizes BM25 ranks into scores in [0, 1]
|
||||
|
||||
The `reciprocal_rank_fusion()` function SHALL fuse ranked lists into a normalized score
|
||||
dict. Candidates with zero BM25 score in all contributing lists SHALL receive a fused
|
||||
score of 0.0. The top-ranked candidate SHALL receive score 1.0.
|
||||
|
||||
#### Scenario: Single-list fusion normalizes top candidate to 1.0
|
||||
|
||||
- **GIVEN** a single ranked list `[("Python", 3.5), ("Python (advanced)", 2.1)]`
|
||||
- **WHEN** `reciprocal_rank_fusion([ranked_list], k=60)` is called
|
||||
- **THEN** `"Python"` receives normalized score `1.0`
|
||||
- **AND** `"Python (advanced)"` receives a score in `(0, 1)`
|
||||
|
||||
#### Scenario: All-zero BM25 list produces empty result
|
||||
|
||||
- **GIVEN** a ranked list `[("JavaScript", 0.0), ("TypeScript", 0.0)]`
|
||||
- **WHEN** `reciprocal_rank_fusion([ranked_list])` is called
|
||||
- **THEN** the function returns an empty dict `{}`
|
||||
- **AND** the caller maps this to `score = 0.0` and `best_match = None`
|
||||
|
||||
### Requirement: Global BM25 index is built once over all filtered candidates
|
||||
|
||||
When `use_bm25_search = true`, the `Matcher` SHALL build one `Bm25Index` over the
|
||||
deduplicated union of all filtered candidates' competences before the per-candidate
|
||||
scoring loop. The same index SHALL be passed into every `compute_competence_similarity()`
|
||||
call and results SHALL be filtered to the current candidate's competence set. This ensures
|
||||
BM25 IDF values reflect the full candidate pool rather than a single person's small corpus.
|
||||
|
||||
#### Scenario: Candidate with unique skill scores above zero in global index
|
||||
|
||||
- **GIVEN** `use_bm25_search = true` in `config.toml`
|
||||
- **AND** two candidates: one with `["Python"]` only, another with 6 unrelated skills
|
||||
- **AND** the task requires `["Python"]`
|
||||
- **WHEN** the system computes `competence_score` for each candidate
|
||||
- **THEN** the candidate with `["Python"]` receives `competence_score > 0.0`
|
||||
- **AND** the candidate with no Python token receives `competence_score == 0.0`
|
||||
|
||||
#### Scenario: Ubiquitous term present in every candidate scores 0.0
|
||||
|
||||
- **GIVEN** a global corpus where a token appears in every candidate's competences
|
||||
- **WHEN** `Bm25Index.rank()` is called with that token as the query
|
||||
- **THEN** the raw BM25 score is clamped to `max(0.0, raw)` (IDF is negative → score is 0.0)
|
||||
- **AND** no exception is raised
|
||||
|
||||
#### Scenario: Global index filters results to current candidate's competence set
|
||||
|
||||
- **GIVEN** a global index built over candidates A (`["Python", "Django"]`) and B (`["Java", "Spring"]`)
|
||||
- **AND** the system scores candidate A for required `["Python"]`
|
||||
- **WHEN** `_bm25_rrf_similarity()` is called with candidate A's competence set
|
||||
- **THEN** only `"Python"` and `"Django"` appear in the filtered results
|
||||
- **AND** `"Java"` and `"Spring"` do NOT appear in candidate A's result
|
||||
|
||||
### Requirement: Optional LLM-based Auto-Tagging expands candidate competences before BM25
|
||||
|
||||
When `use_auto_tagging = true` (and `use_bm25_search = true`), the system SHALL call the
|
||||
`AutoTagger` before each BM25 scoring call to identify which required competences are
|
||||
already covered by the candidate's existing entries via synonym, abbreviation, or
|
||||
cross-language equivalence. The LLM returns canonical required-competence names as
|
||||
additions. The expanded list is ephemeral — it SHALL be used only for the current BM25
|
||||
scoring call and SHALL NOT be written back to any database or persisted between calls.
|
||||
|
||||
#### Scenario: Auto-tagging bridges synonym gap so candidate matches required competence
|
||||
|
||||
- **GIVEN** `use_bm25_search = true` and `use_auto_tagging = true` in `config.toml`
|
||||
- **AND** a task requires competences `["Machine Learning"]`
|
||||
- **AND** a candidate has competences `["ML", "Python"]`
|
||||
- **AND** the LLM identifies `"ML"` as covering `"Machine Learning"`
|
||||
- **WHEN** the system computes `competence_score`
|
||||
- **THEN** `competence_score > 0.0` (auto-tagging added `"Machine Learning"` to working list)
|
||||
- **AND** the candidate's stored competences remain `["ML", "Python"]` (not mutated)
|
||||
|
||||
#### Scenario: Auto-tagging bridges cross-language gap
|
||||
|
||||
- **GIVEN** `use_bm25_search = true` and `use_auto_tagging = true` in `config.toml`
|
||||
- **AND** a task requires competences `["Software Architecture"]`
|
||||
- **AND** a candidate has competences `["Softwarearchitektur"]`
|
||||
- **AND** the LLM identifies `"Softwarearchitektur"` as covering `"Software Architecture"`
|
||||
- **WHEN** the system computes `competence_score`
|
||||
- **THEN** `competence_score > 0.0`
|
||||
|
||||
#### Scenario: Auto-tagging graceful degradation on LLM failure
|
||||
|
||||
- **GIVEN** `use_bm25_search = true` and `use_auto_tagging = true` in `config.toml`
|
||||
- **AND** the LLM call raises an exception (timeout, API error, or malformed JSON)
|
||||
- **WHEN** the system computes `competence_score`
|
||||
- **THEN** no exception is propagated to the caller
|
||||
- **AND** BM25 scoring continues on the original (unexpanded) candidate list
|
||||
- **AND** a warning is logged
|
||||
|
||||
#### Scenario: Auto-tagging LLM cannot invent skills not in required list
|
||||
|
||||
- **GIVEN** `use_auto_tagging = true`
|
||||
- **AND** the LLM response includes an addition that is NOT in the `required` list
|
||||
- **WHEN** `AutoTagger.expand_competences()` processes the response
|
||||
- **THEN** the hallucinated addition is silently dropped
|
||||
- **AND** only valid required-competence names are appended to the working list
|
||||
|
||||
#### Scenario: Auto-tagging is no-op when use_auto_tagging is false
|
||||
|
||||
- **GIVEN** `use_bm25_search = true` and `use_auto_tagging = false` (default) in `config.toml`
|
||||
- **WHEN** the system computes `competence_score`
|
||||
- **THEN** no LLM call is made
|
||||
- **AND** BM25 runs on the raw candidate competence list without expansion
|
||||
|
||||
### Requirement: `use_auto_tagging` configuration key with default false
|
||||
|
||||
`config.toml` SHALL support a `matching.similarity.use_auto_tagging` boolean key.
|
||||
The default value SHALL be `false`. The key SHALL be documented in `config.toml.example`.
|
||||
`use_auto_tagging = true` is only effective when `use_bm25_search = true`.
|
||||
|
||||
#### Scenario: Config key defaults to false when omitted
|
||||
|
||||
- **GIVEN** a `config.toml` that does not contain `use_auto_tagging`
|
||||
- **WHEN** the server starts and loads configuration
|
||||
- **THEN** `SimilarityConfig.use_auto_tagging == False`
|
||||
- **AND** no `AutoTagger` is constructed
|
||||
- **AND** no LLM client is constructed for auto-tagging
|
||||
|
||||
#### Scenario: Config key enables auto-tagging when set to true
|
||||
|
||||
- **GIVEN** `config.toml` contains `use_auto_tagging = true` under `[matching.similarity]`
|
||||
- **AND** `chat_deployment` is set and `AZURE_OPENAI_LLM_API_KEY` is present
|
||||
- **WHEN** the server starts and loads configuration
|
||||
- **THEN** `SimilarityConfig.use_auto_tagging == True`
|
||||
- **AND** an `AutoTagger` is constructed and passed to `SimilarityEngine`
|
||||
|
||||
### Requirement: `azure_openai.chat_deployment` configuration key for auto-tagging
|
||||
|
||||
`config.toml` SHALL support an `azure_openai.chat_deployment` string key naming the Azure
|
||||
OpenAI chat model deployment used by the `AutoTagger`. This key is required when
|
||||
`use_auto_tagging = true`. The `AZURE_OPENAI_LLM_API_KEY` environment variable SHALL
|
||||
supply the API key for that deployment. The server SHALL raise a `ConfigurationError` at
|
||||
startup if `use_auto_tagging = true` and `AZURE_OPENAI_LLM_API_KEY` is absent.
|
||||
|
||||
#### Scenario: Server raises ConfigurationError when API key missing with auto-tagging enabled
|
||||
|
||||
- **GIVEN** `use_auto_tagging = true` in `config.toml`
|
||||
- **AND** the environment variable `AZURE_OPENAI_LLM_API_KEY` is not set
|
||||
- **WHEN** the server starts and loads configuration
|
||||
- **THEN** a `ConfigurationError` is raised before any MCP tool is registered
|
||||
- **AND** the error message indicates the missing environment variable
|
||||
|
||||
#### Scenario: chat_deployment key is read from config when auto-tagging enabled
|
||||
|
||||
- **GIVEN** `config.toml` contains `chat_deployment = "gpt-4.1"` under `[azure_openai]`
|
||||
- **AND** `use_auto_tagging = true` and `AZURE_OPENAI_LLM_API_KEY` is set
|
||||
- **WHEN** the server starts
|
||||
- **THEN** the `AzureOpenAIClient` for auto-tagging is initialized with deployment `"gpt-4.1"`
|
||||
|
||||
### Requirement: `use_bm25_search` configuration key with default false
|
||||
|
||||
`config.toml` SHALL support a `matching.similarity.use_bm25_search` boolean key.
|
||||
The default value SHALL be `false`. The key SHALL be documented in `config.toml.example`.
|
||||
|
||||
#### Scenario: Config key defaults to false when omitted
|
||||
|
||||
- **GIVEN** a `config.toml` that does not contain `use_bm25_search`
|
||||
- **WHEN** the server starts and loads configuration
|
||||
- **THEN** `SimilarityConfig.use_bm25_search == False`
|
||||
- **AND** the embedding-based strategy is used unchanged
|
||||
|
||||
#### Scenario: Config key enables BM25 mode when set to true
|
||||
|
||||
- **GIVEN** `config.toml` contains `use_bm25_search = true` under `[matching.similarity]`
|
||||
- **WHEN** the server starts and loads configuration
|
||||
- **THEN** `SimilarityConfig.use_bm25_search == True`
|
||||
- **AND** `compute_competence_similarity()` delegates to `_bm25_rrf_similarity()`
|
||||
|
||||
## MODIFIED Requirements
|
||||
|
||||
### Requirement: Role similarity is always embedding-based regardless of use_bm25_search
|
||||
|
||||
Role similarity computation SHALL remain embedding-based even when `use_bm25_search = true`.
|
||||
The BM25 strategy applies only to competence similarity.
|
||||
|
||||
#### Scenario: Role scoring unaffected by use_bm25_search flag
|
||||
|
||||
- **GIVEN** `use_bm25_search = true` in `config.toml`
|
||||
- **AND** a task requiring role "Backend Developer"
|
||||
- **AND** a capacity with role "Backend Developer"
|
||||
- **WHEN** the system computes `role_score`
|
||||
- **THEN** `role_score` is computed via embedding similarity (unchanged behavior)
|
||||
- **AND** embedding cache and Azure OpenAI calls for role embeddings proceed as before
|
||||
|
||||
## REMOVED Requirements
|
||||
|
||||
### Requirement: Legacy debug scripts are removed from the repository root
|
||||
|
||||
The files `debug_similarity_analysis.py`, `debug_categorization.py`,
|
||||
`SEMANTIC_SIMILARITY_ANALYSIS.md`, and `SCORE_BUG_FIX.md` SHALL be removed from the
|
||||
repository root. They are superseded by the test suite and `docs/semantic_similarity_tradeoffs.md`.
|
||||
|
||||
#### Scenario: Legacy files do not exist after the change is applied
|
||||
|
||||
- **GIVEN** the change `add-bm25-rrf-competence-matching` is fully applied
|
||||
- **WHEN** the repository root is listed
|
||||
- **THEN** `debug_similarity_analysis.py` does not exist
|
||||
- **AND** `debug_categorization.py` does not exist
|
||||
- **AND** `SEMANTIC_SIMILARITY_ANALYSIS.md` does not exist
|
||||
- **AND** `SCORE_BUG_FIX.md` does not exist
|
||||
@@ -1,586 +0,0 @@
|
||||
# Implementation Tasks: BM25 + RRF Competence Matching
|
||||
|
||||
## Status: Implemented
|
||||
|
||||
## Summary
|
||||
|
||||
This change introduces an optional BM25-based competence matching strategy with optional
|
||||
LLM-based Auto-Tagging as an alternative to the existing embedding-based approach:
|
||||
|
||||
1. **Restore LLM client**: `AzureOpenAIClient.chat_completion()` restored; `AzureOpenAIConfig.chat_deployment` + `llm_api_key` added.
|
||||
2. **New `auto_tagger.py` module**: `AutoTagger` class with `expand_competences()` — ephemeral LLM-driven competence expansion.
|
||||
3. **New `bm25.py` module**: `Bm25Index` dataclass + `bm25_rank_competences()` convenience function.
|
||||
4. **New `rrf.py` module**: `reciprocal_rank_fusion()` fuses ranked lists into a normalized score dict.
|
||||
5. **`similarity.py` extended**: `_bm25_rrf_similarity()` internal method + auto-tag branch + flag routing in `compute_competence_similarity()`.
|
||||
6. **`SimilarityConfig` extended**: new `use_bm25_search: bool = False` and `use_auto_tagging: bool = False` fields.
|
||||
7. **`config.py` parser extended**: reads `use_bm25_search`, `use_auto_tagging`, `chat_deployment`, and `AZURE_OPENAI_LLM_API_KEY`.
|
||||
8. **`config.toml.example` updated**: documents all new keys.
|
||||
9. **`mcp_server.py` updated**: constructs `AutoTagger` when `use_auto_tagging=True`; passes flags to `SimilarityEngine`.
|
||||
10. **Legacy debug files removed**: 4 root-level files no longer part of the module.
|
||||
11. **Tests added**: `tests/test_bm25.py`, `tests/test_rrf.py`, `tests/test_auto_tagger.py`, extensions to `tests/test_similarity.py` and `tests/test_matcher_integration.py`.
|
||||
12. **Documentation updated**: architecture, tradeoffs doc, assistant prompt, README.
|
||||
|
||||
## Clarifications / Decisions
|
||||
|
||||
- **BM25 scope**: BM25 + RRF replaces **competence** similarity only. Role similarity is
|
||||
always embedding-based, regardless of `use_bm25_search`.
|
||||
- **Auto-tagging scope**: `use_auto_tagging` only applies when `use_bm25_search = true`.
|
||||
When `use_auto_tagging = true`, `AutoTagger.expand_competences()` is called once per
|
||||
candidate per `compute_competence_similarity()` invocation. Result is ephemeral.
|
||||
- **Auto-tagging works in both directions**: both `find_matching_capacities` and
|
||||
`find_matching_tasks` pass through `compute_competence_similarity()` and benefit.
|
||||
- **Graceful degradation**: if the LLM call in `AutoTagger` fails for any reason, the
|
||||
original candidate list is used unchanged. BM25 scoring continues.
|
||||
- **LLM client restored**: `AzureOpenAIClient.chat_completion()` is re-added, scoped
|
||||
exclusively to auto-tagging. The embedding-related API surface is unchanged.
|
||||
- **Zero-out rule**: If all BM25 scores for a required competence are 0.0 (no token
|
||||
overlap at all), the RRF score for that required competence is also 0.0.
|
||||
- **Global index** *(revised)*: Originally the `Bm25Index` was to be built fresh per
|
||||
`compute_competence_similarity()` call. This was identified as an IDF pathology: with
|
||||
only one person's competences as the corpus, BM25 IDF is always 0 or negative for any
|
||||
term that appears in every document (and with N=1 all terms do). The fix (Phase 9) builds
|
||||
one `Bm25Index` over the union of all filtered candidates' competences before the loop
|
||||
and passes it in as `global_index`.
|
||||
- **Single ranked list**: Initial implementation fuses one BM25 ranked list per required
|
||||
competence. The `reciprocal_rank_fusion()` signature accepts multiple lists for future
|
||||
hybrid extension.
|
||||
- **Output shape unchanged**: `_bm25_rrf_similarity()` returns the same
|
||||
`dict[str, dict[str, object]]` shape as `_per_skill_similarity()`. Matcher, scorer,
|
||||
and all output formatting remain unchanged.
|
||||
- **`rank-bm25` dependency**: Added to `pyproject.toml` `[project.dependencies]`.
|
||||
- **All new code uses Google Docstrings** per project convention.
|
||||
- **Default `false`**: Existing deployments are unaffected until opt-in.
|
||||
|
||||
## Estimated Effort
|
||||
|
||||
- **Phase 0 (Restore LLM client)**: 1 hour
|
||||
- **Phase 1 (New modules — bm25, rrf, auto_tagger)**: 3–4 hours
|
||||
- **Phase 2 (Config + wiring)**: 1–1.5 hours
|
||||
- **Phase 3 (Similarity integration)**: 2 hours
|
||||
- **Phase 4 (Legacy cleanup)**: 0.5 hour
|
||||
- **Phase 5 (Tests)**: 4–5 hours
|
||||
- **Phase 6 (Documentation)**: 2 hours
|
||||
- **Phase 7 (Dependency)**: 0.5 hour
|
||||
- **Phase 8 (Quality gates)**: 0.5 hour
|
||||
- **Total**: 15–17 hours
|
||||
|
||||
---
|
||||
|
||||
## Phase 0: Restore LLM Client
|
||||
|
||||
### Task 0.1: Extend `AzureOpenAIConfig`
|
||||
**File:** `src/teamlandkarte_mcp/config.py`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Add `chat_deployment: str = ""` field to `AzureOpenAIConfig` dataclass
|
||||
- [x] Add Google Docstring field comment:
|
||||
`"""Azure deployment name for the chat model used in auto-tagging (e.g. 'gpt-4.1'). Required when use_auto_tagging = true."""`
|
||||
- [x] Update the TOML parser to read `azure_openai.chat_deployment` with default `""`
|
||||
- [x] Read `AZURE_OPENAI_LLM_API_KEY` from environment (store as `llm_api_key: str = ""` on `AzureOpenAIConfig`)
|
||||
- [x] When `use_auto_tagging = true` and `llm_api_key` is empty, raise `ConfigurationError` at startup
|
||||
|
||||
**Acceptance:**
|
||||
- `AzureOpenAIConfig()` instantiates with `chat_deployment=""` and `llm_api_key=""`
|
||||
- TOML `azure_openai.chat_deployment = "gpt-4.1"` is read correctly
|
||||
- Missing `AZURE_OPENAI_LLM_API_KEY` with `use_auto_tagging=true` raises at startup
|
||||
|
||||
---
|
||||
|
||||
### Task 0.2: Add `chat_completion()` to `AzureOpenAIClient`
|
||||
**File:** `src/teamlandkarte_mcp/azure/openai_client.py`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Add `chat_deployment: str` and `llm_api_key: str` constructor parameters
|
||||
- [x] Implement `chat_completion(system: str, user: str) -> str` method:
|
||||
- Calls Azure OpenAI chat completions API with `response_format={"type": "json_object"}`
|
||||
- Uses `self.chat_deployment` as the model/deployment
|
||||
- Uses `self.llm_api_key` for auth (separate from embedding key)
|
||||
- Returns the raw JSON string from the first choice's message content
|
||||
- Raises `AzureOpenAIError` on API errors (consistent with existing error handling)
|
||||
- [x] Add Google Docstring to `chat_completion()`
|
||||
|
||||
**Acceptance:**
|
||||
- `chat_completion(system="...", user="...")` returns a JSON string
|
||||
- API errors surface as `AzureOpenAIError`
|
||||
- `chat_deployment=""` + `chat_completion()` call raises `ConfigurationError` (misconfigured guard)
|
||||
|
||||
---
|
||||
|
||||
## Phase 1: New Modules
|
||||
|
||||
### Task 1.1: Add `bm25.py`
|
||||
**File:** `src/teamlandkarte_mcp/matching/bm25.py` (**new**)
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Create `bm25.py` with `_tokenize(text: str) -> list[str]` helper (lowercase, split on `[\W_]+`, filter empty)
|
||||
- [x] Implement `Bm25Index` dataclass:
|
||||
- `corpus: list[str]` field
|
||||
- Internal `_bm25: BM25Okapi | None` built in `__post_init__`
|
||||
- `rank(query: str) -> list[tuple[str, float]]` method:
|
||||
- returns `[]` for empty corpus
|
||||
- returns `(corpus_text, bm25_score)` list sorted by score descending
|
||||
- score 0.0 entries are included (caller decides cutoff)
|
||||
- [x] Implement `bm25_rank_competences(required: str, candidates: list[str]) -> list[tuple[str, float]]` convenience function
|
||||
- [x] Add Google Docstrings to all public symbols
|
||||
|
||||
**Acceptance:**
|
||||
- `Bm25Index(corpus=[]).rank("Python")` → `[]`
|
||||
- `Bm25Index(corpus=["Python"]).rank("Python")` → `[("Python", score > 0)]`
|
||||
- `Bm25Index(corpus=["JavaScript"]).rank("Python")` → `[("JavaScript", 0.0)]`
|
||||
- `Bm25Index(corpus=["Machine Learning"]).rank("machine learning")` → score > 0 (case-insensitive)
|
||||
|
||||
---
|
||||
|
||||
### Task 1.2: Add `rrf.py`
|
||||
**File:** `src/teamlandkarte_mcp/matching/rrf.py` (**new**)
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Implement `reciprocal_rank_fusion(ranked_lists, *, k=60) -> dict[str, float]`:
|
||||
- Skip any list where all scores are 0.0 (no token-overlap signal)
|
||||
- For each candidate in each non-zero list, accumulate `1 / (k + rank)` (1-based rank)
|
||||
- Normalize so the highest-scoring candidate maps to 1.0
|
||||
- Return `{}` if all lists are empty or all-zero
|
||||
- [x] Add Google Docstring including mathematical formula, zero-out rule, and normalization description
|
||||
|
||||
**Acceptance:**
|
||||
- Single list, one candidate → `{"A": 1.0}`
|
||||
- Single list `[("A", 1.0), ("B", 0.5)]` → `{"A": 1.0, "B": ...}` with `0 < B < 1`
|
||||
- Single all-zero list `[("A", 0.0), ("B", 0.0)]` → `{}` (empty, no signal)
|
||||
- `k=60` default: rank-1 score = `1/61`, rank-2 score = `1/62`, normalized → `61/62 ≈ 0.984`
|
||||
|
||||
---
|
||||
|
||||
### Task 1.3: Add `auto_tagger.py`
|
||||
**File:** `src/teamlandkarte_mcp/matching/auto_tagger.py` (**new**)
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Implement `AutoTagger` class:
|
||||
- Constructor: `__init__(self, client: AzureOpenAIClient) -> None`
|
||||
- Stores the client reference
|
||||
- [x] Implement `expand_competences(self, required: list[str], existing: list[str]) -> list[str]`:
|
||||
- Build system prompt: instructs LLM to return only required-competence names already
|
||||
covered by existing entries (synonym/abbreviation/cross-language), as JSON
|
||||
`{"additions": ["...", ...]}`
|
||||
- Build user prompt: `REQUIRED: {required}\nEXISTING: {existing}`
|
||||
- Call `self._client.chat_completion(system=..., user=...)` → JSON string
|
||||
- Parse JSON, extract `additions` list
|
||||
- Validate: keep only items present in `required` (guard against LLM hallucination)
|
||||
- Return `existing + [a for a in additions if a not in existing]`
|
||||
- On any exception (API error, JSON parse, key error): log warning, return `existing` unchanged
|
||||
- [x] Add Google Docstrings to class and all public methods
|
||||
|
||||
**Acceptance:**
|
||||
- LLM returns `{"additions": ["Machine Learning"]}` for existing `["ML"]`, required `["Machine Learning"]`
|
||||
→ returns `["ML", "Machine Learning"]`
|
||||
- LLM returns `{"additions": []}` → returns `existing` unchanged
|
||||
- LLM raises exception → returns `existing` unchanged (no exception propagation)
|
||||
- Addition not in `required` is silently dropped (hallucination guard)
|
||||
- No duplicate entries if `existing` already contains the addition
|
||||
|
||||
---
|
||||
|
||||
## Phase 2: Configuration Extension
|
||||
|
||||
### Task 2.1: Extend `SimilarityConfig`
|
||||
**File:** `src/teamlandkarte_mcp/config.py`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Add `use_bm25_search: bool = False` field to `SimilarityConfig` dataclass
|
||||
- [x] Add `use_auto_tagging: bool = False` field to `SimilarityConfig` dataclass
|
||||
- [x] Add Google Docstring comments to both fields
|
||||
- [x] Update the TOML parser to read `matching.similarity.use_bm25_search` (default `False`)
|
||||
- [x] Update the TOML parser to read `matching.similarity.use_auto_tagging` (default `False`)
|
||||
|
||||
**Acceptance:**
|
||||
- `SimilarityConfig()` instantiates with both flags `False`
|
||||
- `SimilarityConfig(use_bm25_search=True, use_auto_tagging=True)` works
|
||||
- Parser reads both flags from TOML correctly
|
||||
|
||||
---
|
||||
|
||||
### Task 2.2: Update `config.toml.example`
|
||||
**File:** `config.toml.example`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Add `use_bm25_search = false` under `[matching.similarity]` with inline comment
|
||||
- [x] Add `use_auto_tagging = false` under `[matching.similarity]` with inline comment:
|
||||
```toml
|
||||
# When true, BM25 + RRF replaces embedding-based competence similarity.
|
||||
# BM25 assigns 0 to candidates with no token overlap, eliminating false positives.
|
||||
# Role similarity is always embedding-based. Default: false.
|
||||
use_bm25_search = false
|
||||
|
||||
# When true (and use_bm25_search = true), an LLM pre-expands each candidate's
|
||||
# competence list with canonical equivalents of required competences it already covers
|
||||
# (synonyms, abbreviations, cross-language). The expansion is ephemeral. Default: false.
|
||||
use_auto_tagging = false
|
||||
```
|
||||
- [x] Add `chat_deployment = ""` under `[azure_openai]` with inline comment:
|
||||
```toml
|
||||
# Azure deployment name for the chat model used in auto-tagging (e.g. "gpt-4.1").
|
||||
# Required when use_auto_tagging = true.
|
||||
chat_deployment = ""
|
||||
```
|
||||
|
||||
**Acceptance:**
|
||||
- Config example is self-documenting for all three new keys
|
||||
- All new defaults are `false` / `""`
|
||||
|
||||
---
|
||||
|
||||
### Task 2.3: Wire `use_bm25_search`, `use_auto_tagging`, and `AutoTagger` in `mcp_server.py`
|
||||
**File:** `src/teamlandkarte_mcp/mcp_server.py`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] When `use_auto_tagging=True`: construct `AzureOpenAIClient` with `chat_deployment`
|
||||
and `llm_api_key`; construct `AutoTagger(client=llm_client)`
|
||||
- [x] Pass `auto_tagger` (or `None`) and both flags to `SimilarityEngine` constructor
|
||||
- [x] When `use_auto_tagging=False`: `auto_tagger=None` — no LLM client constructed
|
||||
|
||||
**Acceptance:**
|
||||
- Setting `use_auto_tagging = true` in `config.toml` activates auto-tagging at runtime
|
||||
- Setting `use_auto_tagging = false` (default) does not construct any LLM client
|
||||
|
||||
---
|
||||
|
||||
## Phase 3: Similarity Engine Integration
|
||||
|
||||
### Task 3.1: Add `_bm25_rrf_similarity()` to `SimilarityEngine`
|
||||
**File:** `src/teamlandkarte_mcp/matching/similarity.py`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Import `bm25_rank_competences` from `.bm25`, `reciprocal_rank_fusion` from `.rrf`,
|
||||
and `AutoTagger` from `.auto_tagger`
|
||||
- [x] Implement `_bm25_rrf_similarity(self, required: list[str], candidate: list[str]) -> dict[str, dict[str, object]]`:
|
||||
- For each `req` in `required`:
|
||||
1. Call `bm25_rank_competences(req, candidate)` → `ranked`
|
||||
2. Call `reciprocal_rank_fusion([ranked])` → `fused: dict[str, float]`
|
||||
3. If `fused` is empty or all scores are 0.0:
|
||||
- `score = 0.0`, `best_match = None`, `rationale = "BM25: no token overlap with any candidate competence."`
|
||||
4. Otherwise:
|
||||
- `best_match = max(fused, key=fused.get)`
|
||||
- `score = fused[best_match]`
|
||||
- `rationale = f"BM25+RRF: best match '{best_match}' (score {score:.3f})."`
|
||||
- Returns dict shaped identically to `_per_skill_similarity()` output
|
||||
- [x] Add Google Docstring to `_bm25_rrf_similarity()`
|
||||
|
||||
---
|
||||
|
||||
### Task 3.2: Route `compute_competence_similarity()` by flags
|
||||
**File:** `src/teamlandkarte_mcp/matching/similarity.py`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Ensure `SimilarityEngine.__init__` accepts and stores `use_bm25_search`,
|
||||
`use_auto_tagging`, and `auto_tagger: AutoTagger | None = None`
|
||||
- [x] In `compute_competence_similarity()`:
|
||||
```python
|
||||
if self.config.use_bm25_search:
|
||||
working = candidate
|
||||
if self.config.use_auto_tagging and self._auto_tagger is not None:
|
||||
working = self._auto_tagger.expand_competences(required, candidate)
|
||||
return self._bm25_rrf_similarity(required, working)
|
||||
# else: existing strategy dispatch (unchanged)
|
||||
```
|
||||
|
||||
**Acceptance:**
|
||||
- `use_bm25_search=False`: behavior bit-for-bit identical to before
|
||||
- `use_bm25_search=True, use_auto_tagging=False`: BM25 on raw candidate list
|
||||
- `use_bm25_search=True, use_auto_tagging=True`: BM25 on LLM-expanded list
|
||||
- `use_bm25_search=True` + required=`["Python"]`, candidate=`["JavaScript"]`: `score == 0.0`
|
||||
- `use_bm25_search=True` + required=`["Python"]`, candidate=`["Python"]`: `score == 1.0`
|
||||
|
||||
---
|
||||
|
||||
## Phase 4: Legacy Cleanup
|
||||
|
||||
### Task 4.1: Remove legacy debug files
|
||||
**Files to delete:**
|
||||
- `debug_similarity_analysis.py`
|
||||
- `debug_categorization.py`
|
||||
- `SEMANTIC_SIMILARITY_ANALYSIS.md`
|
||||
- `SCORE_BUG_FIX.md`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Verify none of these files are imported or referenced in source code or tests
|
||||
- [x] Delete `debug_similarity_analysis.py`
|
||||
- [x] Delete `debug_categorization.py`
|
||||
- [x] Delete `SEMANTIC_SIMILARITY_ANALYSIS.md`
|
||||
- [x] Delete `SCORE_BUG_FIX.md`
|
||||
|
||||
**Acceptance:**
|
||||
- `git status` shows 4 deleted files
|
||||
- No import errors in existing code
|
||||
|
||||
---
|
||||
|
||||
## Phase 5: Tests
|
||||
|
||||
### Task 5.1: Unit tests for `bm25.py`
|
||||
**File:** `tests/test_bm25.py` (**new**)
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Test `Bm25Index` with empty corpus → `rank()` returns `[]`
|
||||
- [x] Test exact match (single token) → score > 0
|
||||
- [x] Test no token overlap (query "Python", corpus "JavaScript") → score == 0.0
|
||||
- [x] Test partial token overlap multi-token query ("Machine Learning") against ("Deep Learning") → partial score > 0
|
||||
- [x] Test case-insensitivity: `rank("Python")` matches "python" in corpus
|
||||
- [x] Test tokenization edge cases:
|
||||
- "Progressive Web App (PWA)" → `["progressive", "web", "app", "pwa"]`
|
||||
- "CI/CD Pipeline" → `["ci", "cd", "pipeline"]`
|
||||
- "React.js" → `["react", "js"]`
|
||||
- [x] Test `bm25_rank_competences()` convenience wrapper produces same result as `Bm25Index.rank()`
|
||||
- [x] Add Google Docstrings to test module
|
||||
|
||||
### Task 5.2: Unit tests for `rrf.py`
|
||||
**File:** `tests/test_rrf.py` (**new**)
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Test single list, single candidate → `{"A": 1.0}`
|
||||
- [x] Test single list multiple candidates: top candidate maps to 1.0, others < 1.0 and > 0.0
|
||||
- [x] Test all-zero list → returns `{}` (empty)
|
||||
- [x] Test mixed: one non-zero list + one all-zero list → only non-zero list contributes
|
||||
- [x] Test `k` parameter: higher `k` flattens differences (rank-1 and rank-2 scores closer together)
|
||||
- [x] Test score ordering is preserved (rank 1 > rank 2 > rank 3 in output)
|
||||
- [x] Add Google Docstrings to test module
|
||||
|
||||
### Task 5.3: Unit tests for `auto_tagger.py`
|
||||
**File:** `tests/test_auto_tagger.py` (**new**)
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Mock `AzureOpenAIClient.chat_completion()` for all tests (no real LLM calls)
|
||||
- [x] Test: LLM returns `{"additions": ["Machine Learning"]}` for existing `["ML"]`,
|
||||
required `["Machine Learning"]` → result is `["ML", "Machine Learning"]`
|
||||
- [x] Test: LLM returns `{"additions": []}` → result is identical to `existing`
|
||||
- [x] Test: LLM `chat_completion` raises exception → result is identical to `existing`,
|
||||
no exception propagated
|
||||
- [x] Test: LLM returns malformed JSON (not parseable) → result is `existing` unchanged
|
||||
- [x] Test: LLM returns addition not in `required` (hallucination) → addition is dropped
|
||||
- [x] Test: LLM returns addition already in `existing` → no duplicate in result
|
||||
- [x] Test: empty `required` list → `expand_competences()` returns `existing` unchanged
|
||||
(no LLM call needed)
|
||||
- [x] Add Google Docstrings to test module
|
||||
|
||||
### Task 5.4: Extend `tests/test_similarity.py`
|
||||
**File:** `tests/test_similarity.py` (extended)
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Add tests with `use_bm25_search=True`:
|
||||
- Required `["Python"]`, candidate `["JavaScript"]` → `competence_score == 0.0`
|
||||
- Required `["Python"]`, candidate `["Python"]` → `competence_score == 1.0`
|
||||
- Required `["Python", "Machine Learning"]`, candidate `["JavaScript", "TypeScript"]` → both individual scores 0.0, overall `competence_score == 0.0`
|
||||
- Required `["Python", "Machine Learning"]`, candidate `["Python", "Machine Learning"]` → both scores 1.0
|
||||
- [x] Add tests with `use_bm25_search=True, use_auto_tagging=True` (mocked `AutoTagger`):
|
||||
- Mocked `AutoTagger.expand_competences` injects `"Machine Learning"` into `["ML"]`
|
||||
- Required `["Machine Learning"]`, candidate `["ML"]` (after expansion) → `score > 0`
|
||||
- [x] Confirm that with `use_bm25_search=False`, behavior is unchanged (existing tests still pass)
|
||||
|
||||
### Task 5.5: Extend `tests/test_matcher_integration.py`
|
||||
**File:** `tests/test_matcher_integration.py` (extended)
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Add integration test with `use_bm25_search=True`:
|
||||
- Task requires: `["Python", "Machine Learning"]`
|
||||
- Candidate A has: `["Python", "Machine Learning", "Pandas"]` → high score
|
||||
- Candidate B has: `["JavaScript", "TypeScript", "Node.js", "Vue.js"]` → score 0.0
|
||||
- Assert: candidate A `competence_score` > candidate B `competence_score`
|
||||
- Assert: candidate B `competence_score == 0.0` (false-positive eliminated)
|
||||
- [x] Add integration test with `use_bm25_search=True, use_auto_tagging=True`
|
||||
(mocked LLM via mocked `AutoTagger`):
|
||||
- Candidate has `["ML", "Python"]`; auto-tag adds `"Machine Learning"` to working list
|
||||
- Required `["Machine Learning", "Python"]` → both scores > 0
|
||||
- [x] These tests explicitly document the false-positive scenario from production
|
||||
|
||||
---
|
||||
|
||||
## Phase 6: Documentation
|
||||
|
||||
### Task 6.1: Update `docs/architecture.md`
|
||||
**File:** `docs/architecture.md`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Add section "BM25 + RRF + Auto-Tagging Competence Matching" under the matching/similarity section:
|
||||
- Describe the three new modules (`auto_tagger.py`, `bm25.py`, `rrf.py`)
|
||||
- Describe the auto-tag → BM25 → RRF pipeline and routing in `compute_competence_similarity()`
|
||||
- Note that role similarity is unaffected
|
||||
- Reference `use_bm25_search` and `use_auto_tagging` config keys
|
||||
|
||||
### Task 6.2: Update `docs/semantic_similarity_tradeoffs.md`
|
||||
**File:** `docs/semantic_similarity_tradeoffs.md`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Add BM25+RRF row to the strategy comparison table:
|
||||
- Strengths: exact/lexical match, no false positives for non-overlapping skills
|
||||
- Weaknesses: misses synonyms and cross-language variants (mitigated by `use_auto_tagging`)
|
||||
- When to use: when skill names in the dataset are mostly canonical English terms
|
||||
- [x] Add BM25+RRF+AutoTag row:
|
||||
- Strengths: exact match + synonym/abbreviation/cross-language bridging via LLM
|
||||
- Weaknesses: LLM latency per candidate, requires `AZURE_OPENAI_LLM_API_KEY`
|
||||
- When to use: mixed-language or abbreviation-heavy skill datasets
|
||||
|
||||
### Task 6.3: Update `docs/assistant_system_prompt.md`
|
||||
**File:** `docs/assistant_system_prompt.md`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Add note on score interpretation when `use_bm25_search=true`:
|
||||
- A `competence_score` of 0.0 means **no lexical token overlap** between required and candidate competences
|
||||
- The score is not a semantic similarity — it is a lexical matching score
|
||||
- Users should be informed when BM25 mode is active so they understand why cross-language synonyms may not match
|
||||
- [x] Add note on `use_auto_tagging=true`:
|
||||
- Auto-tagging bridges common synonym/abbreviation gaps (e.g. "ML" → "Machine Learning")
|
||||
- A positive `competence_score` with auto-tagging enabled may reflect an LLM-inferred equivalence,
|
||||
not a literal token match; the rationale field will say "BM25+RRF" regardless
|
||||
|
||||
### Task 6.4: Update `README.md`
|
||||
**File:** `README.md`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Add all three new config keys to the config reference section:
|
||||
- `matching.similarity.use_bm25_search` (boolean, default `false`): Enables BM25+RRF lexical competence matching
|
||||
- `matching.similarity.use_auto_tagging` (boolean, default `false`): Enables LLM pre-expansion before BM25
|
||||
- `azure_openai.chat_deployment` (string, default `""`): Chat model deployment name for auto-tagging
|
||||
- [x] Document `AZURE_OPENAI_LLM_API_KEY` environment variable in the env-vars section
|
||||
|
||||
---
|
||||
|
||||
## Phase 7: Dependency
|
||||
|
||||
### Task 7.1: Add `rank-bm25` to project dependencies
|
||||
**File:** `pyproject.toml`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Add `rank-bm25` to `[project.dependencies]` in `pyproject.toml`
|
||||
- [x] Run `uv lock` (or equivalent) to update the lockfile
|
||||
- [x] Verify `from rank_bm25 import BM25Okapi` works in the environment
|
||||
|
||||
**Acceptance:**
|
||||
- `uv run python -c "from rank_bm25 import BM25Okapi; print('ok')"` exits 0
|
||||
|
||||
---
|
||||
|
||||
## Phase 8: Quality Gates
|
||||
|
||||
### Task 8.1: Run unit tests
|
||||
- [x] `uv run pytest tests/test_bm25.py tests/test_rrf.py tests/test_auto_tagger.py -v`
|
||||
- [x] `uv run pytest -m "not integration" -q`
|
||||
- [x] All tests pass
|
||||
|
||||
### Task 8.2: Run linters
|
||||
- [x] `uv run ruff check src/ tests/`
|
||||
- [x] `uv run mypy src/teamlandkarte_mcp/matching/bm25.py src/teamlandkarte_mcp/matching/rrf.py src/teamlandkarte_mcp/matching/auto_tagger.py`
|
||||
- [x] Zero errors
|
||||
|
||||
### Task 8.3: OpenSpec validation
|
||||
- [x] `openspec validate add-bm25-rrf-competence-matching --strict`
|
||||
- [x] All checks pass
|
||||
|
||||
### Task 8.4: Confirm legacy files are removed
|
||||
- [x] `ls debug_*.py` → no such files
|
||||
- [x] `ls SEMANTIC_SIMILARITY_ANALYSIS.md SCORE_BUG_FIX.md` → no such files
|
||||
|
||||
---
|
||||
|
||||
## Phase 9: Architectural Fix — Global BM25 Index
|
||||
|
||||
**Problem identified post-spec**: When `Bm25Index` was built per candidate (N = number of that
|
||||
person's competences), IDF values were always 0 or negative for every term — the library
|
||||
`rank_bm25` computes `log((N - df + 0.5) / (df + 0.5))`, and with N=1 every term has df=1,
|
||||
yielding `log(0)` = negative. All BM25 scores were therefore ≤ 0, making the `max(0.0, raw)`
|
||||
clamp produce a flat 0.0 for everything. The fix builds one global corpus from the union of
|
||||
all filtered candidates' competences, ensuring N is large enough for IDF to be meaningful.
|
||||
|
||||
### Task 9.1: Remove `1e-10` clamp in `bm25.py`
|
||||
**File:** `src/teamlandkarte_mcp/matching/bm25.py`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Remove the special-case block that returned `1e-10` for zero/negative BM25 scores
|
||||
when token overlap existed (a band-aid for the IDF pathology)
|
||||
- [x] Replace with `max(0.0, float(raw))` — ubiquitous terms (negative IDF) simply score 0.0
|
||||
- [x] Update `Bm25Index` and `bm25_rank_competences()` docstrings to reflect the change
|
||||
|
||||
**Acceptance:**
|
||||
- A term present in every document of a sufficiently large corpus scores 0.0, not 1e-10
|
||||
|
||||
---
|
||||
|
||||
### Task 9.2: Add `use_bm25_search` property to `SimilarityEngine`
|
||||
**File:** `src/teamlandkarte_mcp/matching/similarity.py`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Expose `use_bm25_search: bool` as a read-only property (or attribute) on `SimilarityEngine`
|
||||
so `Matcher` can check it without accessing internal config directly
|
||||
|
||||
**Acceptance:**
|
||||
- `engine.use_bm25_search` returns the value of `engine.config.use_bm25_search`
|
||||
|
||||
---
|
||||
|
||||
### Task 9.3: Thread `global_index` parameter through similarity layer
|
||||
**File:** `src/teamlandkarte_mcp/matching/similarity.py`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Add `global_index: Bm25Index | None = None` parameter to `compute_competence_similarity()`
|
||||
- [x] Pass it through to `_bm25_rrf_similarity()`
|
||||
- [x] In `_bm25_rrf_similarity()`: when `global_index is not None`, call
|
||||
`global_index.rank(req)` and filter the result list to only entries whose text is in
|
||||
`set(candidate)`; otherwise fall back to `bm25_rank_competences(req, candidate)`
|
||||
|
||||
**Acceptance:**
|
||||
- `global_index.rank(req)` is called once per required competence, not once per candidate
|
||||
- Results are correctly filtered to the current candidate's competence set
|
||||
- Fall-back path (`global_index=None`) still works unchanged
|
||||
|
||||
---
|
||||
|
||||
### Task 9.4: Build global corpus in `Matcher` before per-candidate loop
|
||||
**File:** `src/teamlandkarte_mcp/matching/matcher.py`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Before the `for cap in filtered` loop: if `self._sim.use_bm25_search` and `filtered`
|
||||
is non-empty, collect the deduplicated union of all candidates' competences into a list
|
||||
`global_corpus`, instantiate `Bm25Index(corpus=global_corpus)`
|
||||
- [x] Pass `global_index` to every `compute_competence_similarity()` call inside the loop
|
||||
- [x] When `use_bm25_search=False` (or `filtered` is empty): `global_bm25_index = None`
|
||||
(no-op, existing behaviour)
|
||||
|
||||
**Acceptance:**
|
||||
- A corpus of N candidates' competences is built once, not N times
|
||||
- Candidates with no overlap with required competences still score 0.0
|
||||
- Candidates with overlap score > 0.0 (IDF is meaningful because N >> 1 in the global corpus)
|
||||
|
||||
---
|
||||
|
||||
### Task 9.5: Update tests for global index
|
||||
**Files:** `tests/test_bm25.py`, `tests/test_similarity.py`, `tests/test_matcher_integration.py`, `tests/test_matching_refinements.py`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] `test_bm25.py`: update small-corpus tests to use ≥ 5-doc corpora so IDF is positive;
|
||||
add `test_bm25_index_rank_filters_to_candidate_subset` and
|
||||
`test_bm25_ubiquitous_term_clamped_to_zero`
|
||||
- [x] `test_similarity.py`: update 4 BM25 tests to pass a 5-doc `global_index`; add
|
||||
`test_use_bm25_search_property_reflects_flag` and `test_bm25_global_index_filters_to_candidate_subset`
|
||||
- [x] `test_matcher_integration.py`: add `use_bm25_search = False` to `_FakeSimilarityEngine`,
|
||||
add `global_index=None` param to `compute_competence_similarity`; update
|
||||
`test_bm25_auto_tag_candidate_matches_after_expansion` to use a 5-doc global corpus;
|
||||
add `test_matcher_builds_global_bm25_index_across_all_candidates`
|
||||
- [x] `test_matching_refinements.py`: add `use_bm25_search = False` and `global_index=None`
|
||||
to local `_FakeSim` stub
|
||||
|
||||
**Acceptance:**
|
||||
- All 140 tests pass, 2 skipped
|
||||
|
||||
---
|
||||
|
||||
### Task 9.6: Update `docs/architecture.md`
|
||||
**File:** `docs/architecture.md`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Replace the incorrect "Per-call index" bullet in §9.3 with a "Global BM25 index" bullet
|
||||
describing the pre-loop construction and per-candidate filtering
|
||||
- [x] Update the §9.1 module table row for `bm25.py` to remove the `1e-10` clamp description
|
||||
and replace with `max(0.0, raw)` description
|
||||
- [x] Update the §9.2 data flow diagram to show `match_candidates` building the global index
|
||||
before the per-candidate loop
|
||||
|
||||
**Acceptance:**
|
||||
- No mention of "per-call index" or `1e-10` clamp remains in the docs
|
||||
|
||||
@@ -1,607 +0,0 @@
|
||||
# Design: Capacity Matching MCP Server
|
||||
|
||||
> **Note**: For comprehensive architecture documentation including ADRs, quality requirements, and deployment view, see [architecture.md](./architecture.md).
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
> **Update (2026-02)**: The server uses **Azure OpenAI** for role/requirements
|
||||
> extraction and similarity scoring via embeddings.
|
||||
> There are **no heuristic fallbacks** for these features.
|
||||
>
|
||||
> The embedding model is `text-embedding-3-large` with **3072 dimensions** and
|
||||
> results are cached in a local SQLite embedding cache.
|
||||
|
||||
## Embedding similarity acceleration (global prefetch + batch embeddings)
|
||||
|
||||
> **Update (2026-02)**: Embedding similarity is optimized via **global bulk prefetch**
|
||||
> plus **true Azure embeddings batch requests**. This is primarily implemented in
|
||||
> `SimilarityEngine` and `AzureOpenAIClient`.
|
||||
|
||||
### Motivation
|
||||
|
||||
The matching pipeline can be slow if embeddings are requested on-demand from inner
|
||||
loops. While the system already uses a persistent SQLite embedding cache
|
||||
(`EmbeddingCache`), additional optimization is necessary to avoid repeated:
|
||||
|
||||
- normalization + cache-key computation
|
||||
- SQLite reads
|
||||
- sequential Azure embedding calls for cache misses
|
||||
|
||||
### Goals
|
||||
|
||||
- Bulk prefetch embeddings for a matching run (roles + competences), across **all free
|
||||
capacities at once** (not person-by-person / incremental embedding).
|
||||
- Maintain deterministic behavior and strict error semantics.
|
||||
- Preserve existing similarity contracts (`per_skill` / `aggregate`) and tool outputs.
|
||||
|
||||
### Data flow
|
||||
|
||||
1. **Collect** all texts that will be embedded in the current run:
|
||||
- required competences
|
||||
- candidate competences (from **all free capacities**)
|
||||
- required roles
|
||||
- candidate roles (from **all free capacities**)
|
||||
2. **Normalize + deduplicate** using existing `_normalize_text()` logic:
|
||||
- skip empty/whitespace-only texts
|
||||
- emit a **Python logger warning** if all candidate competences normalize to empty
|
||||
3. **Resolve embeddings** using a layered cache approach:
|
||||
1) in-memory cache (instance-level, attached to `SimilarityEngine`; not created/cleared per invocation)
|
||||
2) persistent SQLite embedding cache (`EmbeddingCache`)
|
||||
3) Azure embeddings API (batch only the cache-missing texts; chunked)
|
||||
4. **Compute** cosine similarities locally using the prefetched mapping.
|
||||
|
||||
### Normalization and cache keys
|
||||
|
||||
Cache key behavior remains unchanged to avoid invalidating the on-disk cache:
|
||||
|
||||
- normalization: trim + collapse whitespace
|
||||
- key: SHA256 of `model|dims|normalized(text).lower()`
|
||||
|
||||
### Azure embeddings batch API + chunking
|
||||
|
||||
The Azure embeddings API supports embedding multiple inputs per request via
|
||||
`input=[...]`. The implementation requirements are:
|
||||
|
||||
- preserve stable mapping from input texts → returned vectors
|
||||
- rely on the `index` field in the API response if available
|
||||
- otherwise assume stable ordering
|
||||
- chunk large requests using `azure_openai.embedding_batch_size` (default: 128)
|
||||
- strict failure semantics:
|
||||
- the matching operation fails immediately on error
|
||||
- log the failing **chunk input list** via Python logging
|
||||
- cost tracking: log once per chunk with a count of embeddings requested
|
||||
|
||||
### Trade-offs
|
||||
|
||||
- Batch calls reduce network requests dramatically but increase blast radius: one
|
||||
failing request affects multiple inputs.
|
||||
- mitigation: keep retry logic at the chunk level
|
||||
- Prefetch holds a larger in-memory mapping for the current run.
|
||||
- mitigation: only store vectors required for that run (after dedup)
|
||||
|
||||
### Tests
|
||||
|
||||
- prefetch/dedup: client is called once per unique normalized text (cache misses only)
|
||||
- cache layering: in-memory first, then SQLite, then Azure; no duplicate calls
|
||||
- chunking: multiple Azure requests when `len(missing_texts) > batch_size`
|
||||
- empty input handling: warning when all candidate competences normalize to empty
|
||||
- strict error propagation: batch failures identify/log which chunk failed
|
||||
|
||||
```text
|
||||
(diagram below is historical; “sampling” arrows represent the client’s LLM runtime.
|
||||
The MCP server itself does not rely on MCP “sampling” for Azure OpenAI calls.)
|
||||
```
|
||||
|
||||
```
|
||||
┌──────────────┐
|
||||
│ MCP Client │
|
||||
│ (with LLM) │
|
||||
└──────┬───────┘
|
||||
│ MCP protocol (tool calls)
|
||||
▼
|
||||
┌───────────────────────────────────────────────────────────────────────┐
|
||||
│ MCP Server │
|
||||
│ │
|
||||
│ Task Management Tools: │
|
||||
│ - list_open_tasks │
|
||||
│ - get_task_details (table-first) │
|
||||
│ - validate_task_requirements (table-first) │
|
||||
│ - find_capacities_for_task (requires confirm) │
|
||||
│ - infer_roles │
|
||||
│ │
|
||||
│ Requirement Gathering Tools: │
|
||||
│ - extract_requirements (writes pending) │
|
||||
│ - update_requirements (writes pending) │
|
||||
│ - collect_structured_requirement_data │
|
||||
│ (writes pending) │
|
||||
│ - start_guided_capture │
|
||||
│ - guided_set_description │
|
||||
│ - guided_set_role │
|
||||
│ - guided_set_time_range (open-ended) │
|
||||
│ - guided_set_competences (writes pending) │
|
||||
│ - confirm_requirements │
|
||||
│ │
|
||||
│ Search Execution & Refinement: │
|
||||
│ - find_matching_capacities (hard-gated) │
|
||||
│ - filter_search_results │
|
||||
│ - get_results_by_category │
|
||||
│ │
|
||||
│ Business Logic: Matcher / Scorer / Caches │
|
||||
│ │
|
||||
│ Integrations: │
|
||||
│ - Trino / Open Data Lake (read-only SQL) │
|
||||
│ - Azure OpenAI (embeddings only) │
|
||||
└───────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌────────────────────────────┐
|
||||
│ Open Data Lake (Trino) │
|
||||
└────────────────────────────┘
|
||||
```
|
||||
|
||||
## Component Design
|
||||
|
||||
### 1. Configuration Layer (`src/teamlandkarte_mcp/config.py`)
|
||||
|
||||
**Purpose**: Load and validate configuration from `config.toml` and environment.
|
||||
|
||||
**Important**:
|
||||
- `config.toml` contains **non-secret** settings only (endpoints, model/deployment names, cache settings).
|
||||
- credentials come from environment (recommended: `.env`) and must not be committed:
|
||||
- `DATA_LAKE_USERNAME`
|
||||
- `DATA_LAKE_PASSWORD`
|
||||
- `AZURE_OPENAI_EMBEDDING_API_KEY`
|
||||
|
||||
**config.toml structure (excerpt)**:
|
||||
```toml
|
||||
[database]
|
||||
host = "..."
|
||||
port = 8446
|
||||
backend = "trino"
|
||||
http_scheme = "https"
|
||||
verify_ssl = true
|
||||
catalog = "hive"
|
||||
schema = "tier1_open_lake"
|
||||
|
||||
[azure_openai]
|
||||
endpoint = "https://<resource>.openai.azure.com"
|
||||
api_version = "2024-02-15-preview"
|
||||
embedding_deployment = "text-embedding-3-large"
|
||||
|
||||
[embedding_cache]
|
||||
path = ".cache/embeddings.sqlite3"
|
||||
ttl_days = 30
|
||||
|
||||
[matching.similarity]
|
||||
# `per_skill` = best-match per required competence
|
||||
# `aggregate` = mean(required) vs mean(candidate)
|
||||
strategy = "per_skill"
|
||||
```
|
||||
|
||||
### Roundtrip examples (tool-level sequences)
|
||||
|
||||
These examples describe the intended end-to-end client/server interaction.
|
||||
They explicitly include the strict **Review → Ask → Confirm** step before any
|
||||
matching execution when `matching.require_confirmation = true`.
|
||||
|
||||
#### Roundtrip A: Ad-hoc search (underspecified) using guided capture
|
||||
|
||||
1. `start_guided_capture()`
|
||||
2. `guided_set_description("...")` (must capture concrete scope/goal; skill-only is insufficient)
|
||||
3. `guided_set_role("Backend Developer")`
|
||||
4. `guided_set_time_range(date_start="2026-04-01", date_end="2026-06-30")`
|
||||
5. `guided_set_competences(["Python", "FastAPI", "Docker"])`
|
||||
6. **Review**: `show_pending_requirements()` (server returns a single review table)
|
||||
7. **Ask** (assistant → user): "Soll ich diese Anforderungen so übernehmen und die Suche starten?" (Ja/Nein)
|
||||
8. **Confirm** (only on "Ja"): `confirm_requirements(confirm=true)`
|
||||
9. Execute: `find_matching_capacities(role_name="Backend Developer", competences=[...], date_start="2026-04-01", date_end="2026-06-30")`
|
||||
- Server returns deterministic headers (`Using SEARCH_ID=...`, `SEARCH_ID=...`, `META=...`) plus `## Summary` and Top results.
|
||||
|
||||
If the user answers "Nein" in step 7, call `confirm_requirements(confirm=false)`
|
||||
and continue capturing/updating requirements.
|
||||
|
||||
#### Roundtrip B: DB task workflow (task_id-based)
|
||||
|
||||
1. `list_open_tasks(limit=...)`
|
||||
2. `get_task_details(task_id)`
|
||||
3. Optional helpers:
|
||||
- `infer_primary_role(task_id=...)` (if role unclear)
|
||||
- `validate_task_requirements(task_id)` (**only if the user explicitly requests validation**; validation is independent from matching)
|
||||
4. Capture/update requirements as needed (e.g. from task details)
|
||||
5. **Review**: `show_pending_requirements()`
|
||||
6. **Ask** (assistant → user): confirm Yes/No
|
||||
7. **Confirm** (only on "Yes"): `confirm_requirements(confirm=true)`
|
||||
8. Execute matching (depending on the client UX):
|
||||
- `find_capacities_for_task(task_id)` OR
|
||||
- `find_matching_capacities(...)`
|
||||
|
||||
> Alternative marker: If the assistant already displayed requirements to the user
|
||||
> outside of the server review table, it MAY call `request_requirements_confirmation()`
|
||||
> instead of `show_pending_requirements()`. The actual confirmation is still
|
||||
> performed exclusively via `confirm_requirements(confirm=true)` after user approval.
|
||||
|
||||
### 2. Database Layer (`src/teamlandkarte_mcp/database/trino_client.py`)
|
||||
|
||||
**Purpose**: Manage Trino/Presto connectivity to the Open Data Lake and execute read-only queries.
|
||||
|
||||
**Database Schema**:
|
||||
|
||||
**Capacity Tables** (existing):
|
||||
- `teamlandkarte_v_capacities_latest`
|
||||
- Fields: `id`, `owner_name`, `role_name`, `role_level`, `begin_date`, `end_date`, `deletion_reason`
|
||||
- Filter: `deletion_reason IS NULL` (active capacities only)
|
||||
- `teamlandkarte_v_capacity_competences_latest`
|
||||
- Fields: `capacity_id` (FK), `competence_id` (FK)
|
||||
- JOIN: Links capacities to competences
|
||||
- `teamlandkarte_v_competences_latest`
|
||||
- Fields: `id`, `name`
|
||||
- Stores competence/skill names
|
||||
|
||||
**Task Tables** (new for Round 5):
|
||||
- `beschaffungstool_kmp_task_latest`
|
||||
- Fields: `id`, `title__c`, `description__c`, `startdate__c`, `enddate__c`, `createddate`, `status__c`
|
||||
- Filter: `status__c = "Veröffentlicht"` (published tasks only)
|
||||
- Note: Role is NOT stored in DB, extracted from description
|
||||
- `beschaffungstool_kmp_skill_latest`
|
||||
- Fields: `task__c` (FK to task.id), `skillname__c`
|
||||
- JOIN: Links tasks to their required skills
|
||||
- Multiple rows per task (one per skill)
|
||||
|
||||
> Note: Trino catalog/schema are configured via `config.toml` (`catalog` / `schema`).
|
||||
> The queries in code use unqualified view names.
|
||||
|
||||
**Key Classes**:
|
||||
```python
|
||||
class TrinoClient:
|
||||
def __init__(self, config: DatabaseConfig):
|
||||
# Initialize Trino DB-API connection
|
||||
|
||||
def get_all_capacities_with_competences(self) -> List[Capacity]:
|
||||
"""Fetch all active capacities with their competences in a single query"""
|
||||
query = """
|
||||
SELECT
|
||||
cap.id,
|
||||
cap.owner_name,
|
||||
cap.role_name,
|
||||
cap.role_level,
|
||||
cap.begin_date,
|
||||
cap.end_date,
|
||||
comp.name as competence_name
|
||||
FROM teamlandkarte_v_capacities_latest cap
|
||||
LEFT JOIN teamlandkarte_v_capacity_competences_latest cap_comp
|
||||
ON cap.id = cap_comp.capacity_id
|
||||
LEFT JOIN teamlandkarte_v_competences_latest comp
|
||||
ON cap_comp.competence_id = comp.id
|
||||
WHERE cap.deletion_reason IS NULL
|
||||
ORDER BY cap.id, comp.name
|
||||
"""
|
||||
# Execute query with proper parameterization
|
||||
# Group results by capacity_id in Python
|
||||
# Return List[Capacity] with competences populated
|
||||
|
||||
def get_open_tasks(self, limit: int = 20) -> List[Task]:
|
||||
"""Fetch latest open tasks with their skills."""
|
||||
|
||||
# Apply LIMIT to tasks before joining skills, otherwise one task with
|
||||
# many skill rows can consume the full LIMIT.
|
||||
query = """
|
||||
WITH latest_tasks AS (
|
||||
SELECT
|
||||
t.id,
|
||||
t.title__c,
|
||||
t.description__c,
|
||||
t.startdate__c,
|
||||
t.enddate__c,
|
||||
t.createddate
|
||||
FROM beschaffungstool_kmp_task_latest t
|
||||
WHERE t.status__c = ?
|
||||
ORDER BY t.createddate DESC
|
||||
LIMIT ?
|
||||
)
|
||||
SELECT
|
||||
t.id,
|
||||
t.title__c,
|
||||
t.description__c,
|
||||
t.startdate__c,
|
||||
t.enddate__c,
|
||||
t.createddate,
|
||||
s.skillname__c
|
||||
FROM latest_tasks t
|
||||
LEFT JOIN beschaffungstool_kmp_skill_latest s
|
||||
ON t.id = s.task__c
|
||||
ORDER BY t.createddate DESC
|
||||
"""
|
||||
# Execute with limit parameter
|
||||
# Group results by task_id (multiple rows per task for skills)
|
||||
# Return List[Task] with skills populated
|
||||
|
||||
def get_task_by_id(self, task_id: str) -> Optional[Task]:
|
||||
"""Fetch a single task with its skills by ID"""
|
||||
query = """
|
||||
SELECT
|
||||
t.id,
|
||||
t.title__c,
|
||||
t.description__c,
|
||||
t.startdate__c,
|
||||
t.enddate__c,
|
||||
t.createddate,
|
||||
s.skillname__c
|
||||
FROM beschaffungstool_kmp_task_latest t
|
||||
LEFT JOIN beschaffungstool_kmp_skill_latest s
|
||||
ON t.id = s.task__c
|
||||
WHERE t.id = ? AND t.status__c = 'Veröffentlicht'
|
||||
"""
|
||||
# Execute with task_id parameter
|
||||
# Group skill rows into single Task object
|
||||
# Return Task or None if not found
|
||||
|
||||
@dataclass
|
||||
class Capacity:
|
||||
id: int
|
||||
owner_name: str
|
||||
role_name: str
|
||||
role_level: str
|
||||
begin_date: date
|
||||
end_date: date
|
||||
competences: List[str] = field(default_factory=list)
|
||||
|
||||
@dataclass
|
||||
class Task:
|
||||
id: str
|
||||
title: str
|
||||
description: str
|
||||
start_date: Optional[date]
|
||||
end_date: Optional[date]
|
||||
created_date: datetime
|
||||
skills: List[str] = field(default_factory.list)
|
||||
```
|
||||
|
||||
**Safety**:
|
||||
- Connection uses read-only user (enforced by DB permissions)
|
||||
- Query validation to prevent any INSERT/UPDATE/DELETE
|
||||
- Parameterized queries to prevent SQL injection
|
||||
- Single join query reduces round-trips and complexity
|
||||
|
||||
**Implementation Note**:
|
||||
The query returns multiple rows per capacity (one per competence). Python code groups these rows by capacity_id to create the final List[Capacity] structure.
|
||||
|
||||
### 3. Cache Layer (`src/cache/`)
|
||||
|
||||
**Purpose**: Provide two-tier caching for database queries and search results.
|
||||
|
||||
**Search session behavior**:
|
||||
- Matching creates a new `search_id` (UUID) and stores full results in-memory.
|
||||
- Filtering creates a new `filter_id` under the same `search_id`.
|
||||
- Search-related tools emit a JSON block so clients can reliably capture IDs.
|
||||
- Tools reject non-UUID `search_id` early to avoid confusing cache-miss flows.
|
||||
|
||||
**Implementation**:
|
||||
```python
|
||||
from cachetools import TTLCache
|
||||
from datetime import timedelta
|
||||
import json
|
||||
import uuid
|
||||
|
||||
class QueryCache:
|
||||
"""Database query cache with 12-hour TTL"""
|
||||
def __init__(self, ttl_hours: int = 12, max_size: int = 100):
|
||||
self.cache = TTLCache(
|
||||
maxsize=max_size,
|
||||
ttl=ttl_hours * 3600
|
||||
)
|
||||
self.stats = {"hits": 0, "misses": 0}
|
||||
|
||||
def get_or_fetch(self, key: str, fetch_fn: callable) -> Any:
|
||||
if key in self.cache:
|
||||
self.stats["hits"] += 1
|
||||
return self.cache[key]
|
||||
|
||||
self.stats["misses"] += 1
|
||||
result = fetch_fn()
|
||||
self.cache[key] = result
|
||||
return result
|
||||
|
||||
class SearchCache:
|
||||
"""Search results cache with 60-minute TTL, keyed by search_id"""
|
||||
def __init__(self, ttl_minutes: int = 60, max_size: int = 100):
|
||||
self.cache = TTLCache(
|
||||
maxsize=max_size,
|
||||
ttl=ttl_minutes * 60
|
||||
)
|
||||
self.stats = {"hits": 0, "misses": 0}
|
||||
|
||||
def store_search(self, search_data: dict) -> str:
|
||||
"""Store search results and return search_id"""
|
||||
search_id = str(uuid.uuid4())
|
||||
self.cache[search_id] = json.dumps(search_data)
|
||||
return search_id
|
||||
|
||||
def get_search(self, search_id: str) -> Optional[dict]:
|
||||
"""Retrieve search results by search_id"""
|
||||
if search_id in self.cache:
|
||||
self.stats["hits"] += 1
|
||||
return json.loads(self.cache[search_id])
|
||||
|
||||
self.stats["misses"] += 1
|
||||
return None
|
||||
|
||||
def update_search(self, search_id: str, search_data: dict) -> bool:
|
||||
"""Update existing search results (e.g., add filters)"""
|
||||
if search_id in self.cache:
|
||||
self.cache[search_id] = json.dumps(search_data)
|
||||
return True
|
||||
return False
|
||||
```
|
||||
|
||||
**Cache Keys**:
|
||||
|
||||
**QueryCache**:
|
||||
- `"all_capacities_with_competences"` - Complete dataset from database
|
||||
|
||||
**SearchCache**:
|
||||
- `<search_id>` (UUID) - Stores complete search results and associated filters as JSON:
|
||||
```json
|
||||
{
|
||||
"task_id": "T-12345", // NEW: Reference to database task (null for ad-hoc searches)
|
||||
"requirements": {
|
||||
"required_competences": ["Python", "FastAPI"],
|
||||
"preferred_role": "Backend Developer",
|
||||
"candidate_roles": ["Backend Developer", "Software Engineer"],
|
||||
"date_start": "2026-04-01",
|
||||
"date_end": "2026-06-30"
|
||||
},
|
||||
"results_by_category": {
|
||||
"Top": [...],
|
||||
"Good": [...],
|
||||
"Partial": [...],
|
||||
"Low": [...]
|
||||
},
|
||||
"filters": {
|
||||
"filter-1": {
|
||||
"criteria": {
|
||||
"role": "Developer",
|
||||
"competences": ["Python", "Docker"],
|
||||
"min_similarity": 0.7
|
||||
},
|
||||
"results_by_category": {
|
||||
"Top": [...],
|
||||
"Good": [...],
|
||||
"Partial": [...],
|
||||
"Low": [...]
|
||||
},
|
||||
"timestamp": "2026-02-11T14:35:00Z"
|
||||
},
|
||||
"filter-2": {
|
||||
"criteria": {
|
||||
"competences": ["Cloud"],
|
||||
"min_similarity": 0.7
|
||||
},
|
||||
"results_by_category": {...},
|
||||
"timestamp": "2026-02-11T14:40:00Z"
|
||||
}
|
||||
},
|
||||
"timestamp": "2026-02-11T14:30:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
**Notes**:
|
||||
- `task_id` field is `null` for ad-hoc searches (via `find_matching_capacities`)
|
||||
- `task_id` field contains database task ID for task-based searches (via `find_capacities_for_task`)
|
||||
- This allows tracking the source of each search for audit and debugging purposes
|
||||
|
||||
**Session State** (in-memory, per client session):
|
||||
- `"last_requirements"` - Stores TaskRequirements object for use with `update_requirements`
|
||||
- Cleared when new search is executed via `find_matching_capacities`
|
||||
|
||||
### 4. Matching Engine (`src/matching/matcher.py`)
|
||||
|
||||
**Purpose**: Analyze tasks and match capacities based on requirements.
|
||||
|
||||
**Key Components**:
|
||||
|
||||
#### Matching Logic
|
||||
```python
|
||||
# NOTE (2026-02): This section is historical pseudocode. The current
|
||||
# implementation uses Azure OpenAI via `AzureOpenAIClient` (chat + embeddings)
|
||||
# and does not rely on MCP sampling.
|
||||
|
||||
class CapacityMatcher:
|
||||
def __init__(
|
||||
self,
|
||||
config: MatchingConfig,
|
||||
similarity_engine,
|
||||
):
|
||||
self.config = config
|
||||
self.similarity_engine = similarity_engine
|
||||
|
||||
async def _match_competences(
|
||||
self,
|
||||
capacity_comps: List[str],
|
||||
required_comps: List[str],
|
||||
) -> float:
|
||||
"""Compute similarity via embeddings + cosine similarity."""
|
||||
|
||||
details = await self.similarity_engine.compute_competence_similarity(
|
||||
required=required_comps,
|
||||
candidate=capacity_comps,
|
||||
)
|
||||
if not details:
|
||||
return 0.0
|
||||
return sum(d["score"] for d in details.values()) / len(details)
|
||||
```
|
||||
|
||||
### 5. Scoring System (`src/matching/scorer.py`)
|
||||
|
||||
**Purpose**: Categorize match results into Top/Good/Partial/Low.
|
||||
|
||||
```python
|
||||
class ScoreCategory(Enum):
|
||||
TOP = "Top"
|
||||
GOOD = "Good"
|
||||
PARTIAL = "Partial"
|
||||
LOW = "Low"
|
||||
|
||||
class Scorer:
|
||||
def __init__(self, config: MatchingConfig):
|
||||
self.thresholds = {
|
||||
ScoreCategory.TOP: config.top_threshold,
|
||||
ScoreCategory.GOOD: config.good_threshold,
|
||||
ScoreCategory.PARTIAL: config.partial_threshold
|
||||
}
|
||||
|
||||
def categorize(self, score: float) -> ScoreCategory:
|
||||
if score >= self.thresholds[ScoreCategory.TOP]:
|
||||
return ScoreCategory.TOP
|
||||
elif score >= self.thresholds[ScoreCategory.GOOD]:
|
||||
return ScoreCategory.GOOD
|
||||
elif score >= self.thresholds[ScoreCategory.PARTIAL]:
|
||||
return ScoreCategory.PARTIAL
|
||||
else:
|
||||
return ScoreCategory.LOW
|
||||
|
||||
def group_by_category(self, results: List[MatchResult]) -> Dict[ScoreCategory, List[MatchResult]]:
|
||||
"""Group results by score category"""
|
||||
grouped = {cat: [] for cat in ScoreCategory}
|
||||
for result in results:
|
||||
category = self.categorize(result.overall_score)
|
||||
result.category = category
|
||||
grouped[category].append(result)
|
||||
return grouped
|
||||
```
|
||||
|
||||
### 6. MCP Server (`src/mcp_server.py`)
|
||||
|
||||
**Purpose**: Expose capacity matching as MCP tools.
|
||||
|
||||
**Workflow Design**:
|
||||
|
||||
The server provides a **multi-step workflow** with separate tools for each phase:
|
||||
|
||||
0. **Task Management Phase** (NEW in Round 5):
|
||||
- `list_open_tasks()` - Browse published tasks from database
|
||||
- `get_task_details()` - View task details with extracted role
|
||||
- `validate_task_requirements()` - Compare DB skills vs description-extracted skills
|
||||
- `find_capacities_for_task()` - Automated capacity search for database task
|
||||
|
||||
1. **Requirement Gathering Phase** (Ad-hoc):
|
||||
- `extract_requirements()` - Extract from natural language task description
|
||||
- `collect_structured_requirement_data()` - Interactive step-by-step collection
|
||||
- `update_requirements()` - Modify previously extracted requirements
|
||||
|
||||
2. **Search Execution Phase**:
|
||||
- `find_matching_capacities()` - Execute search with confirmed requirements
|
||||
|
||||
3. **Result Refinement Phase**:
|
||||
- `filter_search_results()` - Apply fuzzy filters to results
|
||||
- `get_results_by_category()` - Browse results by score category
|
||||
|
||||
**Key Design Principles**:
|
||||
- **Separation of Concerns**: Each tool has a single, clear responsibility
|
||||
- **LLM State Management**: The client-side LLM tracks requirement state across tool calls
|
||||
- **Explicit Confirmation**: Requirements are returned for user confirmation before search execution
|
||||
- **Filter Chain Tracking**: Filters use `filter_id` to maintain navigation hierarchy
|
||||
- **Database Task Integration**: Seamless workflow for published tasks with automatic validation
|
||||
- **Common Extraction Logic**: DRY principle with shared `_extract_requirements_from_description()` function
|
||||
- **Availability as filter**: Availability is displayed and can be used as a date-range filter (not part of scoring)
|
||||
|
||||
**MCP Tools**:
|
||||
@@ -1,141 +0,0 @@
|
||||
# Change: Add Capacity Matching MCP Server (Stage 1)
|
||||
|
||||
## Why
|
||||
DB Systel employees have free work capacities stored in the Open Data Lake, but there's no efficient way for AI assistants to match these capacities with tasks. Tasks are also stored in the database, but finding suitable capacities for them requires manual work. This MCP server will enable AI clients to automatically:
|
||||
1. Browse and search published tasks from the database
|
||||
2. Validate task requirements (comparing DB skills vs description-extracted skills)
|
||||
3. Find the best-suited employees for both database tasks and ad-hoc requirements
|
||||
4. Match capacities based on competences and role, with optional availability (date-range) filtering
|
||||
|
||||
## What Changes
|
||||
- Add MCP server implementation with **10-tool workflow** (task management, requirement gathering, search, and refinement)
|
||||
- Add **database task management** (4 new tools): list tasks, view details, validate requirements, automated search
|
||||
- Add **task database integration**: Query published tasks with skills from Beschaffungstool KMP tables
|
||||
- Add **requirement validation**: Compare database skills vs LLM-extracted skills from description
|
||||
- Add **common extraction function**: DRY principle with `_extract_requirements_from_description()`
|
||||
- Add **internal helper functions**: `extract_requirements_from_task()`, `extract_roles_from_task()`
|
||||
- Add database connection layer for Trino/Presto with single join query
|
||||
- Requirement extraction and semantic matching are implemented via **Azure OpenAI**:
|
||||
- Chat completions for role/requirement extraction
|
||||
- Embeddings + cosine similarity for semantic similarity scoring
|
||||
- Errors are deterministic (no heuristic fallbacks)
|
||||
|
||||
> **Historical note (2026-02)**: The initial Stage-1 proposal assumed only
|
||||
> client-side LLM access via MCP sampling and used heuristic fallbacks because
|
||||
> sampling was not available in the runtime. The current implementation has been
|
||||
> migrated to an Azure OpenAI-only integration (see change
|
||||
> `replace-heuristics-with-azure-openai`).
|
||||
|
||||
- Add **session state management** for requirement tracking across tool calls
|
||||
- Add configurable scoring system with categorical results (Top/Good/Partial/Low)
|
||||
- Add **two-tier caching**: database cache (12h TTL) and search results cache (60min TTL) with task_id tracking
|
||||
- Emit deterministic, machine-readable headers in search-related tool outputs so MCP
|
||||
clients can reliably capture `search_id` / `filter_id`:
|
||||
- first line: `Using SEARCH_ID=<uuid>`
|
||||
- plus `SEARCH_ID=<uuid>`, `FILTER_ID=<uuid>` (if applicable), `META=<json>`
|
||||
- Reject non-UUID `search_id` values early with a clear message
|
||||
- Track the latest search id internally for debugging/consistency, but do **not** expose a "last search id" tool (multi-user/session unsafe)
|
||||
- Add **requirement workflow**: extract from free text, collect structured, or update requirements
|
||||
- Add **explicit confirmation** before search execution
|
||||
- Remove availability from weighted scoring (availability is displayed and can be used as a filter)
|
||||
- Update default weights: competence 0.8, role 0.2
|
||||
- Keep date range as an optional availability filter (when provided)
|
||||
- Add interactive result exploration with **search session management** (search_id)
|
||||
- Add **fuzzy filtering** for post-search refinement (role and/or competences as list)
|
||||
- Add **filter tracking** with filter_id (no search_id proliferation)
|
||||
- Add **pagination** for result browsing (20 results per page)
|
||||
- Add database configuration in `database.toml` (Trino)
|
||||
- Add **Arc42 architecture documentation** with ADRs
|
||||
|
||||
### Strict two-step confirmation (Review → Ask → Confirm)
|
||||
|
||||
When `matching.require_confirmation = true` (default), the server enforces a
|
||||
strict, user-driven confirmation sequence before any matching run:
|
||||
|
||||
1. Requirements are captured/updated into **pending** state (e.g. via
|
||||
`extract_requirements`, `collect_structured_requirement_data`, `update_requirements`,
|
||||
or guided capture tools).
|
||||
2. The assistant must **review** pending requirements using
|
||||
`show_pending_requirements()` (preferred) or, if requirements were already
|
||||
shown to the user in chat, call `request_requirements_confirmation()` as a
|
||||
marker.
|
||||
3. The assistant must **ask** the user explicitly for Yes/No confirmation.
|
||||
4. Only after user approval, the assistant calls `confirm_requirements(confirm=true)`.
|
||||
If the user says No, it calls `confirm_requirements(confirm=false)` and
|
||||
continues requirement capture.
|
||||
|
||||
The server refuses to confirm requirements unless step 2 has happened first
|
||||
(prevents auto-confirm in the same turn as requirement capture).
|
||||
|
||||
**Stage 1 Scope:**
|
||||
This is the first development stage focusing on basic matching functionality and database task integration. Future stages will add more sophisticated features (to be specified later).
|
||||
|
||||
**Key Architectural Decisions:**
|
||||
- **Azure OpenAI integration**: Server calls Azure OpenAI directly for extraction
|
||||
and embeddings (no sampling dependency)
|
||||
- **10-tool workflow**: Separate tools for task management (4), requirement gathering (3), search execution (1), and result refinement (2)
|
||||
- **Database task integration**: Direct access to published tasks with skills from Beschaffungstool KMP
|
||||
- **Validation-only approach**: Validation compares DB vs description but always uses DB skills for search
|
||||
- **Common extraction function**: `_extract_requirements_from_description()` as single source of truth (DRY principle)
|
||||
- **Automatic task workflow**: `find_capacities_for_task()` fully automated with validation display
|
||||
- **Task reference tracking**: Search cache includes `task_id` field (null for ad-hoc, task ID for DB tasks)
|
||||
- **Explicit confirmation**: Requirements returned for user confirmation before search
|
||||
- **Session state**: In-memory storage of requirements for update workflow
|
||||
- **Filter tracking**: Filters stored within search cache using filter_id (not new search_id)
|
||||
- **Search sessions**: Complete results stored server-side, accessed via search_id (60min expiry)
|
||||
- **Single query pattern**: One JOIN query fetches all data, processed in Python
|
||||
- **Markdown output**: All tables formatted as markdown for better client rendering
|
||||
- **No truncation**: Full competence lists always shown
|
||||
- **No availability scoring**: Availability is displayed and can be used as a date-range filter, but does not influence the score
|
||||
- **Weights**: competence 0.8, role 0.2
|
||||
- **Fuzzy matching**: Filter tool uses fuzzywuzzy for flexible result refinement
|
||||
- **LLM integration**: Azure OpenAI (`gpt-4.1` chat + `text-embedding-3-large`)
|
||||
with embedding cache; no heuristic fallback
|
||||
|
||||
## Impact
|
||||
- **New capability**: `capacity-matching` - Core matching functionality with database task integration
|
||||
- **New files**:
|
||||
- `database.toml` - Database connection configuration
|
||||
- `architecture.md` - Arc42 architecture documentation with ADRs
|
||||
- `src/teamlandkarte_mcp/mcp_server.py` - MCP server implementation (10 tools)
|
||||
- `src/teamlandkarte_mcp/database/trino_client.py` - Trino client (capacity + task queries)
|
||||
- `src/teamlandkarte_mcp/matching/task_analyzer.py` - Azure-based task analysis
|
||||
- `src/matching/matcher.py` - Matching algorithm with common extraction function
|
||||
- `src/matching/scorer.py` - Scoring logic
|
||||
- `src/cache/query_cache.py` - Database query caching (12h TTL)
|
||||
- `src/cache/search_cache.py` - Search results caching (60min TTL) with task_id tracking
|
||||
- `src/config.py` - Configuration management
|
||||
- `requirements.txt` or updated `pyproject.toml` - Dependencies
|
||||
- **Modified files**:
|
||||
- `main.py` - Launch MCP server
|
||||
- `pyproject.toml` - Add dependencies
|
||||
- `README.md` - Update with task management features, validation workflow
|
||||
- **Dependencies added**:
|
||||
- `mcp` - Model Context Protocol SDK
|
||||
- `trino` - Trino/Presto database client
|
||||
- `cachetools` - Caching implementation
|
||||
- `fuzzywuzzy` - Fuzzy string matching for filters
|
||||
|
||||
## Security & Constraints
|
||||
- **Read-only operations**: Server must NEVER modify database data
|
||||
- **Credential storage**: Credentials are provided via environment variables
|
||||
(recommended: local `.env`): `DATA_LAKE_USERNAME`, `DATA_LAKE_PASSWORD`.
|
||||
`database.toml` contains non-secret connection settings and must not be
|
||||
committed.
|
||||
- **Network security**: Trino/Presto connection over port 8446
|
||||
- **Data filtering**:
|
||||
- Capacities: Only consider where `deletion_reason IS NULL`
|
||||
- Tasks: Only consider where `status__c = "Veröffentlicht"`
|
||||
- **LLM access**: Not available in the current server runtime. If reintroduced,
|
||||
it must use a supported MCP API.
|
||||
- **SQL injection prevention**: All queries use proper parameterization
|
||||
- **Validation scope**: Comparison only (skills and dates), role not validated (not in DB yet)
|
||||
|
||||
## Non-Goals (Future Stages)
|
||||
- Advanced filtering options beyond fuzzy matching
|
||||
- Capacity booking/reservation
|
||||
- Task modification or status updates
|
||||
- Historical analysis
|
||||
- Integration with other systems
|
||||
- Multi-language support
|
||||
- Role storage in database (currently extracted from description)
|
||||
-99
@@ -1,99 +0,0 @@
|
||||
# Capacity Matching Specification
|
||||
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: Workflow overview (Round 6)
|
||||
|
||||
The system SHALL support two primary workflows:
|
||||
|
||||
1. **Database task workflow (Phase 0)**: Users browse published tasks from the database, inspect details, optionally validate, and run a capacity search for an existing task.
|
||||
2. **Ad-hoc workflow (Phases 1-3)**: Users gather requirements (free-text extraction, structured input, or guided capture), optionally update requirements, explicitly confirm, execute search, and refine results.
|
||||
|
||||
The system SHALL expose the MCP tools required to support the workflows below. The implementation MAY add additional tools over time.
|
||||
|
||||
#### Scenario: Tool surface includes both DB-task and ad-hoc workflows
|
||||
|
||||
**Phase 0 – Task Management (Database tasks)**
|
||||
- `list_open_tasks(limit=20)`
|
||||
- `get_task_details(task_id)`
|
||||
- `validate_task_requirements(task_id)`
|
||||
- `find_capacities_for_task(task_id)`
|
||||
- `infer_roles(task_id? | task_description?, limit=5)`
|
||||
|
||||
**Phase 1 – Requirement Gathering (Ad-hoc)**
|
||||
- `extract_requirements(task_description)` → writes pending requirements
|
||||
- `collect_structured_requirement_data(role_name?, competences?, date_start?, date_end?, description?)` → writes pending requirements
|
||||
- `update_requirements(change_description)` → writes pending requirements
|
||||
- Guided capture tools:
|
||||
- `start_guided_capture()`
|
||||
- `guided_set_description(description)`
|
||||
- `guided_set_role(role_name)`
|
||||
- `guided_set_time_range(date_start?, date_end?)` (open-ended allowed)
|
||||
- `guided_set_competences(competences)` → writes pending requirements
|
||||
- Review/ask tools:
|
||||
- `show_pending_requirements()` → returns a review table and marks that user confirmation was requested
|
||||
- `request_requirements_confirmation()` → marks that the assistant asked the user to confirm (no table)
|
||||
- `confirm_requirements(confirm: bool = True)`
|
||||
|
||||
**Phase 2 – Search Execution (Ad-hoc)**
|
||||
- `find_matching_capacities(role_name, competences, date_start?, date_end?)`
|
||||
|
||||
**Phase 3 – Result Refinement**
|
||||
- `filter_search_results(search_id, role_filter?, competence_filter?, availability_date_start?, availability_date_end?, min_similarity=0.7)` → returns `filter_id`
|
||||
- `get_results_by_category(search_id, filter_id?, category="Top", page=1, page_size=20)`
|
||||
|
||||
### Requirement: Confirmation gating
|
||||
|
||||
The system SHALL ensure that matching execution is confirmation-gated by default and behaves deterministically based on configuration.
|
||||
|
||||
#### Scenario: Matching is refused until requirements are confirmed (default)
|
||||
|
||||
- Confirmation gating:
|
||||
- The server's confirmation state is stored in in-memory session state and is **not** safe as a multi-client / multi-tenant source of truth.
|
||||
Therefore, treat confirmation gating primarily as a **client-side UX rule**: clients should only trigger matching after an explicit user Yes/No.
|
||||
- If `matching.require_confirmation = true` (default), matching tools MUST refuse to run unless requirements have been explicitly confirmed.
|
||||
- If `matching.require_confirmation = false`, clients MUST NOT ask for confirmation and MUST NOT call confirmation tools; matching MUST proceed once requirements are complete.
|
||||
- The confirmation flow MUST be two-step:
|
||||
1) `show_pending_requirements()` (preferred) OR `request_requirements_confirmation()`
|
||||
2) user explicitly confirms in chat
|
||||
3) `confirm_requirements(confirm=true)`
|
||||
- Servers MAY refuse `confirm_requirements(confirm=true)` if step (1) did not occur.
|
||||
|
||||
### Requirement: Tool outputs and result headers
|
||||
|
||||
The system SHALL emit table-first tool output formats and deterministic, machine-readable headers for search-session tools.
|
||||
|
||||
#### Scenario: Tool results are table-first and include deterministic header metadata
|
||||
|
||||
- Tool outputs:
|
||||
- `get_task_details` MUST be table-first (summary table + description below)
|
||||
- `validate_task_requirements` MUST be table-first (single comparison table + short written summary)
|
||||
- `infer_roles` MUST return a ranked table: `Rank | Role | Rationale`
|
||||
|
||||
- Search session output headers (robust client parsing):
|
||||
- The tools `find_matching_capacities`, `filter_search_results`, and `get_results_by_category` MUST emit deterministic, machine-readable header lines.
|
||||
- On success, the first line MUST be exactly:
|
||||
- `Using SEARCH_ID=<uuid>`
|
||||
- The output MUST also contain:
|
||||
- `SEARCH_ID=<uuid>`
|
||||
- `FILTER_ID=<uuid>` (only if a filter was created/used)
|
||||
- `META=<json>` (MUST include at least `{ "search_id": "...", "status": "ok"|... }`)
|
||||
|
||||
### Requirement: Search session validation and table guarantees
|
||||
|
||||
The system SHALL validate search session identifiers deterministically and MUST always return well-formed Markdown tables for search result tools.
|
||||
|
||||
#### Scenario: Invalid or expired search_id inputs produce deterministic errors and tables are always present
|
||||
|
||||
- Search session validation:
|
||||
- `search_id` inputs MUST be validated as UUIDs.
|
||||
- If the `search_id` is not a UUID, the tool MUST return `META.status=invalid_search_id_format` and guidance to copy the id from tool output.
|
||||
- If the `search_id` is unknown/expired, the tool MUST return `META.status=unknown_or_expired` and guidance to re-run the matching tool.
|
||||
|
||||
- Table-first guarantees for search results:
|
||||
- `find_matching_capacities` MUST include the `## Summary` table.
|
||||
- `find_matching_capacities` MUST include a results table for the **first non-empty category** in this order: Top → Good → Partial → Low.
|
||||
- If all categories are empty, it MUST still emit a valid Markdown table (a single dummy row is acceptable).
|
||||
- `filter_search_results` and `get_results_by_category` MUST NOT repeat the summary counts table.
|
||||
- `filter_search_results` MUST output a flat results table (not split by category) and MUST include a `Category` column **separate** from the numeric `Score`.
|
||||
- `filter_search_results` MUST always return the results as a valid Markdown table (even for exactly one match, and also when there are zero matches).
|
||||
@@ -1,230 +0,0 @@
|
||||
## 1. Project Setup
|
||||
- [x] 1.1 Update `pyproject.toml` with required dependencies (mcp, trino, cachetools)
|
||||
- [x] 1.2 Create `database.toml` with connection configuration structure (template only, no real credentials)
|
||||
- [x] 1.3 Create project structure: `src/`, `src/database/`, `src/matching/`, `src/cache/`
|
||||
- [x] 1.4 Add `.gitignore` entry for `database.toml` to prevent credential exposure
|
||||
|
||||
## 2. Configuration Management
|
||||
- [x] 2.1 Create `src/config.py` to load database and matching configuration
|
||||
- [x] 2.2 Implement configuration validation (ensure required fields present)
|
||||
- [x] 2.3 Add configurable scoring thresholds (Top: ≥80%, Good: 60-79%, Partial: 40-59%, Low: <40%)
|
||||
- [x] 2.4 Add configurable matching weights (competence: 0.8, role: 0.2)
|
||||
- [x] 2.5 Validate that weights sum to 1.0
|
||||
- [x] 2.6 Support two cache TTL settings: db_ttl_hours (12h), search_ttl_minutes (60min)
|
||||
- [x] 2.7 Add fuzzy matching configuration: min_similarity (default: 0.7)
|
||||
|
||||
## 3. Database Layer
|
||||
- [x] 3.1 Create Trino client with connection handling
|
||||
- [x] 3.2 Implement connection pooling/management
|
||||
- [x] 3.3 **VERIFY SCHEMA**: Confirm database schema and field names with actual Open Data Lake tables (via Trino)
|
||||
- [x] 3.4 Implement single join query method:
|
||||
- [x] 3.4.1 `get_all_capacities_with_competences()` - Single query joining all 3 tables
|
||||
- [x] 3.4.2 Filter: deletion_reason IS NULL
|
||||
- [x] 3.4.3 Use proper parameterized queries (no string interpolation)
|
||||
- [x] 3.4.4 Group results by capacity_id in Python to build List[Capacity]
|
||||
- [x] 3.5 Ensure read-only operations (no INSERT/UPDATE/DELETE queries)
|
||||
- [x] 3.6 Add error handling for connection failures with retry logic (3x, exponential backoff)
|
||||
- [x] 3.7 Add query logging for debugging (without sensitive data)
|
||||
- [x] 3.8 **NEW**: Implement task database queries:
|
||||
- [x] 3.8.1 Create `Task` dataclass with fields: id, title__c, description__c, startdate__c, enddate__c, createddate, skills
|
||||
- [x] 3.8.2 `get_open_tasks(limit)` - Fetch published tasks sorted by createddate DESC
|
||||
- [x] 3.8.3 JOIN task table with task skills table (task.id = skill.task__c)
|
||||
- [x] 3.8.4 Filter: status__c = "Veröffentlicht"
|
||||
- [x] 3.8.5 Group results by task_id in Python to build List[Task]
|
||||
- [x] 3.8.6 `get_task_by_id(task_id)` - Fetch single task with skills by ID
|
||||
- [x] 3.8.7 Use parameterized queries for task_id parameter
|
||||
|
||||
## 4. Caching Layer
|
||||
- [x] 4.1 Create `src/cache/query_cache.py` for database cache (12h TTL)
|
||||
- [x] 4.2 Create `src/cache/search_cache.py` for search results cache (60min TTL)
|
||||
- [x] 4.3 Implement QueryCache with cachetools.TTLCache:
|
||||
- [x] 4.3.1 Key: "all_capacities_with_competences"
|
||||
- [x] 4.3.2 TTL: 12 hours
|
||||
- [x] 4.3.3 Cache statistics (hits/misses)
|
||||
- [x] 4.4 Implement SearchCache with cachetools.TTLCache:
|
||||
- [x] 4.4.1 Key: search_id (UUID)
|
||||
- [x] 4.4.2 Value: JSON-serialized search results with task_id (null or task ID), filters dict
|
||||
- [x] 4.4.3 TTL: 60 minutes
|
||||
- [x] 4.4.4 Methods: store_search() returns search_id, get_search(search_id) returns dict, update_search() for adding filters
|
||||
- [x] 4.5 Ensure thread-safe cache operations
|
||||
|
||||
## 5. Matching Engine
|
||||
- [x] 5.1 Create `src/matching/matcher.py` for matching logic
|
||||
- [x] 5.2 Implement task analysis using Azure OpenAI (chat completion):
|
||||
- [x] 5.2.1 **NEW**: Create common internal function `_extract_requirements_from_description(description)` as single source of truth
|
||||
- [x] 5.2.2 Free-text mode: `analyze_free_text()` delegates to common extraction function
|
||||
- [x] 5.2.3 Structured mode: Parse provided parameters directly
|
||||
- [x] 5.2.4 Generate and return task analysis summary for free-text mode
|
||||
- [x] 5.2.5 Handle Azure API errors (timeouts, invalid JSON) with retry logic
|
||||
- [x] 5.2.6 **NEW**: Implement internal helper `extract_requirements_from_task(task_id)` - fetches task from DB and uses common extraction
|
||||
- [x] 5.2.7 **UPDATED**: Implement internal helper `extract_roles_from_task(description)` - returns ranked role list; primary role is roles[0] (if any)
|
||||
- [x] 5.3 Implement matching algorithm:
|
||||
- [x] 5.3.1 Semantic competence matching via Azure OpenAI embeddings
|
||||
- [x] 5.3.2 Role name matching (exact and similarity)
|
||||
- [x] 5.3.3 Availability date range filtering (open-ended capacities: missing end_date means available without limit)
|
||||
- [x] 5.3.4 Availability is NOT scored. If date_start/date_end are provided, availability is used as a filter only.
|
||||
- [x] 5.3.5 Apply configured weights (competence + role) to calculate overall score
|
||||
- [x] 5.4 Create `src/matching/scorer.py` for scoring logic
|
||||
- [x] 5.5 Implement categorical scoring (Top/Good/Partial/Low) with configurable thresholds
|
||||
- [x] 5.6 Generate match score and category for each capacity
|
||||
- [x] 5.7 Group results by category for storage
|
||||
|
||||
## 6. MCP Server Implementation
|
||||
- [x] 6.1 Create `src/mcp_server.py` with MCP SDK integration
|
||||
- [x] 6.2 Integrate Azure OpenAI client for LLM access (chat + embeddings)
|
||||
- [x] 6.3 Implement session state management (in-memory, per client session)
|
||||
- [x] 6.3.1 Store `last_requirements` for use with update_requirements
|
||||
- [x] 6.3.2 Clear session state when new search is executed
|
||||
- [x] 6.4 **NEW**: Implement MCP tool: `list_open_tasks` (Phase 0)
|
||||
- [x] 6.4.1 Tool parameters: limit (default: 20)
|
||||
- [x] 6.4.2 Query database for published tasks (status = "Veröffentlicht")
|
||||
- [x] 6.4.3 Sort by createddate DESC (newest first)
|
||||
- [x] 6.4.4 Format as markdown table: Task ID, Title, Created Date, Start/End Dates, Skills Count
|
||||
- [x] 6.4.5 Return table with next steps hints
|
||||
- [x] 6.5 **NEW**: Implement MCP tool: `get_task_details` (Phase 0)
|
||||
- [x] 6.5.1 Tool parameters: task_id (required)
|
||||
- [x] 6.5.2 Fetch task from database by ID
|
||||
- [x] 6.5.3 Extract ranked roles from description using `extract_roles_from_task()` helper; assign primary role as roles[0]
|
||||
- [x] 6.5.4 Format task details with metadata, description, DB skills, extracted role
|
||||
- [x] 6.5.5 Return formatted details with next steps hints
|
||||
- [x] 6.6 **NEW**: Implement MCP tool: `validate_task_requirements` (Phase 0)
|
||||
- [x] 6.6.1 Tool parameters: task_id (required)
|
||||
- [x] 6.6.2 Fetch task from database
|
||||
- [x] 6.6.3 Extract requirements from description using `extract_requirements_from_task()` helper
|
||||
- [x] 6.6.4 Compare DB skills vs extracted skills (set operations)
|
||||
- [x] 6.6.5 Compare DB dates vs extracted dates
|
||||
- [x] 6.6.6 Build comparison table: Type, In Description, In Database, Status
|
||||
- [x] 6.6.7 Generate summary with matched/missing counts
|
||||
- [x] 6.6.8 Return validation table (informational only, no modifications)
|
||||
- [x] 6.7 **NEW**: Implement MCP tool: `find_capacities_for_task` (Phase 0)
|
||||
- [x] 6.7.1 Tool parameters: task_id (required)
|
||||
- [x] 6.7.2 Fetch task from database
|
||||
- [x] 6.7.3 Extract requirements for validation display
|
||||
- [x] 6.7.4 Run validation internally (build comparison table)
|
||||
- [x] 6.7.5 Build search requirements using DB skills, extracted role, DB dates
|
||||
- [x] 6.7.6 Execute capacity search (same as find_matching_capacities)
|
||||
- [x] 6.7.7 Store in search cache with task_id field set to task ID
|
||||
- [x] 6.7.8 Return validation table + search results (combined response)
|
||||
- [x] 6.8 **EXISTING**: Implement MCP tool: `extract_requirements` (Phase 1)
|
||||
- [x] 6.8.1 Tool parameters: task_description (required), confirm_requirements (default: True)
|
||||
- [x] 6.8.2 Use Azure OpenAI (chat completion) to extract structured requirements from free text
|
||||
- [x] 6.8.3 Store extracted requirements in session state
|
||||
- [x] 6.8.4 Return JSON-formatted requirements
|
||||
- [x] 6.8.5 Include confirmation prompt if confirm_requirements=True
|
||||
- [x] 6.9 **EXISTING**: Implement MCP tool: `collect_structured_requirement_data` (Phase 1)
|
||||
- [x] 6.9.1 Tool parameters: role_name, competences, date_start, date_end (all optional), confirm_requirements (default: True)
|
||||
- [x] 6.9.2 Check completeness: require at least role_name AND competences
|
||||
- [x] 6.9.3 Prompt for missing required parameters with examples
|
||||
- [x] 6.9.4 Store collected requirements in session state
|
||||
- [x] 6.9.5 Return JSON-formatted requirements with confirmation prompt
|
||||
- [x] 6.10 **EXISTING**: Implement MCP tool: `update_requirements` (Phase 1)
|
||||
- [x] 6.10.1 Tool parameters: change_description (required), confirm_requirements (default: True)
|
||||
- [x] 6.10.2 Check if requirements exist in session state
|
||||
- [x] 6.10.3 Use Azure OpenAI (chat completion) to interpret change and update requirements
|
||||
- [x] 6.10.4 Update session state with modified requirements
|
||||
- [x] 6.10.5 Return updated JSON-formatted requirements with confirmation prompt
|
||||
- [x] 6.11 **EXISTING**: Implement MCP tool: `find_matching_capacities` (Phase 2)
|
||||
- [x] 6.11.1 Tool parameters: role_name (required), competences (required), date_start (optional), date_end (optional)
|
||||
- [x] 6.11.2 **NO extraction or interpretation** - accept only explicit structured parameters
|
||||
- [x] 6.11.3 Execute matching algorithm (uses Azure OpenAI embeddings for competence matching; score uses competences+role only; availability is filter-only)
|
||||
- [x] 6.11.4 Generate summary table with count by score category (markdown format)
|
||||
- [x] 6.11.5 Store complete results in SearchCache with task_id=null, empty filters dict
|
||||
- [x] 6.11.6 Return search_id + summary + Top category results (markdown table)
|
||||
- [x] 6.11.7 Clear session state after successful search
|
||||
- [x] 6.12 **EXISTING**: Implement MCP tool: `filter_search_results` (Phase 3)
|
||||
- [x] 6.12.1 Ensure fuzzywuzzy dependency is installed
|
||||
- [x] 6.12.2 Tool parameters: search_id (required), role_filter (optional), competence_filter (optional List[str]), availability_date_start (optional), availability_date_end (optional), min_similarity (default: 0.7)
|
||||
- [x] 6.12.3 **ALWAYS use original search_id as input** (not filtered search_id)
|
||||
- [x] 6.12.4 Validate at least one filter is provided
|
||||
- [x] 6.12.5 Retrieve original search results from SearchCache
|
||||
- [x] 6.12.6 Collect all results across all categories
|
||||
- [x] 6.12.7 Apply fuzzy matching: role_filter (fuzzy match on role_name), competence_filter (ALL competences must match at least one capacity competence)
|
||||
- [x] 6.12.8 Combine filters with AND logic
|
||||
- [x] 6.12.9 Re-categorize filtered results using Scorer
|
||||
- [x] 6.12.10 Generate unique filter_id (e.g., "filter-1", "filter-2")
|
||||
- [x] 6.12.11 Store filtered results under search_data["filters"][filter_id]
|
||||
- [x] 6.12.12 Update search cache with filter data
|
||||
- [x] 6.12.13 Return filter_id + search_id + filter summary + Top category results
|
||||
- [x] 6.13 **EXISTING**: Implement MCP tool: `get_results_by_category` (Phase 3)
|
||||
- [x] 6.13.1 Tool parameters: search_id (required), filter_id (optional), category (default: "Top"), page (default: 1), page_size (default: 20)
|
||||
- [x] 6.13.2 Retrieve search results from SearchCache by search_id
|
||||
- [x] 6.13.3 If filter_id provided, retrieve filtered results from search_data["filters"][filter_id]
|
||||
- [x] 6.13.4 If filter_id not provided, use original search results
|
||||
- [x] 6.13.5 Extract category results, apply pagination
|
||||
- [x] 6.13.6 Format as markdown table with full competence lists
|
||||
- [x] 6.13.7 Return paginated results with page info and next page hint
|
||||
- [x] 6.14 Format results as markdown tables with fields: id, owner_name, role_name, role_level, begin_date, end_date, competences
|
||||
- [x] 6.15 Show full competence lists (no truncation)
|
||||
- [x] 6.16 Add tool descriptions and parameter schemas for MCP client
|
||||
|
||||
> **Update (2026-02)**: Items previously described as "via MCP sampling" are now
|
||||
> implemented with **Azure OpenAI** (chat + embeddings) as part of the change
|
||||
> `replace-heuristics-with-azure-openai`.
|
||||
>
|
||||
> There is **no runtime MCP sampling dependency** and **no heuristic fallback**
|
||||
> path in the current implementation.
|
||||
|
||||
## 7. Integration & Entry Point
|
||||
- [x] 7.1 Update `main.py` to launch MCP server
|
||||
- [x] 7.2 Add command-line arguments for server configuration (port, debug mode, etc.)
|
||||
- [x] 7.3 Implement graceful startup and shutdown
|
||||
- [x] 7.4 Add logging configuration
|
||||
|
||||
## 8. Testing & Validation
|
||||
- [x] 8.1 Manual testing with sample task descriptions
|
||||
- [x] 8.2 Verify database connection and single join query execution
|
||||
- [x] 8.3 Test both input modes (free-text via Azure OpenAI and structured)
|
||||
- [x] 8.4 **NEW**: Test incremental structured mode (partial parameters with prompts)
|
||||
- [x] 8.5 Validate scoring and categorization with updated weights (0.8/0.2)
|
||||
- [x] 8.6 Test that availability does not affect score and is only applied as an optional filter
|
||||
- [x] 8.7 Test two-tier caching behavior (DB cache 12h, search cache 60min)
|
||||
- [x] 8.8 Verify read-only constraint (no writes to database)
|
||||
- [x] 8.9 Test error handling (database unavailable, Azure OpenAI timeout, invalid search_id)
|
||||
- [x] 8.10 Test search session management (search_id creation and retrieval)
|
||||
- [x] 8.11 Test pagination in get_results_by_category
|
||||
- [x] 8.12 Verify markdown table formatting and full competence display
|
||||
- [x] 8.13 **NEW**: Test filter_search_results with role filter only
|
||||
- [x] 8.14 **NEW**: Test filter_search_results with competence filter only
|
||||
- [x] 8.15 **NEW**: Test filter_search_results with combined filters
|
||||
- [x] 8.16 **NEW**: Test multi-step filtering (filter a filtered search)
|
||||
- [x] 8.17 **NEW**: Test fuzzy matching with various similarity thresholds
|
||||
|
||||
## 9. Documentation
|
||||
- [x] 9.1 Update `README.md` with project description and setup instructions
|
||||
- [x] 9.2 Document `database.toml` structure and required fields
|
||||
- [x] 9.3 Document three MCP tools and their parameters (find_matching_capacities, get_results_by_category, filter_search_results)
|
||||
- [x] 9.4 Add usage examples for both input modes (free-text and incremental structured)
|
||||
- [x] 9.5 Document configuration options (weights, thresholds, cache TTLs, fuzzy matching)
|
||||
- [x] 9.6 Add troubleshooting guide
|
||||
- [x] 9.7 Document Azure OpenAI architecture and client-side LLM requirement
|
||||
- [x] 9.8 Document pagination and search session workflow
|
||||
- [x] 9.9 Document availability behavior (display + date-range filtering; open-ended capacities)
|
||||
- [x] 9.10 **NEW**: Document fuzzy filtering workflow and multi-step filtering
|
||||
- [x] 9.11 **NEW**: Add examples of filter_search_results usage
|
||||
|
||||
## 10. Security & Configuration
|
||||
- [x] 10.1 Ensure `database.toml` is in `.gitignore`
|
||||
- [x] 10.2 Create `database.toml.example` as template (already done)
|
||||
- [x] 10.3 Validate secure credential handling
|
||||
- [x] 10.4 Document security best practices
|
||||
- [x] 10.5 Add warning about credential rotation if database.toml was committed
|
||||
|
||||
## 11. Architecture Documentation
|
||||
- [x] 11.1 Review architecture.md (Arc42 format) - already created with 9 ADRs
|
||||
- [x] 11.2 Validate ADRs (Architecture Decision Records) align with implementation
|
||||
- [x] 11.3 Update component diagrams if needed during implementation
|
||||
- [x] 11.4 Verify ADR-007 (Availability is a Filter, Not a Scoring Criterion) implementation
|
||||
- [x] 11.5 **NEW**: Verify ADR-008 (Incremental Structured Mode) implementation
|
||||
- [x] 11.6 **NEW**: Verify ADR-009 (Fuzzy Filtering) implementation
|
||||
- [x] 11.7 Document any additional architectural decisions made during development
|
||||
|
||||
## Notes
|
||||
- All database operations must be read-only
|
||||
- LLM access only via Azure OpenAI (chat + embeddings)
|
||||
- Two separate caches with different TTLs (DB: 12h, Search: 60min)
|
||||
- Single join query for all data retrieval
|
||||
- Markdown tables for all output
|
||||
- Full competence lists (no truncation)
|
||||
- Search sessions via search_id (60min expiry)
|
||||
- Initial implementation focuses on core functionality; refinements in future stages
|
||||
- Total: tracked via `openspec list` (this markdown count is not authoritative)
|
||||
-135
@@ -1,135 +0,0 @@
|
||||
# Design: Capacity→Task Matching and Embedding-based Inference
|
||||
|
||||
## Context
|
||||
|
||||
The system was originally optimized for task→capacity matching and relied on Azure chat completions for role/requirements extraction.
|
||||
|
||||
To enable a symmetrical workflow and reduce latency/cost, this change:
|
||||
|
||||
- adds tools to browse capacities and match a capacity against open tasks
|
||||
- reworks role and competence inference to use embeddings only (Data Lake vocabularies)
|
||||
- preloads and caches role/competence vocab embeddings on startup (with runtime on-demand fallback)
|
||||
- preloads and caches task text embeddings by `task_id`
|
||||
|
||||
## Key decisions
|
||||
|
||||
### A. Vocabulary-driven inference (roles + competences)
|
||||
|
||||
- Maintain two vocabularies sourced from the Data Lake:
|
||||
- roles: `teamlandkarte_v_capacity_roles_latest` filtered by `active=true` and `staffing_board_relevant=true`
|
||||
- competences: unique values from `beschaffungstool_kmp_skill_latest.skillname__c`
|
||||
- Generate embeddings for these vocabularies and store them in the local SQLite embedding cache.
|
||||
|
||||
### B. Task text embedding stored by task_id
|
||||
|
||||
To minimize embedding transaction costs for repeated analysis of the same task:
|
||||
|
||||
- build `task_text = f"{title}\n\n{description}"` (title optional)
|
||||
- embed `task_text` once
|
||||
- store that embedding under a stable key derived from `task_id` (and embedding model/dims)
|
||||
|
||||
**Explicit cache key formats**:
|
||||
|
||||
```python
|
||||
# Role vocabulary entries
|
||||
key = f"role_vocab:{model}:{dims}:{normalized_role_name}"
|
||||
|
||||
# Competence vocabulary entries
|
||||
key = f"comp_vocab:{model}:{dims}:{normalized_competence_name}"
|
||||
|
||||
# Task text embeddings by task_id
|
||||
key = f"task_id:{model}:{dims}:{task_id}"
|
||||
```
|
||||
|
||||
Where:
|
||||
- `model` = embedding model name (e.g., `text-embedding-3-large`)
|
||||
- `dims` = embedding dimensions (e.g., `3072`)
|
||||
- `normalized_role_name` = normalized role text (trimmed, whitespace-collapsed, lowercased)
|
||||
- `normalized_competence_name` = normalized competence text (trimmed, whitespace-collapsed, lowercased)
|
||||
- `task_id` = database task ID (string or integer, converted to string)
|
||||
|
||||
### C. Inference logic (embedding similarity)
|
||||
|
||||
- Role inference:
|
||||
- compute cosine similarity between `task_text_embedding` and all role vocabulary embeddings
|
||||
- return the single best matching role with similarity score as a Markdown table: `Role | Similarity`
|
||||
- Competence inference (for validation and display):
|
||||
- compute cosine similarity between `task_text_embedding` and all competence vocabulary embeddings
|
||||
- return top-N competences with similarity scores as a Markdown table: `Competence | Similarity` (descending order)
|
||||
|
||||
**Configuration** (under `[matching.inference]`):
|
||||
```toml
|
||||
[matching.inference]
|
||||
max_competences = 8 # default: 8; omit to disable limit (no max)
|
||||
min_similarity = 0.4 # default: 0.4; omit to disable threshold filtering
|
||||
```
|
||||
|
||||
**Configuration rules**:
|
||||
- If `max_competences` is not specified: no limit on number of competences returned.
|
||||
- If `min_similarity` is not specified: no threshold filtering applied (include all).
|
||||
- At least one of `max_competences` or `min_similarity` MUST be specified in config to avoid unbounded results.
|
||||
- Selection rule: if both are set, return top `max_competences` among those with similarity `>= min_similarity`.
|
||||
|
||||
### D. Component structure for vocabulary preload
|
||||
|
||||
**Component: VocabularyCache**
|
||||
- **Location**: `src/teamlandkarte_mcp/matching/vocabulary.py` (new module)
|
||||
- **Responsibilities**:
|
||||
- Fetch role + competence vocabularies from Data Lake via database client
|
||||
- Normalize and deduplicate vocabulary entries
|
||||
- Embed missing entries using batch + chunking via `SimilarityEngine.prefetch_embeddings(...)`
|
||||
- Store embeddings in SQLite cache with stable keys
|
||||
- Provide in-memory memoization for fast repeated inference during tool execution
|
||||
- **Initialization**: Called from `build_server()` after DB client + similarity engine initialization
|
||||
- **Refresh strategy**: On startup only; no TTL-based refresh during runtime
|
||||
|
||||
**Task embedding preload**:
|
||||
- Preload **all open tasks** on startup (no limit).
|
||||
- Build `task_text = title + "\n\n" + description` for each task (title optional).
|
||||
- Embed only cache-missing `task_id` vectors (check SQLite cache first).
|
||||
- Store into SQLite embedding cache with key format: `task_id:{model}:{dims}:{task_id}`.
|
||||
- Runtime fallback: if a task embedding is missing during tool execution, embed on-demand (strict semantics).
|
||||
|
||||
### E. Determinism and strict failure semantics
|
||||
|
||||
- Runs are deterministic given fixed models and cached embeddings.
|
||||
- Azure embedding failures fail the operation immediately.
|
||||
- The existing global embedding improvements (dedup + batch + layered caches) should be reused.
|
||||
|
||||
## Data flow
|
||||
|
||||
### 1) Vocabulary refresh
|
||||
|
||||
1. fetch role list + competence list from Data Lake
|
||||
2. normalize/deduplicate
|
||||
3. embed missing entries via Azure batch API
|
||||
4. store embeddings in SQLite cache
|
||||
|
||||
### 2) Task validation
|
||||
|
||||
1. fetch task (title+description) + DB skills
|
||||
2. compute/retrieve `task_text_embedding` by task_id
|
||||
3. infer:
|
||||
- top-K roles
|
||||
- top-K competences
|
||||
4. compare:
|
||||
- DB skills vs inferred competences (plus similarity thresholds)
|
||||
- dates (unchanged)
|
||||
|
||||
### 3) Capacity→task matching
|
||||
|
||||
1. load the target capacity (role + competences + availability)
|
||||
2. list open tasks
|
||||
3. compute/retrieve `task_text_embedding` for each task
|
||||
4. compute overall similarity (role + competences) between the capacity and each task
|
||||
5. categorize + return table-first results (same output conventions as existing matching tools)
|
||||
|
||||
## Alternatives considered
|
||||
|
||||
- Keep chat API for role inference: rejected (latency/cost + redundant).
|
||||
- Use raw string matching for vocab mapping: rejected (less robust than embeddings).
|
||||
|
||||
## Risks
|
||||
|
||||
- Vocabulary size (competences) may be large → requires batch embedding + caching and careful refresh strategy.
|
||||
- Model updates can change nearest neighbors → mitigate with pinned model/deployment and cache key including model/dims.
|
||||
-82
@@ -1,82 +0,0 @@
|
||||
# Change: Add capacity→task matching tools and embedding-based inference
|
||||
|
||||
## Why
|
||||
|
||||
The server currently supports **task→capacity matching** (`list_open_tasks`, `get_task_details`, `find_matching_capacities`, `find_capacities_for_task`). There is no symmetrical workflow to start from a capacity/person and discover relevant tasks.
|
||||
|
||||
Additionally, role/requirements inference and task validation still rely on the Azure chat API. This introduces latency/cost, complicates failure semantics, and is redundant given the system already uses embeddings + cosine similarity for matching.
|
||||
|
||||
## What changes
|
||||
|
||||
### New MCP tools
|
||||
|
||||
- `list_free_capacities(limit=20)`
|
||||
- Show the X most recent active capacities in a table-first Markdown output.
|
||||
- `get_capacity_details(capacity_id)`
|
||||
- Show a single capacity/person entry (same fields as the list, table-first).
|
||||
- `find_matching_tasks(capacity_id)`
|
||||
- Match a capacity against all open tasks based on role + competences.
|
||||
- Returns paginated Markdown table with task_id, title, required competences, availability, score, and category.
|
||||
|
||||
### Rework role + competence inference (remove chat API)
|
||||
|
||||
- Replace chat-based role inference with **embedding-based role inference**:
|
||||
- infer roles from a combined text: **task title (if present) + task description**
|
||||
- embed this combined text once and match against a pre-embedded **role vocabulary** from the Data Lake.
|
||||
- Replace chat-based competence extraction/validation with **embedding-based competence matching**:
|
||||
- embed the task text and match against a pre-embedded **competence vocabulary** from the Data Lake.
|
||||
- Store the embedding for `task_id` (combined title+description text) in the local SQLite embedding cache to minimize repeated embedding costs.
|
||||
|
||||
### Data Lake sourcing
|
||||
|
||||
- Competence vocabulary: `hive.tier1_open_lake.beschaffungstool_kmp_skill_latest` (existing source)
|
||||
- Role vocabulary: `teamlandkarte_v_capacity_roles_latest` via columns:
|
||||
- role name column: `name`
|
||||
- filters: `active=true` and `staffing_board_relevant=true`
|
||||
|
||||
### Azure OpenAI usage
|
||||
|
||||
- Azure embeddings remain required.
|
||||
- Azure chat API usage is removed completely once all dependent features are migrated.
|
||||
|
||||
## Impact
|
||||
|
||||
- **MCP API surface**: Adds 3 tools (capacity browsing + capacity→task matching).
|
||||
- **Business logic**: TaskAnalyzer and validation logic migrate to embeddings-only.
|
||||
- **Data access**: Requires new read queries to fetch role/competence vocabularies.
|
||||
- **Caching**: Reuses existing SQLite embedding cache; adds stable keys for task_id text embeddings.
|
||||
- **Docs**: README, assistant prompt, and architecture must be updated.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Changing scoring weights or category thresholds.
|
||||
- Building a full-text search for tasks/capacities.
|
||||
|
||||
## Open questions (need confirmation)
|
||||
|
||||
1. **“Most recent capacities”**: should ordering be by `begin_date` descending, by `end_date` descending, or by a DB “created/updated” timestamp (if available)?
|
||||
2. `list_free_capacities` “owner/team”: do you want **owner_name only**, or both `owner_name` and an explicit `team` field (if available in the view)?
|
||||
3. Do we keep the existing `infer_roles(...)` tool name/shape, but change its implementation to embeddings-only (recommended), or introduce a new tool and deprecate the old one?
|
||||
4. For embedding-based competence inference for validation: should the output be a **ranked list with similarity scores**, or a **binary chosen set** (top-N)? If top-N, what default N?
|
||||
|
||||
---
|
||||
|
||||
## Confirmed decisions
|
||||
|
||||
- `list_free_capacities`: order by **most recent `creation_date`**.
|
||||
- Capacity “Owner/Team”: use the `owner_name` column (contains the **team name**).
|
||||
- Role inference tool:
|
||||
- introduce a **new tool** and deprecate `infer_roles`
|
||||
- the new tool returns only the **single closest role** (no rank/rationale)
|
||||
- Competence inference:
|
||||
- return a maximum of **X** competences (**configurable**, default: 8)
|
||||
- apply a configurable **similarity score threshold** (default: 0.4)
|
||||
- if `max_competences` is not specified: no limit on number of competences
|
||||
- if `min_similarity` is not specified: no threshold filtering
|
||||
- at least one parameter must be specified to avoid unbounded results
|
||||
- if both are set: select **top X** among those **>= threshold**
|
||||
- Refresh strategy:
|
||||
- on startup only
|
||||
- only cache-missing roles/competences are embedded
|
||||
- embed **all open tasks** (combined title+description) on startup
|
||||
- if something is still missing at runtime, embed on-demand (strict semantics)
|
||||
-150
@@ -1,150 +0,0 @@
|
||||
# Spec Delta: Capacity→Task Matc### Requirement: Match tasks### Requirement: Role inference tool is embe### Requirement: Task validation uses embedding-based role and competence inference
|
||||
|
||||
Task validation SHALL use embedding similarity against role and competence vocabularies and SHALL NOT call the Azure chat API. Output SHALL include Markdown tables for inferred roles and competences with similarity scores.
|
||||
|
||||
#### Scenario: Validate task requirements without chat calls
|
||||
|
||||
- **WHEN** the client calls `validate_task_requirements(task_id=...)`
|
||||
- **THEN** the server infers the task's primary role via the role vocabulary (embeddings)
|
||||
- **AND** the output includes a Markdown table with columns `Role | Similarity` (single row)
|
||||
- **AND** the server infers competences via the competence vocabulary (embeddings) honoring configured max/threshold
|
||||
- **AND** the output includes a Markdown table with columns `Competence | Similarity` in descending order by similarity
|
||||
- **AND** the output remains table-first and includes inferred role/competences with similarity scoresy and returns a single closest role
|
||||
|
||||
Role inference SHALL be based on embedding similarity between a task's combined text and the Data Lake role vocabulary and SHALL return only the single closest role as a Markdown table.
|
||||
|
||||
#### Scenario: Infer primary role from task_id or task_text
|
||||
|
||||
- **WHEN** the client calls `infer_primary_role(task_id=...)`
|
||||
- **THEN** the system builds a combined text consisting of the task title (if present) and the task description
|
||||
- **AND** the combined text is embedded and compared against the available role list
|
||||
- **AND** the output is a Markdown table with columns `Role | Similarity` containing the single best matching role
|
||||
|
||||
- **WHEN** the client calls `infer_primary_role(task_text=...)`
|
||||
- **THEN** the system embeds the provided text and compares it against the available role list
|
||||
- **AND** the output is a Markdown table with columns `Role | Similarity` containing the single best matching role
|
||||
|
||||
- **AND** exactly one of `task_id` or `task_text` MUST be provided
|
||||
- **AND** the legacy `infer_roles(...)` tool MUST be removed completelyThe system SHALL expose a tool `find_matching_tasks(capacity_id)` that matches a capacity against all published tasks using role + competence similarity.
|
||||
|
||||
#### Scenario: Match tasks for a capacity using role + competence similarity
|
||||
|
||||
- **GIVEN** a capacity has a role name and a competence list
|
||||
- **AND** at least one open/published task exists in the Data Lake
|
||||
- **WHEN** the client calls `find_matching_tasks(capacity_id=<id>)`
|
||||
- **THEN** the server infers each task's primary role before computing similarity
|
||||
- **AND** the server infers each task's competences using embedding similarity with configured max/threshold
|
||||
- **AND** the server returns a Markdown table with columns: `task_id`, `Title`, `Required Competences`, `Availability`, `Score`, `Category`
|
||||
- **AND** results are sorted by similarity score in descending order
|
||||
- **AND** the output structure matches `find_matching_capacities` (summary counts table + results table)
|
||||
- **AND** the server returns categorized results (Top/Good/Partial/Low) using the same conventions as `find_matching_capacities`+ Embedding-based Inference
|
||||
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: List free capacities
|
||||
|
||||
The system SHALL expose a tool `list_free_capacities(limit=20)` to list the most recent active capacities in a table-first Markdown format.
|
||||
|
||||
#### Scenario: List free capacities ordered by creation_date
|
||||
|
||||
- **WHEN** the client calls `list_free_capacities(limit=3)`
|
||||
- **THEN** the server returns a Markdown table with columns:
|
||||
- `capacity_id`
|
||||
- `Owner/Team` (sourced from the capacity’s `owner_name` column)
|
||||
- `Role`
|
||||
- `Competences`
|
||||
- `Availability`
|
||||
- **AND** the entries are ordered by `creation_date` descending
|
||||
- **AND** `Competences` is rendered as a comma-separated list (or `(none)`)
|
||||
- **AND** `Availability` is rendered as `begin_date .. end_date` (using `(open)` when the end date is missing)
|
||||
- **AND** when there are zero results, the output still contains a valid Markdown table (a dummy row is acceptable)
|
||||
|
||||
### Requirement: Get capacity details
|
||||
|
||||
The system SHALL expose a tool `get_capacity_details(capacity_id)` that returns a single capacity entry in the same table-first format as `list_free_capacities`.
|
||||
|
||||
#### Scenario: Get capacity details for a known capacity_id
|
||||
|
||||
- **WHEN** the client calls `get_capacity_details(capacity_id=<id>)`
|
||||
- **THEN** the server returns a table-first Markdown output containing the same fields as `list_free_capacities`
|
||||
- **AND** the output MAY include a “Next steps” section pointing to `find_matching_tasks(capacity_id=...)`
|
||||
|
||||
### Requirement: Match tasks for a capacity
|
||||
|
||||
The system SHALL expose a tool `find_matching_tasks(capacity_id, ...)` that matches a capacity against all published tasks using role + competence similarity.
|
||||
|
||||
#### Scenario: Match tasks for a capacity using role + competence similarity
|
||||
|
||||
- **GIVEN** a capacity has a role name and a competence list
|
||||
- **AND** at least one open/published task exists in the Data Lake
|
||||
- **WHEN** the client calls `find_matching_tasks(capacity_id=<id>)`
|
||||
- **THEN** the server infers each task’s primary role before computing similarity
|
||||
- **AND** the server returns categorized results (Top/Good/Partial/Low) using the same conventions as `find_matching_capacities`
|
||||
|
||||
## MODIFIED Requirements
|
||||
|
||||
### Requirement: Role inference tool is embeddings-only and returns a single closest role
|
||||
|
||||
Role inference SHALL be based on embedding similarity between a task’s combined text and the Data Lake role vocabulary and SHALL return only the single closest role.
|
||||
|
||||
#### Scenario: Infer primary role from task_id or task_text
|
||||
|
||||
- **WHEN** the client calls `infer_primary_role(task_id=...)`
|
||||
- **THEN** the system builds a combined text consisting of the task title (if present) and the task description
|
||||
- **AND** the combined text is embedded and compared against the available role list
|
||||
- **AND** the output is the single best matching role
|
||||
|
||||
- **WHEN** the client calls `infer_primary_role(task_text=...)`
|
||||
- **THEN** the system embeds the provided text and compares it against the available role list
|
||||
- **AND** the output is the single best matching role
|
||||
|
||||
- **AND** exactly one of `task_id` or `task_text` MUST be provided
|
||||
- **AND** the legacy `infer_roles(...)` tool MUST be removed completely
|
||||
|
||||
### Requirement: Competence inference is configurable by max count and threshold
|
||||
|
||||
Competence inference for validation SHALL be derived via embedding similarity against the competence vocabulary and MUST support limiting by both maximum count and a minimum similarity threshold. Output SHALL be a Markdown table.
|
||||
|
||||
#### Scenario: Select top X competences above threshold
|
||||
|
||||
- **GIVEN** `max_competences = X` and `min_similarity = T` are configured
|
||||
- **WHEN** competences are inferred for a task
|
||||
- **THEN** only competences with similarity `>= T` are considered
|
||||
- **AND** from those, the top `X` competences by similarity are returned
|
||||
- **AND** the output is a Markdown table with columns `Competence | Similarity` in descending order by similarity
|
||||
|
||||
### Requirement: Task validation uses embedding-based role and competence inference
|
||||
|
||||
Task validation SHALL use embedding similarity against role and competence vocabularies and SHALL NOT call the Azure chat API.
|
||||
|
||||
#### Scenario: Validate task requirements without chat calls
|
||||
|
||||
- **WHEN** the client calls `validate_task_requirements(task_id=...)`
|
||||
- **THEN** the server infers the task’s primary role via the role vocabulary (embeddings)
|
||||
- **AND** the server infers competences via the competence vocabulary (embeddings) honoring configured max/threshold
|
||||
- **AND** the output remains table-first and includes inferred role/competences with similarity scores
|
||||
|
||||
### Requirement: Startup-only embedding preload with runtime fallback
|
||||
|
||||
The server SHOULD preload embeddings on startup to reduce runtime Azure calls, but MUST still embed on-demand for any cache misses encountered during tool execution.
|
||||
|
||||
#### Scenario: Preload only cache-missing embeddings and fall back at runtime
|
||||
|
||||
- **WHEN** the server starts
|
||||
- **THEN** it fetches role vocabulary + competence vocabulary from the Data Lake
|
||||
- **AND** it embeds only cache-missing role/competence vocabulary entries
|
||||
- **AND** it fetches **all open tasks** (no limit)
|
||||
- **AND** it builds combined task text (`title + "\n\n" + description`) and embeds only cache-missing task text embeddings keyed by task_id
|
||||
- **AND** during runtime, if an embedding is still missing, the server retrieves it on-demand (strict semantics)
|
||||
|
||||
## REMOVED Requirements
|
||||
|
||||
### Requirement: Chat-based extraction for roles and competences
|
||||
|
||||
The system SHALL NOT require Azure chat completions for role inference or task validation once this change is implemented.
|
||||
|
||||
#### Scenario: Azure chat API is not required for matching and validation
|
||||
|
||||
- **WHEN** a matching or validation tool is executed
|
||||
- **THEN** no Azure chat completion request is made
|
||||
- **AND** only Azure embeddings are used where LLM intelligence is required
|
||||
@@ -1,149 +0,0 @@
|
||||
# Tasks: Add capacity→task matching tools and embedding-based inference
|
||||
|
||||
> Change: `add-capacity-to-task-matching-tools`
|
||||
>
|
||||
> Status: **Implemented**
|
||||
|
||||
## Phase 1: Requirements + Configuration decisions
|
||||
|
||||
- [x] 1.1 Define `list_free_capacities` ordering precisely:
|
||||
- order by most recent `creation_date` (descending)
|
||||
- [x] 1.2 Confirm “Owner/Team” display:
|
||||
- use `owner_name` (contains the team name)
|
||||
- [x] 1.3 Introduce a new role inference tool and remove the old:
|
||||
- add `infer_primary_role(task_id? | task_text?)` that returns the single closest role as a Markdown table with `Role | Similarity` columns
|
||||
- remove `infer_roles(...)` MCP tool completely (no deprecation period)
|
||||
- remove `TaskAnalyzer.extract_ranked_roles()` method completely
|
||||
- update `extract_requirements()` and other callers to use new primary-role inference
|
||||
- [x] 1.4 Add configuration for competence inference:
|
||||
- `matching.inference.max_competences` (int, default: 8; if not specified, no limit)
|
||||
- `matching.inference.min_similarity` (float in [0,1], default: 0.4; if not specified, no threshold)
|
||||
- at least one parameter MUST be specified to avoid unbounded results
|
||||
- selection rule: pick top X among those >= threshold when both are set
|
||||
- [x] 1.5 Define startup-only refresh behavior:
|
||||
- on startup embed only cache-missing role/competence vocab entries
|
||||
- on startup embed **all open tasks** (title+description by task_id)
|
||||
- at runtime still embed on-demand for misses (strict failure semantics)
|
||||
|
||||
## Phase 2: Data Lake queries (roles, competences, capacities)
|
||||
|
||||
- [x] 2.0 Verify Data Lake schema:
|
||||
- confirm `teamlandkarte_v_capacity_roles_latest` has columns: `name`, `active`, `staffing_board_relevant`
|
||||
- confirm capacities view has `creation_date` column
|
||||
- [x] 2.1 Add `TrinoClient.get_recent_free_capacities(limit)` query returning:
|
||||
- `capacity_id`, `owner_name` (team), `role_name`, `creation_date`, `begin_date`, `end_date`, `competences[]`
|
||||
- ordering: `creation_date DESC`
|
||||
- [x] 2.2 Add `TrinoClient.get_capacity_by_id(capacity_id)` query returning the same fields.
|
||||
- [x] 2.3 Add `TrinoClient.get_all_role_names()`:
|
||||
- source: `teamlandkarte_v_capacity_roles_latest`
|
||||
- filter: `active=true AND staffing_board_relevant=true`
|
||||
- return unique, non-empty role names
|
||||
- [x] 2.4 Add `TrinoClient.get_all_competence_names()`:
|
||||
- source: `beschaffungstool_kmp_skill_latest.skillname__c`
|
||||
- return unique, non-empty competence names
|
||||
- [x] 2.5 Add unit tests for SQL read-only guard + query shape (where applicable).
|
||||
|
||||
## Phase 3: Startup embedding preload (vocabularies + tasks)
|
||||
|
||||
- [x] 3.1 Define stable cache-key scheme (explicit format as documented in design.md):
|
||||
- role vocab entries: `f"role_vocab:{model}:{dims}:{normalized_role_name}"`
|
||||
- competence vocab entries: `f"comp_vocab:{model}:{dims}:{normalized_competence_name}"`
|
||||
- task text embeddings by task_id: `f"task_id:{model}:{dims}:{task_id}"`
|
||||
- [x] 3.2 Implement `VocabularyCache` component (`src/teamlandkarte_mcp/matching/vocabulary.py`):
|
||||
- fetch role vocabulary + competence vocabulary from Trino
|
||||
- normalize/deduplicate
|
||||
- embed only cache-missing entries (batch + chunking via `SimilarityEngine.prefetch_embeddings(...)`)
|
||||
- store into SQLite embedding cache
|
||||
- keep in-memory memoization for fast repeated inference
|
||||
- invoked from `build_server()` after DB + similarity engine init
|
||||
- [x] 3.3 Implement a “task embedding preload” routine executed on startup:
|
||||
- fetch **all open tasks** (no limit)
|
||||
- build `task_text = title + "\n\n" + description` (title optional)
|
||||
- embed only cache-missing `task_id` vectors
|
||||
- store into SQLite embedding cache
|
||||
- [x] 3.4 Runtime fallback:
|
||||
- if a role/competence/task embedding is missing during a tool call, embed on-demand (strict semantics)
|
||||
- [x] 3.5 Tests:
|
||||
- preload embeds only cache misses
|
||||
- preload writes to SQLite
|
||||
- missing-at-runtime triggers on-demand embedding
|
||||
|
||||
## Phase 4: Embedding-only inference (roles + competences)
|
||||
|
||||
- [x] 4.1 Implement primary role inference against role vocabulary:
|
||||
- input: combined `title+description` (or description-only for ad-hoc)
|
||||
- output: single closest role with similarity score as Markdown table: `Role | Similarity`
|
||||
- [x] 4.2 Add a new MCP tool:
|
||||
- `infer_primary_role(task_id? | task_text?)`
|
||||
- returns: Markdown table `Role | Similarity` (single best matching role)
|
||||
- validation: exactly one of `task_id` or `task_text` must be provided
|
||||
- [x] 4.3 Remove deprecated tools and methods:
|
||||
- remove `infer_roles(...)` MCP tool completely
|
||||
- remove `TaskAnalyzer.extract_ranked_roles()` method completely
|
||||
- [x] 4.4 Update callers to use new primary-role inference:
|
||||
- update `get_task_details` to use `infer_primary_role`
|
||||
- update `extract_requirements()` to use new primary-role inference
|
||||
- update any internal helpers relying on `TaskAnalyzer.extract_ranked_roles()`
|
||||
|
||||
## Phase 5: Rework task validation (embedding-only)
|
||||
|
||||
- [x] 5.1 Replace chat-based extraction in `validate_task_requirements` / `extract_requirements_from_task` with embedding-based inference:
|
||||
- roles: infer primary role via role vocabulary (Markdown table output)
|
||||
- competences: infer competences via competence vocabulary with max X and threshold (Markdown table output)
|
||||
- [x] 5.2 Update output format:
|
||||
- keep current table-first display
|
||||
- add inferred primary role as Markdown table: `Role | Similarity` (single row)
|
||||
- add inferred competences as Markdown table: `Competence | Similarity` (descending order by similarity)
|
||||
- [x] 5.3 Tests:
|
||||
- validation uses embeddings path only (no chat calls)
|
||||
- respects max X and threshold rules
|
||||
- output includes role and competence similarity tables
|
||||
|
||||
## Phase 6: New MCP tools (capacity browsing + capacity→task matching)
|
||||
|
||||
- [x] 6.1 Implement `list_free_capacities(limit=20)`:
|
||||
- ordering: `creation_date DESC`
|
||||
- Markdown table columns: `capacity_id`, `Owner/Team`, `Role`, `Competences`, `Availability`
|
||||
- stable empty-table handling (dummy row)
|
||||
- [x] 6.2 Implement `get_capacity_details(capacity_id)`:
|
||||
- table-first, same fields as list
|
||||
- include a “Next steps” pointer to `find_matching_tasks(capacity_id=...)`
|
||||
- [x] 6.3 Implement `find_matching_tasks(capacity_id)`:
|
||||
- signature: takes only `capacity_id` (no optional params)
|
||||
- fetch capacity + all open tasks
|
||||
- infer each task’s primary role via embedding similarity (title+description)
|
||||
- infer each task’s competences via embedding similarity (title+description) with max/threshold
|
||||
- compute a similarity score (role + competences) and categorize results
|
||||
- return output as Markdown table with columns:
|
||||
- `task_id`, `Title`, `Required Competences`, `Availability`, `Score`, `Category`
|
||||
- results are sorted by similarity score (descending)
|
||||
- output structure matches `find_matching_capacities` (includes summary counts table + results table for first non-empty category)
|
||||
- paginated via same search session conventions used by `find_matching_capacities`
|
||||
- [x] 6.4 Tests for new tools:
|
||||
- `list_free_capacities`: ordering by `creation_date DESC`, empty-table handling
|
||||
- `get_capacity_details`: not-found handling, "Next steps" section
|
||||
- `find_matching_tasks`: categorization (Top/Good/Partial/Low), search session conventions, table structure, score sorting
|
||||
|
||||
## Phase 7: Remove Azure chat API
|
||||
|
||||
- [x] 7.1 Remove chat config fields and environment variables if no longer needed (`AZURE_OPENAI_LLM_API_KEY`, chat deployment, chat codepaths).
|
||||
- [x] 7.2 Delete/retire chat client methods and `TaskAnalyzer` chat-based methods.
|
||||
- [x] 7.3 Update docs to remove chat setup instructions.
|
||||
- [x] 7.4 Ensure all gates pass (ruff/mypy/pytest).
|
||||
|
||||
## Phase 8: Documentation + assistant prompt + architecture
|
||||
|
||||
- [x] 8.1 Update `README.md`:
|
||||
- describe new tools and capacity→task workflow
|
||||
- document startup embedding preload and inference config knobs
|
||||
- [x] 8.2 Update `docs/assistant_system_prompt.md`:
|
||||
- add guidance for new tools and matching direction
|
||||
- update role/competence inference description (embeddings-only)
|
||||
- [x] 8.3 Update architecture docs (Arc42 / design docs):
|
||||
- reflect removal of chat API
|
||||
- describe vocab/task embedding preload and cache keys
|
||||
|
||||
## Phase 9: OpenSpec hygiene
|
||||
|
||||
- [x] 9.1 Update spec deltas to reflect confirmed tool shapes + config.
|
||||
- [x] 9.2 Run `openspec validate add-capacity-to-task-matching-tools --strict`.
|
||||
-262
@@ -1,262 +0,0 @@
|
||||
# Change Proposal: Improve Task Details, Role Inference, Guided Requirements Capture, and Search Reliability
|
||||
|
||||
- **Change ID**: `improve-task-details-role-inference-and-guided-flow`
|
||||
- **Status**: Implemented
|
||||
- **Target**: `teamlandkarte-mcp`
|
||||
- **Author**: (to fill)
|
||||
- **Date**: 2026-02-12
|
||||
|
||||
## Summary
|
||||
|
||||
This proposal refines several MCP tools and interaction flows to improve usability in Cherry Studio and other MCP clients:
|
||||
|
||||
1. `get_task_details` should return a concise markdown summary (table) of the task plus description below.
|
||||
2. Introduce a dedicated tool `infer_roles` for role inference from either a DB task ID or free-text description.
|
||||
3. `validate_task_requirements` should return a single comparison table plus a short written summary.
|
||||
4. When a user requests a person search without providing project/task details, switch to a **step-by-step guided capture** (no "all questions at once").
|
||||
5. Enforce a **mandatory confirmation step** before triggering matching, for both free-text and guided capture / structured entry.
|
||||
6. Fix a suspected competence matching regression (cloud expert + AWS scoring ends up `Low`), likely caused by extraction/matching issues.
|
||||
7. Improve search session robustness for pagination/filtering:
|
||||
- search tools emit a JSON block containing `search_id`/`filter_id`
|
||||
- non-UUID `search_id` inputs are rejected early with a clear message
|
||||
- internal tracking of the latest search id (session state) for diagnostics only; **no** "get last search id" tool is provided
|
||||
8. Update documentation, architecture, and design markdown accordingly.
|
||||
|
||||
## Motivation
|
||||
|
||||
- Current outputs are verbose and inconsistent across tools.
|
||||
- Role inference exists but is not clearly available as an explicit tool.
|
||||
- Guided capture and confirmation are required to prevent accidental matching runs.
|
||||
- Cache invalidation/expiry breaks multi-step flows (`filter_search_results`, pagination, etc.).
|
||||
|
||||
## Goals
|
||||
|
||||
- Produce stable, markdown-first outputs that are easy to consume in chat UIs.
|
||||
- Make role inference available as an explicit, reusable capability for both DB-backed tasks and free-text.
|
||||
- Improve correctness and robustness of role/requirements extraction and semantic matching via the server’s Azure OpenAI integration.
|
||||
- Ensure multi-step search workflows remain functional for realistic time spans.
|
||||
- Remove artificial constraints on the number of exposed MCP tools.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- Re-introducing Hive/PyHive support.
|
||||
- Changing the underlying DB schema.
|
||||
- Replacing LLM-based role inference with a numeric-only scorer (categorical matching output should remain the main UX).
|
||||
|
||||
## Proposed Changes
|
||||
|
||||
### 1) `get_task_details` output format
|
||||
|
||||
Change `get_task_details(task_id)` to return:
|
||||
|
||||
1. A markdown table with columns:
|
||||
- Task ID
|
||||
- Title
|
||||
- Created (date only, no time)
|
||||
- Start
|
||||
- End
|
||||
- Competences (comma-separated, no numbering)
|
||||
- Inferred Role
|
||||
2. Below the table:
|
||||
- The task description (plain markdown, no table)
|
||||
|
||||
Notes:
|
||||
- "Inferred Role" is derived using `infer_roles` (see section 2). If no role is inferred, display `(none)`.
|
||||
|
||||
### 2) Dedicated role inference tool: `infer_roles`
|
||||
|
||||
Add a tool to infer roles from either:
|
||||
|
||||
- A DB task via `task_id`, or
|
||||
- A free-text `task_description`
|
||||
|
||||
Signature:
|
||||
|
||||
- `infer_roles(task_id: Optional[str] = None, task_description: Optional[str] = None, limit: int = 5) -> str`
|
||||
|
||||
Output:
|
||||
- A markdown table containing all recognized roles, each with:
|
||||
- Rank
|
||||
- Role
|
||||
- Rationale
|
||||
|
||||
Behavior:
|
||||
- Exactly one of `task_id` or `task_description` must be provided.
|
||||
- Uses Azure OpenAI chat completion for role inference (no heuristic fallback).
|
||||
|
||||
### 3) `validate_task_requirements` output format
|
||||
|
||||
Change output to:
|
||||
|
||||
1. A single markdown comparison table that contrasts DB vs extracted information.
|
||||
2. Table must include:
|
||||
- Task ID
|
||||
- Title
|
||||
- Time ranges (DB vs extracted)
|
||||
- Competences (DB vs extracted)
|
||||
3. Below the table, a short written summary, including:
|
||||
- Skill overlap and notable differences
|
||||
- Any missing DB fields (dates/skills)
|
||||
|
||||
### 4) Step-by-step guided capture mode (step-specific tools)
|
||||
|
||||
Introduce a guided capture workflow for the case:
|
||||
|
||||
> User requests people/capacity search but provides insufficient task/project details.
|
||||
|
||||
The guided flow should collect input sequentially using step-specific tools (to avoid free-text ambiguity):
|
||||
|
||||
1. Task description
|
||||
2. Role
|
||||
3. Time range
|
||||
4. Competences
|
||||
|
||||
Implementation idea:
|
||||
- Add a server-side state machine stored in `SessionState`.
|
||||
- Add tools such as:
|
||||
- `start_guided_capture()`
|
||||
- `guided_set_description(description: str)`
|
||||
- `guided_set_role(role_name: str)`
|
||||
- `guided_set_time_range(date_start: Optional[str] = None, date_end: Optional[str] = None)`
|
||||
- `guided_set_competences(competences: list[str])`
|
||||
|
||||
### 5) Enforce confirmation before matching (shared mechanism, hard-gated)
|
||||
|
||||
`extract_requirements` and structured/guided capture must not directly trigger matching.
|
||||
|
||||
Instead, both should:
|
||||
- produce a "pending" `Requirements` object, and
|
||||
- require a confirmation step **before matching tools can execute**.
|
||||
|
||||
Hard enforcement:
|
||||
- `find_matching_capacities` (and any DB-backed matching tool) must refuse to run unless requirements have been explicitly confirmed **or** confirmation is auto-skipped by configuration (see below).
|
||||
|
||||
Config key (name + location):
|
||||
- Add a matching section flag in `database.toml`:
|
||||
|
||||
```toml
|
||||
[matching]
|
||||
require_confirmation = true
|
||||
```
|
||||
|
||||
- Default: `true` (strict).
|
||||
- When `false`: pending requirements are treated as auto-confirmed (skip confirmation).
|
||||
|
||||
Implementation idea:
|
||||
- Keep `confirm_requirements` parameters for backwards compatibility, but route the behavior through a shared internal helper and a shared tool:
|
||||
- internal: `_set_pending_requirements(req)`
|
||||
- tool: `confirm_requirements(confirm: bool = True)`
|
||||
|
||||
### 6) Fix competence matching producing incorrect `Low` results
|
||||
|
||||
Investigate and fix why "Cloud expert" + "AWS" competences yields `Low` across all capacities.
|
||||
|
||||
Hypotheses:
|
||||
- Requirements extraction produces empty or wrong competences.
|
||||
- Competence normalization / matching thresholds could be misapplied.
|
||||
|
||||
Deliverables:
|
||||
- Add a regression test reproducing the AWS case.
|
||||
- Ensure competence scoring uses the extracted competences correctly.
|
||||
|
||||
Capacity scoring UX note:
|
||||
- Keep numeric scores (`overall_score`, `competence_score`, `role_score`) visible.
|
||||
- Emphasize categorical results (`Top`/`Good`/`Partial`/`Low`) as the primary interpretation in docs and tables.
|
||||
|
||||
### 7) Fix search cache expiry / invalid IDs in multi-step tools
|
||||
|
||||
Observed:
|
||||
- `filter_search_results` and `get_results_by_category` sometimes return
|
||||
"Unknown or expired search_id".
|
||||
|
||||
Additional observed behavior in chat clients:
|
||||
- Some clients shorten, reformat, or accidentally reuse a previous `search_id`
|
||||
when the user requests another category (e.g. "show Good results").
|
||||
|
||||
Assumption:
|
||||
- The server process is continuous (no restart between tool calls).
|
||||
|
||||
Likely causes to verify (not only TTL):
|
||||
- **Eviction due to `[cache].max_size`** (entry removed even within TTL).
|
||||
- Search IDs **not stored** on all paths (serialization error, early return, etc.).
|
||||
- **ID formatting/copy issues** (backticks/whitespace) in client UIs.
|
||||
- Hidden **process restarts / multiple instances** in the client environment.
|
||||
|
||||
Proposed fixes:
|
||||
- Verify `SearchCache` TTL configuration and access.
|
||||
- Verify and test behavior under max-size eviction.
|
||||
- Ensure IDs are stored on every `store_search` call.
|
||||
- Add troubleshooting guidance to copy IDs without surrounding formatting.
|
||||
- Ensure search tools return a machine-readable JSON block containing IDs:
|
||||
- `find_matching_capacities` returns `{search_id, filter_id=null}`
|
||||
- `filter_search_results` returns `{search_id, filter_id}`
|
||||
- `get_results_by_category` echoes `{search_id, filter_id, category, page, ...}`
|
||||
- Reject non-UUID `search_id` values early (clear message: expected UUID).
|
||||
- Track latest `search_id` server-side (session state) for diagnostics only; **no** "get last search id" tool is provided.
|
||||
|
||||
### 8) Documentation updates
|
||||
|
||||
Update:
|
||||
- `README.md`
|
||||
- `docs/troubleshooting.md`
|
||||
- OpenSpec design/architecture docs
|
||||
|
||||
…to reflect:
|
||||
- new/changed tool outputs
|
||||
- `infer_roles`
|
||||
- guided capture step tools
|
||||
- confirmation requirement
|
||||
- role inference and capacity scoring UX (categorical)
|
||||
- cache behavior expectations
|
||||
|
||||
## Implementation Task List
|
||||
|
||||
### A. Tool output refactors
|
||||
|
||||
1. Update `get_task_details` to produce the new table-first format.
|
||||
2. Implement `infer_roles` and reuse it from `get_task_details`.
|
||||
3. Update `validate_task_requirements` output as specified.
|
||||
4. Add/adjust markdown table helpers if needed (column wrapping, newline handling).
|
||||
|
||||
### B. Role inference and extraction robustness
|
||||
|
||||
5. Review role/requirements extraction prompts and output validation.
|
||||
6. Add regression tests for:
|
||||
- role inference
|
||||
- AWS/cloud expert competence matching not collapsing to `Low`
|
||||
|
||||
### C. Guided capture + confirmation flow
|
||||
|
||||
7. Design a minimal state machine for guided capture in `SessionState`.
|
||||
8. Implement step-specific guided tools:
|
||||
- `start_guided_capture`
|
||||
- `guided_set_description`
|
||||
- `guided_set_role`
|
||||
- `guided_set_time_range`
|
||||
- `guided_set_competences`
|
||||
9. Add `matching.require_confirmation` to the configuration model + loader (default: `true`).
|
||||
10. Implement shared confirmation tool `confirm_requirements(confirm: bool = True)`.
|
||||
11. Refactor `extract_requirements` and structured/guided collection tools to set pending requirements and instruct confirmation.
|
||||
12. Refactor matching tools to hard-enforce confirmation unless `matching.require_confirmation = false`.
|
||||
|
||||
### D. Search cache reliability
|
||||
|
||||
13. Diagnose and fix `SearchCache` invalidation/expiry issues.
|
||||
14. Add tests covering:
|
||||
- storing search results
|
||||
- subsequent filtering
|
||||
- pagination
|
||||
- TTL behavior
|
||||
- max-size eviction behavior
|
||||
15. Add (optional) diagnostic support for identifying server instance continuity.
|
||||
|
||||
### E. Documentation and specs
|
||||
|
||||
16. Update `README.md` for the new workflow and new tools.
|
||||
17. Update `docs/troubleshooting.md` with Cherry Studio guidance for multi-step flows (including ID copy hygiene and cache eviction notes).
|
||||
18. Update OpenSpec design and architecture notes for guided capture, confirmation, and categorical-first scoring (numeric scores remain visible).
|
||||
|
||||
## Open Questions / Clarifications Needed
|
||||
|
||||
- (resolved) `infer_roles` output should be `Rank | Role | Rationale` only (no numeric score).
|
||||
- (resolved) Guided time ranges should allow open-ended ranges (either start or end may be omitted).
|
||||
-128
@@ -1,128 +0,0 @@
|
||||
# Tasks: Improve Task Details, Role Inference, Guided Requirements Capture, and Search Reliability
|
||||
|
||||
> Change: `improve-task-details-role-inference-and-guided-flow`
|
||||
>
|
||||
> Notes:
|
||||
> - All code/doc/spec content stays in English.
|
||||
> - Tool count is **not** limited.
|
||||
> - Confirmation is **hard-gated** by default; can be auto-skipped via config.
|
||||
|
||||
## Status
|
||||
|
||||
- ✅ **Implemented** (core behavior + UX contracts shipped).
|
||||
- 📝 Some follow-up items remain for additional hardening/coverage (diagnostics/logging and a few missing tests), but they do not block the change being considered implemented.
|
||||
|
||||
## 1. Proposal Alignment & Baseline
|
||||
|
||||
- [x] 1.1 Re-scan current tools in `src/teamlandkarte_mcp/mcp_server.py` and confirm required changes vs. current behavior.
|
||||
- [ ] 1.2 Confirm existing `SearchCache` semantics (`ttl_minutes`, `max_size`) and how eviction is handled.
|
||||
- [x] 1.3 Add/update a short architectural note: numeric scores remain visible, categories are primary.
|
||||
|
||||
## 2. Configuration: hard confirmation gate (+ optional auto-skip)
|
||||
|
||||
- [x] 2.1 Extend matching config model in `src/teamlandkarte_mcp/config.py`:
|
||||
- add `require_confirmation: bool = True` under `[matching]`.
|
||||
- [x] 2.2 Update TOML loading/validation to accept `[matching].require_confirmation`.
|
||||
- [x] 2.3 Update `database.toml.example` to document `require_confirmation` (default `true`).
|
||||
- [ ] 2.4 Add tests for config parsing (default true, explicit false).
|
||||
|
||||
## 3. Session State: pending + confirmed requirements
|
||||
|
||||
- [x] 3.1 Extend `SessionState` in `src/teamlandkarte_mcp/mcp_server.py`:
|
||||
- pending requirements object
|
||||
- confirmed requirements object (or a confirmed flag/version)
|
||||
- guided capture state (see section 6)
|
||||
- [x] 3.2 Implement internal helper(s):
|
||||
- `_set_pending_requirements(req: Requirements)`
|
||||
- `_require_confirmed_requirements_or_throw()` (implemented as `_require_confirmed_or_auto`)
|
||||
- [x] 3.3 Implement tool: `confirm_requirements(confirm: bool = True)`.
|
||||
- [x] 3.4 Update `extract_requirements`, `collect_structured_requirement_data`, `update_requirements`:
|
||||
- always write **pending** requirements
|
||||
- never implicitly run matching
|
||||
- include next-step guidance to call `confirm_requirements()`
|
||||
- [x] 3.5 Update `find_matching_capacities` (and DB-backed matching tools) to **hard-enforce** confirmation when `matching.require_confirmation = true`.
|
||||
- [x] 3.6 Auto-skip behavior:
|
||||
- when `matching.require_confirmation = false`, treat pending requirements as confirmed (no refusal).
|
||||
- [x] 3.7 Add unit tests for confirmation gate behavior.
|
||||
|
||||
## 4. Tool: `infer_roles` (ranked roles, no score column)
|
||||
|
||||
- [x] 4.1 Add tool `infer_roles(task_id: Optional[str] = None, task_description: Optional[str] = None, limit: int = 5)`.
|
||||
- [x] 4.2 Enforce input rules:
|
||||
- exactly one of `task_id` or `task_description` must be provided.
|
||||
- [x] 4.3 If `task_id` provided:
|
||||
- fetch task from DB, use its description.
|
||||
- [x] 4.4 Use `TaskAnalyzer.extract_ranked_roles(description, limit=...)`.
|
||||
- [x] 4.5 Output a markdown table with columns:
|
||||
- `Rank | Role | Rationale`
|
||||
- [ ] 4.6 Add tests for both modes (DB id mocked + free-text).
|
||||
|
||||
## 5. Tool output refactors
|
||||
|
||||
### 5.1 `get_task_details`
|
||||
|
||||
- [x] 5.1.1 Replace current multi-section output with:
|
||||
- a single markdown table: `Task ID | Title | Created (date only) | Start | End | Competences | Inferred Role`
|
||||
- below: task description
|
||||
- [x] 5.1.2 Created date formatting: date-only (YYYY-MM-DD).
|
||||
- [x] 5.1.3 Competences formatting: comma-separated, no numbering.
|
||||
- [x] 5.1.4 Inferred role:
|
||||
- derive from role inference (reuse same analyzer call used by `infer_roles`)
|
||||
- show `(none)` if empty
|
||||
- [ ] 5.1.5 Add/update tests asserting output format.
|
||||
|
||||
### 5.2 `validate_task_requirements`
|
||||
|
||||
- [x] 5.2.1 Replace current multi-table output with a **single** comparison table.
|
||||
- [x] 5.2.2 Add a short textual summary below the table.
|
||||
- [ ] 5.2.3 Add/update tests asserting table shape + summary presence.
|
||||
|
||||
## 6. Guided capture: step-specific tools
|
||||
|
||||
- [x] 6.1 Define guided capture state model (internal only).
|
||||
- [x] 6.2 Implement tool `start_guided_capture()`.
|
||||
- [x] 6.3 Implement tool `guided_set_description(description: str)`.
|
||||
- [x] 6.4 Implement tool `guided_set_role(role_name: str)`.
|
||||
- [x] 6.5 Implement tool `guided_set_time_range(date_start: Optional[str] = None, date_end: Optional[str] = None)`.
|
||||
- [x] 6.6 Implement tool `guided_set_competences(competences: list[str])`.
|
||||
- [ ] 6.7 Add tests covering the step transitions and open-ended ranges.
|
||||
|
||||
## 7. Competence matching regression (AWS/cloud)
|
||||
|
||||
- [x] 7.1 Reproduce reported case with a focused test (added deterministic similarity fallback to avoid collapse).
|
||||
- [x] 7.2 Inspect `TaskAnalyzer` heuristic competence extraction and normalization.
|
||||
- [x] 7.3 Inspect matching/scoring pipeline.
|
||||
- [x] 7.4 Fix competence similarity computation so obvious matches ("AWS" vs "AWS") do not collapse.
|
||||
- [ ] 7.5 Add regression test(s) to prevent reintroduction.
|
||||
|
||||
## 8. Search cache reliability
|
||||
|
||||
- [ ] 8.1 Add debug-level logging (stderr-safe) around:
|
||||
- `SearchCache.store_search` creation
|
||||
- `SearchCache.get` misses (include age/ttl if available)
|
||||
- eviction events (max_size)
|
||||
- [ ] 8.2 Verify `max_size` eviction behavior in `SearchCache` implementation.
|
||||
- [ ] 8.3 Add tests:
|
||||
- store -> filter -> paginate within TTL
|
||||
- behavior under forced max-size eviction
|
||||
- [ ] 8.4 (Optional) Add a diagnostic tool returning `server_instance_id` to confirm process continuity during client tests.
|
||||
- [x] 8.5 Update troubleshooting docs with:
|
||||
- ID copy hygiene (avoid extra backticks/whitespace)
|
||||
- max_size eviction symptom/fix
|
||||
|
||||
## 9. Documentation updates
|
||||
|
||||
- [x] 9.1 Update `README.md`.
|
||||
- [x] 9.2 Update `docs/troubleshooting.md`.
|
||||
- [x] 9.3 Update OpenSpec design/architecture docs under:
|
||||
- `openspec/changes/add-capacity-matching-mcp-server/` (historical baseline)
|
||||
|
||||
## 10. QA / Validation
|
||||
|
||||
- [x] 10.1 Run full test suite.
|
||||
- [ ] 10.2 Manually validate in Cherry Studio (guide):
|
||||
- guided capture step tools
|
||||
- confirmation gating (on/off via config)
|
||||
- infer_roles output
|
||||
- get_task_details formatting
|
||||
- search cache persistence across multi-step tools
|
||||
-99
@@ -1,99 +0,0 @@
|
||||
# CHANGES_APPLIED — Refine Matching and Filtering
|
||||
|
||||
This file tracks which items from `openspec/changes/refine-matching-and-filtering/` have been implemented in the codebase and documentation.
|
||||
|
||||
## Status
|
||||
|
||||
- Change proposal: **Implemented**
|
||||
- Tests: **Passing** (`uv run pytest -q`)
|
||||
- Lint: **Passing** (`uv run ruff check src/teamlandkarte_mcp`)
|
||||
|
||||
## Implemented requirements (high level)
|
||||
|
||||
1. **Title-first role inference with description fallback**
|
||||
- Role inference prefers task title text; if title missing/empty, falls back to full task text.
|
||||
- Competence inference continues to use the full task text (title + description).
|
||||
|
||||
2. **Separate embedding cache keys for role vs competence**
|
||||
- Embeddings are cached separately for:
|
||||
- task role embeddings vs task competence embeddings
|
||||
- capacity role embeddings vs capacity competence embeddings
|
||||
|
||||
3. **Filtering improvements**
|
||||
- `filter_search_results(...)` works symmetrically for both directions:
|
||||
- task → capacity (“capacity_search”)
|
||||
- capacity → task (“task_search”)
|
||||
- Added full-coverage availability filtering via `is_fully_available`.
|
||||
- Added task-search specific filters:
|
||||
- `task_text_filter`
|
||||
- `task_competence_filter`
|
||||
|
||||
4. **Pagination and category browsing**
|
||||
- `get_results_by_category(...)` supports both search types.
|
||||
- Added new category bucket: **Irrelevant** (`overall_score < matching.thresholds.low`).
|
||||
|
||||
5. **Availability display refinement**
|
||||
- Availability is displayed as **overlap percentage** with the reference range.
|
||||
|
||||
6. **Result table column consistency**
|
||||
- Matching outputs show **Role Score / Competence Score / Overall Score** consistently.
|
||||
- Removed `Level` column from capacity-search matching result tables (Level remains in listing/inspection tools).
|
||||
|
||||
7. **Bug fixes**
|
||||
- Fixed task-search score display bug by standardizing the stored score key to `overall_score`.
|
||||
|
||||
## Code changes (by area)
|
||||
|
||||
### Matching + scoring
|
||||
|
||||
- `src/teamlandkarte_mcp/matching/scorer.py`
|
||||
- Added **Irrelevant** categorization for scores below `matching.thresholds.low`.
|
||||
|
||||
- `src/teamlandkarte_mcp/matching/matcher.py`
|
||||
- Added **Irrelevant** bucket to `by_category`.
|
||||
|
||||
### Embedding cache separation
|
||||
|
||||
- `src/teamlandkarte_mcp/matching/vocabulary.py`
|
||||
- Added stable cache keys and separate ensure methods:
|
||||
- `ensure_task_role_embedding` / `ensure_task_competence_embedding`
|
||||
- `ensure_capacity_role_embedding` / `ensure_capacity_competence_embedding`
|
||||
- Retained legacy `ensure_task_embedding` as deprecated.
|
||||
|
||||
### MCP tools + search payload + filtering
|
||||
|
||||
- `src/teamlandkarte_mcp/mcp_server.py`
|
||||
- Restored MCP tool `infer_primary_role` (title-first behavior).
|
||||
- Role inference uses title-first text; competence inference uses full text.
|
||||
- Matching payload stores:
|
||||
- `search_type: "capacity_search" | "task_search"`
|
||||
- reference dates
|
||||
- `role_score`, `competence_score`, `overall_score`
|
||||
- category buckets including `Irrelevant`
|
||||
- `filter_search_results(...)`:
|
||||
- symmetric for both search types
|
||||
- supports `is_fully_available`
|
||||
- supports `task_text_filter` / `task_competence_filter` for task search
|
||||
- `get_results_by_category(...)`:
|
||||
- supports `Irrelevant`
|
||||
- consistent score columns
|
||||
- availability shown as overlap %
|
||||
- no `Level` in matching result tables
|
||||
|
||||
## Documentation updates
|
||||
|
||||
- `README.md`
|
||||
- Documented refined matching semantics, Irrelevant category, overlap % availability, and symmetric filtering.
|
||||
|
||||
- `docs/assistant_system_prompt.md`
|
||||
- Updated guidance to include Irrelevant, overlap %, score breakdown columns, and new task-search filters.
|
||||
|
||||
- OpenSpec docs updated to match final behavior:
|
||||
- `openspec/changes/refine-matching-and-filtering/tasks.md`
|
||||
- `openspec/changes/refine-matching-and-filtering/proposal.md`
|
||||
- `openspec/changes/refine-matching-and-filtering/design.md`
|
||||
- `openspec/changes/refine-matching-and-filtering/specs/matching-tools/spec.md`
|
||||
|
||||
## Known follow-ups / non-code items
|
||||
|
||||
- Editor diagnostics still appear to enforce a 79-char line limit despite Ruff being configured for 110. This is likely an editor/extension setting mismatch (not a repo lint failure).
|
||||
@@ -1,231 +0,0 @@
|
||||
# Design: Refine Matching and Filtering
|
||||
|
||||
## Context
|
||||
|
||||
This change improves matching accuracy and filtering capabilities across five areas: role inference precision, validation completeness, availability filtering, task search result management, and result categorization.
|
||||
|
||||
## Key Decisions
|
||||
|
||||
### 1. Title-first Role Inference (with fallback)
|
||||
|
||||
**Decision:** Use task **title first** for role inference; if title is missing/empty, fall back to full task text (title + description).
|
||||
|
||||
**Rationale:**
|
||||
- Task titles typically contain clear role indicators (e.g., "Senior Backend Developer")
|
||||
- Descriptions contain detailed requirements/context that dilute role signals
|
||||
- Fallback ensures robustness when titles are absent
|
||||
|
||||
**Implementation (final):**
|
||||
- Role inference text: `title.strip()` if present else `full_text`
|
||||
- Competence inference text: always uses `full_text`
|
||||
|
||||
### 2. Separate cache keys for role vs competence embeddings
|
||||
|
||||
**Decision:** Use separate embedding cache keys for role inference vs competence inference.
|
||||
|
||||
**Rationale:**
|
||||
- The same entity (task/capacity) is embedded with different texts depending on purpose.
|
||||
- Sharing the same cache key would cause collisions (role embedding overwriting competence embedding or vice versa).
|
||||
|
||||
**Implementation (final):**
|
||||
- Task:
|
||||
- role embedding cache key
|
||||
- competence embedding cache key
|
||||
- Capacity:
|
||||
- role embedding cache key
|
||||
- competence embedding cache key
|
||||
|
||||
### 2. Validation Displays All Returned Competences
|
||||
|
||||
**Decision:** Keep `max_competences` config active in validation and display all competences returned by inference (i.e., do not apply any additional output truncation).
|
||||
|
||||
**Rationale:**
|
||||
- `max_competences` is a deliberate, configurable guardrail
|
||||
- Validation should still show the complete inferred set produced under that configuration
|
||||
- The problem to fix is an *extra* display truncation, not the config itself
|
||||
|
||||
### C. Full-coverage availability filter
|
||||
|
||||
**Decision:** Add `is_fully_available` boolean to `filter_search_results(...)`.
|
||||
|
||||
**Semantics:**
|
||||
- Overlap (default): candidate overlaps the reference period.
|
||||
- Full coverage (`is_fully_available=true`): candidate fully covers the reference period.
|
||||
|
||||
**Reference period source:**
|
||||
- If `availability_date_start` / `availability_date_end` is provided in the filter call, use those.
|
||||
- Otherwise, use reference dates stored in the search payload.
|
||||
|
||||
### D. Task search filtering
|
||||
|
||||
**Decision:** Extend `filter_search_results(...)` with task-specific parameters.
|
||||
|
||||
**Parameters:**
|
||||
- `task_competence_filter` (list[str]): Filter tasks that require at least one of the specified competences (OR semantics)
|
||||
- `task_text_filter` (str): Case-insensitive substring match in title + description
|
||||
|
||||
### E. Irrelevant category
|
||||
|
||||
**Decision:** Add "Irrelevant" category for scores below the configured `matching.thresholds.low`.
|
||||
|
||||
**Thresholds (final):**
|
||||
```python
|
||||
if score >= top: "Top"
|
||||
elif score >= good: "Good"
|
||||
elif score >= partial:"Partial"
|
||||
elif score >= low: "Low"
|
||||
else: "Irrelevant"
|
||||
```
|
||||
|
||||
**Config:**
|
||||
- `matching.thresholds.low` in `config.toml` controls the Low vs Irrelevant boundary.
|
||||
|
||||
### F. Availability as percentage overlap
|
||||
|
||||
**Decision:** Display availability as **overlap percentage**.
|
||||
|
||||
**Interpretation:**
|
||||
- Capacity search (task → capacity): overlap % is relative to the **task** period.
|
||||
- Task search (capacity → task): overlap % is relative to the **capacity** period.
|
||||
|
||||
**Display:** integer percentage `0–100%`.
|
||||
|
||||
### G. Remove Level column from capacity search
|
||||
|
||||
**Decision:** Remove "Level" column from capacity-search matching result tables.
|
||||
|
||||
**Rationale:**
|
||||
- Level is not used in matching and clutters output.
|
||||
- Level remains visible in capacity listing/inspection tools.
|
||||
|
||||
### H. Fix task-search score display bug
|
||||
|
||||
**Root cause:** Field name mismatch in stored search results (using `score` vs `overall_score`).
|
||||
|
||||
**Fix:** Standardize stored key to `overall_score` in both directions.
|
||||
|
||||
### I. Separate role and competence scores
|
||||
|
||||
**Decision:** Display `role_score` and `competence_score` alongside `overall_score` in all matching result tables.
|
||||
|
||||
## Data Flow Changes
|
||||
|
||||
### Startup preload (modified)
|
||||
|
||||
```
|
||||
1. Fetch all open tasks
|
||||
2. For each task:
|
||||
- Build task_text = title.strip() # Changed: was title + "\n\n" + description
|
||||
- Embed with key f"task_id:{model}:{dims}:{task_id}"
|
||||
- Store in SQLite cache
|
||||
```
|
||||
|
||||
### Role inference (modified)
|
||||
|
||||
```
|
||||
1. Get task title (not full text)
|
||||
2. Retrieve/compute title embedding
|
||||
3. Compare against role vocabulary embeddings
|
||||
4. Return best match
|
||||
```
|
||||
|
||||
### Validation (modified)
|
||||
|
||||
```
|
||||
1. Get task (title + description + DB skills)
|
||||
2. Infer role from title only
|
||||
3. Infer competences from title+description (respect max_competences, threshold only)
|
||||
4. Display:
|
||||
- DB requirements table
|
||||
- Inferred role table (1 row)
|
||||
- Inferred competences table (all rows returned by inference)
|
||||
```
|
||||
|
||||
### Task filtering (new)
|
||||
|
||||
```
|
||||
1. Retrieve search results from SearchCache
|
||||
2. Detect search_type from payload
|
||||
3. Apply filters based on type:
|
||||
- capacity_search: role, competence, availability, similarity
|
||||
- task_search: task_competence, task_text, availability
|
||||
4. Store filtered results with new filter_id
|
||||
5. Return table with applied filters summary
|
||||
```
|
||||
|
||||
### Result display (modified)
|
||||
|
||||
```
|
||||
1. Fetch search results from SearchCache
|
||||
2. Calculate overlap percentage for each result
|
||||
3. Format table with columns:
|
||||
- Entity fields (ID, Owner/Title, Role, Competences)
|
||||
- Availability (percentage)
|
||||
- Role Score, Competence Score, Overall Score
|
||||
4. Return Markdown table
|
||||
```
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
### Alternative: Keep title+description for role inference
|
||||
**Rejected:** Descriptions contain too much noise for role matching. Titles are more precise.
|
||||
|
||||
### Alternative: Add separate validation tool for unlimited competences
|
||||
**Rejected:** Better to fix the existing tool than add complexity.
|
||||
|
||||
### Alternative: Separate tools for task filtering
|
||||
**Rejected:** Agent UX is better with unified tools. Shared tools reduce learning curve.
|
||||
|
||||
### Alternative: "None" or "No Match" instead of "Irrelevant"
|
||||
**Rejected:** "Irrelevant" is clearer and more user-friendly.
|
||||
|
||||
## Risks and Mitigations
|
||||
|
||||
### Risk: Role inference quality degrades with short titles
|
||||
**Mitigation:**
|
||||
- Monitor role inference accuracy
|
||||
- Consider fallback to description if title is too short (<3 words)
|
||||
- Document best practices for task title format
|
||||
|
||||
### Risk: Free-text search is too simple
|
||||
**Mitigation:**
|
||||
- Start with case-insensitive substring match
|
||||
- Can enhance later if needed (fuzzy, tokenization, etc.)
|
||||
|
||||
### Risk: Cache invalidation when changing embedded text
|
||||
**Mitigation:**
|
||||
- Accept that old entries will age out naturally (30-day TTL)
|
||||
- Or clear cache on deployment
|
||||
- Document in migration notes
|
||||
|
||||
### Risk: Too many filter parameters confuse agents
|
||||
**Mitigation:**
|
||||
- Clear docstrings with examples
|
||||
- Update assistant prompt with filtering patterns
|
||||
- Test with actual agent interactions
|
||||
|
||||
### Risk: Overlap percentage calculation edge cases
|
||||
**Mitigation:**
|
||||
- Handle NULL/missing dates gracefully (show "Open" or "N/A")
|
||||
- Round to nearest integer percentage
|
||||
- Test with various date combinations
|
||||
- Document calculation logic in code comments
|
||||
|
||||
### Risk: Breaking change for clients parsing tables
|
||||
**Mitigation:**
|
||||
- Document table format changes in release notes
|
||||
- Most clients use semantic parsing (LLMs), not brittle column parsing
|
||||
- Provide migration guide if needed
|
||||
|
||||
## Success Metrics
|
||||
|
||||
- Role inference accuracy improves (measure manually on sample tasks)
|
||||
- Validation always shows complete competence list
|
||||
- Full-coverage filtering works reliably in both directions
|
||||
- Task filtering works as documented
|
||||
- "Irrelevant" category appears for low-score matches
|
||||
- Availability percentages are accurate and intuitive
|
||||
- Task search results show correct non-zero scores
|
||||
- Separate scores help users understand match quality
|
||||
- All tests pass
|
||||
- No performance degradation
|
||||
@@ -1,169 +0,0 @@
|
||||
# Change Proposal: Refine Matching and Filtering
|
||||
|
||||
## Summary
|
||||
|
||||
Improve matching accuracy and filtering capabilities by refining role inference, competence validation display, availability filtering, task search result management, and scoring categorization.
|
||||
|
||||
## Why
|
||||
|
||||
The current matching and filtering implementation has several areas where behavior can be improved:
|
||||
|
||||
1. **Role inference uses full task text:** Using both title and description for role inference can introduce noise, as descriptions often contain detailed requirements rather than role indicators. Titles typically contain clearer role signals.
|
||||
|
||||
2. **Validation truncates competence list:** The current validation output respects the global `max_competences` limit, which hides potentially relevant competences from the user during inspection.
|
||||
|
||||
3. **No full-coverage availability filter:** Users cannot filter for capacities that are available for the *entire* task duration—only overlap filtering is supported.
|
||||
|
||||
4. **Task search results lack refinement tools:** The capacity→task matching flow stores search results but provides no tools to filter or refine them, unlike the task→capacity direction.
|
||||
|
||||
5. **No category for irrelevant results:** All matches fall into Top/Good/Partial/Low even when there is zero overlap, making it harder to identify truly irrelevant results.
|
||||
|
||||
6. **Availability display shows exact dates:** Search results show full date ranges (e.g., "2025-03-01 .. 2025-06-30"), which clutters the output. A percentage-based overlap indicator would be more concise.
|
||||
|
||||
7. **Capacity search shows unnecessary Level column:** When matching capacities to a task, the "Level" column appears in results but adds no value in this context.
|
||||
|
||||
8. **Bug: Zero scores displayed in task search results:** When matching tasks to a capacity, all results show a score of 0.0 even though higher scores are calculated internally. This is a display formatting bug.
|
||||
|
||||
9. **Missing score breakdown:** Users cannot see how role match and competence match contribute to the overall score, making it harder to understand why a match scored a certain way.
|
||||
|
||||
## What changes
|
||||
|
||||
### 1. Use task title first for role inference (with description fallback)
|
||||
|
||||
**Change:** Role inference should use **task title first**. If title is missing/empty, fall back to full task text (title + description).
|
||||
|
||||
**Scope:**
|
||||
- `validate_task_requirements(task_id)` tool
|
||||
- `find_matching_capacities(...)` tool (internal role inference)
|
||||
- `find_matching_tasks(capacity_id)` tool (internal role inference)
|
||||
- `infer_primary_role(task_id=...|task_text=...)` tool
|
||||
|
||||
**Exclusions:**
|
||||
- `extract_requirements(task_description, ...)` continues using full text for competence inference.
|
||||
|
||||
**Implementation notes:**
|
||||
- Use separate cache keys for task role embeddings vs task competence embeddings.
|
||||
|
||||
### 2. Show all inferred competences in validation
|
||||
|
||||
**Change:** The `validate_task_requirements(task_id)` tool should display **all competences returned by inference** that meet the `min_similarity` threshold, up to the configured `max_competences`.
|
||||
|
||||
**Scope:**
|
||||
- `validate_task_requirements(task_id)` tool only
|
||||
|
||||
**Implementation notes:**
|
||||
- Keep calling competence inference with `max_competences=cfg.matching.inference.max_competences`.
|
||||
- Ensure table rendering shows all returned rows (no extra hard-coded truncation).
|
||||
|
||||
### 3. Add full-coverage availability filter
|
||||
|
||||
**Change:** Add a boolean parameter `is_fully_available` to `filter_search_results(...)` that filters results to only those fully covering the reference time range.
|
||||
|
||||
**Scope:**
|
||||
- `filter_search_results(...)` tool
|
||||
- Applies to both task→capacity and capacity→task directions
|
||||
|
||||
### 4. Extend search result tools for task direction
|
||||
|
||||
**Change:** Make `get_results_by_category(...)` and `filter_search_results(...)` work seamlessly for **both** task→capacity and capacity→task search directions.
|
||||
|
||||
**Additions:**
|
||||
- Store `search_type` in the cached search payload: `"capacity_search"` or `"task_search"`.
|
||||
- Add task-search filters:
|
||||
- `task_competence_filter` (list[str])
|
||||
- `task_text_filter` (str)
|
||||
|
||||
### 5. Add "Irrelevant" category
|
||||
|
||||
**Change:** Introduce a new scoring category **"Irrelevant"** for matches with `overall_score` below the configured `matching.thresholds.low` value.
|
||||
|
||||
**Config:**
|
||||
- Add `matching.thresholds.low` to `config.toml`.
|
||||
- Scores `>= low` are categorized as "Low".
|
||||
- Scores `< low` are categorized as "Irrelevant".
|
||||
|
||||
### 6. Update tool docstrings
|
||||
|
||||
**Change:** Update docstrings for all affected tools with:
|
||||
- Clear parameter descriptions
|
||||
- Examples showing both directions (where applicable)
|
||||
- Notes about filtering semantics
|
||||
|
||||
### 7. Show availability as percentage overlap
|
||||
|
||||
**Change:** Replace exact date ranges in search result tables with an overlap percentage indicator.
|
||||
|
||||
**Interpretation:**
|
||||
- task→capacity: overlap % relative to the task period
|
||||
- capacity→task: overlap % relative to the capacity period
|
||||
|
||||
### 8. Remove Level column from capacity search results
|
||||
|
||||
**Change:** Remove the "Level" column from result tables when matching capacities to a task.
|
||||
|
||||
### 9. Fix zero score display bug in task search
|
||||
|
||||
**Fix:** Standardize on `'overall_score'` field name for stored results in both directions.
|
||||
|
||||
### 10. Show separate role and competence scores
|
||||
|
||||
**Change:** Display separate scores for role match and competence match in addition to overall score:
|
||||
- `role_score`
|
||||
- `competence_score`
|
||||
- `overall_score`
|
||||
|
||||
## Impact
|
||||
|
||||
### API surface
|
||||
- **Breaking:**
|
||||
- Table column changes (removed "Level" from capacity search, added score columns, changed "Availability" format)
|
||||
- Output format changes may affect clients parsing Markdown tables
|
||||
- **New parameters:** `is_fully_available`, `task_competence_filter`, `task_text_filter`
|
||||
- **New category:** "Irrelevant"
|
||||
- **Bug fixes:** Zero score display in task search results
|
||||
|
||||
### Business logic
|
||||
- Role inference behavior changes (title-only)
|
||||
- Validation shows more competences
|
||||
- New filtering options
|
||||
- Availability display changed from dates to percentage
|
||||
|
||||
### Data access
|
||||
- May need separate cache keys for title-only embeddings
|
||||
- Search payloads now store `role_score` and `competence_score` separately
|
||||
|
||||
### Performance
|
||||
- Minimal impact (same number of embeddings, just different text)
|
||||
- Overlap percentage calculation adds negligible overhead
|
||||
|
||||
### Documentation
|
||||
- README.md: document new filters, category, and output format changes
|
||||
- assistant_system_prompt.md: update category list, add filtering examples, describe new table format
|
||||
- architecture.md: update scoring thresholds and output format
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Changing scoring weights or thresholds for existing categories (Top/Good/Partial/Low)
|
||||
- Adding fuzzy/approximate text search (simple substring match is sufficient)
|
||||
- Changing the fundamental matching algorithm
|
||||
- Showing individual competence match percentages (only overall competence score)
|
||||
- Changing the date overlap logic itself (only the display format)
|
||||
|
||||
## Open Questions
|
||||
|
||||
None—all design decisions have been clarified.
|
||||
|
||||
## Success Criteria
|
||||
|
||||
- Role inference uses task title only (verified by tests)
|
||||
- Validation shows all competences above threshold
|
||||
- `is_fully_available` filter works in both directions
|
||||
- Task search results can be filtered by competences and text
|
||||
- "Irrelevant" category appears when `overall_score < matching.thresholds.low`
|
||||
- Availability shown as percentage overlap in all result tables
|
||||
- Level column removed from capacity search results
|
||||
- Task search results show correct (non-zero) scores
|
||||
- All result tables show Role Score, Competence Score, and Overall Score columns
|
||||
- All tool docstrings are clear and include examples
|
||||
- All tests pass
|
||||
- `openspec validate refine-matching-and-filtering --strict` passes
|
||||
-197
@@ -1,197 +0,0 @@
|
||||
# Spec Delta: Matching Tools Refinements
|
||||
|
||||
## MODIFIED Requirements
|
||||
|
||||
### Requirement: Role inference uses task title only
|
||||
|
||||
Role inference for validation and matching tools SHALL use **only task title** for embedding and matching against role vocabulary. The `extract_requirements` tool continues to use combined text.
|
||||
|
||||
#### Scenario: Validate task requirements with title-only role inference
|
||||
|
||||
- **GIVEN** task ID `T-123` with title "Senior Frontend Developer" and description containing multiple competences
|
||||
- **AND** role vocabulary includes "Frontend Developer" and "Backend Developer"
|
||||
- **WHEN** the client calls `validate_task_requirements(task_id="T-123")`
|
||||
- **THEN** the system embeds **only** the title "Senior Frontend Developer"
|
||||
- **AND** the system matches against role vocabulary
|
||||
- **AND** the system returns inferred role based on title embedding only
|
||||
- **AND** the system still infers competences from full task text (title + description)
|
||||
|
||||
#### Scenario: Find matching capacities with title-only role inference
|
||||
|
||||
- **GIVEN** task with title "DevOps Engineer" and description "Experience with Kubernetes, Docker, CI/CD"
|
||||
- **WHEN** the client calls `find_matching_capacities(...)`
|
||||
- **THEN** the system infers task role from title "DevOps Engineer" only
|
||||
- **AND** the system infers task competences from full text (title + description)
|
||||
|
||||
#### Scenario: Find matching tasks with title-only role inference
|
||||
|
||||
- **GIVEN** capacity with role "Backend Developer"
|
||||
- **AND** task with title "Java Backend Developer" and description "Spring Boot, PostgreSQL"
|
||||
- **WHEN** the client calls `find_matching_tasks(capacity_id=...)`
|
||||
- **THEN** the system infers task role from title "Java Backend Developer" only
|
||||
- **AND** the system matches capacity role against task role
|
||||
|
||||
### Requirement: Validation shows all inferred competences returned by inference
|
||||
|
||||
Task validation SHALL display all inferred competences meeting `min_similarity` threshold, up to the configured `max_competences` limit. The output MUST NOT apply any additional hard-coded truncation (e.g., always showing only the top 8 rows).
|
||||
|
||||
#### Scenario: Validation displays up to max_competences competences
|
||||
|
||||
- **GIVEN** task with description mentioning many competences
|
||||
- **AND** config: `matching.inference.max_competences = 16`, `matching.inference.min_similarity = 0.4`
|
||||
- **AND** at least 16 competences have similarity >= 0.4
|
||||
- **WHEN** the client calls `validate_task_requirements(task_id="T-456")`
|
||||
- **THEN** the system returns and displays 16 competences (not fewer)
|
||||
- **AND** the output is NOT additionally limited to 8 competences
|
||||
- **AND** results are sorted by similarity score in descending order
|
||||
|
||||
### Requirement: Full availability filter parameter
|
||||
|
||||
Search result filtering SHALL support `is_fully_available` boolean parameter for date-coverage filtering in both directions.
|
||||
|
||||
#### Scenario: Filter capacities fully available for task duration
|
||||
|
||||
- **GIVEN** capacity search with 5 results
|
||||
- **AND** task period: 2025-03-01 to 2025-06-30
|
||||
- **AND** capacities: C1 (2025-02-01 to 2025-07-31), C2 (2025-03-15 to 2025-06-15), C3 (2025-01-01 to 2025-12-31)
|
||||
- **WHEN** the client calls `filter_search_results(search_id="...", availability_date_start="2025-03-01", availability_date_end="2025-06-30", is_fully_available=true)`
|
||||
- **THEN** the system returns filtered results containing only C1 and C3
|
||||
- **AND** applied filters table shows: `is_fully_available = true`
|
||||
|
||||
#### Scenario: Filter tasks within capacity availability window
|
||||
|
||||
- **GIVEN** task search with 4 results
|
||||
- **AND** capacity period: 2025-04-01 to 2025-08-31
|
||||
- **AND** tasks: T1 (2025-03-01 to 2025-05-31), T2 (2025-05-01 to 2025-07-31), T3 (2025-06-01 to 2025-09-30)
|
||||
- **WHEN** the client calls `filter_search_results(search_id="...", availability_date_start="2025-04-01", availability_date_end="2025-08-31", is_fully_available=true)`
|
||||
- **THEN** the system returns filtered results containing only T2
|
||||
- **AND** T1 excluded (starts before capacity)
|
||||
- **AND** T3 excluded (ends after capacity)
|
||||
|
||||
### Requirement: Task-specific search filters
|
||||
|
||||
Search result filtering SHALL support task-specific filters `task_competence_filter` and `task_text_filter` when filtering task search results.
|
||||
|
||||
#### Scenario: Filter tasks by required competences
|
||||
|
||||
- **GIVEN** task search with 6 results
|
||||
- **AND** tasks requiring various competences including Python, React, Docker
|
||||
- **WHEN** the client calls `filter_search_results(search_id="...", task_competence_filter=["Python", "Docker"])`
|
||||
- **THEN** the system returns only tasks that require either Python OR Docker
|
||||
- **AND** applied filters table shows: `task_competence_filter = ["Python", "Docker"]`
|
||||
|
||||
#### Scenario: Filter tasks by text search
|
||||
|
||||
- **GIVEN** task search with 5 results
|
||||
- **AND** task T1: title "Frontend Developer", description "React and TypeScript"
|
||||
- **AND** task T2: title "Backend Engineer", description "Python and PostgreSQL"
|
||||
- **AND** task T3: title "Full-Stack Developer", description "React, Node.js, TypeScript"
|
||||
- **WHEN** the client calls `filter_search_results(search_id="...", task_text_filter="typescript")`
|
||||
- **THEN** the system returns T1 and T3 (case-insensitive match)
|
||||
- **AND** applied filters table shows: `task_text_filter = "typescript"`
|
||||
|
||||
### Requirement: Irrelevant match category
|
||||
|
||||
Matching tools SHALL categorize matches with `overall_score` below `matching.thresholds.low` as "Irrelevant" and include them in summary tables and result displays.
|
||||
|
||||
#### Scenario: Match is categorized as Irrelevant
|
||||
|
||||
- **GIVEN** capacity with role "Data Scientist" and competences ["Python", "Machine Learning"]
|
||||
- **AND** task requiring role "Frontend Developer" and competences ["React", "TypeScript"]
|
||||
- **AND** computed match score: 0.05
|
||||
- **WHEN** the system categorizes the match
|
||||
- **THEN** the match is categorized as "Irrelevant"
|
||||
- **AND** summary table includes "Irrelevant" category with count
|
||||
- **AND** results include Irrelevant section if such matches exist
|
||||
|
||||
### Requirement: Enhanced tool docstrings with examples
|
||||
|
||||
All affected matching and filtering tools SHALL include updated docstrings with usage examples demonstrating new features.
|
||||
|
||||
#### Scenario: Tool docstrings show usage examples
|
||||
|
||||
- **GIVEN** tools with new parameters and features
|
||||
- **WHEN** a client inspects tool descriptions
|
||||
- **THEN** each tool docstring includes clear parameter descriptions
|
||||
- **AND** usage examples for new features
|
||||
- **AND** guidance on when to use each feature
|
||||
- **AND** cross-references to related tools
|
||||
|
||||
### Requirement: Availability displayed as percentage overlap
|
||||
|
||||
Search result tables SHALL display availability as percentage overlap instead of exact date ranges.
|
||||
|
||||
#### Scenario: Capacity search shows percentage overlap
|
||||
|
||||
- **GIVEN** task with period 2025-03-01 to 2025-06-30 (122 days)
|
||||
- **AND** capacity with period 2025-03-15 to 2025-06-30 (108 days overlap)
|
||||
- **WHEN** the client views capacity search results
|
||||
- **THEN** the Availability column shows "88%" (108/122 rounded)
|
||||
- **AND** the Availability column is positioned left of score columns
|
||||
|
||||
#### Scenario: Task search shows percentage overlap
|
||||
|
||||
- **GIVEN** capacity with period 2025-04-01 to 2025-08-31 (153 days)
|
||||
- **AND** task with period 2025-05-01 to 2025-07-31 (92 days overlap)
|
||||
- **WHEN** the client views task search results
|
||||
- **THEN** the Availability column shows "60%" (92/153 rounded)
|
||||
|
||||
#### Scenario: Open-ended availability period
|
||||
|
||||
- **GIVEN** capacity with open end date (end_date is NULL)
|
||||
- **WHEN** calculating overlap percentage
|
||||
- **THEN** the system displays "Open" or "∞" instead of a percentage
|
||||
|
||||
### Requirement: Level column removed from capacity search
|
||||
|
||||
Capacity search result tables SHALL NOT include the "Level" column.
|
||||
|
||||
#### Scenario: Find matching capacities output excludes Level
|
||||
|
||||
- **GIVEN** user searches for matching capacities
|
||||
- **WHEN** the client calls find_matching_capacities(...)
|
||||
- **THEN** the result table includes columns: ID, Owner, Role, Competences, Availability, Role Score, Competence Score, Overall Score
|
||||
- **AND** the table does NOT include a Level column
|
||||
|
||||
### Requirement: Task search score display bug fixed
|
||||
|
||||
The system SHALL correctly display non-zero scores for task search results in get_results_by_category.
|
||||
|
||||
#### Scenario: Task search results show correct scores
|
||||
|
||||
- **GIVEN** capacity with role "Backend Developer" and competences ["Python", "PostgreSQL"]
|
||||
- **AND** task requiring role "Backend Developer" and competences ["Python", "Django"]
|
||||
- **AND** computed overall score is 0.85
|
||||
- **WHEN** the client calls find_matching_tasks(capacity_id=...)
|
||||
- **THEN** the result table shows Overall Score as "0.850"
|
||||
- **WHEN** the client calls get_results_by_category(search_id=..., category="Top")
|
||||
- **THEN** the result table shows Overall Score as "0.850" (not "0.000")
|
||||
|
||||
### Requirement: Separate role and competence scores displayed
|
||||
|
||||
All matching result tables SHALL display separate Role Score, Competence Score, and Overall Score columns.
|
||||
|
||||
#### Scenario: Capacity search shows separate scores
|
||||
|
||||
- **GIVEN** task requiring role "Frontend Developer" and competences ["React", "TypeScript", "CSS"]
|
||||
- **AND** capacity with role "Frontend Developer" and competences ["React", "TypeScript"]
|
||||
- **AND** computed role_score = 1.0, competence_score = 0.667, overall_score = 0.750
|
||||
- **WHEN** the client views capacity search results
|
||||
- **THEN** the result table includes columns: Role Score, Competence Score, Overall Score (rightmost)
|
||||
- **AND** the row shows: 1.000, 0.667, 0.750
|
||||
|
||||
#### Scenario: Task search shows separate scores
|
||||
|
||||
- **GIVEN** capacity with role "DevOps Engineer" and competences ["Kubernetes", "Docker"]
|
||||
- **AND** task requiring role "DevOps Engineer" and competences ["Kubernetes", "Terraform", "AWS"]
|
||||
- **AND** computed role_score = 1.0, competence_score = 0.333, overall_score = 0.500
|
||||
- **WHEN** the client views task search results
|
||||
- **THEN** the result table includes columns: Role Score, Competence Score, Overall Score (rightmost)
|
||||
- **AND** the row shows: 1.000, 0.333, 0.500
|
||||
|
||||
#### Scenario: Categorization uses overall score only
|
||||
|
||||
- **GIVEN** a match with role_score = 1.0, competence_score = 0.2, overall_score = 0.45
|
||||
- **WHEN** the system categorizes the match
|
||||
- **THEN** the match is categorized as "Partial" (based on overall_score >= 0.4)
|
||||
- **AND** role_score and competence_score are NOT used for categorization
|
||||
@@ -1,259 +0,0 @@
|
||||
# Implementation Tasks: Refine Matching and Filtering
|
||||
|
||||
## Status: Completed
|
||||
|
||||
## Summary
|
||||
|
||||
This change implements 10 improvements to matching and filtering:
|
||||
|
||||
1. **Title-only role inference**: Use task title (not description) for role matching
|
||||
2. **Unlimited validation**: Show all competences above threshold in validation
|
||||
3. **Full-coverage filter**: Add `is_fully_available` for complete date coverage filtering
|
||||
4. **Task filtering**: Add competence and text filters for task search results
|
||||
5. **Irrelevant category**: Add an "Irrelevant" category for scores below `matching.thresholds.low`
|
||||
6. **Enhanced docstrings**: Comprehensive examples for all tools
|
||||
7. **Percentage availability**: Show overlap as percentage instead of date ranges
|
||||
8. **Remove Level column**: Clean up capacity search output
|
||||
9. **Fix score bug**: Correct zero score display in task search results
|
||||
10. **Separate scores**: Display Role Score, Competence Score, Overall Score
|
||||
|
||||
## Clarifications / Decisions (added during review)
|
||||
|
||||
- **Separate embedding keys (MUST):** Roles and competences MUST use separate embedding cache keys/paths for both tasks and capacities.
|
||||
- If such separation does not exist yet, it MUST be introduced.
|
||||
- For task-role inference specifically, we do NOT need separate keys for title-only vs title+description; the role embedding key can stay stable while input text changes per fallback logic.
|
||||
- **Role inference fallback:** If task title is missing/too short/generic, infer role from task description instead. If that still yields no role above threshold (or no vocab match), role_score MUST be `0.0`.
|
||||
- **`is_fully_available` without explicit date params:** `filter_search_results(..., is_fully_available=true)` MUST work even when `availability_date_start/end` are omitted by using reference dates stored in the search context.
|
||||
- **Availability % with open-ended other_end:** If the reference period is fully defined and `other_end is None`, availability percentage MUST still be computed (overlap is well-defined until ref_end).
|
||||
- **Overall score formula (fixed):** `overall_score` MUST be a weighted mean of role_score and competence_score using config weights:
|
||||
- `overall_score = role_weight * role_score + competence_weight * competence_score`
|
||||
- Weights are read from `cfg.matching.role_weight` and `cfg.matching.competence_weight`.
|
||||
- **Irrelevant always displayed:** The "Irrelevant" category MUST always be shown in summary tables and category lists.
|
||||
- **Task filters are case-insensitive:** task competence and text filtering MUST be case-insensitive.
|
||||
- `required_competences` are structured fields from task details and MUST be present in task-search results.
|
||||
|
||||
## Estimated Effort
|
||||
|
||||
- **Phase 1 (Core changes)**: 6-8 hours
|
||||
- **Phase 2 (Testing)**: 3-4 hours
|
||||
- **Phase 3 (Documentation)**: 2-3 hours
|
||||
- **Phase 4 (Validation)**: 1 hour
|
||||
- **Total**: 12-16 hours
|
||||
|
||||
## Phase 1: Core Changes
|
||||
|
||||
### Task 1.1: Title-only role inference ✅
|
||||
**Files:** `src/teamlandkarte_mcp/matching/vocabulary.py`, `src/teamlandkarte_mcp/mcp_server.py`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Identify where task text is built for role inference (startup preload, validation, matching)
|
||||
- [x] Modify role inference code paths to use title only (not title + description):
|
||||
- `startup_preload()` for task embeddings
|
||||
- `validate_task_requirements()` for role inference
|
||||
- `find_matching_capacities()` for task role inference
|
||||
- `find_matching_tasks()` for task role inference
|
||||
- [x] **Important**: Competence inference still uses title + description (unchanged)
|
||||
- [x] Verify cache key strategy: reuse stable role cache key; competence embeddings use separate key
|
||||
- [x] Document: role embeddings now use title-first text with fallback; competence embeddings use full text
|
||||
- [x] Add fallback when title is empty
|
||||
|
||||
**Acceptance:**
|
||||
- Role inference uses title-first for role embedding
|
||||
- Competence inference still uses title + description
|
||||
- Cache separation prevents collisions
|
||||
- Fallback handles empty titles gracefully
|
||||
|
||||
---
|
||||
|
||||
### Task 1.2: Unlimited competences in validation ✅
|
||||
**Files:** `src/teamlandkarte_mcp/mcp_server.py`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Keep using `cfg.matching.inference.max_competences` in `validate_task_requirements()` (do NOT override it)
|
||||
- [x] Ensure the output table displays **all competences returned by inference**
|
||||
- [x] Ensure the output is not hard-limited to 8 competences anywhere in formatting
|
||||
- [x] Keep threshold behavior (`min_similarity`) unchanged
|
||||
- [x] Update docstring to clarify: validation shows up to `max_competences` competences meeting threshold
|
||||
|
||||
**Acceptance:**
|
||||
- Validation displays all inferred competences returned by inference (up to configured `max_competences`)
|
||||
- No additional hard-coded limit is applied in output formatting
|
||||
- Threshold still applies (`min_similarity` config)
|
||||
|
||||
---
|
||||
|
||||
### Task 1.3: Add `is_fully_available` filter ✅
|
||||
**Files:** `src/teamlandkarte_mcp/mcp_server.py`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Add `search_type` field to search payloads:
|
||||
- Update `find_matching_capacities()` to store `"capacity_search"`
|
||||
- Update `find_matching_tasks()` to store `"task_search"`
|
||||
- [x] Add `is_fully_available` parameter to `filter_search_results()`
|
||||
- [x] Detect `search_type` from search payload metadata
|
||||
- [x] Implement full-coverage logic for `capacity_search`
|
||||
- [x] Implement full-coverage logic for `task_search`
|
||||
- [x] Use stored reference dates when explicit availability_date_* are omitted
|
||||
|
||||
**Acceptance:**
|
||||
- Search payloads include `search_type` field
|
||||
- `is_fully_available=True` filters for full coverage in both directions
|
||||
- Default keeps overlap behavior
|
||||
- Edge cases handled (missing dates)
|
||||
|
||||
---
|
||||
|
||||
### Task 1.4: Add task filtering parameters ✅
|
||||
**Files:** `src/teamlandkarte_mcp/mcp_server.py`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Add `task_competence_filter` parameter to `filter_search_results()`
|
||||
- [x] Add `task_text_filter` parameter to `filter_search_results()`
|
||||
- [x] Implement competence filtering for tasks
|
||||
- [x] Implement text filtering for tasks (case-insensitive)
|
||||
- [x] Apply filters only when `search_type == "task_search"`
|
||||
|
||||
**Acceptance:**
|
||||
- Task competence filter works for capacity→task searches
|
||||
- Task text filter works for capacity→task searches
|
||||
- Filters are ignored for task→capacity searches
|
||||
|
||||
---
|
||||
|
||||
### Task 1.5: Add "Irrelevant" category ✅
|
||||
**Files:** `src/teamlandkarte_mcp/matching/scorer.py`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Modify categorization to add `score < matching.thresholds.low` → "Irrelevant"
|
||||
- [x] Update category order: Top, Good, Partial, Low, Irrelevant
|
||||
- [x] Ensure "Irrelevant" appears in summary tables and category lists
|
||||
|
||||
**Acceptance:**
|
||||
- Scores below `matching.thresholds.low` are categorized as "Irrelevant"
|
||||
- All tools display consistent categories
|
||||
|
||||
---
|
||||
|
||||
### Task 1.6: Update tool docstrings ✅
|
||||
**Files:** `src/teamlandkarte_mcp/mcp_server.py`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Update `filter_search_results()` docstring with bidirectional examples and new params
|
||||
- [x] Update `get_results_by_category()` docstring with both search types examples
|
||||
- [x] Update `find_matching_capacities()` docstring with title-first role inference note
|
||||
- [x] Update `find_matching_tasks()` docstring with title-first role inference note
|
||||
- [x] Update `validate_task_requirements()` docstring with title-first + max_competences note
|
||||
|
||||
**Acceptance:**
|
||||
- All tool docstrings are clear and comprehensive
|
||||
- Examples demonstrate both search directions
|
||||
|
||||
---
|
||||
|
||||
### Task 1.7: Show availability as percentage overlap ✅
|
||||
**Files:** `src/teamlandkarte_mcp/mcp_server.py`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Create helper function `_calculate_overlap_percentage(...)`
|
||||
- [x] Update `find_matching_capacities()` and `find_matching_tasks()` output tables
|
||||
- [x] Update `get_results_by_category()` and `filter_search_results()` preview tables
|
||||
|
||||
**Acceptance:**
|
||||
- All result tables show availability as percentage
|
||||
- Open-ended periods handled
|
||||
|
||||
---
|
||||
|
||||
### Task 1.8: Remove Level column from capacity search ✅
|
||||
**Files:** `src/teamlandkarte_mcp/mcp_server.py`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Remove "Level" from capacity-search matching result tables
|
||||
- [x] Keep Level in `list_free_capacities()` (inspection)
|
||||
|
||||
**Acceptance:**
|
||||
- Capacity search matching outputs do NOT show "Level"
|
||||
- Listing/inspection still shows Level
|
||||
|
||||
---
|
||||
|
||||
### Task 1.9: Fix zero score display bug ✅
|
||||
**Files:** `src/teamlandkarte_mcp/mcp_server.py`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Standardize on `overall_score` in task-search stored payloads
|
||||
- [x] Verify display in `get_results_by_category()`
|
||||
|
||||
**Acceptance:**
|
||||
- Task search results show correct non-zero scores
|
||||
|
||||
---
|
||||
|
||||
### Task 1.10: Display separate role and competence scores ✅
|
||||
**Files:** `src/teamlandkarte_mcp/mcp_server.py`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Store and display `role_score`, `competence_score`, `overall_score` for both searches
|
||||
- [x] Ensure consistent formatting in result tables
|
||||
|
||||
**Acceptance:**
|
||||
- All result tables show Role/Competence/Overall score columns
|
||||
|
||||
---
|
||||
|
||||
## Phase 2: Testing
|
||||
|
||||
### Task 2.1: Unit tests ✅
|
||||
**Files:** `tests/test_matching_refinements.py` (new)
|
||||
|
||||
**Subtasks:**
|
||||
- [x] Test title-first role inference helpers and fallback behavior
|
||||
- [x] Test "Irrelevant" category threshold
|
||||
- [x] Test availability percentage calculation incl. open-ended end date
|
||||
|
||||
**Acceptance:**
|
||||
- Unit tests cover key refinements
|
||||
- Tests pass with `pytest -q`
|
||||
|
||||
---
|
||||
|
||||
### Task 2.2: Integration tests ✅
|
||||
**Files:** `tests/test_integration_matching.py`
|
||||
|
||||
**Subtasks:**
|
||||
- [x] End-to-end test: capacity search with new filters and display
|
||||
- [x] End-to-end test: task search with new filters and display
|
||||
- [x] Test filtering with multiple parameters
|
||||
- [x] Test pagination with new table format
|
||||
- [x] Verify Markdown table format consistency
|
||||
|
||||
**Acceptance:**
|
||||
- Integration tests demonstrate full workflows
|
||||
- All tests pass
|
||||
|
||||
---
|
||||
|
||||
## Phase 3: Documentation
|
||||
|
||||
### Task 3.1: Update README ✅
|
||||
**Files:** `README.md`
|
||||
|
||||
### Task 3.2: Update assistant prompt ✅
|
||||
**Files:** `docs/assistant_system_prompt.md`
|
||||
|
||||
### Task 3.3: Update architecture docs ✅
|
||||
**Files:** `docs/architecture.md`
|
||||
|
||||
---
|
||||
|
||||
## Phase 4: Validation and Completion
|
||||
|
||||
### Task 4.1: OpenSpec validation ✅
|
||||
|
||||
- [x] All tasks in this change marked as completed
|
||||
- [x] Unit + integration tests passing (`uv run pytest -q`)
|
||||
|
||||
### Task 4.2: Final testing and deployment ✅
|
||||
|
||||
- [x] Final local test run completed
|
||||
- [x] No external deployment steps are automated in this repository
|
||||
-8
@@ -1,8 +0,0 @@
|
||||
+# Changelog
|
||||
+
|
||||
+## replace-heuristics-with-azure-openai
|
||||
+
|
||||
+- Removed heuristic/sampling-based similarity and replaced role similarity with Azure embedding cosine similarity.
|
||||
+- Tightened docstrings and typing for `TaskAnalyzer` and `SimilarityEngine`.
|
||||
+- Removed stray debug output and ensured stderr-safe logging.
|
||||
+
|
||||
-752
@@ -1,752 +0,0 @@
|
||||
# Change Proposal: Replace Heuristics with Azure OpenAI Embeddings and LLM
|
||||
|
||||
- **Change ID**: `replace-heuristics-with-azure-openai`
|
||||
- **Status**: Implemented
|
||||
- **Target**: `teamlandkarte-mcp`
|
||||
- **Author**: Thomas Handke
|
||||
- **Date**: 2026-02-13
|
||||
|
||||
## Summary
|
||||
|
||||
Replace the current FastMCP sampling (unavailable) and deterministic heuristic fallbacks with direct Azure OpenAI API integration for:
|
||||
|
||||
1. **Competence and role similarity matching** via embeddings (`text-embedding-3-large`, 3072 dimensions)
|
||||
2. **Requirements extraction and validation** via LLM (`gpt-4.1`)
|
||||
|
||||
Additionally:
|
||||
- Rename `database.toml` → `config.toml` (and template accordingly)
|
||||
- Implement persistent embedding cache using a local SQLite database
|
||||
- Support two similarity scoring strategies (per-skill vs. aggregate) via configuration
|
||||
- Remove all FastMCP sampling code and heuristic fallbacks completely
|
||||
|
||||
## Motivation
|
||||
|
||||
### Current State Problems
|
||||
|
||||
1. **FastMCP sampling is unavailable**: The `mcp` package does not expose a `FastMCP.sampling` API, forcing the system to rely entirely on weak deterministic heuristics.
|
||||
|
||||
2. **Heuristic limitations**:
|
||||
- Role inference: simple keyword matching (`"cloud" → "Cloud Engineer"`)
|
||||
- Competence extraction: small hardcoded keyword list (~15 terms)
|
||||
- Competence similarity: exact normalized match (1.0) or substring (0.7), otherwise 0.0
|
||||
- No true semantic understanding
|
||||
|
||||
3. **Poor matching quality**:
|
||||
- Synonyms not recognized (e.g., "React.js" vs "ReactJS")
|
||||
- Related skills missed (e.g., "FastAPI" vs "REST API Development")
|
||||
- Role extraction frequently returns `"(unknown)"`
|
||||
|
||||
4. **Maintenance burden**: Heuristic keyword lists require manual updates
|
||||
|
||||
### Proposed Benefits
|
||||
|
||||
1. **Semantic matching**: True similarity via embeddings (e.g., "Kubernetes" ↔ "K8s", "Python" ↔ "Python 3")
|
||||
2. **Robust extraction**: LLM-backed role and competence extraction from free text
|
||||
3. **Better user experience**: More accurate Top/Good/Partial/Low categorization
|
||||
4. **Reduced maintenance**: No manual keyword list updates
|
||||
5. **Production-ready**: Direct API integration, no dependency on MCP sampling feature
|
||||
|
||||
## Goals
|
||||
|
||||
1. Replace `TaskAnalyzer.semantic_competence_similarity` with embedding-based similarity
|
||||
2. Replace `TaskAnalyzer.extract_ranked_roles` with Azure OpenAI LLM
|
||||
3. Replace `TaskAnalyzer._extract_requirements_from_description` with Azure OpenAI LLM
|
||||
4. Replace `TaskAnalyzer.apply_requirement_update` with Azure OpenAI LLM
|
||||
5. Implement persistent embedding cache (SQLite)
|
||||
6. Support two similarity strategies via config
|
||||
7. Remove all FastMCP sampling and heuristic code
|
||||
8. Rename `database.toml` → `config.toml`
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- Changing the overall matching workflow (confirmation gate, guided capture, etc.)
|
||||
- Changing the scoring weights (competence 0.8, role 0.2)
|
||||
- Changing the categorical thresholds (Top/Good/Partial/Low)
|
||||
- Supporting multiple embedding models
|
||||
- Supporting other LLM providers (only Azure OpenAI)
|
||||
|
||||
## Proposed Changes
|
||||
|
||||
### 1. Configuration Changes
|
||||
|
||||
#### 1.1 Rename configuration file
|
||||
- `database.toml` → `config.toml`
|
||||
- `database.toml.example` → `config.toml.example`
|
||||
- Update all references in code, docs, README
|
||||
|
||||
#### 1.2 New configuration sections in `config.toml`
|
||||
|
||||
```toml
|
||||
[azure_openai]
|
||||
endpoint = "https://aiservice-ca00361106.cognitiveservices.azure.com/"
|
||||
api_version_embeddings = "2024-02-01"
|
||||
api_version_llm = "2024-12-01-preview"
|
||||
|
||||
[azure_openai.embeddings]
|
||||
model = "text-embedding-3-large"
|
||||
# Azure deployment name is equal to model name
|
||||
deployment = "text-embedding-3-large"
|
||||
|
||||
[azure_openai.llm]
|
||||
model = "gpt-4.1"
|
||||
# Azure deployment name is equal to model name
|
||||
deployment = "gpt-4.1"
|
||||
temperature = 0.2
|
||||
max_tokens = 2000
|
||||
|
||||
[embedding_cache]
|
||||
enabled = true
|
||||
db_path = "embeddings_cache.db"
|
||||
ttl_days = 30
|
||||
|
||||
[matching.similarity]
|
||||
# Strategy: "per_skill" or "aggregate"
|
||||
# - per_skill: Match each required skill to best candidate skill (current behavior)
|
||||
# - aggregate: Compare average embedding of all required vs all candidate skills
|
||||
strategy = "per_skill"
|
||||
# Chat model/deployment name (Azure OpenAI)
|
||||
chat_model = "gpt-4.1"
|
||||
```
|
||||
|
||||
#### 1.3 Environment variables (`.env`)
|
||||
```bash
|
||||
AZURE_OPENAI_EMBEDDING_API_KEY="..."
|
||||
AZURE_OPENAI_LLM_API_KEY="..."
|
||||
```
|
||||
|
||||
### 2. Embedding Cache Implementation
|
||||
|
||||
Create `src/teamlandkarte_mcp/cache/embedding_cache.py`:
|
||||
|
||||
```python
|
||||
class EmbeddingCache:
|
||||
"""Persistent embedding cache using SQLite.
|
||||
|
||||
Schema:
|
||||
embeddings(
|
||||
id INTEGER PRIMARY KEY,
|
||||
text TEXT UNIQUE NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
embedding BLOB NOT NULL, -- UTF-8 bytes of JSON-serialized list[float]
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
|
||||
Notes:
|
||||
- Store embeddings as UTF-8 encoded JSON in a BLOB column for portability.
|
||||
- Serialize with: json.dumps(embedding).encode("utf-8")
|
||||
- Deserialize with: json.loads(blob.decode("utf-8"))
|
||||
"""
|
||||
|
||||
def __init__(self, db_path: str, ttl_days: int = 30):
|
||||
"""Initialize cache, create DB if not exists."""
|
||||
|
||||
def get(self, text: str, model: str) -> Optional[list[float]]:
|
||||
"""Retrieve cached embedding, return None if missing/expired."""
|
||||
|
||||
def put(self, text: str, model: str, embedding: list[float]) -> None:
|
||||
"""Store embedding in cache."""
|
||||
|
||||
def cleanup_expired(self) -> int:
|
||||
"""Remove embeddings older than TTL, return count deleted."""
|
||||
```
|
||||
|
||||
### 3. Azure OpenAI Client Layer
|
||||
|
||||
Create `src/teamlandkarte_mcp/azure/openai_client.py`:
|
||||
|
||||
```python
|
||||
class AzureOpenAIClient:
|
||||
"""Wrapper for Azure OpenAI API calls using AsyncAzureOpenAI client."""
|
||||
|
||||
def __init__(self, config: AzureOpenAIConfig, embedding_cache: EmbeddingCache):
|
||||
"""Initialize AsyncAzureOpenAI clients for embeddings and LLM.
|
||||
|
||||
Uses openai.AsyncAzureOpenAI for all async API calls.
|
||||
"""
|
||||
|
||||
async def get_embedding(self, text: str) -> list[float]:
|
||||
"""Get embedding for text, using cache if available.
|
||||
|
||||
Raises:
|
||||
AzureAPIError: If API call fails (no fallback).
|
||||
"""
|
||||
|
||||
async def get_embeddings_batch(self, texts: list[str]) -> list[list[float]]:
|
||||
"""Get embeddings for multiple texts (not batched per user requirement)."""
|
||||
|
||||
async def chat_completion(
|
||||
self,
|
||||
system: str,
|
||||
user: str,
|
||||
response_format: dict = None
|
||||
) -> str:
|
||||
"""LLM completion with structured JSON response.
|
||||
|
||||
Raises:
|
||||
AzureAPIError: If API call fails (no fallback).
|
||||
"""
|
||||
```
|
||||
|
||||
### 4. Similarity Computation
|
||||
|
||||
Create `src/teamlandkarte_mcp/matching/similarity.py`:
|
||||
|
||||
```python
|
||||
def cosine_similarity(vec_a: list[float], vec_b: list[float]) -> float:
|
||||
"""Compute cosine similarity between two vectors."""
|
||||
|
||||
class SimilarityEngine:
|
||||
"""Compute semantic similarity using embeddings."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
client: AzureOpenAIClient,
|
||||
strategy: str = "per_skill"
|
||||
):
|
||||
"""Initialize with Azure client and strategy."""
|
||||
|
||||
async def compute_competence_similarity(
|
||||
self,
|
||||
required: list[str],
|
||||
candidate: list[str],
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""Compute similarity using configured strategy.
|
||||
|
||||
Returns same format as current semantic_competence_similarity:
|
||||
{required_comp: {best_match: str|null, score: float, rationale: str}}
|
||||
|
||||
Behavioral guarantees:
|
||||
- `rationale` is always included (even when best_match is null).
|
||||
- If `required` is empty, return {}.
|
||||
- If `candidate` is empty, return an entry per required competence with:
|
||||
best_match=null, score=0.0, and a short rationale.
|
||||
"""
|
||||
|
||||
async def _per_skill_similarity(
|
||||
self, required: list[str], candidate: list[str]
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""For each required skill, find best matching candidate skill."""
|
||||
|
||||
async def _aggregate_similarity(
|
||||
self, required: list[str], candidate: list[str]
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""Compare average embeddings of required vs candidate skills."""
|
||||
|
||||
async def compute_role_similarity(
|
||||
self, required_role: str, candidate_role: str
|
||||
) -> float:
|
||||
"""Compute role similarity (0.0 to 1.0).
|
||||
|
||||
Behavioral guarantees:
|
||||
- If either role is empty/None, return 0.0.
|
||||
- If either role is "(unknown)" (case-insensitive, trimmed), return 0.0.
|
||||
- Otherwise compute cosine similarity of the role embeddings.
|
||||
|
||||
Note:
|
||||
- Role similarity returns only a float score. Rationales are provided by
|
||||
`compute_competence_similarity()` results.
|
||||
"""
|
||||
```
|
||||
|
||||
### 5. TaskAnalyzer Refactoring
|
||||
|
||||
**Remove**:
|
||||
- `_sample_json()` method
|
||||
- `_heuristic_competences_from_text()` static method
|
||||
- All heuristic fallback code in:
|
||||
- `extract_ranked_roles()`
|
||||
- `_extract_requirements_from_description()`
|
||||
- `apply_requirement_update()`
|
||||
- `semantic_competence_similarity()`
|
||||
|
||||
**Replace with**:
|
||||
|
||||
```python
|
||||
class TaskAnalyzer:
|
||||
"""Task text analyzer using Azure OpenAI."""
|
||||
|
||||
def __init__(self, azure_client: AzureOpenAIClient):
|
||||
self._client = azure_client
|
||||
|
||||
async def extract_ranked_roles(
|
||||
self, description: str, limit: int = 5
|
||||
) -> list[RankedRole]:
|
||||
"""Extract ranked roles using Azure OpenAI LLM.
|
||||
|
||||
Raises:
|
||||
AzureAPIError: If API call fails.
|
||||
"""
|
||||
|
||||
async def _extract_requirements_from_description(
|
||||
self, description: str
|
||||
) -> ExtractedRequirements:
|
||||
"""Extract requirements using Azure OpenAI LLM.
|
||||
|
||||
Raises:
|
||||
AzureAPIError: If API call fails.
|
||||
"""
|
||||
|
||||
async def apply_requirement_update(
|
||||
self, current: Requirements, change_description: str
|
||||
) -> Requirements:
|
||||
"""Update requirements using Azure OpenAI LLM.
|
||||
|
||||
Raises:
|
||||
AzureAPIError: If API call fails.
|
||||
"""
|
||||
```
|
||||
|
||||
**Additional behavior change**:
|
||||
- The current `TaskAnalyzer` raises `AnalysisError` only when MCP sampling is present but fails/returns invalid JSON, and otherwise silently falls back to heuristics.
|
||||
- After this change, heuristics are removed. Any Azure OpenAI failure (HTTP error, timeout after retries, invalid JSON/schema) will result in an analysis exception (e.g. `AzureAPIError` and/or a narrower `AnalysisError`) and should be surfaced to callers.
|
||||
|
||||
**Testing impact**:
|
||||
- Existing tests that currently assert heuristic fallback behavior in `TaskAnalyzer` must be updated.
|
||||
- Replace “heuristic fallback expected” assertions with either:
|
||||
- mocked Azure responses (unit tests), or
|
||||
- explicit error expectations when Azure is unavailable.
|
||||
|
||||
### 6. Matcher & Scorer Integration
|
||||
|
||||
Update `src/teamlandkarte_mcp/matching/matcher.py`:
|
||||
|
||||
**Key Changes**:
|
||||
1. Accept `SimilarityEngine` in constructor
|
||||
2. Replace `TaskAnalyzer.semantic_competence_similarity()` calls with `SimilarityEngine.compute_competence_similarity()`
|
||||
3. Replace the existing `_role_similarity()` function with `SimilarityEngine.compute_role_similarity()`
|
||||
4. Maintain current scoring weights (competence 0.8, role 0.2)
|
||||
|
||||
**Details**:
|
||||
- The `Matcher` currently uses `TaskAnalyzer.semantic_competence_similarity()` for competence matching (per-skill, best-match strategy)
|
||||
- It also has a standalone `_role_similarity()` function for role matching (currently uses simple string comparison)
|
||||
- Both will be replaced by `SimilarityEngine` methods, which provide semantic similarity via embeddings
|
||||
- The `SimilarityEngine.compute_competence_similarity()` returns the same dict structure as current `semantic_competence_similarity()`, ensuring drop-in compatibility
|
||||
- The `SimilarityEngine.compute_role_similarity()` returns a float (0.0 to 1.0), matching current `_role_similarity()` signature
|
||||
|
||||
```python
|
||||
class Matcher:
|
||||
"""Capacity matcher with semantic similarity."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
analyzer: TaskAnalyzer,
|
||||
similarity_engine: SimilarityEngine,
|
||||
config: MatchingConfig
|
||||
):
|
||||
"""Initialize matcher with analyzer, similarity engine, and config."""
|
||||
self._analyzer = analyzer
|
||||
self._similarity = similarity_engine
|
||||
self._config = config
|
||||
|
||||
async def _compute_match_score(
|
||||
self,
|
||||
required: Requirements,
|
||||
capacity: Capacity
|
||||
) -> MatchScore:
|
||||
"""Compute match score using SimilarityEngine.
|
||||
|
||||
Competence matching:
|
||||
comp_sim = await self._similarity.compute_competence_similarity(
|
||||
required.competences, capacity.competences
|
||||
)
|
||||
|
||||
Role matching:
|
||||
role_score = await self._similarity.compute_role_similarity(
|
||||
required.role, capacity.role
|
||||
)
|
||||
|
||||
Final score: (0.8 * avg_comp_score) + (0.2 * role_score)
|
||||
"""
|
||||
```
|
||||
|
||||
**Owning module note**:
|
||||
- The end-to-end match scoring aggregation currently lives in `src/teamlandkarte_mcp/matching/matcher.py` (with supporting score shaping asserted by `tests/test_scorer.py`).
|
||||
|
||||
### 7. Server Initialization Changes
|
||||
|
||||
Update `src/teamlandkarte_mcp/mcp_server.py`:
|
||||
|
||||
```python
|
||||
# Initialize Azure OpenAI components
|
||||
embedding_cache = EmbeddingCache(
|
||||
db_path=cfg.embedding_cache.db_path,
|
||||
ttl_days=cfg.embedding_cache.ttl_days,
|
||||
) if cfg.embedding_cache.enabled else None
|
||||
|
||||
azure_client = AzureOpenAIClient(cfg.azure_openai, embedding_cache)
|
||||
similarity_engine = SimilarityEngine(
|
||||
azure_client,
|
||||
strategy=cfg.matching.similarity.strategy,
|
||||
)
|
||||
|
||||
# Initialize analyzer with Azure client
|
||||
analyzer = TaskAnalyzer(azure_client)
|
||||
|
||||
# Initialize matcher with similarity engine
|
||||
matcher = Matcher(analyzer, similarity_engine, cfg.matching)
|
||||
```
|
||||
|
||||
### 8. Cost Estimation & Monitoring
|
||||
|
||||
Add cost tracking module `src/teamlandkarte_mcp/azure/cost_tracker.py`:
|
||||
|
||||
```python
|
||||
class CostTracker:
|
||||
"""Track and estimate Azure OpenAI API costs."""
|
||||
|
||||
# Pricing (as of 2026-02, may change)
|
||||
EMBEDDING_COST_PER_1K_TOKENS = 0.00013 # text-embedding-3-large
|
||||
LLM_INPUT_COST_PER_1K_TOKENS = 0.03 # gpt-4.1
|
||||
LLM_OUTPUT_COST_PER_1K_TOKENS = 0.06 # gpt-4.1
|
||||
|
||||
def log_embedding_request(self, text: str, cached: bool):
|
||||
"""Log embedding request for cost tracking."""
|
||||
|
||||
def log_llm_request(self, input_tokens: int, output_tokens: int):
|
||||
"""Log LLM request for cost tracking."""
|
||||
|
||||
def get_session_costs(self) -> dict:
|
||||
"""Return estimated costs for current session."""
|
||||
```
|
||||
|
||||
Add to `README.md`:
|
||||
|
||||
```markdown
|
||||
## Azure OpenAI Cost Estimation
|
||||
|
||||
Typical matching workflow costs (estimated):
|
||||
|
||||
| Operation | API Calls | Estimated Cost |
|
||||
|-----------|-----------|----------------|
|
||||
| Single capacity match (100 candidates) | ~200 embeddings (mostly cached), 0 LLM | $0.001 - $0.01 |
|
||||
| Task requirements extraction | 0 embeddings, 1 LLM call (~1000 tokens) | $0.03 - $0.06 |
|
||||
| Role inference | 0 embeddings, 1 LLM call (~500 tokens) | $0.015 - $0.03 |
|
||||
|
||||
With embedding caching enabled (default), repeated searches are significantly cheaper.
|
||||
|
||||
**Monthly cost estimate** (100 searches/day, 20 days):
|
||||
- Without cache: ~$60-120/month
|
||||
- With cache (90% hit rate): ~$10-20/month
|
||||
```
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
### Unit Tests (Mocked Azure)
|
||||
|
||||
All unit tests should mock Azure OpenAI API responses to avoid API costs and ensure deterministic behavior:
|
||||
|
||||
**Embedding Mocks**:
|
||||
- Use fixed-dimension vectors matching `text-embedding-3-large` (3072 dimensions)
|
||||
- Create realistic patterns:
|
||||
- Similar terms: `[0.9, 0.8, 0.1, ..., 0.0]` and `[0.85, 0.82, 0.15, ..., 0.0]` → high cosine similarity (~0.95)
|
||||
- Different terms: `[0.9, 0.1, 0.0, ..., 0.0]` and `[0.1, 0.9, 0.0, ..., 0.0]` → low cosine similarity (~0.1)
|
||||
- Use `unittest.mock.AsyncMock` or `pytest-mock` for `AsyncAzureOpenAI` client
|
||||
|
||||
**LLM Mocks**:
|
||||
- Mock `chat.completions.create()` to return structured JSON responses
|
||||
- Example: `{"roles": [{"role": "Backend Developer", "confidence": 0.9, "rationale": "..."}], ...}`
|
||||
|
||||
**Example Mock Pattern**:
|
||||
```python
|
||||
@pytest.mark.asyncio
|
||||
async def test_compute_competence_similarity():
|
||||
mock_client = AsyncMock()
|
||||
mock_client.get_embedding.side_effect = [
|
||||
[0.9, 0.8, 0.1] + [0.0] * 3069, # "Python"
|
||||
[0.85, 0.82, 0.15] + [0.0] * 3069, # "Python 3"
|
||||
]
|
||||
engine = SimilarityEngine(mock_client, strategy="per_skill")
|
||||
result = await engine.compute_competence_similarity(["Python"], ["Python 3"])
|
||||
assert result["Python"]["score"] > 0.9
|
||||
```
|
||||
|
||||
### Integration Tests (Real Azure)
|
||||
|
||||
- Mark all integration tests with `@pytest.mark.integration`
|
||||
- Configure `pytest.ini`:
|
||||
```ini
|
||||
[pytest]
|
||||
markers =
|
||||
integration: marks tests that call real Azure OpenAI API (deselect with '-m "not integration"')
|
||||
```
|
||||
- Run integration tests manually before deployment: `pytest -m integration`
|
||||
- Run fast tests in CI: `pytest -m "not integration"`
|
||||
- Integration tests should use small, cheap prompts to minimize costs
|
||||
|
||||
### Test Coverage Goals
|
||||
|
||||
- **Unit tests**: ≥90% coverage for all new code (cache, client, similarity, analyzer)
|
||||
- **Integration tests**: Cover happy path + common error scenarios (API timeout, invalid credentials)
|
||||
- **Manual tests**: Full end-to-end workflow in Cherry Studio (search → filter → confirm)
|
||||
|
||||
## Implementation Task List
|
||||
|
||||
### Phase 1: Configuration & Infrastructure (3-4 hours)
|
||||
|
||||
- [ ] 1.1 Rename `database.toml` → `config.toml` and template
|
||||
- [ ] 1.2 Update all file references in code
|
||||
- [ ] 1.3 Add `[azure_openai]` section to config model (`src/config.py`)
|
||||
- [ ] 1.4 Add `[embedding_cache]` section to config model
|
||||
- [ ] 1.5 Add `[matching.similarity]` section to config model
|
||||
- [ ] 1.6 Update `config.toml.example` with new sections
|
||||
- [ ] 1.7 Load `AZURE_OPENAI_EMBEDDING_API_KEY` from `.env`
|
||||
- [ ] 1.8 Load `AZURE_OPENAI_LLM_API_KEY` from `.env`
|
||||
- [ ] 1.9 Add `openai` package to `pyproject.toml` dependencies
|
||||
|
||||
### Phase 2: Embedding Cache (2-3 hours)
|
||||
|
||||
- [ ] 2.1 Create `src/teamlandkarte_mcp/cache/embedding_cache.py`
|
||||
- [ ] 2.2 Implement `EmbeddingCache.__init__` (create DB/schema if not exists)
|
||||
- [ ] 2.3 Implement `EmbeddingCache.get()` (with TTL check)
|
||||
- [ ] 2.4 Implement `EmbeddingCache.put()`
|
||||
- [ ] 2.5 Implement `EmbeddingCache.cleanup_expired()`
|
||||
- [ ] 2.6 Add SQLite schema migration support (if DB exists but schema old)
|
||||
- [ ] 2.7 Add unit tests for `EmbeddingCache`
|
||||
- [ ] 2.8 Add `.gitignore` entry for `embeddings_cache.db`
|
||||
|
||||
### Phase 3: Azure OpenAI Client (3-4 hours)
|
||||
|
||||
**Testing Strategy**:
|
||||
- Unit tests should mock Azure API responses with realistic embedding vectors (3072 dimensions for text-embedding-3-large)
|
||||
- Use fixed seed embeddings for deterministic test behavior (e.g., `[0.1, 0.2, ..., 0.0]` patterns)
|
||||
- Integration tests marked with `@pytest.mark.integration` use real API calls
|
||||
|
||||
- [ ] 3.1 Create `src/teamlandkarte_mcp/azure/__init__.py`
|
||||
- [ ] 3.2 Create `src/teamlandkarte_mcp/azure/openai_client.py`
|
||||
- [ ] 3.3 Implement `AzureOpenAIClient.__init__`
|
||||
- [ ] 3.4 Implement `AzureOpenAIClient.get_embedding()` with cache integration
|
||||
- [ ] 3.5 Implement `AzureOpenAIClient.get_embeddings_batch()` (individual calls)
|
||||
- [ ] 3.6 Implement `AzureOpenAIClient.chat_completion()` with retry logic
|
||||
- [ ] 3.7 Add `AzureAPIError` exception class
|
||||
- [ ] 3.8 Add unit tests for client (mocked API)
|
||||
- [ ] 3.9 Add integration test with real API (marked as `@pytest.mark.integration`)
|
||||
|
||||
### Phase 4: Similarity Engine (4-5 hours)
|
||||
|
||||
- [ ] 4.1 Create `src/teamlandkarte_mcp/matching/similarity.py`
|
||||
- [ ] 4.2 Implement `cosine_similarity()` function
|
||||
- [ ] 4.3 Implement `SimilarityEngine.__init__`
|
||||
- [ ] 4.4 Implement `SimilarityEngine._per_skill_similarity()`
|
||||
- [ ] 4.5 Implement `SimilarityEngine._aggregate_similarity()`
|
||||
- [ ] 4.6 Implement `SimilarityEngine.compute_competence_similarity()` (router)
|
||||
- [ ] 4.7 Implement `SimilarityEngine.compute_role_similarity()`
|
||||
- [ ] 4.8 Add unit tests for similarity computation (mocked embeddings)
|
||||
- [ ] 4.9 Add integration test comparing both strategies
|
||||
|
||||
### Phase 5: TaskAnalyzer Refactoring (3-4 hours)
|
||||
|
||||
- [ ] 5.1 Remove `TaskAnalyzer.__init__(self, mcp: FastMCP)` signature
|
||||
- [ ] 5.2 Add new `TaskAnalyzer.__init__(self, azure_client: AzureOpenAIClient)`
|
||||
- [ ] 5.3 Remove `_sample_json()` method completely
|
||||
- [ ] 5.4 Remove `_heuristic_competences_from_text()` method completely
|
||||
- [ ] 5.5 Refactor `extract_ranked_roles()` to use `azure_client.chat_completion()`
|
||||
- [ ] 5.6 Remove heuristic fallback from `extract_ranked_roles()`
|
||||
- [ ] 5.7 Refactor `_extract_requirements_from_description()` to use LLM
|
||||
- [ ] 5.8 Remove heuristic fallback from `_extract_requirements_from_description()`
|
||||
- [ ] 5.9 Refactor `apply_requirement_update()` to use LLM
|
||||
- [ ] 5.10 Remove heuristic fallback from `apply_requirement_update()`
|
||||
- [ ] 5.11 Remove `semantic_competence_similarity()` method (replaced by `SimilarityEngine`)
|
||||
- [ ] 5.12 Update all `TaskAnalyzer` unit tests
|
||||
|
||||
### Phase 6: Matcher & Scorer Integration (2-3 hours)
|
||||
|
||||
- [ ] 6.1 Update `Matcher.__init__` to accept `SimilarityEngine`
|
||||
- [ ] 6.2 Replace competence similarity calls with `SimilarityEngine.compute_competence_similarity()`
|
||||
- [ ] 6.3 Replace role similarity calls with `SimilarityEngine.compute_role_similarity()`
|
||||
- [ ] 6.4 Update scorer aggregation logic if needed
|
||||
- [ ] 6.5 Update integration tests for matching pipeline
|
||||
|
||||
### Phase 7: Server Initialization (2 hours)
|
||||
|
||||
- [ ] 7.1 Update `build_server()` in `mcp_server.py` to load Azure config
|
||||
- [ ] 7.2 Initialize `EmbeddingCache` instance
|
||||
- [ ] 7.3 Initialize `AzureOpenAIClient` instance
|
||||
- [ ] 7.4 Initialize `SimilarityEngine` instance
|
||||
- [ ] 7.5 Update `TaskAnalyzer` instantiation
|
||||
- [ ] 7.6 Update `Matcher` instantiation
|
||||
- [ ] 7.7 Add startup logging for Azure OpenAI connection
|
||||
- [ ] 7.8 Add graceful error handling if Azure credentials missing
|
||||
|
||||
### Phase 8: Cost Tracking & Monitoring (2 hours)
|
||||
|
||||
- [ ] 8.1 Create `src/teamlandkarte_mcp/azure/cost_tracker.py`
|
||||
- [ ] 8.2 Implement `CostTracker` class with logging methods
|
||||
- [ ] 8.3 Integrate cost tracking into `AzureOpenAIClient`
|
||||
- [ ] 8.4 Add periodic cost report logging (stderr)
|
||||
- [ ] 8.5 Add cost summary to tool outputs (optional, via config)
|
||||
|
||||
### Phase 9: Documentation (2-3 hours)
|
||||
|
||||
- [ ] 9.1 Update `README.md` with Azure OpenAI setup instructions
|
||||
- [ ] 9.2 Add Azure cost estimation section to `README.md`
|
||||
- [ ] 9.3 Update `docs/troubleshooting.md` with Azure API error guidance
|
||||
- [ ] 9.4 Document similarity strategies (`per_skill` vs `aggregate`)
|
||||
- [ ] 9.5 Update `config.toml.example` with comprehensive comments
|
||||
- [ ] 9.6 Update OpenSpec architecture docs
|
||||
- [ ] 9.7 Add migration guide from old `database.toml` to new `config.toml`
|
||||
|
||||
### Phase 10: Testing & Validation (3-4 hours)
|
||||
|
||||
- [ ] 10.1 Configure `pytest.ini` with integration marker: `markers = integration: marks tests that call real Azure OpenAI API (deselect with '-m "not integration"')`
|
||||
- [ ] 10.2 Run full test suite after refactoring (`pytest -m "not integration"` for fast CI)
|
||||
- [ ] 10.3 Add new integration tests for Azure API paths (marked with `@pytest.mark.integration`)
|
||||
- [ ] 10.4 Test embedding cache persistence across server restarts
|
||||
- [ ] 10.5 Test both similarity strategies with real data
|
||||
- [ ] 10.6 Validate cost tracking accuracy
|
||||
- [ ] 10.7 Test error handling when Azure API unavailable
|
||||
- [ ] 10.8 Performance benchmark: compare cache hit/miss scenarios
|
||||
- [ ] 10.9 Manual validation in Cherry Studio
|
||||
|
||||
### Phase 11: Cleanup (1 hour)
|
||||
|
||||
- [ ] 11.1 Remove all commented-out FastMCP sampling code
|
||||
- [ ] 11.2 Remove unused imports (`from mcp.server.fastmcp import FastMCP` from `TaskAnalyzer`)
|
||||
- [ ] 11.3 Update type hints and docstrings
|
||||
- [ ] 11.4 Run linter and fix style issues
|
||||
- [ ] 11.5 Final code review
|
||||
|
||||
## Impact Analysis
|
||||
|
||||
### Changed Files
|
||||
|
||||
**New files**:
|
||||
- `config.toml` (renamed from `database.toml`)
|
||||
- `config.toml.example` (renamed from `database.toml.example`)
|
||||
- `src/teamlandkarte_mcp/cache/embedding_cache.py`
|
||||
- `src/teamlandkarte_mcp/azure/__init__.py`
|
||||
- `src/teamlandkarte_mcp/azure/openai_client.py`
|
||||
- `src/teamlandkarte_mcp/azure/cost_tracker.py`
|
||||
- `src/teamlandkarte_mcp/matching/similarity.py`
|
||||
- `embeddings_cache.db` (generated at runtime, gitignored)
|
||||
|
||||
**Modified files**:
|
||||
- `src/teamlandkarte_mcp/config.py` (new config sections)
|
||||
- `src/teamlandkarte_mcp/matching/task_analyzer.py` (complete refactor)
|
||||
- `src/teamlandkarte_mcp/matching/matcher.py` (similarity engine integration)
|
||||
- `src/teamlandkarte_mcp/mcp_server.py` (initialization changes)
|
||||
- `README.md` (setup instructions, cost docs)
|
||||
- `docs/troubleshooting.md` (Azure error guidance)
|
||||
- `pyproject.toml` (add `openai` dependency)
|
||||
- `.gitignore` (add `embeddings_cache.db`, `config.toml`)
|
||||
- All OpenSpec architecture/design docs
|
||||
|
||||
**Deleted code**:
|
||||
- All FastMCP sampling code in `TaskAnalyzer`
|
||||
- All heuristic fallback code in `TaskAnalyzer`
|
||||
- `_sample_json()`, `_heuristic_competences_from_text()` methods
|
||||
|
||||
### Breaking Changes
|
||||
|
||||
1. **Configuration file renamed**: Users must rename `database.toml` → `config.toml`
|
||||
2. **New required environment variables**: `AZURE_OPENAI_EMBEDDING_API_KEY`, `AZURE_OPENAI_LLM_API_KEY`
|
||||
3. **Hard dependency on Azure OpenAI**: No offline/fallback mode
|
||||
4. **New dependency**: `openai` Python package
|
||||
5. **TaskAnalyzer constructor signature changed**: `TaskAnalyzer.__init__(self, mcp: FastMCP)` → `TaskAnalyzer.__init__(self, azure_client: AzureOpenAIClient)`
|
||||
- This affects any code that instantiates `TaskAnalyzer` directly
|
||||
- Server initialization in `mcp_server.py` must be updated
|
||||
6. **Matcher constructor signature changed**: `Matcher.__init__(self, analyzer: TaskAnalyzer, config: MatchingConfig)` → `Matcher.__init__(self, analyzer: TaskAnalyzer, similarity_engine: SimilarityEngine, config: MatchingConfig)`
|
||||
- Adds new required `similarity_engine` parameter
|
||||
- Server initialization must pass `SimilarityEngine` instance
|
||||
|
||||
### Migration Path
|
||||
|
||||
1. Rename `database.toml` → `config.toml`
|
||||
2. Add new Azure OpenAI sections to `config.toml`
|
||||
3. Add Azure API keys to `.env`
|
||||
4. Install updated dependencies: `uv sync`
|
||||
5. First run will create `embeddings_cache.db` automatically
|
||||
|
||||
## Security & Constraints
|
||||
|
||||
### Security
|
||||
|
||||
- **API keys in `.env`**: Never commit `.env` or `config.toml` with credentials
|
||||
- **Embedding cache**: Contains only embeddings, not raw capacity data (safe to persist)
|
||||
- **Network**: All API calls over HTTPS to Azure OpenAI endpoint
|
||||
- **Error messages**: Do not log API keys in error messages
|
||||
|
||||
### Operational Constraints
|
||||
|
||||
- **Azure OpenAI dependency**: System will fail if Azure API unavailable (no fallback)
|
||||
- **API rate limits**: Azure OpenAI enforces rate limits (TPM/RPM); implement retry with backoff
|
||||
- **Cost**: Real monetary cost per API call (mitigated by caching)
|
||||
- **Latency**: First search for a capacity will be slower (embedding generation); subsequent searches fast (cached)
|
||||
|
||||
### Configuration Defaults
|
||||
|
||||
```toml
|
||||
[embedding_cache]
|
||||
enabled = true
|
||||
db_path = "embeddings_cache.db"
|
||||
ttl_days = 30
|
||||
|
||||
[matching.similarity]
|
||||
strategy = "per_skill" # Maintains current behavior
|
||||
```
|
||||
|
||||
## Cost Estimation Details
|
||||
|
||||
### API Pricing (as of 2026-02)
|
||||
|
||||
| Service | Model | Cost |
|
||||
|---------|-------|------|
|
||||
| Embeddings | text-embedding-3-large | $0.00013 per 1K tokens |
|
||||
| LLM (input) | gpt-4.1 | $0.03 per 1K tokens |
|
||||
| LLM (output) | gpt-4.1 | $0.06 per 1K tokens |
|
||||
|
||||
### Typical Workflow Costs
|
||||
|
||||
**Scenario 1: Task requirements extraction**
|
||||
- Input: ~500 tokens (task description)
|
||||
- Output: ~300 tokens (JSON with roles/competences/dates)
|
||||
- Cost: (500 × $0.03 / 1000) + (300 × $0.06 / 1000) = **$0.033**
|
||||
|
||||
**Scenario 2: Capacity matching (100 candidates, 5 required skills)**
|
||||
- First run (cold cache):
|
||||
- Required skills: 5 × ~10 tokens = 50 tokens → ~$0.0000065
|
||||
- Candidate skills: 100 candidates × 3 skills avg × 10 tokens = 3000 tokens → ~$0.00039
|
||||
- Total: **~$0.0004** (negligible)
|
||||
- Subsequent runs (warm cache): **$0** (all cached)
|
||||
|
||||
**Scenario 3: Role inference**
|
||||
- Input: ~400 tokens
|
||||
- Output: ~200 tokens
|
||||
- Cost: **~$0.024**
|
||||
|
||||
### Monthly Estimates
|
||||
|
||||
**Light usage** (20 searches/month, 10 extractions):
|
||||
- Embeddings: ~$0.05
|
||||
- LLM: ~$0.50
|
||||
- **Total: ~$0.55/month**
|
||||
|
||||
**Moderate usage** (100 searches/month, 50 extractions):
|
||||
- Embeddings: ~$0.20 (with 90% cache hit rate)
|
||||
- LLM: ~$2.50
|
||||
- **Total: ~$2.70/month**
|
||||
|
||||
**Heavy usage** (500 searches/month, 200 extractions):
|
||||
- Embeddings: ~$1.00 (with 90% cache hit rate)
|
||||
- LLM: ~$10.00
|
||||
- **Total: ~$11/month**
|
||||
|
||||
## Open Questions
|
||||
|
||||
None (all clarifications provided by user).
|
||||
|
||||
## Approval & Timeline
|
||||
|
||||
- **Estimated effort**: 28-35 hours (1 week full-time or 2 weeks part-time)
|
||||
- **Risk level**: Medium (API dependency, cost implications, major refactor)
|
||||
- **Approval required**: Yes (architecture change, new external dependency)
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. Review and approve this proposal
|
||||
2. Create detailed implementation branch
|
||||
3. Implement phases 1-11 sequentially
|
||||
4. Conduct thorough testing (unit + integration + manual)
|
||||
5. Document migration guide
|
||||
6. Deploy to staging environment
|
||||
7. Monitor costs and performance
|
||||
8. Deploy to production
|
||||
-359
@@ -1,359 +0,0 @@
|
||||
# Tasks: Replace Heuristics with Azure OpenAI
|
||||
|
||||
> Change: `replace-heuristics-with-azure-openai`
|
||||
>
|
||||
> Status: **Approved / Implemented**
|
||||
|
||||
## Overview
|
||||
|
||||
Replace legacy client-side sampling/heuristic logic with direct Azure OpenAI integration:
|
||||
- Embeddings (`text-embedding-3-large`) for competence/role similarity
|
||||
- LLM (`gpt-4.1`) for requirements extraction and validation
|
||||
- Persistent SQLite embedding cache
|
||||
- Two configurable similarity strategies
|
||||
|
||||
# Removed: migration note about database.toml rename (now a completed internal detail).
|
||||
|
||||
## Estimated Effort
|
||||
|
||||
**Total: 28-35 hours** (1 week full-time or 2 weeks part-time)
|
||||
|
||||
## Task Breakdown
|
||||
|
||||
### Phase 1: Configuration & Infrastructure (3-4 hours)
|
||||
|
||||
- [x] 1.1 Rename `database.toml` → `config.toml`
|
||||
- [x] 1.2 Rename `database.toml.example` → `config.toml.example`
|
||||
- [x] 1.3 Update all file references in code (`README.md`, `docs/`, `src/config.py`, etc.)
|
||||
- [x] 1.4 Extend `src/config.py`: Add `AzureOpenAIConfig` model
|
||||
- `endpoint`, `api_version_embeddings`, `api_version_llm`
|
||||
- `embeddings.model`, `embeddings.deployment` (deployment must equal model name)
|
||||
- `llm.model`, `llm.deployment` (deployment must equal model name), `llm.temperature`, `llm.max_tokens`
|
||||
- [x] 1.5 Extend `src/config.py`: Add `EmbeddingCacheConfig` model
|
||||
- `enabled`, `db_path`, `ttl_days`
|
||||
- [x] 1.6 Extend `src/config.py`: Add `matching.similarity.strategy` field (`"per_skill"` or `"aggregate"`)
|
||||
- [x] 1.7 Update `config.toml.example` with new `[azure_openai]`, `[embedding_cache]`, `[matching.similarity]` sections
|
||||
- [x] 1.8 Load `AZURE_OPENAI_EMBEDDING_API_KEY` from environment in `load_config()`
|
||||
- [x] 1.9 Load `AZURE_OPENAI_LLM_API_KEY` from environment in `load_config()`
|
||||
- [x] 1.10 Add `openai` package to `pyproject.toml` dependencies
|
||||
- [x] 1.11 Run `uv sync` to install dependencies
|
||||
- [x] 1.12 Add `config.toml` to `.gitignore` (if not already there)
|
||||
- [x] 1.13 Configure pytest integration marker in `pytest.ini` (if not already present):
|
||||
- `integration: marks tests that call real Azure OpenAI API (deselect with '-m "not integration"')`
|
||||
|
||||
### Phase 2: Embedding Cache (2-3 hours)
|
||||
|
||||
- [x] 2.1 Create `src/teamlandkarte_mcp/cache/embedding_cache.py`
|
||||
- [x] 2.2 Implement `EmbeddingCache` class:
|
||||
- `__init__(db_path: str, ttl_days: int)`
|
||||
- Create SQLite database and schema if not exists:
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS embeddings (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
text TEXT UNIQUE NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
embedding BLOB NOT NULL,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
CREATE INDEX IF NOT EXISTS idx_text_model ON embeddings(text, model)
|
||||
```
|
||||
- [x] 2.3 Implement `EmbeddingCache.get(text: str, model: str) -> Optional[list[float]]`
|
||||
- Query by text + model
|
||||
- Check if `created_at` is within TTL
|
||||
- Deserialize BLOB to `list[float]`
|
||||
- Return `None` if missing or expired
|
||||
- [x] 2.4 Implement `EmbeddingCache.put(text: str, model: str, embedding: list[float]) -> None`
|
||||
- Serialize embedding as UTF-8 bytes of JSON: `json.dumps(embedding).encode("utf-8")`
|
||||
- INSERT OR REPLACE into database
|
||||
- [x] 2.5 Implement `EmbeddingCache.cleanup_expired() -> int`
|
||||
- DELETE embeddings older than TTL
|
||||
- Return count of deleted rows
|
||||
- [x] 2.6 Add graceful handling for concurrent access (SQLite locks)
|
||||
- [x] 2.7 Create `tests/test_embedding_cache.py`
|
||||
- Test cache hit/miss
|
||||
- Test TTL expiration
|
||||
- Test cleanup
|
||||
- Test concurrent access
|
||||
- [x] 2.8 Add `embeddings_cache.db` to `.gitignore`
|
||||
|
||||
### Phase 3: Azure OpenAI Client (3-4 hours)
|
||||
|
||||
- [x] 3.1 Create `src/teamlandkarte_mcp/azure/__init__.py`
|
||||
- [x] 3.2 Create `src/teamlandkarte_mcp/azure/openai_client.py`
|
||||
- [x] 3.3 Define `AzureAPIError(RuntimeError)` exception class
|
||||
- [x] 3.4 Implement `AzureOpenAIClient` class:
|
||||
- `__init__(config: AzureOpenAIConfig, embedding_cache: Optional[EmbeddingCache])`
|
||||
- Initialize two `AzureOpenAI` clients (embeddings + LLM) using `AzureKeyCredential`
|
||||
- [x] 3.5 Implement `AzureOpenAIClient.get_embedding(text: str) -> list[float]`
|
||||
- Check cache first (if enabled)
|
||||
- If cache miss: call Azure API
|
||||
- Store in cache (if enabled)
|
||||
- Raise `AzureAPIError` on failure (no fallback)
|
||||
- [x] 3.6 Implement `AzureOpenAIClient.get_embeddings_batch(texts: list[str]) -> list[list[float]]`
|
||||
- Call `get_embedding()` for each text individually (per user requirement #3)
|
||||
- Return list of embeddings in same order
|
||||
- [x] 3.7 Implement `AzureOpenAIClient.chat_completion(system: str, user: str, response_format: Optional[dict] = None) -> str`
|
||||
- Call Azure OpenAI chat completions API
|
||||
- Use `gpt-4.1` model with configured temperature/max_tokens
|
||||
- Support `response_format={"type": "json_object"}` for structured JSON
|
||||
- Implement retry logic (3 attempts, exponential backoff)
|
||||
- Raise `AzureAPIError` on final failure (no fallback)
|
||||
- [x] 3.8 Add logging for API calls (cache hit/miss, request/response sizes)
|
||||
- [x] 3.9 Create `tests/test_azure_client.py`
|
||||
- Mock Azure API with `unittest.mock`
|
||||
- Test embedding retrieval with cache hit/miss
|
||||
- Test LLM completion
|
||||
- Test error handling and retries
|
||||
- [x] 3.10 Create `tests/test_azure_client_integration.py` (marked `@pytest.mark.integration`)
|
||||
- Real API calls (requires valid credentials)
|
||||
- Test embedding generation
|
||||
- Test LLM structured JSON response
|
||||
|
||||
### Phase 4: Similarity Engine (4-5 hours)
|
||||
|
||||
- [x] 4.1 Create `src/teamlandkarte_mcp/matching/similarity.py`
|
||||
- [x] 4.2 Implement `cosine_similarity(vec_a: list[float], vec_b: list[float]) -> float`
|
||||
- Compute dot product and magnitudes
|
||||
- Return cosine similarity in [0.0, 1.0]
|
||||
- Handle zero vectors gracefully
|
||||
- [x] 4.3 Implement `SimilarityEngine` class:
|
||||
- `__init__(client: AzureOpenAIClient, strategy: str = "per_skill")`
|
||||
- Validate strategy is `"per_skill"` or `"aggregate"`
|
||||
- [x] 4.4 Implement `SimilarityEngine._per_skill_similarity(required: list[str], candidate: list[str]) -> dict[str, dict[str, Any]]`
|
||||
- For each required skill:
|
||||
- Get embedding
|
||||
- Compare with all candidate skill embeddings
|
||||
- Find best match (highest cosine similarity)
|
||||
- Return `{required: {best_match: str|None, score: float, rationale: str}}`
|
||||
- Rationale example: `"Cosine similarity: 0.92 (best match: 'React.js')"`
|
||||
- [x] 4.5 Implement `SimilarityEngine._aggregate_similarity(required: list[str], candidate: list[str]) -> dict[str, dict[str, Any]]`
|
||||
- Compute average embedding of all required skills
|
||||
- Compute average embedding of all candidate skills
|
||||
- Compute cosine similarity between averages
|
||||
- Return same format as `_per_skill_similarity` (each required skill gets same aggregate score)
|
||||
- Rationale example: `"Aggregate embedding similarity: 0.85"`
|
||||
- [x] 4.6 Implement `SimilarityEngine.compute_competence_similarity(required: list[str], candidate: list[str]) -> dict[str, dict[str, Any]]`
|
||||
- Route to `_per_skill_similarity()` or `_aggregate_similarity()` based on configured strategy
|
||||
- Maintain same output format as current `TaskAnalyzer.semantic_competence_similarity()`
|
||||
- Guarantee `rationale` is always present (even when best_match is None)
|
||||
- Edge cases:
|
||||
- `required == []` → return `{}`
|
||||
- `candidate == []` → return per-required entry with `best_match=None`, `score=0.0`, and rationale
|
||||
- [x] 4.7 Implement `SimilarityEngine.compute_role_similarity(required_role: str, candidate_role: str) -> float`
|
||||
- Edge cases:
|
||||
- If either role is empty/None → return 0.0
|
||||
- If either role is "(unknown)" (case-insensitive, trimmed) → return 0.0
|
||||
- Otherwise compute cosine similarity of role embeddings
|
||||
- [x] 4.8 Create `tests/test_similarity.py`
|
||||
- Test `cosine_similarity()` with known vectors
|
||||
- Mock embeddings and test both strategies
|
||||
- Test edge cases (empty lists, identical skills, no matches)
|
||||
- [x] 4.9 Create `tests/test_similarity_integration.py` (marked `@pytest.mark.integration`)
|
||||
- Compare both strategies with real embeddings
|
||||
- Validate scores are reasonable (e.g., "Python" vs "Python 3" → high similarity)
|
||||
|
||||
### Phase 5: TaskAnalyzer Refactoring (3-4 hours)
|
||||
|
||||
- [x] 5.1 Update `src/teamlandkarte_mcp/matching/task_analyzer.py`
|
||||
- [x] 5.2 Change `TaskAnalyzer.__init__` signature:
|
||||
- Remove: `__init__(self, mcp: FastMCP)`
|
||||
- Add: `__init__(self, azure_client: AzureOpenAIClient)`
|
||||
- [x] 5.3 **Delete** `_sample_json()` method completely
|
||||
- [x] 5.4 **Delete** `_heuristic_competences_from_text()` static method completely
|
||||
- [x] 5.5 Refactor `extract_ranked_roles(description: str, limit: int = 5) -> list[RankedRole]`:
|
||||
- Remove all heuristic fallback code
|
||||
- Build prompt for structured JSON response:
|
||||
```
|
||||
system: "Extract a ranked list of possible roles from a task description. Return STRICT JSON: {roles:[{rank:int, role:str, rationale:str}]}"
|
||||
user: "Limit: {limit}\n\nTask description:\n{description}"
|
||||
```
|
||||
- Call `self._client.chat_completion(system, user, response_format={"type": "json_object"})`
|
||||
- Parse JSON response and build `list[RankedRole]`
|
||||
- Raise `AzureAPIError` on failure (no fallback)
|
||||
- [x] 5.6 Refactor `_extract_requirements_from_description(description: str) -> ExtractedRequirements`:
|
||||
- Remove all heuristic fallback code
|
||||
- Build prompt for structured JSON:
|
||||
```
|
||||
system: "Extract structured requirements from a task description. Return STRICT JSON with keys: competences (list[str]), date_start (YYYY-MM-DD|null), date_end (YYYY-MM-DD|null), roles ([{rank, role, rationale}])."
|
||||
user: "Task description:\n{description}"
|
||||
```
|
||||
- Call `self._client.chat_completion(system, user, response_format={"type": "json_object"})`
|
||||
- Parse JSON and build `ExtractedRequirements`
|
||||
- Raise `AzureAPIError` on failure (no fallback)
|
||||
- [x] 5.7 Refactor `apply_requirement_update(current: Requirements, change_description: str) -> Requirements`:
|
||||
- Remove all heuristic fallback code
|
||||
- Build prompt with current requirements + change request
|
||||
- Call `self._client.chat_completion(system, user, response_format={"type": "json_object"})`
|
||||
- Parse JSON and build updated `Requirements`
|
||||
- Raise `AzureAPIError` on failure (no fallback)
|
||||
- [x] 5.8 **Delete** `semantic_competence_similarity()` method completely (replaced by `SimilarityEngine`)
|
||||
- [x] 5.9 Update `ExtractedRequirements` dataclass docstring (no longer mentions heuristics)
|
||||
- [x] 5.10 Update `TaskAnalyzer` class docstring (remove FastMCP sampling references, mention Azure OpenAI)
|
||||
- [x] 5.11 Remove unused imports (`FastMCP`, `random`, `asyncio` if no longer needed)
|
||||
- [x] 5.12 Update all `tests/test_task_analyzer.py` (or similar):
|
||||
- Mock `AzureOpenAIClient` instead of `FastMCP`
|
||||
- Remove heuristic fallback tests
|
||||
- Add tests for Azure API error propagation
|
||||
- [x] 5.13 Update `AnalysisError` semantics (or rename/remove if no longer needed):
|
||||
- Previously: heuristic fallback masked missing sampling
|
||||
- Now: Azure failures must surface as exceptions (no heuristics)
|
||||
|
||||
### Phase 6: Matcher & Scorer Integration (2-3 hours)
|
||||
|
||||
- [x] 6.1 Identify where `Matcher` or scoring logic calls `TaskAnalyzer.semantic_competence_similarity()`
|
||||
- [x] 6.2 Update `Matcher.__init__` to accept `SimilarityEngine` as parameter
|
||||
- [x] 6.3 Replace `analyzer.semantic_competence_similarity(required, candidate)` with `similarity_engine.compute_competence_similarity(required, candidate)`
|
||||
- [x] 6.4 Update role matching to use `similarity_engine.compute_role_similarity(required_role, candidate_role)`
|
||||
- [x] 6.5 Ensure scorer aggregation logic works with new similarity output format (should be identical)
|
||||
- [x] 6.6 Update `tests/test_scorer.py` (and/or add `tests/test_matcher.py` if introduced):
|
||||
- Mock `SimilarityEngine` instead of `TaskAnalyzer.semantic_competence_similarity`
|
||||
- Test that both similarity strategies produce valid scores
|
||||
- [x] 6.7 Update matcher/scorer tests to reflect new edge-case contracts:
|
||||
- role similarity returns 0.0 for empty/"(unknown)" roles
|
||||
- competence similarity returns 0.0 when candidate list is empty
|
||||
|
||||
### Phase 7: Server Initialization (2 hours)
|
||||
|
||||
- [x] 7.1 Update `build_server()` in `src/teamlandkarte_mcp/mcp_server.py`
|
||||
- [x] 7.2 Update server build to use top-level embedding cache config (not nested under azure):
|
||||
- `cfg.embedding_cache.enabled/db_path/ttl_days`
|
||||
- [x] 7.3 Initialize `EmbeddingCache` (if enabled) using `cfg.embedding_cache`
|
||||
- [x] 7.4 Initialize `AzureOpenAIClient` using `cfg.azure_openai`
|
||||
- [x] 7.5 Initialize `SimilarityEngine` using `cfg.matching.similarity.strategy`
|
||||
- [x] 7.6 Update `TaskAnalyzer` instantiation
|
||||
- [x] 7.7 Update `Matcher` instantiation
|
||||
- [x] 7.8 Add startup logging (stderr):
|
||||
- Log Azure OpenAI endpoint
|
||||
- Log embedding cache status (enabled/disabled, path)
|
||||
- Log similarity strategy
|
||||
- [x] 7.9 Add graceful error handling if Azure credentials missing:
|
||||
- Log error to stderr
|
||||
- Exit with clear message: "Azure OpenAI credentials not found in environment"
|
||||
|
||||
### Phase 8: Cost Tracking & Monitoring (2 hours)
|
||||
|
||||
- [x] 8.1 Create `src/teamlandkarte_mcp/azure/cost_tracker.py`
|
||||
- [x] 8.2 Implement `CostTracker` class:
|
||||
- Class-level constants for pricing
|
||||
- `log_embedding_request(text: str, cached: bool)`
|
||||
- `log_llm_request(input_tokens: int, output_tokens: int)`
|
||||
- `get_session_costs() -> dict` (return totals)
|
||||
- [x] 8.3 Integrate `CostTracker` into `AzureOpenAIClient`:
|
||||
- Track embedding requests (cache hit/miss)
|
||||
- Track LLM requests (token counts from API response)
|
||||
- [x] 8.4 Add periodic cost logging to stderr:
|
||||
- Every 10 API calls or every 5 minutes
|
||||
- Log: `"Azure OpenAI costs this session: embeddings=$X.XX, llm=$Y.YY, total=$Z.ZZ"`
|
||||
|
||||
### Phase 9: Documentation (2-3 hours)
|
||||
|
||||
- [x] 9.1 Update `README.md`:
|
||||
- Add "Azure OpenAI Setup" section with prerequisites
|
||||
- Document environment variables (`AZURE_OPENAI_EMBEDDING_API_KEY`, `AZURE_OPENAI_LLM_API_KEY`)
|
||||
- Document `config.toml` Azure sections
|
||||
- Add cost estimation table
|
||||
- Add embedding cache explanation
|
||||
- Add similarity strategy comparison
|
||||
- [x] 9.2 Add migration guide to `README.md`:
|
||||
- Step 1: Rename `database.toml` → `config.toml`
|
||||
- Step 2: Add Azure sections to config
|
||||
- Step 3: Add Azure API keys to `.env`
|
||||
- Step 4: Run `uv sync`
|
||||
- Step 5: Start server (embedding cache will be created automatically)
|
||||
- [x] 9.3 Update `docs/troubleshooting.md`:
|
||||
- Add section: "Azure OpenAI API Errors"
|
||||
- Document common errors (rate limits, invalid credentials, network issues)
|
||||
- Document `AzureAPIError` and how to debug
|
||||
- [x] 9.4 Update `config.toml.example`:
|
||||
- Add comprehensive comments for all Azure sections
|
||||
- Document both similarity strategies with examples
|
||||
- Document embedding cache TTL trade-offs
|
||||
- [x] 9.5 Update OpenSpec architecture docs:
|
||||
- `openspec/changes/add-capacity-matching-mcp-server/architecture.md`
|
||||
- `openspec/changes/add-capacity-matching-mcp-server/design.md`
|
||||
- Update diagrams to show Azure OpenAI dependency
|
||||
- Update component descriptions
|
||||
- [x] 9.6 Create `docs/azure_openai_setup.md` (detailed setup guide):
|
||||
- Prerequisites (Azure subscription, OpenAI resource)
|
||||
- API key generation steps
|
||||
- Cost monitoring in Azure portal
|
||||
- Troubleshooting deployment/model availability
|
||||
|
||||
### Phase 10: Testing & Validation (3-4 hours)
|
||||
|
||||
- [x] 10.1 Run fast unit tests in CI mode: `pytest -m "not integration" -q`
|
||||
- [x] 10.2 Run integration tests with real Azure API: `pytest -m integration`
|
||||
- [x] 10.3 Test embedding cache behavior:
|
||||
- Start server, run search → check cache DB created
|
||||
- Restart server, run same search → verify cache hit
|
||||
- Wait past TTL, run cleanup → verify expired entries removed
|
||||
- [x] 10.4 Test both similarity strategies:
|
||||
- Set `matching.similarity.strategy = "per_skill"` → run search
|
||||
- Set `matching.similarity.strategy = "aggregate"` → run search
|
||||
- Compare results and validate scoring differences
|
||||
- [x] 10.5 Test cost tracking:
|
||||
- Run several searches
|
||||
- Check stderr for cost logs
|
||||
- Validate cost estimates are reasonable
|
||||
- [x] 10.6 Test error handling:
|
||||
- Comment out Azure API keys → verify graceful startup error
|
||||
- Mock API failure → verify `AzureAPIError` propagates correctly
|
||||
- Mock rate limit error → verify retry logic works
|
||||
- [x] 10.7 Performance benchmark:
|
||||
- Cold cache: measure time for first search (100 candidates)
|
||||
- Warm cache: measure time for repeated search
|
||||
- Document speedup ratio
|
||||
- [x] 10.8 Manual validation in Cherry Studio:
|
||||
- Browse tasks
|
||||
- Extract requirements from complex task description
|
||||
- Run matching workflow
|
||||
- Validate similarity scores look reasonable
|
||||
- Test guided capture with Azure LLM
|
||||
- Test update_requirements
|
||||
|
||||
### Phase 11: Cleanup & Final Review (2-3 hours)
|
||||
|
||||
- [x] 11.1 Final sweep: ensure there are no remaining FastMCP sampling references
|
||||
- [x] 11.2 Final sweep: ensure no heuristic fallback paths remain
|
||||
- [x] 11.3 Tighten type hints, docstrings, and error semantics
|
||||
- [x] 11.4 Run `ruff check` (via `uv`) and fix all issues
|
||||
- [x] 11.5 Run `mypy` (via `uv`) and fix all issues
|
||||
- [x] 11.6 Final code review: validate all modules, remove stray prints/debug logs
|
||||
- [x] 11.7 Git commit review: ensure no secrets (config/.env/cache DB) are tracked
|
||||
- [x] 11.7a Cleanup: remove obsolete `scripts/trino_smoke_check.py`
|
||||
- [x] 11.7b Dependency hygiene: keep `trino` as a required runtime dependency
|
||||
- [ ] 11.8 Merge to main and tag release
|
||||
|
||||
## Risk Mitigation
|
||||
|
||||
**Risk**: Azure API downtime → System unusable
|
||||
**Mitigation**: Document SLA expectations; consider adding a simple health check tool/endpoint and a clear user-facing error message (`AzureAPIError`).
|
||||
|
||||
**Risk**: Unexpected high costs
|
||||
**Mitigation**: Enable embedding cache (default); monitor costs weekly; set Azure budget alerts.
|
||||
|
||||
**Risk**: Embedding cache grows too large
|
||||
**Mitigation**: Run periodic cleanup; set reasonable TTL (default 30 days); ensure cache DB files stay ignored.
|
||||
|
||||
**Risk**: Secrets accidentally committed (`config.toml`, `.env`, caches)
|
||||
**Mitigation**: Keep `config.toml` and `.env` ignored; review `git status` before commit; run a secret scan locally (e.g. `git grep` for key patterns) before merging.
|
||||
|
||||
**Risk**: Migration breaks existing deployments
|
||||
**Mitigation**: Provide clear migration guide; test migration path manually; run integration tests against Azure in at least one real environment.
|
||||
|
||||
## Success Criteria
|
||||
|
||||
- [x] Unit tests pass (non-integration)
|
||||
- [ ] Integration tests pass (`pytest -m integration`) in an environment with valid Azure credentials
|
||||
- [ ] Server starts successfully with Azure credentials
|
||||
- [ ] Matching results show improved semantic understanding vs. heuristics
|
||||
- [ ] Embedding cache demonstrates significant performance improvement on repeated searches
|
||||
- [ ] Cost tracking shows accurate estimates
|
||||
- [ ] Documentation complete and clear
|
||||
- [ ] Repo clean: no secrets or local config files tracked (verify before merge)
|
||||
|
||||
---
|
||||
|
||||
**Total estimated effort**: 28-35 hours
|
||||
**Priority**: Medium-High (significant quality improvement, but breaking change)
|
||||
**Approval status**: ✅ Approved
|
||||
@@ -1,55 +0,0 @@
|
||||
# Project Context
|
||||
|
||||
## Purpose
|
||||
This project is an MCP (Model Context Protocol) server for "Teamlandkarte" (Team Map). The server provides AI assistants with tools to browse tasks and match DB Systel employees with free capacity to task requirements via a confirmation-gated, table-first workflow.
|
||||
|
||||
## Tech Stack
|
||||
- **Language**: Python 3.13
|
||||
- **Package Manager**: uv (based on pyproject.toml)
|
||||
- **Configuration**: TOML format (pyproject.toml, config.toml)
|
||||
- **Architecture**: MCP Server (Model Context Protocol) exposing table-first tools and search sessions (`search_id` / `filter_id`)
|
||||
- **Version Control**: Git (hosted on GitLab - git.tech.rz.db.de)
|
||||
|
||||
## Project Conventions
|
||||
|
||||
### Code Style
|
||||
- Python 3.13+ syntax and features
|
||||
- Follow PEP 8 style guidelines
|
||||
- Use type hints where appropriate
|
||||
- Entry point: `main.py` with standard `if __name__ == "__main__"` pattern
|
||||
- Configuration files use TOML format
|
||||
|
||||
### Architecture Patterns
|
||||
- MCP Server architecture for AI assistant integration
|
||||
- OpenSpec workflow for managing change proposals and specifications
|
||||
- Modular structure with dedicated `openspec/` directory for specs and planning
|
||||
- Entry point in `main.py` for simple execution
|
||||
|
||||
### Testing Strategy
|
||||
- Test suite uses **pytest** (see `tests/`)
|
||||
- Focus areas:
|
||||
- confirmation gating
|
||||
- search id robustness and error handling
|
||||
- filtering and pagination workflows
|
||||
- read-only DB guard
|
||||
|
||||
### Git Workflow
|
||||
- Main branch: `main`
|
||||
- Hosted on internal GitLab instance (git.tech.rz.db.de)
|
||||
- Follow GitLab merge request workflow
|
||||
- Use OpenSpec for planning significant changes before implementation
|
||||
|
||||
## Domain Context
|
||||
- **Teamlandkarte** (Team Map): A team mapping or visualization tool
|
||||
- **MCP (Model Context Protocol)**: A standardized protocol for AI assistants to interact with external tools and data sources
|
||||
- The server acts as a bridge between AI assistants and team mapping functionality
|
||||
|
||||
## Important Constraints
|
||||
- Python version must be >= 3.13
|
||||
- Internal project hosted on Deutsche Bahn GitLab infrastructure
|
||||
- Must comply with MCP protocol specifications
|
||||
|
||||
## External Dependencies
|
||||
- MCP SDK/libraries (to be added as project develops)
|
||||
- Database connectivity (config.toml suggests DB integration)
|
||||
- Other dependencies to be defined in pyproject.toml as needed
|
||||
@@ -1,57 +0,0 @@
|
||||
[project]
|
||||
name = "teamlandkarte-mcp"
|
||||
version = "0.1.0"
|
||||
description = "MCP server for matching DB Systel employees with free work capacity to task requirements"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.13"
|
||||
dependencies = [
|
||||
"mcp",
|
||||
# Trino / PrestoSQL connectivity (DBeaver driver: PrestoSQL)
|
||||
"trino",
|
||||
# Caching
|
||||
"cachetools",
|
||||
# Fuzzy filtering
|
||||
"fuzzywuzzy",
|
||||
"python-Levenshtein",
|
||||
# BM25 lexical ranking (no native extensions, MIT licensed)
|
||||
"rank-bm25",
|
||||
# Testing
|
||||
"pytest",
|
||||
"pytest-asyncio",
|
||||
"hypothesis",
|
||||
"python-dotenv>=1.2.1",
|
||||
"openai>=2.20.0",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
teamlandkarte-mcp = "teamlandkarte_mcp.__main__:main"
|
||||
|
||||
[build-system]
|
||||
requires = ["setuptools>=68", "wheel"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[tool.setuptools]
|
||||
package-dir = {"" = "src"}
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["src"]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"mypy>=1.19.1",
|
||||
"ruff>=0.15.1",
|
||||
"types-cachetools>=6.2.0.20251022",
|
||||
]
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 110
|
||||
|
||||
[tool.ruff.lint]
|
||||
# Extend line length to 110 for all linting rules
|
||||
select = ["E", "F", "W"]
|
||||
|
||||
[tool.ruff.lint.pycodestyle]
|
||||
max-line-length = 110
|
||||
|
||||
[tool.ruff.format]
|
||||
line-ending = "lf"
|
||||
@@ -1,7 +0,0 @@
|
||||
[pytest]
|
||||
addopts = -q
|
||||
markers =
|
||||
integration: marks tests that require external services (Azure OpenAI) and are not run by default
|
||||
allow_network: opt-out marker for the global network-blocking fixture in tests/conftest.py
|
||||
testpaths = tests
|
||||
pythonpath = src
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user