419 lines
16 KiB
Python
419 lines
16 KiB
Python
"""Property-Based Tests für Enrichment Agent.
|
|
|
|
**Validates: Requirements 9.2, 9.4, 9.5**
|
|
|
|
Property 15: Wiki-Link Format for Entities
|
|
*For any* person name detected by the Enrichment Agent, the generated wiki-link
|
|
SHALL match the format `[[people/{slugified-name}]]`. For any project name, it
|
|
SHALL match `[[projects/{slugified-name}]]`. Slugification SHALL produce
|
|
lowercase, hyphen-separated strings.
|
|
|
|
Property 16: Confidence Threshold Filtering
|
|
*For any* set of detected entities with varying confidence scores, only entities
|
|
with confidence >= 0.7 SHALL appear in the final enrichment output. Entities
|
|
below 0.7 SHALL be excluded.
|
|
|
|
Property 17: Action Items Section Generation
|
|
*For any* non-empty list of `ActionItem` objects, the generated markdown SHALL
|
|
contain a `## Action Items` section header followed by a markdown list item for
|
|
each action item's description.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
from hypothesis import given, settings, assume, HealthCheck
|
|
from hypothesis import strategies as st
|
|
|
|
from monorepo.knowledge.ingestion.enrichment import (
|
|
CONFIDENCE_THRESHOLD,
|
|
ActionItem,
|
|
EnrichmentAgent,
|
|
_slugify,
|
|
generate_action_items_section,
|
|
generate_wiki_links,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Strategies
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Person name strategy: "Vorname Nachname" with Latin characters
|
|
_name_part_st = st.text(
|
|
alphabet=st.characters(
|
|
whitelist_categories=("Lu", "Ll"),
|
|
max_codepoint=255, # Latin-1 range for realistic names
|
|
),
|
|
min_size=3,
|
|
max_size=12,
|
|
).map(lambda s: s[0].upper() + s[1:].lower() if len(s) >= 2 else s.capitalize())
|
|
|
|
|
|
@st.composite
|
|
def person_name_st(draw: st.DrawFn) -> str:
|
|
"""Generate a realistic 'Vorname Nachname' string."""
|
|
first = draw(_name_part_st)
|
|
last = draw(_name_part_st)
|
|
assume(len(first) >= 3 and len(last) >= 3)
|
|
assume(first[0].isupper() and last[0].isupper())
|
|
name = f"{first} {last}"
|
|
# Ensure slug is at least 2 chars
|
|
slug = _slugify(name)
|
|
assume(len(slug) >= 2)
|
|
return name
|
|
|
|
|
|
@st.composite
|
|
def project_name_st(draw: st.DrawFn) -> str:
|
|
"""Generate a project name (alphanumeric with hyphens/spaces).
|
|
|
|
Ensures slugified result has at least 2 chars (realistic project names).
|
|
"""
|
|
parts = draw(st.lists(
|
|
st.text(
|
|
alphabet=st.characters(
|
|
whitelist_categories=("Ll", "Lu"),
|
|
whitelist_characters="-",
|
|
),
|
|
min_size=3,
|
|
max_size=12,
|
|
).filter(lambda s: any(c.isalpha() for c in s)),
|
|
min_size=1,
|
|
max_size=3,
|
|
))
|
|
name = " ".join(parts)
|
|
assume(len(name) >= 3)
|
|
# Ensure slug would be at least 2 chars
|
|
slug = _slugify(name)
|
|
assume(len(slug) >= 2)
|
|
return name
|
|
|
|
|
|
# Action item description strategy
|
|
_action_description_st = st.text(
|
|
alphabet=st.characters(
|
|
whitelist_categories=("L", "N", "P", "Z"),
|
|
blacklist_characters="\x00\r\n",
|
|
),
|
|
min_size=5,
|
|
max_size=80,
|
|
).filter(lambda s: s.strip() != "" and len(s.strip()) >= 5)
|
|
|
|
|
|
# Confidence scores
|
|
_confidence_above_st = st.floats(min_value=0.7, max_value=1.0, allow_nan=False)
|
|
_confidence_below_st = st.floats(min_value=0.0, max_value=0.69, allow_nan=False)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Property 15: Wiki-Link Format for Entities
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestWikiLinkFormatForEntities:
|
|
"""**Validates: Requirements 9.2**
|
|
|
|
Property 15: For any person name, the generated wiki-link SHALL match
|
|
`[[people/{slugified-name}]]`. For any project name, it SHALL match
|
|
`[[projects/{slugified-name}]]`. Slugification SHALL produce lowercase,
|
|
hyphen-separated strings.
|
|
"""
|
|
|
|
@given(names=st.lists(person_name_st(), min_size=1, max_size=5))
|
|
@settings(max_examples=50, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
|
def test_people_wiki_links_have_correct_format(self, names: list[str]) -> None:
|
|
"""All person wiki-links match [[people/{slug}]] format."""
|
|
result = generate_wiki_links(people=names, projects=[])
|
|
|
|
assert len(result["people"]) == len(names)
|
|
|
|
for link in result["people"]:
|
|
# Must match [[people/...]] pattern: slug is lowercase, hyphen-separated
|
|
assert re.match(r"^\[\[people/[a-z0-9]([a-z0-9-]*[a-z0-9])?\]\]$", link), (
|
|
f"Wiki-link {link!r} does not match expected format [[people/slug]]"
|
|
)
|
|
|
|
@given(names=st.lists(project_name_st(), min_size=1, max_size=5))
|
|
@settings(max_examples=50, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
|
def test_project_wiki_links_have_correct_format(self, names: list[str]) -> None:
|
|
"""All project wiki-links match [[projects/{slug}]] format."""
|
|
result = generate_wiki_links(people=[], projects=names)
|
|
|
|
assert len(result["projects"]) == len(names)
|
|
|
|
for link in result["projects"]:
|
|
# Must match [[projects/...]] pattern: slug is lowercase, hyphen-separated
|
|
assert re.match(r"^\[\[projects/[a-z0-9]([a-z0-9-]*[a-z0-9])?\]\]$", link), (
|
|
f"Wiki-link {link!r} does not match expected format [[projects/slug]]"
|
|
)
|
|
|
|
@given(name=person_name_st())
|
|
@settings(max_examples=50, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
|
def test_slugification_is_lowercase(self, name: str) -> None:
|
|
"""Slugified names are always lowercase."""
|
|
slug = _slugify(name)
|
|
assert slug == slug.lower(), (
|
|
f"Slug {slug!r} for name {name!r} is not fully lowercase"
|
|
)
|
|
|
|
@given(name=person_name_st())
|
|
@settings(max_examples=50, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
|
def test_slugification_is_hyphen_separated(self, name: str) -> None:
|
|
"""Slugified names use hyphens as separators (no spaces, underscores, etc.)."""
|
|
slug = _slugify(name)
|
|
# Must contain only lowercase alphanumeric and hyphens
|
|
assert re.match(r"^[a-z0-9]+(-[a-z0-9]+)*$", slug), (
|
|
f"Slug {slug!r} for name {name!r} contains invalid characters "
|
|
f"(expected only lowercase alphanumeric and hyphens)"
|
|
)
|
|
|
|
@given(
|
|
people=st.lists(person_name_st(), min_size=1, max_size=3),
|
|
projects=st.lists(project_name_st(), min_size=1, max_size=3),
|
|
)
|
|
@settings(max_examples=30, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
|
def test_combined_links_have_separate_namespaces(
|
|
self, people: list[str], projects: list[str]
|
|
) -> None:
|
|
"""People links use 'people/' prefix and project links use 'projects/' prefix."""
|
|
result = generate_wiki_links(people=people, projects=projects)
|
|
|
|
for link in result["people"]:
|
|
assert link.startswith("[[people/"), f"Person link {link!r} missing 'people/' prefix"
|
|
|
|
for link in result["projects"]:
|
|
assert link.startswith("[[projects/"), f"Project link {link!r} missing 'projects/' prefix"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Property 16: Confidence Threshold Filtering
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestConfidenceThresholdFiltering:
|
|
"""**Validates: Requirements 9.5**
|
|
|
|
Property 16: For any set of detected entities with varying confidence
|
|
scores, only entities with confidence >= 0.7 SHALL appear in the final
|
|
enrichment output. Entities below 0.7 SHALL be excluded.
|
|
"""
|
|
|
|
@given(
|
|
high_conf_names=st.lists(
|
|
st.tuples(person_name_st(), _confidence_above_st),
|
|
min_size=0,
|
|
max_size=3,
|
|
),
|
|
low_conf_names=st.lists(
|
|
st.tuples(person_name_st(), _confidence_below_st),
|
|
min_size=0,
|
|
max_size=3,
|
|
),
|
|
)
|
|
@settings(max_examples=50, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
|
def test_only_high_confidence_entities_pass_threshold(
|
|
self,
|
|
high_conf_names: list[tuple[str, float]],
|
|
low_conf_names: list[tuple[str, float]],
|
|
) -> None:
|
|
"""Entities with confidence >= 0.7 are included, those below are excluded."""
|
|
assume(len(high_conf_names) + len(low_conf_names) > 0)
|
|
# Ensure no name overlap between high and low sets (same name at different
|
|
# confidence levels is not a meaningful test case for threshold filtering)
|
|
high_names = {name for name, _ in high_conf_names}
|
|
low_names = {name for name, _ in low_conf_names}
|
|
assume(high_names.isdisjoint(low_names))
|
|
|
|
# Simulate the threshold filtering logic used in EnrichmentAgent.enrich()
|
|
all_entities = high_conf_names + low_conf_names
|
|
filtered = [name for name, conf in all_entities if conf >= CONFIDENCE_THRESHOLD]
|
|
|
|
# All high-confidence names should be in filtered
|
|
for name, conf in high_conf_names:
|
|
assert name in filtered, (
|
|
f"High-confidence entity {name!r} (conf={conf}) was excluded"
|
|
)
|
|
|
|
# No low-confidence names should be in filtered
|
|
for name, conf in low_conf_names:
|
|
assert name not in filtered, (
|
|
f"Low-confidence entity {name!r} (conf={conf}) was not excluded"
|
|
)
|
|
|
|
def test_threshold_constant_is_0_7(self) -> None:
|
|
"""The confidence threshold constant is exactly 0.7."""
|
|
assert CONFIDENCE_THRESHOLD == 0.7
|
|
|
|
@given(
|
|
content=st.just(
|
|
"Meeting mit André Knie und Hans Schmidt.\n"
|
|
"Betrifft Projekt Knowledge-Pipeline.\n"
|
|
"@Max assigned to review."
|
|
)
|
|
)
|
|
@settings(max_examples=5, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
|
def test_enrichment_agent_applies_threshold(self, content: str) -> None:
|
|
"""The EnrichmentAgent.enrich() method applies the confidence threshold."""
|
|
agent = EnrichmentAgent(provider="kiro")
|
|
result = agent.enrich(content)
|
|
|
|
# All returned people should have had confidence >= 0.7 internally
|
|
# We can't directly check confidence on the output (it's filtered already),
|
|
# but we can verify no obviously-invalid entities slip through
|
|
for person in result.people:
|
|
assert isinstance(person, str)
|
|
assert len(person) >= 2, (
|
|
f"Person name {person!r} is suspiciously short (likely low-confidence noise)"
|
|
)
|
|
|
|
@given(
|
|
high_conf_projects=st.lists(
|
|
st.tuples(project_name_st(), _confidence_above_st),
|
|
min_size=0,
|
|
max_size=3,
|
|
),
|
|
low_conf_projects=st.lists(
|
|
st.tuples(project_name_st(), _confidence_below_st),
|
|
min_size=0,
|
|
max_size=3,
|
|
),
|
|
)
|
|
@settings(max_examples=50, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
|
def test_project_entities_also_filtered_by_threshold(
|
|
self,
|
|
high_conf_projects: list[tuple[str, float]],
|
|
low_conf_projects: list[tuple[str, float]],
|
|
) -> None:
|
|
"""Project entities are also subject to the confidence threshold."""
|
|
assume(len(high_conf_projects) + len(low_conf_projects) > 0)
|
|
# Ensure no name overlap between high and low sets
|
|
high_names = {name for name, _ in high_conf_projects}
|
|
low_names = {name for name, _ in low_conf_projects}
|
|
assume(high_names.isdisjoint(low_names))
|
|
|
|
all_entities = high_conf_projects + low_conf_projects
|
|
filtered = [name for name, conf in all_entities if conf >= CONFIDENCE_THRESHOLD]
|
|
|
|
for name, conf in high_conf_projects:
|
|
assert name in filtered
|
|
|
|
for name, conf in low_conf_projects:
|
|
assert name not in filtered
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Property 17: Action Items Section Generation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestActionItemsSectionGeneration:
|
|
"""**Validates: Requirements 9.4**
|
|
|
|
Property 17: For any non-empty list of ActionItem objects, the generated
|
|
markdown SHALL contain a `## Action Items` section header followed by a
|
|
markdown list item for each action item's description.
|
|
"""
|
|
|
|
@given(
|
|
descriptions=st.lists(
|
|
_action_description_st,
|
|
min_size=1,
|
|
max_size=10,
|
|
unique=True,
|
|
)
|
|
)
|
|
@settings(max_examples=50, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
|
def test_section_header_present(self, descriptions: list[str]) -> None:
|
|
"""Generated markdown contains '## Action Items' header."""
|
|
items = [ActionItem(description=desc) for desc in descriptions]
|
|
section = generate_action_items_section(items)
|
|
|
|
assert "## Action Items" in section, (
|
|
f"Missing '## Action Items' header in generated section:\n{section}"
|
|
)
|
|
|
|
@given(
|
|
descriptions=st.lists(
|
|
_action_description_st,
|
|
min_size=1,
|
|
max_size=10,
|
|
unique=True,
|
|
)
|
|
)
|
|
@settings(max_examples=50, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
|
def test_each_action_item_has_list_entry(self, descriptions: list[str]) -> None:
|
|
"""Each action item appears as a markdown list item in the section."""
|
|
items = [ActionItem(description=desc) for desc in descriptions]
|
|
section = generate_action_items_section(items)
|
|
|
|
for desc in descriptions:
|
|
assert desc in section, (
|
|
f"Action item description {desc!r} not found in generated section:\n{section}"
|
|
)
|
|
|
|
@given(
|
|
descriptions=st.lists(
|
|
_action_description_st,
|
|
min_size=1,
|
|
max_size=10,
|
|
unique=True,
|
|
)
|
|
)
|
|
@settings(max_examples=50, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
|
def test_list_items_use_markdown_format(self, descriptions: list[str]) -> None:
|
|
"""Each action item is formatted as a markdown checkbox list item '- [ ] ...'."""
|
|
items = [ActionItem(description=desc) for desc in descriptions]
|
|
section = generate_action_items_section(items)
|
|
|
|
# Count markdown list items
|
|
list_item_count = section.count("- [ ]")
|
|
assert list_item_count == len(descriptions), (
|
|
f"Expected {len(descriptions)} list items but found {list_item_count} "
|
|
f"in:\n{section}"
|
|
)
|
|
|
|
@given(
|
|
descriptions=st.lists(
|
|
_action_description_st,
|
|
min_size=1,
|
|
max_size=5,
|
|
unique=True,
|
|
),
|
|
assignees=st.lists(
|
|
st.one_of(
|
|
st.none(),
|
|
st.text(
|
|
alphabet=st.characters(whitelist_categories=("L",)),
|
|
min_size=3,
|
|
max_size=10,
|
|
),
|
|
),
|
|
min_size=1,
|
|
max_size=5,
|
|
),
|
|
)
|
|
@settings(max_examples=30, deadline=None, suppress_health_check=[HealthCheck.function_scoped_fixture])
|
|
def test_assignees_included_when_present(
|
|
self, descriptions: list[str], assignees: list[str | None]
|
|
) -> None:
|
|
"""When an action item has an assignee, it appears with @ prefix."""
|
|
# Align lists
|
|
n = min(len(descriptions), len(assignees))
|
|
items = [
|
|
ActionItem(description=descriptions[i], assignee=assignees[i])
|
|
for i in range(n)
|
|
]
|
|
section = generate_action_items_section(items)
|
|
|
|
for item in items:
|
|
if item.assignee:
|
|
assert f"@{item.assignee}" in section, (
|
|
f"Assignee @{item.assignee} not found in section:\n{section}"
|
|
)
|
|
|
|
def test_empty_list_returns_empty_string(self) -> None:
|
|
"""An empty action items list produces an empty string (no section)."""
|
|
assert generate_action_items_section([]) == ""
|