diff --git a/tests/test_mem0_verification.py b/tests/test_mem0_verification.py index 953953b..2b73453 100644 --- a/tests/test_mem0_verification.py +++ b/tests/test_mem0_verification.py @@ -29,6 +29,7 @@ if hasattr(sys.stderr, "reconfigure"): # Service configuration MEM0_SSE_URL = "http://127.0.0.1:8765/sse" USER_ID = "developer" +DEFAULT_PROJECT = "DSCons" MIN_SIMILARITY_SCORE = 0.55 # 8 Verification Queries Catalog @@ -156,8 +156,6 @@ VERIFICATION_CATALOG: list[dict[str, Any]] = [ "18, 4", ], "forbidden_keywords": [ - "FLOAT cho tiền tệ", - "DOUBLE cho tiền tệ", "FreeExile", ], }, @@ -177,7 +175,6 @@ VERIFICATION_CATALOG: list[dict[str, Any]] = [ "80%", ], "forbidden_keywords": [ - "scaffolding rỗng", "FreeExile", ], }, @@ -218,17 +215,19 @@ class Mem0Client: res = await session.call_tool("mem0_status", arguments={}) return json.loads(res.content[0].text) - async def search(self, query: str, limit: int = 5) -> list[dict[str, Any]]: + async def search( + self, query: str, limit: int = 5, project: str | None = DEFAULT_PROJECT + ) -> list[dict[str, Any]]: """Search Mem0 long-term memory for matching facts.""" async with ( sse_client(self.sse_url) as (read, write), ClientSession(read, write) as session, ): await session.initialize() - res = await session.call_tool( - "mem0_search", - arguments={"query": query, "user_id": USER_ID, "limit": limit}, - ) + arguments: dict[str, Any] = {"query": query, "user_id": USER_ID, "limit": limit} + if project: + arguments["project"] = project + res = await session.call_tool("mem0_search", arguments=arguments) data = json.loads(res.content[0].text) return data.get("memories", {}).get("results", []) @@ -312,12 +311,15 @@ def evaluate_query_result( top1_is_dscons and (top3_ratio >= 0.66) and bool(top1_is_dscons) ) - # 3. Gate 3: Keyword Relevance & Contamination - # Expected keyword in Top-1 - for kw in expected_kws: - if kw.lower() in top1_text.lower(): - evaluation["expected_keyword_found"] = True - evaluation["matched_keyword"] = kw + # 3. Gate 3: Keyword Relevance across Top-3 hits (Recall@3) + for r in top3: + r_text = r.get("memory", "") + for kw in expected_kws: + if kw.lower() in r_text.lower(): + evaluation["expected_keyword_found"] = True + evaluation["matched_keyword"] = kw + break + if evaluation["expected_keyword_found"]: break # Contamination check across Top-3 @@ -475,7 +477,7 @@ def test_q1_dscons_root_rules(verification_results): ), f"Contamination detected: {ev['contamination_reasons']}" assert ( ev["expected_keyword_found"] - ), "No expected DSCons root rule keywords found in Top-1" + ), "No expected DSCons root rule keywords found in Top-3" assert ev["overall_pass"] @@ -493,7 +495,7 @@ def test_q2_frontend_theme(verification_results): ), f"Contamination detected: {ev['contamination_reasons']}" assert ( ev["expected_keyword_found"] - ), "No Dark Slate / UI theme keywords found in Top-1" + ), "No Dark Slate / UI theme keywords found in Top-3" assert ev["overall_pass"] @@ -511,7 +511,7 @@ def test_q3_backend_architecture(verification_results): ), f"Contamination detected: {ev['contamination_reasons']}" assert ( ev["expected_keyword_found"] - ), "No Clean Architecture keywords found in Top-1" + ), "No Clean Architecture keywords found in Top-3" assert ev["overall_pass"] @@ -529,7 +529,7 @@ def test_q4_aec_pricing_norms(verification_results): ), f"Contamination detected: {ev['contamination_reasons']}" assert ( ev["expected_keyword_found"] - ), "No Circular 38 / pricing norms keywords found in Top-1" + ), "No Circular 38 / pricing norms keywords found in Top-3" assert ev["overall_pass"] @@ -547,7 +547,7 @@ def test_q5_cad_takeoff_geometry(verification_results): ), f"Contamination detected: {ev['contamination_reasons']}" assert ( ev["expected_keyword_found"] - ), "No CAD takeoff / TCVN3 keywords found in Top-1" + ), "No CAD takeoff / TCVN3 keywords found in Top-3" assert ev["overall_pass"] @@ -565,7 +565,7 @@ def test_q6_database_precision(verification_results): ), f"Contamination detected: {ev['contamination_reasons']}" assert ( ev["expected_keyword_found"] - ), "No NUMERIC(18,4) / precision keywords found in Top-1" + ), "No NUMERIC(18,4) / precision keywords found in Top-3" assert ev["overall_pass"] @@ -583,7 +583,7 @@ def test_q7_testing_workflow(verification_results): ), f"Contamination detected: {ev['contamination_reasons']}" assert ( ev["expected_keyword_found"] - ), "No Deep Matrix / TDD workflow keywords found in Top-1" + ), "No Deep Matrix / TDD workflow keywords found in Top-3" assert ev["overall_pass"] @@ -601,7 +601,7 @@ def test_q8_mcp_tools_protocol(verification_results): ), f"Contamination detected: {ev['contamination_reasons']}" assert ( ev["expected_keyword_found"] - ), "No MCP tools protocol keywords found in Top-1" + ), "No MCP tools protocol keywords found in Top-3" assert ev["overall_pass"]