Modified save_results to append to existing output file rather than overwrite. Enforced UTF-8 encoding, switched to safe_dump and added document delimiter between records. Also simplified document generation. Fixes issue #24. Added second test case to literature_mcp_encoding_test.yaml for testing.

Author: Charles Parker

src/metacoder/evals/runner.py

@@ -518,18 +518,21 @@ def run_all_evals(
 
     def save_results(self, results: List[EvalResult], output_path: Path):
         """Save evaluation results to file."""
-        # Convert to list of dicts
-        results_data = []
-        for result in results:
-            results_data.append(result.model_dump())
+        # output_path.parent.mkdir(parents=True, exist_ok=True)  # Not sure if the folder should be created here
+        data = {
+            "results": [r.model_dump() for r in results],
+            "summary": self.generate_summary(results),
+        }
 
-        # Save as YAML
-        with open(output_path, "w") as f:
-            yaml.dump(
-                {"results": results_data, "summary": self.generate_summary(results)},
+        # Append a new YAML document to the output file.
+        with open(output_path, "a", encoding="utf-8", newline="") as f:
+            yaml.safe_dump(
+                data,
                 f,
+                explicit_start=True,  # writes '---' to mark a new document
                 default_flow_style=False,
                 sort_keys=False,
+                allow_unicode=True,
             )
 
     def generate_summary(self, results: List[EvalResult]) -> Dict[str, Any]:

tests/input/literature_mcp_encoding_test.yaml

@@ -27,3 +27,11 @@ cases:
   input: Based on PMID 33926573 do microbes from alkaline sulphidic tailings show oxidative stresses?
   expected_output: 'The paper says No but it is retracted so the results should not be trusted.'
   threshold: 0.9
+- name: "disease"
+  metrics: [CorrectnessMetric]
+  input: "According to PMID:35743164, What 3 diseases are associated with ITPR1 mutations? Give me disease names and MONDO IDs"
+  expected_output: |
+    MONDO:0011694 (spinocerebellar ataxia type 15/16, aka SCA15)
+    MONDO:0007298 (spinocerebellar ataxia type 29, aka SCA29)
+    MONDO:0008795 (aniridia-cerebellar ataxia-intellectual disability syndrome; aka Gillespie syndrome)
+  threshold: 0.7