feat(engine): add export_traffic to dump observations as savings-report JSONL

benchmarks/INSPECTING_RULES.md

@@ -80,3 +80,27 @@ rulechef-savings \
 Traffic JSONL: one call per line, `{"text": ..., "llm_label": ..., "gold_label": (optional)}`.
 Print to PDF from the browser, or:
 `chrome --headless --print-to-pdf=savings.pdf savings.html`
+
+### Producing the traffic file from observation mode
+
+If you're already using `chef.start_observing(...)` / `chef.add_observation(...)`
+to capture real LLM calls, you don't need to hand-write the traffic JSONL —
+`export_traffic` writes it directly from the observation buffer:
+
+```python
+chef = RuleChef(client=client)
+wrapped = chef.start_observing(client)
+# ... traffic flows through `wrapped` as usual ...
+
+chef.export_traffic("traffic.jsonl")
+```
+
+```bash
+rulechef-savings --rules rules.json --traffic traffic.jsonl --out savings.html
+```
+
+Only LLM-sourced observations are exported (human-labeled examples aren't
+traffic). Classification output (`{"label": ...}`) is written as `llm_label`;
+NER/extraction output (`{"entities": [...]}` or `{"spans": [...]}`) is written
+as `llm_entities` — note `rulechef-savings` itself currently only scores the
+classification (`llm_label`) shape.

rulechef/engine.py

@@ -432,6 +432,42 @@ def add_raw_observation(
         """
         self._observations.add_raw_observation(messages, response, metadata)
 
+    def export_traffic(self, path: str | Path) -> int:
+        """Export observed LLM traffic as savings-report JSONL.
+
+        Writes one line per LLM-sourced observation (see add_observation /
+        start_observing) in the format ``rulechef-savings`` expects:
+        ``{"text": ..., "llm_label": ...}`` for classification output, or
+        ``{"text": ..., "llm_entities": [...]}`` for NER/extraction output.
+        Human-sourced examples (add_human_example, add_human_correction) are
+        not traffic and are skipped, as are observations missing a "text"
+        input field or a recognized output shape.
+
+        Args:
+            path: Destination JSONL file path.
+
+        Returns:
+            Number of rows written.
+        """
+        rows: list[dict[str, Any]] = []
+        for example in self.buffer.get_all_examples():
+            if example.source != "llm":
+                continue
+            text = example.input.get("text")
+            if text is None:
+                continue
+            if "label" in example.output:
+                rows.append({"text": text, "llm_label": example.output["label"]})
+            elif "entities" in example.output:
+                rows.append({"text": text, "llm_entities": example.output["entities"]})
+            elif "spans" in example.output:
+                rows.append({"text": text, "llm_entities": example.output["spans"]})
+
+        with open(path, "w") as f:
+            for row in rows:
+                f.write(json.dumps(row) + "\n")
+        return len(rows)
+
     # ========================================
     # Learning
     # ========================================

tests/test_engine_observation.py

@@ -276,3 +276,100 @@ def test_mixed_raw_and_monkey_patch(self):
         assert len(chef._pending_raw_observations) == 1
 
         chef.stop_observing()
+
+
+# =========================================================================
+# export_traffic()
+# =========================================================================
+
+
+class TestExportTraffic:
+    def test_writes_classification_jsonl(self, tmp_path):
+        chef = RuleChef(task=_make_task(), client=MagicMock())
+        chef.add_observation({"text": "what is the exchange rate"}, {"label": "exchange_rate"})
+        chef.add_observation({"text": "card is missing"}, {"label": "card_arrival"})
+
+        out = tmp_path / "traffic.jsonl"
+        count = chef.export_traffic(out)
+
+        assert count == 2
+        lines = [json.loads(line) for line in out.read_text().splitlines()]
+        assert lines[0] == {"text": "what is the exchange rate", "llm_label": "exchange_rate"}
+        assert lines[1] == {"text": "card is missing", "llm_label": "card_arrival"}
+
+    def test_writes_ner_jsonl(self, tmp_path):
+        chef = RuleChef(client=MagicMock())
+        entities = [{"text": "Paris", "start": 0, "end": 5, "type": "LOC"}]
+        chef.add_observation({"text": "Paris is nice"}, {"entities": entities})
+
+        out = tmp_path / "traffic.jsonl"
+        count = chef.export_traffic(out)
+
+        assert count == 1
+        row = json.loads(out.read_text().splitlines()[0])
+        assert row == {"text": "Paris is nice", "llm_entities": entities}
+
+    def test_skips_human_examples(self, tmp_path):
+        chef = RuleChef(task=_make_task(), client=MagicMock())
+        chef.add_example({"text": "human labeled"}, {"label": "exchange_rate"})
+        chef.add_observation({"text": "llm labeled"}, {"label": "card_arrival"})
+
+        out = tmp_path / "traffic.jsonl"
+        count = chef.export_traffic(out)
+
+        assert count == 1
+        row = json.loads(out.read_text().splitlines()[0])
+        assert row["text"] == "llm labeled"
+
+    def test_skips_observations_without_text_or_recognized_output(self, tmp_path):
+        chef = RuleChef(client=MagicMock())
+        chef.add_observation({"other_field": "no text here"}, {"label": "x"})
+        chef.add_observation({"text": "unrecognized output shape"}, {"unrecognized_key": "x"})
+
+        out = tmp_path / "traffic.jsonl"
+        count = chef.export_traffic(out)
+
+        assert count == 0
+        assert out.read_text() == ""
+
+    def test_round_trips_through_rulechef_savings(self, tmp_path, monkeypatch):
+        from rulechef.savings import main as savings_main
+
+        chef = RuleChef(task=_make_task(), client=MagicMock())
+        chef.add_observation({"text": "what is the exchange rate"}, {"label": "exchange_rate"})
+        chef.add_observation({"text": "card is missing"}, {"label": "card_arrival"})
+        traffic = tmp_path / "traffic.jsonl"
+        assert chef.export_traffic(traffic) == 2
+
+        rules = {
+            "rules": [
+                {
+                    "name": "rate",
+                    "format": "regex",
+                    "content": r"(?i)exchange rate",
+                    "output_template": {"label": "exchange_rate"},
+                    "output_key": "label",
+                }
+            ]
+        }
+        rules_file = tmp_path / "rules.json"
+        rules_file.write_text(json.dumps(rules))
+        out = tmp_path / "savings.html"
+
+        monkeypatch.setattr(
+            "sys.argv",
+            [
+                "rulechef-savings",
+                "--rules",
+                str(rules_file),
+                "--traffic",
+                str(traffic),
+                "--out",
+                str(out),
+            ],
+        )
+        savings_main()
+
+        html_doc = out.read_text()
+        assert "replaceable" in html_doc
+        assert "50%" in html_doc  # 1 of 2 calls answered