diff --git a/benchmarks/INSPECTING_RULES.md b/benchmarks/INSPECTING_RULES.md index 751acda..b56b0cf 100644 --- a/benchmarks/INSPECTING_RULES.md +++ b/benchmarks/INSPECTING_RULES.md @@ -80,3 +80,27 @@ rulechef-savings \ Traffic JSONL: one call per line, `{"text": ..., "llm_label": ..., "gold_label": (optional)}`. Print to PDF from the browser, or: `chrome --headless --print-to-pdf=savings.pdf savings.html` + +### Producing the traffic file from observation mode + +If you're already using `chef.start_observing(...)` / `chef.add_observation(...)` +to capture real LLM calls, you don't need to hand-write the traffic JSONL — +`export_traffic` writes it directly from the observation buffer: + +```python +chef = RuleChef(client=client) +wrapped = chef.start_observing(client) +# ... traffic flows through `wrapped` as usual ... + +chef.export_traffic("traffic.jsonl") +``` + +```bash +rulechef-savings --rules rules.json --traffic traffic.jsonl --out savings.html +``` + +Only LLM-sourced observations are exported (human-labeled examples aren't +traffic). Classification output (`{"label": ...}`) is written as `llm_label`; +NER/extraction output (`{"entities": [...]}` or `{"spans": [...]}`) is written +as `llm_entities` — note `rulechef-savings` itself currently only scores the +classification (`llm_label`) shape. diff --git a/rulechef/engine.py b/rulechef/engine.py index 3981a34..f6ca3c9 100644 --- a/rulechef/engine.py +++ b/rulechef/engine.py @@ -432,6 +432,42 @@ def add_raw_observation( """ self._observations.add_raw_observation(messages, response, metadata) + def export_traffic(self, path: str | Path) -> int: + """Export observed LLM traffic as savings-report JSONL. + + Writes one line per LLM-sourced observation (see add_observation / + start_observing) in the format ``rulechef-savings`` expects: + ``{"text": ..., "llm_label": ...}`` for classification output, or + ``{"text": ..., "llm_entities": [...]}`` for NER/extraction output. + Human-sourced examples (add_human_example, add_human_correction) are + not traffic and are skipped, as are observations missing a "text" + input field or a recognized output shape. + + Args: + path: Destination JSONL file path. + + Returns: + Number of rows written. + """ + rows: list[dict[str, Any]] = [] + for example in self.buffer.get_all_examples(): + if example.source != "llm": + continue + text = example.input.get("text") + if text is None: + continue + if "label" in example.output: + rows.append({"text": text, "llm_label": example.output["label"]}) + elif "entities" in example.output: + rows.append({"text": text, "llm_entities": example.output["entities"]}) + elif "spans" in example.output: + rows.append({"text": text, "llm_entities": example.output["spans"]}) + + with open(path, "w") as f: + for row in rows: + f.write(json.dumps(row) + "\n") + return len(rows) + # ======================================== # Learning # ======================================== diff --git a/tests/test_engine_observation.py b/tests/test_engine_observation.py index 7461c67..81872f3 100644 --- a/tests/test_engine_observation.py +++ b/tests/test_engine_observation.py @@ -276,3 +276,100 @@ def test_mixed_raw_and_monkey_patch(self): assert len(chef._pending_raw_observations) == 1 chef.stop_observing() + + +# ========================================================================= +# export_traffic() +# ========================================================================= + + +class TestExportTraffic: + def test_writes_classification_jsonl(self, tmp_path): + chef = RuleChef(task=_make_task(), client=MagicMock()) + chef.add_observation({"text": "what is the exchange rate"}, {"label": "exchange_rate"}) + chef.add_observation({"text": "card is missing"}, {"label": "card_arrival"}) + + out = tmp_path / "traffic.jsonl" + count = chef.export_traffic(out) + + assert count == 2 + lines = [json.loads(line) for line in out.read_text().splitlines()] + assert lines[0] == {"text": "what is the exchange rate", "llm_label": "exchange_rate"} + assert lines[1] == {"text": "card is missing", "llm_label": "card_arrival"} + + def test_writes_ner_jsonl(self, tmp_path): + chef = RuleChef(client=MagicMock()) + entities = [{"text": "Paris", "start": 0, "end": 5, "type": "LOC"}] + chef.add_observation({"text": "Paris is nice"}, {"entities": entities}) + + out = tmp_path / "traffic.jsonl" + count = chef.export_traffic(out) + + assert count == 1 + row = json.loads(out.read_text().splitlines()[0]) + assert row == {"text": "Paris is nice", "llm_entities": entities} + + def test_skips_human_examples(self, tmp_path): + chef = RuleChef(task=_make_task(), client=MagicMock()) + chef.add_example({"text": "human labeled"}, {"label": "exchange_rate"}) + chef.add_observation({"text": "llm labeled"}, {"label": "card_arrival"}) + + out = tmp_path / "traffic.jsonl" + count = chef.export_traffic(out) + + assert count == 1 + row = json.loads(out.read_text().splitlines()[0]) + assert row["text"] == "llm labeled" + + def test_skips_observations_without_text_or_recognized_output(self, tmp_path): + chef = RuleChef(client=MagicMock()) + chef.add_observation({"other_field": "no text here"}, {"label": "x"}) + chef.add_observation({"text": "unrecognized output shape"}, {"unrecognized_key": "x"}) + + out = tmp_path / "traffic.jsonl" + count = chef.export_traffic(out) + + assert count == 0 + assert out.read_text() == "" + + def test_round_trips_through_rulechef_savings(self, tmp_path, monkeypatch): + from rulechef.savings import main as savings_main + + chef = RuleChef(task=_make_task(), client=MagicMock()) + chef.add_observation({"text": "what is the exchange rate"}, {"label": "exchange_rate"}) + chef.add_observation({"text": "card is missing"}, {"label": "card_arrival"}) + traffic = tmp_path / "traffic.jsonl" + assert chef.export_traffic(traffic) == 2 + + rules = { + "rules": [ + { + "name": "rate", + "format": "regex", + "content": r"(?i)exchange rate", + "output_template": {"label": "exchange_rate"}, + "output_key": "label", + } + ] + } + rules_file = tmp_path / "rules.json" + rules_file.write_text(json.dumps(rules)) + out = tmp_path / "savings.html" + + monkeypatch.setattr( + "sys.argv", + [ + "rulechef-savings", + "--rules", + str(rules_file), + "--traffic", + str(traffic), + "--out", + str(out), + ], + ) + savings_main() + + html_doc = out.read_text() + assert "replaceable" in html_doc + assert "50%" in html_doc # 1 of 2 calls answered