python-math #40: feat(math): mode collapse C4 — delete-only improved branches (ban filter, tally sources, blob shape)

delphi/polismath/pca_kmeans_rep/legacy_kmeans.py

@@ -12,8 +12,8 @@
 This is a DIFFERENT algorithm from the off-production ``clusters.py`` warm start
 (split-largest / merge-closest, clusters.py:302-364), which is NOT a port of the
 Clojure ``clean-start-clusters``. That module is intentionally left untouched;
-this one is the faithful port and is wired only into the ``clojure-legacy``
-engine mode (see ``polismath.utils.engine_mode``).
+this one is the faithful port wired into the engine (the only clustering path
+since the mode collapse).
 
 Data model (mirrors Clojure's named-matrix + cluster maps):
 

delphi/polismath/pca_kmeans_rep/repness.py

@@ -9,7 +9,7 @@
 import pandas as pd
 from typing import Any, Dict, Iterable, List, Optional, Tuple
 
-from polismath.utils.engine_mode import ENGINE_MODE_LEGACY, resolve_engine_mode
+
 from polismath.utils.general import AGREE, DISAGREE
 
 
@@ -232,31 +232,23 @@ def compute_group_comment_stats_df(votes_long: pd.DataFrame,
 
     # Totals feed the "other" (rest) side of the comparison below.
     #
-    # clojure-legacy: Clojure's rest-stats sum per-group comment-stats over
-    # the OTHER GROUPS only (utils/mapv-rest, repness.clj:125-131), and group
-    # membership is unfolded through base clusters — so votes from
-    # participants in NO cluster never enter the comparison. Totals must
-    # therefore come from clustered voters only (FP-69c7a13580/FP-faac8c6125).
-    #
-    # improved: keeps the historical behavior where "other" included ALL
-    # participants not in the current group (even those not in any cluster).
+    # Clojure's rest-stats sum per-group comment-stats over the OTHER GROUPS
+    # only (utils/mapv-rest, repness.clj:125-131), and group membership is
+    # unfolded through base clusters — so votes from participants in NO
+    # cluster never enter the comparison. Totals must therefore come from
+    # clustered voters only (FP-69c7a13580/FP-faac8c6125).
     #
     # total_votes counts agree + disagree + PASS, matching Clojure's
     # `count-votes` (math/src/polismath/math/repness.clj:56-61, :70).
     # `count-votes` called with no `vote` arg uses `identity` as the filter
     # predicate; in Clojure 0 is truthy, so PASS (0) votes are kept. NaN
     # entries are already dropped above. Use size() to count non-NaN rows.
-    total_source = (
-        votes_in_groups
-        if resolve_engine_mode() == ENGINE_MODE_LEGACY
-        else votes_only
-    )
-    total_counts = total_source.groupby('comment').agg(
+    total_counts = votes_in_groups.groupby('comment').agg(
         total_agree=('vote', lambda x: (x == AGREE).sum()),
         total_disagree=('vote', lambda x: (x == DISAGREE).sum()),
         total_votes=('vote', 'size'),
     )
-    # The comment universe stays votes_only-based in BOTH modes (Clojure
+    # The comment universe stays votes_only-based (Clojure
     # iterates every matrix column; a comment voted on only by unclustered
     # participants still gets an all-zero stats row).
     all_voted_comments = votes_only['comment'].unique()

delphi/tests/test_clj_hash_order.py

@@ -111,9 +111,3 @@ def test_legacy_greedy_tie_follows_clojure_hash_order_string_pids(monkeypatch):
     assert in_conv & {"14", "15", "16", "17"} == {"15", "17"}
 
 
-def test_improved_greedy_unaffected(monkeypatch):
-    monkeypatch.setenv(ENGINE_MODE_ENV_VAR, "improved")
-    conv = _tie_conv()
-    in_conv = conv._get_in_conv_participants()
-    # Improved mode: threshold-only (min(7, n_cmts)=7 votes) — only pid 1.
-    assert in_conv == {1}

delphi/tests/test_in_conv_greedy_carry.py

@@ -64,13 +64,6 @@ def _mode(monkeypatch, mode):
 
 class TestGreedyFloor:
 
-    def test_improved_has_no_greedy_floor(self, monkeypatch):
-        # Current/improved behavior: only the 2 threshold-qualifiers cluster.
-        _mode(monkeypatch, 'improved')
-        conv = Conversation('g').update_votes(_votes(_TICK1_SPECS))
-        assert _clustered_pids(conv) == {'H0', 'H1'}
-        assert conv.in_conv == set()  # improved never populates the carry set
-
     def test_legacy_greedy_fills_to_fifteen(self, monkeypatch):
         _mode(monkeypatch, 'clojure-legacy')
         conv = Conversation('g').update_votes(_votes(_TICK1_SPECS))
@@ -111,17 +104,6 @@ def test_legacy_greedy_admits_persist_after_growth(self, monkeypatch):
         # And the new qualifiers are in too.
         assert {f'Q{i}' for i in range(20)}.issubset(clustered)
 
-    def test_improved_drops_non_qualifiers_after_growth(self, monkeypatch):
-        _mode(monkeypatch, 'improved')
-        conv = Conversation('carry').update_votes(_votes(_TICK1_SPECS))
-        conv = conv.update_votes(self._tick2_new_qualifiers())
-        clustered = _clustered_pids(conv)
-        # No carry, no greedy: the below-threshold lows are NOT clustered.
-        assert not any(f'L{i}' in clustered for i in range(20))
-        # Only the threshold-qualifiers (H0,H1 + Q0..Q19) cluster.
-        assert clustered == {'H0', 'H1'} | {f'Q{i}' for i in range(20)}
-
-
 class TestSerializedInConv:
 
     def test_legacy_blob_in_conv_includes_greedy_admits(self, monkeypatch):
@@ -132,13 +114,6 @@ def test_legacy_blob_in_conv_includes_greedy_admits(self, monkeypatch):
         assert {'H0', 'H1'}.issubset(blob_in_conv)
         assert {f'L{i}' for i in range(13)}.issubset(blob_in_conv)  # greedy admits
 
-    def test_improved_blob_in_conv_is_threshold_only(self, monkeypatch):
-        _mode(monkeypatch, 'improved')
-        conv = Conversation('blob').update_votes(_votes(_TICK1_SPECS))
-        blob_in_conv = {str(p) for p in conv.to_dict()['in-conv']}
-        assert blob_in_conv == {'H0', 'H1'}  # no greedy floor in improved
-
-
 class TestThresholdMonotonicity:
 
     @pytest.mark.parametrize('mode', ['improved', 'clojure-legacy'])

delphi/tests/test_legacy_blob_shape.py

@@ -80,11 +80,6 @@ def legacy(monkeypatch):
     monkeypatch.setenv(ENGINE_MODE_ENV_VAR, "clojure-legacy")
 
 
-@pytest.fixture()
-def improved(monkeypatch):
-    monkeypatch.setenv(ENGINE_MODE_ENV_VAR, "improved")
-
-
 def _sorted_base_clusters(conv):
     return sorted(conv.base_clusters, key=lambda c: c["id"])
 
@@ -105,13 +100,6 @@ def test_legacy_group_clusters_members_are_bids(conv, legacy):
     assert sorted(all_members) == sorted(bc_ids)
 
 
-def test_improved_group_clusters_members_stay_pids(conv, improved):
-    result = conv.to_dict()
-    pids = set(conv.rating_mat.index)
-    for gc in result["group-clusters"]:
-        assert set(gc["members"]) <= pids
-
-
 # ---------------------------------------------------------------------------
 # votes-base: per-base-cluster bucket lists in legacy mode.
 # ---------------------------------------------------------------------------
@@ -154,14 +142,6 @@ def test_legacy_votes_base_excludes_unclustered_votes(conv, legacy):
         assert sum(entry["S"]) == 20
 
 
-def test_improved_votes_base_stays_int_totals(conv, improved):
-    result = conv.to_dict()
-    entry = next(iter(result["votes-base"].values()))
-    assert isinstance(entry["A"], int)
-    assert isinstance(entry["D"], int)
-    assert isinstance(entry["S"], int)
-
-
 # ---------------------------------------------------------------------------
 # pca: comment-projection / comment-extremity emitted + sign parity.
 # ---------------------------------------------------------------------------
@@ -211,16 +191,6 @@ def test_legacy_sign_negation_of_center_and_projections(conv, legacy):
     )
 
 
-def test_improved_pca_emission_unchanged(conv, improved):
-    result = conv.to_dict()
-    np.testing.assert_allclose(result["pca"]["center"], np.asarray(conv.pca["center"]))
-    assert "comment-projection" not in result["pca"]
-    bc = result["base-clusters"]
-    by_id = {c["id"]: c for c in conv.base_clusters}
-    for i, bid in enumerate(bc["id"]):
-        assert bc["x"][i] == pytest.approx(by_id[bid]["center"][0])
-
-
 # ---------------------------------------------------------------------------
 # repness: Clojure finalize-cmt-stats shape in legacy mode.
 # ---------------------------------------------------------------------------
@@ -258,13 +228,6 @@ def test_legacy_repness_shape_and_direction_mapping(conv, legacy):
             assert "comment_id" not in got and "na" not in got and "rat" not in got
 
 
-def test_improved_repness_stays_internal_shape(conv, improved):
-    result = conv.to_dict()
-    assert set(result["repness"].keys()) == {
-        "comment_ids", "group_repness", "comment_repness", "consensus_comments",
-    }
-
-
 # ---------------------------------------------------------------------------
 # repness rest-domain: "other" = the OTHER GROUPS only in legacy mode.
 # ---------------------------------------------------------------------------
@@ -301,16 +264,6 @@ def test_legacy_repness_rest_domain_excludes_unclustered(legacy):
     assert row["ra"] == pytest.approx(3.0)
 
 
-def test_improved_repness_rest_domain_includes_all_voters(improved):
-    from polismath.pca_kmeans_rep.repness import compute_group_comment_stats_df
-
-    votes_long, groups = _rest_domain_fixture()
-    df = compute_group_comment_stats_df(votes_long, groups)
-    row = df.loc[(0, 0)]
-    # rest = group 1 + p99: na=1 ns=3 → other_pa = (1+1)/(3+2) = 0.4; ra = 1.875
-    assert row["ra"] == pytest.approx(0.75 / 0.4)
-
-
 # ---------------------------------------------------------------------------
 # group-aware-consensus: zero-S groups contribute (A+1)/(S+2) = 1/2 in legacy.
 # ---------------------------------------------------------------------------
@@ -337,16 +290,6 @@ def test_legacy_gac_multiplies_zero_s_groups(conv, legacy):
     assert result["group-aware-consensus"][10] == pytest.approx(expected)
 
 
-def test_improved_gac_skips_zero_s_groups(conv, improved):
-    result = conv.to_dict()
-    stats = _gac_group_stats(result, 10)
-    expected = 1.0
-    for a, s in stats.values():
-        if s > 0:
-            expected *= (a + 1.0) / (s + 2.0)
-    assert result["group-aware-consensus"][10] == pytest.approx(expected)
-
-
 # ---------------------------------------------------------------------------
 # moderation-state semantics: None until moderation applied (legacy).
 # ---------------------------------------------------------------------------
@@ -367,13 +310,6 @@ def test_legacy_mod_keys_populated_after_moderation(conv, legacy):
     assert result["lastModTimestamp"] is None
 
 
-def test_improved_mod_keys_stay_lists(conv, improved):
-    result = conv.to_dict()
-    assert result["mod-in"] == []
-    assert result["mod-out"] == []
-    assert result["lastModTimestamp"] == conv.last_updated
-
-
 # ---------------------------------------------------------------------------
 # Arrival-order parity: Clojure's named-matrix column order is first-vote
 # arrival order (update-nmat appends unseen colnames in encounter order);
@@ -422,11 +358,6 @@ def test_legacy_tids_emitted_in_arrival_order_with_aligned_pca(conv, legacy):
         assert result["pca"]["comment-extremity"][i] == pytest.approx(ext[tid])
 
 
-def test_improved_tids_stay_natsorted(conv, improved):
-    result = conv.to_dict()
-    assert result["tids"] == list(conv.rating_mat.columns)
-
-
 def test_legacy_from_dict_restores_arrival_order(conv, legacy):
     restored = Conversation.from_dict(conv.to_dict())
     assert restored.tid_arrival_order == conv.tid_arrival_order
@@ -458,12 +389,6 @@ def test_legacy_from_dict_restores_base_clusters_and_zid(conv, legacy):
     _assert_base_clusters_round_trip(conv, restored)
 
 
-def test_improved_from_dict_restores_base_clusters_and_zid(conv, improved):
-    restored = Conversation.from_dict(conv.to_dict())
-    assert restored.conversation_id == "legacy_blob_shape"
-    _assert_base_clusters_round_trip(conv, restored)
-
-
 def test_from_dict_preserves_falsy_conversation_id():
     # #2656 review finding 2: `data.get('conversation_id') or data.get('zid')`
     # would discard a legitimately-falsy id (e.g. 0) — the key-presence check
@@ -582,15 +507,6 @@ def test_legacy_single_vote_repness_and_consensus(legacy):
     assert d["consensus"]["disagree"] == []
 
 
-def test_improved_single_vote_guards_unchanged(improved):
-    d = _tiny_conv().to_dict()
-    assert d["pca"]["center"] == [0.0]
-    assert d["repness"]["group_repness"] == {0: []} or all(
-        not v for v in d["repness"]["group_repness"].values()
-    )
-    assert d["consensus"] == {"agree": [], "disagree": []}
-
-
 # ---------------------------------------------------------------------------
 # from_dict inverse: legacy round-trip restores the internal convention.
 # ---------------------------------------------------------------------------
@@ -628,13 +544,6 @@ def test_legacy_from_dict_round_trips_center_sign(conv, legacy):
     )
 
 
-def test_improved_from_dict_round_trips_center_sign(conv, improved):
-    restored = Conversation.from_dict(conv.to_dict())
-    np.testing.assert_allclose(
-        np.asarray(restored.pca["center"]), np.asarray(conv.pca["center"])
-    )
-
-
 # ---------------------------------------------------------------------------
 # Tiny SHAPES beyond 1x1 (review finding on #2653): the relaxed small-dim
 # guards cover any `rows < 2 OR cols < 2` matrix. Expectations are REAL

delphi/tests/test_mod_ptpt_leak_parity.py

@@ -51,12 +51,6 @@ def legacy_mode(monkeypatch):
     monkeypatch.setenv(ENGINE_MODE_ENV_VAR, 'clojure-legacy')
 
 
-@pytest.fixture
-def improved_mode(monkeypatch):
-    monkeypatch.delenv(PCA_IMPL_ENV_VAR, raising=False)
-    monkeypatch.setenv(ENGINE_MODE_ENV_VAR, 'improved')
-
-
 class TestLegacyBanLeak:
 
     def test_banned_participant_rows_kept(self, legacy_mode):
@@ -89,15 +83,5 @@ def test_banned_participant_stays_in_conv(self, legacy_mode):
         assert 'a0' in conv.in_conv
 
 
-class TestImprovedBanKept:
-    """Improved mode keeps the real ban feature byte-for-byte."""
-
-    def test_banned_participant_dropped_and_not_clustered(self, improved_mode):
-        conv = Conversation('leak').update_votes(_bloc_votes())
-        conv = conv.update_moderation({'mod_out_ptpts': ['a0']})
-        assert 'a0' not in conv.rating_mat.index
-        assert 'a0' not in _clustered_pids(conv)
-
-
 if __name__ == '__main__':
     pytest.main([__file__, '-v'])