wip

Deep-SAD-PyTorch/src/main.py

@@ -898,7 +898,7 @@ def main(
                     device=device,
                     n_jobs_dataloader=cfg.settings["n_jobs_dataloader"],
                     k_fold_idx=fold_idx,
-                    batch_size=256,
+                    batch_size=32,
                 )
 
             retest_output_path = load_model / "retest_output"

Deep-SAD-PyTorch/src/optim/DeepSAD_trainer.py

@@ -366,7 +366,9 @@ class DeepSADTrainer(BaseTrainer):
             scores_exp_valid = scores_exp[valid_mask_exp]
 
             self.test_auc_exp_based = roc_auc_score(labels_exp_binary, scores_exp_valid)
-            self.test_roc_exp_based = roc_curve(labels_exp_binary, scores_exp_valid)
+            self.test_roc_exp_based = roc_curve(
+                labels_exp_binary, scores_exp_valid, drop_intermediate=False
+            )
             self.test_prc_exp_based = precision_recall_curve(
                 labels_exp_binary, scores_exp_valid
             )
@@ -403,7 +405,7 @@ class DeepSADTrainer(BaseTrainer):
                 labels_manual_binary, scores_manual_valid
             )
             self.test_roc_manual_based = roc_curve(
-                labels_manual_binary, scores_manual_valid
+                labels_manual_binary, scores_manual_valid, drop_intermediate=False
             )
             self.test_prc_manual_based = precision_recall_curve(
                 labels_manual_binary, scores_manual_valid

thesis/Main.bbl

@@ -1591,6 +1591,80 @@
       \verb http://dx.doi.org/10.1109/5.726791
       \endverb
     \endentry
+    \entry{ef_concept_source}{article}{}{}
+      \name{author}{8}{}{%
+        {{hash=cde98454252ce53a9838df6015a87166}{%
+           family={Ye},
+           familyi={Y\bibinitperiod},
+           given={Min},
+           giveni={M\bibinitperiod}}}%
+        {{hash=99c2139712828885b388f7f7e83f4134}{%
+           family={Nie},
+           familyi={N\bibinitperiod},
+           given={Jie},
+           giveni={J\bibinitperiod}}}%
+        {{hash=8b4ed7eebb280dc2d0df1bec91e27c18}{%
+           family={Liu},
+           familyi={L\bibinitperiod},
+           given={Anan},
+           giveni={A\bibinitperiod}}}%
+        {{hash=9f4446a6d583221d3d173f806ecf8627}{%
+           family={Wang},
+           familyi={W\bibinitperiod},
+           given={Zhigang},
+           giveni={Z\bibinitperiod}}}%
+        {{hash=ebcb470ee777cbc56984a83379d29819}{%
+           family={Huang},
+           familyi={H\bibinitperiod},
+           given={Lei},
+           giveni={L\bibinitperiod}}}%
+        {{hash=d761d577d421d6a0566473ae6b8b342f}{%
+           family={Tian},
+           familyi={T\bibinitperiod},
+           given={Hao},
+           giveni={H\bibinitperiod}}}%
+        {{hash=2e463ab20a44bc493252994ca77f0fca}{%
+           family={Song},
+           familyi={S\bibinitperiod},
+           given={Dehai},
+           giveni={D\bibinitperiod}}}%
+        {{hash=eeea6461d631d4e9cb07d2abc2de6885}{%
+           family={Wei},
+           familyi={W\bibinitperiod},
+           given={Zhiqiang},
+           giveni={Z\bibinitperiod}}}%
+      }
+      \list{publisher}{1}{%
+        {Frontiers Media SA}%
+      }
+      \strng{namehash}{0fca66725a9966a967fc7893b180ddef}
+      \strng{fullhash}{0e37676c60146890b0c3819a1c8e441b}
+      \strng{fullhashraw}{0e37676c60146890b0c3819a1c8e441b}
+      \strng{bibnamehash}{0e37676c60146890b0c3819a1c8e441b}
+      \strng{authorbibnamehash}{0e37676c60146890b0c3819a1c8e441b}
+      \strng{authornamehash}{0fca66725a9966a967fc7893b180ddef}
+      \strng{authorfullhash}{0e37676c60146890b0c3819a1c8e441b}
+      \strng{authorfullhashraw}{0e37676c60146890b0c3819a1c8e441b}
+      \field{sortinit}{7}
+      \field{sortinithash}{108d0be1b1bee9773a1173443802c0a3}
+      \field{labelnamesource}{author}
+      \field{labeltitlesource}{title}
+      \field{issn}{2296-7745}
+      \field{journaltitle}{Frontiers in Marine Science}
+      \field{month}{8}
+      \field{title}{Multi-Year ENSO Forecasts Using Parallel Convolutional Neural Networks With Heterogeneous Architecture}
+      \field{volume}{8}
+      \field{year}{2021}
+      \verb{doi}
+      \verb 10.3389/fmars.2021.717184
+      \endverb
+      \verb{urlraw}
+      \verb http://dx.doi.org/10.3389/fmars.2021.717184
+      \endverb
+      \verb{url}
+      \verb http://dx.doi.org/10.3389/fmars.2021.717184
+      \endverb
+    \endentry
   \enddatalist
 \endrefsection
 \endinput

thesis/Main.tex

@@ -65,6 +65,7 @@
 % \draftcopyName{ENTWURF}{160}
  
 \usepackage{xcolor}
+\usepackage{xfrac}
 \usepackage{booktabs}
 \usepackage{multirow}
 \usepackage[colorinlistoftodos]{todonotes}
@@ -1067,11 +1068,11 @@ The decoder network (see figure~\ref{fig:setup_arch_lenet_decoder}) mirrors the
 
 Even though the LeNet-inspired encoder proved capable of achieving our degradation quantification objective in initial experiments, we identified several shortcomings that motivated the design of a second, more efficient architecture. The most important issue concerns the shape of the CNN's receptive field (RF) which describes the region of the input that influences a single output activation. Its size and aspect ratio determine which structures the network can effectively capture: if the RF is too small, larger patterns cannot be detected, while an excessively large RF may hinder the network from learning to recognize fine details. For standard image data, the RF is often expressed as a symmetric $n \times n$ region, but in principle it can be computed independently per axis.
 
-\todo[inline]{RF concept figur}
+\fig{setup_ef_concept}{figures/setup_ef_concept}{Receptive fields in a CNN. Each output activation aggregates information from a region of the input; stacking layers expands this region, while kernel size, stride, and padding control how quickly it grows and what shape it takes. (A) illustrates slower, fine-grained growth; (B) shows faster expansion, producing a larger—potentially anisotropic—receptive field and highlighting the trade-off between detail and context. Reproduced from~\cite{ef_concept_source}}
 
-The RF shape's issue arises from the fact that spinning multi-beam LiDAR oftentimes produce point clouds posessing dense horizontal but limited vertical resolution. In our case this, this results in a pixel-per-degree resolution of approximately $0.99^{\circ}$/pixel vertically and $0.18^{\circ}$/pixel horizontally \todo[inline]{double-check with calculation graphic/table}. Consequently, the LeNet-inspired encoder’s calculated receptive field of $16 \times 16$ pixels translates to an angular size of $15.88^{\circ} \times 2.81^{\circ}$, which is highly rectangular in angular space. Such a mismatch risks limiting the network’s ability to capture degradation patterns that extend differently across the two axes.
+The RF shape's issue arises from the fact that spinning multi-beam LiDAR oftentimes produce point clouds posessing dense horizontal but limited vertical resolution. In our case this, this results in a pixel-per-degree resolution of approximately $5.69\,\sfrac{pixel}{deg}$ vertically and $1.01\,\sfrac{pixel}{deg}$ horizontally. Consequently, the LeNet-inspired encoder’s calculated receptive field of $16 \times 16$ pixels translates to an angular size of $15.88^{\circ} \times 2.81^{\circ}$, which is highly rectangular in angular space. Such a mismatch risks limiting the network’s ability to capture degradation patterns that extend differently across the two axes.
 
-\todo[inline]{add schematic showing rectangular angular RF overlaid on LiDAR projection}
+%\todo[inline]{add schematic showing rectangular angular RF overlaid on LiDAR projection}
 
 %\todo[inline]{start by explaining lenet architecture, encoder and decoder split, encoder network is the one being trained during the main training step, together as autoencoder during pretraining, decoder of lenet pretty much mirrored architecture of encoder, after preprocessing left with image data (2d projections, grayscale = 1 channel) so input is 2048x32x1. convolutional layers with pooling afterwards (2 convolution + pooling) convolutions to multiple channels (8, 4?) each channel capable of capturing a different pattern/structure of input. fully connected layer before latent space, latent space size not fixed since its also a hyperparameter and depended on how well the normal vs anomalous data can be captured and differentiated in the dimensionality of the latent space}
 %\todo[inline]{batch normalization, relu? something....}

thesis/bib/bibliography.bib

@@ -549,6 +549,20 @@
 	author = {Lecun, Y. and Bottou, L. and Bengio, Y. and Haffner, P.},
 	year = {1998},
 	pages = {2278–2324},
+},
+@article{ef_concept_source,
+	title = {Multi-Year ENSO Forecasts Using Parallel Convolutional Neural
+	         Networks With Heterogeneous Architecture},
+	volume = {8},
+	ISSN = {2296-7745},
+	url = {http://dx.doi.org/10.3389/fmars.2021.717184},
+	DOI = {10.3389/fmars.2021.717184},
+	journal = {Frontiers in Marine Science},
+	publisher = {Frontiers Media SA},
+	author = {Ye, Min and Nie, Jie and Liu, Anan and Wang, Zhigang and Huang, Lei
+	          and Tian, Hao and Song, Dehai and Wei, Zhiqiang},
+	year = {2021},
+	month = aug,
 }
 
 

tools/calculate_rf_sizes.py

@@ -63,6 +63,19 @@ def calculate_angular_receptive_field(
     return rf_vertical_deg, rf_horizontal_deg
 
 
+def calculate_pixels_per_degree(resolution: int, fov: float) -> float:
+    """Calculate pixels per degree for a given resolution and field of view.
+
+    Args:
+        resolution: Number of pixels
+        fov: Field of view in degrees
+
+    Returns:
+        float: Pixels per degree
+    """
+    return resolution / fov
+
+
 horizontal_resolution = 2048
 horizontal_fov = 360.0
 vertical_resolution = 32
@@ -100,3 +113,12 @@ print(f"SubTer LeNet (Asymmetric kernels) RF size: {rf_h} × {rf_w} pixels")
 print(
     f"SubTer LeNet (Asymmetric kernels) RF angular size: {rf_vert_deg:.2f}° × {rf_horiz_deg:.2f}°"
 )
+
+# Calculate pixels per degree
+horizontal_ppd = calculate_pixels_per_degree(horizontal_resolution, horizontal_fov)
+vertical_ppd = calculate_pixels_per_degree(vertical_resolution, vertical_fov)
+
+print("\nPixels per Degree:")
+print(f"Horizontal: {horizontal_ppd:.2f} px/°")
+print(f"Vertical:   {vertical_ppd:.2f} px/°")
+print()

tools/diff_df.py

@@ -0,0 +1,544 @@
+import json
+import math
+from typing import Any, Dict, Iterable, List, Optional, Tuple
+
+import polars as pl
+
+Number = (int, float)
+
+FLOAT_DTYPES = {pl.Float32, pl.Float64}
+SIMPLE_CASTABLE_DTYPES = (
+    pl.Int8,
+    pl.Int16,
+    pl.Int32,
+    pl.Int64,
+    pl.UInt8,
+    pl.UInt16,
+    pl.UInt32,
+    pl.UInt64,
+    pl.Float32,
+    pl.Float64,
+    pl.Utf8,
+    pl.Boolean,
+    pl.Date,
+    pl.Datetime,
+    pl.Time,
+    pl.Duration,
+)
+
+
+def _is_nan(x):
+    try:
+        return isinstance(x, float) and math.isnan(x)
+    except Exception:
+        return False
+
+
+def _repr_safe(v):
+    try:
+        return json.dumps(v, default=str, ensure_ascii=False)
+    except Exception:
+        return repr(v)
+
+
+def _to_python(v):
+    """
+    Convert any leaf-ish object to plain Python types:
+    - pl.Series -> list (or scalar if length==1)
+    - objects with .to_list()/.tolist() -> list
+    - dict stays dict; list/tuple become list
+    """
+    # Polars Series
+    if isinstance(v, pl.Series):
+        seq = v.to_list()
+        return seq[0] if len(seq) == 1 else seq
+    # Numpy scalars/arrays or anything with tolist()
+    if hasattr(v, "tolist"):
+        try:
+            return v.tolist()
+        except Exception:
+            pass
+    # Polars expressions should not appear; stringify them
+    # Anything iterable that isn't list/dict/str -> convert carefully
+    if isinstance(v, tuple):
+        return [_to_python(x) for x in v]
+    if isinstance(v, list):
+        return [_to_python(x) for x in v]
+    if isinstance(v, dict):
+        return {k: _to_python(val) for k, val in v.items()}
+    return v
+
+
+def _safe_equal(a, b):
+    """
+    Return a plain bool saying whether a and b are equal,
+    without ever producing a vector/Series.
+    """
+    # exact same object
+    if a is b:
+        return True
+    # normalize
+    a_n = _to_python(a)
+    b_n = _to_python(b)
+    # handle NaNs
+    if _is_nan(a_n) and _is_nan(b_n):
+        return True
+    # plain scalars/containers
+    try:
+        eq = a_n == b_n
+        if isinstance(eq, bool):
+            return eq
+    except Exception:
+        pass
+    # fallback: compare stable JSON-ish reprs
+    return _repr_safe(a_n) == _repr_safe(b_n)
+
+
+def _num_close(a: float, b: float, atol: float, rtol: float) -> bool:
+    # NaN==NaN treated equal
+    if _is_nan(a) and _is_nan(b):
+        return True
+    return abs(a - b) <= (atol + rtol * abs(b))
+
+
+def _to_python(v: Any) -> Any:
+    """
+    Convert Polars value to a Python object. Struct -> dict, List -> list, scalars stay scalars.
+    Values coming from Series[i] / .to_list() are already Python, so this usually no-ops.
+    """
+    return v
+
+
+def _repr_safe(v: Any) -> str:
+    try:
+        return json.dumps(v, default=str, ensure_ascii=False)
+    except Exception:
+        return repr(v)
+
+
+def _iter_dict_keys(d: Dict[str, Any]) -> Iterable[str]:
+    # stable order, useful for predictable output
+    return sorted(d.keys())
+
+
+def _recursive_leaf_diffs(a, b, path, out, float_atol, float_rtol):
+    # treat None==None
+    if a is None and b is None:
+        return
+
+    # normalize early
+    a = _to_python(a)
+    b = _to_python(b)
+
+    # tuples -> lists
+    if isinstance(a, tuple):
+        a = list(a)
+    if isinstance(b, tuple):
+        b = list(b)
+
+    # numbers
+    if isinstance(a, (int, float)) and isinstance(b, (int, float)):
+        if _is_nan(a) and _is_nan(b):
+            return
+        # |a-b| <= atol + rtol*|b|
+        if abs(float(a) - float(b)) > (float_atol + float_rtol * abs(float(b))):
+            out.append(
+                {
+                    "path": path or "$",
+                    "left": a,
+                    "right": b,
+                    "abs_delta": abs(float(a) - float(b)),
+                }
+            )
+        return
+
+    # exact types for strings/bools
+    if type(a) is type(b) and isinstance(a, (str, bool)):
+        if not _safe_equal(a, b):
+            out.append({"path": path or "$", "left": a, "right": b, "abs_delta": None})
+        return
+
+    # lists
+    if isinstance(a, list) and isinstance(b, list):
+        if len(a) != len(b):
+            out.append(
+                {
+                    "path": f"{path or '$'}.length",
+                    "left": len(a),
+                    "right": len(b),
+                    "abs_delta": None,
+                }
+            )
+        n = min(len(a), len(b))
+        for i in range(n):
+            _recursive_leaf_diffs(
+                a[i], b[i], f"{path or '$'}[{i}]", out, float_atol, float_rtol
+            )
+        for i in range(n, len(a)):
+            out.append(
+                {
+                    "path": f"{path or '$'}[{i}]",
+                    "left": a[i],
+                    "right": None,
+                    "abs_delta": None,
+                }
+            )
+        for i in range(n, len(b)):
+            out.append(
+                {
+                    "path": f"{path or '$'}[{i}]",
+                    "left": None,
+                    "right": b[i],
+                    "abs_delta": None,
+                }
+            )
+        return
+
+    # dicts
+    if isinstance(a, dict) and isinstance(b, dict):
+        keys = sorted(set(a.keys()) | set(b.keys()))
+        for k in keys:
+            ak = a.get(k, None)
+            bk = b.get(k, None)
+            if k not in a:
+                out.append(
+                    {
+                        "path": f"{path or '$'}.{k}",
+                        "left": None,
+                        "right": bk,
+                        "abs_delta": None,
+                    }
+                )
+            elif k not in b:
+                out.append(
+                    {
+                        "path": f"{path or '$'}.{k}",
+                        "left": ak,
+                        "right": None,
+                        "abs_delta": None,
+                    }
+                )
+            else:
+                _recursive_leaf_diffs(
+                    ak, bk, f"{path or '$'}.{k}", out, float_atol, float_rtol
+                )
+        return
+
+    # fallback (type mismatch / opaque objects)
+    if not _safe_equal(a, b):
+        out.append({"path": path or "$", "left": a, "right": b, "abs_delta": None})
+
+
+def _boolean_mask_simple_equals(s1: pl.Series, s2: pl.Series) -> pl.Series:
+    both_null = s1.is_null() & s2.is_null()
+    return ((s1 == s2) | both_null).fill_null(True)
+
+
+def _boolean_mask_float_close(
+    s1: pl.Series, s2: pl.Series, atol: float, rtol: float
+) -> pl.Series:
+    both_null = s1.is_null() & s2.is_null()
+    both_nan = s1.is_nan() & s2.is_nan()
+    abs_diff = (s1 - s2).abs()
+    near = abs_diff <= (atol + rtol * s2.abs())
+    return (near | both_null | both_nan).fill_null(False)
+
+
+def _candidate_rows_for_nested(col_left: pl.Series, col_right: pl.Series) -> List[int]:
+    """
+    Cheap way to find rows that might differ for nested types:
+    compare JSON dumps of values. This is only a prefilter.
+    """
+    a = col_left.to_list()
+    b = col_right.to_list()
+    cand = []
+    for i, (x, y) in enumerate(zip(a, b)):
+        if _repr_safe(x) != _repr_safe(y):
+            cand.append(i)
+    return cand
+
+
+def recursive_diff_frames(
+    left: pl.DataFrame,
+    right: pl.DataFrame,
+    ignore: Optional[List[str]] = None,
+    float_atol: float = 0.0,
+    float_rtol: float = 0.0,
+    max_rows_per_column: int = 20,
+    max_leafs_per_row: int = 200,
+) -> Tuple[pl.DataFrame, pl.DataFrame]:
+    """
+    Deep diff DataFrames, recursing into List/Struct/dict-like values.
+    Returns (diff_summary, diff_leaves).
+    - diff_summary: [column, n_rows_with_diffs]
+    - diff_leaves: [column, row, path, left, right, abs_delta]
+        left/right are Python values (JSON-serializable where possible).
+    """
+    ignore = set(ignore or [])
+
+    # basic guards
+    if left.height != right.height:
+        raise ValueError(f"Row count differs: {left.height} vs {right.height}")
+
+    lcols = set(left.columns) - ignore
+    rcols = set(right.columns) - ignore
+    if lcols != rcols:
+        raise ValueError(
+            f"Column sets differ after ignoring.\nleft_only={sorted(lcols - rcols)}\nright_only={sorted(rcols - lcols)}"
+        )
+
+    cols = sorted(lcols)
+
+    summary_rows: List[Tuple[str, int]] = []
+    leaves_rows: List[Dict[str, Any]] = []
+
+    for c in cols:
+        s1, s2 = left[c], right[c]
+
+        # Fast path for simple, non-nested types with vectorized comparison
+        simple_dtype = (
+            s1.dtype in SIMPLE_CASTABLE_DTYPES and s2.dtype in SIMPLE_CASTABLE_DTYPES
+        )
+        is_floaty = s1.dtype in FLOAT_DTYPES and s2.dtype in FLOAT_DTYPES
+
+        if simple_dtype and not is_floaty:
+            equal_mask = _boolean_mask_simple_equals(s1, s2)
+            diff_idx = [i for i, ok in enumerate(equal_mask) if not ok]
+        elif simple_dtype and is_floaty:
+            close_mask = _boolean_mask_float_close(s1, s2, float_atol, float_rtol)
+            diff_idx = [i for i, ok in enumerate(close_mask) if not ok]
+        else:
+            # nested or exotic dtype → candidate rows via JSON compare
+            diff_idx = _candidate_rows_for_nested(s1, s2)
+
+        if not diff_idx:
+            continue
+
+        summary_rows.append((c, len(diff_idx)))
+
+        # limit how many rows per column we fully expand
+        for row in diff_idx[:max_rows_per_column]:
+            a = s1[row]
+            b = s2[row]
+            leaf_diffs: List[Dict[str, Any]] = []
+            _recursive_leaf_diffs(
+                a,
+                b,
+                path="",
+                out=leaf_diffs,
+                float_atol=float_atol,
+                float_rtol=float_rtol,
+            )
+
+            # If all leaf_diffs are only float-close (within tol), suppress (can happen for nested)
+            # The recursive function already filters by tolerance for numbers, so we keep what's left.
+
+            # cap the number of leaf diffs to avoid explosion
+            for d in leaf_diffs[:max_leafs_per_row]:
+                left_norm = _repr_safe(_to_python(d["left"]))  # -> str
+                right_norm = _repr_safe(_to_python(d["right"]))  # -> str
+
+                abs_delta_val = d.get("abs_delta", None)
+                try:
+                    abs_delta_norm = (
+                        float(abs_delta_val) if abs_delta_val is not None else None
+                    )
+                except Exception:
+                    abs_delta_norm = None  # just in case something weird sneaks in
+
+                leaves_rows.append(
+                    {
+                        "column": str(c),
+                        "row": int(row),
+                        "path": str(d["path"] or "$"),
+                        "left": left_norm,  # str
+                        "right": right_norm,  # str
+                        "abs_delta": abs_delta_norm,  # float or None
+                    }
+                )
+
+    diff_summary = (
+        pl.DataFrame(summary_rows, schema=["column", "n_rows_with_diffs"]).sort(
+            "n_rows_with_diffs", descending=True
+        )
+        if summary_rows
+        else pl.DataFrame(
+            {
+                "column": pl.Series([], pl.Utf8),
+                "n_rows_with_diffs": pl.Series([], pl.Int64),
+            }
+        )
+    )
+
+    # Build diff_leaves with stable schema; stringify complex left/right to avoid concat issues
+    if leaves_rows:
+        diff_leaves = pl.DataFrame(
+            {
+                "column": [r["column"] for r in leaves_rows],
+                "row": pl.Series([r["row"] for r in leaves_rows], dtype=pl.Int64),
+                "path": [r["path"] for r in leaves_rows],
+                "left": [r["left"] for r in leaves_rows],  # Utf8
+                "right": [r["right"] for r in leaves_rows],  # Utf8
+                "abs_delta": pl.Series(
+                    [r["abs_delta"] for r in leaves_rows], dtype=pl.Float64
+                ),
+            },
+            schema={
+                "column": pl.Utf8,
+                "row": pl.Int64,
+                "path": pl.Utf8,
+                "left": pl.Utf8,
+                "right": pl.Utf8,
+                "abs_delta": pl.Float64,
+            },
+        )
+    else:
+        diff_leaves = pl.DataFrame(
+            {
+                "column": [],
+                "row": [],
+                "path": [],
+                "left": [],
+                "right": [],
+                "abs_delta": [],
+            }
+        )
+
+    return diff_summary, diff_leaves
+
+    # FLOAT_DTYPES = {pl.Float32, pl.Float64}
+
+    # def diff_frames(
+    #     left: pl.DataFrame,
+    #     right: pl.DataFrame,
+    #     ignore: Optional[List[str]] = None,
+    #     float_atol: float = 0.0,
+    #     float_rtol: float = 0.0,
+    #     sample: int = 20,
+    # ) -> Tuple[pl.DataFrame, pl.DataFrame]:
+    #     ignore = set(ignore or [])
+
+    #     if left.height != right.height:
+    #         raise ValueError(f"Row count differs: {left.height} vs {right.height}")
+
+    #     lcols = set(left.columns) - ignore
+    #     rcols = set(right.columns) - ignore
+    #     if lcols != rcols:
+    #         raise ValueError(
+    #             f"Column sets differ after ignoring.\nleft_only={sorted(lcols - rcols)}\nright_only={sorted(rcols - lcols)}"
+    #         )
+
+    #     cols = sorted(lcols)
+    #     row_idx = pl.Series("row", range(left.height), dtype=pl.Int64)
+
+    #     def _float_diff_mask(s1: pl.Series, s2: pl.Series) -> pl.Series:
+    #         both_null = s1.is_null() & s2.is_null()
+    #         both_nan = s1.is_nan() & s2.is_nan()
+    #         abs_diff = (s1 - s2).abs()
+    #         near = abs_diff <= (float_atol + float_rtol * s2.abs())
+    #         return ~(near | both_null | both_nan)
+
+    #     def _nonfloat_diff_mask(s1: pl.Series, s2: pl.Series) -> pl.Series:
+    #         both_null = s1.is_null() & s2.is_null()
+    #         return ~((s1 == s2) | both_null).fill_null(True)
+
+    #     examples_frames = []
+    #     summary_rows = []
+
+    #     for c in cols:
+    #         s1, s2 = left[c], right[c]
+    #         if s1.dtype in FLOAT_DTYPES and s2.dtype in FLOAT_DTYPES:
+    #             diff_mask = _float_diff_mask(s1, s2)
+    #             abs_delta = (s1 - s2).abs()
+    #         else:
+    #             diff_mask = _nonfloat_diff_mask(s1, s2)
+    #             abs_delta = None
+
+    #         diff_mask = diff_mask.cast(pl.Boolean)
+    #         n_diff = int(diff_mask.sum())
+    #         if n_diff == 0:
+    #             continue
+
+    #         summary_rows.append((c, n_diff))
+    #         k = min(sample, n_diff)
+
+    #         idx = row_idx.filter(diff_mask)[:k]
+
+    #         def to_utf8_safe(s: pl.Series) -> pl.Series:
+    #             # Fast path for simple scalars
+    #             if s.dtype in (
+    #                 pl.Int8,
+    #                 pl.Int16,
+    #                 pl.Int32,
+    #                 pl.Int64,
+    #                 pl.UInt8,
+    #                 pl.UInt16,
+    #                 pl.UInt32,
+    #                 pl.UInt64,
+    #                 pl.Float32,
+    #                 pl.Float64,
+    #                 pl.Utf8,
+    #                 pl.Boolean,
+    #                 pl.Date,
+    #                 pl.Datetime,
+    #                 pl.Time,
+    #                 pl.Duration,
+    #             ):
+    #                 return s.cast(pl.Utf8)
+    #             # Fallback for nested/complex types: List, Struct, etc.
+    #             return s.map_elements(
+    #                 lambda v: json.dumps(v, default=str, allow_nan=True),
+    #                 return_dtype=pl.Utf8,
+    #             )
+
+    #         ex_left = to_utf8_safe(s1.filter(diff_mask)[:k])
+    #         ex_right = to_utf8_safe(s2.filter(diff_mask)[:k])
+
+    #         ex = pl.DataFrame(
+    #             {
+    #                 "column": [c] * k,
+    #                 "row": idx,
+    #                 "left": ex_left,
+    #                 "right": ex_right,
+    #                 "dtype_left": [str(s1.dtype)] * k,
+    #                 "dtype_right": [str(s2.dtype)] * k,
+    #             }
+    #         )
+
+    #         # unify schema: always have abs_delta as Float64 (None for non-floats)
+    #         if abs_delta is not None:
+    #             ex = ex.with_columns(
+    #                 abs_delta.filter(diff_mask)[:k].cast(pl.Float64).alias("abs_delta")
+    #             )
+    #         else:
+    #             ex = ex.with_columns(pl.lit(None, dtype=pl.Float64).alias("abs_delta"))
+
+    #         examples_frames.append(ex)
+
+    #     diff_summary = (
+    #         pl.DataFrame(summary_rows, schema=["column", "n_different"]).sort(
+    #             "n_different", descending=True
+    #         )
+    #         if summary_rows
+    #         else pl.DataFrame(
+    #             {
+    #                 "column": pl.Series([], pl.Utf8),
+    #                 "n_different": pl.Series([], pl.Int64),
+    #             }
+    #         )
+    #     )
+    #     diff_examples = (
+    #         pl.concat(examples_frames) if examples_frames else pl.DataFrame()
+    #     )
+
+    #     return diff_summary, diff_examples
+
+    # # --- usage ---
+    # # diff_summary: one row per column with a count of differing rows
+    # # diff_examples: sample rows showing left/right values (and abs_delta for floats)
+    # summary, examples = diff_frames(
+    #     df1, df2, ignore=["timestamp"], float_atol=0.1, float_rtol=0.0, sample=25
+    # )
+
+    # print(summary)  # which columns differ and how much
+    # print(examples)  # sample mismatches with row indices

tools/load_results.py

@@ -3,10 +3,13 @@ from __future__ import annotations
 import json
 import pickle
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
 
 import numpy as np
 import polars as pl
+from polars.testing import assert_frame_equal
+
+from diff_df import recursive_diff_frames
 
 # ------------------------------------------------------------
 # Config you can tweak
@@ -247,6 +250,14 @@ def read_pickle(p: Path) -> Any:
 # ------------------------------------------------------------
 # Extractors for each model
 # ------------------------------------------------------------
+
+counting = {
+    (label_method, eval_method): []
+    for label_method in ["exp_based", "manual_based"]
+    for eval_method in ["roc", "prc"]
+}
+
+
 def rows_from_deepsad(data: dict, evals: List[str]) -> Dict[str, dict]:
     """
     deepsad under data['test'][eval], with extra per-eval arrays and AP present.
@@ -257,6 +268,8 @@ def rows_from_deepsad(data: dict, evals: List[str]) -> Dict[str, dict]:
         evd = test.get(ev)
         if not isinstance(evd, dict):
             continue
+        counting[(ev, "roc")].append(len(evd["roc"][0]))
+        counting[(ev, "prc")].append(len(evd["prc"][0]))
         out[ev] = {
             "auc": float(evd["auc"])
             if "auc" in evd and evd["auc"] is not None
@@ -585,12 +598,53 @@ def load_pretraining_results_dataframe(
 
 
 def main():
-    root = Path("/home/fedex/mt/results/done")
+    root = Path("/home/fedex/mt/results/copy")
-    df = load_results_dataframe(root, allow_cache=True)
+    df1 = load_results_dataframe(root, allow_cache=True)
-    print(df.shape, df.head())
+    exit(0)
 
-    df_pre = load_pretraining_results_dataframe(root, allow_cache=True)
+    retest_root = Path("/home/fedex/mt/results/copy/retest_nodrop")
-    print("pretraining:", df_pre.shape, df_pre.head())
+    df2 = load_results_dataframe(retest_root, allow_cache=False).drop("folder")
+
+    # exact schema & shape first (optional but helpful messages)
+    assert df1.shape == df2.shape, f"Shape differs: {df1.shape} vs {df2.shape}"
+    assert set(df1.columns) == set(df2.columns), (
+        f"Column sets differ: {df1.columns} vs {df2.columns}"
+    )
+
+    # allow small float diffs, ignore column order differences if you want
+    df1_sorted = df1.select(sorted(df1.columns))
+    df2_sorted = df2.select(sorted(df2.columns))
+
+    # Optionally pre-align/sort both frames by a stable key before diffing.
+    summary, leaves = recursive_diff_frames(
+        df1,
+        df2,
+        ignore=["timestamp"],  # columns to ignore
+        float_atol=0.1,  # absolute tolerance for floats
+        float_rtol=0.0,  # relative tolerance for floats
+        max_rows_per_column=20,  # limit expansion per column
+        max_leafs_per_row=200,  # cap leaves per row
+    )
+
+    pl.Config.set_fmt_table_cell_list_len(100)
+    pl.Config.set_tbl_rows(100)
+
+    print(summary)  # which columns differ & how many rows
+    print(leaves)  # exact nested paths + scalar diffs
+
+    # check_exact=False lets us use atol/rtol for floats
+    assert_frame_equal(
+        df1_sorted,
+        df2_sorted,
+        check_exact=False,
+        atol=0.1,  # absolute tolerance for floats
+        rtol=0.0,  # relative tolerance (set if you want % based)
+        check_dtypes=True,  # set False if you only care about values
+    )
+    print("DataFrames match within tolerance ✅")
+
+    # df_pre = load_pretraining_results_dataframe(root, allow_cache=True)
+    # print("pretraining:", df_pre.shape, df_pre.head())
 
 
 if __name__ == "__main__":

tools/plot_scripts/results_latent_space_comparisons.py

@@ -15,7 +15,7 @@ from load_results import load_results_dataframe
 # ----------------------------
 # Config
 # ----------------------------
-ROOT = Path("/home/fedex/mt/results/done")  # experiments root you pass to the loader
+ROOT = Path("/home/fedex/mt/results/copy")  # experiments root you pass to the loader
 OUTPUT_DIR = Path("/home/fedex/mt/plots/results_latent_space_comparisons")
 
 SEMI_LABELING_REGIMES = [(0, 0), (50, 10), (500, 100)]

tools/plot_scripts/results_semi_labels_comparison.py

@@ -17,7 +17,7 @@ from load_results import load_results_dataframe
 # ---------------------------------
 # Config
 # ---------------------------------
-ROOT = Path("/home/fedex/mt/results/done")
+ROOT = Path("/home/fedex/mt/results/copy")
 OUTPUT_DIR = Path("/home/fedex/mt/plots/results_semi_labels_comparison")
 
 LATENT_DIMS = [32, 64, 128, 256, 512, 768, 1024]