From 661d2bbb69bfbbc39a81e69a2fe888d4358d64dc Mon Sep 17 00:00:00 2001 From: "henrydingliu@gmail.com" Date: Sun, 26 Jul 2026 01:49:44 +0000 Subject: [PATCH 01/10] fixing fixture expecting errors --- conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conftest.py b/conftest.py index 867c8e0bf..9f1fd175c 100644 --- a/conftest.py +++ b/conftest.py @@ -57,12 +57,14 @@ def _sample_fixture( """ # Set the backend to sparse for a sparse-only-run. + cl.options.set_option("AUTO_SPARSE", False) cl.options.set_option("ARRAY_BACKEND", "sparse" if request.param == "sparse_only_run" else "numpy") # Load the sample data. tri = cl.load_sample(sample) # Apply a transformation if supplied, then yield the triangle to the test. yield transform(tri) if transform else tri # After the test, reset the backend to default numpy. + cl.options.set_option("AUTO_SPARSE", True) cl.options.set_option("ARRAY_BACKEND", "numpy") From 5f6fa5ac358aa8dae47c0c9d70a7a631ad6b94bb Mon Sep 17 00:00:00 2001 From: "henrydingliu@gmail.com" Date: Sun, 26 Jul 2026 02:01:05 +0000 Subject: [PATCH 02/10] fixing prism --- conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index 9f1fd175c..76cc3bc67 100644 --- a/conftest.py +++ b/conftest.py @@ -28,7 +28,7 @@ def pytest_generate_tests(metafunc): "prism_dense", ["normal_run", "sparse_only_run"], indirect=True ) if "prism" in metafunc.fixturenames: - metafunc.parametrize("prism", ["normal_run"], indirect=True) + metafunc.parametrize("prism", ["sparse_only_run"], indirect=True) if "xyz" in metafunc.fixturenames: metafunc.parametrize("xyz", ["normal_run", "sparse_only_run"], indirect=True) From 29e1be1ee5a11cf36c3c1fffa68489c55785ac9d Mon Sep 17 00:00:00 2001 From: "henrydingliu@gmail.com" Date: Sun, 26 Jul 2026 03:12:38 +0000 Subject: [PATCH 03/10] trying another approach --- conftest.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/conftest.py b/conftest.py index 76cc3bc67..407c29708 100644 --- a/conftest.py +++ b/conftest.py @@ -56,16 +56,12 @@ def _sample_fixture( """ - # Set the backend to sparse for a sparse-only-run. - cl.options.set_option("AUTO_SPARSE", False) - cl.options.set_option("ARRAY_BACKEND", "sparse" if request.param == "sparse_only_run" else "numpy") # Load the sample data. tri = cl.load_sample(sample) - # Apply a transformation if supplied, then yield the triangle to the test. - yield transform(tri) if transform else tri - # After the test, reset the backend to default numpy. - cl.options.set_option("AUTO_SPARSE", True) - cl.options.set_option("ARRAY_BACKEND", "numpy") + # Apply a transformation if supplied + tri = transform(tri) if transform else tri + # Set the backend to sparse for a sparse-only-run, then yield the triangle to the test. + yield tri.set_backend("sparse" if request.param == "sparse_only_run" else "numpy") @pytest.fixture From 2718584a8fd62dc8ce4bbcb08e015e53ec377de0 Mon Sep 17 00:00:00 2001 From: henrydingliu <106109320+henrydingliu@users.noreply.github.com> Date: Sun, 26 Jul 2026 10:09:19 -0700 Subject: [PATCH 04/10] Sparse test fixture (#16) * fixing fixture expecting errors * fixing prism * trying another approach * some fixes * more fixes * more fixes * fix for learning * more learning fix * restoring test still fails --- chainladder/core/tests/test_triangle.py | 7 +++---- chainladder/development/learning.py | 9 ++++++--- .../development/tests/test_development.py | 19 +++++++++++-------- conftest.py | 7 +++++++ 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/chainladder/core/tests/test_triangle.py b/chainladder/core/tests/test_triangle.py index 1a0857e79..e0ba3caa0 100644 --- a/chainladder/core/tests/test_triangle.py +++ b/chainladder/core/tests/test_triangle.py @@ -164,8 +164,7 @@ def test_printer(raa): def test_value_order(clrd): a = clrd[["CumPaidLoss", "BulkLoss"]] b = clrd[["BulkLoss", "CumPaidLoss"]] - xp = a.get_array_module() - xp.testing.assert_array_equal(a.values[:, -1], b.values[:, 0]) + assert a.iloc[:, -1] == b.iloc[:, 0] def test_trend(raa, atol): @@ -436,7 +435,7 @@ def test_groupby_agg_auto_sparse(prism: Triangle) -> None: assert result_default == result_no_sparse -def test_auto_sparse_disabled_returns_self(prism: Triangle) -> None: +def test_auto_sparse_disabled_returns_self(prism_convert: Triangle) -> None: """ When cl.options.AUTO_SPARSE is False, _auto_sparse() returns the triangle unchanged without switching backends. @@ -450,7 +449,7 @@ def test_auto_sparse_disabled_returns_self(prism: Triangle) -> None: ------- None """ - dense = prism.set_backend("numpy") + dense = prism_convert.set_backend("numpy") cl.options.set_option("AUTO_SPARSE", False) try: result = dense._auto_sparse() diff --git a/chainladder/development/learning.py b/chainladder/development/learning.py index 0d9ab2ac3..01b76fb35 100644 --- a/chainladder/development/learning.py +++ b/chainladder/development/learning.py @@ -250,11 +250,14 @@ def _prep_X_ml(self, X): return df def _prep_w_ml(self,X,sample_weight=None): - weight_base = (~np.isnan(X.values)).astype(float) + #scikit-learn requires a dense sample_weight + backend = "cupy" if X.array_backend == "cupy" else "numpy" + obj = X.set_backend(backend) + weight_base = (~np.isnan(obj.values)).astype(float) weight = weight_base.copy() - weight = weight * TriangleWeight(drop=self.drop,drop_valuation=self.drop_valuation).fit(X).w_.fillzero().values + weight = weight * TriangleWeight(drop=self.drop,drop_valuation=self.drop_valuation).fit(obj).w_.fillzero().values if sample_weight is not None: - weight = weight * sample_weight.values + weight = weight * sample_weight.set_backend(backend).values return weight.flatten()[weight_base.flatten()>0] def fit(self, X, y=None, sample_weight=None): diff --git a/chainladder/development/tests/test_development.py b/chainladder/development/tests/test_development.py index d92a05935..f8bc6687b 100644 --- a/chainladder/development/tests/test_development.py +++ b/chainladder/development/tests/test_development.py @@ -22,20 +22,23 @@ def __init__(self,dev): def fit(self, X, y: None = None, sample_weight: None = None): if hasattr(X,'age_to_age'): - super().fit(X.incr_to_cum().age_to_age) - xp = X.get_array_module() - indices = X.values.shape[0] - columns = X.values.shape[1] - origins = X.age_to_age.values.shape[2] - reg_x = X.incr_to_cum().values[...,:origins,:-1] - reg_y = X.incr_to_cum().values[...,:origins,1:] + #following precedent _set_fit_groups() from DevelopmentBase + backend = "numpy" if X.array_backend in ["sparse", "numpy"] else "cupy" + obj = X.set_backend(backend) + super().fit(obj.incr_to_cum().age_to_age) + xp = obj.get_array_module() + indices = obj.values.shape[0] + columns = obj.values.shape[1] + origins = obj.age_to_age.values.shape[2] + reg_x = obj.incr_to_cum().values[...,:origins,:-1] + reg_y = obj.incr_to_cum().values[...,:origins,1:] dev_len = reg_x.shape[3] average_param = self._cascade_param(dev_len, self.average, "volume") average_param = np.tile(average_param,(indices,columns,1,1)) params = cl.WeightedRegression(axis=2, thru_orig=True, xp=xp).fit( reg_x, reg_y, self.w_.values, average_param ) - self.ldf_ = self.dev._param_property(X, params.slope_.swapaxes(2, 3), 0) + self.ldf_ = self.dev._param_property(obj, params.slope_.swapaxes(2, 3), 0) return self def test_full_slice(genins): diff --git a/conftest.py b/conftest.py index 407c29708..be08e77e1 100644 --- a/conftest.py +++ b/conftest.py @@ -23,6 +23,10 @@ def pytest_generate_tests(metafunc): metafunc.parametrize("clrd", ["normal_run", "sparse_only_run"], indirect=True) if "genins" in metafunc.fixturenames: metafunc.parametrize("genins", ["normal_run", "sparse_only_run"], indirect=True) + if "prism_convert" in metafunc.fixturenames: + metafunc.parametrize( + "prism_convert", ["normal_run", "sparse_only_run"], indirect=True + ) if "prism_dense" in metafunc.fixturenames: metafunc.parametrize( "prism_dense", ["normal_run", "sparse_only_run"], indirect=True @@ -88,6 +92,9 @@ def genins(request): def prism(request): yield from _sample_fixture(request, "prism") +@pytest.fixture +def prism_convert(request): + yield from _sample_fixture(request, "prism", transform=lambda t: t.iloc[:5000]) @pytest.fixture def prism_dense(request): From e1c7f239c014d81ff35c1b4fb1137753eb1783d7 Mon Sep 17 00:00:00 2001 From: "henrydingliu@gmail.com" Date: Sun, 26 Jul 2026 17:23:14 +0000 Subject: [PATCH 05/10] further improving prism_convert --- conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conftest.py b/conftest.py index be08e77e1..20a5a1a57 100644 --- a/conftest.py +++ b/conftest.py @@ -25,7 +25,7 @@ def pytest_generate_tests(metafunc): metafunc.parametrize("genins", ["normal_run", "sparse_only_run"], indirect=True) if "prism_convert" in metafunc.fixturenames: metafunc.parametrize( - "prism_convert", ["normal_run", "sparse_only_run"], indirect=True + "prism_convert", ["sparse_only_run"], indirect=True ) if "prism_dense" in metafunc.fixturenames: metafunc.parametrize( @@ -94,7 +94,7 @@ def prism(request): @pytest.fixture def prism_convert(request): - yield from _sample_fixture(request, "prism", transform=lambda t: t.iloc[:5000]) + yield from _sample_fixture(request, "prism", transform=lambda t: t.iloc[:200]) @pytest.fixture def prism_dense(request): From 2d8435f075743c92b0b3e3dfc6d1fc1237676f84 Mon Sep 17 00:00:00 2001 From: "henrydingliu@gmail.com" Date: Mon, 27 Jul 2026 01:56:30 +0000 Subject: [PATCH 06/10] capecod & mack fixes --- chainladder/methods/mack.py | 16 ++++++++++------ chainladder/methods/tests/test_capecod.py | 2 +- chainladder/methods/tests/test_mack.py | 9 ++++----- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/chainladder/methods/mack.py b/chainladder/methods/mack.py index ea1a2b6b7..e6556df0f 100644 --- a/chainladder/methods/mack.py +++ b/chainladder/methods/mack.py @@ -335,6 +335,7 @@ def total_process_risk_(self): def _mack_recursion(self, est, X=None): obj = X.copy() + backend = X.array_backend xp = obj.get_array_module() risk_arr = xp.zeros((*X.shape[:3], 1)) if est == "total_parameter_risk_": @@ -343,14 +344,16 @@ def _mack_recursion(self, est, X=None): future_std_err = ( X._full_triangle_ - X[X.valuation < X.valuation_date] ).iloc[:, :, :, : X.shape[3]] * X.std_err_.values - t1_t = xp.nan_to_num(future_std_err.sum("origin").values) + #sum applies auto_sparse, so backend needs to be forced + t1_t = xp.nan_to_num(future_std_err.sum("origin").set_backend(backend).values) obj.odims = obj.odims[0:1] else: nans = xp.nan_to_num(X.nan_triangle[None, None]) nans = 1 - xp.concatenate((nans, xp.zeros((1, 1, X.shape[2], 1))), 3) full_tri = X._full_triangle_.values[..., : len(X.ddims)] if est == "parameter_risk_": - t1_t = xp.nan_to_num(full_tri) * obj.std_err_.values + #std_err_ is always numpy + t1_t = xp.nan_to_num(full_tri) * obj.std_err_.set_backend(backend).values else: t1_t = xp.nan_to_num(full_tri) * self._get_full_std_err_(X).values extend = X.ldf_.shape[-1] - X.shape[-1] + 1 @@ -490,11 +493,12 @@ def summary_(self): """ # This might be better as a dataframe obj = self.ultimate_.copy() + backend = obj.array_backend cols = ( - self.X_.latest_diagonal.values, - self.ibnr_.values, - self.ultimate_.values, - self.mack_std_err_.values[..., -1:], + self.X_.latest_diagonal.set_backend(backend).values, + self.ibnr_.set_backend(backend).values, + self.ultimate_.set_backend(backend).values, + self.mack_std_err_.set_backend(backend).values[..., -1:], ) obj.values = obj.get_array_module().concatenate(cols, 3) obj.ddims = ["Latest", "IBNR", "Ultimate", "Mack Std Err"] diff --git a/chainladder/methods/tests/test_capecod.py b/chainladder/methods/tests/test_capecod.py index 70becc879..55e05f96c 100644 --- a/chainladder/methods/tests/test_capecod.py +++ b/chainladder/methods/tests/test_capecod.py @@ -35,7 +35,7 @@ def test_groupby(clrd): def test_capecod_zero_tri(raa): premium = raa.latest_diagonal * 0 + 50000 - raa.loc[:,:,'1987',48] = 0 + raa.at['Total','values','1987',48] = 0 assert cl.CapeCod().fit(raa, sample_weight=premium).ultimate_.loc[:,:,'1987'].sum() > 0 diff --git a/chainladder/methods/tests/test_mack.py b/chainladder/methods/tests/test_mack.py index 86c3b69d7..828d9f190 100644 --- a/chainladder/methods/tests/test_mack.py +++ b/chainladder/methods/tests/test_mack.py @@ -15,11 +15,10 @@ def test_mack_to_triangle(): .summary_ ) -def test_mack_malformed(): - a = cl.load_sample('raa') - b = a.iloc[:, :, :-1] - x = cl.MackChainladder().fit(a) - y = cl.MackChainladder().fit(b) +def test_mack_malformed(raa): + raa_alt = raa.copy().iloc[:, :, :-1] + x = cl.MackChainladder().fit(raa) + y = cl.MackChainladder().fit(raa_alt) assert x.process_risk_.iloc[:,:,:-1] == y.process_risk_ def test_multi_triangle_mack(clrd,atol): From 09364a14e8e40af2af160ee2d54686891486ceab Mon Sep 17 00:00:00 2001 From: "henrydingliu@gmail.com" Date: Mon, 27 Jul 2026 02:53:12 +0000 Subject: [PATCH 07/10] fixing test_utilities and removing sparse.testing --- chainladder/core/tests/test_grain.py | 4 ++-- .../development/tests/test_development.py | 5 ++--- chainladder/utils/sparse.py | 6 ++--- chainladder/utils/tests/test_utilities.py | 22 +++++++++++-------- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/chainladder/core/tests/test_grain.py b/chainladder/core/tests/test_grain.py index 978aecdb8..9f19cfb47 100644 --- a/chainladder/core/tests/test_grain.py +++ b/chainladder/core/tests/test_grain.py @@ -6,8 +6,8 @@ def test_grain(qtr): + #this is a dense only test, as grain() follows auto_sparse actual = qtr.iloc[0, 0].grain("OYDY") - xp = actual.get_array_module() nan = xp.nan expected = xp.array( [ @@ -25,7 +25,7 @@ def test_grain(qtr): [13, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan], ] ) - xp.testing.assert_array_equal(actual.values[0, 0, :, :], expected) + np.testing.assert_array_equal(actual.values[0, 0, :, :], expected) def test_grain_returns_valid_tri(qtr): diff --git a/chainladder/development/tests/test_development.py b/chainladder/development/tests/test_development.py index f8bc6687b..d88f67a8e 100644 --- a/chainladder/development/tests/test_development.py +++ b/chainladder/development/tests/test_development.py @@ -822,9 +822,8 @@ def test_pipeline(clrd): def test_pct_reported_and_unreported(raa): dev = cl.Development().fit(raa) - xp = dev.cdf_.get_array_module() - xp.testing.assert_array_equal(dev.pct_reported_.values, (1 / dev.cdf_).values) - xp.testing.assert_array_equal(dev.pct_unreported_.values, (1 - 1 / dev.cdf_).values) + np.testing.assert_array_equal(dev.pct_reported_.values, (1 / dev.cdf_).values) + np.testing.assert_array_equal(dev.pct_unreported_.values, (1 - 1 / dev.cdf_).values) # reported + unreported percentages sum to one where defined total = (dev.pct_reported_ + dev.pct_unreported_).values assert np.allclose(np.nan_to_num(total), np.nan_to_num(dev.cdf_.values * 0 + 1)) diff --git a/chainladder/utils/sparse.py b/chainladder/utils/sparse.py index e3dee3063..c5a6313a1 100644 --- a/chainladder/utils/sparse.py +++ b/chainladder/utils/sparse.py @@ -15,13 +15,13 @@ sp.abs = np.abs -def nan_to_num(a): +def nan_to_num(a,nan=0.0): if type(a) in [int, float, np.int64, np.float64]: return np.nan_to_num(a) if hasattr(a, "fill_value"): a = a.copy() - a.data[np.isnan(a.data)] = 0.0 - return COO(coords=a.coords, data=a.data, fill_value=0.0, shape=a.shape) + a.data[np.isnan(a.data)] = nan + return COO(coords=a.coords, data=a.data, fill_value=nan, shape=a.shape) def ones(*args, **kwargs): diff --git a/chainladder/utils/tests/test_utilities.py b/chainladder/utils/tests/test_utilities.py index 9c4615b27..5a0b8a422 100644 --- a/chainladder/utils/tests/test_utilities.py +++ b/chainladder/utils/tests/test_utilities.py @@ -61,11 +61,11 @@ def from_sequence(seq): def test_triangle_json_io(clrd): xp = clrd.get_array_module() clrd2 = cl.read_json(clrd.to_json(), array_backend=clrd.array_backend) - xp.testing.assert_array_equal(clrd.values, clrd2.values) - xp.testing.assert_array_equal(clrd.kdims, clrd2.kdims) - xp.testing.assert_array_equal(clrd.vdims, clrd2.vdims) - xp.testing.assert_array_equal(clrd.odims, clrd2.odims) - xp.testing.assert_array_equal(clrd.ddims, clrd2.ddims) + assert clrd == clrd2 + assert np.all(clrd.kdims == clrd2.kdims) + assert np.all(clrd.vdims == clrd2.vdims) + assert np.all(clrd.odims == clrd2.odims) + assert np.all(clrd.ddims == clrd2.ddims) assert np.all(clrd.valuation == clrd2.valuation) @@ -222,17 +222,21 @@ def test_to_pickle_read_pickle(raa): os.remove(path) -def test_maximum(raa): +def test_maximum_minimum_1(raa): ult_vol = cl.Chainladder().fit( cl.Development(average="volume").fit_transform(raa) ).ultimate_ ult_sim = cl.Chainladder().fit( cl.Development(average="simple").fit_transform(raa) ).ultimate_ - high_side = cl.maximum(ult_vol, ult_sim) + high_side = maximum(ult_vol, ult_sim) + low_side = minimum(ult_vol, ult_sim) np.testing.assert_array_almost_equal( high_side.values, np.maximum(ult_vol.values, ult_sim.values) ) + np.testing.assert_array_almost_equal( + low_side.values, np.minimum(ult_vol.values, ult_sim.values) + ) def test_invalid_sample() -> None: @@ -596,7 +600,7 @@ def test_concat_axis1_duplicate_columns(raa: Triangle) -> None: cl.concat([raa, raa], axis=1) -def test_maximum(raa: Triangle) -> None: +def test_maximum_2(raa: Triangle) -> None: """ Run cl.maximum(raa, 5000) and check if each element in the resulting triangle is at least 5000. @@ -615,7 +619,7 @@ def test_maximum(raa: Triangle) -> None: assert xp.all(xp.nan_to_num(result.values, nan=5000) >= 5000) -def test_minimum(raa): +def test_minimum_2(raa): """ Run cl.minimum(raa, 5000) and check if each element in the resulting triangle is at most 5000. From 14e1f091c414b33b42d746f5958b0d61ac499b1a Mon Sep 17 00:00:00 2001 From: "henrydingliu@gmail.com" Date: Mon, 27 Jul 2026 03:03:07 +0000 Subject: [PATCH 08/10] more fixes --- chainladder/core/tests/test_grain.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chainladder/core/tests/test_grain.py b/chainladder/core/tests/test_grain.py index 9f19cfb47..701a07d17 100644 --- a/chainladder/core/tests/test_grain.py +++ b/chainladder/core/tests/test_grain.py @@ -8,8 +8,8 @@ def test_grain(qtr): #this is a dense only test, as grain() follows auto_sparse actual = qtr.iloc[0, 0].grain("OYDY") - nan = xp.nan - expected = xp.array( + nan = np.nan + expected = np.array( [ [44, 621, 950, 1020, 1070, 1069, 1089, 1094, 1097, 1099, 1100, 1100], [42, 541, 1052, 1169, 1238, 1249, 1266, 1269, 1296, 1300, 1300, nan], From 7c8e5febce4b34cecc85e4eca029258d5b7bfee3 Mon Sep 17 00:00:00 2001 From: "henrydingliu@gmail.com" Date: Mon, 27 Jul 2026 04:34:00 +0000 Subject: [PATCH 09/10] implementing nanquantile and nanmedian --- chainladder/core/tests/test_triangle.py | 4 +- chainladder/utils/sparse.py | 80 ++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 4 deletions(-) diff --git a/chainladder/core/tests/test_triangle.py b/chainladder/core/tests/test_triangle.py index e0ba3caa0..b43fe9b8c 100644 --- a/chainladder/core/tests/test_triangle.py +++ b/chainladder/core/tests/test_triangle.py @@ -179,9 +179,7 @@ def test_shift(qtr): def test_quantile_vs_median(clrd): xp = clrd.get_array_module() - xp.testing.assert_array_equal( - clrd.quantile(q=0.5)["CumPaidLoss"].values, clrd.median()["CumPaidLoss"].values - ) + assert clrd.quantile(q=0.5)["CumPaidLoss"] == clrd.median()["CumPaidLoss"] def test_base_minimum_exposure_triangle(raa): diff --git a/chainladder/utils/sparse.py b/chainladder/utils/sparse.py index c5a6313a1..5aca743d3 100644 --- a/chainladder/utils/sparse.py +++ b/chainladder/utils/sparse.py @@ -33,7 +33,83 @@ def nansum(a, axis=None, keepdims=None, *args, **kwargs): axis=axis, keepdims=keepdims, *args, **kwargs ) - +def nanquantile(a:COO, q:float, axis:int=0, keepdims:bool=False): + """ + mimics np.nanquantile + + Parameters + ---------- + x : COO + input sparse array. + q : float + quantiles in [0, 1]. + axis : int + Axis over which the quantile is applied. + keepdims : bool + Match NumPy keepdims behavior. + + Returns + ------- + out : COO + result cast back to COO for consistency. + """ + + # coordinates that survive the reduction + keep_axes = tuple(i for i in range(a.ndim) if i != axis) + keep_shape = tuple(a.shape[i] for i in keep_axes) + + if not keep_axes: + out = np.nanquantile(a.data, q) + if keepdims: + out = np.asarray(out).reshape( + tuple(1 for _ in range(a.ndim)) + ) + return COO(out) + + # map every stored value to an output location + keep_coords = a.coords[list(keep_axes)] + group_ids = np.ravel_multi_index(keep_coords, keep_shape) + + # sort by group + order = np.argsort(group_ids) + group_ids = group_ids[order] + values = a.data[order] + + out = np.full(np.prod(keep_shape), np.nan) + + starts = np.r_[0, np.flatnonzero(np.diff(group_ids)) + 1] + ends = np.r_[starts[1:], len(group_ids)] + + for start, end in zip(starts, ends): + gid = group_ids[start] + out[gid] = np.nanquantile(values[start:end], q) + + out = out.reshape(keep_shape) + + if keepdims: + out = np.expand_dims(out,axis) + + return COO(out) + +def nanmedian(a:COO, axis:int=0, keepdims:bool=False): + """ + mimics np.nanmean + + Parameters + ---------- + x : COO + input sparse array. + axis : int + Axis over which the median is applied. + keepdims : bool + Match NumPy keepdims behavior. + + Returns + ------- + out : COO + result cast back to COO for consistency. + """ + return nanquantile(a,0.5,axis,keepdims) def nanmean(a, axis=None, keepdims=None): n = nansum(a, axis=axis, keepdims=keepdims) @@ -85,3 +161,5 @@ def floor(x: COO) -> COO: COO.cumprod = cumprod sp.nanmean = nanmean sp.sum = COO.sum +sp.nanquantile = nanquantile +sp.nanmedian = nanmedian \ No newline at end of file From ac9e1b5381daf696a3abf4c2be76213ab835ab8a Mon Sep 17 00:00:00 2001 From: henrydingliu <106109320+henrydingliu@users.noreply.github.com> Date: Tue, 28 Jul 2026 22:00:08 -0700 Subject: [PATCH 10/10] Sparse fixture (#1166) * further improving prism_convert * capecod & mack fixes * fixing test_utilities and removing sparse.testing * more fixes * implementing nanquantile and nanmedian * improving comment, adding tests, fixing bugbot * fixing test * removing empty check --- chainladder/core/tests/test_grain.py | 2 +- chainladder/core/tests/test_triangle.py | 4 +-- .../development/tests/test_development.py | 2 +- chainladder/methods/mack.py | 1 + chainladder/utils/tests/test_sparse.py | 27 ++++++++++++++++++- 5 files changed, 31 insertions(+), 5 deletions(-) diff --git a/chainladder/core/tests/test_grain.py b/chainladder/core/tests/test_grain.py index 701a07d17..362c848ac 100644 --- a/chainladder/core/tests/test_grain.py +++ b/chainladder/core/tests/test_grain.py @@ -6,7 +6,7 @@ def test_grain(qtr): - #this is a dense only test, as grain() follows auto_sparse + #this test is dense only in practice, since grain() applies auto_sparse, which is True by default actual = qtr.iloc[0, 0].grain("OYDY") nan = np.nan expected = np.array( diff --git a/chainladder/core/tests/test_triangle.py b/chainladder/core/tests/test_triangle.py index b43fe9b8c..9b7514fa0 100644 --- a/chainladder/core/tests/test_triangle.py +++ b/chainladder/core/tests/test_triangle.py @@ -440,8 +440,8 @@ def test_auto_sparse_disabled_returns_self(prism_convert: Triangle) -> None: Parameters ---------- - prism : Triangle - The prism sample data set Triangle. + prism_convert : Triangle + The prism sample data set Triangle, reduced for test speed. Returns ------- diff --git a/chainladder/development/tests/test_development.py b/chainladder/development/tests/test_development.py index d88f67a8e..9adb39c77 100644 --- a/chainladder/development/tests/test_development.py +++ b/chainladder/development/tests/test_development.py @@ -22,7 +22,7 @@ def __init__(self,dev): def fit(self, X, y: None = None, sample_weight: None = None): if hasattr(X,'age_to_age'): - #following precedent _set_fit_groups() from DevelopmentBase + #following precedent set by _set_fit_groups() from DevelopmentBase to force triangle to be dense backend = "numpy" if X.array_backend in ["sparse", "numpy"] else "cupy" obj = X.set_backend(backend) super().fit(obj.incr_to_cum().age_to_age) diff --git a/chainladder/methods/mack.py b/chainladder/methods/mack.py index e6556df0f..4ded3f876 100644 --- a/chainladder/methods/mack.py +++ b/chainladder/methods/mack.py @@ -494,6 +494,7 @@ def summary_(self): # This might be better as a dataframe obj = self.ultimate_.copy() backend = obj.array_backend + # forcing these four triangles to the same backend cols = ( self.X_.latest_diagonal.set_backend(backend).values, self.ibnr_.set_backend(backend).values, diff --git a/chainladder/utils/tests/test_sparse.py b/chainladder/utils/tests/test_sparse.py index d92cdbf35..344c8850a 100644 --- a/chainladder/utils/tests/test_sparse.py +++ b/chainladder/utils/tests/test_sparse.py @@ -4,9 +4,11 @@ array, floor, COO, - where + where, + nanquantile ) +from sparse import all as sparse_all def test_array_from_list_default_fill_value() -> None: """ @@ -111,3 +113,26 @@ def test_floor_returns_copy() -> None: assert result is not a np.testing.assert_array_equal(result.todense(), [1.0, 2.0, -1.0]) np.testing.assert_array_equal(a.todense(), [1.2, 2.7, -0.3]) + +def test_1D_nanquantile() -> None: + """ + Checks that nanquantile performs in 1D special case. + + Returns + ------- + None + """ + a = COO(np.array([1,2,3,4])) + assert nanquantile(a,0.5) == 2.5 + assert sparse_all(nanquantile(a,0.5,keepdims = True) == COO(np.array([2.5]))) + +def test_keepdims_nanquantile() -> None: + """ + Checks that nanquantile keeps dimension when instructed. + + Returns + ------- + None + """ + a = COO(np.array([[1,2,3,4],[3,4,5,6]])) + assert sparse_all(nanquantile(a,0.5,keepdims = True) == COO(np.array([[2,3,4,5]]))) \ No newline at end of file