Sparse test

chainladder/core/tests/test_grain.py

@@ -6,10 +6,10 @@
 
 
 def test_grain(qtr):
+    #this test is dense only in practice, since grain() applies auto_sparse, which is True by default
     actual = qtr.iloc[0, 0].grain("OYDY")
-    xp = actual.get_array_module()
-    nan = xp.nan
-    expected = xp.array(
+    nan = np.nan
+    expected = np.array(
         [
             [44, 621, 950, 1020, 1070, 1069, 1089, 1094, 1097, 1099, 1100, 1100],
             [42, 541, 1052, 1169, 1238, 1249, 1266, 1269, 1296, 1300, 1300, nan],
@@ -25,7 +25,7 @@ def test_grain(qtr):
             [13, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan],
         ]
     )
-    xp.testing.assert_array_equal(actual.values[0, 0, :, :], expected)
+    np.testing.assert_array_equal(actual.values[0, 0, :, :], expected)
 
 
 def test_grain_returns_valid_tri(qtr):

chainladder/core/tests/test_triangle.py

@@ -164,8 +164,7 @@ def test_printer(raa):
 def test_value_order(clrd):
     a = clrd[["CumPaidLoss", "BulkLoss"]]
     b = clrd[["BulkLoss", "CumPaidLoss"]]
-    xp = a.get_array_module()
-    xp.testing.assert_array_equal(a.values[:, -1], b.values[:, 0])
+    assert a.iloc[:, -1] == b.iloc[:, 0]
 
 
 def test_trend(raa, atol):
@@ -180,9 +179,7 @@ def test_shift(qtr):
 
 def test_quantile_vs_median(clrd):
     xp = clrd.get_array_module()
-    xp.testing.assert_array_equal(
-        clrd.quantile(q=0.5)["CumPaidLoss"].values, clrd.median()["CumPaidLoss"].values
-    )
+    assert clrd.quantile(q=0.5)["CumPaidLoss"] == clrd.median()["CumPaidLoss"]
 
 
 def test_base_minimum_exposure_triangle(raa):
@@ -436,21 +433,21 @@ def test_groupby_agg_auto_sparse(prism: Triangle) -> None:
     assert result_default == result_no_sparse
 
 
-def test_auto_sparse_disabled_returns_self(prism: Triangle) -> None:
+def test_auto_sparse_disabled_returns_self(prism_convert: Triangle) -> None:
     """
     When cl.options.AUTO_SPARSE is False, _auto_sparse() returns the triangle
     unchanged without switching backends.
 
     Parameters
     ----------
-    prism : Triangle
-        The prism sample data set Triangle.
+    prism_convert : Triangle
+        The prism sample data set Triangle, reduced for test speed.
 
     Returns
     -------
     None
     """
-    dense = prism.set_backend("numpy")
+    dense = prism_convert.set_backend("numpy")
     cl.options.set_option("AUTO_SPARSE", False)
     try:
         result = dense._auto_sparse()

chainladder/development/learning.py

@@ -250,11 +250,14 @@ def _prep_X_ml(self, X):
         return df
 
     def _prep_w_ml(self,X,sample_weight=None):
-        weight_base = (~np.isnan(X.values)).astype(float)
+        #scikit-learn requires a dense sample_weight
+        backend = "cupy" if X.array_backend == "cupy" else "numpy"
+        obj = X.set_backend(backend)
+        weight_base = (~np.isnan(obj.values)).astype(float)
         weight = weight_base.copy()
-        weight = weight * TriangleWeight(drop=self.drop,drop_valuation=self.drop_valuation).fit(X).w_.fillzero().values
+        weight = weight * TriangleWeight(drop=self.drop,drop_valuation=self.drop_valuation).fit(obj).w_.fillzero().values
         if sample_weight is not None:
-            weight = weight * sample_weight.values 
+            weight = weight * sample_weight.set_backend(backend).values 
         return weight.flatten()[weight_base.flatten()>0]
 
     def fit(self, X, y=None, sample_weight=None):

chainladder/development/tests/test_development.py

@@ -22,20 +22,23 @@ def __init__(self,dev):
 
     def fit(self, X, y: None = None, sample_weight: None = None):
         if hasattr(X,'age_to_age'):
-            super().fit(X.incr_to_cum().age_to_age)
-            xp = X.get_array_module()
-            indices = X.values.shape[0]
-            columns = X.values.shape[1]
-            origins = X.age_to_age.values.shape[2]
-            reg_x = X.incr_to_cum().values[...,:origins,:-1]
-            reg_y = X.incr_to_cum().values[...,:origins,1:]
+            #following precedent set by _set_fit_groups() from DevelopmentBase to force triangle to be dense
+            backend = "numpy" if X.array_backend in ["sparse", "numpy"] else "cupy"
+            obj = X.set_backend(backend)
+            super().fit(obj.incr_to_cum().age_to_age)
+            xp = obj.get_array_module()
+            indices = obj.values.shape[0]
+            columns = obj.values.shape[1]
+            origins = obj.age_to_age.values.shape[2]
+            reg_x = obj.incr_to_cum().values[...,:origins,:-1]
+            reg_y = obj.incr_to_cum().values[...,:origins,1:]
             dev_len = reg_x.shape[3]
             average_param = self._cascade_param(dev_len, self.average, "volume")
             average_param = np.tile(average_param,(indices,columns,1,1))
             params = cl.WeightedRegression(axis=2, thru_orig=True, xp=xp).fit(
                 reg_x, reg_y, self.w_.values, average_param
             )
-            self.ldf_ = self.dev._param_property(X, params.slope_.swapaxes(2, 3), 0)
+            self.ldf_ = self.dev._param_property(obj, params.slope_.swapaxes(2, 3), 0)
         return self
 
 def test_full_slice(genins):
@@ -819,9 +822,8 @@ def test_pipeline(clrd):
 
 def test_pct_reported_and_unreported(raa):
     dev = cl.Development().fit(raa)
-    xp = dev.cdf_.get_array_module()
-    xp.testing.assert_array_equal(dev.pct_reported_.values, (1 / dev.cdf_).values)
-    xp.testing.assert_array_equal(dev.pct_unreported_.values, (1 - 1 / dev.cdf_).values)
+    np.testing.assert_array_equal(dev.pct_reported_.values, (1 / dev.cdf_).values)
+    np.testing.assert_array_equal(dev.pct_unreported_.values, (1 - 1 / dev.cdf_).values)
     # reported + unreported percentages sum to one where defined
     total = (dev.pct_reported_ + dev.pct_unreported_).values
     assert np.allclose(np.nan_to_num(total), np.nan_to_num(dev.cdf_.values * 0 + 1))

chainladder/methods/mack.py

@@ -335,6 +335,7 @@ def total_process_risk_(self):
 
     def _mack_recursion(self, est, X=None):
         obj = X.copy()
+        backend = X.array_backend
         xp = obj.get_array_module()
         risk_arr = xp.zeros((*X.shape[:3], 1))
         if est == "total_parameter_risk_":
@@ -343,14 +344,16 @@ def _mack_recursion(self, est, X=None):
             future_std_err = (
                 X._full_triangle_ - X[X.valuation < X.valuation_date]
             ).iloc[:, :, :, : X.shape[3]] * X.std_err_.values
-            t1_t = xp.nan_to_num(future_std_err.sum("origin").values)
+            #sum applies auto_sparse, so backend needs to be forced
+            t1_t = xp.nan_to_num(future_std_err.sum("origin").set_backend(backend).values)
             obj.odims = obj.odims[0:1]
         else:
             nans = xp.nan_to_num(X.nan_triangle[None, None])
             nans = 1 - xp.concatenate((nans, xp.zeros((1, 1, X.shape[2], 1))), 3)
             full_tri = X._full_triangle_.values[..., : len(X.ddims)]
             if est == "parameter_risk_":
-                t1_t = xp.nan_to_num(full_tri) * obj.std_err_.values
+                #std_err_ is always numpy
+                t1_t = xp.nan_to_num(full_tri) * obj.std_err_.set_backend(backend).values
             else:
                 t1_t = xp.nan_to_num(full_tri) * self._get_full_std_err_(X).values
         extend = X.ldf_.shape[-1] - X.shape[-1] + 1
@@ -490,11 +493,13 @@ def summary_(self):
         """
         # This might be better as a dataframe
         obj = self.ultimate_.copy()
+        backend = obj.array_backend
+        # forcing these four triangles to the same backend
         cols = (
-            self.X_.latest_diagonal.values,
-            self.ibnr_.values,
-            self.ultimate_.values,
-            self.mack_std_err_.values[..., -1:],
+            self.X_.latest_diagonal.set_backend(backend).values,
+            self.ibnr_.set_backend(backend).values,
+            self.ultimate_.set_backend(backend).values,
+            self.mack_std_err_.set_backend(backend).values[..., -1:],
         )
         obj.values = obj.get_array_module().concatenate(cols, 3)
         obj.ddims = ["Latest", "IBNR", "Ultimate", "Mack Std Err"]

chainladder/methods/tests/test_capecod.py

@@ -35,7 +35,7 @@ def test_groupby(clrd):
 
 def test_capecod_zero_tri(raa):
     premium = raa.latest_diagonal * 0 + 50000
-    raa.loc[:,:,'1987',48] = 0
+    raa.at['Total','values','1987',48] = 0
     assert cl.CapeCod().fit(raa, sample_weight=premium).ultimate_.loc[:,:,'1987'].sum() > 0
 
 

chainladder/methods/tests/test_mack.py

@@ -15,11 +15,10 @@ def test_mack_to_triangle():
         .summary_
     )
 
-def test_mack_malformed():
-    a  = cl.load_sample('raa')
-    b = a.iloc[:, :, :-1]
-    x = cl.MackChainladder().fit(a) 
-    y = cl.MackChainladder().fit(b)
+def test_mack_malformed(raa):
+    raa_alt = raa.copy().iloc[:, :, :-1]
+    x = cl.MackChainladder().fit(raa) 
+    y = cl.MackChainladder().fit(raa_alt)
     assert x.process_risk_.iloc[:,:,:-1] == y.process_risk_
 
 def test_multi_triangle_mack(clrd,atol):

chainladder/utils/sparse.py

@@ -15,13 +15,13 @@
 sp.abs = np.abs
 
 
-def nan_to_num(a):
+def nan_to_num(a,nan=0.0):
     if type(a) in [int, float, np.int64, np.float64]:
         return np.nan_to_num(a)
     if hasattr(a, "fill_value"):
         a = a.copy()
-        a.data[np.isnan(a.data)] = 0.0
-    return COO(coords=a.coords, data=a.data, fill_value=0.0, shape=a.shape)
+        a.data[np.isnan(a.data)] = nan
+    return COO(coords=a.coords, data=a.data, fill_value=nan, shape=a.shape)
 
 
 def ones(*args, **kwargs):
@@ -33,7 +33,83 @@ def nansum(a, axis=None, keepdims=None, *args, **kwargs):
         axis=axis, keepdims=keepdims, *args, **kwargs
     )
 
-
+def nanquantile(a:COO, q:float, axis:int=0, keepdims:bool=False):
+    """
+    mimics np.nanquantile
+
+    Parameters
+    ----------
+    x : COO
+        input sparse array.
+    q : float
+        quantiles in [0, 1].
+    axis : int
+        Axis over which the quantile is applied.
+    keepdims : bool
+        Match NumPy keepdims behavior.
+
+    Returns
+    -------
+    out : COO
+        result cast back to COO for consistency.
+    """
+
+    # coordinates that survive the reduction
+    keep_axes = tuple(i for i in range(a.ndim) if i != axis)
+    keep_shape = tuple(a.shape[i] for i in keep_axes)
+
+    if not keep_axes:
+        out = np.nanquantile(a.data, q)
+        if keepdims:
+            out = np.asarray(out).reshape(
+                tuple(1 for _ in range(a.ndim))
+            )
+        return COO(out)
+
+    # map every stored value to an output location
+    keep_coords = a.coords[list(keep_axes)]
+    group_ids = np.ravel_multi_index(keep_coords, keep_shape)
+
+    # sort by group
+    order = np.argsort(group_ids)
+    group_ids = group_ids[order]
+    values = a.data[order]
+
+    out = np.full(np.prod(keep_shape), np.nan)
+
+    starts = np.r_[0, np.flatnonzero(np.diff(group_ids)) + 1]
+    ends = np.r_[starts[1:], len(group_ids)]
+
+    for start, end in zip(starts, ends):
+        gid = group_ids[start]
+        out[gid] = np.nanquantile(values[start:end], q)
+
+    out = out.reshape(keep_shape)
+
+    if keepdims:
+        out = np.expand_dims(out,axis)
+
+    return COO(out)
+
+def nanmedian(a:COO, axis:int=0, keepdims:bool=False):
+    """
+    mimics np.nanmean
+
+    Parameters
+    ----------
+    x : COO
+        input sparse array.
+    axis : int
+        Axis over which the median is applied.
+    keepdims : bool
+        Match NumPy keepdims behavior.
+
+    Returns
+    -------
+    out : COO
+        result cast back to COO for consistency.
+    """
+    return nanquantile(a,0.5,axis,keepdims)
 
 def nanmean(a, axis=None, keepdims=None):
     n = nansum(a, axis=axis, keepdims=keepdims)
@@ -85,3 +161,5 @@ def floor(x: COO) -> COO:
 COO.cumprod = cumprod
 sp.nanmean = nanmean
 sp.sum = COO.sum
+sp.nanquantile = nanquantile
+sp.nanmedian = nanmedian

chainladder/utils/tests/test_sparse.py

@@ -4,9 +4,11 @@
     array,
     floor,
     COO,
-    where
+    where,
+    nanquantile
 )
 
+from sparse import all as sparse_all
 
 def test_array_from_list_default_fill_value() -> None:
     """
@@ -111,3 +113,26 @@ def test_floor_returns_copy() -> None:
     assert result is not a
     np.testing.assert_array_equal(result.todense(), [1.0, 2.0, -1.0])
     np.testing.assert_array_equal(a.todense(), [1.2, 2.7, -0.3])
+
+def test_1D_nanquantile() -> None:
+    """
+    Checks that nanquantile performs in 1D special case.
+
+    Returns
+    -------
+    None
+    """
+    a = COO(np.array([1,2,3,4]))
+    assert nanquantile(a,0.5) == 2.5
+    assert sparse_all(nanquantile(a,0.5,keepdims = True) == COO(np.array([2.5])))
+
+def test_keepdims_nanquantile() -> None:
+    """
+    Checks that nanquantile keeps dimension when instructed.
+
+    Returns
+    -------
+    None
+    """
+    a = COO(np.array([[1,2,3,4],[3,4,5,6]]))
+    assert sparse_all(nanquantile(a,0.5,keepdims = True) == COO(np.array([[2,3,4,5]])))