From 66c4baeabba39f063f4998917609c3705c3148ab Mon Sep 17 00:00:00 2001 From: Michael Skinner Date: Sun, 19 Apr 2020 18:38:14 -0700 Subject: [PATCH 1/2] alternative API for `multi_check` to allow parameterless checks, using *args --- bulwark/checks.py | 23 ++++++++++++++++------ bulwark/decorators.py | 27 +++++++++++++++++++++++++ tests/test_checks.py | 46 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 6 deletions(-) diff --git a/bulwark/checks.py b/bulwark/checks.py index 6c83484..f5f70be 100644 --- a/bulwark/checks.py +++ b/bulwark/checks.py @@ -7,6 +7,7 @@ - return the original, unaltered pd.DataFrame """ +import collections import warnings import numpy as np @@ -477,22 +478,32 @@ def is_same_as(df, df_to_compare, **kwargs): return df -def multi_check(df, checks, warn=False): +def multi_check(df, checks, *more_checks, warn=False): """Asserts that all checks pass. - Args: df (pd.DataFrame): Any pd.DataFrame. - checks (dict): Mapping of check functions to parameters for those check functions. + checks: A single function specification, as below. + This exists for backwards compatibility, and to ensure there's at least one check. + more_checks: Any number of check function specifications. + The simplest specification is a check function, which assumes no parameters. + Or it may be a Mapping of check functions to parameters for those functions. + In the event of duplicates, the last specification will win. warn (bool): Indicates whether an error should be raised or only a warning notification should be displayed. Default is to error. - Returns: Original `df`. - """ + + checks_map = {} + for check in [checks, *more_checks]: + if isinstance(check, collections.Mapping): + checks_map.update(check) + else: + checks_map[check] = {} + error_msgs = [] - for func, params in checks.items(): + for func, params in checks_map.items(): try: func(df, **params) except AssertionError as e: diff --git a/bulwark/decorators.py b/bulwark/decorators.py index 4718a05..66f8708 100644 --- a/bulwark/decorators.py +++ b/bulwark/decorators.py @@ -45,6 +45,33 @@ class decorator_name(BaseDecorator): setattr(this_module, decorator_name, decorator_factory(decorator_name, func)) +class MultiCheck: + """ + Notes: + - This code is purposefully located below the auto-generation of decorators, + so this overwrites the auto-generated MultiCheck. + - `MultiCheck`'s __init__ diverges from `BaseDecorator`, due to the use of *args. + + TODO: Work this into BaseDecorator? + + """ + + def __init__(self, checks, *more_checks, warn=False, enabled=True): + self.checks = checks + self.more_checks = more_checks + self.warn = warn + self.enabled = enabled + + def __call__(self, f): + @functools.wraps(f) + def decorated(*args, **kwargs): + df = f(*args, **kwargs) + if self.enabled: + ck.multi_check(df, self.checks, *self.more_checks, warn=self.warn) + return df + return decorated + + class CustomCheck: """ Notes: diff --git a/tests/test_checks.py b/tests/test_checks.py index c1707b6..b0c1690 100644 --- a/tests/test_checks.py +++ b/tests/test_checks.py @@ -515,6 +515,52 @@ def total_sum_not_equal(df, amt): # with pytest.raises(AssertionError): +def test_multi_check_varargs(): + df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) + result = ck.multi_check(df, ck.has_no_nans, warn=False) + tm.assert_frame_equal(df, result) + + result = ck.multi_check(df, + ck.has_no_nans, + {ck.is_shape: {"shape": (3, 2)}}, + warn=False) + tm.assert_frame_equal(df, result) + + result = dc.MultiCheck(ck.has_no_nans, + {ck.is_shape: {"shape": (3, 2)}}, + warn=False)(_noop)(df) + tm.assert_frame_equal(df, result) + + def total_sum_not_equal(df, amt): + if amt == 51: + raise AssertionError("Test") + return True + + result = ck.multi_check(df, + ck.has_no_nans, + {total_sum_not_equal: {"amt": 52}}, + warn=False) + tm.assert_frame_equal(df, result) + + result = dc.MultiCheck(ck.has_no_nans, + {total_sum_not_equal: {"amt": 52}}, + warn=False)(_noop)(df) + tm.assert_frame_equal(df, result) + + result = dc.MultiCheck(ck.has_no_nans, + {total_sum_not_equal: {"amt": 51}}, + {total_sum_not_equal: {"amt": 52}}, # last wins + warn=False)(_noop)(df) + tm.assert_frame_equal(df, result) + + with pytest.raises(AssertionError): + result = dc.MultiCheck(ck.has_no_nans, + {total_sum_not_equal: {"amt": 51}}, + warn=False)(_noop)(df) + + # with pytest.raises(AssertionError): + + def test_custom_check(): def f(df, length): if len(df) <= length: From b983765ce4e2f0f0f0bcc9ba48fa10527605b1b0 Mon Sep 17 00:00:00 2001 From: Michael Skinner Date: Sun, 19 Apr 2020 18:41:07 -0700 Subject: [PATCH 2/2] fix some whitespace oddness --- bulwark/checks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bulwark/checks.py b/bulwark/checks.py index f5f70be..53988aa 100644 --- a/bulwark/checks.py +++ b/bulwark/checks.py @@ -491,8 +491,10 @@ def multi_check(df, checks, *more_checks, warn=False): warn (bool): Indicates whether an error should be raised or only a warning notification should be displayed. Default is to error. + Returns: Original `df`. + """ checks_map = {}