feat: null classification

great-docs.yml

@@ -131,6 +131,8 @@ reference:
           members: false
         - name: DraftValidation
           members: false
+        - name: MissingSpec
+          members: true
 
     - title: Contracts and Pipelines
       desc: >
@@ -189,6 +191,10 @@ reference:
         - Validate.col_vals_expr
         - Validate.col_exists
         - Validate.col_pct_null
+        - Validate.col_pct_missing
+        - Validate.col_missing_coded
+        - Validate.col_missing_only_coded
+        - Validate.col_missing_consistent
         - Validate.rows_distinct
         - Validate.rows_complete
         - Validate.col_schema_match

pointblank/__init__.py

@@ -57,6 +57,7 @@
 from pointblank.generate.base import GeneratorConfig
 from pointblank.inspect import has_columns, has_rows
 from pointblank.integrations.otel import emit_otel
+from pointblank.missing import MissingSpec
 from pointblank.metadata import (
     ADaMDatasetTemplate,
     ADaMVariableSpec,
@@ -120,6 +121,7 @@
     "PipelineResult",
     "DataScan",
     "DraftValidation",
+    "MissingSpec",
     "col",
     "ref",
     "expr_col",

pointblank/_constants.py

@@ -21,10 +21,16 @@
     "within_spec": ["str"],
     "null": ["str", "numeric", "bool", "datetime", "duration"],
     "not_null": ["str", "numeric", "bool", "datetime", "duration"],
+    "missing_coded": ["str", "numeric", "bool", "datetime", "duration"],
+    "missing_only_coded": ["str", "numeric", "bool", "datetime", "duration"],
 }
 
 ASSERTION_TYPE_METHOD_MAP: dict[str, str] = {
     "col_pct_null": "pct_null",
+    "col_pct_missing": "pct_missing",
+    "col_missing_coded": "missing_coded",
+    "col_missing_only_coded": "missing_only_coded",
+    "col_missing_consistent": "missing_consistent",
     "col_vals_gt": "gt",
     "col_vals_lt": "lt",
     "col_vals_eq": "eq",
@@ -91,6 +97,8 @@
     "col_vals_decreasing",
     "col_vals_null",
     "col_vals_not_null",
+    "col_missing_coded",
+    "col_missing_only_coded",
     "col_vals_expr",
     "conjointly",
     "prompt",
@@ -640,6 +648,18 @@
             </g>
         </g>
     </g>
+</svg>""",
+    "col_pct_missing": """<?xml version="1.0" encoding="UTF-8"?>
+<svg width="67px" height="67px" viewBox="0 0 67 67" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" style="background: #FFFFFF;">
+    <title>pct_missing</title>
+    <g id="Icons" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="col_pct_missing" transform="translate(1.000000, 1.581717)" fill-rule="nonzero">
+            <path d="M55,0 C57.4852813,0 59.7352813,1.00735931 61.363961,2.63603897 C62.9926407,4.26471863 64,6.51471863 64,9 L64,9 L64,64 L9,64 C6.51471862,64 4.26471862,62.9926407 2.63603897,61.363961 C1.00735931,59.7352814 0,57.4852814 0,55 L0,55 L0,9 C0,6.51471863 1.00735931,4.26471863 2.63603897,2.63603897 C4.26471862,1.00735931 6.51471862,0 9,0 L9,0 L55,0 Z" id="rectangle" stroke="#000000" stroke-width="2" fill="#FFFFFF"></path>
+            <g id="percent" transform="translate(11.268508, 23.854373)" fill="#000000">
+                <path d="M1.89920553,17.2037988 C1.78720553,17.1237988 1.664,17.0296274 1.584,16.8936274 C1.504,16.7576274 1.464,16.6096274 1.464,16.4496274 C1.464,16.2416274 1.536,16.0336274 1.68,15.8256274 L12.24,0.489627434 C12.432,0.185627434 12.5942662,0 12.9462662,0 C13.1542662,0 13.472,0.089627434 13.68,0.201627434 C14.048,0.425627434 14.232,0.681627434 14.232,0.969627434 C14.232,1.14562743 14.16,1.34562743 14.016,1.56962743 L3.432,16.9776274 C3.176,17.2816274 2.88,17.4336274 2.544,17.4336274 C2.336,17.4336274 2.13920553,17.3477988 1.89920553,17.2037988 Z M3.864,7.47362743 C3.176,7.47362743 2.536,7.30562743 1.944,6.96962743 C1.352,6.63362743 0.88,6.18162743 0.528,5.61362743 C0.176,5.04562743 0,4.42562743 0,3.75362743 C0,3.08162743 0.172,2.46162743 0.516,1.89362743 C0.86,1.32562743 1.328,0.877627434 1.92,0.549627434 C2.512,0.221627434 3.16,0.057627434 3.864,0.057627434 C4.568,0.057627434 5.216,0.221627434 5.808,0.549627434 C6.4,0.877627434 6.864,1.32562743 7.2,1.89362743 C7.536,2.46162743 7.704,3.08162743 7.704,3.75362743 C7.704,4.42562743 7.532,5.04562743 7.188,5.61362743 C6.844,6.18162743 6.38,6.63362743 5.796,6.96962743 C5.212,7.30562743 4.568,7.47362743 3.864,7.47362743 Z M3.864,5.69762743 C4.408,5.69762743 4.852,5.51362743 5.196,5.14562743 C5.54,4.77762743 5.712,4.31362743 5.712,3.75362743 C5.712,3.17762743 5.54,2.70562743 5.196,2.33762743 C4.852,1.96962743 4.408,1.78562743 3.864,1.78562743 C3.304,1.78562743 2.848,1.96962743 2.496,2.33762743 C2.144,2.70562743 1.968,3.17762743 1.968,3.75362743 C1.968,4.31362743 2.144,4.77762743 2.496,5.14562743 C2.848,5.51362743 3.304,5.69762743 3.864,5.69762743 Z M11.952,17.3856274 C11.248,17.3856274 10.6,17.2176274 10.008,16.8816274 C9.416,16.5456274 8.948,16.0936274 8.604,15.5256274 C8.26,14.9576274 8.088,14.3376274 8.088,13.6656274 C8.088,12.9936274 8.26,12.3736274 8.604,11.8056274 C8.948,11.2376274 9.416,10.7896274 10.008,10.4616274 C10.6,10.1336274 11.248,9.96962743 11.952,9.96962743 C12.656,9.96962743 13.3,10.1336274 13.884,10.4616274 C14.468,10.7896274 14.928,11.2376274 15.264,11.8056274 C15.6,12.3736274 15.768,12.9936274 15.768,13.6656274 C15.768,14.3376274 15.596,14.9576274 15.252,15.5256274 C14.908,16.0936274 14.444,16.5456274 13.86,16.8816274 C13.276,17.2176274 12.64,17.3856274 11.952,17.3856274 Z M11.952,15.6096274 C12.48,15.6096274 12.92,15.4296274 13.272,15.0696274 C13.624,14.7096274 13.8,14.2416274 13.8,13.6656274 C13.8,13.1056274 13.624,12.6416274 13.272,12.2736274 C12.92,11.9056274 12.48,11.7216274 11.952,11.7216274 C11.392,11.7216274 10.932,11.9056274 10.572,12.2736274 C10.212,12.6416274 10.032,13.1056274 10.032,13.6656274 C10.032,14.2416274 10.212,14.7096274 10.572,15.0696274 C10.932,15.4296274 11.392,15.6096274 11.952,15.6096274 Z"></path>
+            </g>
+        </g>
+    </g>
 </svg>""",
     "col_vals_not_null": """<?xml version="1.0" encoding="UTF-8"?>
 <svg width="67px" height="67px" viewBox="0 0 67 67" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
@@ -652,6 +672,36 @@
             <polygon id="line_white" fill="#FFFFFF" transform="translate(34.899496, 32.153303) rotate(-320.000000) translate(-34.899496, -32.153303) " points="34.3994962 8.54160469 35.3994962 8.54160469 35.3994962 55.7650019 34.3994962 55.7650019"></polygon>
         </g>
     </g>
+</svg>""",
+    "col_missing_coded": """<?xml version="1.0" encoding="UTF-8"?>
+<svg width="67px" height="67px" viewBox="0 0 67 67" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <title>col_missing_coded</title>
+    <g id="Icons" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="col_missing_coded" transform="translate(0.000000, 0.551724)">
+            <path d="M56.712234,1 C59.1975153,1 61.4475153,2.00735931 63.076195,3.63603897 C64.7048747,5.26471863 65.712234,7.51471863 65.712234,10 L65.712234,10 L65.712234,65 L10.712234,65 C8.22695259,65 5.97695259,63.9926407 4.34827294,62.363961 C2.71959328,60.7352814 1.71223397,58.4852814 1.71223397,56 L1.71223397,56 L1.71223397,10 C1.71223397,7.51471863 2.71959328,5.26471863 4.34827294,3.63603897 C5.97695259,2.00735931 8.22695259,1 10.712234,1 L10.712234,1 Z" id="rectangle" stroke="#000000" stroke-width="2" fill="#FFFFFF"></path>
+            <path d="M40.6120805,47.037834 C37.4692348,47.037834 35.0126139,45.9348613 33.712234,44.0140597 C32.4118541,45.9348613 29.9552331,47.037834 26.8123883,47.037834 C22.6574397,47.037834 16.0646712,43.4437723 16.0646712,33.8021619 C16.0646712,29.3401361 17.4715879,18.962166 30.5035862,18.962166 C30.9454018,18.962166 31.3057481,19.3225124 31.3057481,19.7643279 L31.3057481,21.3686518 C31.3057481,21.8104674 30.9454018,22.1708138 30.5035862,22.1708138 C26.6400486,22.1708138 22.4819668,25.8118774 22.4819668,33.8021619 C22.4819668,37.5090277 23.7635456,43.0270243 27.2949384,43.0270243 C29.795428,43.0270243 31.224279,40.4231312 32.0985095,38.2861221 C30.5067194,35.6101596 29.7014243,33.1034035 29.7014243,30.8347892 C29.7014243,25.6238707 31.8603677,23.7751377 33.712234,23.7751377 C35.5641002,23.7751377 37.7230437,25.6238707 37.7230437,30.8347892 C37.7230437,33.1347383 36.9396828,35.5788255 35.3290916,38.2861221 C36.6294715,41.4321009 38.243196,43.0270243 40.1295295,43.0270243 C43.6609223,43.0270243 44.9425012,37.5090277 44.9425012,33.8021619 C44.9425012,25.8118774 40.7844193,22.1708138 36.9208817,22.1708138 C36.4759329,22.1708138 36.1187198,21.8104674 36.1187198,21.3686518 L36.1187198,19.7643279 C36.1187198,19.3225124 36.4759329,18.962166 36.9208817,18.962166 C49.9528801,18.962166 51.3597967,29.3401361 51.3597967,33.8021619 C51.3597967,43.4437723 44.7670282,47.037834 40.6120805,47.037834 Z" id="omega" fill="#000000" fill-rule="nonzero"></path>
+        </g>
+    </g>
+</svg>""",
+    "col_missing_only_coded": """<?xml version="1.0" encoding="UTF-8"?>
+<svg width="67px" height="67px" viewBox="0 0 67 67" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <title>col_missing_only_coded</title>
+    <g id="Icons" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="col_missing_only_coded" transform="translate(0.000000, 0.551724)">
+            <path d="M56.712234,1 C59.1975153,1 61.4475153,2.00735931 63.076195,3.63603897 C64.7048747,5.26471863 65.712234,7.51471863 65.712234,10 L65.712234,10 L65.712234,65 L10.712234,65 C8.22695259,65 5.97695259,63.9926407 4.34827294,62.363961 C2.71959328,60.7352814 1.71223397,58.4852814 1.71223397,56 L1.71223397,56 L1.71223397,10 C1.71223397,7.51471863 2.71959328,5.26471863 4.34827294,3.63603897 C5.97695259,2.00735931 8.22695259,1 10.712234,1 L10.712234,1 Z" id="rectangle" stroke="#000000" stroke-width="2" fill="#FFFFFF"></path>
+            <path d="M40.6120805,47.037834 C37.4692348,47.037834 35.0126139,45.9348613 33.712234,44.0140597 C32.4118541,45.9348613 29.9552331,47.037834 26.8123883,47.037834 C22.6574397,47.037834 16.0646712,43.4437723 16.0646712,33.8021619 C16.0646712,29.3401361 17.4715879,18.962166 30.5035862,18.962166 C30.9454018,18.962166 31.3057481,19.3225124 31.3057481,19.7643279 L31.3057481,21.3686518 C31.3057481,21.8104674 30.9454018,22.1708138 30.5035862,22.1708138 C26.6400486,22.1708138 22.4819668,25.8118774 22.4819668,33.8021619 C22.4819668,37.5090277 23.7635456,43.0270243 27.2949384,43.0270243 C29.795428,43.0270243 31.224279,40.4231312 32.0985095,38.2861221 C30.5067194,35.6101596 29.7014243,33.1034035 29.7014243,30.8347892 C29.7014243,25.6238707 31.8603677,23.7751377 33.712234,23.7751377 C35.5641002,23.7751377 37.7230437,25.6238707 37.7230437,30.8347892 C37.7230437,33.1347383 36.9396828,35.5788255 35.3290916,38.2861221 C36.6294715,41.4321009 38.243196,43.0270243 40.1295295,43.0270243 C43.6609223,43.0270243 44.9425012,37.5090277 44.9425012,33.8021619 C44.9425012,25.8118774 40.7844193,22.1708138 36.9208817,22.1708138 C36.4759329,22.1708138 36.1187198,21.8104674 36.1187198,21.3686518 L36.1187198,19.7643279 C36.1187198,19.3225124 36.4759329,18.962166 36.9208817,18.962166 C49.9528801,18.962166 51.3597967,29.3401361 51.3597967,33.8021619 C51.3597967,43.4437723 44.7670282,47.037834 40.6120805,47.037834 Z" id="omega" fill="#000000" fill-rule="nonzero"></path>
+        </g>
+    </g>
+</svg>""",
+    "col_missing_consistent": """<?xml version="1.0" encoding="UTF-8"?>
+<svg width="67px" height="67px" viewBox="0 0 67 67" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <title>col_missing_consistent</title>
+    <g id="Icons" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="col_missing_consistent" transform="translate(0.000000, 0.551724)">
+            <path d="M56.712234,1 C59.1975153,1 61.4475153,2.00735931 63.076195,3.63603897 C64.7048747,5.26471863 65.712234,7.51471863 65.712234,10 L65.712234,10 L65.712234,65 L10.712234,65 C8.22695259,65 5.97695259,63.9926407 4.34827294,62.363961 C2.71959328,60.7352814 1.71223397,58.4852814 1.71223397,56 L1.71223397,56 L1.71223397,10 C1.71223397,7.51471863 2.71959328,5.26471863 4.34827294,3.63603897 C5.97695259,2.00735931 8.22695259,1 10.712234,1 L10.712234,1 Z" id="rectangle" stroke="#000000" stroke-width="2" fill="#FFFFFF"></path>
+            <path d="M40.6120805,47.037834 C37.4692348,47.037834 35.0126139,45.9348613 33.712234,44.0140597 C32.4118541,45.9348613 29.9552331,47.037834 26.8123883,47.037834 C22.6574397,47.037834 16.0646712,43.4437723 16.0646712,33.8021619 C16.0646712,29.3401361 17.4715879,18.962166 30.5035862,18.962166 C30.9454018,18.962166 31.3057481,19.3225124 31.3057481,19.7643279 L31.3057481,21.3686518 C31.3057481,21.8104674 30.9454018,22.1708138 30.5035862,22.1708138 C26.6400486,22.1708138 22.4819668,25.8118774 22.4819668,33.8021619 C22.4819668,37.5090277 23.7635456,43.0270243 27.2949384,43.0270243 C29.795428,43.0270243 31.224279,40.4231312 32.0985095,38.2861221 C30.5067194,35.6101596 29.7014243,33.1034035 29.7014243,30.8347892 C29.7014243,25.6238707 31.8603677,23.7751377 33.712234,23.7751377 C35.5641002,23.7751377 37.7230437,25.6238707 37.7230437,30.8347892 C37.7230437,33.1347383 36.9396828,35.5788255 35.3290916,38.2861221 C36.6294715,41.4321009 38.243196,43.0270243 40.1295295,43.0270243 C43.6609223,43.0270243 44.9425012,37.5090277 44.9425012,33.8021619 C44.9425012,25.8118774 40.7844193,22.1708138 36.9208817,22.1708138 C36.4759329,22.1708138 36.1187198,21.8104674 36.1187198,21.3686518 L36.1187198,19.7643279 C36.1187198,19.3225124 36.4759329,18.962166 36.9208817,18.962166 C49.9528801,18.962166 51.3597967,29.3401361 51.3597967,33.8021619 C51.3597967,43.4437723 44.7670282,47.037834 40.6120805,47.037834 Z" id="omega" fill="#000000" fill-rule="nonzero"></path>
+        </g>
+    </g>
 </svg>""",
     "col_vals_regex": """<?xml version="1.0" encoding="UTF-8"?>
 <svg width="67px" height="67px" viewBox="0 0 67 67" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">