- Analyzer: Properly detects optional fields when None values present

- Analyzer: Filters None/unknown before type conflict detection
- Schema: Smart conflict resolution (None + single type → optional field)

Author: MS-32154

json_explorer/analyzer.py

@@ -4,7 +4,7 @@
 
 
 def detect_timestamp(value):
-    if not isinstance(value, str):
+    if not isinstance(value, str) or len(value) < 4:
         return False
     parsed = dateparser.parse(value)
     return parsed is not None
@@ -27,6 +27,7 @@ def analyze_node(node):
                     progress.update(task, advance=1)
                     children[key] = analyze_node(val)
                 return {"type": "object", "children": children}
+
             elif isinstance(node, list):
                 # Skip empty or null-only lists
                 non_empty_items = [
@@ -67,6 +68,11 @@ def analyze_node(node):
                     }
 
                 return {"type": "list", "child_type": "mixed"}
+
+            elif node is None:
+                # Explicitly handle None - mark as unknown but with a flag
+                return {"type": "unknown", "is_none": True}
+
             else:
                 if isinstance(node, str):
                     if detect_timestamp(node):
@@ -79,11 +85,20 @@ def analyze_node(node):
         def merge_object_summaries(summaries):
             key_structures = {}
             key_counts = Counter()
+            key_none_counts = Counter()
             total = len(summaries)
 
             for summary in summaries:
+                seen_keys = set()
+
                 for key, val in summary.get("children", {}).items():
                     key_counts[key] += 1
+                    seen_keys.add(key)
+
+                    # Track if this value is None/unknown
+                    if val.get("type") == "unknown":
+                        key_none_counts[key] += 1
+
                     if key not in key_structures:
                         key_structures[key] = []
                     key_structures[key].append(val)
@@ -93,19 +108,34 @@ def merge_object_summaries(summaries):
 
             for key, structures in key_structures.items():
                 count = key_counts[key]
-                optional = count < total
+                none_count = key_none_counts[key]
+
+                # Field is optional if:
+                # 1. Missing from some objects (count < total)
+                # 2. Has None in some objects (none_count > 0)
+                optional = (count < total) or (none_count > 0)
 
-                # Get unique types for this key
-                types = {s["type"] for s in structures}
+                # Filter out None/unknown types to find concrete types
+                concrete_structures = [
+                    s for s in structures if s.get("type") != "unknown"
+                ]
+
+                # If we have concrete types, use those; otherwise use all structures
+                working_structures = (
+                    concrete_structures if concrete_structures else structures
+                )
+
+                # Get unique types from working structures
+                types = {s["type"] for s in working_structures}
 
                 if len(types) == 1:
-                    # All structures have the same type
+                    # Single type (possibly with None values)
                     structure_type = list(types)[0]
 
                     if structure_type == "object":
                         # Recursively merge object structures
                         merged_children, child_conflicts = merge_object_summaries(
-                            structures
+                            working_structures
                         )
                         merged[key] = {
                             "type": "object",
@@ -117,21 +147,26 @@ def merge_object_summaries(summaries):
 
                     elif structure_type == "list":
                         # Merge list structures
-                        merged_list = merge_list_summaries(structures)
+                        merged_list = merge_list_summaries(working_structures)
                         merged[key] = {
                             "type": "list",
                             "optional": optional,
                             **{k: v for k, v in merged_list.items() if k != "type"},
                         }
 
                     else:
-                        # Primitive type
+                        # Primitive type (possibly with None)
                         merged[key] = {"type": structure_type, "optional": optional}
-                else:
-                    # Type conflict
+
+                elif len(types) > 1:
+                    # Multiple different types = real conflict
                     merged[key] = {"type": "conflict", "optional": optional}
                     conflicts[key] = list(types)
 
+                else:
+                    # Should not happen, but handle gracefully
+                    merged[key] = {"type": "unknown", "optional": optional}
+
             return merged, conflicts
 
         def merge_list_summaries(summaries):

json_explorer/codegen/core/schema.py

@@ -349,13 +349,53 @@ def _create_field_from_node(
         optional=optional,
     )
 
-    # Handle type conflicts
+    # Handle type conflicts with None/unknown handling
     if field_name in conflicts:
-        field_obj.type = FieldType.CONFLICT
-        field_obj.conflicting_types = [
-            map_analyzer_type(t) for t in conflicts[field_name]
-        ]
-        logger.warning(f"Type conflict in {parent_schema_name}.{field_name}")
+        conflict_types = conflicts[field_name]
+
+        if "unknown" in conflict_types:
+            # Filter out "unknown" to see what concrete types remain
+            concrete_types = [t for t in conflict_types if t != "unknown"]
+
+            if len(concrete_types) == 1:
+                # Single concrete type + None → Optional[ConcreteType]
+                # This is the most common case and should NOT be treated as a conflict
+                field_obj.type = map_analyzer_type(concrete_types[0])
+                field_obj.optional = True
+                logger.debug(
+                    f"Resolved None conflict in {parent_schema_name}.{field_name}: "
+                    f"using {concrete_types[0]} as optional"
+                )
+
+            elif len(concrete_types) > 1:
+                # Multiple concrete types + None → Real conflict
+                # Example: [{"value": None}, {"value": 1}, {"value": "text"}]
+                field_obj.type = FieldType.CONFLICT
+                field_obj.conflicting_types = [
+                    map_analyzer_type(t) for t in concrete_types
+                ]
+                field_obj.optional = True  # Can also be None
+                logger.warning(
+                    f"Type conflict in {parent_schema_name}.{field_name}: "
+                    f"{', '.join(concrete_types)} (plus None)"
+                )
+
+            else:
+                # Only "unknown" types → Keep as unknown but mark optional
+                field_obj.type = FieldType.UNKNOWN
+                field_obj.optional = True
+                logger.debug(
+                    f"Unknown type in {parent_schema_name}.{field_name} "
+                    f"(only None values found)"
+                )
+        else:
+            # Real conflict without None involved
+            field_obj.type = FieldType.CONFLICT
+            field_obj.conflicting_types = [map_analyzer_type(t) for t in conflict_types]
+            logger.warning(
+                f"Type conflict in {parent_schema_name}.{field_name}: "
+                f"{', '.join(conflict_types)}"
+            )
 
     # Handle specific field types
     elif field_type == FieldType.OBJECT: