From 3be8f5ede5ebd3f7825904877f1f590202eb7457 Mon Sep 17 00:00:00 2001 From: Evan Erwee Date: Fri, 26 Jun 2026 11:14:43 -0400 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20add=20document-graph=20=E2=80=94=20?= =?UTF-8?q?structured=20data=20ETL=20for=20graph-enhanced=20GenAI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document-graph extends graphrag-toolkit with typed node support for structured formats (CSV, Excel, JSON, Parquet, XML). It complements lexical-graph by handling deterministic ETL while lexical-graph handles LLM-based extraction. Key features: - Schema providers (CSV, JSON, S3, Static, Glue) with auto-discovery - 20+ transformers (normalizers, field, document, filter, graph, truncators) - Cypher generation with tenant-scoped labels - Query engine for typed node retrieval - Multi-tenancy (same Neptune cluster, isolated by labels) - Hybrid integration: index structured data into lexical-graph for semantic search Coexists with lexical-graph in the same Neptune database: - lexical-graph: unstructured text → Source/Chunk/Topic/Entity - document-graph: structured data → typed nodes (__User__, __Account__, etc.) 59 tests passing. 100% docstring coverage. Sphinx docs included. --- document-graph/CHANGELOG.md | 58 ++ document-graph/README.md | 204 +++++++ document-graph/pyproject.toml | 52 ++ document-graph/src/document_graph/__init__.py | 34 ++ document-graph/src/document_graph/config.py | 6 + document-graph/src/document_graph/errors.py | 130 ++++ .../document_graph/graph_build/__init__.py | 3 + .../graph_build/build_result.py | 262 +++++++++ .../graph_build/constructors/__init__.py | 11 + .../constructors/constructors_plan.py | 93 +++ .../constructors_provider_base.py | 70 +++ .../constructors_provider_config.py | 51 ++ .../constructors_provider_factory.py | 32 + .../constructors_provider_registry.py | 74 +++ .../graph_build/constructors/core/__init__.py | 16 + .../constructors/core/edge_constructor.py | 47 ++ .../constructors/core/node_constructor.py | 46 ++ .../constructors/core/property_constructor.py | 39 ++ .../core/schema_driven_constructor.py | 42 ++ .../constructors/list_constructors.py | 72 +++ .../constructors/optimizations/__init__.py | 12 + .../optimizations/batch_constructor.py | 36 ++ .../deduplication_constructor.py | 41 ++ .../constructors/patterns/__init__.py | 12 + .../patterns/many_to_many_constructor.py | 17 + .../patterns/one_to_many_constructor.py | 17 + .../constructors/register_constructors.py | 33 ++ .../graph_build/cypher_builder.py | 47 ++ .../src/document_graph/ingest/__init__.py | 1 + .../document_graph/ingest/column/__init__.py | 15 + .../ingest/column/column_renamer.py | 41 ++ .../ingest/column/column_reorder.py | 53 ++ .../ingest/column/column_selector.py | 52 ++ .../ingest/column/column_type_converter.py | 48 ++ .../ingest/column/register_ingestors.py | 14 + .../document_graph/ingest/field/__init__.py | 14 + .../field/numeric_id_cleanup_ingestor.py | 40 ++ .../ingest/field/register_ingestors.py | 7 + .../ingest/ingestors_error_handler.py | 112 ++++ .../document_graph/ingest/ingestors_plan.py | 171 ++++++ .../ingest/ingestors_provider_base.py | 70 +++ .../ingest/ingestors_provider_config.py | 52 ++ .../ingest/ingestors_provider_factory.py | 34 ++ .../ingest/ingestors_provider_registry.py | 77 +++ .../document_graph/ingest/ingestors_schema.py | 223 +++++++ .../ingest/ingestors_validator.py | 246 ++++++++ .../document_graph/ingest/list_ingestors.py | 89 +++ .../src/document_graph/ingest/row/__init__.py | 10 + .../ingest/row/date_range_filter.py | 82 +++ .../ingest/row/register_ingestors.py | 9 + .../src/document_graph/ingest/row/skip_row.py | 71 +++ document-graph/src/document_graph/model.py | 66 +++ .../src/document_graph/model_elements.py | 163 ++++++ .../src/document_graph/pipeline/__init__.py | 1 + .../pipeline/extract/__init__.py | 1 + .../pipeline/extract/extract_provider_base.py | 77 +++ .../extract/extract_provider_config.py | 257 ++++++++ .../pipeline/extract/extract_provider_csv.py | 413 +++++++++++++ .../extract/extract_provider_excel.py | 165 ++++++ .../extract/extract_provider_factory.py | 77 +++ .../pipeline/extract/extract_provider_json.py | 235 ++++++++ .../extract/extract_provider_parquet.py | 160 +++++ .../extract/extract_provider_registry.py | 98 ++++ .../pipeline/extract/extraction_result.py | 14 + .../pipeline/extract/utils/__init__.py | 1 + .../pipeline/extract/utils/csv_fixer.py | 279 +++++++++ .../extract/utils/csv_line_normalizer.py | 108 ++++ .../pipeline/extract/utils/csv_to_json.py | 112 ++++ .../extract/utils/dataframe_reader.py | 13 + .../extract/utils/graph_element_utils.py | 57 ++ .../pipeline/extract/utils/parsing_utils.py | 48 ++ .../document_graph/pipeline/load/__init__.py | 1 + .../pipeline/load/load_context.py | 93 +++ .../pipeline/load/load_provider_base.py | 170 ++++++ .../pipeline/load/load_provider_config.py | 208 +++++++ .../pipeline/load/load_result.py | 189 ++++++ .../src/document_graph/pipeline_executor.py | 224 +++++++ .../src/document_graph/plugins/__init__.py | 1 + .../src/document_graph/query/__init__.py | 3 + .../src/document_graph/query/query_engine.py | 43 ++ .../src/document_graph/schema/__init__.py | 61 ++ .../schema/discovery/__init__.py | 41 ++ .../discovery/csv_discovery_provider.py | 144 +++++ .../discovery/excel_discovery_provider.py | 133 +++++ .../discovery/json_discovery_provider.py | 143 +++++ .../discovery/parquet_discovery_provider.py | 137 +++++ .../schema/discovery/schema_discovery_base.py | 114 ++++ .../discovery/schema_discovery_registry.py | 117 ++++ .../schema_discovery_registry_class.py | 66 +++ .../discovery/tabular_discovery_base.py | 85 +++ .../discovery/xml_discovery_provider.py | 171 ++++++ .../discovery/yaml_discovery_provider.py | 181 ++++++ .../schema/document_graph_schema.py | 65 ++ .../document_graph/schema/etl_schema_model.py | 316 ++++++++++ .../schema/providers/__init__.py | 100 ++++ .../schema/providers/csv_schema_provider.py | 157 +++++ .../schema/providers/excel_schema_provider.py | 161 +++++ .../schema/providers/file_schema_provider.py | 174 ++++++ .../schema/providers/glue_schema_provider.py | 246 ++++++++ .../schema/providers/json_schema_provider.py | 160 +++++ .../providers/parquet_schema_provider.py | 159 +++++ .../schema/providers/s3_schema_provider.py | 203 +++++++ .../schema/providers/schema_provider_base.py | 148 +++++ .../providers/schema_provider_config.py | 69 +++ .../schema_provider_config_aws_base.py | 63 ++ .../providers/schema_provider_factory.py | 193 ++++++ .../providers/schema_provider_registry.py | 169 ++++++ .../schema/providers/xml_schema_provider.py | 156 +++++ .../schema/providers/yaml_schema_provider.py | 157 +++++ .../src/document_graph/schema/schema_io.py | 112 ++++ .../schema/static_schema_provider.py | 134 +++++ .../src/document_graph/transform/__init__.py | 1 + .../document_transformers/__init__.py | 22 + .../document_transformers/json_to_rows.py | 78 +++ .../pii_redactor_provider.py | 76 +++ .../document_transformers/text_chunker.py | 101 ++++ .../transform/enrichers/__init__.py | 24 + .../enrichers/bedrock_enricher_plugin.py | 203 +++++++ .../enrichers/language_enricher_provider.py | 76 +++ .../enrichers/llm_enricher_plugin.py | 152 +++++ .../enrichers/prompts/evidence_summarize.txt | 8 + .../enrichers/prompts/evidence_tag.txt | 8 + .../enrichers/remediation_factory_provider.py | 128 ++++ .../transform/field_transformers/__init__.py | 42 ++ .../field_transformers/comma_flattener.py | 193 ++++++ .../comma_split_provider.py | 112 ++++ .../field_transformers/embedded_json.py | 87 +++ .../field_transformers/json_array_expander.py | 333 +++++++++++ .../json_array_flattener.py | 98 ++++ .../json_array_paired_flattener.py | 72 +++ .../field_transformers/json_flattener.py | 174 ++++++ .../json_value_flattener.py | 53 ++ .../field_transformers/normalize_timestamp.py | 86 +++ .../field_transformers/paired_flattener.py | 56 ++ .../regex_cleaner_provider.py | 98 ++++ .../field_transformers/standardize_enum.py | 104 ++++ .../field_transformers/uuid_generator.py | 22 + .../transform/filter_transformers/__init__.py | 22 + .../filter_transformers/column_pruner.py | 54 ++ .../filter_transformers/row_filter.py | 107 ++++ .../transform/graph_transformers/__init__.py | 22 + .../graph_transformers/infer_edges.py | 78 +++ .../graph_transformers/row_to_node.py | 72 +++ .../transform/normalizers/__init__.py | 32 + .../transform/normalizers/normalize_case.py | 41 ++ .../normalizers/normalize_case_provider.py | 71 +++ .../transform/normalizers/normalize_enum.py | 61 ++ .../normalizers/normalize_enum_provider.py | 76 +++ .../transform/normalizers/normalize_nulls.py | 54 ++ .../normalizers/normalize_nulls_provider.py | 73 +++ .../normalizers/normalize_spelling.py | 55 ++ .../normalize_spelling_provider.py | 83 +++ .../normalizers/normalize_timestamp.py | 56 ++ .../normalize_timestamp_provider.py | 76 +++ .../normalizers/normalize_whitespace.py | 42 ++ .../normalize_whitespace_provider.py | 69 +++ .../transform/registry/__init__.py | 20 + .../registry/transformer_provider_factory.py | 74 +++ .../registry/transformer_provider_registry.py | 75 +++ .../transform/transformation_plan.py | 91 +++ .../transform/transformer_provider_base.py | 114 ++++ .../transform/transformer_provider_config.py | 73 +++ .../transform/transformer_provider_factory.py | 119 ++++ .../transformer_provider_registry.py | 248 ++++++++ .../transform/truncators/__init__.py | 1 + .../truncators/field_count_truncator.py | 61 ++ .../transform/truncators/length_truncator.py | 65 ++ .../transform/truncators/token_truncator.py | 91 +++ document-graph/tests/conftest.py | 7 + document-graph/tests/test_document_graph.py | 553 ++++++++++++++++++ 170 files changed, 15843 insertions(+) create mode 100644 document-graph/CHANGELOG.md create mode 100644 document-graph/README.md create mode 100644 document-graph/pyproject.toml create mode 100644 document-graph/src/document_graph/__init__.py create mode 100644 document-graph/src/document_graph/config.py create mode 100644 document-graph/src/document_graph/errors.py create mode 100644 document-graph/src/document_graph/graph_build/__init__.py create mode 100644 document-graph/src/document_graph/graph_build/build_result.py create mode 100644 document-graph/src/document_graph/graph_build/constructors/__init__.py create mode 100644 document-graph/src/document_graph/graph_build/constructors/constructors_plan.py create mode 100644 document-graph/src/document_graph/graph_build/constructors/constructors_provider_base.py create mode 100644 document-graph/src/document_graph/graph_build/constructors/constructors_provider_config.py create mode 100644 document-graph/src/document_graph/graph_build/constructors/constructors_provider_factory.py create mode 100644 document-graph/src/document_graph/graph_build/constructors/constructors_provider_registry.py create mode 100644 document-graph/src/document_graph/graph_build/constructors/core/__init__.py create mode 100644 document-graph/src/document_graph/graph_build/constructors/core/edge_constructor.py create mode 100644 document-graph/src/document_graph/graph_build/constructors/core/node_constructor.py create mode 100644 document-graph/src/document_graph/graph_build/constructors/core/property_constructor.py create mode 100644 document-graph/src/document_graph/graph_build/constructors/core/schema_driven_constructor.py create mode 100644 document-graph/src/document_graph/graph_build/constructors/list_constructors.py create mode 100644 document-graph/src/document_graph/graph_build/constructors/optimizations/__init__.py create mode 100644 document-graph/src/document_graph/graph_build/constructors/optimizations/batch_constructor.py create mode 100644 document-graph/src/document_graph/graph_build/constructors/optimizations/deduplication_constructor.py create mode 100644 document-graph/src/document_graph/graph_build/constructors/patterns/__init__.py create mode 100644 document-graph/src/document_graph/graph_build/constructors/patterns/many_to_many_constructor.py create mode 100644 document-graph/src/document_graph/graph_build/constructors/patterns/one_to_many_constructor.py create mode 100644 document-graph/src/document_graph/graph_build/constructors/register_constructors.py create mode 100644 document-graph/src/document_graph/graph_build/cypher_builder.py create mode 100644 document-graph/src/document_graph/ingest/__init__.py create mode 100644 document-graph/src/document_graph/ingest/column/__init__.py create mode 100644 document-graph/src/document_graph/ingest/column/column_renamer.py create mode 100644 document-graph/src/document_graph/ingest/column/column_reorder.py create mode 100644 document-graph/src/document_graph/ingest/column/column_selector.py create mode 100644 document-graph/src/document_graph/ingest/column/column_type_converter.py create mode 100644 document-graph/src/document_graph/ingest/column/register_ingestors.py create mode 100644 document-graph/src/document_graph/ingest/field/__init__.py create mode 100644 document-graph/src/document_graph/ingest/field/numeric_id_cleanup_ingestor.py create mode 100644 document-graph/src/document_graph/ingest/field/register_ingestors.py create mode 100644 document-graph/src/document_graph/ingest/ingestors_error_handler.py create mode 100644 document-graph/src/document_graph/ingest/ingestors_plan.py create mode 100644 document-graph/src/document_graph/ingest/ingestors_provider_base.py create mode 100644 document-graph/src/document_graph/ingest/ingestors_provider_config.py create mode 100644 document-graph/src/document_graph/ingest/ingestors_provider_factory.py create mode 100644 document-graph/src/document_graph/ingest/ingestors_provider_registry.py create mode 100644 document-graph/src/document_graph/ingest/ingestors_schema.py create mode 100644 document-graph/src/document_graph/ingest/ingestors_validator.py create mode 100644 document-graph/src/document_graph/ingest/list_ingestors.py create mode 100644 document-graph/src/document_graph/ingest/row/__init__.py create mode 100644 document-graph/src/document_graph/ingest/row/date_range_filter.py create mode 100644 document-graph/src/document_graph/ingest/row/register_ingestors.py create mode 100644 document-graph/src/document_graph/ingest/row/skip_row.py create mode 100644 document-graph/src/document_graph/model.py create mode 100644 document-graph/src/document_graph/model_elements.py create mode 100644 document-graph/src/document_graph/pipeline/__init__.py create mode 100644 document-graph/src/document_graph/pipeline/extract/__init__.py create mode 100644 document-graph/src/document_graph/pipeline/extract/extract_provider_base.py create mode 100644 document-graph/src/document_graph/pipeline/extract/extract_provider_config.py create mode 100644 document-graph/src/document_graph/pipeline/extract/extract_provider_csv.py create mode 100644 document-graph/src/document_graph/pipeline/extract/extract_provider_excel.py create mode 100644 document-graph/src/document_graph/pipeline/extract/extract_provider_factory.py create mode 100644 document-graph/src/document_graph/pipeline/extract/extract_provider_json.py create mode 100644 document-graph/src/document_graph/pipeline/extract/extract_provider_parquet.py create mode 100644 document-graph/src/document_graph/pipeline/extract/extract_provider_registry.py create mode 100644 document-graph/src/document_graph/pipeline/extract/extraction_result.py create mode 100644 document-graph/src/document_graph/pipeline/extract/utils/__init__.py create mode 100644 document-graph/src/document_graph/pipeline/extract/utils/csv_fixer.py create mode 100644 document-graph/src/document_graph/pipeline/extract/utils/csv_line_normalizer.py create mode 100644 document-graph/src/document_graph/pipeline/extract/utils/csv_to_json.py create mode 100644 document-graph/src/document_graph/pipeline/extract/utils/dataframe_reader.py create mode 100644 document-graph/src/document_graph/pipeline/extract/utils/graph_element_utils.py create mode 100644 document-graph/src/document_graph/pipeline/extract/utils/parsing_utils.py create mode 100644 document-graph/src/document_graph/pipeline/load/__init__.py create mode 100644 document-graph/src/document_graph/pipeline/load/load_context.py create mode 100644 document-graph/src/document_graph/pipeline/load/load_provider_base.py create mode 100644 document-graph/src/document_graph/pipeline/load/load_provider_config.py create mode 100644 document-graph/src/document_graph/pipeline/load/load_result.py create mode 100644 document-graph/src/document_graph/pipeline_executor.py create mode 100644 document-graph/src/document_graph/plugins/__init__.py create mode 100644 document-graph/src/document_graph/query/__init__.py create mode 100644 document-graph/src/document_graph/query/query_engine.py create mode 100644 document-graph/src/document_graph/schema/__init__.py create mode 100644 document-graph/src/document_graph/schema/discovery/__init__.py create mode 100644 document-graph/src/document_graph/schema/discovery/csv_discovery_provider.py create mode 100644 document-graph/src/document_graph/schema/discovery/excel_discovery_provider.py create mode 100644 document-graph/src/document_graph/schema/discovery/json_discovery_provider.py create mode 100644 document-graph/src/document_graph/schema/discovery/parquet_discovery_provider.py create mode 100644 document-graph/src/document_graph/schema/discovery/schema_discovery_base.py create mode 100644 document-graph/src/document_graph/schema/discovery/schema_discovery_registry.py create mode 100644 document-graph/src/document_graph/schema/discovery/schema_discovery_registry_class.py create mode 100644 document-graph/src/document_graph/schema/discovery/tabular_discovery_base.py create mode 100644 document-graph/src/document_graph/schema/discovery/xml_discovery_provider.py create mode 100644 document-graph/src/document_graph/schema/discovery/yaml_discovery_provider.py create mode 100644 document-graph/src/document_graph/schema/document_graph_schema.py create mode 100644 document-graph/src/document_graph/schema/etl_schema_model.py create mode 100644 document-graph/src/document_graph/schema/providers/__init__.py create mode 100644 document-graph/src/document_graph/schema/providers/csv_schema_provider.py create mode 100644 document-graph/src/document_graph/schema/providers/excel_schema_provider.py create mode 100644 document-graph/src/document_graph/schema/providers/file_schema_provider.py create mode 100644 document-graph/src/document_graph/schema/providers/glue_schema_provider.py create mode 100644 document-graph/src/document_graph/schema/providers/json_schema_provider.py create mode 100644 document-graph/src/document_graph/schema/providers/parquet_schema_provider.py create mode 100644 document-graph/src/document_graph/schema/providers/s3_schema_provider.py create mode 100644 document-graph/src/document_graph/schema/providers/schema_provider_base.py create mode 100644 document-graph/src/document_graph/schema/providers/schema_provider_config.py create mode 100644 document-graph/src/document_graph/schema/providers/schema_provider_config_aws_base.py create mode 100644 document-graph/src/document_graph/schema/providers/schema_provider_factory.py create mode 100644 document-graph/src/document_graph/schema/providers/schema_provider_registry.py create mode 100644 document-graph/src/document_graph/schema/providers/xml_schema_provider.py create mode 100644 document-graph/src/document_graph/schema/providers/yaml_schema_provider.py create mode 100644 document-graph/src/document_graph/schema/schema_io.py create mode 100644 document-graph/src/document_graph/schema/static_schema_provider.py create mode 100644 document-graph/src/document_graph/transform/__init__.py create mode 100644 document-graph/src/document_graph/transform/document_transformers/__init__.py create mode 100644 document-graph/src/document_graph/transform/document_transformers/json_to_rows.py create mode 100644 document-graph/src/document_graph/transform/document_transformers/pii_redactor_provider.py create mode 100644 document-graph/src/document_graph/transform/document_transformers/text_chunker.py create mode 100644 document-graph/src/document_graph/transform/enrichers/__init__.py create mode 100644 document-graph/src/document_graph/transform/enrichers/bedrock_enricher_plugin.py create mode 100644 document-graph/src/document_graph/transform/enrichers/language_enricher_provider.py create mode 100644 document-graph/src/document_graph/transform/enrichers/llm_enricher_plugin.py create mode 100644 document-graph/src/document_graph/transform/enrichers/prompts/evidence_summarize.txt create mode 100644 document-graph/src/document_graph/transform/enrichers/prompts/evidence_tag.txt create mode 100644 document-graph/src/document_graph/transform/enrichers/remediation_factory_provider.py create mode 100644 document-graph/src/document_graph/transform/field_transformers/__init__.py create mode 100644 document-graph/src/document_graph/transform/field_transformers/comma_flattener.py create mode 100644 document-graph/src/document_graph/transform/field_transformers/comma_split_provider.py create mode 100644 document-graph/src/document_graph/transform/field_transformers/embedded_json.py create mode 100644 document-graph/src/document_graph/transform/field_transformers/json_array_expander.py create mode 100644 document-graph/src/document_graph/transform/field_transformers/json_array_flattener.py create mode 100644 document-graph/src/document_graph/transform/field_transformers/json_array_paired_flattener.py create mode 100644 document-graph/src/document_graph/transform/field_transformers/json_flattener.py create mode 100644 document-graph/src/document_graph/transform/field_transformers/json_value_flattener.py create mode 100644 document-graph/src/document_graph/transform/field_transformers/normalize_timestamp.py create mode 100644 document-graph/src/document_graph/transform/field_transformers/paired_flattener.py create mode 100644 document-graph/src/document_graph/transform/field_transformers/regex_cleaner_provider.py create mode 100644 document-graph/src/document_graph/transform/field_transformers/standardize_enum.py create mode 100644 document-graph/src/document_graph/transform/field_transformers/uuid_generator.py create mode 100644 document-graph/src/document_graph/transform/filter_transformers/__init__.py create mode 100644 document-graph/src/document_graph/transform/filter_transformers/column_pruner.py create mode 100644 document-graph/src/document_graph/transform/filter_transformers/row_filter.py create mode 100644 document-graph/src/document_graph/transform/graph_transformers/__init__.py create mode 100644 document-graph/src/document_graph/transform/graph_transformers/infer_edges.py create mode 100644 document-graph/src/document_graph/transform/graph_transformers/row_to_node.py create mode 100644 document-graph/src/document_graph/transform/normalizers/__init__.py create mode 100644 document-graph/src/document_graph/transform/normalizers/normalize_case.py create mode 100644 document-graph/src/document_graph/transform/normalizers/normalize_case_provider.py create mode 100644 document-graph/src/document_graph/transform/normalizers/normalize_enum.py create mode 100644 document-graph/src/document_graph/transform/normalizers/normalize_enum_provider.py create mode 100644 document-graph/src/document_graph/transform/normalizers/normalize_nulls.py create mode 100644 document-graph/src/document_graph/transform/normalizers/normalize_nulls_provider.py create mode 100644 document-graph/src/document_graph/transform/normalizers/normalize_spelling.py create mode 100644 document-graph/src/document_graph/transform/normalizers/normalize_spelling_provider.py create mode 100644 document-graph/src/document_graph/transform/normalizers/normalize_timestamp.py create mode 100644 document-graph/src/document_graph/transform/normalizers/normalize_timestamp_provider.py create mode 100644 document-graph/src/document_graph/transform/normalizers/normalize_whitespace.py create mode 100644 document-graph/src/document_graph/transform/normalizers/normalize_whitespace_provider.py create mode 100644 document-graph/src/document_graph/transform/registry/__init__.py create mode 100644 document-graph/src/document_graph/transform/registry/transformer_provider_factory.py create mode 100644 document-graph/src/document_graph/transform/registry/transformer_provider_registry.py create mode 100644 document-graph/src/document_graph/transform/transformation_plan.py create mode 100644 document-graph/src/document_graph/transform/transformer_provider_base.py create mode 100644 document-graph/src/document_graph/transform/transformer_provider_config.py create mode 100644 document-graph/src/document_graph/transform/transformer_provider_factory.py create mode 100644 document-graph/src/document_graph/transform/transformer_provider_registry.py create mode 100644 document-graph/src/document_graph/transform/truncators/__init__.py create mode 100644 document-graph/src/document_graph/transform/truncators/field_count_truncator.py create mode 100644 document-graph/src/document_graph/transform/truncators/length_truncator.py create mode 100644 document-graph/src/document_graph/transform/truncators/token_truncator.py create mode 100644 document-graph/tests/conftest.py create mode 100644 document-graph/tests/test_document_graph.py diff --git a/document-graph/CHANGELOG.md b/document-graph/CHANGELOG.md new file mode 100644 index 00000000..a87881dd --- /dev/null +++ b/document-graph/CHANGELOG.md @@ -0,0 +1,58 @@ +# Changelog + +## 3.0.4 (2026-06-26) + +### Added +- **100% docstring coverage** across all 163 source files and 141 classes +- **59 tests passing** — schema providers, transformers, ETL model, cypher edge cases, CSV discovery +- **Edge relationships** — AUTHORED_BY, CITES edge generation in hybrid pipeline +- **Lineage preservation** — source metadata structure for graphrag-toolkit (`source.sourceId` + `source.metadata`) +- **Cleanup notebook** (00-Cleanup) — remove old test tenants from Neptune + +### Fixed +- `S3SchemaProvider` — removed dead `DocumentGraphConfig` references, uses `boto3.Session()` directly +- `StaticSchemaProvider` — added missing `from_config()` classmethod +- `PIIRedactorProvider` — lazy import for `sanitary` (optional dependency) +- MockStore tests — `query()` → `execute_query()` to match actual API + +## 3.0.3 (2026-06-25) + +### Added +- **Sphinx docs** scaffolding +- **Hybrid notebooks** — 07a (data processing), 07b (lexical-graph query + correlation) +- **Mandela demo** — 08a (build), 08b (query) +- **Multi-tenant demo** — 09 (isolation proven) +- **Schema notebook** — 005 (providers, integration, validation) +- **Transformer notebook** — 006 (all 7 categories) + +### Changed +- `graphrag-lexical-graph` moved to optional dependency (avoids pip resolver conflicts) +- `push-to-sagemaker.sh` — S3 cleanup before upload, only latest wheel +- Neptune upgraded to 1.4.7.0 (graphrag-toolkit nested UNWIND compatibility) +- Batch writes enabled (Neptune 1.4.x supports it) + +### Fixed +- All corrupt notebooks (ai4triage removal artifacts) — JSON repaired +- Notebook numbering consolidated: 00–09 sequential + +## 3.0.0 (2026-06-24) + +### Breaking Changes +- **Removed `root_id`** — use `tenant_id` only (aligned with graphrag-toolkit) +- **Removed vendored storage** — uses graphrag-toolkit as dependency +- **Removed `tenant_id.py`, `versioning.py`, `root_id.py`** — sourced from graphrag-toolkit + +### Added +- `storage/` — thin extension layer (ReadOnlyGraphStore, factory extensions, complementary drivers) +- `graph_build/` — cypher_builder (node_to_cypher, edge_to_cypher, batch) +- `query/` — DocumentGraphQueryEngine +- Schema providers: CSV, JSON, S3, Static, File, Glue, Parquet, Excel, XML, YAML +- Schema discovery: auto-infer ETLSchema from data files +- 20+ transformers across 6 categories + +## 2.0.0 (2026-06-24) + +### Breaking Changes +- Pydantic V2 migration (all validators, ConfigDict) +- Removed visualisation, resilient_client, graph_id +- graphrag-toolkit as dependency (not vendored) diff --git a/document-graph/README.md b/document-graph/README.md new file mode 100644 index 00000000..06a45918 --- /dev/null +++ b/document-graph/README.md @@ -0,0 +1,204 @@ +# Document Graph + +> Structured data ETL for graph-enhanced GenAI applications. Extends [graphrag-toolkit](https://github.com/awslabs/graphrag-toolkit) with typed node support, schema providers, and hybrid search. + +## What it does + +| Input | Process | Output | +|-------|---------|--------| +| CSV, Excel, JSON, Parquet, XML | Deterministic ETL (no LLM) | Typed Neptune nodes + edges | + +**Hybrid with Lexical Graph:** Same Neptune cluster holds both structured (document-graph) and unstructured (lexical-graph) data. Query both with a single semantic search. + +## Features + +| Feature | Module | Description | +|---------|--------|-------------| +| **Schema Providers** | `schema.providers` | CSV, JSON, S3, Static, File, Glue — auto-discover or define schemas | +| **Schema Discovery** | `schema.discovery` | Infer ETL schema from data files (CSV, Excel, JSON, Parquet, XML, YAML) | +| **Transformers** | `transform` | Normalizers, field, document, filter, graph, truncators (20+ built-in) | +| **Graph Build** | `graph_build` | Cypher generation (node_to_cypher, edge_to_cypher) with tenant scoping | +| **Query Engine** | `query` | Typed node queries with tenant isolation | +| **Multi-Tenancy** | Labels | `__Type__tenant_id__` — complete data isolation per tenant | +| **Hybrid Search** | Integration | Lexical-graph indexes document-graph content for semantic retrieval | + +## Architecture + +``` +graphrag-toolkit (PyPI: graphrag-lexical-graph) +├── GraphStore, VectorStore, Factories +├── LexicalGraphIndex (unstructured → lexical graph) +└── LexicalGraphQueryEngine (semantic search) + +document-graph (this package) +├── schema/ → ETL schema model, providers (CSV/JSON/S3/Static), discovery +├── transform/ → 20+ transformers (normalizers, field, document, filter, graph, truncators) +├── graph_build/ → Cypher generation with tenant-scoped labels +├── query/ → DocumentGraphQueryEngine (typed node queries) +├── pipeline/ → Extract providers (CSV, Excel, JSON, Parquet) +├── ingest/ → Column selectors, row filters, renamers +└── storage/ → ReadOnlyGraphStore, factory extensions +``` + +## Infrastructure + +This project relies on [graphrag-toolkit](https://github.com/awslabs/graphrag-toolkit) for AWS infrastructure. Deploy the CloudFormation stack: + +> https://github.com/awslabs/graphrag-toolkit/tree/main/examples/lexical-graph/cloudformation-templates + +This provisions Neptune, OpenSearch Serverless, and a SageMaker notebook instance. All example notebooks (`examples/cloud/notebooks/`) must run on **SageMaker** within the provisioned VPC. + +## Install + +```bash +pip install document-graph # core (no heavy deps) +pip install document-graph[graphrag] # with lexical-graph integration +pip install graphrag-lexical-graph # for hybrid search +``` + +## Quick Start + +### 1. Write nodes to Neptune + +```python +from graphrag_toolkit.lexical_graph.storage import GraphStoreFactory +from document_graph.graph_build import node_to_cypher +from document_graph import Node + +gs = GraphStoreFactory.for_graph_store('neptune-db://endpoint:8182').__enter__() + +node = Node(id='u1', labels=['User'], properties={'name': 'Alice', 'role': 'admin'}) +cypher, params = node_to_cypher(node, tenant_id='my_app') +gs.execute_query(cypher, params) +``` + +### 2. Query nodes + +```python +from document_graph.query import DocumentGraphQueryEngine + +engine = DocumentGraphQueryEngine(gs, tenant_id='my_app') +users = engine.get_nodes('User') +admins = engine.find_by_property('User', 'role', 'admin') +``` + +### 3. Schema-driven pipeline + +```python +from document_graph.schema.providers.schema_provider_config import SchemaProviderConfig +from document_graph.schema.providers.csv_schema_provider import CSVSchemaProvider + +config = SchemaProviderConfig(type='csv', connection_config={'path': 'data/users.csv'}) +provider = CSVSchemaProvider(config) +schema = provider.load_schema() # Returns ETLSchema with discovered fields +``` + +### 4. Transform data + +```python +from document_graph.transform.transformer_provider_config import TransformerProviderConfig +from document_graph.transform.normalizers.normalize_whitespace_provider import NormalizeWhitespaceProvider +from document_graph.transform.graph_transformers.row_to_node import RowToNodeTransformer + +# Normalize +records = NormalizeWhitespaceProvider(TransformerProviderConfig(name='ws', args={})).transform(records) + +# Convert to nodes +nodes = RowToNodeTransformer(TransformerProviderConfig(name='r2n', args={'type': 'User'})).transform(records) +``` + +### 5. Hybrid search (document-graph + lexical-graph) + +```python +from graphrag_toolkit.lexical_graph import LexicalGraphIndex, LexicalGraphQueryEngine +from graphrag_toolkit.lexical_graph.storage import VectorStoreFactory +from llama_index.core.schema import Document + +# Index document-graph data into lexical-graph +docs = [Document(text=f'[User | {r["id"]} | my_app]\nname: {r["name"]}', metadata={...}) for r in records] +vector_store = VectorStoreFactory.for_vector_store('aoss://endpoint') +graph_index = LexicalGraphIndex(graph_store, vector_store) +graph_index.extract_and_build(docs, show_progress=True) + +# Semantic query +query_engine = LexicalGraphQueryEngine.for_traversal_based_search(graph_store, vector_store) +results = query_engine.retrieve('Who are the admin users?') +``` + +## Lexical-Graph Integration + +Document-graph and lexical-graph coexist in the same Neptune database: + +| | Lexical Graph | Document Graph | +|--|---------------|----------------| +| **Input** | Unstructured text (PDF, web) | Structured data (CSV, Excel, JSON) | +| **Extraction** | LLM-based (Claude) | Deterministic ETL (pandas) | +| **Graph Model** | Source → Chunk → Topic → Statement → Entity | Row → Typed Node + Edges | +| **Query** | Traversal-based semantic search | Cypher over typed nodes | +| **Labels** | `__Source__`, `__Chunk__`, `__Topic__` | `__User__tenant__`, `__Account__tenant__` | +| **Coexistence** | Same Neptune, different labels | Complete isolation | + +### Lineage + +When indexing document-graph data into lexical-graph, embed lineage in the text: + +```python +text = f'[{node_type} | {node_id} | {tenant}]\n...' +``` + +This survives the lexical-graph chunking pipeline and enables correlation back to the original nodes. + +## Multi-Tenancy + +All operations use tenant-scoped labels: `__Type__tenant_id__` + +```python +# Write to tenant A +node_to_cypher(node, tenant_id='acme_corp') # → MERGE (n:`__User__acme_corp__` ...) + +# Query tenant A only +engine = DocumentGraphQueryEngine(gs, tenant_id='acme_corp') +engine.get_nodes('User') # Only sees acme_corp's users +``` + +## Schema Providers + +| Provider | Source | Usage | +|----------|--------|-------| +| `CSVSchemaProvider` | CSV file | Auto-discover schema | +| `JSONSchemaProvider` | JSON file | Load pre-defined schema | +| `S3SchemaProvider` | S3 bucket | Shared schemas across environments | +| `StaticSchemaProvider` | Code | Programmatic definition | +| `SchemaProviderFactory` | Config dict | Unified creation | + +## Transformers + +| Category | Examples | Purpose | +|----------|----------|---------| +| Normalizers | whitespace, nulls, case, enum, timestamp | Clean & standardize | +| Field | json_flattener, uuid_gen, regex_clean | Reshape fields | +| Document | json_to_rows, text_chunker, pii_redactor | Split/transform docs | +| Filter | row_filter, column_pruner | Remove unwanted data | +| Graph | row_to_node, infer_edges | Create graph structures | +| Truncators | length, field_count, token | Limit data size | + +All follow: `TransformerProvider(config).transform(records) → records` + +## Testing + +```bash +pip install pytest +pytest tests/ -q # 59 tests +``` + +## Requirements + +- Python >= 3.10 +- `pydantic >= 2.0` +- `pandas >= 2.0` +- `boto3 >= 1.26` +- Optional: `graphrag-lexical-graph >= 3.18.0` (for hybrid search) + +## License + +Copyright © 2024-2026 Evan Erwee. All rights reserved. diff --git a/document-graph/pyproject.toml b/document-graph/pyproject.toml new file mode 100644 index 00000000..f625f62c --- /dev/null +++ b/document-graph/pyproject.toml @@ -0,0 +1,52 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/document_graph"] + +[project] +name = "document-graph" +version = "3.0.4" +description = "Graph-enhanced document processing toolkit for Neptune and Neo4j" +readme = "README.md" +requires-python = ">=3.10" +license = "Apache-2.0" +dependencies = [ + "pydantic>=2.0.0", + "networkx>=3.0", + "boto3>=1.26.0", + "pandas>=2.0.0", + "pluggy>=1.0.0", +] + +[project.optional-dependencies] +graphrag = [ + "graphrag-lexical-graph>=3.18.0", +] +neptune = [ + "boto3>=1.26.0", +] +neo4j = [ + "neo4j>=5.0.0", +] +visualization = [ + "graph-notebook>=4.0.0", +] +dev = [ + "pytest>=7.0", + "pytest-cov>=4.0", + "pytest-mock>=3.0", +] +all = [ + "document-graph[neptune,neo4j,visualization]", +] + +[project.urls] +Homepage = "https://github.com/awslabs/graphrag-toolkit/tree/main/document-graph" +Repository = "https://github.com/awslabs/graphrag-toolkit" +Issues = "https://github.com/awslabs/graphrag-toolkit/issues" + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] diff --git a/document-graph/src/document_graph/__init__.py b/document-graph/src/document_graph/__init__.py new file mode 100644 index 00000000..f4a41d24 --- /dev/null +++ b/document-graph/src/document_graph/__init__.py @@ -0,0 +1,34 @@ +"""Document Graph — structured data → typed graph nodes. + +A transformation library that extends graphrag-toolkit with typed node support +for structured formats (CSV, Excel, JSON, Parquet). + +Architecture: + Toolkit Reader → [ingest → transform → build] → Toolkit GraphStore + +Usage: + from document_graph import NodeModel, EdgeModel + from document_graph.transform.graph_transformers.row_to_node import RowToNode + from document_graph.graph_build.cypher_builder import CypherBuilder +""" + +__version__ = "3.0.3" +__author__ = "Evan Erwee" + +from .model import NodeModel, EdgeModel +from .model_elements import Node, Edge +from .pipeline_executor import PipelineExecutor +from .errors import ( + ModelError, + DatabaseConnectionError, + ConfigurationError, + QueryExecutionError, +) + +__all__ = [ + "PipelineExecutor", + "NodeModel", "EdgeModel", + "Node", "Edge", + "ModelError", "DatabaseConnectionError", + "ConfigurationError", "QueryExecutionError", +] diff --git a/document-graph/src/document_graph/config.py b/document-graph/src/document_graph/config.py new file mode 100644 index 00000000..cf0a12d8 --- /dev/null +++ b/document-graph/src/document_graph/config.py @@ -0,0 +1,6 @@ +"""Document Graph Config — minimal wrapper.""" +class DocumentGraphConfig: + """Minimal config. AWS operations go through graphrag-toolkit.""" + def __init__(self, **kwargs): + for k, v in kwargs.items(): + setattr(self, k, v) diff --git a/document-graph/src/document_graph/errors.py b/document-graph/src/document_graph/errors.py new file mode 100644 index 00000000..d35ddc41 --- /dev/null +++ b/document-graph/src/document_graph/errors.py @@ -0,0 +1,130 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Document Graph Exception Classes and Error Handling. + +This module defines a comprehensive set of custom exceptions for document graph operations, +providing specific error types for different failure scenarios. These exceptions enable +precise error handling and debugging throughout the document graph toolkit. + +Exception Hierarchy: + - ModelError: Data model validation and processing failures + - BatchJobError: Batch processing and ETL pipeline failures + - IndexError: Graph indexing and search index failures + - DatabaseConnectionError: Database connection and communication failures + - ConfigurationError: Configuration validation and parameter failures + - QueryExecutionError: Graph query execution and syntax failures + +Error Handling Patterns: + All exceptions follow consistent patterns for error reporting and debugging: + - Clear error messages with context information + - Preservation of original exception details when wrapping errors + - Integration with logging system for error tracking + - Support for error recovery and retry mechanisms + +Usage: + from document_graph.errors import ( + ModelError, DatabaseConnectionError, QueryExecutionError + ) + + try: + result = execute_graph_operation() + except DatabaseConnectionError as e: + logger.error(f"Database connection failed: {e}") + # Implement retry logic + except QueryExecutionError as e: + logger.error(f"Query failed: {e}") + # Handle query errors + except ModelError as e: + logger.error(f"Model validation failed: {e}") + # Handle data model issues +""" + +import logging + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +class ModelError(Exception): + """Raised when model validation or processing fails. + + This exception indicates issues with data model validation, + schema mismatches, or model processing errors. + + Examples: + >>> try: + ... validate_document_model(invalid_data) + ... except ModelError as e: + ... print(f"Model validation failed: {e}") + """ + pass + +class BatchJobError(Exception): + """Raised when batch job operations fail. + + This exception indicates failures in batch processing operations, + such as ETL pipeline errors or bulk data processing issues. + + Examples: + >>> try: + ... run_batch_job(job_config) + ... except BatchJobError as e: + ... print(f"Batch job failed: {e}") + """ + pass + +class IndexError(Exception): + """Raised when graph indexing operations fail. + + This exception indicates failures in graph database indexing, + search index creation, or index maintenance operations. + + Examples: + >>> try: + ... create_graph_index(index_config) + ... except IndexError as e: + ... print(f"Index creation failed: {e}") + """ + pass + +class DatabaseConnectionError(Exception): + """Raised when database connection fails. + + This exception indicates failures in establishing or maintaining + connections to graph databases ( Neptune, Neo4j). + + Examples: + >>> try: + ... connect_to_database(connection_config) + ... except DatabaseConnectionError as e: + ... print(f"Database connection failed: {e}") + """ + pass + +class ConfigurationError(Exception): + """Raised when configuration is invalid or missing. + + This exception indicates issues with provider configuration, + missing required parameters, or invalid configuration values. + + Examples: + >>> try: + ... validate_config(user_config) + ... except ConfigurationError as e: + ... print(f"Configuration error: {e}") + """ + pass + +class QueryExecutionError(Exception): + """Raised when graph query execution fails. + + This exception indicates failures in executing Cypher or Gremlin + queries against graph databases. + + Examples: + >>> try: + ... execute_query("MATCH (n) RETURN n LIMIT 10") + ... except QueryExecutionError as e: + ... print(f"Query execution failed: {e}") + """ + pass diff --git a/document-graph/src/document_graph/graph_build/__init__.py b/document-graph/src/document_graph/graph_build/__init__.py new file mode 100644 index 00000000..5849870b --- /dev/null +++ b/document-graph/src/document_graph/graph_build/__init__.py @@ -0,0 +1,3 @@ +"""Build stage — Node/Edge → Cypher → GraphStore.""" + +from .cypher_builder import node_to_cypher, edge_to_cypher, batch_nodes_to_cypher, batch_edges_to_cypher diff --git a/document-graph/src/document_graph/graph_build/build_result.py b/document-graph/src/document_graph/graph_build/build_result.py new file mode 100644 index 00000000..d7bf99ac --- /dev/null +++ b/document-graph/src/document_graph/graph_build/build_result.py @@ -0,0 +1,262 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Build Result module for document graph operations. + +This module defines the BuildResult class, which represents the complete +output of a build operation. It includes information about the build status, +output files, graph statistics, and metadata. It also provides factory methods +for creating success, empty, and failed results. +""" + +import logging +from typing import Optional, Dict, Any, List +from datetime import datetime, timezone +from pydantic import BaseModel, ConfigDict, computed_field +from graphrag_toolkit.lexical_graph import TenantId, to_tenant_id + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +class BuildResult(BaseModel): + """ + Represents the result of a build operation. + + The BuildResult class models the outcome of a build process, providing details + such as the document identifier, tenant information, build mode, and status. It + is designed to encapsulate various states including successful, empty, or + failed build results. This class includes helper methods to simplify the + creation of instances based on different build scenarios. + + Attributes: + document_id (str): The unique identifier of the document associated with the + build operation. + tenant_id (TenantId): The tenant information to which this build result + belongs. + build_mode (str): The mode in which the build operation was performed. Can + be one of: "graph", "code", "schema", or "sample". + build_time (str): The timestamp indicating when the build was created. + status (str): The status of the build operation, default is "success". Other + valid statuses include "empty_input" and "failed". + output_files (Optional[Dict[str, str]]): A dictionary containing the output + files resulting from the build operation. + graph_stats (Optional[Dict[str, Any]]): Optional metrics related to the + generated graph during the build. + metadata (Optional[Dict[str, Any]]): Additional metadata about the build + operation. + """ + + #Add this to allow custom TenantId types + model_config = ConfigDict(arbitrary_types_allowed=True) + + document_id: str + tenant_id: TenantId + build_mode: str # graph, code, schema, sample + build_time: str + status: str = "success" + output_files: Optional[Dict[str, str]] = None + graph_stats: Optional[Dict[str, Any]] = None + metadata: Optional[Dict[str, Any]] = None + tenant_id: Optional[str] = None + + @computed_field + @property + def scope_id(self) -> Optional[str]: + """Returns 'tenant_id.tenant_id' when both present, None otherwise.""" + if self.tenant_id is not None and self.tenant_id is not None: + return f"{self.tenant_id}.{self.tenant_id}" + return None + + def __init__(self, **data): + """ + [Brief one-line description of the function's purpose] + + [Detailed explanation of what the function does and how it works] + + Args: + [param_name] ([param_type]): [Description of parameter] + [param_name] ([param_type], optional): [Description of parameter]. Defaults to [default_value]. + + Returns: + [return_type]: [Description of return value] + + Raises: + [exception_type]: [Conditions under which exception is raised] + """ + if 'build_time' not in data: + data['build_time'] = datetime.now(timezone.utc).isoformat() + if 'tenant_id' in data: + data['tenant_id'] = to_tenant_id(data['tenant_id']) + super().__init__(**data) + logger.debug(f"Created BuildResult for document: {self.document_id} (mode: {self.build_mode})") + + def __str__(self) -> str: + return f"BuildResult(document_id={self.document_id}, mode={self.build_mode}, status={self.status})" + + def is_successful(self) -> bool: + """ + Determines whether the operation was successful based on the status attribute. + + Returns + ------- + bool + True if the status attribute equals "success", otherwise False. + """ + return self.status == "success" + + def is_empty(self) -> bool: + """ + Determines if the current instance represents an empty state. + + This method checks whether the instance is in an empty state based on its + `status` attribute or the lack of output files. + + Returns: + bool: True if the instance is in an empty state, False otherwise. + """ + return self.status == "empty_input" or not self.output_files + + @classmethod + def create_success( + cls, + document_id: str, + build_mode: str, + output_files: Dict[str, str], + tenant_id: Optional[str] = None, + graph_stats: Optional[Dict[str, Any]] = None, + metadata: Optional[Dict[str, Any]] = None, + tenant_id: Optional[str] = None + ) -> "BuildResult": + """ + Creates a successful build result object for a document build process. + + Summary: + This class method is responsible for generating a `BuildResult` instance + representing a successful document build operation. The method logs the process + and initializes the `BuildResult` object with relevant details such as document + ID, build mode, output files, and optional metadata or stats. + + Args: + document_id (str): The unique identifier of the document being built. + build_mode (str): The mode used during the build process (e.g., production, + draft). + output_files (Dict[str, str]): A mapping of output file types to their + corresponding paths or locations. + tenant_id (Optional[str]): The identifier of the tenant, if applicable. + graph_stats (Optional[Dict[str, Any]]): Statistics about graph data, if + available. + metadata (Optional[Dict[str, Any]]): Additional metadata associated with + the build. + tenant_id (Optional[str]): The root identifier for scoped operation. + + Returns: + BuildResult: A `BuildResult` instance initialized with the provided data. + + Raises: + None + """ + logger.info(f"Creating successful build result: {document_id} (mode: {build_mode})") + return cls( + document_id=document_id, + tenant_id=to_tenant_id(tenant_id), + build_mode=build_mode, + status="success", + output_files=output_files, + graph_stats=graph_stats or {}, + metadata=metadata or {}, + ) + + @classmethod + def create_empty( + cls, + document_id: str, + build_mode: str, + tenant_id: Optional[str] = None, + metadata: Optional[Dict[str, Any]] = None, + tenant_id: Optional[str] = None + ) -> "BuildResult": + """ + Creates an empty BuildResult object with a predefined status and optional metadata. + + This class method serves as a utility to instantiate a BuildResult object in situations + where no input is provided or where an empty initialization is required. It initializes + the object with the provided `document_id`, `build_mode`, `tenant_id`, and optional + metadata. The status of the created object is set to "empty_input" by default. + + Arguments: + document_id: str + Unique identifier of the document associated with the build result. + build_mode: str + Specifies the mode in which the build is executed. + tenant_id: Optional[str] + Identifier for the tenant, if applicable. If not provided, + a default tenant ID will be used. + metadata: Optional[Dict[str, Any]] + Additional information related to the build result, provided + as a dictionary. Defaults to an empty dictionary if not provided. + tenant_id: Optional[str] + The root identifier for scoped operation. + + Returns: + BuildResult + A new instance of BuildResult with the specified attributes and + a predefined status "empty_input". + + Additional Notes: + This method logs a warning message indicating that an empty build + result is being created for the specified document. + """ + logger.warning(f"Creating empty build result: {document_id}") + return cls( + document_id=document_id, + tenant_id=to_tenant_id(tenant_id), + build_mode=build_mode, + status="empty_input", + metadata=metadata or {}, + ) + + @classmethod + def create_failed( + cls, + document_id: str, + build_mode: str, + error_message: str, + tenant_id: Optional[str] = None, + metadata: Optional[Dict[str, Any]] = None, + tenant_id: Optional[str] = None + ) -> "BuildResult": + """ + Creates a failed build result with the provided details. + + This method initializes a failed build result instance for the specified document, + document creation/build mode, and error message. + + Parameters: + document_id: str + Identifier of the document for which the build has failed. + build_mode: str + Mode in which the build was attempted. + error_message: str + Detailed error message explaining the failure. + tenant_id: Optional[str] + Optional identifier of the tenant. Defaults to None. + metadata: Optional[Dict[str, Any]] + Optional additional metadata associated with the failure. Defaults to None. + tenant_id: Optional[str] + The root identifier for scoped operation. + + Returns: + BuildResult + An instance of the BuildResult class in a failed state with the provided information. + """ + logger.error(f"Creating failed build result: {document_id} - {error_message}") + metadata = metadata or {} + metadata["error_message"] = error_message + return cls( + document_id=document_id, + tenant_id=to_tenant_id(tenant_id), + build_mode=build_mode, + status="failed", + metadata=metadata, + ) diff --git a/document-graph/src/document_graph/graph_build/constructors/__init__.py b/document-graph/src/document_graph/graph_build/constructors/__init__.py new file mode 100644 index 00000000..30822315 --- /dev/null +++ b/document-graph/src/document_graph/graph_build/constructors/__init__.py @@ -0,0 +1,11 @@ +# Copyright (c) Evan Erwee. All rights reserved. +"""Graph build constructors — registry, factory, and base classes for graph construction.""" + +from document_graph.graph_build.constructors.constructors_provider_registry import ConstructorProviderRegistry +from document_graph.graph_build.constructors.constructors_provider_config import ConstructorProviderConfig +from document_graph.graph_build.constructors.constructors_provider_factory import ConstructorProviderFactory +from document_graph.graph_build.constructors.constructors_plan import ConstructorPlan + +# Auto-register all constructors +from document_graph.graph_build.constructors import register_constructors + diff --git a/document-graph/src/document_graph/graph_build/constructors/constructors_plan.py b/document-graph/src/document_graph/graph_build/constructors/constructors_plan.py new file mode 100644 index 00000000..06cafc8c --- /dev/null +++ b/document-graph/src/document_graph/graph_build/constructors/constructors_plan.py @@ -0,0 +1,93 @@ +# Copyright (c) Evan Erwee. All rights reserved. +"""Constructors plan — orchestrates execution of constructor providers on data.""" + +import logging +from typing import List, Tuple +import pandas as pd +from document_graph.graph_build.constructors.constructors_provider_config import ConstructorProviderConfig +from document_graph.graph_build.constructors.constructors_provider_factory import ConstructorProviderFactory +from document_graph.model_elements import Node, Edge + +# Use document-graph logging +logger = logging.getLogger(__name__) + +class ConstructorPlan: + """ + Manages a construction plan for executing multiple constructor providers. + + The `ConstructorPlan` class is responsible for handling a list of constructor + provider configurations. It executes all constructors in sequence, processes + an input DataFrame, and combines the resulting nodes and edges. Appropriate + logging and error handling are applied throughout the execution of the + constructor pipeline. + """ + + def __init__(self, configs: List[ConstructorProviderConfig]): + """ + Initializes an instance of the ConstructorPlan with a given list of constructor + configurations. Logs the number of configurations provided during + initialization. + + Attributes: + configs: List of ConstructorProviderConfig + A list of configuration objects provided for the ConstructorPlan + instance. + + Parameters: + configs: List[ConstructorProviderConfig] + List of constructor configuration objects passed during the + initialization of the class. + """ + self.configs = configs + logger.info(f"ConstructorPlan initialized with {len(configs)} constructors") + + def execute(self, data: pd.DataFrame) -> Tuple[List[Node], List[Edge]]: + """ + Executes a pipeline of constructors to generate nodes and edges from a given data. + + This method processes an input DataFrame and iteratively applies a series of constructors + defined in the instance configuration. Each constructor generates a set of nodes and edges + based on its specific implementation logic and the provided data. The pipeline continues + even if individual constructors fail, logging warnings or errors at each step. At the end + of the pipeline, the method aggregates all the nodes and edges created by the constructors + and returns them as a tuple. + + Raises: + Logger warnings and errors for an empty DataFrame or for failures during the execution + of individual constructors. + + Parameters: + data (pd.DataFrame): The input data to be processed. + + Returns: + Tuple[List[Node], List[Edge]]: A tuple containing two lists: + - A list of all nodes created by the constructors in the pipeline. + - A list of all edges created by the constructors in the pipeline. + """ + if data.empty: + logger.warning("Input DataFrame is empty") + return [], [] + + all_nodes = [] + all_edges = [] + + logger.info(f"Starting constructor pipeline: {len(data)} rows") + + for i, config in enumerate(self.configs): + try: + logger.info(f"Executing constructor {i+1}/{len(self.configs)}: {config.name} ({config.type})") + + constructor = ConstructorProviderFactory.create(config) + nodes, edges = constructor.construct(data) + + all_nodes.extend(nodes) + all_edges.extend(edges) + + logger.info(f" Completed: +{len(nodes)} nodes, +{len(edges)} edges") + + except Exception as e: + logger.error(f"Constructor {config.name} failed: {e}") + # Continue with other constructors + + logger.info(f"Constructor pipeline completed: {len(all_nodes)} total nodes, {len(all_edges)} total edges") + return all_nodes, all_edges \ No newline at end of file diff --git a/document-graph/src/document_graph/graph_build/constructors/constructors_provider_base.py b/document-graph/src/document_graph/graph_build/constructors/constructors_provider_base.py new file mode 100644 index 00000000..e91ec172 --- /dev/null +++ b/document-graph/src/document_graph/graph_build/constructors/constructors_provider_base.py @@ -0,0 +1,70 @@ +# Copyright (c) Evan Erwee. All rights reserved. +"""Constructors provider base — abstract base class for constructor providers.""" + +import logging +from abc import ABC, abstractmethod +from typing import List, Tuple + +import pandas as pd + +from document_graph.model_elements import Node, Edge + +# Use document-graph logging +logger = logging.getLogger(__name__) + +class ConstructorProvider(ABC): + """ + Provides an interface for constructing graph nodes and edges from data. + + This is an abstract base class designed to standardize the construction + of graph components (nodes and edges) from a given dataset. By defining + a common interface, it ensures a consistent approach and validation for + turning raw data into graph representations. Extend this class and provide + implementations for the abstract methods to use it effectively. + """ + + def __init__(self, config): + """Initialize constructor with configuration.""" + self.config = config + self.args = config.args + logger.debug(f"Initialized {self.__class__.__name__} with config: {config.type}") + + @abstractmethod + def construct(self, data: pd.DataFrame) -> Tuple[List[Node], List[Edge]]: + """ + Defines an abstract method for constructing graph nodes and edges from a given + data source. This method is intended to be implemented by subclasses to provide + specific logic for parsing data and generating corresponding graph structures. + + Parameters: + data (pd.DataFrame): A pandas DataFrame that serves as the input data + for constructing the graph structure. + + Returns: + Tuple[List[Node], List[Edge]]: A tuple containing a list of Node objects + and a list of Edge objects. These represent the constructed graph's nodes + and edges respectively. + + Raises: + None + """ + pass + + def validate_input(self, data: pd.DataFrame) -> bool: + """ + Validates the input DataFrame to ensure it is not empty. + + This method checks whether the provided DataFrame is valid by verifying that + it is neither None nor empty. It returns a boolean indicating the validity + of the DataFrame. If the DataFrame is invalid, a warning will be logged. + + Parameters: + data (pd.DataFrame): The DataFrame to be validated. + + Returns: + bool: True if the DataFrame is valid (not None and not empty), otherwise False. + """ + if data is None or data.empty: + logger.warning(f"{self.__class__.__name__}: Input DataFrame is empty") + return False + return True \ No newline at end of file diff --git a/document-graph/src/document_graph/graph_build/constructors/constructors_provider_config.py b/document-graph/src/document_graph/graph_build/constructors/constructors_provider_config.py new file mode 100644 index 00000000..c28e4a77 --- /dev/null +++ b/document-graph/src/document_graph/graph_build/constructors/constructors_provider_config.py @@ -0,0 +1,51 @@ +# Copyright (c) Evan Erwee. All rights reserved. +"""Constructors provider config — configuration dataclass for constructor providers.""" + +from dataclasses import dataclass, field +from typing import Dict, Any, Optional, List + +@dataclass +class ConstructorProviderConfig: + """ + Represents the configuration for a constructor provider. + + This class defines the parameters required to initialize and validate a constructor provider. It ensures + that mandatory fields are present and allows for optional customization of additional configurations. + + Attributes: + name: A string representing the name of the constructor. + type: A string representing the type of the constructor. + args: A dictionary containing additional arguments for the constructor configuration. Defaults to + an empty dictionary. + description: An optional string providing a description for the constructor. Defaults to None. + required_columns: An optional list of strings specifying the required columns for the constructor. + Defaults to None. + batch_size: An optional integer specifying the batch size. Defaults to None. + """ + name: str + type: str + args: Dict[str, Any] = field(default_factory=dict) + description: Optional[str] = None + required_columns: Optional[List[str]] = None + batch_size: Optional[int] = None + + def __post_init__(self): + """ + Validates and initializes post-construction logic for the object. + + Ensures that the `name` and `type` attributes are not empty upon initialization. + If the `description` attribute is not provided, it sets a default description + using the `type` and `name` attributes. + + Raises: + ValueError: If `name` is empty. + ValueError: If `type` is empty. + """ + if not self.name: + raise ValueError("Constructor name cannot be empty") + if not self.type: + raise ValueError("Constructor type cannot be empty") + + # Set default description if not provided + if not self.description: + self.description = f"{self.type} constructor: {self.name}" \ No newline at end of file diff --git a/document-graph/src/document_graph/graph_build/constructors/constructors_provider_factory.py b/document-graph/src/document_graph/graph_build/constructors/constructors_provider_factory.py new file mode 100644 index 00000000..87d03d6f --- /dev/null +++ b/document-graph/src/document_graph/graph_build/constructors/constructors_provider_factory.py @@ -0,0 +1,32 @@ +# Copyright (c) Evan Erwee. All rights reserved. +"""Constructors provider factory — creates constructor provider instances from config.""" + +from document_graph.graph_build.constructors.constructors_provider_registry import ConstructorProviderRegistry +from document_graph.graph_build.constructors.constructors_provider_config import ConstructorProviderConfig + +class ConstructorProviderFactory: + """ + Factory class for creating instances of constructor providers. + + This class provides a method to create a specific constructor provider + instance based on the configuration provided. It utilizes a registry to + look up the appropriate constructor provider based on the type specified + in the configuration. + """ + + @staticmethod + def create(config: ConstructorProviderConfig): + """ + This static method is responsible for creating an instance of a provider class based on the provided + configuration. It retrieves the appropriate class from the ConstructorProviderRegistry based on the + type specified in the configuration and initializes it using the given configuration. + + Parameters: + config (ConstructorProviderConfig): The configuration object used to determine the provider + type and to initialize the provider instance. + + Returns: + Any: An instance of the provider class corresponding to the type defined in the configuration. + """ + provider_class = ConstructorProviderRegistry.get(config.type) + return provider_class(config) \ No newline at end of file diff --git a/document-graph/src/document_graph/graph_build/constructors/constructors_provider_registry.py b/document-graph/src/document_graph/graph_build/constructors/constructors_provider_registry.py new file mode 100644 index 00000000..5004fda0 --- /dev/null +++ b/document-graph/src/document_graph/graph_build/constructors/constructors_provider_registry.py @@ -0,0 +1,74 @@ +# Copyright (c) Evan Erwee. All rights reserved. +"""Constructors provider registry — centralized registry of constructor provider classes.""" + +from typing import Dict, Type +from document_graph.graph_build.constructors.constructors_provider_base import ConstructorProvider + +class ConstructorProviderRegistry: + """ + Manages the registration and access of constructor providers. + + This class serves as a centralized registry for constructor provider classes, + allowing registration, retrieval, and listing of available providers. It is + commonly used to dynamically manage and access different provider classes + by their unique names. + """ + + _providers: Dict[str, Type[ConstructorProvider]] = {} + + @classmethod + def register(cls, name: str, provider_class: Type[ConstructorProvider]): + """ + Registers a provider with the given name. + + This class method allows registering a new provider by associating a name with + a provider class. + + Parameters: + name: str + The unique name used to identify the provider. + provider_class: Type[ConstructorProvider] + A class representing the provider to be registered, typically inheriting + from ConstructorProvider. + + Raises: + KeyError + If the provided name is already registered in the providers. + """ + cls._providers[name] = provider_class + + @classmethod + def get(cls, name: str) -> Type[ConstructorProvider]: + """ + Retrieves a constructor provider based on the provided name. + + This method accesses a registry of constructor providers and retrieves the provider + corresponding to the given name. If the name is not found within the registry, an + exception is raised. + + Args: + name (str): The name of the desired constructor provider. + + Returns: + Type[ConstructorProvider]: The constructor provider associated with the given name. + + Raises: + ValueError: If the specified name is not found in the registry of constructor providers. + """ + if name not in cls._providers: + raise ValueError(f"Unknown constructor provider: {name}") + return cls._providers[name] + + @classmethod + def list_providers(cls) -> list: + """ + Provides functionality to retrieve the list of available providers. + + This method is a class-level method that retrieves and returns a list of + all the available providers registered within the class. + + Returns: + list: A list containing the keys of the registered providers within + the class. + """ + return list(cls._providers.keys()) \ No newline at end of file diff --git a/document-graph/src/document_graph/graph_build/constructors/core/__init__.py b/document-graph/src/document_graph/graph_build/constructors/core/__init__.py new file mode 100644 index 00000000..553666b6 --- /dev/null +++ b/document-graph/src/document_graph/graph_build/constructors/core/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) Evan Erwee. All rights reserved. +"""Core constructors — basic graph building operations for nodes, edges, and properties.""" + +# Core constructors for basic graph building operations + +from .schema_driven_constructor import SchemaDrivenConstructor +from .node_constructor import NodeConstructor +from .edge_constructor import EdgeConstructor +from .property_constructor import PropertyConstructor + +__all__ = [ + 'SchemaDrivenConstructor', + 'NodeConstructor', + 'EdgeConstructor', + 'PropertyConstructor' +] \ No newline at end of file diff --git a/document-graph/src/document_graph/graph_build/constructors/core/edge_constructor.py b/document-graph/src/document_graph/graph_build/constructors/core/edge_constructor.py new file mode 100644 index 00000000..151fdf73 --- /dev/null +++ b/document-graph/src/document_graph/graph_build/constructors/core/edge_constructor.py @@ -0,0 +1,47 @@ +# Copyright (c) Evan Erwee. All rights reserved. +"""Edge constructor — generates graph edges from DataFrame relationships.""" + +from document_graph.graph_build.constructors.constructors_provider_base import ConstructorProvider +from typing import List, Tuple +import pandas as pd +from document_graph.model_elements import Node, Edge + +class EdgeConstructor(ConstructorProvider): + """ + Provides functionality for constructing edges from DataFrame relationships. + + This class is responsible for generating a list of nodes and edges based on the + relationships defined within a given pandas DataFrame. It extends functionality from + a ConstructorProvider class, allowing it to process data for graph-based representations + or analysis. + + Methods + ------- + construct(data) + Constructs edges and nodes from the provided DataFrame. + """ + + def construct(self, data: pd.DataFrame) -> Tuple[List[Node], List[Edge]]: + """ + Converts a given DataFrame into nodes and edges. + + This method processes the input DataFrame and triage_constructs a list of nodes + and edges based on its data. The structure of the DataFrame is expected to + conform to the necessary format suitable for the transformation. + + Parameters: + data: pd.DataFrame + The input DataFrame containing the data to be converted + into nodes and edges. The DataFrame must contain the + required information for defining both the nodes and their connections. + + Returns: + Tuple[List[Node], List[Edge]] + A tuple where the first element is a list of nodes and the second + element is a list of edges derived from the input DataFrame. + """ + # TODO: Implement DataFrame to edges conversion + nodes = [] + edges = [] + return nodes, edges + diff --git a/document-graph/src/document_graph/graph_build/constructors/core/node_constructor.py b/document-graph/src/document_graph/graph_build/constructors/core/node_constructor.py new file mode 100644 index 00000000..dd9a398a --- /dev/null +++ b/document-graph/src/document_graph/graph_build/constructors/core/node_constructor.py @@ -0,0 +1,46 @@ +# Copyright (c) Evan Erwee. All rights reserved. +"""Node constructor — converts DataFrame rows into graph nodes.""" + +from document_graph.graph_build.constructors.constructors_provider_base import ConstructorProvider +from typing import List, Tuple +import pandas as pd +from document_graph.model_elements import Node, Edge + +class NodeConstructor(ConstructorProvider): + """ + Represents a node constructor that converts data into nodes and edges. + + This class is used to construct nodes and edges from the input data provided + as a DataFrame. It inherits from the `ConstructorProvider` base class and + implements the `construct` method, which processes the data to generate a + list of `Node` objects and a list of `Edge` objects. + + Methods + ------- + construct(data: pd.DataFrame) -> Tuple[List[Node], List[Edge]] + Constructs and returns the nodes and edges based on the provided DataFrame. + """ + + def construct(self, data: pd.DataFrame) -> Tuple[List[Node], List[Edge]]: + """ + Converts a DataFrame into nodes and edges. + + This method takes a pandas DataFrame, processes its content, and produces a list + of nodes and edges based on the data. Each row in the DataFrame may represent + relationships or entities that are used to construct nodes and edges. + + Parameters: + data : pd.DataFrame + A pandas DataFrame containing raw data from which nodes and edges are to be + constructed. + + Returns: + Tuple[List[Node], List[Edge]] + A tuple containing two lists: + - nodes: A list of Node objects created from the DataFrame. + - edges: A list of Edge objects defining relationships between the nodes. + """ + # TODO: Implement DataFrame to nodes conversion + nodes = [] + edges = [] + return nodes, edges \ No newline at end of file diff --git a/document-graph/src/document_graph/graph_build/constructors/core/property_constructor.py b/document-graph/src/document_graph/graph_build/constructors/core/property_constructor.py new file mode 100644 index 00000000..1bb0c1c9 --- /dev/null +++ b/document-graph/src/document_graph/graph_build/constructors/core/property_constructor.py @@ -0,0 +1,39 @@ +# Copyright (c) Evan Erwee. All rights reserved. +"""Property constructor — constructs node and edge properties from data.""" + +from document_graph.graph_build.constructors.constructors_provider_base import ConstructorProvider +from typing import List, Tuple +import pandas as pd +from document_graph.model_elements import Node, Edge + +class PropertyConstructor(ConstructorProvider): + """ + Handles the construction of properties from given data. + + This class is specifically designed to generate properties, represented as nodes + and edges, from the provided data encapsulated in a DataFrame. It serves as a + provider that processes this data and triage_constructs meaningful structures that can + be used in further processing or analysis. + """ + + def construct(self, data: pd.DataFrame) -> Tuple[List[Node], List[Edge]]: + """ + Construct nodes and edges from a given DataFrame. + + This function is intended to map the properties of a DataFrame to a list of + nodes and edges. The details of the mapping process need to be implemented. + It will create and return a list of nodes and edges based on the provided + data structure. + + Args: + data (pd.DataFrame): The input DataFrame that contains the necessary data + for constructing nodes and edges. + + Returns: + Tuple[List[Node], List[Edge]]: A tuple containing a list of nodes and a + list of edges. + """ + # TODO: Implement DataFrame columns to properties mapping + nodes = [] + edges = [] + return nodes, edges \ No newline at end of file diff --git a/document-graph/src/document_graph/graph_build/constructors/core/schema_driven_constructor.py b/document-graph/src/document_graph/graph_build/constructors/core/schema_driven_constructor.py new file mode 100644 index 00000000..0e3e3804 --- /dev/null +++ b/document-graph/src/document_graph/graph_build/constructors/core/schema_driven_constructor.py @@ -0,0 +1,42 @@ +# Copyright (c) Evan Erwee. All rights reserved. +"""Schema-driven constructor — creates nodes and edges based on schema definitions.""" + +from document_graph.graph_build.constructors.constructors_provider_base import ConstructorProvider +from typing import List, Tuple +import pandas as pd +from document_graph.model_elements import Node, Edge + +class SchemaDrivenConstructor(ConstructorProvider): + """ + Represents a schema-driven constructor for creating nodes and edges. + + This class provides functionality to construct nodes and edges + from a schema-driven configuration. It works based on the input + data and uses specific logic defined in its implementation to + derive graph elements. Designed to be extended or instantiated + in systems where schema-driven graph construction is required. + """ + + def construct(self, data: pd.DataFrame) -> Tuple[List[Node], List[Edge]]: + """ + Construct a schema-driven graph from a given DataFrame. + + This method processes a pandas DataFrame and creates a graph representation + composed of nodes and edges based on a predefined schema. The exact schema + for constructing the graph must be implemented in this method. This function + returns two lists: one containing Node objects and another containing Edge + objects that define relationships between the nodes. + + Args: + data: The input pandas DataFrame containing the data to construct the graph + from. + + Returns: + A tuple where the first element is a list of Node objects representing + the entities in the graph, and the second element is a list of Edge + objects representing the relationships between the entities. + """ + # TODO: Implement schema-driven graph construction + nodes = [] + edges = [] + return nodes, edges \ No newline at end of file diff --git a/document-graph/src/document_graph/graph_build/constructors/list_constructors.py b/document-graph/src/document_graph/graph_build/constructors/list_constructors.py new file mode 100644 index 00000000..172d1ddd --- /dev/null +++ b/document-graph/src/document_graph/graph_build/constructors/list_constructors.py @@ -0,0 +1,72 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""API to list available constructors.""" + +from document_graph.graph_build.constructors.constructors_provider_registry import ConstructorProviderRegistry + +# Import registration module to ensure constructors are registered +from . import register_constructors + +def list_available_constructors(): + """ + Lists all registered constructor providers available in the ConstructorProviderRegistry. + + Returns: + list: A list containing all registered providers in the + ConstructorProviderRegistry. + """ + return ConstructorProviderRegistry.list_providers() + +def list_core_constructors(): + """ + Filters and returns a list of core constructors from all available constructors. + + This function retrieves a list of all constructors from the + ConstructorProviderRegistry and filters them to match a predefined + list of core constructor names. The filtered list contains + only the constructors that are considered core. + + Returns: + list: A list of constructor names that match the predefined + list of core constructors. + """ + all_constructors = ConstructorProviderRegistry.list_providers() + core = ["schema_driven", "node_constructor", "edge_constructor", "property_constructor"] + return [const for const in all_constructors if const in core] + +def list_pattern_constructors(): + """ + Lists all constructors that match specific patterns. + + This function retrieves all available constructor providers and filters + them based on predefined patterns. + + Returns: + list: A list of constructor providers that match the specified patterns. + """ + all_constructors = ConstructorProviderRegistry.list_providers() + patterns = ["one_to_many", "many_to_many"] + return [const for const in all_constructors if const in patterns] + +def list_optimization_constructors(): + """ + Filters and returns a list of constructors that match specified optimization features. + + This function retrieves all available constructors from the + `ConstructorProviderRegistry`, compares them against a set of predefined + optimization features, and returns only the ones that are part of the + optimization list. + + Returns: + list[str]: A list of constructors matching the specified optimization + features. + """ + all_constructors = ConstructorProviderRegistry.list_providers() + optimizations = ["batch_constructor", "deduplication"] + return [const for const in all_constructors if const in optimizations] + +if __name__ == "__main__": + print("Available Constructors:") + print("Core:", list_core_constructors()) + print("Patterns:", list_pattern_constructors()) + print("Optimizations:", list_optimization_constructors()) \ No newline at end of file diff --git a/document-graph/src/document_graph/graph_build/constructors/optimizations/__init__.py b/document-graph/src/document_graph/graph_build/constructors/optimizations/__init__.py new file mode 100644 index 00000000..a25efef8 --- /dev/null +++ b/document-graph/src/document_graph/graph_build/constructors/optimizations/__init__.py @@ -0,0 +1,12 @@ +# Copyright (c) Evan Erwee. All rights reserved. +"""Optimization constructors — batch and deduplication strategies for graph building.""" + +# Optimization constructors for performance and efficiency + +from .batch_constructor import BatchConstructor +from .deduplication_constructor import DeduplicationConstructor + +__all__ = [ + 'BatchConstructor', + 'DeduplicationConstructor' +] \ No newline at end of file diff --git a/document-graph/src/document_graph/graph_build/constructors/optimizations/batch_constructor.py b/document-graph/src/document_graph/graph_build/constructors/optimizations/batch_constructor.py new file mode 100644 index 00000000..6611124e --- /dev/null +++ b/document-graph/src/document_graph/graph_build/constructors/optimizations/batch_constructor.py @@ -0,0 +1,36 @@ +# Copyright (c) Evan Erwee. All rights reserved. +"""Batch constructor — groups graph operations into batches for efficiency.""" + +from document_graph.graph_build.constructors.constructors_provider_base import ConstructorProvider +from typing import List, Tuple +import pandas as pd +from document_graph.model_elements import Node, Edge + +class BatchConstructor(ConstructorProvider): + """ + + """ + + def construct(self, data: pd.DataFrame) -> Tuple[List[Node], List[Edge]]: + """ + Construct nodes and edges from a given DataFrame. + + This method processes the given DataFrame and triage_constructs a list of nodes and edges + based on its content. The method is designed to be extended with additional processing + to handle large DataFrames in batches. + + Raises: + ValueError: If the DataFrame is invalid or does not meet the required format. + + Args: + data (pd.DataFrame): Input DataFrame containing information to construct nodes + and edges. + + Returns: + Tuple[List[Node], List[Edge]]: A tuple containing a list of constructed nodes + and edges. + """ + # TODO: Implement batch processing for large DataFrames + nodes = [] + edges = [] + return nodes, edges \ No newline at end of file diff --git a/document-graph/src/document_graph/graph_build/constructors/optimizations/deduplication_constructor.py b/document-graph/src/document_graph/graph_build/constructors/optimizations/deduplication_constructor.py new file mode 100644 index 00000000..35ebc6f2 --- /dev/null +++ b/document-graph/src/document_graph/graph_build/constructors/optimizations/deduplication_constructor.py @@ -0,0 +1,41 @@ +# Copyright (c) Evan Erwee. All rights reserved. +"""Deduplication constructor — ensures unique graph objects during construction.""" + +from document_graph.graph_build.constructors.constructors_provider_base import ConstructorProvider +from typing import List, Tuple +import pandas as pd +from document_graph.model_elements import Node, Edge + +class DeduplicationConstructor(ConstructorProvider): + """ + DeduplicationConstructor is responsible for constructing unique graph objects. + + This class provides implementations for the construction of deduplicated nodes + and edges from the given input data. It ensures that no duplicate graph elements + are created, facilitating the creation of a cleaner graph structure. + + Attributes + ---------- + None + """ + + def construct(self, data: pd.DataFrame) -> Tuple[List[Node], List[Edge]]: + """ + Construct a graph representation from the given data. + + This method processes a DataFrame to generate a list of unique nodes and + edges, suitable for representing a graph structure. Deduplication logic + for nodes and edges is currently to be implemented. + + Args: + data (pd.DataFrame): The input data containing information required to + construct nodes and edges. + + Returns: + Tuple[List[Node], List[Edge]]: A tuple containing a list of unique nodes + and a list of unique edges derived from the input data. + """ + # TODO: Implement deduplication logic for nodes and edges + nodes = [] + edges = [] + return nodes, edges \ No newline at end of file diff --git a/document-graph/src/document_graph/graph_build/constructors/patterns/__init__.py b/document-graph/src/document_graph/graph_build/constructors/patterns/__init__.py new file mode 100644 index 00000000..d0295e66 --- /dev/null +++ b/document-graph/src/document_graph/graph_build/constructors/patterns/__init__.py @@ -0,0 +1,12 @@ +# Copyright (c) Evan Erwee. All rights reserved. +"""Pattern constructors — common relationship patterns like one-to-many and many-to-many.""" + +# Pattern-based constructors for common relationship patterns + +from .one_to_many_constructor import OneToManyConstructor +from .many_to_many_constructor import ManyToManyConstructor + +__all__ = [ + 'OneToManyConstructor', + 'ManyToManyConstructor' +] \ No newline at end of file diff --git a/document-graph/src/document_graph/graph_build/constructors/patterns/many_to_many_constructor.py b/document-graph/src/document_graph/graph_build/constructors/patterns/many_to_many_constructor.py new file mode 100644 index 00000000..3c7bb0de --- /dev/null +++ b/document-graph/src/document_graph/graph_build/constructors/patterns/many_to_many_constructor.py @@ -0,0 +1,17 @@ +# Copyright (c) Evan Erwee. All rights reserved. +"""Many-to-many constructor — handles M:N relationships via junction patterns.""" + +from document_graph.graph_build.constructors.constructors_provider_base import ConstructorProvider +from typing import List, Tuple +import pandas as pd +from document_graph.model_elements import Node, Edge + +class ManyToManyConstructor(ConstructorProvider): + """Handle M:N relationships via junction patterns.""" + + def construct(self, data: pd.DataFrame) -> Tuple[List[Node], List[Edge]]: + """Construct M:N relationship patterns.""" + # TODO: Implement many-to-many relationship construction + nodes = [] + edges = [] + return nodes, edges diff --git a/document-graph/src/document_graph/graph_build/constructors/patterns/one_to_many_constructor.py b/document-graph/src/document_graph/graph_build/constructors/patterns/one_to_many_constructor.py new file mode 100644 index 00000000..4f13a075 --- /dev/null +++ b/document-graph/src/document_graph/graph_build/constructors/patterns/one_to_many_constructor.py @@ -0,0 +1,17 @@ +# Copyright (c) Evan Erwee. All rights reserved. +"""One-to-many constructor — handles 1:N parent-child relationships.""" + +from ..constructors_provider_base import ConstructorProvider +from typing import List, Tuple +import pandas as pd +from document_graph.model_elements import Node, Edge + +class OneToManyConstructor(ConstructorProvider): + """Handle 1:N relationships (one parent to many children).""" + + def construct(self, data: pd.DataFrame) -> Tuple[List[Node], List[Edge]]: + """Construct 1:N relationship patterns.""" + # TODO: Implement one-to-many relationship construction + nodes = [] + edges = [] + return nodes, edges \ No newline at end of file diff --git a/document-graph/src/document_graph/graph_build/constructors/register_constructors.py b/document-graph/src/document_graph/graph_build/constructors/register_constructors.py new file mode 100644 index 00000000..049c6000 --- /dev/null +++ b/document-graph/src/document_graph/graph_build/constructors/register_constructors.py @@ -0,0 +1,33 @@ +# Copyright (c) Evan Erwee. All rights reserved. +"""Register constructors — auto-registers all constructor providers with the registry.""" + +# Register all constructor providers +from document_graph.graph_build.constructors.constructors_provider_registry import ConstructorProviderRegistry + +# Core constructors +from document_graph.graph_build.constructors.core.schema_driven_constructor import SchemaDrivenConstructor +from document_graph.graph_build.constructors.core.node_constructor import NodeConstructor +from document_graph.graph_build.constructors.core.edge_constructor import EdgeConstructor +from document_graph.graph_build.constructors.core.property_constructor import PropertyConstructor + +# Pattern constructors +from document_graph.graph_build.constructors.patterns.one_to_many_constructor import OneToManyConstructor +from document_graph.graph_build.constructors.patterns.many_to_many_constructor import ManyToManyConstructor + +# Optimization constructors +from document_graph.graph_build.constructors.optimizations.batch_constructor import BatchConstructor +from document_graph.graph_build.constructors.optimizations.deduplication_constructor import DeduplicationConstructor + +# Register core constructors +ConstructorProviderRegistry.register("schema_driven", SchemaDrivenConstructor) +ConstructorProviderRegistry.register("node_constructor", NodeConstructor) +ConstructorProviderRegistry.register("edge_constructor", EdgeConstructor) +ConstructorProviderRegistry.register("property_constructor", PropertyConstructor) + +# Register pattern constructors +ConstructorProviderRegistry.register("one_to_many", OneToManyConstructor) +ConstructorProviderRegistry.register("many_to_many", ManyToManyConstructor) + +# Register optimization constructors +ConstructorProviderRegistry.register("batch_constructor", BatchConstructor) +ConstructorProviderRegistry.register("deduplication", DeduplicationConstructor) diff --git a/document-graph/src/document_graph/graph_build/cypher_builder.py b/document-graph/src/document_graph/graph_build/cypher_builder.py new file mode 100644 index 00000000..10f63e1b --- /dev/null +++ b/document-graph/src/document_graph/graph_build/cypher_builder.py @@ -0,0 +1,47 @@ +"""Cypher Builder — generates openCypher MERGE statements from Node/Edge models. + +Uses graphrag-toolkit's GraphStore.query() for execution. +""" + +from typing import Optional +from ..model_elements import Node, Edge + + +def node_to_cypher(node: Node, tenant_id: Optional[str] = None) -> tuple[str, dict]: + """Generate MERGE statement for a typed node.""" + labels = node.labels or ["Node"] + if tenant_id: + labels = [f"__{l}__{tenant_id}__" for l in labels] + label_str = ":".join(labels) + props = node.properties or {} + id_val = node.id + + query = f"MERGE (n:{label_str} {{id: $id_val}}) SET n += $props" + params = {"id_val": id_val, "props": props} + return query, params + + +def edge_to_cypher(edge: Edge, tenant_id: Optional[str] = None) -> tuple[str, dict]: + """Generate MERGE statement for a typed edge.""" + rel_type = edge.label + + query = ( + f"MATCH (a {{id: $src_id}}), (b {{id: $tgt_id}}) " + f"MERGE (a)-[r:{rel_type}]->(b) SET r += $props" + ) + params = { + "src_id": edge.source_id, + "tgt_id": edge.target_id, + "props": edge.properties or {}, + } + return query, params + + +def batch_nodes_to_cypher(nodes: list[Node], tenant_id: Optional[str] = None) -> list[tuple[str, dict]]: + """Generate MERGE statements for a batch of nodes.""" + return [node_to_cypher(n, tenant_id) for n in nodes] + + +def batch_edges_to_cypher(edges: list[Edge], tenant_id: Optional[str] = None) -> list[tuple[str, dict]]: + """Generate MERGE statements for a batch of edges.""" + return [edge_to_cypher(e, tenant_id) for e in edges] diff --git a/document-graph/src/document_graph/ingest/__init__.py b/document-graph/src/document_graph/ingest/__init__.py new file mode 100644 index 00000000..b53db66f --- /dev/null +++ b/document-graph/src/document_graph/ingest/__init__.py @@ -0,0 +1 @@ +"""Ingest stage — data preparation (column/row/field operations).""" diff --git a/document-graph/src/document_graph/ingest/column/__init__.py b/document-graph/src/document_graph/ingest/column/__init__.py new file mode 100644 index 00000000..8b8b45be --- /dev/null +++ b/document-graph/src/document_graph/ingest/column/__init__.py @@ -0,0 +1,15 @@ +"""Column-level ingestors — renaming, reordering, selection, and type conversion.""" +# Column-level ingestors + +from document_graph.ingest.column.column_renamer import ColumnRenamerProvider + +from document_graph.ingest.column.column_reorder import ColumnReorderProvider +from document_graph.ingest.column.column_selector import ColumnSelectorProvider +from document_graph.ingest.column.column_type_converter import ColumnTypeConverterProvider + +__all__ = [ + 'ColumnRenamerProvider', + 'ColumnReorderProvider', + 'ColumnSelectorProvider', + 'ColumnTypeConverterProvider' +] \ No newline at end of file diff --git a/document-graph/src/document_graph/ingest/column/column_renamer.py b/document-graph/src/document_graph/ingest/column/column_renamer.py new file mode 100644 index 00000000..2663da0c --- /dev/null +++ b/document-graph/src/document_graph/ingest/column/column_renamer.py @@ -0,0 +1,41 @@ +import pandas as pd +from document_graph.ingest.ingestors_provider_base import IngestorProvider + +class ColumnRenamerProvider(IngestorProvider): + """ + A provider class for column renaming in a DataFrame. + + The ColumnRenamerProvider allows ingestion of a DataFrame with column + renaming applied based on a provided mapping. The mapping should be + supplied via the arguments dictionary and contains the old column names + as keys and the new column names as values. Only columns that exist in + the DataFrame will be renamed. + + Methods: + ingest(data: pd.DataFrame) -> pd.DataFrame + Renames DataFrame columns based on the provided mapping. + + """ + + def ingest(self, data: pd.DataFrame) -> pd.DataFrame: + """ + Processes and renames columns in the given DataFrame based on a mapping + configuration. If a mapping is provided, only the columns that exist + in the DataFrame and are included in the mapping will be renamed. + + Parameters: + data (pd.DataFrame): The input DataFrame to process. + + Returns: + pd.DataFrame: A new DataFrame with renamed columns according to the + provided mapping. + """ + mapping = self.args.get("mapping", {}) + + if not mapping: + return data + + # Only rename columns that exist + existing_mapping = {old: new for old, new in mapping.items() if old in data.columns} + + return data.rename(columns=existing_mapping) \ No newline at end of file diff --git a/document-graph/src/document_graph/ingest/column/column_reorder.py b/document-graph/src/document_graph/ingest/column/column_reorder.py new file mode 100644 index 00000000..f167a978 --- /dev/null +++ b/document-graph/src/document_graph/ingest/column/column_reorder.py @@ -0,0 +1,53 @@ +import pandas as pd +from document_graph.ingest.ingestors_provider_base import IngestorProvider + +class ColumnReorderProvider(IngestorProvider): + """ + Provides functionality to reorder the columns of a given pandas DataFrame + based on a specified column order. + + The class is designed to take a pandas DataFrame and rearrange its columns + based on a predefined order specified in the `args` attribute. If no column + order is specified, the original DataFrame is returned unchanged. The primary + purpose of the class is to modify column arrangements while preserving columns + that are not explicitly mentioned in the provided order. This ensures all + columns in the DataFrame are retained, even if not reordered. + + Methods + ------- + ingest(data: pd.DataFrame) -> pd.DataFrame + Reorders the columns of the provided DataFrame as per the specified + column order or retains the original order if no preference is provided. + """ + + def ingest(self, data: pd.DataFrame) -> pd.DataFrame: + """ + Reorders the columns of a DataFrame based on the specified order provided in the arguments. + + If a specific column order is provided in the arguments (`column_order`), the method will + rearrange the DataFrame's columns to match the specified order. Columns not present in the + `column_order` list will retain their positions and be appended after the ordered columns. + + Parameters: + data (pd.DataFrame): The DataFrame to be ingested and reordered. The input DataFrame + should contain columns that may partially or fully match the `column_order` list in + the instance's arguments. + + Returns: + pd.DataFrame: A DataFrame with columns reordered based on the `column_order` provided + in the arguments. If `column_order` is not specified or empty, the original DataFrame + is returned without any modifications. + """ + column_order = self.args.get("column_order", []) + + if not column_order: + return data + + # Keep existing columns that aren't in the order list + existing_columns = [col for col in column_order if col in data.columns] + remaining_columns = [col for col in data.columns if col not in column_order] + + # Combine ordered columns with remaining columns + new_order = existing_columns + remaining_columns + + return data[new_order] diff --git a/document-graph/src/document_graph/ingest/column/column_selector.py b/document-graph/src/document_graph/ingest/column/column_selector.py new file mode 100644 index 00000000..528004b7 --- /dev/null +++ b/document-graph/src/document_graph/ingest/column/column_selector.py @@ -0,0 +1,52 @@ +import pandas as pd +from document_graph.ingest.ingestors_provider_base import IngestorProvider + +class ColumnSelectorProvider(IngestorProvider): + """ + Provides functionality to either select specific columns or drop specific columns + from a given pandas DataFrame. + + The ColumnSelectorProvider class processes incoming data based on the specified + action ("select" or "drop") and a list of columns. It is useful when there is a + need to streamline DataFrame operations by filtering or excluding certain + columns dynamically during data ingestion. + + Methods + ------- + ingest(data: pd.DataFrame) -> pd.DataFrame + Processes the DataFrame according to the specified action and columns. + """ + + def ingest(self, data: pd.DataFrame) -> pd.DataFrame: + """ + Processes a DataFrame by either selecting or dropping specified columns based on + the provided action. The method utilizes the arguments passed to dynamically + filter columns of the DataFrame. This is useful for cleaning or transforming + datasets by retaining or removing specific columns. + + Parameters: + data (pd.DataFrame): The input DataFrame that needs to be processed. + + Returns: + pd.DataFrame: The modified DataFrame with columns either selected or dropped, + based on the specified action. + + Raises: + ValueError: Raised when the provided action is unsupported. Supported values + are 'select' or 'drop'. + """ + action = self.args.get("action", "select") # "select" or "drop" + columns = self.args.get("columns", []) + + if not columns: + return data + + if action == "select": + # Keep only specified columns + available_columns = [col for col in columns if col in data.columns] + return data[available_columns] + elif action == "drop": + # Drop specified columns + return data.drop(columns=[col for col in columns if col in data.columns]) + else: + raise ValueError(f"Unsupported action: {action}. Use 'select' or 'drop'.") \ No newline at end of file diff --git a/document-graph/src/document_graph/ingest/column/column_type_converter.py b/document-graph/src/document_graph/ingest/column/column_type_converter.py new file mode 100644 index 00000000..332d0dc1 --- /dev/null +++ b/document-graph/src/document_graph/ingest/column/column_type_converter.py @@ -0,0 +1,48 @@ +import pandas as pd +from document_graph.ingest.ingestors_provider_base import IngestorProvider + +class ColumnTypeConverterProvider(IngestorProvider): + """ + Provides functionality to convert dataframe column types based on a specified mapping. + + This class extends the base IngestorProvider and allows for dynamic conversion of column data + types in a pandas DataFrame. It is particularly useful when ingesting data with columns that + need to adhere to a specific type structure for further processing or analysis. + + Methods + ------- + ingest(data: pd.DataFrame) -> pd.DataFrame + Converts column types in the provided DataFrame according to the 'type_mapping' argument. + + """ + + def ingest(self, data: pd.DataFrame) -> pd.DataFrame: + type_mapping = self.args.get("type_mapping", {}) + + if not type_mapping: + return data + + result = data.copy() + + for column, target_type in type_mapping.items(): + if column not in result.columns: + continue + + try: + if target_type == "string": + result[column] = result[column].astype(str) + elif target_type == "int": + result[column] = pd.to_numeric(result[column], errors='coerce').astype('Int64') + elif target_type == "float": + result[column] = pd.to_numeric(result[column], errors='coerce') + elif target_type == "datetime": + result[column] = pd.to_datetime(result[column], errors='coerce') + elif target_type == "bool": + result[column] = result[column].astype(bool) + else: + # Direct pandas dtype + result[column] = result[column].astype(target_type) + except Exception as e: + print(f"Warning: Could not convert column '{column}' to {target_type}: {e}") + + return result \ No newline at end of file diff --git a/document-graph/src/document_graph/ingest/column/register_ingestors.py b/document-graph/src/document_graph/ingest/column/register_ingestors.py new file mode 100644 index 00000000..31d02024 --- /dev/null +++ b/document-graph/src/document_graph/ingest/column/register_ingestors.py @@ -0,0 +1,14 @@ +"""Register column ingestors — registers all column-level ingestor providers.""" +# Register all column_level ingestors +from document_graph.ingest.ingestors_provider_registry import IngestorProviderRegistry +from document_graph.ingest.column.column_selector import ColumnSelectorProvider +from document_graph.ingest.column.column_renamer import ColumnRenamerProvider +from document_graph.ingest.column.column_reorder import ColumnReorderProvider +from document_graph.ingest.column.column_type_converter import ColumnTypeConverterProvider + +# Register column_level ingestors +IngestorProviderRegistry.register("column_selector", ColumnSelectorProvider) +IngestorProviderRegistry.register("column_renamer", ColumnRenamerProvider) +IngestorProviderRegistry.register("column_reorder", ColumnReorderProvider) +IngestorProviderRegistry.register("column_type_converter", ColumnTypeConverterProvider) + diff --git a/document-graph/src/document_graph/ingest/field/__init__.py b/document-graph/src/document_graph/ingest/field/__init__.py new file mode 100644 index 00000000..1e80fcd9 --- /dev/null +++ b/document-graph/src/document_graph/ingest/field/__init__.py @@ -0,0 +1,14 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Field-level ingestors for data transformation and cleanup. + +This module contains ingestors that operate on individual fields or field-level +transformations during the data extraction phase. +""" + +from .numeric_id_cleanup_ingestor import NumericIdCleanupIngestor + +__all__ = [ + 'NumericIdCleanupIngestor', +] \ No newline at end of file diff --git a/document-graph/src/document_graph/ingest/field/numeric_id_cleanup_ingestor.py b/document-graph/src/document_graph/ingest/field/numeric_id_cleanup_ingestor.py new file mode 100644 index 00000000..f3100d8b --- /dev/null +++ b/document-graph/src/document_graph/ingest/field/numeric_id_cleanup_ingestor.py @@ -0,0 +1,40 @@ +"""Numeric ID cleanup ingestor — removes trailing decimals from numeric ID fields.""" +import pandas as pd +from document_graph.ingest.ingestors_provider_base import IngestorProvider + + +class NumericIdCleanupIngestor(IngestorProvider): + """ + Ingestor provider for cleaning up numeric IDs with specific formatting. + + This class processes a pandas DataFrame to clean up numeric ID fields, + specifically by converting them to string format and removing any `.0` + suffixes. It ensures the field specified in the arguments is cleaned if + it exists in the DataFrame. Intended for use in data ingestion workflows + where numeric IDs might be improperly formatted or need standardization. + + Methods: + ingest: Processes and cleans a DataFrame based on the specified field. + + """ + + def ingest(self, data: pd.DataFrame) -> pd.DataFrame: + """ + Processes a DataFrame to modify a specified field by converting its values to strings + and removing ".0" suffix if present. The operation is applied only when the field + is specified and exists in the DataFrame columns. + + Parameters: + data (pd.DataFrame): The input DataFrame containing the data to be processed. + + Returns: + pd.DataFrame: The processed DataFrame with the specified field modified, if applicable. + """ + field = self.args.get("field") + if not field or field not in data.columns: + return data + + # Convert to string and remove .0 suffix + data[field] = data[field].astype(str).str.replace(r'\.0$', '', regex=True) + + return data \ No newline at end of file diff --git a/document-graph/src/document_graph/ingest/field/register_ingestors.py b/document-graph/src/document_graph/ingest/field/register_ingestors.py new file mode 100644 index 00000000..5e500959 --- /dev/null +++ b/document-graph/src/document_graph/ingest/field/register_ingestors.py @@ -0,0 +1,7 @@ +"""Register field ingestors — registers all field-level ingestor providers.""" +# Register all field_level ingestors +from ..ingestors_provider_registry import IngestorProviderRegistry +from .numeric_id_cleanup_ingestor import NumericIdCleanupIngestor + +# Register field_level ingestors +IngestorProviderRegistry.register("numeric_id_cleanup", NumericIdCleanupIngestor) \ No newline at end of file diff --git a/document-graph/src/document_graph/ingest/ingestors_error_handler.py b/document-graph/src/document_graph/ingest/ingestors_error_handler.py new file mode 100644 index 00000000..164f533a --- /dev/null +++ b/document-graph/src/document_graph/ingest/ingestors_error_handler.py @@ -0,0 +1,112 @@ +"""Ingestors error handler — error handling and validation utilities for ingestion.""" +import logging +import pandas as pd +from typing import List, Dict, Any + +# Use document-graph logging +logger = logging.getLogger(__name__) + +class IngestorErrorHandler: + """ + Provides error handling and validation utilities for ingestor processes. + + IngestorErrorHandler contains static methods that validate the presence of required + columns in a DataFrame, ensure configuration fields are complete, and handle failures + by rolling back to a safe state. This class is designed to support robust error + management in data ingestion pipelines, promoting reliability and easier debugging. + """ + + @staticmethod + def validate_column_exists(df: pd.DataFrame, column: str, ingestor_name: str): + """ + Validates the existence of a specified column in a DataFrame for the given ingestor. + + If the specified column does not exist in the DataFrame, an error message will be + logged and a ValueError will be raised. + + Parameters: + df (pd.DataFrame): The input DataFrame in which to check the availability of the column. + column (str): The name of the column to verify existence. + ingestor_name (str): The name of the ingestor to include in error logging. + + Raises: + ValueError: If the specified column is not found in the DataFrame's columns. + """ + if column not in df.columns: + error_msg = f"{ingestor_name}: Column '{column}' not found. Available columns: {list(df.columns)}" + logger.error(error_msg) + raise ValueError(error_msg) + + @staticmethod + def validate_columns_exist(df: pd.DataFrame, columns: List[str], ingestor_name: str): + """ + Validates if the specified columns exist in the provided DataFrame. This method checks + for missing columns in the DataFrame compared to the list of required columns. If any + columns are missing, an error is logged and an exception is raised. + + Args: + df (pd.DataFrame): The DataFrame to validate. + columns (List[str]): A list of column names to check for existence in the DataFrame. + ingestor_name (str): The name of the ingestor for logging purposes. + + Raises: + ValueError: If any of the specified columns are missing in the DataFrame. + """ + if missing := [col for col in columns if col not in df.columns]: + error_msg = f"{ingestor_name}: Columns not found: {missing}. Available: {list(df.columns)}" + logger.error(error_msg) + raise ValueError(error_msg) + + @staticmethod + def handle_ingestor_failure(ingestor_name: str, error: Exception, df: pd.DataFrame) -> pd.DataFrame: + """ + Handles the failure of an ingestor operation and reverts to the original DataFrame. + + The method logs an error and a warning message whenever an ingestor fails. It ensures + that in case of any error during ingestion, the original DataFrame is retained and + returned unchanged. + + Parameters: + ingestor_name: str + The name of the ingestor that encountered a failure. + error: Exception + The exception instance describing the failure. + df: pd.DataFrame + The original DataFrame prior to the failed ingestion attempt. + + Returns: + pd.DataFrame + The original DataFrame unchanged, ensuring data consistency. + """ + logger.error(f"{ingestor_name} failed: {error}") + logger.warning(f"Rolling back to original DataFrame with {len(df)} rows") + return df + + @staticmethod + def validate_config(config_dict: Dict[str, Any], required_fields: List[str], ingestor_name: str): + """ + Validates a configuration dictionary against a list of required fields. + + This method checks that all required fields are present in the provided + configuration dictionary. If any required fields are missing, an error + is logged, and a ValueError is raised. + + Parameters: + config_dict : Dict[str, Any] + The configuration dictionary to validate. + required_fields : List[str] + A list of field names required to be in the configuration dictionary. + ingestor_name : str + The name of the ingestor, included in error messages for better + context. + + Raises: + ValueError + Raised when one or more required fields are missing from the + configuration dictionary. + """ + missing = [field for field in required_fields if field not in config_dict] + if missing: + error_msg = f"{ingestor_name}: Missing required config fields: {missing}" + logger.error(error_msg) + raise ValueError(error_msg) \ No newline at end of file diff --git a/document-graph/src/document_graph/ingest/ingestors_plan.py b/document-graph/src/document_graph/ingest/ingestors_plan.py new file mode 100644 index 00000000..9e44bec8 --- /dev/null +++ b/document-graph/src/document_graph/ingest/ingestors_plan.py @@ -0,0 +1,171 @@ +"""Ingestors plan — orchestrates execution of ingestor providers on DataFrames.""" +import logging +from typing import List +import pandas as pd +from .ingestors_provider_config import IngestorProviderConfig +from .ingestors_provider_factory import IngestorProviderFactory +from .ingestors_validator import IngestorConfigValidator +from .ingestors_schema import IngestorSchemaTracker +from .ingestors_error_handler import IngestorErrorHandler + +# Use document-graph logging +logger = logging.getLogger(__name__) + +class IngestorPlan: + """ + Represents a plan for executing a sequence of ingestor operations on a dataset. + + The IngestorPlan class is used to manage and execute a series of data ingestion steps + defined by a set of configurations. Each ingestor performs a specific processing step, + and the class ensures that errors are handled and schema changes are tracked throughout + the execution. The class also provides functionalities for validating configurations + upon initialization and tracking schema modifications to aid in debugging or pipeline analysis. + """ + + def __init__(self, configs: List[IngestorProviderConfig], validate: bool = True): + """ + Initialize an IngestorPlan instance. + + The IngestorPlan class manages the initialization and validation of ingestor + configurations. It tracks the state of ingestor schemas and ensures the + provided configurations are valid based on the validation rules. + + Arguments: + configs: List of IngestorProviderConfig objects containing the configurations + for initializing ingestors. + validate: Boolean flag to specify whether to validate the configurations. + Default is True. + + Raises: + ValueError: If validation is enabled and any configuration errors are detected. + """ + self.configs = configs + self.schema_tracker = None + + if validate: + errors = IngestorConfigValidator.validate_configs(configs) + if errors: + error_msg = "Ingestor configuration errors:\n" + "\n".join(errors) + logger.error(error_msg) + raise ValueError(error_msg) + + logger.info(f"IngestorPlan initialized with {len(configs)} ingestors") + + def execute(self, data: pd.DataFrame) -> pd.DataFrame: + """ + Executes a pipeline of ingestors on the input DataFrame. + + This method processes the provided DataFrame by sequentially applying + configured ingestors in the pipeline. Each ingestor modifies the DataFrame + and tracks schema changes using an internal schema tracker. If any ingestor + fails, the failure is logged, and an error handler processes the issue. + + Parameters: + data (pd.DataFrame): The input DataFrame to process. + + Returns: + pd.DataFrame: The resulting DataFrame after applying the pipeline of ingestors. + + Raises: + Exception: If an unexpected error occurs during the execution of the pipeline. + """ + if data.empty: + logger.warning("Input DataFrame is empty") + return data + + # Initialize schema tracking + self.schema_tracker = IngestorSchemaTracker(data) + result = data.copy() + + logger.info(f"Starting ingestor pipeline: {len(result)} rows, {len(result.columns)} columns") + + for i, config in enumerate(self.configs): + try: + logger.info(f"Executing ingestor {i+1}/{len(self.configs)}: {config.name} ({config.type})") + + # Store state before ingestor + before_df = result.copy() + + # Execute ingestor + ingestor = IngestorProviderFactory.create(config) + result = ingestor.ingest(result) + + # Track schema changes + change_type = self._determine_change_type(config.type, before_df, result) + self.schema_tracker.track_change( + ingestor_name=config.name, + change_type=change_type, + details=config.args, + df_before=before_df, + df_after=result + ) + + logger.info(f" Completed: {len(before_df)} → {len(result)} rows") + + except Exception as e: + logger.error(f"Ingestor {config.name} failed: {e}") + result = IngestorErrorHandler.handle_ingestor_failure(config.name, e, result) + + # Print final summary + self.schema_tracker.print_summary() + + logger.info(f"Ingestor pipeline completed: {len(result)} rows, {len(result.columns)} columns") + return result + + def get_schema_changes(self): + """ + Retrieves schema changes tracked by the schema tracker. + + This method checks if a schema tracker exists and, if so, returns the + changes tracked by it. If no schema tracker is available, an empty list + is returned. + + Returns: + list: A list containing the tracked schema changes, or an empty list + if no schema tracker is present. + """ + if self.schema_tracker: + return self.schema_tracker.changes + return [] + + def _determine_change_type(self, ingestor_type: str, before_df: pd.DataFrame, after_df: pd.DataFrame) -> str: + """ + Determines the type of change between two dataframes given their structure and content. + + This method compares two dataframes to determine the nature of the changes + between them, such as removal or addition of rows, modification of data, or + alteration of column structure. It also includes logic to detect specific + column-related changes like renaming, addition, or removal. The determination + of the change type relies entirely on structural and content differences + of the dataframes. + + Parameters: + ingestor_type: str + Identifier for the type of ingestor or process responsible for + generating the dataframes. + before_df: pd.DataFrame + The initial dataframe before the change was applied. + after_df: pd.DataFrame + The modified dataframe after the change was applied. + + Returns: + str: A string indicating the type of change detected. Possible values are: + - "rows_filtered": If the number of rows has changed. + - "columns_removed": If fewer columns exist after the change. + - "columns_added": If additional columns exist after the change. + - "columns_renamed": If the column names differ without a change + in the column count. + - "data_modified": If neither the rows nor columns changed, but + data content was altered. + """ + if len(before_df) != len(after_df): + return "rows_filtered" + elif list(before_df.columns) != list(after_df.columns): + if len(before_df.columns) > len(after_df.columns): + return "columns_removed" + elif len(before_df.columns) < len(after_df.columns): + return "columns_added" + else: + return "columns_renamed" + else: + return "data_modified" \ No newline at end of file diff --git a/document-graph/src/document_graph/ingest/ingestors_provider_base.py b/document-graph/src/document_graph/ingest/ingestors_provider_base.py new file mode 100644 index 00000000..cba3cf99 --- /dev/null +++ b/document-graph/src/document_graph/ingest/ingestors_provider_base.py @@ -0,0 +1,70 @@ +"""Ingestors provider base — abstract base class for ingestor providers.""" +import logging +from abc import ABC, abstractmethod +from typing import List, Dict, Any +import pandas as pd + +# Use document-graph logging +logger = logging.getLogger(__name__) + +class IngestorProvider(ABC): + """ + Abstract base class that defines the interface and behavior for an ingestor provider. + + This class is intended to be inherited and implemented by subclasses to perform + specific ingestion tasks. It provides the basic blueprint and utility methods for + ingesting and validating data. Subclasses must implement the `ingest` method to + define the ingestion logic. The class also includes a utility method for input + validation. + + Attributes: + config: Configuration object for the ingestor, which provides necessary + parameters and settings. + + Methods must be overridden or used directly where applicable in the defined + workflow for ingestion. + """ + + def __init__(self, config): + """ + Initializes an instance of the class with the provided configuration. + + Attributes: + config: The configuration object associated with the instance. + args: Parsed arguments extracted from the provided configuration. + + Args: + config: The configuration object used to initialize the instance. + """ + self.config = config + self.args = config.args + logger.debug(f"Initialized {self.__class__.__name__} with config: {config.type}") + + @abstractmethod + def ingest(self, data: pd.DataFrame) -> pd.DataFrame: + """ + Abstract method that must be implemented by derived classes to process and + transform the input data. + + Parameters: + data (pd.DataFrame): Input data in the form of a Pandas DataFrame. + + Returns: + pd.DataFrame: Processed and transformed data as a Pandas DataFrame. + """ + pass + + def validate_input(self, data: pd.DataFrame) -> bool: + """ + Validates the input DataFrame to ensure it is not None or empty. + + This method checks if the input DataFrame is valid by ensuring it is not + None and contains data. + + Returns: + bool: True if the input DataFrame is valid, otherwise False. + """ + if data is None or data.empty: + logger.warning(f"{self.__class__.__name__}: Input DataFrame is empty") + return False + return True \ No newline at end of file diff --git a/document-graph/src/document_graph/ingest/ingestors_provider_config.py b/document-graph/src/document_graph/ingest/ingestors_provider_config.py new file mode 100644 index 00000000..7eb5c9bc --- /dev/null +++ b/document-graph/src/document_graph/ingest/ingestors_provider_config.py @@ -0,0 +1,52 @@ +"""Ingestors provider config — configuration dataclass for ingestor providers.""" +from dataclasses import dataclass, field +from typing import Dict, Any, Optional, List + +@dataclass +class IngestorProviderConfig: + """ + Represents the configuration for an ingestor provider. + + This class encapsulates the necessary attributes and validation rules for + defining an ingestor provider's configuration. It is mainly used to store + the metadata and validation requirements for the ingestor. + + Attributes: + name: A unique name identifying the ingestor. + type: The type or category of the ingestor. + args: A dictionary containing additional parameters for the ingestor. + description: An optional description for the ingestor; defaults to a + combination of type and name if not provided. + required_columns: An optional list of column names required by the + ingestor. + + Raises: + ValueError: Raised if `name` or `type` is empty during initialization. + + """ + name: str + type: str + args: Dict[str, Any] = field(default_factory=dict) + description: Optional[str] = None + required_columns: Optional[List[str]] = None + + def __post_init__(self): + """ + Performs post-initialization validation and sets default values for some + attributes of the ingestor instance after the object is instantiated. + + Raises + ------ + ValueError + If the 'name' attribute is empty or None. + ValueError + If the 'type' attribute is empty or None. + """ + if not self.name: + raise ValueError("Ingestor name cannot be empty") + if not self.type: + raise ValueError("Ingestor type cannot be empty") + + # Set default description if not provided + if not self.description: + self.description = f"{self.type} ingestor: {self.name}" \ No newline at end of file diff --git a/document-graph/src/document_graph/ingest/ingestors_provider_factory.py b/document-graph/src/document_graph/ingest/ingestors_provider_factory.py new file mode 100644 index 00000000..7ee66eec --- /dev/null +++ b/document-graph/src/document_graph/ingest/ingestors_provider_factory.py @@ -0,0 +1,34 @@ +from document_graph.ingest.ingestors_provider_registry import IngestorProviderRegistry +from document_graph.ingest.ingestors_provider_config import IngestorProviderConfig + +class IngestorProviderFactory: + """ + Provides a factory for creating instances of ingestor providers based on a + given configuration. + + This class serves as a factory for instantiating ingestor providers that + are registered in the `IngestorProviderRegistry`. It uses the type specified + in the configuration to determine the appropriate provider class and returns + an instance of it. + """ + + @staticmethod + def create(config: IngestorProviderConfig): + """ + Creates an instance of an ingestor provider based on the provided configuration. + + This method retrieves the appropriate ingestor provider class from the + IngestorProviderRegistry based on the type specified in the given configuration. + It then initializes an instance of this class using the configuration and + returns the created instance. + + Args: + config (IngestorProviderConfig): Configuration object containing + details necessary for selecting and initializing the ingestor provider. + + Returns: + IngestorProvider: An instance of the provider class initialized with + the given configuration. + """ + provider_class = IngestorProviderRegistry.get(config.type) + return provider_class(config) diff --git a/document-graph/src/document_graph/ingest/ingestors_provider_registry.py b/document-graph/src/document_graph/ingest/ingestors_provider_registry.py new file mode 100644 index 00000000..0d870a99 --- /dev/null +++ b/document-graph/src/document_graph/ingest/ingestors_provider_registry.py @@ -0,0 +1,77 @@ +"""Ingestors provider registry — centralized registry of ingestor provider classes.""" +from typing import Dict, Type +from document_graph.ingest.ingestors_provider_base import IngestorProvider +from document_graph.ingest.field.numeric_id_cleanup_ingestor import NumericIdCleanupIngestor + +class IngestorProviderRegistry: + """ + Provides a registry for managing ingestor providers. + + The IngestorProviderRegistry class allows for the registration, retrieval, and + listing of ingestor providers by name. This serves as a central management system for + providers, enabling dynamic customization and extension of functionality. + + Methods: + register: Registers a new ingestor provider with a specified name. + get: Retrieves a registered ingestor provider by its name. + list_providers: Lists all currently registered provider names. + """ + + _providers: Dict[str, Type[IngestorProvider]] = { + 'numeric_id_cleanup': NumericIdCleanupIngestor, + } + + @classmethod + def register(cls, name: str, provider_class: Type[IngestorProvider]): + """ + Registers a new ingestor provider with a specified name. + + This class method allows adding a provider to the internal registry by associating it + with a unique name. The registered ingestor provider can be used in the system for + handling specific ingestion tasks. The uniqueness of the provider name must be ensured + by the caller to avoid accidental overwrites. + + Parameters: + name (str): The unique identifier for the ingestor provider. + provider_class (Type[IngestorProvider]): The class implementing the ingestor + provider functionality. + + Raises: + KeyError: Raises an error if the provider name already exists in the registry. + """ + cls._providers[name] = provider_class + + @classmethod + def get(cls, name: str) -> Type[IngestorProvider]: + """ + Retrieves an ingestor provider class by its name. + + Raises a ValueError if the provided name does not exist in the registered + ingestor providers. + + Args: + name: The name of the ingestor provider to retrieve. + + Returns: + The ingestor provider class associated with the given name. + + Raises: + ValueError: If the given name is not a valid provider. + """ + if name not in cls._providers: + raise ValueError(f"Unknown ingestor provider: {name}") + return cls._providers[name] + + @classmethod + def list_providers(cls) -> list: + """ + Returns a list of available providers for the class. + + This method retrieves the keys from the internal `_providers` dictionary + and returns them as a list. It is used to view all registered providers + accessible for the class. + + Returns: + list: A list containing the names of all available providers. + """ + return list(cls._providers.keys()) \ No newline at end of file diff --git a/document-graph/src/document_graph/ingest/ingestors_schema.py b/document-graph/src/document_graph/ingest/ingestors_schema.py new file mode 100644 index 00000000..6617702d --- /dev/null +++ b/document-graph/src/document_graph/ingest/ingestors_schema.py @@ -0,0 +1,223 @@ +"""Ingestors schema — tracks schema changes during ingestion pipeline execution.""" +import logging +import pandas as pd +from typing import Dict, List, Any, Optional +from dataclasses import dataclass + +# Use document-graph logging +logger = logging.getLogger(__name__) + +@dataclass +class IngestorSchemaChange: + """ + Represents a schema change event in an ingestor. + + This data structure is used to track schema-related changes occurring + during data ingestion. It stores the details of the change, including + the type of change, the affected rows, and the affected columns. + + Attributes: + ingestor_name: Name of the ingestor where the schema change occurred. + change_type: Type of schema change. Possible values include "column_added", + "column_removed", "column_renamed", "rows_filtered". + details: A dictionary containing additional information about the schema change. + rows_before: Number of rows before the change occurred. + rows_after: Number of rows after the change occurred. + columns_before: List of column names present before the change occurred. + columns_after: List of column names present after the change occurred. + """ + ingestor_name: str + change_type: str # "column_added", "column_removed", "column_renamed", "rows_filtered" + details: Dict[str, Any] + rows_before: int + rows_after: int + columns_before: List[str] + columns_after: List[str] + +@dataclass +class SourceSchema: + """ + Represents a source schema with information about dataset structure. + + This class provides metadata and details about the structure of a dataset, + including its columns, data types, the number of rows, and sample data + for initial validation or analysis. + + Attributes: + columns: List of column names in the dataset. + dtypes: Mapping of column names to their respective data types. + row_count: Total number of rows in the dataset. + sample_data: A sample set of data values from the dataset, useful + for verification or inspection. + """ + columns: List[str] + dtypes: Dict[str, str] + row_count: int + sample_data: Dict[str, Any] + +@dataclass +class IngestorSchema: + """ + Represents the schema for an ingestor. + + The IngestorSchema class defines the necessary schema required for data ingestion + operations. It encapsulates the source schema, final columns and their data types, + final row count, and the list of changes applied during ingestion. + + Attributes: + source_schema: SourceSchema + The original schema from the source data being ingested. + final_columns: List[str] + The list of column names that will exist in the final ingested data. + final_dtypes: Dict[str, str] + A dictionary mapping each column name to its data type in the final data. + final_row_count: int + The total row count in the final ingested dataset. + changes: List[IngestorSchemaChange] + A list of changes or transformations made to the schema during the ingestion + process. + """ + source_schema: SourceSchema + final_columns: List[str] + final_dtypes: Dict[str, str] + final_row_count: int + changes: List[IngestorSchemaChange] + +class IngestorSchemaTracker: + """ + Tracks schema changes for a source DataFrame during data ingestion processes. + + This class is designed to monitor and document changes to a source DataFrame's schema + caused by data ingestors. It keeps track of initial schema details, lists all changes made, + and provides a mechanism to summarize these changes. The class also offers a way to + retrieve the final schema after all transformations. + + Attributes: + source_schema: An object representing the schema of the source DataFrame, including + column names, data types, row count, and an optional sample of data. + changes: A list of schema change events recorded during data ingestion. + + Methods: + __init__(source_df) + Initializes the schema tracker with the structure of the source DataFrame. + track_change(ingestor_name, change_type, details, df_before, df_after) + Records a schema change caused by an ingestor. + get_final_schema(final_df) + Constructs and retrieves the final schema after all data ingestion operations. + print_summary() + Displays a logged summary of all schema changes. + """ + + def __init__(self, source_df: pd.DataFrame): + """ + Initializes a schema tracker for a source DataFrame. + + Summary: + The constructor method initializes the schema tracking by capturing details + about the column names, data types, row count, and sample data from the provided + DataFrame. It also sets up an empty list to track schema changes and logs an + initialization message. + + Parameters: + source_df (pd.DataFrame): The DataFrame for which schema tracking is to be + initialized. + + Attributes: + source_schema (SourceSchema): Contains information about the structure of the + source DataFrame, including columns, data types, row count, and a sample record. + changes (List[IngestorSchemaChange]): A list intended to track schema changes + that occur during operations. + """ + self.source_schema = SourceSchema( + columns=list(source_df.columns), + dtypes={col: str(dtype) for col, dtype in source_df.dtypes.items()}, + row_count=len(source_df), + sample_data=source_df.head(1).to_dict('records')[0] if not source_df.empty else {} + ) + self.changes: List[IngestorSchemaChange] = [] + logger.info(f"Schema tracker initialized: {len(self.source_schema.columns)} columns, {self.source_schema.row_count} rows") + + def track_change(self, ingestor_name: str, change_type: str, details: Dict[str, Any], + df_before: pd.DataFrame, df_after: pd.DataFrame): + """ + Tracks changes in the ingested data and logs the details of the changes, including the + number of rows and columns before and after the change. + + Parameters: + ingestor_name: str + The name of the ingestor responsible for the dataset change. + change_type: str + A description of the type of change applied (e.g., "schema_update"). + details: Dict[str, Any] + Additional information or metadata about the change. + df_before: pd.DataFrame + The dataframe representing the state of the data before the change. + df_after: pd.DataFrame + The dataframe representing the state of the data after the change. + """ + change = IngestorSchemaChange( + ingestor_name=ingestor_name, + change_type=change_type, + details=details, + rows_before=len(df_before), + rows_after=len(df_after), + columns_before=list(df_before.columns), + columns_after=list(df_after.columns) + ) + self.changes.append(change) + + logger.info(f"{ingestor_name}: {change_type} - {change.rows_before}→{change.rows_after} rows, " + f"{len(change.columns_before)}→{len(change.columns_after)} columns") + + def get_final_schema(self, final_df: pd.DataFrame) -> IngestorSchema: + """ + Generate the final schema for the given DataFrame. + + This method prepares an instance of IngestorSchema by collecting metadata + from the provided DataFrame, including column names, data types, row count, + and any registered changes. It helps to encapsulate the final schema details + for use in data ingestion or transformation processes. + + Args: + final_df (pd.DataFrame): A DataFrame which contains the finalized data structure. + + Returns: + IngestorSchema: An object encapsulating the schema details including + source schema, final column names, data types, row count, and applied changes. + """ + return IngestorSchema( + source_schema=self.source_schema, + final_columns=list(final_df.columns), + final_dtypes={col: str(dtype) for col, dtype in final_df.dtypes.items()}, + final_row_count=len(final_df), + changes=self.changes + ) + + def print_summary(self): + """ + Logs a summary of the schema changes during the ingestion process. + + The method provides a detailed summary of the source schema and all + the changes applied during the ingestion. Each change is logged with + its name, type of change, number of rows and columns before and after + the change, and additional details if provided. The final state of the + data after all changes is also logged. + + Raises: + AttributeError: If required attributes like `source_schema` or + `changes` are missing or improperly formatted. + """ + logger.info("=== INGESTOR SCHEMA SUMMARY ===") + logger.info(f"Source: {len(self.source_schema.columns)} columns, {self.source_schema.row_count} rows") + + for change in self.changes: + logger.info(f" {change.ingestor_name}: {change.change_type}") + logger.info(f" Rows: {change.rows_before} → {change.rows_after}") + logger.info(f" Columns: {len(change.columns_before)} → {len(change.columns_after)}") + if change.details: + logger.info(f" Details: {change.details}") + + if self.changes: + final_change = self.changes[-1] + logger.info(f"Final: {len(final_change.columns_after)} columns, {final_change.rows_after} rows") + logger.info("=== END SUMMARY ===") \ No newline at end of file diff --git a/document-graph/src/document_graph/ingest/ingestors_validator.py b/document-graph/src/document_graph/ingest/ingestors_validator.py new file mode 100644 index 00000000..8bb24186 --- /dev/null +++ b/document-graph/src/document_graph/ingest/ingestors_validator.py @@ -0,0 +1,246 @@ +"""Ingestors validator — validates ingestor configurations before execution.""" +import logging +from typing import List, Dict, Any +from document_graph.ingest.ingestors_provider_config import IngestorProviderConfig +from document_graph.ingest.ingestors_provider_registry import IngestorProviderRegistry + +# Use document-graph logging +logger = logging.getLogger(__name__) + +class IngestorConfigValidator: + """ + This class provides methods to validate ingestor configurations for an ingestion pipeline. + + The class aims to ensure that the configuration settings for different types of ingestors are + compatible and meet the required standards. It validates individual configuration parameters, + checks for missing required fields, and ensures there are no dependency conflicts between + multiple ingestors. This helps ensure a robust and error-free ingestion process. + """ + + # Required fields for each ingestor type + REQUIRED_FIELDS = { + "skip_row": ["conditions"], + "date_range_filter": ["date_field"], + "column_selector": ["columns"], + "column_renamer": ["mapping"], + "column_reorder": ["column_order"], + "column_type_converter": ["type_mapping"] + } + + @classmethod + def validate_config(cls, config: IngestorProviderConfig) -> List[str]: + """ + Validates the configuration for an ingestor type. + + This method checks if the provided ingestor type is registered, ensures that + all required fields for the specified ingestor type are present, and performs + type-specific validations based on the type of the ingestor. Errors found + during the validation process are collected and returned as a list of strings. + + Args: + config (IngestorProviderConfig): Configuration object containing the type + of the ingestor and its arguments. + + Returns: + List[str]: A list of error messages, if any validation issues are found. + """ + errors = [] + + # Check if ingestor type is registered + try: + IngestorProviderRegistry.get(config.type) + except ValueError: + errors.append(f"Unknown ingestor type: {config.type}") + return errors + + # Check required fields + required = cls.REQUIRED_FIELDS.get(config.type, []) + for field in required: + if field not in config.args: + errors.append(f"{config.type}: Missing required field '{field}'") + + # Type-specific validation + if config.type == "skip_row": + errors.extend(cls._validate_skip_row(config.args)) + elif config.type == "date_range_filter": + errors.extend(cls._validate_date_range_filter(config.args)) + elif config.type == "column_selector": + errors.extend(cls._validate_column_selector(config.args)) + + return errors + + @classmethod + def validate_configs(cls, configs: List[IngestorProviderConfig]) -> List[str]: + """ + Validates a list of ingestor configuration objects and checks for possible issues. + + This method performs validation on each provided configuration by using + the `validate_config` method. It also checks for dependency conflicts + among configurations and collects any validation errors. + + Args: + configs (List[IngestorProviderConfig]): A list of configuration objects + to be validated. + + Returns: + List[str]: A list of error messages for invalid configurations, or an + empty list if all configurations are valid. + """ + all_errors = [] + + for i, config in enumerate(configs): + errors = cls.validate_config(config) + for error in errors: + all_errors.append(f"Ingestor {i+1} ({config.name}): {error}") + + # Check for dependency conflicts + all_errors.extend(cls._check_dependencies(configs)) + + return all_errors + + @classmethod + def _validate_skip_row(cls, args: Dict[str, Any]) -> List[str]: + """ + Validates the provided conditions for skipping rows against the required + format and applicable operator requirements. + + The method checks if the 'conditions' in `args` is a list and validates each + condition entry to ensure it is a dictionary containing required keys such as + 'field' and 'operator'. It also ensures that specific operators have associated + 'required' values. Returns a list of error messages detailing any validation + failures. + + Arguments: + args (Dict[str, Any]): A dictionary containing the conditions to validate. + + Returns: + List[str]: A list of error messages resulting from the validation process. + """ + errors = [] + conditions = args.get("conditions", []) + + if not isinstance(conditions, list): + errors.append("conditions must be a list") + return errors + + for i, condition in enumerate(conditions): + if not isinstance(condition, dict): + errors.append(f"Condition {i+1} must be a dictionary") + continue + + if "field" not in condition: + errors.append(f"Condition {i+1}: Missing 'field'") + if "operator" not in condition: + errors.append(f"Condition {i+1}: Missing 'operator'") + + operator = condition.get("operator") + if operator in ["eq", "ne", "lt", "le", "gt", "ge", "in", "not_in", "contains", "not_contains", "startswith", "endswith", "regex", "length_eq", "length_gt", "length_lt"]: + if "value" not in condition: + errors.append(f"Condition {i+1}: Operator '{operator}' requires 'value'") + + return errors + + @classmethod + def _validate_date_range_filter(cls, args: Dict[str, Any]) -> List[str]: + """ + Validates the presence of at least one date constraint in the given arguments. + + This method ensures that the arguments dictionary contains at least one of the + specified date constraint keys: "start_date", "end_date", "days_back", + "weeks_back", or "months_back". If none of these keys are present, it + accumulates an error message. + + Args: + args (Dict[str, Any]): The dictionary of arguments to be validated. + + Returns: + List[str]: A list of error messages. An empty list indicates that the + validation was successful. + """ + errors = [] + + # Must have at least one date constraint + date_constraints = ["start_date", "end_date", "days_back", "weeks_back", "months_back"] + if not any(constraint in args for constraint in date_constraints): + errors.append("Must specify at least one date constraint") + + return errors + + @classmethod + def _validate_column_selector(cls, args: Dict[str, Any]) -> List[str]: + """ + Validates the column selector arguments and returns a list of any detected + validation errors. This method ensures that the provided arguments for the + column selection or dropping process conform to the expected format and + requirements. + + Parameters: + args (Dict[str, Any]): A dictionary containing the arguments for the + column selector. Must include 'action' and 'columns'. + + Returns: + List[str]: A list of errors as strings, if any. An empty list is returned + if no errors are detected. + """ + errors = [] + + action = args.get("action", "select") + if action not in ["select", "drop"]: + errors.append(f"Invalid action '{action}'. Must be 'select' or 'drop'") + + columns = args.get("columns", []) + if not isinstance(columns, list): + errors.append("columns must be a list") + elif not columns: + errors.append("columns list cannot be empty") + + return errors + + @classmethod + def _check_dependencies(cls, configs: List[IngestorProviderConfig]) -> List[str]: + """ + Checks for dependencies and conflicts among a list of ingestor configurations. + + This method ensures that column operations, such as renaming and filtering, do not + depend on columns that have been removed by prior operations. It analyzes the sequence + of configurations and identifies any conflicting dependencies which might cause runtime + issues. + + Parameters: + configs : List[IngestorProviderConfig] + List of configurations for ingestors that define various operations such as + column selection, renaming, filtering, etc. + + Returns: + List[str] + A list of error messages describing any conflicts detected between column operations + performed by the ingestors. Each message specifies the index and type of conflict + in the configurations. + """ + errors = [] + + # Check for column operations after column removal + column_removers = [] + column_users = [] + + for i, config in enumerate(configs): + if config.type == "column_selector" and config.args.get("action") == "drop": + column_removers.append((i, config.args.get("columns", []))) + elif config.type in ["column_renamer", "skip_row", "date_range_filter"]: + if config.type == "column_renamer": + column_users.append((i, list(config.args.get("mapping", {}).keys()))) + elif config.type == "skip_row": + fields = [cond.get("field") for cond in config.args.get("conditions", [])] + column_users.append((i, fields)) + elif config.type == "date_range_filter": + column_users.append((i, [config.args.get("date_field")])) + + # Check if any column user comes after a column remover that removes its columns + for user_idx, user_columns in column_users: + for remover_idx, removed_columns in column_removers: + if remover_idx < user_idx: # Remover comes before user + conflicts = set(user_columns) & set(removed_columns) + if conflicts: + errors.append(f"Ingestor {user_idx+1} uses columns {list(conflicts)} that were removed by ingestor {remover_idx+1}") + + return errors \ No newline at end of file diff --git a/document-graph/src/document_graph/ingest/list_ingestors.py b/document-graph/src/document_graph/ingest/list_ingestors.py new file mode 100644 index 00000000..88e09d63 --- /dev/null +++ b/document-graph/src/document_graph/ingest/list_ingestors.py @@ -0,0 +1,89 @@ +"""API to list available ingestors.""" + +from .ingestors_provider_registry import IngestorProviderRegistry + +# Import registration modules to ensure ingestors are registered +try: + from .row_level.register_ingestors import * +except ImportError: + # Handle hyphenated directory names + import importlib + row_module = importlib.import_module('.row_level.register_ingestors', package=__package__) + +try: + from .column_level.register_ingestors import * +except ImportError: + # Handle hyphenated directory names + import importlib + col_module = importlib.import_module('.column_level.register_ingestors', package=__package__) + +# Import field_level ingestors (no register_ingestors file needed, they're in registry) +# The numeric_id_cleanup is already registered in ingestors_provider_registry.py + +def list_available_ingestors(): + """ + Lists all available ingestor providers. + + This function retrieves and returns a list of all registered ingestor + providers from the IngestorProviderRegistry. + + Returns: + list: A list of available ingestor providers. + """ + return IngestorProviderRegistry.list_providers() + +def list_row_level_ingestors(): + """ + Lists all row-level ingestors. + + This function retrieves the list of all ingestors available in the + IngestorProviderRegistry and filters only those that operate at the + row level. It checks against predefined row-level ingestors and + returns a filtered list. + + Returns: + list[str]: A list of names of ingestors that process data at + the row level. + """ + all_ingestors = IngestorProviderRegistry.list_providers() + row_level = ["skip_row", "date_range_filter"] + return [ing for ing in all_ingestors if ing in row_level] + +def list_column_level_ingestors(): + """ + Filters and retrieves column-level ingestors from a list of all available ingestors. + + This function identifies specific ingestors operating at the column level, such as + selectors, renamers, reorders, and type converters, from the general list of + available ingestor providers. + + Returns + ------- + list[str] + A list of ingestor names corresponding to column-level operations. + """ + all_ingestors = IngestorProviderRegistry.list_providers() + column_level = ["column_selector", "column_renamer", "column_reorder", "column_type_converter"] + return [ing for ing in all_ingestors if ing in column_level] + +def list_field_level_ingestors(): + """ + Lists all field-level ingestors available in the IngestorProviderRegistry. + + This function retrieves a list of all ingestors registered in the + IngestorProviderRegistry and filters them to return only those that + operate at the field level. Field-level ingestors are predefined + and identified within the function. + + Returns: + list: A list of ingestor names that operate at the field level. + """ + all_ingestors = IngestorProviderRegistry.list_providers() + field_level = ["numeric_id_cleanup"] + return [ing for ing in all_ingestors if ing in field_level] + +if __name__ == "__main__": + print("Available Ingestors:") + print("Row-level:", list_row_level_ingestors()) + print("Column-level:", list_column_level_ingestors()) + print("Field-level:", list_field_level_ingestors()) \ No newline at end of file diff --git a/document-graph/src/document_graph/ingest/row/__init__.py b/document-graph/src/document_graph/ingest/row/__init__.py new file mode 100644 index 00000000..fbdb5365 --- /dev/null +++ b/document-graph/src/document_graph/ingest/row/__init__.py @@ -0,0 +1,10 @@ +"""Row-level ingestors — filtering and skipping rows during ingestion.""" +# Row-level ingestors + +from document_graph.ingest.row.date_range_filter import DateRangeFilterProvider +from .skip_row import SkipRowProvider + +__all__ = [ + 'DateRangeFilterProvider', + 'SkipRowProvider' +] diff --git a/document-graph/src/document_graph/ingest/row/date_range_filter.py b/document-graph/src/document_graph/ingest/row/date_range_filter.py new file mode 100644 index 00000000..090a18d3 --- /dev/null +++ b/document-graph/src/document_graph/ingest/row/date_range_filter.py @@ -0,0 +1,82 @@ +"""Date range filter — filters DataFrame rows based on date range criteria.""" +import pandas as pd +from datetime import datetime, timedelta, timezone +from typing import Optional +from document_graph.ingest.ingestors_provider_base import IngestorProvider + + +class DateRangeFilterProvider(IngestorProvider): + """Filter rows based on date ranges.""" + + def ingest(self, data: pd.DataFrame) -> pd.DataFrame: + date_field = self.args.get("date_field") + if not date_field or date_field not in data.columns: + return data + + # Convert to datetime if not already + date_column = pd.to_datetime(data[date_field], errors='coerce') + + # Start with all rows included + mask = pd.Series([True] * len(data), index=data.index) + + # Handle different date range specifications + start_date = self._parse_date(self.args.get("start_date")) + end_date = self._parse_date(self.args.get("end_date")) + + # Relative date ranges + days_back = self.args.get("days_back") + weeks_back = self.args.get("weeks_back") + months_back = self.args.get("months_back") + + # Auto-detect timezone handling - match data timezone + data_has_tz = date_column.dt.tz is not None + + # Calculate relative dates matching data timezone + if data_has_tz: + now = datetime.now(timezone.utc) # Use UTC for timezone-aware data + else: + now = datetime.now() # Use naive datetime for naive data + + if days_back: + start_date = now - timedelta(days=days_back) + elif weeks_back: + start_date = now - timedelta(weeks=weeks_back) + elif months_back: + start_date = now - timedelta(days=months_back * 30) # Approximate + + # Apply date filters - ensure timezone compatibility + def make_compatible(dt, target_has_tz): + if target_has_tz and dt.tzinfo is None: + return dt.replace(tzinfo=timezone.utc) + elif not target_has_tz and dt.tzinfo is not None: + return dt.replace(tzinfo=None) + return dt + + if start_date: + start_date = make_compatible(start_date, data_has_tz) + mask &= (date_column >= start_date) + if end_date: + end_date = make_compatible(end_date, data_has_tz) + mask &= (date_column <= end_date) + + # Handle null dates + include_null = self.args.get("include_null", False) + if not include_null: + mask &= date_column.notna() + + return data[mask] + + def _parse_date(self, date_value) -> Optional[datetime]: + """Parse various date formats.""" + if not date_value: + return None + + if isinstance(date_value, str): + try: + return pd.to_datetime(date_value) + except: + return None + elif isinstance(date_value, datetime): + return date_value + + return None \ No newline at end of file diff --git a/document-graph/src/document_graph/ingest/row/register_ingestors.py b/document-graph/src/document_graph/ingest/row/register_ingestors.py new file mode 100644 index 00000000..0363fc03 --- /dev/null +++ b/document-graph/src/document_graph/ingest/row/register_ingestors.py @@ -0,0 +1,9 @@ +"""Register row ingestors — registers all row-level ingestor providers.""" +# Register all row_level ingestors +from ..ingestors_provider_registry import IngestorProviderRegistry +from .skip_row import SkipRowProvider +from .date_range_filter import DateRangeFilterProvider + +# Register row_level ingestors +IngestorProviderRegistry.register("skip_row", SkipRowProvider) +IngestorProviderRegistry.register("date_range_filter", DateRangeFilterProvider) \ No newline at end of file diff --git a/document-graph/src/document_graph/ingest/row/skip_row.py b/document-graph/src/document_graph/ingest/row/skip_row.py new file mode 100644 index 00000000..867ca241 --- /dev/null +++ b/document-graph/src/document_graph/ingest/row/skip_row.py @@ -0,0 +1,71 @@ +"""Skip row — skips DataFrame rows based on flexible conditions.""" +import pandas as pd +from typing import List, Dict, Any, Union +from ..ingestors_provider_base import IngestorProvider + +class SkipRowProvider(IngestorProvider): + """Skip rows based on flexible conditions.""" + + def ingest(self, data: pd.DataFrame) -> pd.DataFrame: + conditions = self.args.get("conditions", []) + if not conditions: + return data + + # Start with all rows included + mask = pd.Series([True] * len(data), index=data.index) + + for condition in conditions: + field = condition["field"] + operator = condition["operator"] + value = condition.get("value") + + # Apply condition logic + if operator == "eq": # equals + mask &= (data[field] == value) + elif operator == "ne": # not equals + mask &= (data[field] != value) + elif operator == "lt": # less than + mask &= (data[field] < value) + elif operator == "le": # less than or equal + mask &= (data[field] <= value) + elif operator == "gt": # greater than + mask &= (data[field] > value) + elif operator == "ge": # greater than or equal + mask &= (data[field] >= value) + elif operator == "in": # in list + mask &= data[field].isin(value) + elif operator == "not_in": # not in list + mask &= ~data[field].isin(value) + elif operator == "contains": # string contains + mask &= data[field].str.contains(str(value), na=False) + elif operator == "not_contains": # string doesn't contain + mask &= ~data[field].str.contains(str(value), na=False) + elif operator == "startswith": # string starts with + mask &= data[field].str.startswith(str(value), na=False) + elif operator == "endswith": # string ends with + mask &= data[field].str.endswith(str(value), na=False) + elif operator == "is_null": # field is null/NaN + mask &= data[field].isna() + elif operator == "not_null": # field is not null/NaN + mask &= data[field].notna() + elif operator == "is_empty": # field is empty string + mask &= (data[field] == "") + elif operator == "not_empty": # field is not empty string + mask &= (data[field] != "") + elif operator == "regex": # regex match + mask &= data[field].str.match(str(value), na=False) + elif operator == "length_eq": # string length equals + mask &= (data[field].str.len() == value) + elif operator == "length_gt": # string length greater than + mask &= (data[field].str.len() > value) + elif operator == "length_lt": # string length less than + mask &= (data[field].str.len() < value) + else: + raise ValueError(f"Unsupported operator: {operator}") + + # Return rows that match the conditions (keep=True) or don't match (keep=False) + keep_matching = self.args.get("keep_matching", True) + if keep_matching: + return data[mask] + else: + return data[~mask] \ No newline at end of file diff --git a/document-graph/src/document_graph/model.py b/document-graph/src/document_graph/model.py new file mode 100644 index 00000000..c71a4559 --- /dev/null +++ b/document-graph/src/document_graph/model.py @@ -0,0 +1,66 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Document Graph Data Models. + +This module defines Pydantic models for graph nodes and edges. +""" + +import logging +from pydantic import BaseModel +from typing import Dict + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + +class NodeModel(BaseModel): + """Pydantic model for graph nodes. + + Attributes: + id: Unique identifier for the node + type: Node type/label + properties: Dictionary of node properties + + Examples: + >>> node = NodeModel( + ... id="doc-123", + ... type="Document", + ... properties={"title": "Example Document", "created_date": "2025-07-25"} + ... ) + >>> node.id + 'doc-123' + >>> node.type + 'Document' + >>> node.properties["title"] + 'Example Document' + """ + id: str + type: str + properties: Dict[str, object] + +class EdgeModel(BaseModel): + """Pydantic model for graph edges. + + Attributes: + source: Source node identifier + target: Target node identifier + type: Edge type/relationship + properties: Dictionary of edge properties + + Examples: + >>> edge = EdgeModel( + ... source="doc-123", + ... target="doc-456", + ... type="REFERENCES", + ... properties={"weight": 0.8, "created_date": "2025-07-25"} + ... ) + >>> edge.source + 'doc-123' + >>> edge.target + 'doc-456' + >>> edge.type + 'REFERENCES' + """ + source: str + target: str + type: str + properties: Dict[str, object] diff --git a/document-graph/src/document_graph/model_elements.py b/document-graph/src/document_graph/model_elements.py new file mode 100644 index 00000000..a14e95bb --- /dev/null +++ b/document-graph/src/document_graph/model_elements.py @@ -0,0 +1,163 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""[Module Name] Module for Document Graph Plugin System. + +This module [provides/defines/implements] [brief description of module purpose]. + +The module [includes/contains/offers] [key components or features]: +- [Component/Feature 1]: [Brief description] +- [Component/Feature 2]: [Brief description] +- [Component/Feature 3]: [Brief description] + +[Additional context about design principles, architecture, or implementation details] + +Usage: + # [Example usage code] + [explanation of example] +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +"""Graph Elements Module for Document Graph Storage. + +This module defines the fundamental data structures used to represent graph elements +(nodes and edges) in the document graph storage system. These classes serve as the +core data transfer objects between different components of the storage layer. + +The module provides two main dataclasses: +- Node: Represents a vertex in the graph with an ID, labels, and properties +- Edge: Represents a relationship between two nodes with source and target IDs, + a label, and properties + +These classes are designed to be simple, serializable data containers that can be +easily converted to and from various graph database formats. + +Usage: + + # Create a node + document_node = Node( + id="doc123", + labels=["Document"], + properties={"title": "Example Document", "created_at": "2023-01-01"} + ) + + # Create an edge + contains_edge = Edge( + id="edge456", + source_id="doc123", + target_id="section789", + label="CONTAINS", + properties={"order": 1} + ) +""" + +from typing import Dict, Any, Optional +from dataclasses import dataclass + + +@dataclass +class Node: + """ + Represents a node (vertex) in the document graph. + + A node is a fundamental element in a graph that represents an entity or concept. + In the document graph context, nodes typically represent documents, sections, + paragraphs, or other document elements. + + This class is implemented as a dataclass for simplicity and serialization support. + It provides default initialization for labels and properties if they are not provided. + + Attributes: + id (str): Unique identifier for the node. + labels (list): List of labels/types associated with this node. + Examples: ["Document", "Section", "Paragraph"]. + Defaults to an empty list if not provided. + properties (Dict[str, Any]): Dictionary of key-value properties for the node. + Examples: {"title": "Introduction", "created_at": "2023-01-01"}. + Defaults to an empty dictionary if not provided. + + Example: + >>> node = Node(id="doc123", labels=["Document"], properties={"title": "Example"}) + >>> print(node.id, node.labels) + doc123 ['Document'] + """ + + id: str + labels: list = None + properties: Dict[str, Any] = None + + def __post_init__(self): + """ + Initialize default values for properties and labels after instance creation. + + This method is automatically called by the dataclass after the object is created. + It ensures that properties and labels are never None by setting them to empty + collections when they are not provided during initialization. + + - Sets properties to an empty dict if None + - Sets labels to an empty list if None + """ + if self.properties is None: + self.properties = {} + if self.labels is None: + self.labels = [] + + +@dataclass +class Edge: + """ + Represents an edge (relationship) in the document graph. + + An edge is a connection between two nodes in a graph that represents a relationship + or association. In the document graph context, edges typically represent relationships + like "CONTAINS", "REFERENCES", "SIMILAR_TO", etc. between document elements. + + This class is implemented as a dataclass for simplicity and serialization support. + It provides default initialization for properties if they are not provided. + + Attributes: + id (str): Unique identifier for the edge. + source_id (str): Identifier of the source/from node. + target_id (str): Identifier of the target/to node. + label (str): Type of relationship this edge represents. + Examples: "CONTAINS", "REFERENCES", "SIMILAR_TO". + properties (Dict[str, Any]): Dictionary of key-value properties for the edge. + Examples: {"weight": 0.8, "created_at": "2023-01-01"}. + Defaults to an empty dictionary if not provided. + + Example: + >>> edge = Edge( + ... id="edge123", + ... source_id="doc1", + ... target_id="section2", + ... label="CONTAINS", + ... properties={"order": 1} + ... ) + >>> print(edge.source_id, edge.label, edge.target_id) + doc1 CONTAINS section2 + """ + + id: str + source_id: str + target_id: str + label: str + properties: Dict[str, Any] = None + + def __post_init__(self): + """ + Initialize default values for properties after instance creation. + + This method is automatically called by the dataclass after the object is created. + It ensures that properties is never None by setting it to an empty + dictionary when it is not provided during initialization. + + - Sets properties to an empty dict if None + """ + if self.properties is None: + self.properties = {} \ No newline at end of file diff --git a/document-graph/src/document_graph/pipeline/__init__.py b/document-graph/src/document_graph/pipeline/__init__.py new file mode 100644 index 00000000..635a9759 --- /dev/null +++ b/document-graph/src/document_graph/pipeline/__init__.py @@ -0,0 +1 @@ +"""Pipeline — extract, transform, and load pipeline components.""" diff --git a/document-graph/src/document_graph/pipeline/extract/__init__.py b/document-graph/src/document_graph/pipeline/extract/__init__.py new file mode 100644 index 00000000..ff034b0b --- /dev/null +++ b/document-graph/src/document_graph/pipeline/extract/__init__.py @@ -0,0 +1 @@ +"""Extract — data extraction providers for various file formats.""" diff --git a/document-graph/src/document_graph/pipeline/extract/extract_provider_base.py b/document-graph/src/document_graph/pipeline/extract/extract_provider_base.py new file mode 100644 index 00000000..f1118691 --- /dev/null +++ b/document-graph/src/document_graph/pipeline/extract/extract_provider_base.py @@ -0,0 +1,77 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Extract Provider Base module for document graph operations.""" + +import logging +from abc import ABC, abstractmethod +from document_graph.pipeline.extract.extract_provider_config import ExtractProviderConfig +from document_graph.config import DocumentGraphConfig +from document_graph.pipeline.extract.extraction_result import ExtractionResult +from document_graph.schema.etl_schema_model import ETLSchema + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available +# logging via graphrag-toolkit # noqa: F401 + + + +class ExtractProvider(ABC): + """ + Abstract base class for an extract provider. + + This class serves as a blueprint for creating extract providers that perform + ETL (Extract, Transform, Load) operations on structured document nodes. + The main purpose of this class is to define the interface and shared + functionality for concrete implementations. Subclasses are required to + implement the `extract` method. + + Attributes: + config: Extract provider configuration. + aws_config: AWS configuration for cloud operations. + """ + + def __init__(self, config: ExtractProviderConfig, aws_config: DocumentGraphConfig): + """Initialize extract provider with configuration. + + Args: + config: Extract provider configuration + aws_config: AWS configuration for cloud operations + """ + self.config = config + self.aws_config = aws_config + logger.debug(f"Initialized {self.__class__.__name__} with config: {config.type}") + + @abstractmethod + def extract(self, source: str, **kwargs) -> ExtractionResult: + """ + Represents an abstract base class for data extraction. + + This class defines an interface for extracting data from a given source string. + Subclasses are required to implement the `extract` method. + + Methods: + extract(source: str, **kwargs) -> ExtractionResult + + Abstract method intended to be implemented by subclasses. This method + extracts specific data from a provided string source using potential + additional arguments. + + Raises: + NotImplementedError: If the method is not implemented in a deriving class. + + """ + pass + + def get_etl_schema(self) -> ETLSchema: + """ + Returns the ETL schema for the current configuration. + + The method retrieves the ETL schema object by loading it from the + config if it hasn't been loaded already. + + Returns: + ETLSchema: The schema object representing the ETL schema. + """ + return self.config.load_schema_if_needed() \ No newline at end of file diff --git a/document-graph/src/document_graph/pipeline/extract/extract_provider_config.py b/document-graph/src/document_graph/pipeline/extract/extract_provider_config.py new file mode 100644 index 00000000..508a7977 --- /dev/null +++ b/document-graph/src/document_graph/pipeline/extract/extract_provider_config.py @@ -0,0 +1,257 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Extract Provider Config module for document graph operations.""" + +import logging +from pydantic import BaseModel, Field +from typing import Literal, Dict, Any, Optional, TYPE_CHECKING + +from document_graph.schema.etl_schema_model import ETLSchema, ExtractConfig +from document_graph.schema.providers.schema_provider_factory import SchemaProviderFactory + +if TYPE_CHECKING: + pass + + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available +# logging via graphrag-toolkit # noqa: F401 + + + +class ExtractProviderConfig(BaseModel): + """ + Represents a configuration for an Extract Provider. + + This class provides a detailed configuration structure for extract providers, + facilitating integration with document sources of various types. It serves + as a foundational element for managing the extraction process, including + the ETL schema, pipeline arguments, transformers, and factory integrations. + + Attributes: + type (str): Type of extract provider. It can be one of the following values: + "csv", "json", "parquet", or "excel". + source (str): Path or URI to the document source. + document_id (str): Document ID used in node construction. + etl_schema (Optional[ETLSchema]): Complete ETL schema configuration. + schema_provider_config (Optional[Dict[str, Any]]): Schema provider configuration. + args (Dict[str, Any]): Arguments for Pandas file reader, such as separators + or encoding specifications. + pipeline_args (Dict[str, Any]): Configurations affecting pipeline behavior, + such as logging settings or skipping conditions. + transformers (Optional[list]): List of transformer configurations. + factory (Optional[S3ArtifactFactory]): Factory used to load sources or pipelines. + model_config (dict): Arbitrary types allowance configuration. + + Methods: + parameters -> Dict[str, Any]: + Retrieves backward-compatible alias for 'args'. + + schema_path -> Optional[str]: + Provides backward-compatible alias for the schema path. + + get_extract_config -> ExtractConfig: + Derives the extract configuration from ETL schema or defaults to the + type configuration. + + load_schema_if_needed -> Optional[ETLSchema]: + Loads ETL schema configuration dynamically if not already available. + + from_factory_source(cls, factory, source_name, pipeline_name=None, **kwargs) -> ExtractProviderConfig: + Creates configuration using a factory source artifact with optional + pipeline arguments. + + from_factory_pipeline(cls, factory, pipeline_name) -> ExtractProviderConfig: + Creates configuration using a factory pipeline artifact. + """ + type: Literal["csv", "json", "parquet", "excel"] = Field(..., description="Type of extract provider") + source: str = Field(..., description="Path or URI to the document source") + document_id: str = Field(..., description="Document ID used in node construction") + + # Optional schema integration + etl_schema: Optional[ETLSchema] = Field(None, description="Complete ETL schema configuration") + schema_provider_config: Optional[Dict[str, Any]] = Field(None, description="Schema provider configuration") + + # Clean separation of arguments + args: Dict[str, Any] = Field(default_factory=dict, + description="Arguments for Pandas file reader (e.g., sep, encoding, on_bad_lines)") + pipeline_args: Dict[str, Any] = Field(default_factory=dict, + description="Pipeline behavior (e.g., logging, skipping)") + transformers: Optional[list] = Field(default_factory=list, + description="List of transformer configurations") + + # Factory support + factory: Optional[Any] = Field(None, description="Artifact factory for loading sources/pipelines") + + model_config = {"arbitrary_types_allowed": True} + + @property + def parameters(self) -> Dict[str, Any]: + """ + Provides access to the parameters of an object. + + This property allows retrieval of the object's parameters in the form of a dictionary. + + Returns: + Dict[str, Any]: A dictionary containing the parameters of the object. + """ + return self.args + + @property + def schema_path(self) -> Optional[str]: + """ + Gets the schema path of the object. + + The `schema_path` property is intended to provide the schema location + for this specific instance, if available. If no schema path is available, + it will return `None`. + + Returns: + Optional[str]: The path to the schema or `None` if no schema path exists. + """ + return None + + def get_extract_config(self) -> ExtractConfig: + """ + Gets the extraction configuration. + + This method retrieves the extraction configuration based on the current ETL schema. + If no specific ETL schema is defined, it returns a default minimal configuration. + + Returns: + ExtractConfig: The extraction configuration object. + + """ + if self.etl_schema: + return self.etl_schema.extract + + # Default minimal config + return ExtractConfig( + source_type="file", + file_type=self.type + ) + + def load_schema_if_needed(self) -> Optional[ETLSchema]: + """ + Loads the ETL schema if it is not already loaded. + + This method attempts to load and return the ETL schema. If the schema is already + available in the `etl_schema` attribute, it will return that. If not, it will try to + load the schema using the provided schema provider configuration. In the event of an + error while loading the schema, it logs a warning and returns None. + + Raises: + Logs a warning if the schema loading process fails due to an invalid configuration + or other errors. + + Returns: + Optional[ETLSchema]: The loaded ETL schema, or None if the schema could not be loaded. + """ + if self.etl_schema: + return self.etl_schema + + if self.schema_provider_config: + try: + schema_provider = SchemaProviderFactory.create(self.schema_provider_config) + return schema_provider.load_schema() + except Exception as e: + # ErrorHandler removed + raise ValueError(f"Schema load failed: {e}") from e + + + + + logger.warning(f"Failed to load schema from provider: {error.message}") + return None + + return None + + @classmethod + def from_factory_source(cls, factory: Any, source_name: str, pipeline_name: str = None, **kwargs) -> "ExtractProviderConfig": + """ + Creates an instance of ExtractProviderConfig from a factory source. + + Detailed Summary: + This class method initializes an ExtractProviderConfig instance using a provided + S3ArtifactFactory and a source name. It pulls metadata and arguments from the specified + source artifact or associated pipeline if applicable. Additional parameters can be + customized using keyword arguments, with a fallback to source-defined attributes. + + Parameters: + factory (S3ArtifactFactory): The factory responsible for managing artifact storage + and retrieval. + source_name (str): The name of the source artifact within the factory to load data from. + pipeline_name (str, optional): The name of the pipeline artifact to extract additional + configuration arguments, if applicable. + **kwargs: Additional keyword arguments for custom configuration, with document_id + being extracted separately, defaulting to the source's name if not provided. + + Raises: + ValueError: If the specified source artifact cannot be found within the provided factory. + + Returns: + ExtractProviderConfig: An initialized configuration object for the extract provider. + """ + source_artifact = factory.load_source(source_name) + if not source_artifact: + raise ValueError(f"Source artifact '{source_name}' not found in factory") + + # Load args from pipeline or source metadata + args = {} + if pipeline_name: + pipeline_artifact = factory.load_pipeline(pipeline_name) + if pipeline_artifact: + args = pipeline_artifact.pipeline_config.get('source', {}).get('args', {}) + + # Fallback to source metadata processing_args + if not args and source_artifact.metadata: + args = source_artifact.metadata.get('processing_args', {}) + + # Extract document_id from kwargs to avoid duplicate + document_id = kwargs.pop('document_id', source_artifact.name) + + return cls( + factory=factory, + type=source_artifact.source_type, + source=f"factory://{source_name}", + document_id=document_id, + args=args, + **kwargs + ) + + @classmethod + def from_factory_pipeline(cls, factory: Any, pipeline_name: str) -> "ExtractProviderConfig": + """ + Creates an `ExtractProviderConfig` instance from a factory and pipeline name. + + This class method allows loading a pipeline artifact using the provided factory + and pipeline name, extracting configuration data, and constructing an instance + of `ExtractProviderConfig` with it. + + Parameters: + factory (S3ArtifactFactory): The factory instance used to load the pipeline artifact. + pipeline_name (str): The identifier of the pipeline whose artifact needs to + be loaded. + + Returns: + ExtractProviderConfig: A new instance of `ExtractProviderConfig` initialized with the + extracted configuration data. + + Raises: + ValueError: If the specified pipeline artifact is not found in the factory. + """ + pipeline_artifact = factory.load_pipeline(pipeline_name) + if not pipeline_artifact: + raise ValueError(f"Pipeline artifact '{pipeline_name}' not found in factory") + + config_data = pipeline_artifact.pipeline_config + return cls( + factory=factory, + type=config_data['source']['type'], + source=config_data['source']['path'], + document_id=config_data['extract']['document_id'], + args=config_data['source'].get('args', {}), + **config_data.get('extract', {}) + ) diff --git a/document-graph/src/document_graph/pipeline/extract/extract_provider_csv.py b/document-graph/src/document_graph/pipeline/extract/extract_provider_csv.py new file mode 100644 index 00000000..b642ed75 --- /dev/null +++ b/document-graph/src/document_graph/pipeline/extract/extract_provider_csv.py @@ -0,0 +1,413 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Extract Provider Csv module for document graph operations.""" + +import logging +from pathlib import Path +from typing import List, Tuple + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available +# logging via graphrag-toolkit # noqa: F401 +from document_graph.config import DocumentGraphConfig +from document_graph.pipeline.extract.extract_provider_base import ExtractProvider +from document_graph.pipeline.extract.extract_provider_config import ExtractProviderConfig +from document_graph.schema.providers.csv_schema_provider import CSVSchemaProvider +from document_graph.schema.etl_schema_model import ETLSchema +from document_graph.pipeline.extract.utils.dataframe_reader import DataFrameReader +# PathResolver removed + + + +from document_graph.schema.providers.schema_provider_config import SchemaProviderConfig + + + +class CSVExtractProvider(ExtractProvider): + """ + Provide functionality to extract data from a CSV file based on a specified ETL configuration. + + This class is responsible for handling CSV files, supporting local file paths or S3 URIs, + and extracting structured information while adhering to ETL schema configurations. It can + apply data transformations, infer schema, save skipped rows if necessary, and stream process + data when ingestors are provided. + + Attributes: + config (ExtractProviderConfig): Configuration details for the extraction provider. + aws_config (DocumentGraphConfig): AWS settings for operations like S3 file handling. + """ + + def __init__(self, config: ExtractProviderConfig, aws_config: DocumentGraphConfig): + """ + Initializes an instance of CSVExtractProvider. + + The initializer sets up the CSVExtractProvider with the provided configuration + objects. It logs the configurations and calls the parent class initializer + to ensure proper initialization. + + Parameters: + config (ExtractProviderConfig): Configuration specific to the extraction provider. + aws_config (DocumentGraphConfig): Configuration related to AWS Document Graph services. + """ + logger.debug(f"Initializing CSVExtractProvider with config: {vars(config)}") + logger.debug(f"AWS config: {vars(aws_config)}") + super().__init__(config, aws_config) + logger.debug(f"[CSVExtractProvider] args: {self.config.args}") + logger.debug(f"[CSVExtractProvider] pipeline_args: {self.config.pipeline_args}") + + def extract(self, source: str, ingestors=None, **kwargs) -> None: + """ + Extracts data from a CSV file at the given source and processes it into a structured format. + The extraction process includes validating the file's existence, handling any parsing errors, + and applying configurations for data transformation. Optionally, it integrates with ETL schemas, + infers the schema from the data, and generates document nodes for downstream processes. + + The function achieves robust CSV parsing and ensures compatibility with specific configurations, + including handling row limits and bad line strategies. + + Parameters: + source (str): The source path of the CSV file to extract data from. + ingestors: Optional list of ingestors to process data in a streaming fashion. + kwargs: Arbitrary keyword arguments for additional extraction configurations. + + Raises: + Exception: If errors occur during file parsing or data inference. + + Returns: + None + """ + logger.debug(f"[extract] Entry point kwargs: {kwargs}") + logger.debug(f"[extract] config.args received: {self.config.args}") + logger.debug(f"[extract] config.pipeline_args received: {self.config.pipeline_args}") + + logger.info(f"Starting CSV extraction from source: {source}") + logger.debug(f"extract called with kwargs: {kwargs}") + resolved_path = source + resolved_path = Path(resolved_path) + logger.debug(f"Resolved path: {resolved_path}") + + # Use centralized error handling + # ErrorHandler removed + assert Path(resolved_path).exists(), f"File not found: {resolved_path}" + + try: + # Filter out custom parameters before passing to pandas + logger.debug(f"Original config.args: {self.config.args}") + pandas_args = { + k: v for k, v in self.config.args.items() if k != 'output_skip_lines' + } + + # Add header config from transformers but never row limits + logger.debug(f"Transformers config: {self.config.transformers}") + if self.config.transformers: + for transformer in self.config.transformers: + if transformer.get('type') == 'row_to_node': + transformer_config = transformer.get('config', {}) + # Keep header override if explicitly provided + if 'header' in transformer_config: + pandas_args['header'] = transformer_config['header'] + + # Never let Transformers set row limits for Extract + pandas_args.pop('nrows', None) + pandas_args.pop('skiprows', None) + + logger.info(f"Final pandas_args: {pandas_args}") + + # Ensure pandas uses the correct engine for robust CSV parsing + if pandas_args.get("on_bad_lines") == "skip": + current_engine = pandas_args.get("engine", "c").lower() + if current_engine != "python": + logger.warning( + "on_bad_lines='skip' requires engine='python'; overriding engine to ensure compatibility." + ) + pandas_args["engine"] = "python" + logger.debug(f"Overridden pandas_args with engine='python': {pandas_args}") + + # Use streaming if ingestors provided + logger.debug(f"Ingestors parameter: {ingestors}") + logger.debug(f"Ingestors type: {type(ingestors)}") + if ingestors: + logger.info("Using streaming extraction with ingestors") + df = self._stream_with_ingestors(resolved_path, pandas_args, ingestors) + else: + logger.info("No ingestors provided, using regular extraction") + logger.debug(f"Reading CSV with final pandas args: {pandas_args}") + df = DataFrameReader(path=resolved_path, **pandas_args).read() + + logger.info(f"Successfully read CSV with shape: {df.shape}") + logger.debug(f"DataFrame dtypes: {df.dtypes.to_dict()}") + logger.debug(f"DataFrame head(2): {df.head(2).to_dict(orient='records')}") + + # Save skipped lines if requested + if self.config.args.get('output_skip_lines', False) and self.config.args.get('on_bad_lines') == 'skip': + self._save_skipped_lines(resolved_path) + + except Exception as e: + logger.error(f"Error during CSV parsing: {e}", exc_info=True) + raise RuntimeError(f"CSV parse error: {resolved_path}: {e}") from e + + if df.empty: + strict_mode = kwargs.get("strict", True) + logger.warning(f"DataFrame is empty, strict_mode: {strict_mode}") + error = ValueError(f"Empty data: {resolved_path}") if strict_mode else None + if error: + raise error + return [], {} + + logger.debug("Inferring schema from DataFrame") + schema = {col: str(df[col].dtype) for col in df.columns} + + logger.debug(f"Inferred schema with {len(schema)} fields: {schema}") + + logger.debug(f"Converting {len(df)} rows to document nodes") + nodes = [] + for i, row in df.iterrows(): + content = ", ".join(f"{k}: {v}" for k, v in row.items()) + node = dict( + id=f"{self.config.document_id}-row-{i}", + type="Row", + content=content, + metadata={"row_index": str(i)}) + nodes.append(node) + if nodes: + logger.debug(f"Sample node: id={nodes[0]['id']}, type={nodes[0]['type']}, metadata={nodes[0]['metadata']}") + + # Load ETL schema if configured + etl_schema = self.config.load_schema_if_needed() + if not etl_schema: + # Dynamically generate schema_id from document_id + document_id = self.config.document_id + schema_id = f"{document_id}-schema" if document_id else None + logger.debug(f"Generated schema_id: {schema_id}") + + # Separate args for schema discovery (exclude batching parameters) + schema_args = {k: v for k, v in pandas_args.items() + if k not in ['skiprows', 'nrows']} + + # Create schema from CSV file using SchemaProviderConfig with args + schema_config = SchemaProviderConfig( + type="csv", + schema_id=schema_id, + connection_config={"path": str(resolved_path), "args": schema_args} + ) + logger.debug(f"SchemaProviderConfig: {vars(schema_config)}") + csv_schema_provider = CSVSchemaProvider(schema_config) + etl_schema = csv_schema_provider.load_schema() + logger.info(f"Generated ETL schema from CSV: {etl_schema.schema_id}") + logger.debug(f"ETL schema details: {vars(etl_schema)}") + + # Create extraction result with ETL schema integration + from document_graph.pipeline.extract.extraction_result import ExtractionResult + + metadata = { + "source": source, + "extraction_type": "csv", + "etl_schema_id": etl_schema.schema_id if etl_schema else None, + "rows_processed": str(len(df)), + "schema_fields": str(len(schema)) + } + logger.debug(f"Extraction metadata: {metadata}") + + result = ExtractionResult( + document_id=self.config.document_id, + extracted_schema=schema, + nodes=nodes, + dataframe=df, + metadata=metadata + ) + + logger.info( + f"CSV extraction completed: {len(nodes)} nodes, ETL schema: {etl_schema.schema_id if etl_schema else 'None'}") + return result + + def _save_skipped_lines(self, csv_path: Path): + """ + Saves lines from a CSV file that were skipped during parsing to a separate file. + + This method processes a given CSV file, identifies lines that do not match the + expected field count based on the file's header, and writes the invalid lines to a + new file. The new file contains information about the skipped lines and their + discrepancies. + + Attributes + ---------- + None + + Parameters + ---------- + csv_path : Path + Path to the CSV file to be checked for skipped lines. + + Raises + ------ + Exception + If an exception occurs during reading, processing, or writing skipped lines, + it logs the warning including an exception traceback. + + Note + ---- + The output file containing the skipped lines is named with a `.skip` suffix before + the original file's extension. + """ + import pandas as pd + import csv + + skip_file = csv_path.with_suffix('.skip' + csv_path.suffix) + logger.debug(f"Checking for skipped lines, output: {skip_file}") + + try: + # Read all lines + with open(csv_path, 'r', encoding='utf-8') as f: + all_lines = f.readlines() + logger.debug(f"Total lines in file: {len(all_lines)}") + + # Read successfully parsed lines + pandas_args = {k: v for k, v in self.config.args.items() if k != 'output_skip_lines'} + logger.debug(f"pandas_args for reading good_df: {pandas_args}") + good_df = pd.read_csv(csv_path, **pandas_args) + logger.debug(f"Good DataFrame shape: {good_df.shape}") + + # Detect delimiter from first line + sniffer = csv.Sniffer() + delimiter = sniffer.sniff(all_lines[0]).delimiter if all_lines else ',' + logger.debug(f"Detected delimiter: '{delimiter}'") + + # Find lines with wrong field count + header_fields = len(all_lines[0].split(delimiter)) if all_lines else 0 + logger.debug(f"Header fields count: {header_fields}") + skipped_lines = [] + + for i, line in enumerate(all_lines[1:], 1): # Skip header + field_count = len(line.split(delimiter)) + if field_count != header_fields: + skipped_lines.append(f"# Line {i + 1}: Expected {header_fields} fields, got {field_count}\n") + skipped_lines.append(line) + + # Save skipped lines if any found + if skipped_lines: + with open(skip_file, 'w', encoding='utf-8') as f: + f.write(f"# Skipped lines from {csv_path.name}\n") + f.write(f"# Header: {all_lines[0]}") + f.writelines(skipped_lines) + logger.info(f"Saved {len(skipped_lines) // 2} skipped lines to: {skip_file}") + else: + logger.debug("No skipped lines detected") + + except Exception as e: + logger.warning(f"Could not save skipped lines: {e}", exc_info=True) + + def _stream_with_ingestors(self, csv_path, pandas_args, ingestors): + """ + Handles reading and processing a CSV file with error tolerance and memory handling. + + The method attempts to read the entire file first, applying custom error handling on bad lines. + If the full read fails, it falls back to reading the file in chunks, leveraging chunked processing + to handle memory constraints. The method integrates the use of ingestors for data filtering + and transformation, which is applied to data either fully or in chunks. + + Attributes: + csv_path (str): The path to the CSV file to be read. + pandas_args (dict): Additional arguments passed to pandas read_csv for configuration. + ingestors: An object or utility providing the `execute` method for applying transformations or + filtering to the loaded data. + + Parameters and Configuration: + csv_path: The file path of the CSV to process. + pandas_args: Contains any additional arguments needed for `pd.read_csv`. The function utilizes + these for fallback operations if required. + ingestors: Utilized to process and filter the read DataFrame. + + Raises: + Various exceptions may be generated on initial full read-pandas arguments under directly/during + ```""" + import pandas as pd + + # First try: Load entire file with robust error handling + logger.info(f"Attempting full file read: {csv_path}") + try: + # Custom bad line handler for debugging + def bad_line_handler(line): + """ + Handles lines considered as "bad" by logging a warning. + + This function is used to process lines that are identified as problematic + or invalid for further operations. It logs a warning with detailed + information about the bad line and then skips it by returning None. + + Parameters: + line (str): The input line identified as bad. The first 200 characters + of this line will be included in the warning message for debugging purposes. + + Returns: + None: Indicates that the bad line has been skipped. + """ + logger.warning(f"Skipping bad line: {repr(line[:200])}...") + return None + + # Read entire file with maximum error tolerance + full_df = pd.read_csv( + csv_path, + encoding='utf-8', + engine='python', + on_bad_lines=bad_line_handler, + low_memory=False, + quoting=1, # QUOTE_ALL + skipinitialspace=True + ) + logger.info(f"Full file loaded: {len(full_df)} rows") + + # Apply ingestors to full dataset + filtered_df = ingestors.execute(full_df) + logger.info(f"Ingestors applied: {len(full_df)} -> {len(filtered_df)} rows") + return filtered_df + + except Exception as e: + logger.warning(f"Full file read failed: {e}, trying chunked approach") + + # Fallback: Try chunked reading + chunk_size = 10000 + filtered_chunks = [] + chunk_args = {'encoding': pandas_args.get('encoding', 'utf-8')} + + logger.info(f"Pandas chunked reading: {csv_path}") + logger.info(f"Chunk args: {chunk_args}") + + try: + chunk_reader = pd.read_csv(csv_path, chunksize=chunk_size, iterator=True, **chunk_args) + + chunk_count = 0 + total_rows_read = 0 + + while True: + try: + chunk = next(chunk_reader) + chunk_count += 1 + total_rows_read += len(chunk) + logger.info(f"Processing chunk {chunk_count}: {len(chunk)} rows (total: {total_rows_read})") + + filtered_chunk = ingestors.execute(chunk) + if not filtered_chunk.empty: + filtered_chunks.append(filtered_chunk) + + except StopIteration: + logger.info("Reached end of file - no more chunks") + break + + logger.info(f"Pandas chunked reading completed: {chunk_count} chunks, {total_rows_read} total rows") + + if filtered_chunks: + result = pd.concat(filtered_chunks, ignore_index=True) + logger.info(f"Combined {len(filtered_chunks)} chunks -> {len(result)} rows") + return result + else: + return pd.DataFrame() + + except Exception as e: + logger.error(f"Pandas chunked reading failed: {e}", exc_info=True) + logger.warning("Falling back to pure pandas read_csv") + return pd.read_csv(csv_path, **pandas_args) + + + diff --git a/document-graph/src/document_graph/pipeline/extract/extract_provider_excel.py b/document-graph/src/document_graph/pipeline/extract/extract_provider_excel.py new file mode 100644 index 00000000..ac1ca531 --- /dev/null +++ b/document-graph/src/document_graph/pipeline/extract/extract_provider_excel.py @@ -0,0 +1,165 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Extract Provider Excel module for document graph operations.""" + +import logging +from pathlib import Path +from typing import List + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available +# logging via graphrag-toolkit # noqa: F401 +from document_graph.config import DocumentGraphConfig +from document_graph.pipeline.extract.extract_provider_base import ExtractProvider +from document_graph.pipeline.extract.extract_provider_config import ExtractProviderConfig +from document_graph.pipeline.extract.utils.dataframe_reader import DataFrameReader +# PathResolver removed +from document_graph.schema.document_node import DocumentNode +from document_graph.schema.schema_inference_utils import SchemaInferenceEngine +from document_graph.schema.schema_io import SchemaWriter +from document_graph.schema.providers.schema_provider_config import SchemaProviderConfig +from document_graph.schema.providers.excel_schema_provider import ExcelSchemaProvider + + + +class ExcelExtractProvider(ExtractProvider): + """ + Provides functionality for extracting, processing, and analyzing Excel data. + + This class is responsible for managing the extraction of Excel data into structured formats including + DataFrames and schema-based nodes. It supports resolving file paths, reading Excel data, applying + ingestors for preprocessing, inferring schemas, saving schemas with mock data, and generating document + nodes based on the content of each row. It also ensures handling of errors and validation during the + entire extraction process. + + Attributes: + config (ExtractProviderConfig): Configuration object for the extraction provider that stores + various extraction-related settings and details. + aws_config (DocumentGraphConfig): AWS-specific configuration for managing data storage, schema + saving, and other AWS-related operations. + """ + def __init__(self, config: ExtractProviderConfig, aws_config: DocumentGraphConfig): + """ + [Brief one-line description of the function's purpose] + + [Detailed explanation of what the function does and how it works] + + Args: + [param_name] ([param_type]): [Description of parameter] + [param_name] ([param_type], optional): [Description of parameter]. Defaults to [default_value]. + + Returns: + [return_type]: [Description of return value] + + Raises: + [exception_type]: [Conditions under which exception is raised] + """ + super().__init__(config, aws_config) + + def extract(self, source: str, ingestors=None, **kwargs): + """ + Extracts data from an Excel file, processes it, and returns a comprehensive + extraction result, including inferred schema, document nodes, and metadata. + + This method performs several actions, including resolving the file path, + verifying the file's existence, reading the Excel file into a DataFrame, + and applying specified ingestors if provided. It further processes the data + to infer its schema, saves the schema with mock data, and creates document nodes + from its rows. If an ETL schema is configured, it uses the schema; otherwise, it + generates an ETL schema dynamically. + + Parameters: + source (str): Source path to the Excel file to be extracted. + ingestors (optional): An optional ingestors instance used to process the DataFrame. + **kwargs: Arbitrary keyword arguments, e.g., `strict` for handling empty data. + + Raises: + Exception: If an error occurs while reading the Excel file or applying the + ingestors. + Error: When the DataFrame is empty, depending on the strict mode configuration. + + Returns: + ExtractionResult: An object encapsulating the document ID, extracted schema, + document nodes, processed DataFrame, and associated metadata. + + """ + logger.info(f"Starting Excel extraction from source: {source}") + resolved_path = PathResolver(self.aws_config).resolve(source) + resolved_path = Path(resolved_path) + logger.debug(f"Resolved path: {resolved_path}") + + # ErrorHandler removed + ErrorHandler.check_file_exists(str(resolved_path), "Excel extraction") + + try: + logger.debug(f"Reading Excel with args: {self.config.args}") + df = DataFrameReader(path=resolved_path, **self.config.args).read() + logger.info(f"Successfully read Excel with shape: {df.shape}") + + # Apply ingestors if provided + if ingestors: + logger.info("Applying ingestors to Excel data") + df = ingestors.execute(df) + except Exception as e: + raise ErrorHandler.file_parse_error(str(resolved_path), "excel", e) from e + + if df.empty: + strict_mode = kwargs.get("strict", True) + error = ErrorHandler.empty_data(str(resolved_path), strict_mode) + if error: + raise error + return [], {} + + logger.debug("Inferring schema from DataFrame") + schema_engine = SchemaInferenceEngine(df) + schema = schema_engine.infer_basic_schema() + logger.debug(f"Inferred schema with {len(schema)} fields") + + logger.debug("Saving schema with mock data") + schema_path = SchemaWriter(self.aws_config).save_schema_with_mock_data( + schema_dict=schema, + df=df, + source=source, + output_dir=Path("schemas"), + document_id=self.config.document_id, + ) + logger.info(f"Schema saved to: {schema_path}") + + logger.debug(f"Converting {len(df)} rows to document nodes") + nodes = [] + for i, row in df.iterrows(): + content = ", ".join(f"{k}: {v}" for k, v in row.items()) + nodes.append(DocumentNode( + id=f"{self.config.document_id}-row-{i}", + type="Row", + content=content, + metadata={"row_index": str(i)} + )) + + # Load ETL schema if provided in config, otherwise fall back to dynamic schema generation + etl_schema = self.config.load_schema_if_needed() + if not etl_schema: + schema_config = SchemaProviderConfig( + type="excel", + connection_config={"path": str(resolved_path)}, + ) + excel_schema_provider = ExcelSchemaProvider(schema_config) + etl_schema = excel_schema_provider.load_schema() + logger.info(f"Generated ETL schema from Excel: {etl_schema.schema_id}") + + from document_graph.pipeline.extract.extraction_result import ExtractionResult + return ExtractionResult( + document_id=self.config.document_id, + extracted_schema=schema, + nodes=nodes, + dataframe=df, + metadata={ + "source": source, + "extraction_type": "excel", + "etl_schema_id": etl_schema.schema_id if etl_schema else None, + "rows_processed": str(len(df)), + "schema_fields": str(len(schema)) + } + ) diff --git a/document-graph/src/document_graph/pipeline/extract/extract_provider_factory.py b/document-graph/src/document_graph/pipeline/extract/extract_provider_factory.py new file mode 100644 index 00000000..4c8c9b70 --- /dev/null +++ b/document-graph/src/document_graph/pipeline/extract/extract_provider_factory.py @@ -0,0 +1,77 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Extract Provider Factory module for document graph operations.""" + +import logging +from typing import Type + +from .extract_provider_base import ExtractProvider +from .extract_provider_config import ExtractProviderConfig +from .extract_provider_registry import extract_provider_registry +from ...document_graph_config import DocumentGraphConfig + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available +# logging via graphrag-toolkit # noqa: F401 + + + +class ExtractProviderFactory: + """ + Factory class for creating and retrieving extract provider instances. + + This class provides a mechanism for creating extract providers based on their + configuration and type. It utilizes a registry of available providers and + performs validation to ensure the requested provider type is supported. + The created provider is configured according to the supplied configurations. + """ + + @classmethod + def get_provider( + cls, + config: ExtractProviderConfig, + aws_config: DocumentGraphConfig + ) -> ExtractProvider: + """ + Creates and returns an instance of the appropriate `ExtractProvider` subclass based on the given + configuration and type. The provider's class is determined dynamically by looking up the + `extract_provider_registry` with the given type. If the type is not registered, an appropriate + validation error is raised. Logs detailed debug and informational messages for the creation process. + + Args: + config (ExtractProviderConfig): Configuration object specifying the type of extract provider + to create and any additional provider-specific configuration details. + aws_config (DocumentGraphConfig): Configuration object containing AWS-related settings for + document processing. + + Raises: + ErrorHandler.ValidationError: If the given `provider_type` is not recognized or not registered + in the `extract_provider_registry`. + + Returns: + ExtractProvider: An instance of the dynamically determined subclass of `ExtractProvider`. + """ + provider_type = config.type + logger.debug(f"Creating extract provider: type={provider_type}, config={config.dict()}") + logger.debug(f"Looking up provider for type: {provider_type}") + + try: + provider_cls: Type[ExtractProvider] = extract_provider_registry.get(provider_type) + logger.debug(f"Found provider class: {provider_cls.__name__}") + except KeyError: + from ...shared.error_handler import ErrorHandler + available_types = extract_provider_registry.list_providers() + logger.error(f"Unknown extract provider type: {provider_type}. Available: {available_types}") + raise ErrorHandler.validation_error( + "provider_type", + provider_type, + f"one of {available_types}" + ) + + logger.debug(f"Instantiating provider: {provider_cls.__name__}") + logger.info(f"Creating extract provider of type: {provider_type}") + provider = provider_cls(config=config, aws_config=aws_config) + logger.debug(f"Successfully created provider: {provider.__class__.__name__}") + return provider diff --git a/document-graph/src/document_graph/pipeline/extract/extract_provider_json.py b/document-graph/src/document_graph/pipeline/extract/extract_provider_json.py new file mode 100644 index 00000000..9d360fbd --- /dev/null +++ b/document-graph/src/document_graph/pipeline/extract/extract_provider_json.py @@ -0,0 +1,235 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Extract Provider Json module for document graph operations.""" + +import logging +from pathlib import Path +from typing import List + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available +# logging via graphrag-toolkit # noqa: F401 +from document_graph.config import DocumentGraphConfig +from document_graph.pipeline.extract.extract_provider_base import ExtractProvider +from document_graph.pipeline.extract.extract_provider_config import ExtractProviderConfig +from document_graph.pipeline.extract.utils.dataframe_reader import DataFrameReader +# PathResolver removed +from document_graph.schema.document_node import DocumentNode +from document_graph.schema.schema_inference_utils import SchemaInferenceEngine +from document_graph.schema.schema_io import SchemaWriter +from document_graph.schema.providers.schema_provider_config import SchemaProviderConfig +from document_graph.schema.providers.json_schema_provider import JSONSchemaProvider + + + +class JSONExtractProvider(ExtractProvider): + """ + Handles JSON data extraction and schema inference for a given source. + + This class extends the ExtractProvider base class and provides functionality + to read data from a JSON source, infer its schema, and generate document + nodes for downstream processing. It also supports integration with external + schema factories and provides mechanisms for ingesting additional data. + + Attributes: + config (ExtractProviderConfig): Configuration for the extract provider, which includes + document ID, extraction arguments, and schema handling details. + aws_config (DocumentGraphConfig): AWS-specific configuration, including factory settings and + schema management. + + Methods: + __init__(config: ExtractProviderConfig, aws_config: DocumentGraphConfig) + Initializes the JSONExtractProvider with the required configuration details. + extract(source: str, ingestors=None, **kwargs) + Extracts data from the JSON source, infers schema, and creates document nodes. + """ + + def __init__(self, config: ExtractProviderConfig, aws_config: DocumentGraphConfig): + """ + [Brief one-line description of the function's purpose] + + [Detailed explanation of what the function does and how it works] + + Args: + [param_name] ([param_type]): [Description of parameter] + [param_name] ([param_type], optional): [Description of parameter]. Defaults to [default_value]. + + Returns: + [return_type]: [Description of return value] + + Raises: + [exception_type]: [Conditions under which exception is raised] + """ + super().__init__(config, aws_config) + + def extract(self, source: str, ingestors=None, **kwargs): + """ + Extracts data from a JSON source, processes it, and returns an ExtractionResult object. + + This method performs the following actions: + 1. Resolves the path to the given source. + 2. Validates the existence of the file. + 3. Reads the JSON data into a DataFrame. + 4. Applies optional ingestors for data transformation. + 5. Infers and/or generates schemas for the data. + 6. Converts the DataFrame rows into document nodes. + 7. Returns an ExtractionResult containing the extracted information, + nodes, and metadata. + + The method supports strict mode for handling empty data. It also integrates + with schema factories for retrieving or storing schemas. + + Parameters: + source: str + Path to the JSON file to be extracted. + ingestors + Optional ingestors to be applied for data transformation. + **kwargs + Additional options for configuration. Supports "strict" flag + to determine behavior for empty data. + + Returns: + ExtractionResult + Contains the document ID, extracted schema, document nodes, + DataFrame, and metadata. + + Raises: + Exception + If an error occurs during reading, parsing, or schema handling. + ErrorHandler.empty_data + If the data is empty and strict mode is enabled. + """ + logger.info(f"Starting JSON extraction from source: {source}") + resolved_path = PathResolver(self.aws_config).resolve(source) + resolved_path = Path(resolved_path) + logger.debug(f"Resolved path: {resolved_path}") + + # ErrorHandler removed + ErrorHandler.check_file_exists(str(resolved_path), "JSON extraction") + + try: + logger.debug(f"Reading JSON with args: {self.config.args}") + df = DataFrameReader(path=resolved_path, **self.config.args).read() + logger.info(f"Successfully read JSON with shape: {df.shape}") + + # Apply ingestors if provided + if ingestors: + logger.info("Applying ingestors to JSON data") + df = ingestors.execute(df) + except Exception as e: + raise ErrorHandler.file_parse_error(str(resolved_path), "json", e) from e + + if df.empty: + strict_mode = kwargs.get("strict", True) + error = ErrorHandler.empty_data(str(resolved_path), strict_mode) + if error: + raise error + return [], {} + + logger.debug("Inferring schema from DataFrame") + schema_engine = SchemaInferenceEngine(df) + schema = schema_engine.infer_basic_schema() + logger.debug(f"Inferred schema with {len(schema)} fields") + + # Check factory for existing schema first (before local generation) + factory_schema_found = False + if hasattr(self.aws_config, 'factory') and self.aws_config.factory: + factory_schema = self.aws_config.factory.find_schema_for_source( + self.config.document_id, "1.0.0" + ) + if factory_schema: + logger.info(f"Using existing factory schema: {factory_schema.name}@{factory_schema.version}") + factory_schema_found = True + + # Only generate local schema if not found in factory + if not factory_schema_found: + logger.debug("Saving schema with mock data") + schema_path = SchemaWriter(self.aws_config).save_schema_with_mock_data( + schema_dict=schema, + df=df, + source=source, + output_dir=Path("schemas"), + document_id=self.config.document_id, + ) + logger.info(f"Schema saved to: {schema_path}") + else: + logger.info("Skipping local schema generation - using factory schema") + + logger.debug(f"Converting {len(df)} rows to document nodes") + nodes = [] + for i, row in df.iterrows(): + content = ", ".join(f"{k}: {v}" for k, v in row.items()) + nodes.append(DocumentNode( + id=f"{self.config.document_id}-row-{i}", + type="Row", + content=content, + metadata={"row_index": str(i)} + )) + + # Load ETL schema if configured + etl_schema = self.config.load_schema_if_needed() + if not etl_schema: + # Check factory for existing schema first + factory_schema = None + if hasattr(self.aws_config, 'factory') and self.aws_config.factory: + factory_schema = self.aws_config.factory.find_schema_for_source( + self.config.document_id, "1.0.0" + ) + if factory_schema: + logger.info(f"Using factory schema: {factory_schema.name}@{factory_schema.version}") + # Convert factory schema to ETL schema format if needed + etl_schema = factory_schema.schema_data + + # Generate new schema if not found in factory + if not factory_schema: + schema_config = SchemaProviderConfig( + type="json", + connection_config={"path": str(resolved_path)}, + ) + json_schema_provider = JSONSchemaProvider(schema_config) + etl_schema = json_schema_provider.load_schema() + logger.info(f"Generated ETL schema from JSON: {etl_schema.schema_id if hasattr(etl_schema, 'schema_id') else 'new-schema'}") + + # Store generated schema in factory + if hasattr(self.aws_config, 'factory') and self.aws_config.factory: + from document_graph.factory.schema_artifact import SchemaArtifact + import json + + # Convert ETLSchema to simple dict for JSON serialization + try: + schema_dict = { + "schema_id": getattr(etl_schema, 'schema_id', 'generated'), + "fields": schema, # Use the inferred basic schema + "document_id": self.config.document_id, + "source_type": "json" + } + schema_artifact = SchemaArtifact( + name=f"{self.config.document_id}-schema", + version="1.0.0", + description=f"Auto-generated schema for {self.config.document_id}", + schema_type="etl", + schema_data=schema_dict, + source_ref=f"{self.config.document_id}@1.0.0" + ) + schema_id = self.aws_config.factory.store_schema(schema_artifact) + logger.info(f"Stored schema in factory: {schema_id}") + except Exception as e: + logger.warning(f"Failed to store schema in factory: {e}") + + + from document_graph.pipeline.extract.extraction_result import ExtractionResult + return ExtractionResult( + document_id=self.config.document_id, + extracted_schema=schema, + nodes=nodes, + dataframe=df, + metadata={ + "source": source, + "extraction_type": "json", + "etl_schema_id": etl_schema.get('schema_id') if isinstance(etl_schema, dict) else (etl_schema.schema_id if etl_schema and hasattr(etl_schema, 'schema_id') else None), + "rows_processed": str(len(df)), + "schema_fields": str(len(schema)) + } + ) diff --git a/document-graph/src/document_graph/pipeline/extract/extract_provider_parquet.py b/document-graph/src/document_graph/pipeline/extract/extract_provider_parquet.py new file mode 100644 index 00000000..fafec833 --- /dev/null +++ b/document-graph/src/document_graph/pipeline/extract/extract_provider_parquet.py @@ -0,0 +1,160 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Extract Provider Parquet module for document graph operations.""" + +import logging +from pathlib import Path +from typing import List + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available +# logging via graphrag-toolkit # noqa: F401 +from document_graph.config import DocumentGraphConfig +from document_graph.pipeline.extract.extract_provider_base import ExtractProvider +from document_graph.pipeline.extract.extract_provider_config import ExtractProviderConfig +from document_graph.pipeline.extract.utils.dataframe_reader import DataFrameReader +# PathResolver removed +from document_graph.schema.document_node import DocumentNode +from document_graph.schema.schema_inference_utils import SchemaInferenceEngine +from document_graph.schema.schema_io import SchemaWriter +from document_graph.schema.providers.schema_provider_config import SchemaProviderConfig +from document_graph.schema.providers.parquet_schema_provider import ParquetSchemaProvider + + + +class ParquetExtractProvider(ExtractProvider): + """ + A provider for extracting data from Parquet files. + + This class is designed to handle the process of reading, processing, and + extracting data from Parquet files. It leverages configurable components + to provide flexibility in handling various data extraction and schema + inference scenarios. The main functionalities include resolving file paths, + reading Parquet files, schema inference, and metadata preparation. + + Attributes: + config (ExtractProviderConfig): Configuration for the extraction process. + aws_config (DocumentGraphConfig): Configuration for AWS-related operations. + """ + + def __init__(self, config: ExtractProviderConfig, aws_config: DocumentGraphConfig): + """ + Initializes an instance of the class. + + This constructor initializes the class instance by calling the parent class + constructor with the provided configuration objects. It ensures that the + necessary configurations are passed for operation. + + Args: + config (ExtractProviderConfig): The configuration object containing + settings for the extract provider. + aws_config (DocumentGraphConfig): The configuration object specific + to the document graph service or AWS-related settings. + """ + super().__init__(config, aws_config) + + def extract(self, source: str, ingestors=None, **kwargs): + """ + Extracts data from a source in Parquet format, processes it, and infers schema and + document nodes. + + This method handles different aspects of the Parquet extraction pipeline including + path resolution, Parquet file reading, schema inference, and document node creation. + Additionally, it supports applying custom ingestors to transform the data. The method + saves the inferred schema along with mock data and returns all results in an + ExtractionResult object. + + Raises: + FileNotFoundError: If the specified Parquet source file does not exist. + RuntimeError: If the schema inference or document node creation fails. + ValueError: If the extracted data is empty and strict mode is enabled. + + Args: + source (str): The source path of the Parquet file. + ingestors (optional): Object containing ingestors to process the DataFrame. Defaults to None. + **kwargs: Additional keyword arguments for configuring extraction behavior. + + Returns: + ExtractionResult: Result of the extraction process, including document nodes, + inferred schema, processed DataFrame, and metadata. + """ + logger.info(f"Starting Parquet extraction from source: {source}") + resolved_path = PathResolver(self.aws_config).resolve(source) + resolved_path = Path(resolved_path) + logger.debug(f"Resolved path: {resolved_path}") + + # ErrorHandler removed + ErrorHandler.check_file_exists(str(resolved_path), "Parquet extraction") + + try: + logger.debug(f"Reading Parquet with args: {self.config.args}") + df = DataFrameReader(path=resolved_path, **self.config.args).read() + logger.info(f"Successfully read Parquet with shape: {df.shape}") + + # Apply ingestors if provided + if ingestors: + logger.info("Applying ingestors to Parquet data") + df = ingestors.execute(df) + except Exception as e: + raise ErrorHandler.csv_parse_error(str(resolved_path), e) from e + + if df.empty: + strict_mode = kwargs.get("strict", True) + error = ErrorHandler.empty_data(str(resolved_path), strict_mode) + if error: + raise error + return [], {} + + logger.debug("Inferring schema from DataFrame") + schema_engine = SchemaInferenceEngine(df) + schema = schema_engine.infer_basic_schema() + logger.debug(f"Inferred schema with {len(schema)} fields") + + logger.debug("Saving schema with mock data") + schema_path = SchemaWriter(self.aws_config).save_schema_with_mock_data( + schema_dict=schema, + df=df, + source=source, + output_dir=Path("schemas"), + document_id=self.config.document_id, + ) + logger.info(f"Schema saved to: {schema_path}") + + logger.debug(f"Converting {len(df)} rows to document nodes") + nodes = [] + for i, row in df.iterrows(): + content = ", ".join(f"{k}: {v}" for k, v in row.items()) + nodes.append(DocumentNode( + id=f"{self.config.document_id}-row-{i}", + type="Row", + content=content, + metadata={"row_index": str(i)} + )) + + # Load ETL schema if configured + etl_schema = self.config.load_schema_if_needed() + if not etl_schema: + schema_config = SchemaProviderConfig( + type="parquet", + connection_config={"path": str(resolved_path)}, + ) + parquet_schema_provider = ParquetSchemaProvider(schema_config) + etl_schema = parquet_schema_provider.load_schema() + logger.info(f"Generated ETL schema from Parquet: {etl_schema.schema_id}") + + from document_graph.pipeline.extract.extraction_result import ExtractionResult + return ExtractionResult( + document_id=self.config.document_id, + extracted_schema=schema, + nodes=nodes, + dataframe=df, + metadata={ + "source": source, + "extraction_type": "parquet", + "etl_schema_id": etl_schema.schema_id if etl_schema else None, + "rows_processed": str(len(df)), + "schema_fields": str(len(schema)) + } + ) diff --git a/document-graph/src/document_graph/pipeline/extract/extract_provider_registry.py b/document-graph/src/document_graph/pipeline/extract/extract_provider_registry.py new file mode 100644 index 00000000..ae9670e5 --- /dev/null +++ b/document-graph/src/document_graph/pipeline/extract/extract_provider_registry.py @@ -0,0 +1,98 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Registry for extract provider implementations. + +Maintains a mapping of provider type names to their concrete implementations. +New extract providers should be registered here to be available through +the ExtractProviderFactory. + +This module provides a registry for extract provider classes, allowing them to be +dynamically registered and retrieved by name or type. The registry is a central +component of the extract framework, enabling the factory to instantiate providers +without direct dependencies on specific implementations. + +The registry follows a singleton pattern, with a single instance (extract_provider_registry) +that is used throughout the application to register and retrieve provider classes. +""" + +import logging +from typing import Dict, Type + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available +# logging via graphrag-toolkit # noqa: F401 + +from .extract_provider_base import ExtractProvider + + + +class ExtractProviderRegistry: + """ + Provides a registry for managing extract provider classes. + + The class enables registering, retrieving, and listing extract provider classes by their type + names. It is case-insensitive and supports runtime replacement of registered providers. This is + commonly used to handle dynamic provider configurations in a flexible and modular way. + """ + + _registry: Dict[str, Type[ExtractProvider]] = {} + + @classmethod + def register(cls, type_: str, provider_cls: Type[ExtractProvider]) -> None: + """ + Registers an extract provider with a specific type. + + This method associates a provider class with a specific `type_` and + stores it in the `_registry`. The type is stored in lowercase form + to ensure case-insensitive lookups. + + Parameters: + type_ (str): The case-insensitive type associated with the provider. + provider_cls (Type[ExtractProvider]): The class of the provider to be registered. + + Returns: + None + """ + logger.debug(f"Registering extract provider: {type_} -> {provider_cls.__name__}") + cls._registry[type_.lower()] = provider_cls + + @classmethod + def get(cls, type_: str) -> Type[ExtractProvider]: + """ + Retrieves an extract provider class registered for the specified type. + + This method is responsible for looking up an extract provider in the + registry using the specified type. If no provider is found for the given + type, a KeyError is raised. + + Args: + type_: The type of extract provider to retrieve. The type should be + provided as a string. + + Raises: + KeyError: If no extract provider is registered for the specified type. + + Returns: + A registered extract provider class corresponding to the specified type. + """ + key = type_.lower() + if key not in cls._registry: + raise KeyError(f"No extract provider registered for type='{type_}'") + return cls._registry[key] + + @classmethod + def list_providers(cls) -> list[str]: + """ + Provides a method to list all registered providers. + + Returns: + list[str]: A list containing the names of all providers currently + registered in the class-level registry. + """ + return list(cls._registry.keys()) + + +# Initialize registry instance +extract_provider_registry = ExtractProviderRegistry() diff --git a/document-graph/src/document_graph/pipeline/extract/extraction_result.py b/document-graph/src/document_graph/pipeline/extract/extraction_result.py new file mode 100644 index 00000000..6e4a179c --- /dev/null +++ b/document-graph/src/document_graph/pipeline/extract/extraction_result.py @@ -0,0 +1,14 @@ +"""Extraction result — output of the extract stage.""" +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional +import pandas as pd + +@dataclass +class ExtractionResult: + """Holds the output of an extraction stage including schema, nodes, and data.""" + + document_id: str = "" + extracted_schema: dict = field(default_factory=dict) + nodes: list = field(default_factory=list) + dataframe: Optional[pd.DataFrame] = None + metadata: Dict[str, Any] = field(default_factory=dict) diff --git a/document-graph/src/document_graph/pipeline/extract/utils/__init__.py b/document-graph/src/document_graph/pipeline/extract/utils/__init__.py new file mode 100644 index 00000000..058253e2 --- /dev/null +++ b/document-graph/src/document_graph/pipeline/extract/utils/__init__.py @@ -0,0 +1 @@ +"""Extract utilities — helper functions for data extraction.""" diff --git a/document-graph/src/document_graph/pipeline/extract/utils/csv_fixer.py b/document-graph/src/document_graph/pipeline/extract/utils/csv_fixer.py new file mode 100644 index 00000000..ba0d5d49 --- /dev/null +++ b/document-graph/src/document_graph/pipeline/extract/utils/csv_fixer.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 +"""CSV Fixer utility using document-graph schema for validation and repair.""" + +import json +import csv +import logging +from pathlib import Path +from typing import List, Dict, Any, Optional + +logger = logging.getLogger(__name__) + +class CSVFixer: + """ + Class to handle fixing malformed CSV files. + + This class supports fixing and processing CSV files based on a predefined schema. It uses a two-stage + process to handle the input, ensuring proper formatting of the records. The process identifies and corrects + issues such as incomplete records caused by multiline fields and provides logging for errors and corrections. + + Attributes + ---------- + schema : dict + Dictionary extracting column metadata and optional fields from the given schema file. + expected_columns : list[str] + List of expected column names as per the schema. + expected_count : int + Total count of expected columns based on the schema. + optional_fields : set[str] + Set of column names identified as optional. + """ + + def __init__(self, schema_path: str): + """Initialize with schema file.""" + with open(schema_path, 'r') as f: + self.schema = json.load(f) + + self.expected_columns = list(self.schema['columns'].keys()) + self.expected_count = len(self.expected_columns) + self.optional_fields = set(self.schema.get('optional_fields', [])) + + logger.info(f"Schema loaded: {self.expected_count} columns expected") + + def _stage_lines(self, input_path: str, staging_path: Path): + """ + Stages the lines from an input file by copying them to a staging file. + + This method reads the contents of the specified input file line by line and + writes them to a designated staging file. Errors encountered during input file + reading are replaced to ensure the process completes without interruption. + + Parameters: + input_path (str): The path to the input file to be staged. + staging_path (Path): The path to the output staging file. + + Returns: + None + """ + with open(input_path, 'r', encoding='utf-8', errors='replace') as infile: + with open(staging_path, 'w', encoding='utf-8') as stagefile: + for line in infile: + stagefile.write(line) + + def fix_csv(self, input_path: str, output_path: str, badlog_path: Optional[str] = None) -> Dict[str, Any]: + """ + Fixes and processes a CSV file, identifying and resolving errors or inconsistencies such as incomplete records, + excess fields, or multiline fields. The method can optionally log bad lines to a specified file. + + Parameters: + input_path: str + Path to the input CSV file. + output_path: str + Path to the corrected output CSV file. + badlog_path: Optional[str] + Path to the log file where bad/incomplete lines will be recorded, if provided. + + Returns: + Dict[str, Any] + A dictionary containing processing statistics: + - 'total_lines': Total lines read from the input file. + - 'valid_records': Number of valid records written to the output file without modification. + - 'fixed_records': Number of records corrected and written. + - 'skipped_lines': Number of invalid or irrecoverable lines that were skipped. + - 'errors': List of error messages encountered during processing. + """ + # Stage 1: Read all lines into staging + staging_path = Path(output_path).with_suffix('.staging') + self._stage_lines(input_path, staging_path) + + # Stage 2: Process staged lines + stats = { + 'total_lines': 0, + 'valid_records': 0, + 'fixed_records': 0, + 'skipped_lines': 0, + 'errors': [] + } + + badlog_file = open(badlog_path, 'w', encoding='utf-8') if badlog_path else None + + with (open(staging_path, 'r', encoding='utf-8') as infile, open(output_path, 'w', encoding='utf-8', newline='') as outfile): + + writer = csv.writer(outfile, quoting=csv.QUOTE_MINIMAL) + + # Write header + writer.writerow(self.expected_columns) + + # Process line by line + current_record = [] + in_multiline_field = False + multiline_buffer = "" + multiline_line_count = 0 + MAX_MULTILINE_LINES = 10 + + for line_num, line in enumerate(infile, 1): + stats['total_lines'] += 1 + line = line.rstrip('\n\r') + + if line_num % 10000 == 0 or line_num > 78000: + print(f"Processing line {line_num}, records output: {stats['valid_records'] + stats['fixed_records']}, multiline: {in_multiline_field}") + + if line_num == 1: # Skip original header + continue + + try: + # Parse line as CSV + parsed = list(csv.reader([line]))[0] + + if not in_multiline_field: + # Start new record + if len(parsed) == self.expected_count: + # Perfect record + writer.writerow(parsed) + stats['valid_records'] += 1 + elif len(parsed) < self.expected_count: + # Potentially incomplete (multiline field) + current_record = parsed + in_multiline_field = True + multiline_buffer = line + multiline_line_count = 1 + else: + # Too many fields - log and skip + error_msg = f"Line {line_num}: Too many fields ({len(parsed)}) expected {self.expected_count}" + stats['errors'].append(error_msg) + if badlog_file: + badlog_file.write(error_msg + "\n") + stats['skipped_lines'] += 1 + else: + # Continue multiline field + multiline_line_count += 1 + + # Safety check: reset if too many lines accumulated + if multiline_line_count > MAX_MULTILINE_LINES: + # Output partial record with padding + while len(current_record) < self.expected_count: + current_record.append("") + writer.writerow(current_record[:self.expected_count]) + stats['fixed_records'] += 1 + + error_msg = f"Line {line_num}: Multiline field exceeded {MAX_MULTILINE_LINES} lines, outputting partial record" + stats['errors'].append(error_msg) + if badlog_file: + badlog_file.write(error_msg + "\n") + in_multiline_field = False + current_record = [] + multiline_buffer = "" + multiline_line_count = 0 + continue + + multiline_buffer += "\n" + line + + # Try to parse accumulated buffer + try: + accumulated = list(csv.reader([multiline_buffer]))[0] + if len(accumulated) == self.expected_count: + # Complete record found + writer.writerow(accumulated) + stats['fixed_records'] += 1 + in_multiline_field = False + current_record = [] + multiline_buffer = "" + multiline_line_count = 0 + elif len(accumulated) > self.expected_count: + # Overshot - reset and try current line as new record + error_msg = f"Line {line_num}: Multiline field overshot, resetting" + stats['errors'].append(error_msg) + if badlog_file: + badlog_file.write(error_msg + "\n") + in_multiline_field = False + current_record = [] + multiline_buffer = "" + multiline_line_count = 0 + # Retry current line as new record + if len(parsed) == self.expected_count: + writer.writerow(parsed) + stats['valid_records'] += 1 + except Exception: + # Continue accumulating + pass + + except Exception as e: + if in_multiline_field: + # Add to multiline buffer + multiline_buffer += "\n" + line + else: + error_msg = f"Line {line_num}: Parse error - {e}" + stats['errors'].append(error_msg) + if badlog_file: + badlog_file.write(error_msg + "\n") + stats['skipped_lines'] += 1 + + # Handle any remaining multiline buffer at end of file + if in_multiline_field and multiline_buffer: + try: + accumulated = list(csv.reader([multiline_buffer]))[0] + if len(accumulated) <= self.expected_count: + # Pad with empty strings if needed + while len(accumulated) < self.expected_count: + accumulated.append("") + writer.writerow(accumulated[:self.expected_count]) + stats['fixed_records'] += 1 + if badlog_file: + badlog_file.write(f"EOF: Flushed incomplete multiline record with {len(accumulated)} fields\n") + except Exception as e: + if badlog_file: + badlog_file.write(f"EOF: Failed to flush multiline buffer: {e}\n") + stats['skipped_lines'] += 1 + + if badlog_file: + badlog_file.close() + + # Cleanup staging file + try: + staging_path.unlink() + except Exception: + pass + + logger.info(f"CSV fix complete: {stats['valid_records']} valid, {stats['fixed_records']} fixed, {stats['skipped_lines']} skipped") + return stats + +def main(): + """ + Fixes a malformed CSV file using a provided JSON schema, saving the corrected + file and optionally logging errors. + + Parameters: + input_csv: str + The path to the input malformed CSV file. + schema_json: str + The path to the JSON file containing the schema to validate and fix the + CSV. + output_csv: str + The path to the output file for the fixed CSV. + badlog: str, optional + The path to the file where errors will be logged during the processing. + + Returns: + None + + Raises: + None + """ + import argparse + + parser = argparse.ArgumentParser(description="Fix malformed CSV using schema") + parser.add_argument("input_csv", help="Input CSV file") + parser.add_argument("schema_json", help="Schema JSON file") + parser.add_argument("output_csv", help="Output fixed CSV file") + parser.add_argument("--badlog", help="Write error log to file") + + args = parser.parse_args() + + fixer = CSVFixer(args.schema_json) + stats = fixer.fix_csv(args.input_csv, args.output_csv, args.badlog) + + print(f"Fixed CSV saved to: {args.output_csv}") + print(f"Stats: {stats}") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/document-graph/src/document_graph/pipeline/extract/utils/csv_line_normalizer.py b/document-graph/src/document_graph/pipeline/extract/utils/csv_line_normalizer.py new file mode 100644 index 00000000..c38f38ba --- /dev/null +++ b/document-graph/src/document_graph/pipeline/extract/utils/csv_line_normalizer.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Binary-level CSV line normalizer to fix corrupted line endings.""" + +import argparse +from pathlib import Path + +def normalize_csv_lines(input_path: str, output_path: str, expected_fields: int = 39): + """ + Normalizes lines in a CSV file to ensure proper field structure and formatting. + + This function reads a CSV file in binary mode, processes its lines to ensure + each record has the expected number of fields, and writes normalized lines + to an output file. Lines with incomplete records are adjusted to maintain + proper field continuity by replacing line breaks with spaces within fields. + + Attributes: + None + + Args: + input_path: str + Path to the input CSV file to be normalized. + output_path: str + Path to the output CSV file where normalized content will be saved. + expected_fields: int, optional + The number of fields expected in each record. Defaults to 39. + + Raises: + None + + Returns: + None + """ + + with open(input_path, 'rb') as infile, open(output_path, 'wb') as outfile: + buffer = b'' + field_count = 0 + in_quotes = False + line_count = 0 + + while True: + chunk = infile.read(8192) + if not chunk: + break + + buffer += chunk + + i = 0 + while i < len(buffer): + byte = buffer[i:i+1] + + if byte == b'"': + in_quotes = not in_quotes + elif byte == b',' and not in_quotes: + field_count += 1 + elif byte in (b'\n', b'\r') and not in_quotes: + # Check if we have the right number of fields + if field_count == expected_fields - 1: # -1 because last field doesn't have comma + # Proper record end - write LF + outfile.write(b'\n') + field_count = 0 + line_count += 1 + if line_count % 10000 == 0: + print(f"Processed {line_count} records") + else: + # Incomplete record - replace with space to continue field + outfile.write(b' ') + + # Skip any additional CR/LF bytes + while i + 1 < len(buffer) and buffer[i+1:i+2] in (b'\n', b'\r'): + i += 1 + else: + # Regular byte - pass through + outfile.write(byte) + + i += 1 + + # Keep last incomplete line in buffer + buffer = b'' + + print(f"Normalized {line_count} records") + +def main(): + """ + Parses command-line arguments to normalize CSV line endings at the binary level + and invokes the function for processing the CSV file. + + Summary: + This script provides functionality to take an input corrupted CSV file and an output file location, + normalize the corrupted line endings by considering the provided number of fields, + and save the normalized version into the output file. + + Args: + input_csv (str): Path to the input corrupted CSV file. + output_csv (str): Path where the output normalized CSV file should be saved. + fields (int, optional): The expected number of fields per record. Defaults to 39. + """ + parser = argparse.ArgumentParser(description="Normalize CSV line endings at binary level") + parser.add_argument("input_csv", help="Input corrupted CSV file") + parser.add_argument("output_csv", help="Output normalized CSV file") + parser.add_argument("--fields", type=int, default=39, help="Expected number of fields per record") + + args = parser.parse_args() + + normalize_csv_lines(args.input_csv, args.output_csv, args.fields) + print(f"Normalized CSV saved to: {args.output_csv}") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/document-graph/src/document_graph/pipeline/extract/utils/csv_to_json.py b/document-graph/src/document_graph/pipeline/extract/utils/csv_to_json.py new file mode 100644 index 00000000..dfa3c41d --- /dev/null +++ b/document-graph/src/document_graph/pipeline/extract/utils/csv_to_json.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Convert CSV to JSON Lines format.""" + +import pandas as pd +import json +import argparse + +def csv_to_jsonl(csv_path: str, json_path: str): + """ + Converts a CSV file to a JSON Lines file with additional data processing and cleaning. + + This function attempts to read a CSV file using multiple strategies to ensure resilience + against format inconsistencies. Once successfully read, the CSV data is processed + row-by-row to handle NaN values, clean problematic characters in string fields, + and parse predefined JSON fields. The processed data is then written to the specified + JSON Lines file. + + Parameters: + csv_path: str + The file path to the input CSV file. + json_path: str + The file path where the output JSON Lines file will be written. + + Raises: + ValueError + If all CSV reading strategies fail. + """ + # Try multiple CSV reading strategies + df = None + strategies = [ + {'sep': ',', 'quotechar': '"', 'quoting': 1}, # QUOTE_ALL + {'sep': ',', 'quotechar': '"', 'quoting': 0}, # QUOTE_MINIMAL + {'sep': ',', 'quotechar': '"', 'quoting': 3, 'skipinitialspace': True}, # QUOTE_NONE + {'sep': ',', 'on_bad_lines': 'skip'}, # Skip bad lines + ] + + for i, strategy in enumerate(strategies): + try: + print(f"Trying CSV read strategy {i+1}...") + df = pd.read_csv(csv_path, **strategy) + print(f"✅ Success with strategy {i+1}") + break + except Exception as e: + print(f"❌ Strategy {i+1} failed: {e}") + continue + + if df is None: + raise ValueError("All CSV reading strategies failed") + + print(f"Successfully read CSV with {len(df)} rows and {len(df.columns)} columns") + + with open(json_path, 'w', encoding='utf-8') as f: + for _, row in df.iterrows(): + # Convert pandas Series to dict, handling NaN values + record = row.to_dict() + # Replace NaN with None for proper JSON serialization + record = {k: (None if pd.isna(v) else v) for k, v in record.items()} + + # Clean string values - remove problematic characters + for k, v in record.items(): + if isinstance(v, str): + # Clean up common problematic characters + v = v.replace('\r\n', '\n').replace('\r', '\n') + # Ensure proper encoding + record[k] = v + + # Handle special JSON fields + json_fields = ['Resource original JSON', 'Evidence'] + for field in json_fields: + if field in record and record[field] and record[field] != 'NaN': + try: + # Try to parse as JSON + if isinstance(record[field], str): + record[field] = json.loads(record[field]) + except (json.JSONDecodeError, TypeError): + # Keep as string if not valid JSON + pass + + # Use json.dumps with ensure_ascii=False for proper encoding + f.write(json.dumps(record, ensure_ascii=False, separators=(',', ':')) + '\n') + + print(f"Converted {len(df)} records from CSV to JSON Lines") + +def main(): + """ + Main function for converting a CSV file into a JSON Lines file. + + This function initializes an argument parser to handle the input and output + file paths. It processes the command line arguments, and executes the + conversion using the csv_to_jsonl function. If the conversion is successful, + it displays a confirmation message; otherwise, it displays an error message + and raises the encountered exception. + + Raises: + Exception: Re-raises any exception encountered during the conversion + process. + """ + parser = argparse.ArgumentParser(description="Convert CSV to JSON Lines") + parser.add_argument("csv_file", help="Input CSV file") + parser.add_argument("json_file", help="Output JSON Lines file") + + args = parser.parse_args() + + try: + csv_to_jsonl(args.csv_file, args.json_file) + print(f"✅ Successfully converted {args.csv_file} to {args.json_file}") + except Exception as e: + print(f"❌ Conversion failed: {e}") + raise + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/document-graph/src/document_graph/pipeline/extract/utils/dataframe_reader.py b/document-graph/src/document_graph/pipeline/extract/utils/dataframe_reader.py new file mode 100644 index 00000000..fb91b8ed --- /dev/null +++ b/document-graph/src/document_graph/pipeline/extract/utils/dataframe_reader.py @@ -0,0 +1,13 @@ +"""DataFrame reader — wraps pandas read_csv with config.""" +import pandas as pd +from pathlib import Path + +class DataFrameReader: + """Reads data files into pandas DataFrames with configurable options.""" + + def __init__(self, path, **kwargs): + self.path = Path(path) + self.kwargs = kwargs + + def read(self) -> pd.DataFrame: + return pd.read_csv(self.path, **self.kwargs) diff --git a/document-graph/src/document_graph/pipeline/extract/utils/graph_element_utils.py b/document-graph/src/document_graph/pipeline/extract/utils/graph_element_utils.py new file mode 100644 index 00000000..b57ac33f --- /dev/null +++ b/document-graph/src/document_graph/pipeline/extract/utils/graph_element_utils.py @@ -0,0 +1,57 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""[Module Name] Module for Document Graph Plugin System. + +This module [provides/defines/implements] [brief description of module purpose]. + +The module [includes/contains/offers] [key components or features]: +- [Component/Feature 1]: [Brief description] +- [Component/Feature 2]: [Brief description] +- [Component/Feature 3]: [Brief description] + +[Additional context about design principles, architecture, or implementation details] + +Usage: + # [Example usage code] + [explanation of example] +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available +# logging via graphrag-toolkit # noqa: F401 + + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Graph element utilities for creating nodes, links, and edges. + +This module provides functions for creating graph elements such as text nodes and links/edges +between nodes. It delegates to the shared implementation in the graph module. +""" + +# Import from the authoritative shared location +from document_graph.shared.graph.graph_element_utils import ( + create_text_node, + create_link +) + +# Alias for backward compatibility +def create_edge(source: str, target: str, edge_type: str): + """Create an edge between two nodes in the graph. + + This is an alias for create_link, maintained for backward compatibility. + + Args: + source: The ID of the source node + target: The ID of the target node + edge_type: The type of edge to create + + Returns: + A dictionary representing the edge/link between the nodes + """ + return create_link(source, target, edge_type) diff --git a/document-graph/src/document_graph/pipeline/extract/utils/parsing_utils.py b/document-graph/src/document_graph/pipeline/extract/utils/parsing_utils.py new file mode 100644 index 00000000..a9553eec --- /dev/null +++ b/document-graph/src/document_graph/pipeline/extract/utils/parsing_utils.py @@ -0,0 +1,48 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Parsing utilities for text processing in document graph operations. + +This module provides utility functions for cleaning and parsing text data +before it is processed further in the document graph pipeline. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available +# logging via graphrag-toolkit # noqa: F401 + + +import re +from typing import List + +def clean_text(text: str) -> str: + """Clean text by removing excess whitespace. + + This function strips leading and trailing whitespace and normalizes + all internal whitespace sequences (spaces, tabs, newlines) to a single space. + + Args: + text: The input text to clean + + Returns: + The cleaned text with normalized whitespace + """ + return re.sub(r'\s+', ' ', text.strip()) + +def split_into_paragraphs(text: str) -> List[str]: + """Split text into logical paragraphs. + + This function divides text into paragraphs by looking for double line breaks + (i.e., an empty line between paragraphs). It strips whitespace from each + paragraph and filters out empty paragraphs. + + Args: + text: The input text to split into paragraphs + + Returns: + A list of paragraphs, with each paragraph as a string + """ + return [p.strip() for p in re.split(r'\n\s*\n', text) if p.strip()] diff --git a/document-graph/src/document_graph/pipeline/load/__init__.py b/document-graph/src/document_graph/pipeline/load/__init__.py new file mode 100644 index 00000000..70b02766 --- /dev/null +++ b/document-graph/src/document_graph/pipeline/load/__init__.py @@ -0,0 +1 @@ +"""Load — data loading providers for graph databases.""" diff --git a/document-graph/src/document_graph/pipeline/load/load_context.py b/document-graph/src/document_graph/pipeline/load/load_context.py new file mode 100644 index 00000000..4fb2fa22 --- /dev/null +++ b/document-graph/src/document_graph/pipeline/load/load_context.py @@ -0,0 +1,93 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Load Context module for document graph operations.""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available +# logging via graphrag-toolkit # noqa: F401 + + +from typing import Optional +from document_graph.pipeline.extract.extracted_data import ExtractedData +from document_graph.storage.graph.storage_provider_factory import StorageProviderFactory +from document_graph.storage.graph.validated_config import StorageProviderConfig + + +class LoadContext: + """ + Represents a context for loading and managing extracted data. + + This class is responsible for handling the management of data extracted from a source. + It provides utilities to access or create relevant storage mechanisms as well as + to interact with the data source registration. The class ensures that the data needed + for processing or analysis is properly handled and stored. + + Attributes: + extracted_data (ExtractedData): The extracted data object that includes information + about the data source and registration. + """ + + def __init__(self, extracted_data: ExtractedData): + """ + Initializes a new instance of the class with provided extracted data. + + Attributes: + extracted_data: ExtractedData + The data extracted and provided for initialization. + + Arguments: + extracted_data: ExtractedData + The extracted data to initialize the instance. + """ + self.extracted_data = extracted_data + self._storage = None + + @property + def storage(self): + """ + Provides accessor for the `storage` property of the class. The property ensures + that a storage provider instance is initialized and configured properly for + the class instance. If the storage has already been initialized, the existing + instance is returned. Otherwise, the storage configuration is built using the + graph information and a new storage provider instance is created. + + Raises: + ValueError: If the graph information is not found in the data registration. + + Returns: + StorageProvider: An instance of a storage provider initialized with the + corresponding configuration. + """ + if self._storage is None: + graph_info = self.extracted_data.source.get_graph_info() + if not graph_info: + raise ValueError(f"Graph {self.extracted_data.graph_id} not found in registration") + + # Create storage config from registration + # This would be configured based on the data plane setup + storage_config = StorageProviderConfig( + provider_type="graphjson", # Default, could be from registration + connection_config={"path": f"/tmp/{self.extracted_data.graph_id}.json"}, + tenant_id=self.extracted_data.tenant_id + ) + + self._storage = StorageProviderFactory.for_writer(storage_config) + + return self._storage + + @property + def registration(self): + """ + Provides access to the 'registration' property. + + This property retrieves and returns the value of the 'registration' attribute + from the 'extracted_data' object. + + Returns: + The registration value fetched from the 'extracted_data' object. + """ + return self.extracted_data.registration \ No newline at end of file diff --git a/document-graph/src/document_graph/pipeline/load/load_provider_base.py b/document-graph/src/document_graph/pipeline/load/load_provider_base.py new file mode 100644 index 00000000..c4ad3db3 --- /dev/null +++ b/document-graph/src/document_graph/pipeline/load/load_provider_base.py @@ -0,0 +1,170 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Load Provider Base module for document graph operations.""" + +import logging +from abc import ABC, abstractmethod +from typing import Dict, Any, List, Optional +from pydantic import BaseModel +from document_graph.pipeline.extract.extraction_result import ExtractionResult +from document_graph.pipeline.load.load_result import LoadResult +from document_graph.pipeline.load.transformers.transformation_plan import TransformationPlan + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available +# logging via graphrag-toolkit # noqa: F401 + + + +class LoadProviderConfig(BaseModel): + """ + Represents the configuration needed to load a provider. + + This class is used to define the necessary information for loading a + specific provider, including its name, type, optional parameters, and + optional configurations such as transformation plans or AWS-specific + settings. It extends the BaseModel class for structured data handling. + """ + name: str + type: str + parameters: Dict[str, Any] = {} + transformation_plan: Optional[TransformationPlan] = None + aws_config: Optional[Any] = None + + +class LoadProvider(ABC): + """ + Abstract base class for implementing load providers. + + The LoadProvider class defines the blueprint for any load provider that processes + and optionally transforms extracted data before persisting it in a target system. + It allows customization through a configuration and manages the lifecycle of loading + and transformation operations. + + Attributes: + config (LoadProviderConfig): Configuration for the load provider, including the transformation plan. + transformation_plan (TransformationPlan): Transformation plan detailing the steps to be applied + before persisting the data. + """ + + def __init__(self, config: LoadProviderConfig): + """Initialize load provider with configuration.""" + self.config = config + self.transformation_plan = config.transformation_plan + logger.debug(f"Initialized {self.__class__.__name__}: name={config.name}, type={config.type}") + logger.debug(f"Load provider parameters: {config.parameters}") + if self.transformation_plan: + logger.debug(f"Transformation plan configured with {len(self.transformation_plan.steps)} steps") + + def load(self, extraction_result: ExtractionResult) -> LoadResult: + """ + Load data into the target destination after applying necessary transformations. + + This method orchestrates the load operation, which involves optional data + transformations and persistence into the targeted destination. Logging is + employed to track each stage of the operation for debugging and informational + purposes. In case of failure, a failed load result is created and returned. + + Parameters: + extraction_result (ExtractionResult): The input data to be loaded, + containing the document ID and other necessary information. + + Returns: + LoadResult: Indicates the success or failure of the load operation, + including any relevant metadata or error details. + """ + logger.debug(f"Starting load operation: document={extraction_result.document_id}, provider={self.__class__.__name__}") + logger.info(f"Starting load operation for document: {extraction_result.document_id}") + + try: + if self.transformation_plan: + logger.debug(f"Applying {len(self.transformation_plan.steps)} transformation steps") + extraction_result = self._apply_transformations(extraction_result) + logger.debug(f"Transformations completed successfully") + + logger.debug(f"Persisting data using {self.__class__.__name__}") + result = self._persist(extraction_result) + logger.debug(f"Persistence completed: success={result.success}") + logger.info(f"Load operation completed for document: {extraction_result.document_id}") + return result + except Exception as e: + logger.error(f"Load operation failed for document {extraction_result.document_id}: {e}") + return LoadResult.create_failed( + document_id=extraction_result.document_id, + error_message=str(e) + ) + + def _apply_transformations(self, extraction_result: ExtractionResult) -> ExtractionResult: + """ + Applies a series of transformations to the given extraction result based on a predefined + transformation plan. Each transformation step is executed sequentially, modifying the + nodes or dataframe of the extraction result as specified. + + Parameters: + extraction_result (ExtractionResult): The initial extraction result to be transformed. + + Returns: + ExtractionResult: The transformed extraction result after applying all transformation + steps. + + Raises: + Exception: If any transformation step fails during the process. + """ + from document_graph.transformers.transformer_provider_factory import TransformerProviderFactory + from document_graph.transformers.transformer_provider_config import TransformerProviderConfig + + transformed_result = extraction_result + + for i, step in enumerate(self.transformation_plan.steps): + logger.debug(f"Applying transformation step {i + 1}/{len(self.transformation_plan.steps)}: {step.name}") + try: + transformer_config = TransformerProviderConfig( + name=step.name, + type=step.type or "transformer", + args=step.args or {} + ) + transformer = TransformerProviderFactory.get_provider(transformer_config) + + if transformed_result.nodes: + transformed_result.nodes = [ + transformer.transform(node, **(step.args or {})) + for node in transformed_result.nodes + ] + + if transformed_result.dataframe is not None: + transformed_result.dataframe = transformer.transform( + transformed_result.dataframe, **(step.args or {}) + ) + + logger.debug(f"Successfully applied transformation: {step.name}") + except Exception as e: + logger.error(f"Failed to apply transformation {step.name}: {e}") + raise + + return transformed_result + + @abstractmethod + def _persist(self, extraction_result: ExtractionResult) -> LoadResult: + """ + Represents an abstract method for persistence of extraction results. This method + must be implemented by subclasses to define how to persist the given + extraction result and return the corresponding load result. + + Parameters: + extraction_result: ExtractionResult + The result of some extraction process that needs to be persisted. + + Returns: + LoadResult + The result of the persistence operation. + + Raises: + NotImplementedError + If this method is not implemented in a subclass. + """ + pass + + def __repr__(self): + return f"<{self.__class__.__name__} name={self.config.name} type={self.config.type}>" diff --git a/document-graph/src/document_graph/pipeline/load/load_provider_config.py b/document-graph/src/document_graph/pipeline/load/load_provider_config.py new file mode 100644 index 00000000..91700fe6 --- /dev/null +++ b/document-graph/src/document_graph/pipeline/load/load_provider_config.py @@ -0,0 +1,208 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Load Provider Configuration Module. + +This module defines configuration classes for load providers in the +document graph pipeline. +""" + +import logging +from typing import Dict, Any, Optional +from pydantic import BaseModel, Field + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available +# logging via graphrag-toolkit # noqa: F401 + + + +class LoadProviderConfig(BaseModel): + """ + Represents the configuration for a load provider. + + Provides detailed setup, validation, and pre-defined methods to configure + load providers based on type. These providers can be file-based, database-based, + or S3 configurations. The class supports defining and customizing configurations + with specific attributes and capabilities depending on the provider type. + + Attributes: + name (str): Unique name for this load provider. + type (str): Load provider type (e.g., 'file', 'database', 's3'). + output_dir (Optional[str]): Output directory for file-based providers. + output_format (str): Output format (json, csv, parquet). + database_config (Optional[Dict[str, Any]]): Database connection configuration. + file_config (Optional[Dict[str, Any]]): File output configuration. + s3_config (Optional[Dict[str, Any]]): S3 output configuration. + parameters (Dict[str, Any]): Additional provider parameters. + batch_size (int): Batch size for processing. + overwrite (bool): Whether to overwrite existing output. + """ + + name: str = Field(..., description="Unique name for this load provider") + type: str = Field(..., description="Load provider type (e.g., 'file', 'database', 's3')") + output_dir: Optional[str] = Field(None, description="Output directory for file-based providers") + output_format: str = Field(default="json", description="Output format (json, csv, parquet)") + + # Database-specific configuration + database_config: Optional[Dict[str, Any]] = Field(None, description="Database connection configuration") + + # File-specific configuration + file_config: Optional[Dict[str, Any]] = Field(None, description="File output configuration") + + # S3-specific configuration + s3_config: Optional[Dict[str, Any]] = Field(None, description="S3 output configuration") + + # General parameters + parameters: Dict[str, Any] = Field(default_factory=dict, description="Additional provider parameters") + + # Processing options + batch_size: int = Field(default=1000, description="Batch size for processing") + overwrite: bool = Field(default=False, description="Whether to overwrite existing output") + + class Config: + """Pydantic configuration.""" + extra = "allow" # Allow additional fields + + def __post_init__(self): + """ + Logs initialization details of the LoadProviderConfig object. + + This method is automatically called after the object is fully + initialized, providing a debug log entry with the name and type + of the configuration. + + Raises: + None + """ + logger.debug(f"Initialized LoadProviderConfig: {self.name} ({self.type})") + + @classmethod + def create_file_config( + cls, + name: str, + output_dir: str, + output_format: str = "json", + **kwargs + ) -> "LoadProviderConfig": + """ + Creates a configuration for a file-based load provider. + + This class method generates and returns a LoadProviderConfig instance + with configuration tailored for file-based data load. It includes specific + parameters such as the file output directory and format and encapsulates + additional configuration options if provided. + + Parameters: + name: str + The name of the configuration. + output_dir: str + The directory where files will be output. + output_format: str, optional + The format of the output files. Defaults to "json". + kwargs: + Additional keyword arguments for custom configurations. + + Returns: + LoadProviderConfig + An instance of LoadProviderConfig initialized with the specified + parameters. + + """ + return cls( + name=name, + type="file", + output_dir=output_dir, + output_format=output_format, + file_config={ + "output_dir": output_dir, + "format": output_format + }, + **kwargs + ) + + @classmethod + def create_database_config( + cls, + name: str, + database_type: str, + connection_config: Dict[str, Any], + **kwargs + ) -> "LoadProviderConfig": + """ + Creates a LoadProviderConfig instance configured for a database load provider. + + This class method initializes a LoadProviderConfig object with parameters suited + to a database setup. The method associates the instance with a specific + database type and a set of connection configurations. Additional keyword arguments + can be supplied to further customize the configuration. + + Parameters: + name: str + The name of the load provider configuration. + database_type: str + The type of database being configured, such as 'PostgreSQL', 'MySQL', etc. + connection_config: Dict[str, Any] + A dictionary containing the database connection configuration settings. + **kwargs + Additional keyword arguments to customize the configuration. + + Returns: + LoadProviderConfig + An instance of LoadProviderConfig initialized with database-related settings. + """ + return cls( + name=name, + type="database", + database_config={ + "type": database_type, + "connection": connection_config + }, + **kwargs + ) + + @classmethod + def create_s3_config( + cls, + name: str, + bucket: str, + prefix: str = "", + output_format: str = "json", + **kwargs + ) -> "LoadProviderConfig": + """ + Creates a LoadProviderConfig instance for an s3 load provider. + + The method acts as a factory method to generate configurations specifically for + an s3-based data load provider. The s3 configuration includes the bucket name, + optional prefix for object paths, and the desired output format for processing. + + Parameters: + name: str + The name of the load provider configuration. + bucket: str + The bucket name of the s3 storage where data resides. + prefix: str, optional + The prefix of object paths within the s3 bucket. Default is an empty string. + output_format: str, optional + The format of output data (e.g., "json"). Default is "json". + **kwargs: dict + Additional arguments which will be passed to the LoadProviderConfig + constructor. + + Returns: + LoadProviderConfig + A configured instance of LoadProviderConfig representing an s3 load provider. + """ + return cls( + name=name, + type="s3", + output_format=output_format, + s3_config={ + "bucket": bucket, + "prefix": prefix, + "format": output_format + }, + **kwargs + ) \ No newline at end of file diff --git a/document-graph/src/document_graph/pipeline/load/load_result.py b/document-graph/src/document_graph/pipeline/load/load_result.py new file mode 100644 index 00000000..5a57d805 --- /dev/null +++ b/document-graph/src/document_graph/pipeline/load/load_result.py @@ -0,0 +1,189 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Load Result module for document graph operations.""" + +import logging +from typing import Optional, Dict, Any +from datetime import datetime, timezone +from pydantic import BaseModel + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available +# logging via graphrag-toolkit # noqa: F401 + + + +class LoadResult(BaseModel): + """ + Represents the outcome of a document load operation. + + This class is used to encapsulate the details and status of a load operation, + including the document identifier, records loaded, load time, overall status, + and additional metadata related to the load. It provides utility methods to + check the success or emptiness of the load result and class methods to create + specific types of results such as successful, empty, or failed load outcomes. + + Attributes: + document_id (str): Identifier of the document being loaded. + records_loaded (int): Number of records successfully loaded. Defaults to 0. + load_time (str): Timestamp when the load operation occurred. If not provided + during initialization, it will be automatically generated in UTC. + status (str): Status of the load operation, defaulting to "success". + transformations_applied (bool): Indicates whether transformations were applied + during the load process. Defaults to False. + output_files (Optional[Dict[str, str]]): Dictionary containing file paths for + generated output files. Defaults to None if not provided. + metadata (Optional[Dict[str, Any]]): Additional metadata related to the load + operation. Defaults to None if not provided. + """ + + document_id: str + records_loaded: int = 0 + load_time: str + status: str = "success" + transformations_applied: bool = False + output_files: Optional[Dict[str, str]] = None + metadata: Optional[Dict[str, Any]] = None + + def __init__(self, **data): + """ + Initializes a LoadResult object. + + This method sets the load_time for the object upon creation if it is not explicitly + provided in the input data. It then invokes the parent class initializer with the + given data. A debug log entry is generated for the initialization of a LoadResult + instance. + + Attributes: + load_time (str): A UTC timestamp indicating the object's creation time. If not + provided, it defaults to the current time. + + Parameters: + data (dict): Arbitrary keyword arguments representing the properties of the + LoadResult object. + + Returns: + None + """ + if 'load_time' not in data: + data['load_time'] = datetime.now(timezone.utc).isoformat() + super().__init__(**data) + logger.debug(f"Created LoadResult for document: {self.document_id}") + + def __str__(self) -> str: + """ + Converts the object to its string representation. + + Returns + ------- + str + A string that includes the document ID, the number of records loaded, and + the status of the operation. + """ + return f"LoadResult(document_id={self.document_id}, records_loaded={self.records_loaded}, status={self.status})" + + def is_successful(self) -> bool: + """ + Determines whether the operation was successful based on its status + and the number of records loaded. + + Returns: + bool: True if the status is "success" and at least one record was + loaded; otherwise, False. + """ + return self.status == "success" and self.records_loaded > 0 + + def is_empty(self) -> bool: + """ + Determines whether the collection is empty. + + This method evaluates whether the number of loaded records equals zero, + indicating that the collection is currently empty. + + Returns: + bool: True if the collection has no loaded records; otherwise, False. + """ + return self.records_loaded == 0 + + @classmethod + def create_success( + cls, + document_id: str, + records_loaded: int, + output_files: Optional[Dict[str, str]] = None, + transformations_applied: bool = False, + metadata: Optional[Dict[str, Any]] = None + ) -> "LoadResult": + """Create a successful load result. + + Args: + document_id: Document identifier + records_loaded: Number of records loaded + output_files: Dictionary of output file paths + transformations_applied: Whether transformations were applied + metadata: Additional metadata + + Returns: + LoadResult: Successful load result + """ + logger.info(f"Creating successful load result: {document_id} ({records_loaded} records)") + return cls( + document_id=document_id, + records_loaded=records_loaded, + status="success", + transformations_applied=transformations_applied, + output_files=output_files or {}, + metadata=metadata or {} + ) + + @classmethod + def create_empty( + cls, + document_id: str, + metadata: Optional[Dict[str, Any]] = None + ) -> "LoadResult": + """Create an empty load result. + + Args: + document_id: Document identifier + metadata: Additional metadata + + Returns: + LoadResult: Empty load result + """ + logger.warning(f"Creating empty load result: {document_id}") + return cls( + document_id=document_id, + records_loaded=0, + status="empty_result", + metadata=metadata or {} + ) + + @classmethod + def create_failed( + cls, + document_id: str, + error_message: str, + metadata: Optional[Dict[str, Any]] = None + ) -> "LoadResult": + """Create a failed load result. + + Args: + document_id: Document identifier + error_message: Error message describing the failure + metadata: Additional metadata + + Returns: + LoadResult: Failed load result + """ + logger.error(f"Creating failed load result: {document_id} - {error_message}") + metadata = metadata or {} + metadata["error_message"] = error_message + return cls( + document_id=document_id, + records_loaded=0, + status="failed", + metadata=metadata + ) \ No newline at end of file diff --git a/document-graph/src/document_graph/pipeline_executor.py b/document-graph/src/document_graph/pipeline_executor.py new file mode 100644 index 00000000..09934328 --- /dev/null +++ b/document-graph/src/document_graph/pipeline_executor.py @@ -0,0 +1,224 @@ +"""Pipeline Executor — schema-driven orchestration of ingest → transform → build. + +Reads an ETLSchema (from JSON/mapping file) and executes the full pipeline: +1. Read source data (via graphrag-toolkit readers) +2. Ingest: column/row/field prep (pandas) +3. Transform: rows → typed Nodes/Edges +4. Build: Cypher generation + write to Neptune via GraphStore +""" + +import json +import logging +from dataclasses import dataclass, field +from typing import Any, Optional + +import pandas as pd + +from document_graph.model_elements import Node, Edge +from document_graph.graph_build.cypher_builder import node_to_cypher, edge_to_cypher + +logger = logging.getLogger(__name__) + + +@dataclass +class PipelineResult: + """Result of a pipeline execution.""" + nodes_created: int = 0 + edges_created: int = 0 + records_processed: int = 0 + errors: list = field(default_factory=list) + + +class PipelineExecutor: + """Schema-driven pipeline executor. + + Reads a mapping/schema JSON and orchestrates: + source → ingest (DataFrame ops) → transform (row→node) → build (Cypher→Neptune) + + Usage: + from document_graph.pipeline import PipelineExecutor + + executor = PipelineExecutor(graph_store, tenant_id='scim') + result = executor.run_from_schema('mapping.json', source_df=df) + """ + + def __init__(self, graph_store, tenant_id: str = "default"): + self._store = graph_store + self._tenant_id = tenant_id + + def run_from_schema(self, schema_path: str, source_df: pd.DataFrame) -> PipelineResult: + """Execute the full pipeline from a schema file. + + Args: + schema_path: Path to mapping.json / ETL schema file + source_df: Source data as pandas DataFrame + """ + with open(schema_path) as f: + schema = json.load(f) + + result = PipelineResult(records_processed=len(source_df)) + + # Extract graph definition from schema + graph_def = schema.get("graph", {}) + node_defs = graph_def.get("nodes", []) + edge_defs = graph_def.get("edges", []) + mappings = schema.get("mappings", schema.get("properties", [])) + + # Build property map: csv_column → (node_type, property_name) + property_map = {} + for m in mappings: + csv_col = m.get("csv_property_name", "") + prop_name = m.get("arrow_property_name", csv_col) + parents = m.get("parents", [m.get("name", "")]) + map_type = m.get("type", "property") + if csv_col: + property_map[csv_col] = { + "property_name": prop_name, + "parents": parents, + "type": map_type, + } + + # Build node type definitions + node_types = {} + for node_def in node_defs: + for label, config in node_def.items(): + id_field = config.get("property", {}).get("csv_map", "id") + node_types[label] = {"id_field": id_field, "properties": {}} + + # Map properties to their node types + for csv_col, mapping in property_map.items(): + if mapping["type"] == "property": + for parent in mapping["parents"]: + if parent in node_types: + node_types[parent]["properties"][csv_col] = mapping["property_name"] + + # Build edge definitions + edge_types = [] + for edge_def in edge_defs: + for source_label, rels in edge_def.items(): + for rel_type, target_label in rels.items(): + edge_types.append({ + "source": source_label, + "target": target_label, + "type": rel_type, + }) + + # Execute: create nodes + nodes = [] + for _, row in source_df.iterrows(): + for label, type_def in node_types.items(): + id_col = type_def["id_field"] + if id_col in row and pd.notna(row[id_col]): + props = {} + for csv_col, prop_name in type_def["properties"].items(): + if csv_col in row and pd.notna(row[csv_col]): + props[prop_name] = str(row[csv_col]) + props["id"] = str(row[id_col]) + props[id_col] = str(row[id_col]) + + node = Node(id=str(row[id_col]), labels=[label], properties=props) + nodes.append(node) + + # Deduplicate nodes by id+label + seen = set() + unique_nodes = [] + for n in nodes: + key = (n.id, tuple(n.labels)) + if key not in seen: + seen.add(key) + unique_nodes.append(n) + + # Write nodes + for node in unique_nodes: + cypher, params = node_to_cypher(node, tenant_id=self._tenant_id) + try: + self._store.execute_query(cypher, params) + result.nodes_created += 1 + except Exception as e: + result.errors.append(f"Node {node.id}: {e}") + + # Write edges (based on shared fields in the data) + for edge_def in edge_types: + src_label = edge_def["source"] + tgt_label = edge_def["target"] + rel_type = edge_def["type"] + + src_id_field = node_types.get(src_label, {}).get("id_field", "") + tgt_id_field = node_types.get(tgt_label, {}).get("id_field", "") + + if src_id_field in source_df.columns and tgt_id_field in source_df.columns: + for _, row in source_df.iterrows(): + src_id = str(row[src_id_field]) if pd.notna(row.get(src_id_field)) else None + tgt_id = str(row[tgt_id_field]) if pd.notna(row.get(tgt_id_field)) else None + if src_id and tgt_id: + edge = Edge( + id=f"{src_id}-{rel_type}-{tgt_id}", + source_id=src_id, + target_id=tgt_id, + label=rel_type, + ) + cypher, params = edge_to_cypher(edge, tenant_id=self._tenant_id) + try: + self._store.execute_query(cypher, params) + result.edges_created += 1 + except Exception as e: + result.errors.append(f"Edge {src_id}->{tgt_id}: {e}") + + logger.info("Pipeline complete: %d nodes, %d edges, %d errors", + result.nodes_created, result.edges_created, len(result.errors)) + return result + + def run_from_dataframe(self, df: pd.DataFrame, node_label: str, + id_field: str = "id", + edge_field: Optional[str] = None, + edge_type: str = "RELATED_TO") -> PipelineResult: + """Simple pipeline: DataFrame → typed nodes (+ optional edges). + + For when you don't have a full schema, just a label and DataFrame. + """ + from document_graph.transform.transformer_provider_config import TransformerProviderConfig + from document_graph.transform.graph_transformers.row_to_node import RowToNodeTransformer + + result = PipelineResult(records_processed=len(df)) + records = df.to_dict("records") + + # Transform + config = TransformerProviderConfig(name="r2n", args={"type": node_label}) + nodes_data = RowToNodeTransformer(config).transform(records) + + # Build + write nodes + for n in nodes_data: + node = Node( + id=n.get(id_field, n.get("_id", "")), + labels=[n.get("node_type", node_label)], + properties=n, + ) + cypher, params = node_to_cypher(node, tenant_id=self._tenant_id) + try: + self._store.execute_query(cypher, params) + result.nodes_created += 1 + except Exception as e: + result.errors.append(str(e)) + + # Optional: infer edges + if edge_field and edge_field in df.columns: + from document_graph.transform.graph_transformers.infer_edges import EdgeInferencer + config = TransformerProviderConfig(name="edges", args={ + "source_field": edge_field, "edge_type": edge_type + }) + edges_data = EdgeInferencer(config).transform(records) + for ed in edges_data: + edge = Edge( + id=ed.get("id", ""), + source_id=ed.get("source_id", ""), + target_id=ed.get("target_id", ""), + label=ed.get("edge_type", edge_type), + ) + cypher, params = edge_to_cypher(edge, tenant_id=self._tenant_id) + try: + self._store.execute_query(cypher, params) + result.edges_created += 1 + except Exception as e: + result.errors.append(str(e)) + + return result diff --git a/document-graph/src/document_graph/plugins/__init__.py b/document-graph/src/document_graph/plugins/__init__.py new file mode 100644 index 00000000..7a21b948 --- /dev/null +++ b/document-graph/src/document_graph/plugins/__init__.py @@ -0,0 +1 @@ +"""Plugins — extension hooks for custom ingest/transform/build steps.""" diff --git a/document-graph/src/document_graph/query/__init__.py b/document-graph/src/document_graph/query/__init__.py new file mode 100644 index 00000000..7b385453 --- /dev/null +++ b/document-graph/src/document_graph/query/__init__.py @@ -0,0 +1,3 @@ +"""Query stage — Cypher queries over typed document graph nodes.""" + +from .query_engine import DocumentGraphQueryEngine diff --git a/document-graph/src/document_graph/query/query_engine.py b/document-graph/src/document_graph/query/query_engine.py new file mode 100644 index 00000000..b2149f03 --- /dev/null +++ b/document-graph/src/document_graph/query/query_engine.py @@ -0,0 +1,43 @@ +"""Document Graph Query Engine — Cypher queries over typed nodes. + +Uses graphrag-toolkit's GraphStore for execution. +""" + +from typing import Optional + + +class DocumentGraphQueryEngine: + """Query typed document graph nodes via GraphStore.""" + + def __init__(self, graph_store, tenant_id: Optional[str] = None): + """Initialize with a graphrag-toolkit GraphStore. + + Args: + graph_store: GraphStore from GraphStoreFactory.for_graph_store() + tenant_id: Optional tenant scope for queries + """ + self._store = graph_store + self._tenant_id = tenant_id + + def query(self, cypher: str, params: dict = None) -> list[dict]: + """Execute a Cypher query against the typed document graph.""" + return self._store.execute_query(cypher, params or {}) + + def get_nodes(self, label: str, limit: int = 100) -> list[dict]: + """Get all nodes of a given type.""" + scoped_label = f"__{label}__{self._tenant_id}__" if self._tenant_id else label + return self.query(f"MATCH (n:{scoped_label}) RETURN n LIMIT $limit", {"limit": limit}) + + def get_relationships(self, source_label: str, rel_type: str, target_label: str, limit: int = 100) -> list[dict]: + """Get relationships between typed nodes.""" + src = f"__{source_label}__{self._tenant_id}__" if self._tenant_id else source_label + tgt = f"__{target_label}__{self._tenant_id}__" if self._tenant_id else target_label + return self.query( + f"MATCH (a:{src})-[r:{rel_type}]->(b:{tgt}) RETURN a, r, b LIMIT $limit", + {"limit": limit}, + ) + + def find_by_property(self, label: str, key: str, value) -> list[dict]: + """Find nodes by property value.""" + scoped_label = f"__{label}__{self._tenant_id}__" if self._tenant_id else label + return self.query(f"MATCH (n:{scoped_label} {{{key}: $val}}) RETURN n", {"val": value}) diff --git a/document-graph/src/document_graph/schema/__init__.py b/document-graph/src/document_graph/schema/__init__.py new file mode 100644 index 00000000..b12b7074 --- /dev/null +++ b/document-graph/src/document_graph/schema/__init__.py @@ -0,0 +1,61 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Schema Package for Document Graph Operations. + +This package provides the core schema definitions and utilities for working with +document graph ETL (Extract, Transform, Load) operations. It includes models for +defining extraction sources, transformation rules, and graph loading configurations. + +The schema package contains: +- ETL schema models for defining document processing pipelines +- Schema providers for loading schemas from various sources +- Schema discovery for automatic schema inference +- Utilities for schema serialization and deserialization +- Static schema definitions for testing and default configurations + +These components work together to provide a flexible and extensible way to define +how documents are processed and loaded into graph databases. +""" + +# Core schema models +from .etl_schema_model import ETLSchema, ExtractConfig, TransformConfig, LoadConfig +from .document_graph_schema import ( + ChunkingConfig, MetadataMapping, EntityExtractionConfig, NormalizeConfig, + NodeDefinition, RelationshipDefinition +) + +# Schema I/O utilities +from .schema_io import save_schema, load_schema + +# Static schema provider +from .static_schema_provider import StaticSchemaProvider + +# Import submodules for convenience +from . import providers +from . import discovery + +__all__ = [ + # Core schema models + 'ETLSchema', + 'ExtractConfig', + 'TransformConfig', + 'LoadConfig', + 'ChunkingConfig', + 'MetadataMapping', + 'EntityExtractionConfig', + 'NormalizeConfig', + 'NodeDefinition', + 'RelationshipDefinition', + + # Schema I/O utilities + 'save_schema', + 'load_schema', + + # Utilities + 'StaticSchemaProvider', + + # Submodules + 'providers', + 'discovery' +] + diff --git a/document-graph/src/document_graph/schema/discovery/__init__.py b/document-graph/src/document_graph/schema/discovery/__init__.py new file mode 100644 index 00000000..bfd65f98 --- /dev/null +++ b/document-graph/src/document_graph/schema/discovery/__init__.py @@ -0,0 +1,41 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Schema Discovery Module for Document Graph Operations. + +This module provides schema discovery capabilities for various file formats, +allowing automatic inference of ETL schemas from source data files. +""" + +from .schema_discovery_base import SchemaDiscoveryProvider +from .tabular_discovery_base import TabularSchemaDiscoveryProvider +from .csv_discovery_provider import CSVSchemaDiscoveryProvider +from .excel_discovery_provider import ExcelSchemaDiscoveryProvider +from .json_discovery_provider import JSONSchemaDiscoveryProvider +from .parquet_discovery_provider import ParquetSchemaDiscoveryProvider +from .xml_discovery_provider import XMLSchemaDiscoveryProvider +from .yaml_discovery_provider import YAMLSchemaDiscoveryProvider +from .schema_discovery_registry import get_discovery_provider, DISCOVERY_REGISTRY +from .schema_discovery_registry_class import SchemaDiscoveryRegistry + +# Aliases for notebook 24 compatibility +CSVDiscoveryProvider = CSVSchemaDiscoveryProvider +JSONDiscoveryProvider = JSONSchemaDiscoveryProvider +ExcelDiscoveryProvider = ExcelSchemaDiscoveryProvider + +__all__ = [ + 'SchemaDiscoveryProvider', + 'TabularSchemaDiscoveryProvider', + 'CSVSchemaDiscoveryProvider', + 'ExcelSchemaDiscoveryProvider', + 'JSONSchemaDiscoveryProvider', + 'ParquetSchemaDiscoveryProvider', + 'XMLSchemaDiscoveryProvider', + 'YAMLSchemaDiscoveryProvider', + 'get_discovery_provider', + 'DISCOVERY_REGISTRY', + # Notebook 24 compatibility + 'CSVDiscoveryProvider', + 'JSONDiscoveryProvider', + 'ExcelDiscoveryProvider', + 'SchemaDiscoveryRegistry' +] \ No newline at end of file diff --git a/document-graph/src/document_graph/schema/discovery/csv_discovery_provider.py b/document-graph/src/document_graph/schema/discovery/csv_discovery_provider.py new file mode 100644 index 00000000..e13f28cd --- /dev/null +++ b/document-graph/src/document_graph/schema/discovery/csv_discovery_provider.py @@ -0,0 +1,144 @@ +# Copyright (c) Evan Erwee. All rights reserved. +"""CSV discovery provider — discovers schema from CSV file structure.""" + +import logging +from typing import List, Dict, Any +import pandas as pd + +from document_graph.schema.etl_schema_model import ( + ETLSchema, ExtractConfig, TransformConfig, LoadConfig, + ChunkingConfig, MetadataMapping, EntityExtractionConfig, + NormalizeConfig, NodeDefinition, RelationshipDefinition +) +from document_graph.schema.discovery.tabular_discovery_base import TabularSchemaDiscoveryProvider + +logger = logging.getLogger(__name__) + + +class CSVSchemaDiscoveryProvider(TabularSchemaDiscoveryProvider): + """ + Stable CSV schema discovery that ignores batching and reads only headers. + + Key design choices: + - Remove batching args (skiprows/nrows) so schema isn't batch-dependent. + - Read only header row (nrows=0) to discover columns quickly & deterministically. + - If header=None, sample one row to count fields and synthesize column names. + - Keep on_bad_lines='skip' resilient by forcing engine='python' when needed. + """ + + _BATCH_ARG_KEYS = {"skiprows", "nrows"} # anything that could make schema batch-specific + + def _sanitize_read_args(self, args: Dict[str, Any]) -> Dict[str, Any]: + """Return a copy of args safe for schema discovery (no batching).""" + clean = dict(args or {}) + # Pull out discovery flags + clean.pop("log_on_extract", None) + + # Remove batch-affecting args + for k in list(clean.keys()): + if k in self._BATCH_ARG_KEYS: + clean.pop(k, None) + + # Be resilient by default + clean.setdefault("on_bad_lines", "skip") + + # Ensure pandas engine compatibility for on_bad_lines='skip' + if str(clean.get("on_bad_lines")).lower() == "skip": + # Pandas requires engine='python' for on_bad_lines='skip' unless already set + engine = clean.get("engine", "python") + if engine.lower() != "python": + logger.warning( + "on_bad_lines='skip' requires engine='python' for reliable behavior during discovery; " + "overriding engine to 'python'." + ) + clean["engine"] = "python" + + # Avoid dtype surprises during discovery + # (Don't set dtype here; let pandas leave them as object when nrows=0) + return clean + + def _discover_headers(self, read_args: Dict[str, Any]) -> List[str]: + """ + Determine headers deterministically: + - If header is provided (e.g., 0), read with nrows=0 and use df.columns. + - If header=None, read a single row with header=None to count fields and synthesize names. + """ + # Case 1: header row exists (header is 0 or an int/list) + header_arg = read_args.get("header", 0) + try: + # Fast path: read only the header row structure + discovery_args = dict(read_args) + discovery_args["nrows"] = 0 + df = pd.read_csv(self.source, **discovery_args) + headers = list(df.columns) + + # If user explicitly said header=None, columns will be RangeIndex([]) with nrows=0. + # In that case, synthesize names from a one-row sample. + if header_arg is None or len(headers) == 0: + sample_args = dict(read_args) + sample_args["header"] = None + sample_args["nrows"] = 1 + sample_df = pd.read_csv(self.source, **sample_args) + n_cols = sample_df.shape[1] + headers = [f"column_{i}" for i in range(n_cols)] + except Exception as e: + raise ErrorHandler.csv_parse_error(str(self.source), e) + + if not headers: + raise ValueError("Validation error") + + return headers + + def discover_schema(self) -> ETLSchema: + """ + Discover and return an ETL schema from a CSV file, ignoring batching. + """ + if not self.source.exists(): + raise FileNotFoundError(f"File not found") + + raw_args = dict(self.args or {}) + log_enabled = bool(raw_args.get("log_on_extract", False)) + + # Build safe args for discovery + read_args = self._sanitize_read_args(raw_args) + + # Optional log file + log_path = self.source.with_suffix(self.source.suffix + ".log") if log_enabled else None + + try: + headers: List[str] = self._discover_headers(read_args) + logger.info(f"Discovered columns from CSV: {headers}") + + if log_path: + with log_path.open("w", encoding="utf-8") as log_file: + log_file.write(f"CSV Discovery Log for {self.source.name}\n") + log_file.write(f"Discovered columns: {headers}\n") + # Show the sanitized args we actually used (without batching) + log_file.write(f"Applied pandas.read_csv args (sanitized): {read_args}\n") + + except Exception as e: + if log_path: + with log_path.open("a", encoding="utf-8") as log_file: + log_file.write("\nException encountered during CSV parsing:\n") + log_file.write(str(e)) + raise + + # Construct a stable ETL schema using discovered headers only + return ETLSchema( + schema_id=f"discovered-{self.source.stem}", + description=f"Discovered ETL schema from CSV file: {self.source.name}", + extract=ExtractConfig(source_type="file", file_type="csv"), + transform=TransformConfig( + chunking=ChunkingConfig(strategy="fixed_length", min_length=100), + metadata_mapping=MetadataMapping(), + entity_extraction=EntityExtractionConfig(method="ner"), + normalize=NormalizeConfig() + ), + load=LoadConfig( + document_node=NodeDefinition(type="Document", fields=headers), + section_node=NodeDefinition(type="Row", fields=headers), + relationships=[ + RelationshipDefinition(type="contains", source="document_id", target="row_id") + ] + ) + ) diff --git a/document-graph/src/document_graph/schema/discovery/excel_discovery_provider.py b/document-graph/src/document_graph/schema/discovery/excel_discovery_provider.py new file mode 100644 index 00000000..7f1e6750 --- /dev/null +++ b/document-graph/src/document_graph/schema/discovery/excel_discovery_provider.py @@ -0,0 +1,133 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Excel Schema Discovery Provider Module for Document Graph Operations. + +This module provides a schema discovery provider for Excel files (.xlsx, .xls). +It analyzes Excel files to infer their structure and generate appropriate ETL schemas +for processing and loading the data into a document graph. + +The module includes the following components: +- ExcelSchemaDiscoveryProvider: Schema discovery provider for Excel files + +The Excel schema discovery provider uses pandas to read and analyze Excel files, +extracting column headers and other metadata to create an ETL schema. It supports +various Excel parsing options through the args parameter, which is passed directly +to pandas.read_excel. + +Usage: + # Get an Excel discovery provider for a specific file + from pathlib import Path + from document_graph.schema.discovery.schema_discovery_registry import get_discovery_provider + + file_path = Path("data/sample.xlsx") + provider = get_discovery_provider(file_path) + + # Or create it directly with custom arguments + from document_graph.schema.discovery.excel_discovery_provider import ExcelSchemaDiscoveryProvider + + provider = ExcelSchemaDiscoveryProvider( + source=file_path, + args={"sheet_name": "Sheet1", "header": 0} + ) + + # Discover the schema + schema = provider.discover_schema() + + # Use the schema for ETL operations + print(f"Discovered schema ID: {schema.schema_id}") + print(f"Fields: {schema.load.document_node.fields}") +""" + + +# Import custom logging to ensure configuration is available + + +import logging +from typing import List +import pandas as pd + +from document_graph.schema.etl_schema_model import * +from document_graph.schema.discovery.tabular_discovery_base import TabularSchemaDiscoveryProvider + + +logger = logging.getLogger(__name__) + +class ExcelSchemaDiscoveryProvider(TabularSchemaDiscoveryProvider): + """ + Schema discovery provider for Excel files (.xlsx, .xls). + + This class analyzes Excel files to infer their structure and generate appropriate + ETL schemas for processing and loading the data into a document graph. It uses + pandas to read and analyze Excel files, extracting column headers and other metadata. + + The provider supports various Excel parsing options through the args parameter, + which is passed directly to pandas.read_excel. This allows for customization of + the Excel parsing process, such as specifying sheet names, header rows, and + other pandas.read_excel options. + + Attributes: + source (Path): The path to the Excel file to analyze. + args (Dict[str, Any]): Optional arguments that control the discovery process. + These arguments are passed directly to pandas.read_excel. + + Config options: + - sheet_name (str or int): Name or index of the sheet to read. Defaults to 0. + - header (int): Row to use for the column labels. Defaults to 0. + - Any other argument accepted by pandas.read_excel. + """ + + def discover_schema(self) -> ETLSchema: + """ + Discover and return an ETL schema from an Excel file. + + This method reads the Excel file using pandas, extracts the column headers, + and generates an appropriate ETL schema. It includes error handling to + manage issues that might arise during the Excel parsing process. + + The method performs the following steps: + 1. Validates that the source file exists + 2. Reads the Excel file using pandas.read_excel with the provided arguments + 3. Extracts the column headers + 4. Validates that the headers are not empty + 5. Logs the discovered columns + 6. Constructs and returns an ETLSchema object + + Returns: + ETLSchema: A complete ETL schema object that describes the structure + of the Excel file and how it should be processed. + + Raises: + FileNotFoundError: If the specified source file does not exist. + ValueError: If the Excel file has no columns or cannot be parsed. + Exception: Any exception raised by pandas.read_excel. + """ + if not self.source.exists(): + raise FileNotFoundError(f"File not found") + + try: + df = pd.read_excel(self.source, **self.args) + headers: List[str] = list(df.columns) + if not headers: + raise ValueError("Validation error") + logger.info(f"Discovered columns from Excel: {headers}") + except Exception as e: + raise ErrorHandler.csv_parse_error(str(self.source), e) + + return ETLSchema( + schema_id=f"discovered-{self.source.stem}", + description=f"Discovered ETL schema from Excel file: {self.source.name}", + extract=ExtractConfig(source_type="file", file_type="excel"), + transform=TransformConfig( + chunking=ChunkingConfig(strategy="fixed_length", min_length=100), + metadata_mapping=MetadataMapping(), + entity_extraction=EntityExtractionConfig(method="ner"), + normalize=NormalizeConfig() + ), + load=LoadConfig( + document_node=NodeDefinition(type="Document", fields=headers), + section_node=NodeDefinition(type="Row", fields=headers), + relationships=[ + RelationshipDefinition(type="contains", source="document_id", target="row_id") + ] + ) + ) diff --git a/document-graph/src/document_graph/schema/discovery/json_discovery_provider.py b/document-graph/src/document_graph/schema/discovery/json_discovery_provider.py new file mode 100644 index 00000000..74f42c67 --- /dev/null +++ b/document-graph/src/document_graph/schema/discovery/json_discovery_provider.py @@ -0,0 +1,143 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""JSON Schema Discovery Provider Module for Document Graph Operations. + +This module provides a schema discovery provider for JSON (JavaScript Object Notation) files. +It analyzes JSON files to infer their structure and generate appropriate ETL schemas +for processing and loading the data into a document graph. + +The module includes the following components: +- JSONSchemaDiscoveryProvider: Schema discovery provider for JSON files + +The JSON schema discovery provider reads and analyzes JSON files, extracting object keys +from the first record (or the entire object if it's not a list) to create an ETL schema. +It handles both array-based JSON files (containing multiple records) and single-object +JSON files. + +Usage: + # Get a JSON discovery provider for a specific file + from pathlib import Path + from document_graph.schema.discovery.schema_discovery_registry import get_discovery_provider + + file_path = Path("data/sample.json") + provider = get_discovery_provider(file_path) + + # Or create it directly + from document_graph.schema.discovery.json_discovery_provider import JSONSchemaDiscoveryProvider + + provider = JSONSchemaDiscoveryProvider(source=file_path) + + # Discover the schema + schema = provider.discover_schema() + + # Use the schema for ETL operations + print(f"Discovered schema ID: {schema.schema_id}") + print(f"Fields: {schema.load.document_node.fields}") +""" + + +# Import custom logging to ensure configuration is available + + +import json +import logging +from typing import List + +from document_graph.schema.etl_schema_model import * +from document_graph.schema.discovery.tabular_discovery_base import TabularSchemaDiscoveryProvider + + +logger = logging.getLogger(__name__) + +class JSONSchemaDiscoveryProvider(TabularSchemaDiscoveryProvider): + """ + Schema discovery provider for JSON (JavaScript Object Notation) files. + + This class analyzes JSON files to infer their structure and generate appropriate + ETL schemas for processing and loading the data into a document graph. It reads + JSON files and extracts object keys to determine the fields that should be included + in the ETL schema. + + The provider handles both array-based JSON files (containing multiple records) + and single-object JSON files. For array-based files, it examines the first record + to determine the schema. For single-object files, it uses the keys of the object. + + Attributes: + source (Path): The path to the JSON file to analyze. + args (Dict[str, Any]): Optional arguments that control the discovery process. + Currently, no specific arguments are used for JSON discovery. + """ + + def discover_schema(self) -> ETLSchema: + """ + Discover and return an ETL schema from a JSON file. + + This method reads the JSON file, extracts the keys from the first record + (or the entire object if it's not a list), and generates an appropriate + ETL schema. It includes error handling to manage issues that might arise + during the JSON parsing process. + + The method performs the following steps: + 1. Validates that the source file exists + 2. Reads the JSON file using json.load + 3. Determines if the data is an array or a single object + 4. Extracts the keys from the first record or the entire object + 5. Validates that the keys are not empty + 6. Logs the discovered fields + 7. Constructs and returns an ETLSchema object + + Returns: + ETLSchema: A complete ETL schema object that describes the structure + of the JSON file and how it should be processed. + + Raises: + FileNotFoundError: If the specified source file does not exist. + ValueError: If the JSON file has no keys or cannot be parsed. + Exception: Any exception raised during JSON parsing. + """ + if not self.source.exists(): + raise FileNotFoundError(f"File not found") + + try: + with self.source.open("r", encoding="utf-8") as f: + # Try regular JSON first + try: + data = json.load(f) + first_record = data[0] if isinstance(data, list) else data + except json.JSONDecodeError: + # If regular JSON fails, try JSONL (newline-delimited JSON) + logger.debug("Regular JSON failed, trying JSONL format") + f.seek(0) # Reset file pointer + first_line = f.readline().strip() + if first_line: + first_record = json.loads(first_line) + else: + raise ValueError("Empty JSONL file") + + headers: List[str] = list(first_record.keys()) + + if not headers: + raise ValueError("Validation error") + + logger.info(f"Discovered fields from JSON/JSONL: {headers}") + except Exception as e: + raise ErrorHandler.csv_parse_error(str(self.source), e) + + return ETLSchema( + schema_id=f"discovered-{self.source.stem}", + description=f"Discovered ETL schema from JSON file: {self.source.name}", + extract=ExtractConfig(source_type="file", file_type="json"), + transform=TransformConfig( + chunking=ChunkingConfig(strategy="fixed_length", min_length=100), + metadata_mapping=MetadataMapping(), + entity_extraction=EntityExtractionConfig(method="ner"), + normalize=NormalizeConfig() + ), + load=LoadConfig( + document_node=NodeDefinition(type="Document", fields=headers), + section_node=NodeDefinition(type="Row", fields=headers), + relationships=[ + RelationshipDefinition(type="contains", source="document_id", target="row_id") + ] + ) + ) diff --git a/document-graph/src/document_graph/schema/discovery/parquet_discovery_provider.py b/document-graph/src/document_graph/schema/discovery/parquet_discovery_provider.py new file mode 100644 index 00000000..5b132e22 --- /dev/null +++ b/document-graph/src/document_graph/schema/discovery/parquet_discovery_provider.py @@ -0,0 +1,137 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Parquet Schema Discovery Provider Module for Document Graph Operations. + +This module provides a schema discovery provider for Parquet files, a columnar storage +format commonly used in big data processing. It analyzes Parquet files to infer their +structure and generate appropriate ETL schemas for processing and loading the data +into a document graph. + +The module includes the following components: +- ParquetSchemaDiscoveryProvider: Schema discovery provider for Parquet files + +The Parquet schema discovery provider uses pandas to read and analyze Parquet files, +extracting column headers and other metadata to create an ETL schema. It supports +various Parquet parsing options through the args parameter, which is passed directly +to pandas.read_parquet. + +Usage: + # Get a Parquet discovery provider for a specific file + from pathlib import Path + from document_graph.schema.discovery.schema_discovery_registry import get_discovery_provider + + file_path = Path("data/sample.parquet") + provider = get_discovery_provider(file_path) + + # Or create it directly with custom arguments + from document_graph.schema.discovery.parquet_discovery_provider import ParquetSchemaDiscoveryProvider + + provider = ParquetSchemaDiscoveryProvider( + source=file_path, + args={"columns": ["id", "name", "value"]} + ) + + # Discover the schema + schema = provider.discover_schema() + + # Use the schema for ETL operations + print(f"Discovered schema ID: {schema.schema_id}") + print(f"Fields: {schema.load.document_node.fields}") +""" + + +# Import custom logging to ensure configuration is available + + +import logging +from typing import List +import pandas as pd + +from document_graph.schema.etl_schema_model import * +from document_graph.schema.discovery.tabular_discovery_base import TabularSchemaDiscoveryProvider + + +logger = logging.getLogger(__name__) + +class ParquetSchemaDiscoveryProvider(TabularSchemaDiscoveryProvider): + """ + Schema discovery provider for Parquet files. + + This class analyzes Parquet files to infer their structure and generate appropriate + ETL schemas for processing and loading the data into a document graph. It uses + pandas to read and analyze Parquet files, extracting column headers and other metadata. + + Parquet is a columnar storage format commonly used in big data processing. It is + designed for efficient data compression and encoding schemes, making it particularly + well-suited for query performance and minimizing I/O operations. + + The provider supports various Parquet parsing options through the args parameter, + which is passed directly to pandas.read_parquet. This allows for customization of + the Parquet parsing process, such as specifying which columns to read. + + Attributes: + source (Path): The path to the Parquet file to analyze. + args (Dict[str, Any]): Optional arguments that control the discovery process. + These arguments are passed directly to pandas.read_parquet. + + Config options: + - columns (List[str]): List of columns to read. If not provided, all columns are read. + - Any other argument accepted by pandas.read_parquet. + """ + + def discover_schema(self) -> ETLSchema: + """ + Discover and return an ETL schema from a Parquet file. + + This method reads the Parquet file using pandas, extracts the column headers, + and generates an appropriate ETL schema. It includes error handling to + manage issues that might arise during the Parquet parsing process. + + The method performs the following steps: + 1. Validates that the source file exists + 2. Reads the Parquet file using pandas.read_parquet with the provided arguments + 3. Extracts the column headers + 4. Validates that the headers are not empty + 5. Logs the discovered columns + 6. Constructs and returns an ETLSchema object + + Returns: + ETLSchema: A complete ETL schema object that describes the structure + of the Parquet file and how it should be processed. + + Raises: + FileNotFoundError: If the specified source file does not exist. + ValueError: If the Parquet file has no columns or cannot be parsed. + Exception: Any exception raised by pandas.read_parquet. + """ + if not self.source.exists(): + raise FileNotFoundError(f"File not found") + + try: + df = pd.read_parquet(self.source, **self.args) + headers: List[str] = list(df.columns) + if not headers: + raise ValueError("Validation error") + + logger.info(f"Discovered columns from Parquet: {headers}") + except Exception as e: + raise ErrorHandler.csv_parse_error(str(self.source), e) + + return ETLSchema( + schema_id=f"discovered-{self.source.stem}", + description=f"Discovered ETL schema from Parquet file: {self.source.name}", + extract=ExtractConfig(source_type="file", file_type="parquet"), + transform=TransformConfig( + chunking=ChunkingConfig(strategy="fixed_length", min_length=100), + metadata_mapping=MetadataMapping(), + entity_extraction=EntityExtractionConfig(method="ner"), + normalize=NormalizeConfig() + ), + load=LoadConfig( + document_node=NodeDefinition(type="Document", fields=headers), + section_node=NodeDefinition(type="Row", fields=headers), + relationships=[ + RelationshipDefinition(type="contains", source="document_id", target="row_id") + ] + ) + ) diff --git a/document-graph/src/document_graph/schema/discovery/schema_discovery_base.py b/document-graph/src/document_graph/schema/discovery/schema_discovery_base.py new file mode 100644 index 00000000..06e4bf6b --- /dev/null +++ b/document-graph/src/document_graph/schema/discovery/schema_discovery_base.py @@ -0,0 +1,114 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Schema Discovery Base Module for Document Graph Operations. + +This module provides the foundation for schema discovery in the document graph system. +It defines the abstract base class that all schema discovery providers must implement. +Schema discovery is the process of inferring a structured ETL schema from raw source data, +which is essential for properly processing and loading data into the document graph. + +The module includes the following components: +- SchemaDiscoveryProvider: Abstract base class for all schema discovery providers + +Schema discovery providers are responsible for examining source data files and +determining their structure, field types, and other metadata needed to create +a valid ETL schema. This schema is then used to guide the extraction, transformation, +and loading of data into the document graph. + +Usage: + # Get a discovery provider for a specific file + from pathlib import Path + from document_graph.schema.discovery.schema_discovery_registry import get_discovery_provider + + file_path = Path("data/sample.csv") + provider = get_discovery_provider(file_path) + + # Discover the schema + schema = provider.discover_schema() + + # Use the schema for ETL operations + print(f"Discovered schema ID: {schema.schema_id}") + print(f"Fields: {schema.load.document_node.fields}") +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Union, Dict, Any, Optional + +from document_graph.schema.etl_schema_model import ETLSchema + + +class SchemaDiscoveryProvider(ABC): + """ + Base class for all schema discovery providers. + + This abstract base class defines the interface that all schema discovery providers + must implement. Schema discovery providers are responsible for examining source + data files and inferring a structured ETL schema that can be used for document + graph operations. + + The provider analyzes the structure, field types, and other metadata of the source + data to create an appropriate ETL schema. This schema is then used to guide the + extraction, transformation, and loading of data into the document graph. + + Subclasses should implement the discover_schema method to provide format-specific + schema discovery logic. + + Attributes: + source (Path): The path to the source data file to analyze. + args (Dict[str, Any]): Optional arguments that control the discovery process. + These arguments are format-specific and are passed to the underlying + data reading functions. + """ + + def __init__(self, source: Union[str, Path], args: Optional[Dict[str, Any]] = None): + """ + Initialize the schema discovery provider. + + This constructor sets up the provider with the source file path and any + format-specific arguments needed for the discovery process. It validates + that the source file exists before proceeding. + + Args: + source (Union[str, Path]): The path to the source data file to analyze. + Can be provided as a string or a Path object. + args (Optional[Dict[str, Any]]): Optional arguments that control the + discovery process. These arguments are format-specific and are + passed to the underlying data reading functions. Defaults to None. + + Raises: + FileNotFoundError: If the specified source file does not exist. + """ + self.source = Path(source) + self.args = args or {} + + if not self.source.exists(): + raise FileNotFoundError(f"File not found") + + @abstractmethod + def discover_schema(self) -> ETLSchema: + """ + Discover and return an ETL schema from the source data. + + This abstract method must be implemented by subclasses to provide + format-specific schema discovery logic. The implementation should + analyze the source data file and infer an appropriate ETL schema + that captures its structure and field types. + + Returns: + ETLSchema: A complete ETL schema object that describes the structure + of the source data and how it should be processed. + + Raises: + Various exceptions: Implementations may raise format-specific exceptions + when the source data cannot be parsed or does not meet expected + format requirements. + """ + pass diff --git a/document-graph/src/document_graph/schema/discovery/schema_discovery_registry.py b/document-graph/src/document_graph/schema/discovery/schema_discovery_registry.py new file mode 100644 index 00000000..94cd9647 --- /dev/null +++ b/document-graph/src/document_graph/schema/discovery/schema_discovery_registry.py @@ -0,0 +1,117 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Schema Discovery Registry Module for Document Graph Operations. + +This module provides a registry system for schema discovery providers, allowing +the appropriate provider to be selected based on file extensions. It serves as +the entry point for schema discovery operations, abstracting away the details of +which provider to use for a given file type. + +The module includes the following components: +- DISCOVERY_REGISTRY: A dictionary mapping file extensions to provider classes +- get_discovery_provider: Function to get the appropriate provider for a file + +The registry system simplifies the process of discovering ETL schemas from various +file formats by automatically selecting the correct provider based on the file extension. +This allows client code to work with a consistent interface regardless of the +underlying file format. + +Supported file formats: +- CSV (.csv): Comma-separated values files +- Parquet (.parquet): Columnar storage format files +- JSON (.json): JavaScript Object Notation files +- Excel (.xlsx, .xls): Microsoft Excel spreadsheet files +- XML (.xml): eXtensible Markup Language files +- YAML (.yaml, .yml): YAML Ain't Markup Language files + +Usage: + # Get a discovery provider for a specific file + from pathlib import Path + from document_graph.schema.discovery.schema_discovery_registry import get_discovery_provider + + # The appropriate provider is selected based on the file extension + csv_provider = get_discovery_provider("data/sample.csv") + excel_provider = get_discovery_provider("data/sample.xlsx") + json_provider = get_discovery_provider("data/sample.json") + parquet_provider = get_discovery_provider("data/sample.parquet") + + # Discover the schema using the provider + schema = csv_provider.discover_schema() + + # Use the schema for ETL operations + print(f"Discovered schema ID: {schema.schema_id}") + print(f"Fields: {schema.load.document_node.fields}") +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + +from pathlib import Path +from typing import Type, Dict, Optional, Any + +from document_graph.schema.discovery.schema_discovery_base import SchemaDiscoveryProvider +from document_graph.schema.discovery.csv_discovery_provider import CSVSchemaDiscoveryProvider +from document_graph.schema.discovery.parquet_discovery_provider import ParquetSchemaDiscoveryProvider +from document_graph.schema.discovery.json_discovery_provider import JSONSchemaDiscoveryProvider +from document_graph.schema.discovery.excel_discovery_provider import ExcelSchemaDiscoveryProvider +from document_graph.schema.discovery.xml_discovery_provider import XMLSchemaDiscoveryProvider +from document_graph.schema.discovery.yaml_discovery_provider import YAMLSchemaDiscoveryProvider + +DISCOVERY_REGISTRY: Dict[str, Type[SchemaDiscoveryProvider]] = { + "csv": CSVSchemaDiscoveryProvider, + "parquet": ParquetSchemaDiscoveryProvider, + "json": JSONSchemaDiscoveryProvider, + "xlsx": ExcelSchemaDiscoveryProvider, + "xls": ExcelSchemaDiscoveryProvider, + "xml": XMLSchemaDiscoveryProvider, + "yaml": YAMLSchemaDiscoveryProvider, + "yml": YAMLSchemaDiscoveryProvider, +} + + +def get_discovery_provider(file_path: Path, args: Optional[Dict[str, Any]] = None) -> SchemaDiscoveryProvider: + """ + Get the appropriate schema discovery provider for a given file. + + This function selects the appropriate schema discovery provider based on the + file extension of the provided file path. It uses the DISCOVERY_REGISTRY + dictionary to map file extensions to provider classes. + + The function handles the instantiation of the provider with the file path + and any provided arguments, returning a ready-to-use provider instance. + + Args: + file_path (Path): The path to the file for which to get a discovery provider. + The file extension determines which provider is selected. + args (Optional[Dict[str, Any]]): Optional arguments to pass to the provider. + These arguments are format-specific and are passed to the underlying + data reading functions. Defaults to None. + + Returns: + SchemaDiscoveryProvider: An instance of the appropriate schema discovery + provider for the given file type, initialized with the provided file + path and arguments. + + Raises: + ValueError: If no discovery provider is registered for the file extension. + + Examples: + >>> from pathlib import Path + >>> provider = get_discovery_provider(Path("data/sample.csv")) + >>> schema = provider.discover_schema() + + >>> # With custom arguments + >>> provider = get_discovery_provider( + ... Path("data/sample.csv"), + ... args={"delimiter": ",", "encoding": "utf-8"} + ... ) + >>> schema = provider.discover_schema() + """ + ext = file_path.suffix.lower().lstrip(".") + provider_class = DISCOVERY_REGISTRY.get(ext) + if not provider_class: + raise ValueError(f"No discovery provider registered for extension: {ext}") + return provider_class(source=file_path, args=args or {}) diff --git a/document-graph/src/document_graph/schema/discovery/schema_discovery_registry_class.py b/document-graph/src/document_graph/schema/discovery/schema_discovery_registry_class.py new file mode 100644 index 00000000..3e887398 --- /dev/null +++ b/document-graph/src/document_graph/schema/discovery/schema_discovery_registry_class.py @@ -0,0 +1,66 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Schema Discovery Registry Class for notebook 24 compatibility.""" + +from typing import Dict, Type +from .schema_discovery_base import SchemaDiscoveryProvider + + +class SchemaDiscoveryRegistry: + """ + Manages the registration and retrieval of schema discovery providers. + + This class provides functionality for registering schema discovery providers, + retrieving providers by name, and listing all available providers. It serves as + a central registry for accessing different schema discovery mechanisms. + """ + + def __init__(self): + self._providers: Dict[str, Type[SchemaDiscoveryProvider]] = {} + + def register_provider(self, name: str, provider_class: Type[SchemaDiscoveryProvider]): + """ + Registers a schema discovery provider with the specified name. + + This method associates a provider class with a given name, allowing the + provider to be referenced and used later. + + Args: + name: The name to register the provider under. + provider_class: The class of the schema discovery provider to register. + """ + self._providers[name] = provider_class + + def get_provider(self, name: str) -> Type[SchemaDiscoveryProvider]: + """ + Retrieve a schema discovery provider by its name. + + This method looks up and returns a schema discovery provider associated with + the provided name. If the name does not correspond to any registered provider, + a KeyError is raised. + + Parameters: + name (str): The name of the schema discovery provider to retrieve. + + Raises: + KeyError: If the specified provider name is not found among registered providers. + + Returns: + Type[SchemaDiscoveryProvider]: The schema discovery provider associated with + the specified name. + """ + if name not in self._providers: + raise KeyError(f"Unknown discovery provider: {name}") + return self._providers[name] + + def list_providers(self) -> list: + """ + Returns a list of registered providers. + + This method retrieves all the registered providers from the internal + providers dictionary and returns their keys as a list. + + Returns: + list: List containing the names of all registered providers. + """ + return list(self._providers.keys()) \ No newline at end of file diff --git a/document-graph/src/document_graph/schema/discovery/tabular_discovery_base.py b/document-graph/src/document_graph/schema/discovery/tabular_discovery_base.py new file mode 100644 index 00000000..7db8aba7 --- /dev/null +++ b/document-graph/src/document_graph/schema/discovery/tabular_discovery_base.py @@ -0,0 +1,85 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Tabular Discovery Base Module for Document Graph Operations. + +This module provides a base class for schema discovery providers that work with +tabular data formats such as CSV, Excel, Parquet, and similar structured data. +It extends the core schema discovery functionality with tabular-specific features. + +The module includes the following components: +- TabularSchemaDiscoveryProvider: Base class for tabular schema discovery providers + +Tabular schema discovery providers are specialized for handling data formats that +have a row-column structure. They share common patterns for extracting column headers, +inferring data types, and creating appropriate ETL schemas for tabular data. + +This module serves as an intermediate layer between the abstract SchemaDiscoveryProvider +and concrete implementations for specific tabular formats like CSV, Excel, and Parquet. + +Usage: + # This class is not typically used directly, but through its subclasses + from document_graph.schema.discovery.schema_discovery_registry import get_discovery_provider + + # Get a provider for a specific tabular format + csv_provider = get_discovery_provider("data/sample.csv") + excel_provider = get_discovery_provider("data/sample.xlsx") + parquet_provider = get_discovery_provider("data/sample.parquet") + + # All these providers inherit from TabularSchemaDiscoveryProvider + # and share common behavior for tabular data +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from pathlib import Path +from typing import Optional, Dict, Any + +from document_graph.schema.discovery.schema_discovery_base import SchemaDiscoveryProvider + + +class TabularSchemaDiscoveryProvider(SchemaDiscoveryProvider): + """ + Base class for tabular schema discovery providers like CSV, Excel, and Parquet. + + This class extends the SchemaDiscoveryProvider to provide common functionality + for tabular data formats. Tabular data is characterized by a row-column structure + where each row represents a record and each column represents a field. + + Concrete subclasses implement format-specific logic for reading and analyzing + tabular data sources like CSV, Excel, and Parquet files. These subclasses share + common patterns for extracting column headers, inferring data types, and creating + appropriate ETL schemas. + + The TabularSchemaDiscoveryProvider simplifies the implementation of concrete + providers by handling common aspects of tabular data processing, allowing + subclasses to focus on format-specific details. + + Attributes: + source (Path): The path to the tabular data file to analyze. + args (Dict[str, Any]): Optional arguments that control the discovery process. + These arguments are format-specific and are passed to the underlying + data reading functions (e.g., pandas.read_csv, pandas.read_excel). + """ + def __init__(self, source: Path, args: Optional[Dict[str, Any]] = None): + """ + Initialize the tabular schema discovery provider. + + This constructor sets up the provider with the source file path and any + format-specific arguments needed for the discovery process. Unlike the + parent class, it does not validate that the source file exists, leaving + that responsibility to the concrete subclasses. + + Args: + source (Path): The path to the tabular data file to analyze. + args (Optional[Dict[str, Any]]): Optional arguments that control the + discovery process. These arguments are format-specific and are + passed to the underlying data reading functions (e.g., pandas.read_csv, + pandas.read_excel). Defaults to None. + """ + self.source = Path(source) + self.args = args or {} diff --git a/document-graph/src/document_graph/schema/discovery/xml_discovery_provider.py b/document-graph/src/document_graph/schema/discovery/xml_discovery_provider.py new file mode 100644 index 00000000..9feb713f --- /dev/null +++ b/document-graph/src/document_graph/schema/discovery/xml_discovery_provider.py @@ -0,0 +1,171 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""XML Schema Discovery Provider Module for Document Graph Operations. + +This module provides a schema discovery provider for XML (eXtensible Markup Language) files. +It analyzes XML files to infer their structure and generate appropriate ETL schemas +for processing and loading the data into a document graph. + +The module includes the following components: +- XMLSchemaDiscoveryProvider: Schema discovery provider for XML files + +The XML schema discovery provider reads and analyzes XML files, extracting element names +and attributes from the root elements to create an ETL schema. It handles both simple +XML structures and more complex nested structures by flattening them into field names. + +Usage: + # Get an XML discovery provider for a specific file + from pathlib import Path + from document_graph.schema.discovery.schema_discovery_registry import get_discovery_provider + + file_path = Path("data/sample.xml") + provider = get_discovery_provider(file_path) + + # Or create it directly + from document_graph.schema.discovery.xml_discovery_provider import XMLSchemaDiscoveryProvider + + provider = XMLSchemaDiscoveryProvider(source=file_path) + + # Discover the schema + schema = provider.discover_schema() + + # Use the schema for ETL operations + print(f"Discovered schema ID: {schema.schema_id}") + print(f"Fields: {schema.load.document_node.fields}") +""" + +import logging +import xml.etree.ElementTree as ET +from typing import List, Set + +# Import custom logging to ensure configuration is available + +from document_graph.schema.etl_schema_model import * +from document_graph.schema.discovery.schema_discovery_base import SchemaDiscoveryProvider + +logger = logging.getLogger(__name__) + + +class XMLSchemaDiscoveryProvider(SchemaDiscoveryProvider): + """ + Schema discovery provider for XML (eXtensible Markup Language) files. + + This class analyzes XML files to infer their structure and generate appropriate + ETL schemas for processing and loading the data into a document graph. It reads + XML files and extracts element names and attributes to determine the fields that + should be included in the ETL schema. + + The provider handles XML structures by examining the first few elements to + determine the schema. It flattens nested structures into field names using + dot notation (e.g., "parent.child"). + + Attributes: + source (Path): The path to the XML file to analyze. + args (Dict[str, Any]): Optional arguments that control the discovery process. + Currently, no specific arguments are used for XML discovery. + """ + + def discover_schema(self) -> ETLSchema: + """ + Discover and return an ETL schema from an XML file. + + This method reads the XML file, extracts element names and attributes + from the structure, and generates an appropriate ETL schema. It includes + error handling to manage issues that might arise during XML parsing. + + The method performs the following steps: + 1. Validates that the source file exists + 2. Parses the XML file using ElementTree + 3. Extracts field names from elements and attributes + 4. Handles nested structures by flattening them + 5. Validates that fields were discovered + 6. Logs the discovered fields + 7. Constructs and returns an ETLSchema object + + Returns: + ETLSchema: A complete ETL schema object that describes the structure + of the XML file and how it should be processed. + + Raises: + FileNotFoundError: If the specified source file does not exist. + ValueError: If the XML file has no discoverable fields. + Exception: Any exception raised during XML parsing. + """ + if not self.source.exists(): + raise FileNotFoundError(f"File not found") + + try: + tree = ET.parse(self.source) + root = tree.getroot() + + # Extract field names from XML structure + fields = self._extract_fields_from_element(root) + + if not fields: + raise ValueError("Validation error") + + logger.info(f"Discovered fields from XML: {sorted(fields)}") + + except ET.ParseError as e: + raise ErrorHandler.file_parse_error(str(self.source), "xml", e) + except Exception as e: + raise ErrorHandler.file_parse_error(str(self.source), "xml", e) + + return ETLSchema( + schema_id=f"discovered-{self.source.stem}", + description=f"Discovered ETL schema from XML file: {self.source.name}", + extract=ExtractConfig(source_type="file", file_type="xml"), + transform=TransformConfig( + chunking=ChunkingConfig(strategy="fixed_length", min_length=100), + metadata_mapping=MetadataMapping(), + entity_extraction=EntityExtractionConfig(method="ner"), + normalize=NormalizeConfig() + ), + load=LoadConfig( + document_node=NodeDefinition(type="Document", fields=sorted(fields)), + section_node=NodeDefinition(type="Element", fields=sorted(fields)), + relationships=[ + RelationshipDefinition(type="contains", source="document_id", target="element_id") + ] + ) + ) + + def _extract_fields_from_element(self, element: ET.Element, prefix: str = "") -> Set[str]: + """ + Recursively extract field names from an XML element. + + This helper method traverses the XML structure and extracts field names + from element tags and attributes. It handles nested structures by using + dot notation to create hierarchical field names. + + Args: + element (ET.Element): The XML element to analyze. + prefix (str): The prefix to use for nested field names. + + Returns: + Set[str]: A set of field names discovered in the XML structure. + """ + fields = set() + + # Add the element's tag name + tag_name = element.tag.split('}')[-1] if '}' in element.tag else element.tag + field_name = f"{prefix}.{tag_name}" if prefix else tag_name + fields.add(field_name) + + # Add attributes + for attr_name in element.attrib: + attr_field = f"{field_name}.{attr_name}" + fields.add(attr_field) + + # Add text content if present + if element.text and element.text.strip(): + text_field = f"{field_name}.text" + fields.add(text_field) + + # Recursively process child elements (limit depth to avoid excessive nesting) + if len(field_name.split('.')) < 3: # Limit nesting depth + for child in element: + child_fields = self._extract_fields_from_element(child, field_name) + fields.update(child_fields) + + return fields \ No newline at end of file diff --git a/document-graph/src/document_graph/schema/discovery/yaml_discovery_provider.py b/document-graph/src/document_graph/schema/discovery/yaml_discovery_provider.py new file mode 100644 index 00000000..687ce712 --- /dev/null +++ b/document-graph/src/document_graph/schema/discovery/yaml_discovery_provider.py @@ -0,0 +1,181 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""YAML Schema Discovery Provider Module for Document Graph Operations. + +This module provides a schema discovery provider for YAML (YAML Ain't Markup Language) files. +It analyzes YAML files to infer their structure and generate appropriate ETL schemas +for processing and loading the data into a document graph. + +The module includes the following components: +- YAMLSchemaDiscoveryProvider: Schema discovery provider for YAML files + +The YAML schema discovery provider reads and analyzes YAML files, extracting keys +from the structure to create an ETL schema. It handles both simple YAML structures +and more complex nested structures by flattening them into field names. + +Usage: + # Get a YAML discovery provider for a specific file + from pathlib import Path + from document_graph.schema.discovery.schema_discovery_registry import get_discovery_provider + + file_path = Path("data/sample.yaml") + provider = get_discovery_provider(file_path) + + # Or create it directly + from document_graph.schema.discovery.yaml_discovery_provider import YAMLSchemaDiscoveryProvider + + provider = YAMLSchemaDiscoveryProvider(source=file_path) + + # Discover the schema + schema = provider.discover_schema() + + # Use the schema for ETL operations + print(f"Discovered schema ID: {schema.schema_id}") + print(f"Fields: {schema.load.document_node.fields}") +""" + +import logging +from typing import List, Set, Dict, Any, Union + +# Import custom logging to ensure configuration is available + +from document_graph.schema.etl_schema_model import * +from document_graph.schema.discovery.schema_discovery_base import SchemaDiscoveryProvider + +logger = logging.getLogger(__name__) + +try: + import yaml +except ImportError: + yaml = None + + +class YAMLSchemaDiscoveryProvider(SchemaDiscoveryProvider): + """ + Schema discovery provider for YAML (YAML Ain't Markup Language) files. + + This class analyzes YAML files to infer their structure and generate appropriate + ETL schemas for processing and loading the data into a document graph. It reads + YAML files and extracts keys from the structure to determine the fields that + should be included in the ETL schema. + + The provider handles both simple YAML structures and complex nested structures + by flattening them into field names using dot notation (e.g., "parent.child"). + It supports both single-document and multi-document YAML files. + + Attributes: + source (Path): The path to the YAML file to analyze. + args (Dict[str, Any]): Optional arguments that control the discovery process. + Currently, no specific arguments are used for YAML discovery. + """ + + def discover_schema(self) -> ETLSchema: + """ + Discover and return an ETL schema from a YAML file. + + This method reads the YAML file, extracts keys from the structure, + and generates an appropriate ETL schema. It includes error handling + to manage issues that might arise during YAML parsing. + + The method performs the following steps: + 1. Validates that the source file exists and PyYAML is available + 2. Parses the YAML file using yaml.safe_load + 3. Extracts field names from the structure + 4. Handles nested structures by flattening them + 5. Validates that fields were discovered + 6. Logs the discovered fields + 7. Constructs and returns an ETLSchema object + + Returns: + ETLSchema: A complete ETL schema object that describes the structure + of the YAML file and how it should be processed. + + Raises: + ImportError: If PyYAML is not installed. + FileNotFoundError: If the specified source file does not exist. + ValueError: If the YAML file has no discoverable fields. + Exception: Any exception raised during YAML parsing. + """ + if yaml is None: + raise ImportError("PyYAML is required for YAML schema discovery. Install with: pip install PyYAML") + + if not self.source.exists(): + raise FileNotFoundError(f"File not found") + + try: + with self.source.open("r", encoding="utf-8") as f: + data = yaml.safe_load(f) + + # Extract field names from YAML structure + fields = self._extract_fields_from_data(data) + + if not fields: + raise ValueError("Validation error") + + logger.info(f"Discovered fields from YAML: {sorted(fields)}") + + except yaml.YAMLError as e: + raise ErrorHandler.file_parse_error(str(self.source), "yaml", e) + except Exception as e: + raise ErrorHandler.file_parse_error(str(self.source), "yaml", e) + + return ETLSchema( + schema_id=f"discovered-{self.source.stem}", + description=f"Discovered ETL schema from YAML file: {self.source.name}", + extract=ExtractConfig(source_type="file", file_type="yaml"), + transform=TransformConfig( + chunking=ChunkingConfig(strategy="fixed_length", min_length=100), + metadata_mapping=MetadataMapping(), + entity_extraction=EntityExtractionConfig(method="ner"), + normalize=NormalizeConfig() + ), + load=LoadConfig( + document_node=NodeDefinition(type="Document", fields=sorted(fields)), + section_node=NodeDefinition(type="Section", fields=sorted(fields)), + relationships=[ + RelationshipDefinition(type="contains", source="document_id", target="section_id") + ] + ) + ) + + def _extract_fields_from_data(self, data: Any, prefix: str = "") -> Set[str]: + """ + Recursively extract field names from YAML data structure. + + This helper method traverses the YAML data structure and extracts field names + from dictionaries, lists, and scalar values. It handles nested structures by + using dot notation to create hierarchical field names. + + Args: + data (Any): The YAML data to analyze (dict, list, or scalar). + prefix (str): The prefix to use for nested field names. + + Returns: + Set[str]: A set of field names discovered in the YAML structure. + """ + fields = set() + + if isinstance(data, dict): + for key, value in data.items(): + field_name = f"{prefix}.{key}" if prefix else key + fields.add(field_name) + + # Recursively process nested structures (limit depth to avoid excessive nesting) + if len(field_name.split('.')) < 3: # Limit nesting depth + nested_fields = self._extract_fields_from_data(value, field_name) + fields.update(nested_fields) + + elif isinstance(data, list) and data: + # For lists, analyze the first item to determine structure + first_item = data[0] + if isinstance(first_item, dict): + # If it's a list of dictionaries, extract fields from the first dictionary + nested_fields = self._extract_fields_from_data(first_item, prefix) + fields.update(nested_fields) + elif prefix: + fields.add(prefix) + + if prefix: + fields.add(prefix) + + return fields \ No newline at end of file diff --git a/document-graph/src/document_graph/schema/document_graph_schema.py b/document-graph/src/document_graph/schema/document_graph_schema.py new file mode 100644 index 00000000..43051d85 --- /dev/null +++ b/document-graph/src/document_graph/schema/document_graph_schema.py @@ -0,0 +1,65 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Document Graph Schema Definitions. + +This module provides a centralized import location for all ETL schema models +used in document graph processing. It re-exports the core schema classes from +the etl_schema_model module to provide a clean, unified interface for working +with document graph schemas. + +The exported classes include: +- ETLSchema: The top-level schema definition for ETL operations +- ExtractConfig: Configuration for data extraction from various sources +- TransformConfig: Configuration for document transformation operations +- LoadConfig: Configuration for loading processed documents into a graph +- Supporting classes for specific aspects of the ETL process + +This module simplifies imports by allowing users to import all schema-related +classes from a single location rather than from multiple modules. + +Example: + from document_graph.schema.document_graph_schema import ( + ETLSchema, ExtractConfig, LoadConfig + ) + + # Create a schema using the imported classes + schema = ETLSchema( + schema_id="example-schema", + extract=ExtractConfig(...), + transform=TransformConfig(...), + load=LoadConfig(...) + ) +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + +from typing import List +from .etl_schema_model import ( + ETLSchema, + ExtractConfig, + TransformConfig, + LoadConfig, + ChunkingConfig, + MetadataMapping, + EntityExtractionConfig, + NormalizeConfig, + NodeDefinition, + RelationshipDefinition, +) + +__all__: List[str] = [ + "ETLSchema", + "ExtractConfig", + "TransformConfig", + "LoadConfig", + "ChunkingConfig", + "MetadataMapping", + "EntityExtractionConfig", + "NormalizeConfig", + "NodeDefinition", + "RelationshipDefinition", +] diff --git a/document-graph/src/document_graph/schema/etl_schema_model.py b/document-graph/src/document_graph/schema/etl_schema_model.py new file mode 100644 index 00000000..2cf652ed --- /dev/null +++ b/document-graph/src/document_graph/schema/etl_schema_model.py @@ -0,0 +1,316 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""ETL Schema Model Module for Document Graph Operations. + +This module defines the data models used for ETL (Extract, Transform, Load) operations +in the document graph system. It provides a comprehensive set of Pydantic models that +represent different aspects of the ETL process: + +1. Extract: Models for configuring data extraction from various sources +2. Transform: Models for text chunking, metadata mapping, entity extraction, and normalization +3. Load: Models for defining graph nodes, relationships, and loading configurations + +These models are used throughout the document graph system to define how documents +are processed, transformed, and loaded into graph databases. The models are designed +to be extensible, allowing for custom configurations through the use of Pydantic's +extra fields feature. + +The top-level ETLSchema class combines all these components into a complete schema +that can be serialized to/from JSON and used to configure the entire ETL pipeline. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from typing import List, Optional +from pydantic import BaseModel, ConfigDict + + +# Extract section +class ExtractConfig(BaseModel): + """ + Configuration for data extraction from various sources. + + This class defines the parameters needed to extract data from different source types, + such as S3 buckets, local files, or APIs. It specifies where to find the data and + how to read it. + + Attributes: + source_type (str): The type of data source (e.g., "s3", "html", "csv", "api"). + bucket (Optional[str]): The S3 bucket name for S3 sources. + prefix (Optional[str]): The S3 prefix/folder path for S3 sources. + key (Optional[str]): The specific S3 key or file path. + file_type (Optional[str]): The type of file to extract (e.g., "pdf", "docx"). + reader (Optional[str]): The reader to use for extraction (e.g., "pymupdf"). + delimiter (Optional[str]): The delimiter for CSV or similar files. + encoding (Optional[str]): The character encoding of the source file. + + Note: + This class allows additional custom fields through the `extra = "allow"` config, + enabling users to add source-specific parameters like API tokens or headers. + """ + source_type: str # Allow user-defined types (e.g., "html", "csv", "api") + bucket: Optional[str] = None + prefix: Optional[str] = None + key: Optional[str] = None + file_type: Optional[str] = None + reader: Optional[str] = None + delimiter: Optional[str] = None + encoding: Optional[str] = None + + model_config = ConfigDict(extra="allow") + + +# Transform section +class ChunkingConfig(BaseModel): + """ + Configuration for chunking text during document transformation. + + This class defines how documents should be divided into smaller, manageable chunks + for processing and storage. Different chunking strategies can be used depending on + the document structure and requirements. + + Attributes: + strategy (str): The chunking strategy to use (e.g., "by_heading", "fixed_length", + "by_customer"). + min_length (Optional[int]): The minimum length of a chunk in characters or tokens, + defaulting to 100. + + Note: + This class allows additional custom fields through the `extra = "allow"` config, + enabling users to add strategy-specific parameters like maximum chunk size or + overlap settings. + """ + strategy: str # e.g., "by_heading", "fixed_length", "by_customer" + min_length: Optional[int] = 100 + + model_config = ConfigDict(extra="allow") + + +class MetadataMapping(BaseModel): + """ + Field-level mapping of document metadata to standardized fields. + + This class defines how metadata fields from source documents should be mapped + to standardized fields in the document graph. It allows for consistent metadata + representation across different document types and sources. + + Attributes: + title (Optional[str]): Field path or expression to extract the document title. + author (Optional[str]): Field path or expression to extract the document author. + created_date (Optional[str]): Field path or expression to extract the document creation date. + category (Optional[str]): Field path or expression to extract the document category. + + Note: + This class allows additional custom fields through the `extra = "allow"` config, + enabling users to map any custom metadata fields specific to their document types. + Field values can be direct field names or dot-notation paths (e.g., "metadata.author"). + """ + title: Optional[str] = None + author: Optional[str] = None + created_date: Optional[str] = None + category: Optional[str] = None # e.g., for classification purposes + + model_config = ConfigDict(extra="allow") + + +class EntityExtractionConfig(BaseModel): + """ + Configuration for extracting named entities from document text. + + This class defines how named entities (such as people, organizations, locations) + should be extracted from document text during the transformation phase. It specifies + the extraction method, model to use, and which entity types to extract. + + Attributes: + method (str): The entity extraction method to use (e.g., "ner", "regex", "rule_based"). + model (Optional[str]): The specific model to use for extraction (e.g., "spacy_en_core_web_lg"). + extract_emails (Optional[bool]): Whether to extract email addresses, defaults to False. + extract_dates (Optional[bool]): Whether to extract date references, defaults to False. + + Note: + This class allows additional custom fields through the `extra = "allow"` config, + enabling users to specify additional entity types to extract or method-specific + parameters like confidence thresholds. + """ + method: str # e.g., "ner", "regex", "rule_based" + model: Optional[str] = None + extract_emails: Optional[bool] = False + extract_dates: Optional[bool] = False + + model_config = ConfigDict(extra="allow") + + +class NormalizeConfig(BaseModel): + """ + Configuration for text normalization during document processing. + + This class defines how document text should be normalized during the transformation + phase. Normalization can include removing headers, standardizing whitespace, + standardizing date formats, and other text cleaning operations. + + Attributes: + remove_headers (bool): Whether to remove headers from the text, defaults to True. + collapse_whitespace (bool): Whether to normalize whitespace (removing extra spaces, + newlines, etc.), defaults to True. + standardize_dates (Optional[bool]): Whether to convert dates to a standard format, + defaults to False. + + Note: + This class allows additional custom fields through the `extra = "allow"` config, + enabling users to specify additional normalization operations specific to their + document types or requirements. + """ + remove_headers: bool = True + collapse_whitespace: bool = True + standardize_dates: Optional[bool] = False + + model_config = ConfigDict(extra="allow") + + +class TransformConfig(BaseModel): + """ + Complete configuration for document transformation operations. + + This class combines all the transformation-related configurations into a single + comprehensive configuration for the transform phase of the ETL process. It includes + settings for chunking, metadata mapping, entity extraction, and text normalization. + + Attributes: + chunking (ChunkingConfig): Configuration for how to chunk the document text. + metadata_mapping (MetadataMapping): Configuration for mapping document metadata fields. + entity_extraction (EntityExtractionConfig): Configuration for extracting named entities. + normalize (NormalizeConfig): Configuration for text normalization operations. + + Note: + This class allows additional custom fields through the `extra = "allow"` config, + enabling users to add custom transformation configurations not covered by the + standard components. + """ + chunking: ChunkingConfig + metadata_mapping: MetadataMapping + entity_extraction: EntityExtractionConfig + normalize: NormalizeConfig + + model_config = ConfigDict(extra="allow") + + +# Load section +class NodeDefinition(BaseModel): + """ + Definition of a graph node type and its associated fields. + + This class defines a type of node in the document graph and specifies which fields + should be included in nodes of this type. It is used in the load phase to determine + how to create nodes in the graph database. + + Attributes: + type (str): The type or label of the node (e.g., "DocumentNode", "SectionNode"). + fields (List[str]): List of field names to include in nodes of this type. + + Note: + This class allows additional custom fields through the `extra = "allow"` config, + enabling users to specify additional node properties like indexes or constraints. + """ + type: str + fields: List[str] + + model_config = ConfigDict(extra="allow") + + +class RelationshipDefinition(BaseModel): + """ + Defines a relationship (edge) between two node types in the document graph. + + This class specifies how nodes in the graph should be connected to each other, + defining the type of relationship and which nodes should be the source and target + of the relationship. It is used in the load phase to create edges in the graph database. + + Attributes: + type (str): The type or label of the relationship (e.g., "has_section", "references"). + source (str): The identifier for the source node of the relationship. + target (str): The identifier for the target node of the relationship. + + Note: + This class allows additional custom fields through the `extra = "allow"` config, + enabling users to specify additional relationship properties like weights or + directional constraints. + """ + type: str + source: str + target: str + + model_config = ConfigDict(extra="allow") + + +class LoadConfig(BaseModel): + """ + Configuration for loading processed documents into a graph database. + + This class defines how processed document data should be loaded into a graph database, + specifying the node types, their fields, and the relationships between them. It is + used in the load phase of the ETL process to create the document graph structure. + + Attributes: + document_node (NodeDefinition): Definition of the document node type and its fields. + section_node (NodeDefinition): Definition of the section node type and its fields. + relationships (List[RelationshipDefinition]): List of relationships between nodes + in the graph. + + Note: + This class allows additional custom fields through the `extra = "allow"` config, + enabling users to specify additional loading configurations like batch sizes, + transaction settings, or custom node types beyond the standard document and + section nodes. + """ + document_node: NodeDefinition + section_node: NodeDefinition + relationships: List[RelationshipDefinition] + + model_config = ConfigDict(extra="allow") + + +# Top-level schema +class ETLSchema(BaseModel): + """ + Complete configuration for ETL (Extract, Transform, Load) processing of document graphs. + + This is the top-level schema class that combines all aspects of the ETL process into + a single comprehensive configuration. It includes configurations for extracting data + from sources, transforming the data through various operations, and loading the + processed data into a graph database. + + Attributes: + schema_id (str): A unique identifier for this schema. + description (Optional[str]): A human-readable description of the schema's purpose. + extract (ExtractConfig): Configuration for the extract phase. + transform (TransformConfig): Configuration for the transform phase. + load (LoadConfig): Configuration for the load phase. + + Note: + This class allows additional custom fields through the `extra = "allow"` config, + enabling users to add custom configurations not covered by the standard components. + + Example: + ```python + schema = ETLSchema( + schema_id="pdf-document-schema", + description="Schema for processing PDF documents", + extract=ExtractConfig(source_type="s3", bucket="documents", file_type="pdf"), + transform=TransformConfig(...), + load=LoadConfig(...) + ) + ``` + """ + schema_id: str + description: Optional[str] = None + extract: ExtractConfig + transform: TransformConfig + load: LoadConfig + + model_config = ConfigDict(extra="allow") diff --git a/document-graph/src/document_graph/schema/providers/__init__.py b/document-graph/src/document_graph/schema/providers/__init__.py new file mode 100644 index 00000000..f980eeff --- /dev/null +++ b/document-graph/src/document_graph/schema/providers/__init__.py @@ -0,0 +1,100 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Schema Providers Package for Document Graph Operations. + +This package provides a collection of schema providers for loading and managing ETL schemas +used in document graph operations. Schema providers are responsible for loading schemas +from various sources such as files, databases, and cloud services. + +The package includes the following components: + +Base Classes: +- SchemaProviderBase: Abstract base class for all schema providers +- SchemaProviderConfig: Base configuration model for schema providers +- AWSSchemaProviderConfig: Base configuration model for AWS-backed schema providers + +Factory and Registry: +- SchemaProviderFactory: Factory for creating schema providers based on configuration +- SchemaProviderRegistry: Registry for managing schema provider classes + +Provider Implementations: +- FileSchemaProvider: Provider for loading schemas from local files +- S3SchemaProvider: Provider for loading schemas from Amazon S3 +- CSVSchemaProvider: Provider for generating schemas from CSV files +- JSONSchemaProvider: Provider for generating schemas from JSON files +- ExcelSchemaProvider: Provider for generating schemas from Excel files +- GlueSchemaProvider: Provider for loading schemas from AWS Glue +- ParquetSchemaProvider: Provider for generating schemas from Parquet files +- YAMLSchemaProvider: Provider for generating schemas from YAML files +- XMLSchemaProvider: Provider for generating schemas from XML files + +Usage: + # Create a schema provider using the factory + from document_graph.schema.providers.schema_provider_factory import get_schema_provider + + config = { + "type": "file", + "schema_id": "my_schema", + "connection_config": { + "path": "/path/to/schema.yaml" + } + } + + provider = get_schema_provider(config) + + # Load the schema + schema = provider.pipeline_schema() + + # Use the schema for ETL operations + print(f"Loaded schema ID: {schema.schema_id}") +""" + +# Base classes +from .schema_provider_base import SchemaProviderBase +from .schema_provider_config import SchemaProviderConfig +from .schema_provider_config_aws_base import AWSSchemaProviderConfig + +# Factory and registry +from .schema_provider_factory import SchemaProviderFactory +from .schema_provider_registry import SchemaProviderRegistry + +# Provider implementations +from .file_schema_provider import FileSchemaProvider +from .s3_schema_provider import S3SchemaProvider +from .csv_schema_provider import CSVSchemaProvider +from .json_schema_provider import JSONSchemaProvider +from .excel_schema_provider import ExcelSchemaProvider +from .glue_schema_provider import GlueSchemaProvider +from .parquet_schema_provider import ParquetSchemaProvider +from .yaml_schema_provider import YAMLSchemaProvider +from .xml_schema_provider import XMLSchemaProvider + +__all__ = [ + # Base classes + 'SchemaProviderBase', + 'SchemaProviderConfig', + 'AWSSchemaProviderConfig', + + # Factory and registry + 'SchemaProviderFactory', + 'SchemaProviderRegistry', + + # Provider implementations + 'FileSchemaProvider', + 'S3SchemaProvider', + 'CSVSchemaProvider', + 'JSONSchemaProvider', + 'ExcelSchemaProvider', + 'GlueSchemaProvider', + 'ParquetSchemaProvider', + 'YAMLSchemaProvider', + 'XMLSchemaProvider', + + # Aliases for backward compatibility + 'CSVSchemaProvider', + 'JSONSchemaProvider' +] + +# Aliases +CSVSchemaProvider = CSVSchemaProvider # Already correct name +JSONSchemaProvider = JSONSchemaProvider # Already correct name \ No newline at end of file diff --git a/document-graph/src/document_graph/schema/providers/csv_schema_provider.py b/document-graph/src/document_graph/schema/providers/csv_schema_provider.py new file mode 100644 index 00000000..b424372b --- /dev/null +++ b/document-graph/src/document_graph/schema/providers/csv_schema_provider.py @@ -0,0 +1,157 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""CSV Schema Provider Module for Document Graph Operations. + +This module provides a schema provider for CSV (Comma-Separated Values) files. +It discovers and generates ETL schemas from CSV files for processing and loading +the data into a document graph. + +The module includes the following components: +- CSVSchemaProvider: Schema provider for CSV files + +The CSV schema provider uses the CSVSchemaDiscoveryProvider to analyze CSV files +and generate appropriate ETL schemas. It supports loading schemas from CSV files +and optionally saving the discovered schemas to JSON files. + +Usage: + # Get a CSV schema provider for a specific file + from document_graph.schema.providers.schema_provider_factory import get_schema_provider + + config = { + "provider_type": "csv", + "schema_id": "my_csv_schema", + "connection_config": { + "path": "/path/to/data.csv" + } + } + + provider = get_schema_provider(config) + + # Load the schema + schema = provider.pipeline_schema() + + # Use the schema for ETL operations + print(f"Loaded schema ID: {schema.schema_id}") + print(f"Fields: {schema.load.document_node.fields}") +""" + +import logging +import csv +import json +from pathlib import Path +from typing import List, Optional, Any + +# Import custom logging to ensure configuration is available + +from document_graph.schema.etl_schema_model import ETLSchema +from document_graph.schema.providers.schema_provider_base import SchemaProviderBase +from document_graph.schema.providers.schema_provider_config import SchemaProviderConfig +from document_graph.schema.discovery.csv_discovery_provider import CSVSchemaDiscoveryProvider + +logger = logging.getLogger(__name__) + + +class CSVSchemaProvider(SchemaProviderBase): + """ + Schema provider for CSV (Comma-Separated Values) files. + + This class provides functionality to discover and generate ETL schemas from CSV files. + It uses the CSVSchemaDiscoveryProvider to analyze CSV files and extract field information + to create appropriate ETL schemas for processing and loading the data into a document graph. + + The provider supports loading schemas directly from CSV files and optionally saving + the discovered schemas to JSON files for later use. + + Attributes: + config (SchemaProviderConfig): Configuration for the provider. + csv_path (Path): The path to the CSV file to analyze. + """ + + def __init__(self, config: SchemaProviderConfig): + """ + Initialize a CSV schema provider with the given configuration. + + Args: + config (SchemaProviderConfig): Configuration for the provider, including + the path to the CSV file in the connection_config. + + Raises: + ValueError: If the path is not provided in the connection_config. + FileNotFoundError: If the specified CSV file does not exist. + """ + self.config = config + path = config.connection_config.get("path") + if not path: + raise ValueError("Validation error") + self.csv_path = Path(path) + + if not self.csv_path.exists(): + raise FileNotFoundError(f"File not found") + + @classmethod + def from_config(cls, config: SchemaProviderConfig) -> "CSVSchemaProvider": + """ + Factory method to construct a CSV provider from a configuration. + + Args: + config (SchemaProviderConfig): Configuration for the provider. + + Returns: + CSVSchemaProvider: A new instance of the CSV schema provider. + """ + return cls(config) + + def load_schema(self, source=None, **kwargs) -> ETLSchema: + """ + Load (or generate) an ETL schema from a CSV file using schema discovery. + + This method uses the CSVSchemaDiscoveryProvider to analyze the CSV file and + generate an appropriate ETL schema based on the structure and content of the file. + + Args: + source: Optional source override (not used in this provider). + **kwargs: Additional provider-specific parameters (not used in this provider). + + Returns: + ETLSchema: A complete ETL schema object that describes the structure + of the CSV file and how it should be processed. + + Raises: + FileNotFoundError: If the CSV file does not exist. + ValueError: If the CSV file cannot be parsed or has no discoverable fields. + """ + discovery = CSVSchemaDiscoveryProvider(source=self.csv_path) + return discovery.discover_schema() + + def save_schema(self, output_path: Path) -> None: + """ + Save the inferred ETL schema to a JSON file at the given output path. + + This method discovers the schema from the CSV file and saves it to a JSON file + at the specified path. The schema can then be loaded directly using a FileSchemaProvider. + + Args: + output_path (Path): The path where the schema JSON file should be saved. + + Raises: + IOError: If the schema cannot be saved to the output path. + """ + schema = self.load_schema() + try: + with output_path.open("w", encoding="utf-8") as f: + json.dump(schema.model_dump(mode="json", exclude_unset=True), f, indent=2) + logger.info(f"Saved CSV schema to file: {output_path}") + except Exception as e: + raise IOError(f"IO error: {output_path, e}") + + def get_schema_id(self) -> str: + """ + Return the unique identifier for the schema. + + This method returns a unique identifier for the schema, either from the + configuration or generated from the CSV file name. + + Returns: + str: A unique identifier for the schema. + """ + return self.config.schema_id or f"csv-schema-{self.csv_path.stem}" diff --git a/document-graph/src/document_graph/schema/providers/excel_schema_provider.py b/document-graph/src/document_graph/schema/providers/excel_schema_provider.py new file mode 100644 index 00000000..116742c1 --- /dev/null +++ b/document-graph/src/document_graph/schema/providers/excel_schema_provider.py @@ -0,0 +1,161 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Excel Schema Provider Module for Document Graph Operations. + +This module provides a schema provider for Excel files. +It discovers and generates ETL schemas from Excel files for processing and loading +the data into a document graph. + +The module includes the following components: +- ExcelSchemaProvider: Schema provider for Excel files + +The Excel schema provider uses the ExcelSchemaDiscoveryProvider to analyze Excel files +and generate appropriate ETL schemas. It supports loading schemas from Excel files +and optionally saving the discovered schemas to JSON files. + +Usage: + # Get an Excel schema provider for a specific file + from document_graph.schema.providers.schema_provider_factory import get_schema_provider + + config = { + "provider_type": "excel", + "schema_id": "my_excel_schema", + "connection_config": { + "path": "/path/to/data.xlsx" + } + } + + provider = get_schema_provider(config) + + # Load the schema + schema = provider.pipeline_schema() + + # Use the schema for ETL operations + print(f"Loaded schema ID: {schema.schema_id}") + print(f"Fields: {schema.load.document_node.fields}") +""" + +import logging + +# Import custom logging to ensure configuration is available + +import json +from pathlib import Path +from typing import Optional, Any + +from document_graph.schema.etl_schema_model import ETLSchema +from document_graph.schema.providers.schema_provider_base import SchemaProviderBase +from document_graph.schema.providers.schema_provider_config import SchemaProviderConfig + +from document_graph.schema.discovery.excel_discovery_provider import ( + ExcelSchemaDiscoveryProvider +) + +logger = logging.getLogger(__name__) + + +class ExcelSchemaProvider(SchemaProviderBase): + """ + Schema provider for Excel files. + + This class provides functionality to discover and generate ETL schemas from Excel files. + It uses the ExcelSchemaDiscoveryProvider to analyze Excel files and extract field information + to create appropriate ETL schemas for processing and loading the data into a document graph. + + The provider supports loading schemas directly from Excel files and optionally saving + the discovered schemas to JSON files for later use. + + Attributes: + config (SchemaProviderConfig): Configuration for the provider. + excel_path (Path): The path to the Excel file to analyze. + """ + + def __init__(self, config: SchemaProviderConfig): + """ + Initialize an Excel schema provider with the given configuration. + + Args: + config (SchemaProviderConfig): Configuration for the provider, including + the path to the Excel file in the connection_config. + + Raises: + ValueError: If the path is not provided in the connection_config. + FileNotFoundError: If the specified Excel file does not exist. + """ + self.config = config + path = config.connection_config.get("path") + if not path: + raise ValueError("Validation error") + self.excel_path = Path(path) + + if not self.excel_path.exists(): + raise FileNotFoundError(f"File not found") + + @classmethod + def from_config(cls, config: SchemaProviderConfig) -> "ExcelSchemaProvider": + """ + Factory method to construct an Excel provider from a configuration. + + Args: + config (SchemaProviderConfig): Configuration for the provider. + + Returns: + ExcelSchemaProvider: A new instance of the Excel schema provider. + """ + return cls(config) + + def load_schema(self, source=None, **kwargs) -> ETLSchema: + """ + Load (or generate) an ETL schema from an Excel file using schema discovery. + + This method uses the ExcelSchemaDiscoveryProvider to analyze the Excel file and + generate an appropriate ETL schema based on the structure and content of the file. + + Args: + source: Optional source override (not used in this provider). + **kwargs: Additional provider-specific parameters (not used in this provider). + + Returns: + ETLSchema: A complete ETL schema object that describes the structure + of the Excel file and how it should be processed. + + Raises: + FileNotFoundError: If the Excel file does not exist. + ValueError: If the Excel file cannot be parsed or has no discoverable fields. + ImportError: If the required Excel libraries are not installed. + """ + discovery = ExcelSchemaDiscoveryProvider(source=self.excel_path) + return discovery.discover_schema() + + def save_schema(self, output_path: Path) -> None: + """ + Save the discovered ETL schema to a JSON file at the given path. + + This method discovers the schema from the Excel file and saves it to a JSON file + at the specified path. The schema can then be loaded directly using a FileSchemaProvider. + + Args: + output_path (Path): The path where the schema JSON file should be saved. + + Raises: + IOError: If the schema cannot be saved to the output path. + """ + schema = self.load_schema() + try: + with output_path.open("w", encoding="utf-8") as f: + json.dump(schema.model_dump(mode="json", exclude_unset=True), f, indent=2) + logger.info(f"Saved Excel schema to file: {output_path}") + except Exception as e: + raise IOError(f"IO error: {output_path, e}") + + def get_schema_id(self) -> str: + """ + Return the unique identifier for the schema. + + This method returns a unique identifier for the schema, either from the + configuration or generated from the Excel file name. + + Returns: + str: A unique identifier for the schema. + """ + return self.config.schema_id or f"excel-schema-{self.excel_path.stem}" diff --git a/document-graph/src/document_graph/schema/providers/file_schema_provider.py b/document-graph/src/document_graph/schema/providers/file_schema_provider.py new file mode 100644 index 00000000..a6f29eaa --- /dev/null +++ b/document-graph/src/document_graph/schema/providers/file_schema_provider.py @@ -0,0 +1,174 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""File Schema Provider Module for Document Graph Operations. + +This module provides a schema provider for loading and saving ETL schemas from/to local JSON files. +Unlike discovery-based providers, this provider works with pre-defined schema files rather than +inferring schemas from data sources. + +The module includes the following components: +- FileSchemaProvider: Schema provider for JSON schema files + +The file schema provider loads ETL schemas directly from JSON files and can also save +schemas back to files. It's commonly used to store and retrieve pre-defined or previously +discovered schemas. + +Usage: + # Get a file schema provider for a specific schema file + from document_graph.schema.providers.schema_provider_factory import get_schema_provider + + config = { + "provider_type": "file", + "schema_id": "my_schema", + "connection_config": { + "path": "/path/to/schema.json" + } + } + + provider = get_schema_provider(config) + + # Load the schema + schema = provider.pipeline_schema() + + # Use the schema for ETL operations + print(f"Loaded schema ID: {schema.schema_id}") + print(f"Fields: {schema.load.document_node.fields}") + + # Save a modified schema + provider.save_schema(schema) +""" + +# Import custom logging to ensure configuration is available + +import json +import logging +from pathlib import Path +from typing import Any, Optional + +from document_graph.schema.providers.schema_provider_base import SchemaProviderBase +from document_graph.schema.providers.schema_provider_config import SchemaProviderConfig +from document_graph.schema.etl_schema_model import ETLSchema + +logger = logging.getLogger(__name__) + + +class FileSchemaProvider(SchemaProviderBase): + """ + Schema provider for loading and saving ETL schemas from/to local JSON files. + + This class provides functionality to load pre-defined ETL schemas from JSON files + and save schemas back to files. Unlike discovery-based providers, this provider + works with existing schema files rather than inferring schemas from data sources. + + The provider is commonly used to store and retrieve pre-defined or previously + discovered schemas for reuse in ETL pipelines. + + Attributes: + config (SchemaProviderConfig): Configuration for the provider. + path (Path): The path to the JSON schema file. + _schema_id (str): The unique identifier for the schema. + """ + + def __init__(self, config: SchemaProviderConfig): + """ + Initialize a file schema provider with the given configuration. + + Args: + config (SchemaProviderConfig): Configuration for the provider, including + the path to the JSON schema file in the connection_config. + + Raises: + ValueError: If the path is not provided in the connection_config. + FileNotFoundError: If the specified JSON file does not exist. + """ + self.config = config + path_str = config.connection_config.get("path") + + if not path_str: + raise ErrorHandler.validation_error( + "schema_config_path", + "None", + "valid file path in config.connection_config['path']" + ) + + self.path: Path = Path(path_str) + if not self.path.exists(): + raise FileNotFoundError(f"File not found: {self.path}") + + self._schema_id: str = config.schema_id or self.path.stem + + @classmethod + def from_config(cls, config: SchemaProviderConfig) -> "FileSchemaProvider": + """ + Factory method to construct a file provider from a configuration. + + Args: + config (SchemaProviderConfig): Configuration for the provider. + + Returns: + FileSchemaProvider: A new instance of the file schema provider. + """ + return cls(config) + + def load_schema(self, source=None, **kwargs) -> ETLSchema: + """ + Load the ETL schema from the JSON file and parse it into an ETLSchema object. + + This method reads the JSON schema file, parses it, and returns an ETLSchema object. + Unlike discovery-based providers, this method loads a pre-defined schema rather + than inferring one from a data source. + + Args: + source: Optional source override (not used in this provider). + **kwargs: Additional provider-specific parameters (not used in this provider). + + Returns: + ETLSchema: The ETL schema loaded from the JSON file. + + Raises: + FileNotFoundError: If the JSON file does not exist. + ValueError: If the JSON file cannot be parsed or is not a valid ETL schema. + IOError: If there's an error reading the file. + """ + try: + with self.path.open("r", encoding="utf-8") as f: + schema_json = json.load(f) + logger.info(f"Loaded schema from file: {self.path}") + return ETLSchema(**schema_json) + except Exception as e: + logger.error(f"Failed to load ETL schema from file {self.path}: {e}") + raise ErrorHandler.database_error("file schema load", e) + + def save_schema(self, schema: ETLSchema) -> None: + """ + Save the ETL schema to the file path specified in the configuration. + + This method serializes the ETL schema to JSON and writes it to the file + specified in the provider's configuration. The schema can then be loaded + later using this or another FileSchemaProvider. + + Args: + schema (ETLSchema): The ETL schema to save. + + Raises: + IOError: If there's an error writing to the file. + """ + try: + with self.path.open("w", encoding="utf-8") as f: + json.dump(schema.model_dump(mode="json", exclude_unset=True), f, indent=2) + logger.info(f"Saved schema to file: {self.path}") + except Exception as e: + logger.error(f"Failed to save ETL schema to file {self.path}: {e}") + raise IOError(f"IO error: {self.path, e}") + + def get_schema_id(self) -> str: + """ + Return the unique identifier for the schema. + + This method returns a unique identifier for the schema, either from the + configuration or generated from the JSON file name. + + Returns: + str: A unique identifier for the schema. + """ + return self._schema_id diff --git a/document-graph/src/document_graph/schema/providers/glue_schema_provider.py b/document-graph/src/document_graph/schema/providers/glue_schema_provider.py new file mode 100644 index 00000000..43d3ea93 --- /dev/null +++ b/document-graph/src/document_graph/schema/providers/glue_schema_provider.py @@ -0,0 +1,246 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""AWS Glue Schema Provider Module for Document Graph Operations. + +This module provides a schema provider for loading ETL schemas from AWS Glue Schema Registry. +It retrieves AVRO schemas from Glue and converts them into ETL schemas for processing +and loading data into a document graph. + +The module includes the following components: +- GlueSchemaProvider: Schema provider for AWS Glue Schema Registry + +The Glue schema provider connects to AWS Glue Schema Registry, retrieves the latest +version of a specified schema, and converts it into an ETL schema. It extracts field +information from AVRO schemas and builds appropriate ETL schemas for document graph +operations. + +Usage: + # Get a Glue schema provider for a specific schema + from document_graph.schema.providers.schema_provider_factory import get_schema_provider + + config = { + "provider_type": "glue", + "schema_id": "my_glue_schema", + "connection_config": { + "registry_name": "my-registry", + "schema_name": "my-schema", + "region": "us-east-1" + } + } + + provider = get_schema_provider(config) + + # Load the schema + schema = provider.pipeline_schema() + + # Use the schema for ETL operations + print(f"Loaded schema ID: {schema.schema_id}") + print(f"Fields: {schema.load.document_node.fields}") +""" + +# Import custom logging to ensure configuration is available + +import json +import logging +from typing import Any, List, Optional + +from document_graph.schema.providers.schema_provider_base import SchemaProviderBase +from document_graph.schema.providers.schema_provider_config import SchemaProviderConfig +from document_graph.schema.etl_schema_model import ( + ETLSchema, ExtractConfig, TransformConfig, LoadConfig, + ChunkingConfig, MetadataMapping, EntityExtractionConfig, NormalizeConfig, + NodeDefinition, RelationshipDefinition +) +from document_graph.config import DocumentGraphConfig + +logger = logging.getLogger(__name__) + + +class GlueSchemaProvider(SchemaProviderBase): + """ + Schema provider for AWS Glue Schema Registry. + + This class provides functionality to load ETL schemas from AWS Glue Schema Registry. + It connects to the registry, retrieves the latest version of a specified schema, + and converts it into an ETL schema for document graph operations. + + The provider extracts field information from AVRO schemas in the Glue registry + and builds appropriate ETL schemas with those fields. It supports AWS authentication + through boto3 sessions and can be configured with different AWS regions. + + Attributes: + config (SchemaProviderConfig): Configuration for the provider. + registry_name (str): The name of the Glue schema registry. + schema_name (str): The name of the schema in the registry. + region (str): The AWS region where the registry is located. + session: The boto3 session for AWS authentication. + """ + + def __init__(self, config: SchemaProviderConfig): + """ + Initialize a Glue schema provider with the given configuration. + + Args: + config (SchemaProviderConfig): Configuration for the provider, including + the registry name and schema name in the connection_config. + + Raises: + ValueError: If the registry name or schema name is not provided in the connection_config. + """ + self.config = config + conn = config.connection_config + + self.registry_name = conn.get("registry_name") + self.schema_name = conn.get("schema_name") + self.region = conn.get("region", DocumentGraphConfig.aws_region) + self.session = conn.get("session") or DocumentGraphConfig.session + + if not self.registry_name or not self.schema_name: + raise ErrorHandler.validation_error( + "glue.connection_config", + conn, + "registry_name and schema_name must be defined" + ) + + @classmethod + def from_config(cls, config: SchemaProviderConfig) -> "GlueSchemaProvider": + """ + Factory method to construct a Glue provider from a configuration. + + Args: + config (SchemaProviderConfig): Configuration for the provider. + + Returns: + GlueSchemaProvider: A new instance of the Glue schema provider. + """ + return cls(config) + + def load_schema(self, source=None, **kwargs) -> ETLSchema: + """ + Load an ETL schema from AWS Glue Schema Registry. + + This method connects to AWS Glue Schema Registry, retrieves the latest version + of the specified schema, extracts field information from the AVRO schema, + and builds an appropriate ETL schema for document graph operations. + + Args: + source: Optional source override (not used in this provider). + **kwargs: Additional provider-specific parameters (not used in this provider). + + Returns: + ETLSchema: A complete ETL schema object built from the Glue schema. + + Raises: + ValueError: If the schema does not exist in the registry or cannot be parsed. + Exception: If there's an error connecting to AWS or retrieving the schema. + """ + glue = self.session.client("glue", region_name=self.region) + try: + response = glue.get_schema_version( + SchemaId={ + "SchemaName": self.schema_name, + "RegistryName": self.registry_name + }, + SchemaVersionNumber={"LatestVersion": True} + ) + schema_definition = response["SchemaDefinition"] + fields = self._parse_avro_fields(schema_definition) + + logger.info(f"Loaded schema from Glue registry: {self.registry_name}/{self.schema_name}") + return self._build_minimal_etl_schema(fields) + + except glue.exceptions.EntityNotFoundException: + raise ErrorHandler.validation_error( + "glue_schema", + f"{self.registry_name}/{self.schema_name}", + "existing schema in Glue registry" + ) + except Exception as e: + logger.error(f"Failed to load Glue schema: {e}") + raise ErrorHandler.database_error("Glue schema load", e) + + def get_schema_id(self) -> str: + """ + Return the unique identifier for the schema. + + This method returns a unique identifier for the schema, either from the + configuration or generated from the registry and schema names. + + Returns: + str: A unique identifier for the schema. + """ + return self.config.schema_id or f"{self.registry_name}:{self.schema_name}" + + def _parse_avro_fields(self, schema_def: str) -> List[str]: + """ + Parse field names from an AVRO schema definition. + + This method parses the AVRO schema definition to extract field names, + safely handling union types and defaults. It validates that the schema + is of AVRO record type and contains fields. + + Args: + schema_def (str): The AVRO schema definition as a JSON string. + + Returns: + List[str]: A list of field names extracted from the AVRO schema. + + Raises: + ValueError: If the schema is not of AVRO record type, is missing fields, + or no fields could be extracted. + Exception: If there's an error parsing the schema. + """ + try: + schema_json = json.loads(schema_def) + + # Handle AVRO record type at root + if schema_json.get("type") != "record" or "fields" not in schema_json: + raise ValueError("Schema is not of AVRO record type or missing 'fields'") + + field_names = [] + for field in schema_json["fields"]: + name = field.get("name") + if not name: + logger.warning(f"Skipping unnamed field in AVRO schema: {field}") + continue + field_names.append(name) + + if not field_names: + raise ValueError("No fields extracted from AVRO schema") + + return field_names + except Exception as e: + raise ErrorHandler.schema_error("parse", "glue_avro", e) + + def _build_minimal_etl_schema(self, field_names: List[str]) -> ETLSchema: + """ + Build a minimal ETL schema from the extracted field names. + + This method creates a basic ETL schema with default configurations for + extraction, transformation, and loading, using the field names extracted + from the AVRO schema. + + Args: + field_names (List[str]): The list of field names to include in the schema. + + Returns: + ETLSchema: A complete ETL schema object with the specified fields. + """ + return ETLSchema( + schema_id=self.get_schema_id(), + description=f"Autogenerated from Glue schema {self.registry_name}/{self.schema_name}", + extract=ExtractConfig(source_type="s3"), + transform=TransformConfig( + chunking=ChunkingConfig(strategy="fixed_length"), + metadata_mapping=MetadataMapping(), + entity_extraction=EntityExtractionConfig(method="ner"), + normalize=NormalizeConfig() + ), + load=LoadConfig( + document_node=NodeDefinition(type="Document", fields=field_names), + section_node=NodeDefinition(type="Section", fields=[]), + relationships=[ + RelationshipDefinition(type="contains", source="document_id", target="section_id") + ] + ) + ) diff --git a/document-graph/src/document_graph/schema/providers/json_schema_provider.py b/document-graph/src/document_graph/schema/providers/json_schema_provider.py new file mode 100644 index 00000000..974eeecc --- /dev/null +++ b/document-graph/src/document_graph/schema/providers/json_schema_provider.py @@ -0,0 +1,160 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""JSON Schema Provider Module for Document Graph Operations. + +This module provides a schema provider for JSON (JavaScript Object Notation) files. +It discovers and generates ETL schemas from JSON files for processing and loading +the data into a document graph. + +The module includes the following components: +- JSONSchemaProvider: Schema provider for JSON files + +The JSON schema provider uses the JSONSchemaDiscoveryProvider to analyze JSON files +and generate appropriate ETL schemas. It supports loading schemas from JSON files +and optionally saving the discovered schemas to JSON files. + +Usage: + # Get a JSON schema provider for a specific file + from document_graph.schema.providers.schema_provider_factory import get_schema_provider + + config = { + "provider_type": "json", + "schema_id": "my_json_schema", + "connection_config": { + "path": "/path/to/data.json" + } + } + + provider = get_schema_provider(config) + + # Load the schema + schema = provider.pipeline_schema() + + # Use the schema for ETL operations + print(f"Loaded schema ID: {schema.schema_id}") + print(f"Fields: {schema.load.document_node.fields}") +""" + +import logging +import json +from pathlib import Path +from typing import Optional, Any, Dict + +# Import custom logging to ensure configuration is available + +from document_graph.schema.etl_schema_model import ETLSchema +from document_graph.schema.providers.schema_provider_base import SchemaProviderBase +from document_graph.schema.providers.schema_provider_config import SchemaProviderConfig +from document_graph.schema.discovery.json_discovery_provider import JSONSchemaDiscoveryProvider + +logger = logging.getLogger(__name__) + + +class JSONSchemaProvider(SchemaProviderBase): + """ + Schema provider for JSON (JavaScript Object Notation) files. + + This class provides functionality to discover and generate ETL schemas from JSON files. + It uses the JSONSchemaDiscoveryProvider to analyze JSON files and extract field + information to create appropriate ETL schemas for processing and loading the data into + a document graph. + + The provider supports loading schemas directly from JSON files and optionally saving + the discovered schemas to JSON files for later use. It handles both array-based JSON + files (containing multiple records) and single-object JSON files. + + Attributes: + config (SchemaProviderConfig): Configuration for the provider. + json_path (Path): The path to the JSON file to analyze. + """ + + def __init__(self, config: SchemaProviderConfig): + """ + Initialize a JSON schema provider with the given configuration. + + Args: + config (SchemaProviderConfig): Configuration for the provider, including + the path to the JSON file in the connection_config. + + Raises: + ValueError: If the path is not provided in the connection_config. + FileNotFoundError: If the specified JSON file does not exist. + """ + self.config = config + path_str = config.connection_config.get("path") + if not path_str: + raise ValueError("Validation error") + self.json_path = Path(path_str) + + if not self.json_path.exists(): + raise FileNotFoundError(f"File not found") + + @classmethod + def from_config(cls, config: SchemaProviderConfig) -> "JSONSchemaProvider": + """ + Factory method to construct a JSON provider from a configuration. + + Args: + config (SchemaProviderConfig): Configuration for the provider. + + Returns: + JSONSchemaProvider: A new instance of the JSON schema provider. + """ + return cls(config) + + def load_schema(self, source=None, **kwargs) -> ETLSchema: + """ + Load (or generate) an ETL schema from a JSON file using schema discovery. + + This method uses the JSONSchemaDiscoveryProvider to analyze the JSON file and + generate an appropriate ETL schema based on the structure and content of the file. + It handles both array-based JSON files and single-object JSON files. + + Args: + source: Optional source override (not used in this provider). + **kwargs: Additional provider-specific parameters (not used in this provider). + + Returns: + ETLSchema: A complete ETL schema object that describes the structure + of the JSON file and how it should be processed. + + Raises: + FileNotFoundError: If the JSON file does not exist. + ValueError: If the JSON file cannot be parsed or has no discoverable fields. + Exception: Any exception raised during JSON parsing. + """ + discovery = JSONSchemaDiscoveryProvider(source=self.json_path) + return discovery.discover_schema() + + def save_schema(self, output_path: Path) -> None: + """ + Save the inferred ETL schema to a JSON file at the given output path. + + This method discovers the schema from the JSON file and saves it to a JSON file + at the specified path. The schema can then be loaded directly using a FileSchemaProvider. + + Args: + output_path (Path): The path where the schema JSON file should be saved. + + Raises: + IOError: If the schema cannot be saved to the output path. + """ + schema = self.load_schema() + try: + with output_path.open("w", encoding="utf-8") as f: + json.dump(schema.model_dump(mode="json", exclude_unset=True), f, indent=2) + logger.info(f"Saved JSON schema to file: {output_path}") + except Exception as e: + raise IOError(f"IO error: {output_path, e}") + + def get_schema_id(self) -> str: + """ + Return the unique identifier for the schema. + + This method returns a unique identifier for the schema, either from the + configuration or generated from the JSON file name. + + Returns: + str: A unique identifier for the schema. + """ + return self.config.schema_id or f"json-schema-{self.json_path.stem}" diff --git a/document-graph/src/document_graph/schema/providers/parquet_schema_provider.py b/document-graph/src/document_graph/schema/providers/parquet_schema_provider.py new file mode 100644 index 00000000..babe8178 --- /dev/null +++ b/document-graph/src/document_graph/schema/providers/parquet_schema_provider.py @@ -0,0 +1,159 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Parquet Schema Provider Module for Document Graph Operations. + +This module provides a schema provider for Parquet files, a columnar storage format +commonly used in big data processing. It discovers and generates ETL schemas from +Parquet files for processing and loading the data into a document graph. + +The module includes the following components: +- ParquetSchemaProvider: Schema provider for Parquet files + +The Parquet schema provider uses the ParquetSchemaDiscoveryProvider to analyze Parquet +files and generate appropriate ETL schemas. It supports loading schemas from Parquet +files and optionally saving the discovered schemas to JSON files. + +Usage: + # Get a Parquet schema provider for a specific file + from document_graph.schema.providers.schema_provider_factory import get_schema_provider + + config = { + "provider_type": "parquet", + "schema_id": "my_parquet_schema", + "connection_config": { + "path": "/path/to/data.parquet" + } + } + + provider = get_schema_provider(config) + + # Load the schema + schema = provider.pipeline_schema() + + # Use the schema for ETL operations + print(f"Loaded schema ID: {schema.schema_id}") + print(f"Fields: {schema.load.document_node.fields}") +""" + +import logging +import json +from pathlib import Path +from typing import Optional, Any, Dict + +# Import custom logging to ensure configuration is available + +from document_graph.schema.etl_schema_model import ETLSchema +from document_graph.schema.providers.schema_provider_base import SchemaProviderBase +from document_graph.schema.providers.schema_provider_config import SchemaProviderConfig +from document_graph.schema.discovery.parquet_discovery_provider import ParquetSchemaDiscoveryProvider + +logger = logging.getLogger(__name__) + + +class ParquetSchemaProvider(SchemaProviderBase): + """ + Schema provider for Parquet files. + + This class provides functionality to discover and generate ETL schemas from Parquet files. + It uses the ParquetSchemaDiscoveryProvider to analyze Parquet files and extract field + information to create appropriate ETL schemas for processing and loading the data into + a document graph. + + The provider supports loading schemas directly from Parquet files and optionally saving + the discovered schemas to JSON files for later use. + + Attributes: + config (SchemaProviderConfig): Configuration for the provider. + parquet_path (Path): The path to the Parquet file to analyze. + """ + + def __init__(self, config: SchemaProviderConfig): + """ + Initialize a Parquet schema provider with the given configuration. + + Args: + config (SchemaProviderConfig): Configuration for the provider, including + the path to the Parquet file in the connection_config. + + Raises: + ValueError: If the path is not provided in the connection_config. + FileNotFoundError: If the specified Parquet file does not exist. + """ + self.config = config + path = config.connection_config.get("path") + if not path: + raise ValueError("Validation error") + self.parquet_path = Path(path) + + if not self.parquet_path.exists(): + raise FileNotFoundError(f"File not found") + + @classmethod + def from_config(cls, config: SchemaProviderConfig) -> "ParquetSchemaProvider": + """ + Factory method to construct a Parquet provider from a configuration. + + Args: + config (SchemaProviderConfig): Configuration for the provider. + + Returns: + ParquetSchemaProvider: A new instance of the Parquet schema provider. + """ + return cls(config) + + def load_schema(self, source=None, **kwargs) -> ETLSchema: + """ + Load (or generate) an ETL schema from a Parquet file using schema discovery. + + This method uses the ParquetSchemaDiscoveryProvider to analyze the Parquet file and + generate an appropriate ETL schema based on the structure and content of the file. + + Args: + source: Optional source override (not used in this provider). + **kwargs: Additional provider-specific parameters that are passed to the + discovery provider, such as specific columns to include. + + Returns: + ETLSchema: A complete ETL schema object that describes the structure + of the Parquet file and how it should be processed. + + Raises: + FileNotFoundError: If the Parquet file does not exist. + ValueError: If the Parquet file cannot be parsed or has no discoverable fields. + Exception: Any exception raised during Parquet parsing. + """ + discovery = ParquetSchemaDiscoveryProvider(source=self.parquet_path, args=kwargs) + return discovery.discover_schema() + + def save_schema(self, output_path: Path) -> None: + """ + Save the inferred ETL schema to a JSON file at the given output path. + + This method discovers the schema from the Parquet file and saves it to a JSON file + at the specified path. The schema can then be loaded directly using a FileSchemaProvider. + + Args: + output_path (Path): The path where the schema JSON file should be saved. + + Raises: + IOError: If the schema cannot be saved to the output path. + """ + schema = self.load_schema() + try: + with output_path.open("w", encoding="utf-8") as f: + json.dump(schema.model_dump(mode="json", exclude_unset=True), f, indent=2) + logger.info(f"Saved Parquet schema to file: {output_path}") + except Exception as e: + raise IOError(f"IO error: {output_path, e}") + + def get_schema_id(self) -> str: + """ + Return the unique identifier for the schema. + + This method returns a unique identifier for the schema, either from the + configuration or generated from the Parquet file name. + + Returns: + str: A unique identifier for the schema. + """ + return self.config.schema_id or f"parquet-schema-{self.parquet_path.stem}" diff --git a/document-graph/src/document_graph/schema/providers/s3_schema_provider.py b/document-graph/src/document_graph/schema/providers/s3_schema_provider.py new file mode 100644 index 00000000..8173db70 --- /dev/null +++ b/document-graph/src/document_graph/schema/providers/s3_schema_provider.py @@ -0,0 +1,203 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""S3 Schema Provider Module for Document Graph Operations. + +This module provides a schema provider for loading and saving ETL schemas from/to Amazon S3. +It retrieves schema files from S3 buckets and can also save schemas back to S3. + +The module includes the following components: +- S3SchemaProvider: Schema provider for Amazon S3 + +The S3 schema provider loads ETL schemas directly from JSON files stored in S3 buckets +and can also save schemas back to S3. It's commonly used to store and retrieve pre-defined +or previously discovered schemas in cloud environments. + +Usage: + # Get an S3 schema provider for a specific schema file in S3 + from document_graph.schema.providers.schema_provider_factory import get_schema_provider + + config = { + "provider_type": "s3", + "schema_id": "my_s3_schema", + "connection_config": { + "bucket": "my-schema-bucket", + "key": "schemas/my-schema.json", + "region": "us-east-1" + } + } + + provider = get_schema_provider(config) + + # Load the schema + schema = provider.pipeline_schema() + + # Use the schema for ETL operations + print(f"Loaded schema ID: {schema.schema_id}") + print(f"Fields: {schema.load.document_node.fields}") + + # Save a modified schema back to S3 + provider.save_schema() +""" + +# Import custom logging to ensure configuration is available + +import json +import logging +from typing import Any, Optional + +import boto3 +from botocore.config import Config as BotoConfig + +from document_graph.schema.providers.schema_provider_base import SchemaProviderBase +from document_graph.schema.providers.schema_provider_config import SchemaProviderConfig +from document_graph.schema.etl_schema_model import ETLSchema + +logger = logging.getLogger(__name__) + + +class S3SchemaProvider(SchemaProviderBase): + """ + Schema provider for loading and saving ETL schemas from/to Amazon S3. + + This class provides functionality to load pre-defined ETL schemas from JSON files + stored in S3 buckets and save schemas back to S3. It handles AWS authentication + and S3 operations to retrieve and store schema files. + + The provider is commonly used in cloud environments to store and retrieve + pre-defined or previously discovered schemas for reuse in ETL pipelines. + + Attributes: + config (SchemaProviderConfig): Configuration for the provider. + bucket (str): The name of the S3 bucket containing the schema file. + key (str): The S3 key (path) to the schema JSON file. + region (str): The AWS region where the S3 bucket is located. + session: The boto3 session for AWS authentication. + s3: The boto3 S3 client used for S3 operations. + """ + + def __init__(self, config: SchemaProviderConfig): + """ + Initialize an S3 schema provider with the given configuration. + + Args: + config (SchemaProviderConfig): Configuration for the provider, including + the bucket and key in the connection_config. + + Raises: + ValueError: If the bucket or key is not provided in the connection_config, + or if the key does not point to a JSON file. + """ + self.config = config + conn = config.connection_config + + self.bucket = conn.get("bucket") + self.key = conn.get("key") + self.region = conn.get("region", "us-east-1") + self.session = conn.get("session") or boto3.Session() + + if not self.bucket or not self.key: + raise ValueError( + "s3.connection_config", + conn, + "bucket and key must be provided" + ) + + if not self.key.endswith(".json"): + raise ValueError( + "s3.key", + self.key, + "S3 key must point to a .json file" + ) + + self.s3 = self.session.client( + "s3", + region_name=self.region, + config=BotoConfig(retries={"max_attempts": 3}) + ) + + @classmethod + def from_config(cls, config: SchemaProviderConfig) -> "S3SchemaProvider": + """ + Factory method to construct an S3 provider from a configuration. + + Args: + config (SchemaProviderConfig): Configuration for the provider. + + Returns: + S3SchemaProvider: A new instance of the S3 schema provider. + """ + return cls(config) + + def load_schema(self, source=None, **kwargs) -> ETLSchema: + """ + Load the ETL schema from the S3 bucket and parse it into an ETLSchema object. + + This method retrieves the JSON schema file from S3, parses it, and returns + an ETLSchema object. It handles AWS authentication and S3 operations to + retrieve the schema file. + + Args: + source: Optional source override (not used in this provider). + **kwargs: Additional provider-specific parameters (not used in this provider). + + Returns: + ETLSchema: The ETL schema loaded from the S3 file. + + Raises: + FileNotFoundError: If the S3 object does not exist. + ValueError: If the S3 object cannot be parsed or is not a valid ETL schema. + Exception: If there's an error connecting to AWS or retrieving the object. + """ + s3_path = f"s3://{self.bucket}/{self.key}" + try: + response = self.s3.get_object(Bucket=self.bucket, Key=self.key) + schema_str = response["Body"].read().decode("utf-8") + schema_json = json.loads(schema_str) + logger.info(f"Successfully loaded schema from {s3_path}") + return ETLSchema(**schema_json) + except self.s3.exceptions.NoSuchKey: + raise FileNotFoundError(f"File not found") + except Exception as e: + logger.error(f"Error loading schema from {s3_path}: {e}") + raise RuntimeError(f"S3 schema load failed: {e}") from e + + def save_schema(self, output_key: Optional[str] = None) -> None: + """ + Save the ETL schema to the S3 bucket under the given key. + + This method retrieves the current schema, serializes it to JSON, and saves + it to the S3 bucket. If no output key is provided, the original key is used. + + Args: + output_key (Optional[str]): The S3 key where the schema should be saved. + If not provided, the original key is used. + + Raises: + IOError: If there's an error writing to S3. + Exception: If there's an error connecting to AWS or retrieving the schema. + """ + schema = self.load_schema() + key_to_use = output_key or self.key + s3_path = f"s3://{self.bucket}/{key_to_use}" + try: + self.s3.put_object( + Bucket=self.bucket, + Key=key_to_use, + Body=json.dumps(schema.model_dump(mode="json", exclude_unset=True), indent=2).encode("utf-8"), + ContentType="application/json" + ) + logger.info(f"Schema saved to {s3_path}") + except Exception as e: + raise IOError(f"IO error: {s3_path, e}") + + def get_schema_id(self) -> str: + """ + Return the unique identifier for the schema. + + This method returns a unique identifier for the schema, either from the + configuration or generated from the S3 key. + + Returns: + str: A unique identifier for the schema. + """ + return self.config.schema_id or self.key.split("/")[-1].replace(".json", "") diff --git a/document-graph/src/document_graph/schema/providers/schema_provider_base.py b/document-graph/src/document_graph/schema/providers/schema_provider_base.py new file mode 100644 index 00000000..52784f85 --- /dev/null +++ b/document-graph/src/document_graph/schema/providers/schema_provider_base.py @@ -0,0 +1,148 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Schema Provider Base Module for Document Graph Operations. + +This module defines the abstract base class that all schema providers must implement. +Schema providers are responsible for loading ETL schemas from various sources such as +files, databases, and cloud services. + +The module includes the following components: +- SchemaProviderBase: Abstract base class for all schema providers + +Schema providers are used to load ETL schemas that define how data should be extracted, +transformed, and loaded into a document graph. Different providers handle different +source types (e.g., files, S3, databases) and formats (e.g., YAML, JSON, CSV). + +Usage: + # Get a schema provider for a specific configuration + from document_graph.schema.providers.schema_provider_factory import get_schema_provider + + config = { + "provider_type": "file", + "schema_id": "my_schema", + "connection_config": { + "path": "/path/to/schema.yaml" + } + } + + provider = get_schema_provider(config) + + # Load the schema + schema = provider.pipeline_schema() + + # Use the schema for ETL operations + print(f"Loaded schema ID: {schema.schema_id}") +""" + +import logging +from abc import ABC, abstractmethod +from typing import Dict, Any, Type + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available +from document_graph.schema.etl_schema_model import ETLSchema + + +class SchemaProviderBase(ABC): + """ + Abstract base class for all ETL schema providers. + + This class defines the interface that all schema providers must implement. + Schema providers are responsible for loading ETL schemas from various sources + including files, databases, and cloud services. Each provider implementation + handles a specific source type or format. + + The SchemaProviderBase class enforces a consistent interface across all providers, + ensuring that they can be used interchangeably in the document graph ETL pipeline. + Concrete implementations must provide methods for loading schemas and retrieving + schema identifiers. + + Attributes: + None directly in the base class. Subclasses typically have: + config: Configuration for the provider, usually a SchemaProviderConfig instance. + schema_id: A unique identifier for the schema being provided. + """ + + @classmethod + @abstractmethod + def from_config(cls: Type["SchemaProviderBase"], config: Dict[str, Any]) -> "SchemaProviderBase": + """ + Factory method to construct a provider from a raw config dictionary. + + This method is responsible for parsing the configuration dictionary and creating + an instance of the appropriate schema provider. It validates the configuration + parameters and raises appropriate exceptions if the configuration is invalid. + + The configuration dictionary typically includes: + - provider_type: The type of provider (e.g., "file", "s3", "glue") + - schema_id: A unique identifier for the schema + - connection_config: Provider-specific connection parameters + + Args: + config: A dictionary containing configuration parameters for the provider. + The exact structure depends on the provider implementation. + + Returns: + An instance of the implementing SchemaProviderBase subclass, properly + configured according to the provided configuration. + + Raises: + ValueError: If the configuration is invalid or missing required parameters. + TypeError: If the configuration contains parameters of the wrong type. + Other exceptions may be raised by specific provider implementations. + """ + pass + + @abstractmethod + def load_schema(self, source = None, **kwargs) -> ETLSchema: + """ + Load and return the ETL schema as a Pydantic model. + + This method is responsible for loading the ETL schema from the source specified + in the provider's configuration. It reads the schema definition, validates it, + and returns it as an ETLSchema object. The schema defines how data should be + extracted, transformed, and loaded into a document graph. + + The method may perform additional processing such as: + - Validating the schema against a schema definition + - Enriching the schema with additional information + - Converting between different schema formats + + Args: + source: Optional DocumentGraphSource containing data source and registration info. + If provided, this may override the source specified in the provider's + configuration. + **kwargs: Additional provider-specific parameters that control how the schema + is loaded and processed. + + Returns: + ETLSchema: A complete ETL schema object that defines how data should be + extracted, transformed, and loaded into a document graph. + + Raises: + FileNotFoundError: If the schema source file does not exist. + ValueError: If the schema is invalid or cannot be parsed. + Other exceptions may be raised by specific provider implementations. + """ + pass + + @abstractmethod + def get_schema_id(self) -> str: + """ + Return the unique identifier for the schema. + + This method returns a unique identifier for the schema that this provider + is responsible for. The schema ID is used to reference the schema in the + document graph ETL pipeline and to identify the schema in storage systems. + + The schema ID should be unique within the context of the application and + should be consistent across multiple invocations of the provider for the + same schema source. + + Returns: + str: A unique identifier for the schema. This is typically derived from + the schema source (e.g., filename, database table) and may include + additional information such as version numbers or timestamps. + """ + pass diff --git a/document-graph/src/document_graph/schema/providers/schema_provider_config.py b/document-graph/src/document_graph/schema/providers/schema_provider_config.py new file mode 100644 index 00000000..9505cca2 --- /dev/null +++ b/document-graph/src/document_graph/schema/providers/schema_provider_config.py @@ -0,0 +1,69 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Schema Provider Config Module for Document Graph Operations. + +This module defines the configuration model for schema providers used in document graph +operations. It provides a standardized way to configure different types of schema providers +through a Pydantic model. + +The module includes the following components: +- SchemaProviderConfig: Base configuration model for all schema providers + +The SchemaProviderConfig class is used to configure schema providers with information +about the provider type, schema ID, and connection-specific parameters. This configuration +is used by the schema provider factory to instantiate the appropriate provider. + +Usage: + # Create a configuration for a file-based schema provider + from document_graph.schema.providers.schema_provider_config import SchemaProviderConfig + + config = SchemaProviderConfig( + type="file", + schema_id="my_schema", + connection_config={ + "path": "/path/to/schema.yaml" + } + ) + + # Use the configuration to create a schema provider + from document_graph.schema.providers.schema_provider_factory import get_schema_provider + + provider = get_schema_provider(config) +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + +from typing import Dict, Any, Literal, Optional +from pydantic import BaseModel, Field + +class SchemaProviderConfig(BaseModel): + """ + Base configuration for schema providers. + + This Pydantic model defines the configuration parameters for schema providers + used in document graph operations. It provides a standardized way to configure + different types of schema providers with information about the provider type, + schema ID, and connection-specific parameters. + + The SchemaProviderConfig is used by the schema provider factory to instantiate + the appropriate provider based on the specified type. Each provider type has + its own specific connection configuration requirements. + + Attributes: + type (str): The type of schema provider. Must be one of the supported types: + "file", "s3", "static", "csv", "json", "excel", "glue", "parquet", "yaml", "xml". + schema_id (Optional[str]): An optional unique identifier for the schema. + If not provided, the provider will generate one based on the source. + connection_config (Dict[str, Any]): A dictionary containing connection-specific + configuration parameters. The exact structure depends on the provider type. + For example: + - "file" provider: {"path": "/path/to/schema.yaml"} + - "s3" provider: {"bucket": "my-bucket", "key": "path/to/schema.yaml"} + """ + type: Literal["file", "s3", "static", "csv", "json", "excel", "glue", "parquet", "yaml", "xml"] = Field(..., description="Type of schema provider") + schema_id: Optional[str] = Field(None, description="Unique ID for the schema (optional override)") + connection_config: Dict[str, Any] = Field(default_factory=dict, description="Connection-specific configuration") diff --git a/document-graph/src/document_graph/schema/providers/schema_provider_config_aws_base.py b/document-graph/src/document_graph/schema/providers/schema_provider_config_aws_base.py new file mode 100644 index 00000000..a3078f63 --- /dev/null +++ b/document-graph/src/document_graph/schema/providers/schema_provider_config_aws_base.py @@ -0,0 +1,63 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Schema Provider Config AWS Base Module for Document Graph Operations. + +This module defines the base configuration model for AWS-backed schema providers +used in document graph operations. It provides a standardized way to configure +schema providers that interact with AWS services like S3 and AWS Glue. + +The module includes the following components: +- AWSSchemaProviderConfig: Base configuration model for AWS-backed schema providers + +The AWSSchemaProviderConfig class is used as a base for more specific AWS provider +configurations, providing common AWS connection parameters such as bucket, key, +region, and profile name. This configuration is extended by specific AWS provider +implementations like S3SchemaProvider and GlueSchemaProvider. + +Usage: + # Create a configuration for an S3-based schema provider + from document_graph.schema.providers.schema_provider_config import SchemaProviderConfig + + config = SchemaProviderConfig( + type="s3", + schema_id="my_s3_schema", + connection_config={ + "bucket": "my-schema-bucket", + "key": "schemas/my-schema.json", + "region": "us-west-2", + "profile_name": "my-aws-profile" + } + ) + + # Use the configuration to create a schema provider + from document_graph.schema.providers.schema_provider_factory import get_schema_provider + + provider = get_schema_provider(config) +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from pydantic import BaseModel, Field +from typing import Optional + + +class AWSSchemaProviderConfig(BaseModel): + """ + Configuration base class for AWS-backed schema providers (e.g., S3, Glue). + + Attributes: + bucket (str): S3 bucket name where the schema is stored. + key (str): S3 key (file path) to the schema JSON file. + region (Optional[str]): AWS region of the S3 bucket (default: inferred from environment or session). + profile_name (Optional[str]): Named AWS CLI profile to use for credentials (optional). + """ + + bucket: str = Field(..., description="S3 bucket name containing the schema") + key: str = Field(..., description="S3 key (path) to the schema JSON file") + region: Optional[str] = Field(None, description="AWS region for the S3 bucket") + profile_name: Optional[str] = Field(None, description="AWS CLI profile to use for credentials (optional)") diff --git a/document-graph/src/document_graph/schema/providers/schema_provider_factory.py b/document-graph/src/document_graph/schema/providers/schema_provider_factory.py new file mode 100644 index 00000000..9bad499a --- /dev/null +++ b/document-graph/src/document_graph/schema/providers/schema_provider_factory.py @@ -0,0 +1,193 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Schema Provider Factory Module for Document Graph Operations. + +This module provides a factory for creating schema providers based on configuration +parameters. It centralizes the instantiation of different schema provider types and +ensures that the appropriate provider is created based on the specified type. + +The module includes the following components: +- SchemaProviderFactory: Factory class for creating schema providers + +The SchemaProviderFactory maintains a registry of available schema provider types +and provides methods for registering new provider types and creating provider instances. +It supports various provider types including file-based, S3-based, and database-based +providers. + +Usage: + # Create a schema provider using a configuration dictionary + from document_graph.schema.providers.schema_provider_factory import SchemaProviderFactory + + config = { + "type": "file", + "schema_id": "my_schema", + "connection_config": { + "path": "/path/to/schema.yaml" + } + } + + provider = SchemaProviderFactory.create(config) + + # Or use the convenience function + from document_graph.schema.providers.schema_provider_factory import get_schema_provider + + provider = get_schema_provider(config) + + # Load the schema + schema = provider.pipeline_schema() +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from typing import Dict, Any, Type +from document_graph.schema.providers.schema_provider_base import SchemaProviderBase + +from document_graph.schema.providers.file_schema_provider import FileSchemaProvider +from document_graph.schema.providers.s3_schema_provider import S3SchemaProvider +from document_graph.schema.providers.csv_schema_provider import CSVSchemaProvider +from document_graph.schema.providers.glue_schema_provider import GlueSchemaProvider +from document_graph.schema.providers.parquet_schema_provider import ParquetSchemaProvider +from document_graph.schema.providers.json_schema_provider import JSONSchemaProvider +from document_graph.schema.providers.excel_schema_provider import ExcelSchemaProvider +from document_graph.schema.providers.yaml_schema_provider import YAMLSchemaProvider +from document_graph.schema.providers.xml_schema_provider import XMLSchemaProvider + + +class SchemaProviderFactory: + """ + Factory class to instantiate schema providers based on a configuration dictionary. + + This class provides a centralized way to create schema provider instances based on + a configuration dictionary. It maintains a registry of available provider types and + maps them to their corresponding provider classes. The factory ensures that the + appropriate provider is created based on the specified type in the configuration. + + The factory supports various provider types including: + - file: For loading schemas from local files + - s3: For loading schemas from Amazon S3 + - static: For using predefined schemas + - csv: For generating schemas from CSV files + - json: For generating schemas from JSON files + - excel: For generating schemas from Excel files + - glue: For loading schemas from AWS Glue + - parquet: For generating schemas from Parquet files + - yaml: For generating schemas from YAML files + - xml: For generating schemas from XML files + + New provider types can be registered using the register_provider method. + + Attributes: + _registry (Dict[str, Type[SchemaProviderBase]]): A mapping of provider type names + to their corresponding provider classes. + """ + + _registry: Dict[str, Type[SchemaProviderBase]] = { + "file": FileSchemaProvider, + "s3": S3SchemaProvider, + "csv": CSVSchemaProvider, + "json": JSONSchemaProvider, + "excel": ExcelSchemaProvider, + "glue": GlueSchemaProvider, + "parquet": ParquetSchemaProvider, + "yaml": YAMLSchemaProvider, + "xml": XMLSchemaProvider, + } + + @classmethod + def _get_static_provider(cls): + """Late import for StaticSchemaProvider to avoid circular imports.""" + from document_graph.schema.static_schema_provider import StaticSchemaProvider + return StaticSchemaProvider + + @classmethod + def register_provider(cls, type_name: str, provider_class: Type[SchemaProviderBase]) -> None: + """ + Register a new schema provider class. + + This method allows for the registration of custom schema provider classes + that are not included in the default registry. Once registered, the provider + can be created using the factory's create method by specifying the registered + type name in the configuration. + + Args: + type_name: A unique string identifier for the provider type. This will be + used as the "type" value in configuration dictionaries to specify + that this provider should be used. + provider_class: The schema provider class to register. This must be a subclass + of SchemaProviderBase and implement the required interface, + including the from_config class method. + + Returns: + None + + Example: + # Register a custom provider + from my_package import MyCustomProvider + SchemaProviderFactory.register_provider("custom", MyCustomProvider) + + # Use the custom provider + config = {"type": "custom", "schema_id": "my_schema", "connection_config": {...}} + provider = SchemaProviderFactory.create(config) + """ + cls._registry[type_name] = provider_class + + @classmethod + def create(cls, config: Dict[str, Any]) -> SchemaProviderBase: + """ + Instantiate a schema provider based on the `type` in the config. + + This method creates an instance of the appropriate schema provider based on + the "type" specified in the configuration dictionary. It validates that the + type is registered and that the provider class implements the required interface. + + The method performs the following steps: + 1. Extracts the "type" from the configuration + 2. Validates that the type is registered in the factory + 3. Gets the provider class from the registry + 4. Validates that the provider class implements the required interface + 5. Converts the configuration dictionary to a SchemaProviderConfig object + 6. Creates and returns an instance of the provider using the from_config method + + Args: + config: A dictionary containing the configuration for the schema provider. + Must include a "type" key with a value that is registered in the factory. + Other required keys depend on the specific provider type. + + Returns: + An instance of a SchemaProviderBase subclass, configured according to the + provided configuration. + + Raises: + ValueError: If the "type" is missing or not registered in the factory. + TypeError: If the provider class does not implement the required interface. + Other exceptions may be raised by the provider's from_config method. + """ + type_name = config.get("type") + + # Handle static provider with late import to avoid circular dependency + if type_name == "static": + provider_class = cls._get_static_provider() + elif type_name and type_name in cls._registry: + provider_class = cls._registry[type_name] + else: + available_types = list(cls._registry.keys()) + ["static"] + raise ErrorHandler.validation_error( + "schema_provider_type", + type_name or "None", + f"one of {available_types}" + ) + + # provider_class is already set above + + if not hasattr(provider_class, "from_config"): + raise TypeError(f"{provider_class.__name__} is missing required `from_config(config)` method.") + + # Convert dict to SchemaProviderConfig object + from document_graph.schema.providers.schema_provider_config import SchemaProviderConfig + config_obj = SchemaProviderConfig(**config) + return provider_class.from_config(config_obj) diff --git a/document-graph/src/document_graph/schema/providers/schema_provider_registry.py b/document-graph/src/document_graph/schema/providers/schema_provider_registry.py new file mode 100644 index 00000000..3bca1cca --- /dev/null +++ b/document-graph/src/document_graph/schema/providers/schema_provider_registry.py @@ -0,0 +1,169 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Schema Provider Registry Module for Document Graph Operations. + +This module provides a registry system for schema providers, allowing for runtime +registration and lookup of provider classes by name. It centralizes the management +of available schema provider types and ensures that providers can be dynamically +added and retrieved. + +The module includes the following components: +- SchemaProviderRegistry: Registry class for managing schema provider classes + +The SchemaProviderRegistry maintains a dictionary of provider names to provider classes +and provides methods for registering new providers, retrieving providers by name, and +listing all available providers. This registry is used by the schema provider factory +to look up provider classes based on configuration. + +Usage: + # Create a registry and register providers + from document_graph.schema.providers.schema_provider_registry import SchemaProviderRegistry + from document_graph.schema.providers.file_schema_provider import FileSchemaProvider + from document_graph.schema.providers.s3_schema_provider import S3SchemaProvider + + registry = SchemaProviderRegistry() + registry.register("file", FileSchemaProvider) + registry.register("s3", S3SchemaProvider) + + # Look up a provider by name + provider_class = registry.get("file") + + # List all registered providers + available_providers = registry.list() +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from typing import Dict, Type +from document_graph.schema.providers.schema_provider_base import SchemaProviderBase + + +class SchemaProviderRegistry: + """ + Central registry for all schema provider classes. + + This class provides a registry system for schema provider classes, allowing for + runtime registration and lookup of providers by name. It centralizes the management + of available schema provider types and ensures that providers can be dynamically + added and retrieved. + + The registry maintains a dictionary mapping provider names to provider classes and + provides methods for registering new providers, retrieving providers by name, and + listing all available providers. This registry is used by the schema provider factory + to look up provider classes based on configuration. + + Attributes: + _registry (Dict[str, Type[SchemaProviderBase]]): A dictionary mapping provider + names to provider classes. + """ + + def __init__(self): + """ + [Brief one-line description of the function's purpose] + + [Detailed explanation of what the function does and how it works] + + Returns: + [return_type]: [Description of return value] + """ + self._registry: Dict[str, Type[SchemaProviderBase]] = {} + + def register(self, name: str, provider_cls: Type[SchemaProviderBase], overwrite: bool = False) -> None: + """ + Register a new schema provider class. + + This method adds a new schema provider class to the registry, associating it + with the specified name. The provider class must be a subclass of SchemaProviderBase. + By default, attempting to register a provider with a name that already exists + in the registry will raise an error, but this behavior can be overridden with + the overwrite parameter. + + Args: + name: Provider type name (e.g., 'csv', 's3'). This is the identifier that + will be used to look up the provider class. + provider_cls: Provider implementation class. Must be a subclass of + SchemaProviderBase and implement the required interface. + overwrite: Allow replacing an existing provider (default: False). If True, + an existing provider with the same name will be replaced. + + Raises: + ValueError: If a provider with the same name already exists and overwrite + is False. + + Example: + # Register a custom provider + registry = SchemaProviderRegistry() + registry.register("custom", MyCustomProvider) + """ + if name in self._registry and not overwrite: + raise ErrorHandler.validation_error( + "schema_provider_registration", + name, + "a unique provider name not already registered" + ) + self._registry[name] = provider_cls + + def get(self, name: str) -> Type[SchemaProviderBase]: + """ + Retrieve a registered schema provider class by name. + + This method looks up a schema provider class in the registry by its registered + name. If the name is not found in the registry, an error is raised with a list + of available provider names. + + Args: + name: The name of the provider to retrieve. This must be a name that has + been previously registered with the register method. + + Returns: + Type[SchemaProviderBase]: The schema provider class associated with the + specified name. + + Raises: + ValueError: If the specified name is not found in the registry. + + Example: + # Get a provider class by name + registry = SchemaProviderRegistry() + provider_class = registry.get("file") + + # Create an instance of the provider + provider = provider_class.from_config(config) + """ + if name not in self._registry: + available = list(self._registry.keys()) + raise ErrorHandler.validation_error( + "schema_provider_lookup", + name, + f"one of: {available}" + ) + return self._registry[name] + + def list(self) -> Dict[str, Type[SchemaProviderBase]]: + """ + Return a copy of all registered providers. + + This method returns a dictionary containing all the schema provider classes + currently registered in the registry. The dictionary maps provider names to + their corresponding provider classes. A copy of the internal registry is + returned to prevent modification of the registry through the returned dictionary. + + Returns: + Dict[str, Type[SchemaProviderBase]]: A dictionary mapping provider names + to their corresponding provider classes. + + Example: + # List all registered providers + registry = SchemaProviderRegistry() + providers = registry.list() + + # Print the names of all registered providers + for name in providers.keys(): + print(f"Provider: {name}") + """ + return self._registry.copy() diff --git a/document-graph/src/document_graph/schema/providers/xml_schema_provider.py b/document-graph/src/document_graph/schema/providers/xml_schema_provider.py new file mode 100644 index 00000000..f0ec67b4 --- /dev/null +++ b/document-graph/src/document_graph/schema/providers/xml_schema_provider.py @@ -0,0 +1,156 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""XML Schema Provider Module for Document Graph Operations. + +This module provides a schema provider for XML (eXtensible Markup Language) files. +It discovers and generates ETL schemas from XML files for processing and loading +the data into a document graph. + +The module includes the following components: +- XMLSchemaProvider: Schema provider for XML files + +The XML schema provider uses the XMLSchemaDiscoveryProvider to analyze XML files +and generate appropriate ETL schemas. It supports loading schemas from XML files +and optionally saving the discovered schemas to JSON files. + +Usage: + # Get an XML schema provider for a specific file + from document_graph.schema.providers.schema_provider_factory import get_schema_provider + + config = { + "provider_type": "xml", + "schema_id": "my_xml_schema", + "connection_config": { + "path": "/path/to/data.xml" + } + } + + provider = get_schema_provider(config) + + # Load the schema + schema = provider.pipeline_schema() + + # Use the schema for ETL operations + print(f"Loaded schema ID: {schema.schema_id}") + print(f"Fields: {schema.load.document_node.fields}") +""" + +import logging +import json +from pathlib import Path +from typing import Optional, Any + +# Import custom logging to ensure configuration is available + +from document_graph.schema.etl_schema_model import ETLSchema +from document_graph.schema.providers.schema_provider_base import SchemaProviderBase +from document_graph.schema.providers.schema_provider_config import SchemaProviderConfig +from document_graph.schema.discovery.xml_discovery_provider import XMLSchemaDiscoveryProvider + +logger = logging.getLogger(__name__) + + +class XMLSchemaProvider(SchemaProviderBase): + """ + Schema provider for XML (eXtensible Markup Language) files. + + This class provides functionality to discover and generate ETL schemas from XML files. + It uses the XMLSchemaDiscoveryProvider to analyze XML files and extract field information + to create appropriate ETL schemas for processing and loading the data into a document graph. + + The provider supports loading schemas directly from XML files and optionally saving + the discovered schemas to JSON files for later use. + + Attributes: + config (SchemaProviderConfig): Configuration for the provider. + xml_path (Path): The path to the XML file to analyze. + """ + + def __init__(self, config: SchemaProviderConfig): + """ + Initialize an XML schema provider with the given configuration. + + Args: + config (SchemaProviderConfig): Configuration for the provider, including + the path to the XML file in the connection_config. + + Raises: + ValueError: If the path is not provided in the connection_config. + FileNotFoundError: If the specified XML file does not exist. + """ + self.config = config + path = config.connection_config.get("path") + if not path: + raise ValueError("Validation error") + self.xml_path = Path(path) + + if not self.xml_path.exists(): + raise FileNotFoundError(f"File not found") + + @classmethod + def from_config(cls, config: SchemaProviderConfig) -> "XMLSchemaProvider": + """ + Factory method to construct an XML provider from a configuration. + + Args: + config (SchemaProviderConfig): Configuration for the provider. + + Returns: + XMLSchemaProvider: A new instance of the XML schema provider. + """ + return cls(config) + + def load_schema(self, source=None, **kwargs) -> ETLSchema: + """ + Load (or generate) an ETL schema from an XML file using schema discovery. + + This method uses the XMLSchemaDiscoveryProvider to analyze the XML file and + generate an appropriate ETL schema based on the structure and content of the file. + + Args: + source: Optional source override (not used in this provider). + **kwargs: Additional provider-specific parameters (not used in this provider). + + Returns: + ETLSchema: A complete ETL schema object that describes the structure + of the XML file and how it should be processed. + + Raises: + FileNotFoundError: If the XML file does not exist. + ValueError: If the XML file cannot be parsed or has no discoverable fields. + """ + discovery = XMLSchemaDiscoveryProvider(source=self.xml_path) + return discovery.discover_schema() + + def save_schema(self, output_path: Path) -> None: + """ + Save the inferred ETL schema to a JSON file at the given output path. + + This method discovers the schema from the XML file and saves it to a JSON file + at the specified path. The schema can then be loaded directly using a FileSchemaProvider. + + Args: + output_path (Path): The path where the schema JSON file should be saved. + + Raises: + IOError: If the schema cannot be saved to the output path. + """ + schema = self.load_schema() + try: + with output_path.open("w", encoding="utf-8") as f: + json.dump(schema.model_dump(mode="json", exclude_unset=True), f, indent=2) + logger.info(f"Saved XML schema to file: {output_path}") + except Exception as e: + raise IOError(f"IO error: {output_path, e}") + + def get_schema_id(self) -> str: + """ + Return the unique identifier for the schema. + + This method returns a unique identifier for the schema, either from the + configuration or generated from the XML file name. + + Returns: + str: A unique identifier for the schema. + """ + return self.config.schema_id or f"xml-schema-{self.xml_path.stem}" \ No newline at end of file diff --git a/document-graph/src/document_graph/schema/providers/yaml_schema_provider.py b/document-graph/src/document_graph/schema/providers/yaml_schema_provider.py new file mode 100644 index 00000000..c23c785a --- /dev/null +++ b/document-graph/src/document_graph/schema/providers/yaml_schema_provider.py @@ -0,0 +1,157 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""YAML Schema Provider Module for Document Graph Operations. + +This module provides a schema provider for YAML (YAML Ain't Markup Language) files. +It discovers and generates ETL schemas from YAML files for processing and loading +the data into a document graph. + +The module includes the following components: +- YAMLSchemaProvider: Schema provider for YAML files + +The YAML schema provider uses the YAMLSchemaDiscoveryProvider to analyze YAML files +and generate appropriate ETL schemas. It supports loading schemas from YAML files +and optionally saving the discovered schemas to JSON files. + +Usage: + # Get a YAML schema provider for a specific file + from document_graph.schema.providers.schema_provider_factory import get_schema_provider + + config = { + "provider_type": "yaml", + "schema_id": "my_yaml_schema", + "connection_config": { + "path": "/path/to/data.yaml" + } + } + + provider = get_schema_provider(config) + + # Load the schema + schema = provider.pipeline_schema() + + # Use the schema for ETL operations + print(f"Loaded schema ID: {schema.schema_id}") + print(f"Fields: {schema.load.document_node.fields}") +""" + +import logging +import json +from pathlib import Path +from typing import Optional, Any + +# Import custom logging to ensure configuration is available + +from document_graph.schema.etl_schema_model import ETLSchema +from document_graph.schema.providers.schema_provider_base import SchemaProviderBase +from document_graph.schema.providers.schema_provider_config import SchemaProviderConfig +from document_graph.schema.discovery.yaml_discovery_provider import YAMLSchemaDiscoveryProvider + +logger = logging.getLogger(__name__) + + +class YAMLSchemaProvider(SchemaProviderBase): + """ + Schema provider for YAML (YAML Ain't Markup Language) files. + + This class provides functionality to discover and generate ETL schemas from YAML files. + It uses the YAMLSchemaDiscoveryProvider to analyze YAML files and extract field information + to create appropriate ETL schemas for processing and loading the data into a document graph. + + The provider supports loading schemas directly from YAML files and optionally saving + the discovered schemas to JSON files for later use. + + Attributes: + config (SchemaProviderConfig): Configuration for the provider. + yaml_path (Path): The path to the YAML file to analyze. + """ + + def __init__(self, config: SchemaProviderConfig): + """ + Initialize a YAML schema provider with the given configuration. + + Args: + config (SchemaProviderConfig): Configuration for the provider, including + the path to the YAML file in the connection_config. + + Raises: + ValueError: If the path is not provided in the connection_config. + FileNotFoundError: If the specified YAML file does not exist. + """ + self.config = config + path = config.connection_config.get("path") + if not path: + raise ValueError("Validation error") + self.yaml_path = Path(path) + + if not self.yaml_path.exists(): + raise FileNotFoundError(f"File not found") + + @classmethod + def from_config(cls, config: SchemaProviderConfig) -> "YAMLSchemaProvider": + """ + Factory method to construct a YAML provider from a configuration. + + Args: + config (SchemaProviderConfig): Configuration for the provider. + + Returns: + YAMLSchemaProvider: A new instance of the YAML schema provider. + """ + return cls(config) + + def load_schema(self, source=None, **kwargs) -> ETLSchema: + """ + Load (or generate) an ETL schema from a YAML file using schema discovery. + + This method uses the YAMLSchemaDiscoveryProvider to analyze the YAML file and + generate an appropriate ETL schema based on the structure and content of the file. + + Args: + source: Optional source override (not used in this provider). + **kwargs: Additional provider-specific parameters (not used in this provider). + + Returns: + ETLSchema: A complete ETL schema object that describes the structure + of the YAML file and how it should be processed. + + Raises: + FileNotFoundError: If the YAML file does not exist. + ValueError: If the YAML file cannot be parsed or has no discoverable fields. + ImportError: If PyYAML is not installed. + """ + discovery = YAMLSchemaDiscoveryProvider(source=self.yaml_path) + return discovery.discover_schema() + + def save_schema(self, output_path: Path) -> None: + """ + Save the inferred ETL schema to a JSON file at the given output path. + + This method discovers the schema from the YAML file and saves it to a JSON file + at the specified path. The schema can then be loaded directly using a FileSchemaProvider. + + Args: + output_path (Path): The path where the schema JSON file should be saved. + + Raises: + IOError: If the schema cannot be saved to the output path. + """ + schema = self.load_schema() + try: + with output_path.open("w", encoding="utf-8") as f: + json.dump(schema.model_dump(mode="json", exclude_unset=True), f, indent=2) + logger.info(f"Saved YAML schema to file: {output_path}") + except Exception as e: + raise IOError(f"IO error: {output_path, e}") + + def get_schema_id(self) -> str: + """ + Return the unique identifier for the schema. + + This method returns a unique identifier for the schema, either from the + configuration or generated from the YAML file name. + + Returns: + str: A unique identifier for the schema. + """ + return self.config.schema_id or f"yaml-schema-{self.yaml_path.stem}" \ No newline at end of file diff --git a/document-graph/src/document_graph/schema/schema_io.py b/document-graph/src/document_graph/schema/schema_io.py new file mode 100644 index 00000000..687a517d --- /dev/null +++ b/document-graph/src/document_graph/schema/schema_io.py @@ -0,0 +1,112 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Schema IO Module for Document Graph Operations. + +This module provides utility functions for saving and loading ETL schemas to and from +various storage formats. It handles serialization and deserialization of ETL schema +objects, making it easy to persist schemas and share them between different components +of the document graph system. + +The module includes functions for: +- Saving ETL schemas to JSON files or file-like objects +- Loading ETL schemas from JSON files or file-like objects + +These functions handle the conversion between ETLSchema objects and their JSON +representations, ensuring that all schema components are properly serialized and +deserialized. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +import json +from pathlib import Path +from typing import Union, IO +from document_graph.schema.etl_schema_model import ETLSchema + +def save_schema(schema: ETLSchema, output: Union[str, Path, IO], indent: int = 2) -> None: + """ + Save an ETL schema to a file or file-like object in JSON format. + + This function serializes an ETLSchema object to JSON and writes it to the specified + output. The output can be a file path (as a string or Path object) or a file-like + object that supports the write method. + + Args: + schema (ETLSchema): The ETL schema to save. + output (Union[str, Path, IO]): The output destination, which can be a file path + (as a string or Path object) or a file-like object. + indent (int, optional): The indentation level for the JSON output. Defaults to 2. + + Raises: + IOError: If there's an error writing to the output. + + Example: + ```python + from document_graph.schema.etl_schema_model import ETLSchema + from document_graph.schema.schema_io import save_schema + + # Create a schema + schema = ETLSchema(...) + + # Save to a file + save_schema(schema, "path/to/schema.json") + + # Or save to a file-like object + with open("path/to/schema.json", "w") as f: + save_schema(schema, f) + ``` + """ + try: + if isinstance(output, (str, Path)): + with Path(output).open("w", encoding="utf-8") as f: + json.dump(schema.model_dump(mode="json", exclude_unset=True), f, indent=indent) + else: + json.dump(schema.model_dump(mode="json", exclude_unset=True), output, indent=indent) + except Exception as e: + raise IOError(f"IO error: {str(output), e}") + + +def load_schema(input_: Union[str, Path, IO]) -> ETLSchema: + """ + Load an ETL schema from a file or file-like object. + + This function deserializes an ETLSchema object from JSON read from the specified + input. The input can be a file path (as a string or Path object) or a file-like + object that supports the read method. + + Args: + input_ (Union[str, Path, IO]): The input source, which can be a file path + (as a string or Path object) or a file-like object. + + Returns: + ETLSchema: The deserialized ETL schema. + + Raises: + IOError: If there's an error reading from the input. + ValueError: If the input contains invalid JSON or the JSON doesn't represent a valid ETLSchema. + + Example: + ```python + from document_graph.schema.schema_io import pipeline_schema + + # Load from a file + schema = pipeline_schema("path/to/schema.json") + + # Or load from a file-like object + with open("path/to/schema.json", "r") as f: + schema = pipeline_schema(f) + ``` + """ + try: + if isinstance(input_, (str, Path)): + with Path(input_).open("r", encoding="utf-8") as f: + return ETLSchema(**json.load(f)) + else: + return ETLSchema(**json.load(input_)) + except Exception as e: + raise IOError(f"IO error: {str(input_), e}") diff --git a/document-graph/src/document_graph/schema/static_schema_provider.py b/document-graph/src/document_graph/schema/static_schema_provider.py new file mode 100644 index 00000000..d0f500fc --- /dev/null +++ b/document-graph/src/document_graph/schema/static_schema_provider.py @@ -0,0 +1,134 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Static Schema Provider Module for Document Graph Operations. + +This module provides a schema provider that returns a hardcoded, static ETL schema. +It's primarily used for testing, development, or as a fallback when no other schema +providers are available. + +The static schema provider implements the SchemaProviderBase interface, making it +compatible with the rest of the document graph system. It provides a default schema +that includes configurations for extracting from S3, transforming with standard +operations, and loading into a graph with document and section nodes. + +This provider is useful for: +- Testing document graph operations without setting up external schema sources +- Providing a fallback schema when user-defined schemas are not available +- Demonstrating the structure of a complete ETL schema +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from typing import Dict, Any +from document_graph.schema.providers.schema_provider_base import SchemaProviderBase +from document_graph.schema.etl_schema_model import ETLSchema, ExtractConfig, TransformConfig, LoadConfig, ChunkingConfig, MetadataMapping, EntityExtractionConfig, NormalizeConfig, NodeDefinition, RelationshipDefinition + + +class StaticSchemaProvider(SchemaProviderBase): + """ + A schema provider that returns a hardcoded, static ETL schema. + + This class implements the SchemaProviderBase interface to provide a default + ETL schema without requiring any external configuration sources. It initializes + with a complete, hardcoded schema that includes all necessary components for + the ETL process. + + The static schema includes: + - Extract configuration for S3 PDF documents + - Transform configuration with chunking, metadata mapping, entity extraction, and normalization + - Load configuration with document and section nodes and their relationships + + This provider is particularly useful for: + - Testing and development environments + - Providing a fallback when no user-defined schema is available + - Demonstrating a complete schema structure + + Attributes: + _schema (ETLSchema): The hardcoded ETL schema instance. + """ + + def __init__(self, config: Dict[str, Any]): + """ + Initialize a static schema provider with a hardcoded ETL schema. + + This constructor creates a complete ETL schema with predefined settings for + all components. The config parameter is accepted to maintain a consistent + interface with other schema providers, but it is not used in this implementation. + + Args: + config (Dict[str, Any]): Configuration dictionary (unused in this provider). + Included for interface consistency with other providers. + """ + # Config is unused for static schema, but accepted to keep constructor consistent + self._schema = ETLSchema( + schema_id="static-default", + description="Default hardcoded ETL schema", + extract=ExtractConfig( + source_type="s3", + bucket="default-bucket", + prefix="docs/", + file_type="pdf", + reader="pymupdf" + ), + transform=TransformConfig( + chunking=ChunkingConfig(strategy="by_heading", min_length=100), + metadata_mapping=MetadataMapping( + title="document.title", + author="metadata.author" + ), + entity_extraction=EntityExtractionConfig(method="ner", model="spacy_en_core_web_lg"), + normalize=NormalizeConfig(remove_headers=True) + ), + load=LoadConfig( + document_node=NodeDefinition( + type="DocumentNode", + fields=["title", "document_type", "metadata"] + ), + section_node=NodeDefinition( + type="SectionNode", + fields=["title", "level", "text_units"] + ), + relationships=[ + RelationshipDefinition( + type="has_section", + source="document_id", + target="section_id" + ) + ] + ) + ) + + @classmethod + def from_config(cls, config): + """Factory method — StaticSchemaProvider ignores config.""" + return cls(config if isinstance(config, dict) else {}) + + def load_schema(self) -> ETLSchema: + """ + Load and return the static ETL schema. + + This method implements the SchemaProviderBase interface's pipeline_schema method. + Unlike other schema providers that might load from external sources, this method + simply returns the hardcoded schema that was created during initialization. + + Returns: + ETLSchema: The static, hardcoded ETL schema. + """ + return self._schema + + def get_schema_id(self) -> str: + """ + Get the unique identifier for the static schema. + + This method implements the SchemaProviderBase interface's get_schema_id method. + It returns the schema_id property from the hardcoded schema. + + Returns: + str: The unique identifier for the schema (e.g., "static-default"). + """ + return self._schema.schema_id diff --git a/document-graph/src/document_graph/transform/__init__.py b/document-graph/src/document_graph/transform/__init__.py new file mode 100644 index 00000000..a8995c45 --- /dev/null +++ b/document-graph/src/document_graph/transform/__init__.py @@ -0,0 +1 @@ +"""Transform stage — rows → typed Nodes/Edges.""" diff --git a/document-graph/src/document_graph/transform/document_transformers/__init__.py b/document-graph/src/document_graph/transform/document_transformers/__init__.py new file mode 100644 index 00000000..7d64abc1 --- /dev/null +++ b/document-graph/src/document_graph/transform/document_transformers/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Document-level transformers for processing and manipulating document content. + +This module provides transformers that operate on document content, such as: +- PII redaction: Removing personally identifiable information +- Text chunking: Splitting long text into manageable pieces +- JSON flattening: Converting nested JSON structures to tabular format + +These transformers can be used in document processing pipelines to prepare +content for storage, analysis, or retrieval. +""" + +from .pii_redactor_provider import PIIRedactorProvider +from .json_to_rows import JSONToRowsTransformer +from .text_chunker import TextChunkerTransformer + +__all__ = [ + 'PIIRedactorProvider', + 'JSONToRowsTransformer', + 'TextChunkerTransformer' +] \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/document_transformers/json_to_rows.py b/document-graph/src/document_graph/transform/document_transformers/json_to_rows.py new file mode 100644 index 00000000..0f6400d5 --- /dev/null +++ b/document-graph/src/document_graph/transform/document_transformers/json_to_rows.py @@ -0,0 +1,78 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""JSON to Rows transformer for flattening nested JSON structures. + +This module provides functionality to transform nested JSON data into a flat +tabular structure, making it easier to process and analyze hierarchical data +in a row-based format. It uses pandas json_normalize to flatten nested JSON +structures into rows with dot-notation column names. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from typing import List, Dict, Any +import json +import pandas as pd +from document_graph.transform.transformer_provider_base import TransformerProvider + +class JSONToRowsTransformer(TransformerProvider): + """Transforms nested JSON fields into flat tabular rows. + + This transformer takes records containing nested JSON data and converts them + into multiple flattened records, where each nested object becomes a separate row. + The nested structure is flattened using dot notation for field names. + + Configuration: + json_field (str): The field name containing the JSON data to flatten. + Defaults to 'content'. + """ + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Transform records with JSON fields into flattened rows. + + This method processes each record by: + 1. Extracting the JSON data from the specified field + 2. Parsing the JSON if it's a string + 3. Normalizing the nested structure into a flat DataFrame + 4. Creating new records for each row in the DataFrame + + If JSON parsing fails, the original record is preserved. + + Args: + records: A list of record dictionaries to transform + + Returns: + A list of transformed records, where each nested JSON object + has been converted to a separate record with flattened fields + prefixed with 'json_'. + """ + json_field = self.args.get('json_field', 'content') + output_records = [] + + for record in records: + try: + json_data = record.get(json_field) + if isinstance(json_data, str): + json_data = json.loads(json_data) + + # Normalize JSON to flat structure + df = pd.json_normalize(json_data) + + for idx, row in df.iterrows(): + new_record = record.copy() + # Add flattened fields to record + for col, val in row.items(): + new_record[f"json_{col}"] = val + new_record['row_index'] = idx + output_records.append(new_record) + + except Exception: + # Keep original record if JSON parsing fails + output_records.append(record) + + return output_records \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/document_transformers/pii_redactor_provider.py b/document-graph/src/document_graph/transform/document_transformers/pii_redactor_provider.py new file mode 100644 index 00000000..240ef7d8 --- /dev/null +++ b/document-graph/src/document_graph/transform/document_transformers/pii_redactor_provider.py @@ -0,0 +1,76 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""PII Redactor Provider for removing sensitive information from documents. + +This module provides functionality to identify and redact Personally Identifiable +Information (PII) from document content. It uses the 'sanitary' library to detect +and replace sensitive information such as IP addresses, login names, credentials, +and other PII with redaction markers. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from typing import Any, Dict, List + +try: + from sanitary import Sanitizer +except ImportError: + Sanitizer = None + +from document_graph.transform.transformer_provider_base import TransformerProvider +from document_graph.transform.transformer_provider_config import TransformerProviderConfig + + +class PIIRedactorProvider(TransformerProvider): + """Redacts PII such as IP addresses, login names, credentials from text fields. + + This transformer identifies and redacts Personally Identifiable Information (PII) + from specified fields in document records. It uses the `sanitary` library to + detect and replace sensitive information with a redaction marker. + + Configuration: + fields (List[str]): List of field names to scan for PII and redact. + If empty, no fields will be redacted. + """ + + def __init__(self, config: TransformerProviderConfig): + """Initialize the PII redactor with configuration. + + Args: + config: Configuration object containing transformer settings. + Must include an 'args' dictionary with a 'fields' list + specifying which fields to redact. + """ + if Sanitizer is None: + raise ImportError("PIIRedactorProvider requires 'sanitary' package: pip install sanitary") + super().__init__(config) + self.fields_to_redact = config.args.get("fields", []) + self.sanitizer = Sanitizer( + keys=self.fields_to_redact, + replacement="***REDACTED***" + ) + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Transform records by redacting PII from specified fields. + + This method processes each record by applying the sanitizer to + redact PII in the configured fields. The original record structure + is preserved, with only the sensitive information replaced. + + Args: + records: A list of record dictionaries to process + + Returns: + A list of records with PII redacted from the specified fields. + Each record maintains its original structure. + """ + redacted_records = [] + for record in records: + redacted = self.sanitizer.sanitize(record) + redacted_records.append(redacted) + return redacted_records diff --git a/document-graph/src/document_graph/transform/document_transformers/text_chunker.py b/document-graph/src/document_graph/transform/document_transformers/text_chunker.py new file mode 100644 index 00000000..4ebc396c --- /dev/null +++ b/document-graph/src/document_graph/transform/document_transformers/text_chunker.py @@ -0,0 +1,101 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Text Chunker module for splitting document content into manageable chunks. + +This module provides functionality to divide long text content into smaller, +overlapping chunks for more efficient processing and analysis. This is particularly +useful for large documents that need to be processed in smaller segments, such as +for embedding generation, semantic search, or other NLP operations that have +input size limitations. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from typing import List, Dict, Any +from document_graph.transform.transformer_provider_base import TransformerProvider + + +class TextChunkerTransformer(TransformerProvider): + """Splits long text content into smaller, potentially overlapping chunks. + + This transformer divides text content from records into smaller chunks + of a specified size, with optional overlap between consecutive chunks. + Each chunk becomes a separate record that maintains the original record's + metadata while adding chunk-specific information. + + Configuration: + chunk_size (int): Maximum size of each text chunk in characters. + Defaults to 512. + overlap (int): Number of characters to overlap between consecutive chunks. + Defaults to 0 (no overlap). + text_field (str): The field name containing the text to chunk. + Defaults to 'content'. + """ + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Transform records by splitting text content into smaller chunks. + + This method processes each record by: + 1. Extracting the text from the specified field + 2. Dividing it into chunks of the configured size + 3. Creating new records for each chunk with additional metadata + + Records without text content or with non-string content in the specified + field are passed through unchanged. + + Each chunk record includes the following additional fields: + - chunk_index: Sequential index of the chunk (0-based) + - chunk_start: Character position where the chunk starts in the original text + - chunk_end: Character position where the chunk ends in the original text + - original_id: The ID of the original record + + If the original record has an 'id' field, each chunk's ID will be + formatted as "{original_id}-chunk-{chunk_index}". + + Args: + records: A list of record dictionaries to transform + + Returns: + A list of transformed records, where text content has been + divided into chunks with each chunk as a separate record. + """ + chunk_size = int(self.args.get("chunk_size", 512)) + overlap = int(self.args.get("overlap", 0)) + text_field = self.args.get("text_field", "content") + + output_records = [] + + for record in records: + text = record.get(text_field, "") + if not text or not isinstance(text, str): + output_records.append(record) + continue + + start = 0 + chunk_index = 0 + + while start < len(text): + end = start + chunk_size + chunk = text[start:end] + + chunk_record = record.copy() + chunk_record[text_field] = chunk + chunk_record['chunk_index'] = chunk_index + chunk_record['chunk_start'] = start + chunk_record['chunk_end'] = min(end, len(text)) + chunk_record['original_id'] = record.get('id', '') + + if 'id' in chunk_record: + chunk_record['id'] = f"{chunk_record['id']}-chunk-{chunk_index}" + + output_records.append(chunk_record) + + chunk_index += 1 + start = end - overlap + + return output_records \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/enrichers/__init__.py b/document-graph/src/document_graph/transform/enrichers/__init__.py new file mode 100644 index 00000000..0bc2a6ef --- /dev/null +++ b/document-graph/src/document_graph/transform/enrichers/__init__.py @@ -0,0 +1,24 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Enricher transformers for adding information to documents. + +This module provides transformers that enrich document content by adding +additional information such as: +- Language detection: Automatically detect document language +- LLM enrichment: Use large language models to add metadata + +These transformers enhance document processing by providing additional +context and metadata that can be used for analysis and retrieval. +""" + +from .language_enricher_provider import LanguageDetectionEnricher +from .llm_enricher_plugin import LLMEnricherPlugin +from .bedrock_enricher_plugin import BedrockEnricherPlugin +from .remediation_factory_provider import RemediationFactoryProvider + +__all__ = [ + 'LanguageDetectionEnricher', + 'LLMEnricherPlugin', + 'BedrockEnricherPlugin', + 'RemediationFactoryProvider' +] \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/enrichers/bedrock_enricher_plugin.py b/document-graph/src/document_graph/transform/enrichers/bedrock_enricher_plugin.py new file mode 100644 index 00000000..d8f352cf --- /dev/null +++ b/document-graph/src/document_graph/transform/enrichers/bedrock_enricher_plugin.py @@ -0,0 +1,203 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Bedrock Enricher Plugin for enhancing documents with AI-generated content.""" + +import logging +import json +from typing import List, Dict, Any +from document_graph.transform.transformer_provider_base import TransformerProvider +from document_graph.document_graph_config import document_graph_config + +logger = logging.getLogger(__name__) + + +class BedrockEnricherPlugin(TransformerProvider): + """Enriches records using AWS Bedrock models for content analysis.""" + + def __init__(self, config): + super().__init__(config) + + self.task = self.args.get("task", "summarize") + self.input_field = self.args.get("field", "content") + self.model_id = self.args.get("model_id", "anthropic.claude-3-haiku-20240307-v1:0") + self.prefix = self.args.get("prefix", "bedrock_") + self.suffix = self.args.get("suffix", "") + self.max_tokens = self.args.get("max_tokens", 200) + self.temperature = self.args.get("temperature", 0.3) + self.enable_dedup = self.args.get("enable_dedup", True) + self.cache_dir = self.args.get("cache_dir", None) + self._permission_error = False # Circuit breaker for permission errors + self._cache = {} # In-memory cache for deduplication + + # Setup persistent cache if cache_dir is provided + if self.cache_dir and self.enable_dedup: + import os + os.makedirs(self.cache_dir, exist_ok=True) + self._load_cache_from_disk() + + # If suffix is provided, use field name as prefix + if self.suffix: + self.prefix = self.args.get('csv_property_name', self.input_field) + + self.client = document_graph_config._get_or_create_client("bedrock-runtime") + + def _load_prompt_template(self, task: str) -> str: + """Load prompt template with fallback to document graph prompts.""" + import os + + # Try enricher-specific prompts first + enricher_prompt_path = os.path.join( + os.path.dirname(__file__), "prompts", f"evidence_{task}.txt" + ) + + if os.path.exists(enricher_prompt_path): + with open(enricher_prompt_path, 'r') as f: + return f.read().strip() + + # Fallback to document graph prompts + try: + from document_graph.prompts import get_prompt + return get_prompt(f"evidence_{task}") + except: + pass + + # Default fallback + if task == "summarize": + return "Summarize the key evidence from this JSON data in 2-3 sentences:\n\n{text}" + elif task == "tag": + return "Extract key tags from this evidence data:\n\n{text}" + else: + return "Analyze this evidence data:\n\n{text}" + + def _make_prompt(self, text: str) -> str: + template = self._load_prompt_template(self.task) + return template.format(text=text) + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + enriched_records = [] + + # Skip all processing if we've hit a permission error + if self._permission_error: + logger.warning("Skipping Bedrock enrichment due to previous permission error") + return records + + for record in records: + enriched_record = record.copy() + text = record.get(self.input_field, "") + + if text and isinstance(text, str) and text.strip(): + field_name = f"{self.prefix}{self.suffix}" if self.suffix else f"{self.prefix}{self.task}" + + # Check cache for deduplication + if self.enable_dedup: + import hashlib + text_hash = hashlib.md5(text.encode()).hexdigest() + if text_hash in self._cache: + logger.debug(f"Using cached result for duplicate evidence (hash: {text_hash[:8]}...)") + enriched_record[field_name] = self._cache[text_hash] + enriched_records.append(enriched_record) + continue + + try: + prompt = self._make_prompt(text) + + # Different request formats for different models + if "anthropic.claude" in self.model_id: + body = json.dumps({ + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": self.max_tokens, + "temperature": self.temperature, + "messages": [{"role": "user", "content": prompt}] + }) + elif "amazon.nova" in self.model_id: + body = json.dumps({ + "messages": [{"role": "user", "content": [{"text": prompt}]}], + "inferenceConfig": { + "maxTokens": self.max_tokens, + "temperature": self.temperature + } + }) + else: + # Default format + body = json.dumps({ + "inputText": prompt, + "textGenerationConfig": { + "maxTokenCount": self.max_tokens, + "temperature": self.temperature + } + }) + + response = self.client.invoke_model( + modelId=self.model_id, + body=body + ) + + response_body = json.loads(response['body'].read()) + + # Different response formats for different models + if "anthropic.claude" in self.model_id: + output = response_body['content'][0]['text'].strip() + elif "amazon.nova" in self.model_id: + output = response_body['output']['message']['content'][0]['text'].strip() + else: + output = response_body.get('results', [{}])[0].get('outputText', '').strip() + + enriched_record[field_name] = output + + # Cache the result for deduplication + if self.enable_dedup: + self._cache[text_hash] = output + logger.info(f"Cached new result for evidence hash: {text_hash[:8]}...") + # Save to disk if cache_dir is configured + if self.cache_dir: + self._save_cache_to_disk() + + except Exception as e: + error_msg = str(e) + + # Check for permission errors and set circuit breaker + if "AccessDeniedException" in error_msg or "bedrock:InvokeModel" in error_msg: + self._permission_error = True + logger.error(f"Bedrock permission error - stopping enrichment: {error_msg}") + error_field = f"{self.prefix}{self.suffix}_error" if self.suffix else f"{self.prefix}{self.task}_error" + enriched_record[error_field] = "Permission denied for Bedrock access" + enriched_records.append(enriched_record) + break # Stop processing immediately + else: + logger.warning(f"Bedrock enrichment failed: {error_msg}") + error_field = f"{self.prefix}{self.suffix}_error" if self.suffix else f"{self.prefix}{self.task}_error" + enriched_record[error_field] = error_msg + + enriched_records.append(enriched_record) + + return enriched_records + + def _load_cache_from_disk(self): + """Load cache from disk if it exists.""" + import os + import json + + cache_file = os.path.join(self.cache_dir, f"bedrock_cache_{self.task}.json") + if os.path.exists(cache_file): + try: + with open(cache_file, 'r') as f: + self._cache = json.load(f) + logger.debug(f"Loaded {len(self._cache)} cached results from {cache_file}") + except Exception as e: + logger.warning(f"Failed to load cache from {cache_file}: {e}") + + def _save_cache_to_disk(self): + """Save cache to disk.""" + if not self.cache_dir: + return + + import os + import json + + cache_file = os.path.join(self.cache_dir, f"bedrock_cache_{self.task}.json") + try: + with open(cache_file, 'w') as f: + json.dump(self._cache, f, indent=2) + logger.debug(f"Saved {len(self._cache)} cached results to {cache_file}") + except Exception as e: + logger.warning(f"Failed to save cache to {cache_file}: {e}") \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/enrichers/language_enricher_provider.py b/document-graph/src/document_graph/transform/enrichers/language_enricher_provider.py new file mode 100644 index 00000000..e8de6ac0 --- /dev/null +++ b/document-graph/src/document_graph/transform/enrichers/language_enricher_provider.py @@ -0,0 +1,76 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Language Enricher Provider for detecting document languages. + +This module provides functionality to automatically detect the language of text +content in documents. It uses the 'langdetect' library to identify the language +of text fields and adds this information as a new field in the document record. +This enrichment is useful for multilingual document collections, enabling filtering, +routing, or specialized processing based on document language. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from typing import List, Dict, Any +from langdetect import detect +from document_graph.transform.transformer_provider_base import TransformerProvider + + +class LanguageDetectionEnricher(TransformerProvider): + """Enriches records with language detection for text content. + + This transformer analyzes text content in records and determines the language + of the text using the 'langdetect' library. It adds a new field to each record + containing the detected language code (e.g., 'en' for English, 'es' for Spanish). + + Configuration: + text_field (str): The field name containing the text to analyze. + Defaults to 'content'. + output_field (str): The field name where the detected language will be stored. + Defaults to 'language'. + """ + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Transform records by detecting and adding language information. + + This method processes each record by: + 1. Extracting the text from the specified field + 2. Using langdetect to identify the language of the text + 3. Adding the detected language code to a new field in the record + + If text detection fails or the text field is empty/non-string, + the language is set to "unknown". + + Args: + records: A list of record dictionaries to process + + Returns: + A list of enriched records with language information added. + Each record maintains its original structure with an additional + language field. + """ + text_field = self.args.get("text_field", "content") + output_field = self.args.get("output_field", "language") + + enriched_records = [] + for record in records: + enriched_record = record.copy() + text = record.get(text_field, "") + + if text and isinstance(text, str): + try: + language = detect(text) + except Exception: + language = "unknown" + enriched_record[output_field] = language + else: + enriched_record[output_field] = "unknown" + + enriched_records.append(enriched_record) + + return enriched_records \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/enrichers/llm_enricher_plugin.py b/document-graph/src/document_graph/transform/enrichers/llm_enricher_plugin.py new file mode 100644 index 00000000..eacdad22 --- /dev/null +++ b/document-graph/src/document_graph/transform/enrichers/llm_enricher_plugin.py @@ -0,0 +1,152 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""LLM Enricher Plugin for enhancing documents with AI-generated content. + +This module provides functionality to enrich document content using Large Language +Models (LLMs) via the OpenAI API. It can perform various tasks such as tagging, +summarization, and classification of document content, adding the AI-generated +outputs as new fields in the document records. + +This enrichment is useful for adding semantic metadata to documents, generating +concise summaries, or categorizing content without manual intervention. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +import os +from typing import List, Dict, Any +from document_graph.transform.transformer_provider_base import TransformerProvider + +try: + from openai import OpenAI +except ImportError: + OpenAI = None + + +class LLMEnricherPlugin(TransformerProvider): + """Enriches records using OpenAI's LLMs for content analysis and enhancement. + + This transformer uses OpenAI's API to process text content in records and + generate additional metadata such as tags, summaries, or classifications. + It adds the LLM-generated content as new fields in each record, preserving + the original content. + + Configuration: + task (str): The type of enrichment to perform. Options are: + - "tag": Generate relevant tags/categories for the content + - "summarize": Create a concise summary of the content + - "classify": Identify the topic or domain of the content + Defaults to "tag". + field (str): The field name containing the text to analyze. + Defaults to "content". + llm_model (str): The OpenAI model to use for generation. + Defaults to "gpt-4". + api_key (str): OpenAI API key. If not provided, will look for + OPENAI_API_KEY environment variable. + """ + + def __init__(self, config): + """Initialize the LLM enricher with configuration. + + This method sets up the OpenAI client and configures the enricher + based on the provided configuration. It validates that the OpenAI + library is installed and that an API key is available. + + Args: + config: Configuration object containing transformer settings. + Must include an 'args' dictionary with task, field, + and model settings. Can optionally include an API key. + + Raises: + ImportError: If the 'openai' library is not installed. + ValueError: If no OpenAI API key is provided in config or environment. + """ + super().__init__(config) + if OpenAI is None: + raise ImportError("Please install the 'openai' library to use LLMEnricherPlugin") + + self.task = self.args.get("task", "tag") + self.input_field = self.args.get("field", "content") + self.model = self.args.get("llm_model", "gpt-4") + + api_key = self.args.get("api_key") or os.environ.get("OPENAI_API_KEY") + if not api_key: + raise ValueError("Missing OpenAI API key in config or environment") + + self.client = OpenAI(api_key=api_key) + + def _make_prompt(self, text: str) -> str: + """Create an appropriate prompt for the configured task. + + This method generates a task-specific prompt to send to the LLM, + based on the task type configured for this enricher (tag, summarize, + or classify). + + Args: + text: The document text to be processed by the LLM + + Returns: + A formatted prompt string that instructs the LLM on the task + to perform on the provided text. + + Raises: + ValueError: If the configured task is not one of the supported types. + """ + if self.task == "tag": + return f"Tag the following content with relevant categories:\n\n{text}" + elif self.task == "summarize": + return f"Summarize the following content:\n\n{text}" + elif self.task == "classify": + return f"Classify the topic or domain of the content below:\n\n{text}" + else: + raise ValueError(f"Unknown task: {self.task}") + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Transform records by enriching them with LLM-generated content. + + This method processes each record by: + 1. Extracting the text from the configured input field + 2. Creating a task-specific prompt for the LLM + 3. Sending the prompt to the OpenAI API + 4. Adding the LLM's response to the record in a new field + + The new field is named based on the task (e.g., "llm_tag", "llm_summarize"). + If the API call fails, an error field is added instead (e.g., "llm_tag_error"). + + Records with empty or non-string content in the input field are passed + through with minimal processing. + + Args: + records: A list of record dictionaries to enrich + + Returns: + A list of enriched records with LLM-generated content added. + Each record maintains its original structure with additional + fields containing the enrichment results. + """ + enriched_records = [] + for record in records: + enriched_record = record.copy() + text = record.get(self.input_field, "") + + if text and isinstance(text, str) and text.strip(): + try: + prompt = self._make_prompt(text) + response = self.client.chat.completions.create( + model=self.model, + messages=[{"role": "user", "content": prompt}], + temperature=0.3, + ) + output = response.choices[0].message.content.strip() + enriched_record[f"llm_{self.task}"] = output + except Exception as e: + enriched_record[f"llm_{self.task}_error"] = str(e) + + enriched_records.append(enriched_record) + + return enriched_records \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/enrichers/prompts/evidence_summarize.txt b/document-graph/src/document_graph/transform/enrichers/prompts/evidence_summarize.txt new file mode 100644 index 00000000..55e572a1 --- /dev/null +++ b/document-graph/src/document_graph/transform/enrichers/prompts/evidence_summarize.txt @@ -0,0 +1,8 @@ +Analyze the following security evidence data and provide a concise 2-3 sentence summary of the key findings and resources involved: + +{text} + +Focus on: +- What type of evidence objects are present +- Key security findings or configuration issues +- Affected resources and their identifiers \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/enrichers/prompts/evidence_tag.txt b/document-graph/src/document_graph/transform/enrichers/prompts/evidence_tag.txt new file mode 100644 index 00000000..dfc4f54b --- /dev/null +++ b/document-graph/src/document_graph/transform/enrichers/prompts/evidence_tag.txt @@ -0,0 +1,8 @@ +Extract relevant security tags and categories from the following evidence data. Return as comma-separated values: + +{text} + +Focus on extracting tags for: +- Evidence types (e.g., DATABASE, FIREWALL, CONFIGURATION_FINDING) +- Security domains (e.g., access-control, encryption, compliance) +- Severity levels and risk categories \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/enrichers/remediation_factory_provider.py b/document-graph/src/document_graph/transform/enrichers/remediation_factory_provider.py new file mode 100644 index 00000000..aba561f6 --- /dev/null +++ b/document-graph/src/document_graph/transform/enrichers/remediation_factory_provider.py @@ -0,0 +1,128 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Remediation Factory Provider for AWS remediation input generation.""" + +import json +import hashlib +import logging +from typing import Dict, Any, List +from document_graph.transform.transformer_provider_base import TransformerProvider +from document_graph.document_graph_config import document_graph_config + +logger = logging.getLogger(__name__) + + +class RemediationFactoryProvider(TransformerProvider): + """Provider that generates AWS remediation inputs using S3 factory pattern.""" + + def __init__(self, config): + super().__init__(config) + self.s3_bucket = self.args.get("s3_bucket") + self.s3_prefix = self.args.get("s3_prefix", "inputs/").rstrip('/') + '/' + self.model_id = self.args.get("model_id", "amazon.nova-pro-v1:0") + self.source_fields = self.args.get("source_fields", ["Resource Type", "Resource external ID", "Title", "Remediation Recommendation"]) + self.target_field = self.args.get("target_field", "remediation_input") + self.prompt_path = self.args.get("prompt_path") + self.cloud_provider_field = self.args.get("cloud_provider_field", "Resource Platform") + + if not self.s3_bucket: + raise ValueError("s3_bucket required") + + self.s3_client = document_graph_config._get_or_create_client("s3") + # Add timeout configuration + self.s3_client.meta.config.read_timeout = 30 + self.s3_client.meta.config.connect_timeout = 10 + self.bedrock_client = document_graph_config._get_or_create_client("bedrock-runtime") + self._prompt_template = None + + def _get_cloud_fallback_prompt(self, cloud_provider): + """Generate cloud-specific fallback prompt""" + prompts = { + "aws": "Generate AWS remediation for:\n{input_text}\nReturn JSON: {{\"service\": \"aws_service\", \"method\": \"boto3_method\", \"description\": \"fix_description\"}}", + "azure": "Generate Azure remediation for:\n{input_text}\nReturn JSON: {{\"service\": \"azure_service\", \"method\": \"azure_method\", \"description\": \"fix_description\"}}", + "gcp": "Generate GCP remediation for:\n{input_text}\nReturn JSON: {{\"service\": \"gcp_service\", \"method\": \"gcp_method\", \"description\": \"fix_description\"}}" + } + return prompts.get(cloud_provider.lower(), prompts["aws"]) + + def _load_prompt(self, cloud_provider="aws"): + if self._prompt_template: + return self._prompt_template + if self.prompt_path: + if self.prompt_path.startswith('s3://'): + bucket, key = self.prompt_path[5:].split('/', 1) + response = self.s3_client.get_object(Bucket=bucket, Key=key) + self._prompt_template = response['Body'].read().decode('utf-8') + else: + with open(self.prompt_path, 'r') as f: + self._prompt_template = f.read() + else: + self._prompt_template = self._get_cloud_fallback_prompt(cloud_provider) + return self._prompt_template + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Transform records by adding remediation inputs.""" + print(f"🔧 RemediationFactory: Processing {len(records)} records") + enriched_records = [] + + for record in records: + enriched_record = record.copy() + input_parts = [ + f"{field}: {record[field]}" + for field in self.source_fields + if field in record and record[field] + ] + if input_parts: + input_text = "\n".join(input_parts) + cache_key = hashlib.sha256(input_text.encode()).hexdigest() + s3_key = f"{self.s3_prefix}{cache_key}.json" + + try: + response = self.s3_client.get_object(Bucket=self.s3_bucket, Key=s3_key) + enriched_record[self.target_field] = json.loads(response['Body'].read().decode('utf-8')) + except: + try: + # Detect cloud provider from record + cloud_provider = "aws" # default + if self.cloud_provider_field in record: + provider_value = str(record[self.cloud_provider_field]).lower() + if "azure" in provider_value or "microsoft" in provider_value: + cloud_provider = "azure" + elif "gcp" in provider_value or "google" in provider_value: + cloud_provider = "gcp" + + prompt = self._load_prompt(cloud_provider).format(input_text=input_text) + body = json.dumps({"messages": [{"role": "user", "content": [{"text": prompt}]}]}) + response = self.bedrock_client.invoke_model(modelId=self.model_id, body=body) + content = json.loads(response['body'].read())['output']['message']['content'][0]['text'] + logger.debug(f"Bedrock response: {content[:200]}...") + + # Simple JSON extraction - just find first { to last } + import re + json_start = content.find('{') + json_end = content.rfind('}') + + if json_start != -1 and json_end != -1: + json_str = content[json_start:json_end + 1] + # Clean control characters + json_str = re.sub(r'[\x00-\x1f\x7f-\x9f]', '', json_str) + logger.debug(f"Extracted JSON: {json_str}") + input_data = json.loads(json_str) + logger.debug(f"Storing to S3: {s3_key}") + try: + self.s3_client.put_object(Bucket=self.s3_bucket, Key=s3_key, Body=json.dumps(input_data, indent=2)) + logger.debug(f"Successfully stored to S3: {s3_key}") + except Exception as s3_error: + logger.error(f"S3 storage failed: {s3_error}") + enriched_record[self.target_field] = input_data + print(f" ✅ Added {self.target_field} to record") + else: + logger.error(f"No JSON found in response: {content}") + raise ValueError("Could not find JSON in response") + except Exception as e: + logger.error(f"Failed to generate remediation input: {e}") + print(f" ❌ Failed to add {self.target_field}: {e}") + + enriched_records.append(enriched_record) + + print(f"✅ RemediationFactory: Completed {len(enriched_records)} records") + return enriched_records \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/field_transformers/__init__.py b/document-graph/src/document_graph/transform/field_transformers/__init__.py new file mode 100644 index 00000000..499c2546 --- /dev/null +++ b/document-graph/src/document_graph/transform/field_transformers/__init__.py @@ -0,0 +1,42 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Field-level transformers for processing individual fields. + +This module provides transformers that operate on individual fields within +documents, such as: +- Regex cleaning: Clean field values using regular expressions +- Embedded JSON: Extract and process JSON within text fields +- Timestamp normalization: Standardize timestamp formats +- Enum standardization: Normalize enumerated values + +These transformers help clean and standardize field-level data during +document processing pipelines. +""" + +from .regex_cleaner_provider import RegexCleanerProvider +from .embedded_json import EmbeddedJSONFieldTransformer +from .normalize_timestamp import TimestampNormalizer +from .standardize_enum import EnumStandardizer +from .comma_split_provider import CommaSplitProvider +from .json_array_expander import JSONArrayExpanderProvider +from .json_array_flattener import JSONArrayFlattenerProvider +from .comma_flattener import CommaFlattenerProvider +from .paired_flattener import PairedFlattenerProvider +from .json_value_flattener import JSONValueFlattenerProvider +from .json_flattener import JSONFlattenerProvider +from .uuid_generator import UuidGeneratorTransformer + +__all__ = [ + 'RegexCleanerProvider', + 'EmbeddedJSONFieldTransformer', + 'TimestampNormalizer', + 'EnumStandardizer', + 'CommaSplitProvider', + 'JSONArrayExpanderProvider', + 'JSONArrayFlattenerProvider', + 'CommaFlattenerProvider', + 'PairedFlattenerProvider', + 'JSONValueFlattenerProvider', + 'JSONFlattenerProvider', + 'UuidGeneratorTransformer' +] \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/field_transformers/comma_flattener.py b/document-graph/src/document_graph/transform/field_transformers/comma_flattener.py new file mode 100644 index 00000000..04b2b26e --- /dev/null +++ b/document-graph/src/document_graph/transform/field_transformers/comma_flattener.py @@ -0,0 +1,193 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Comma Flattener transformer for enriching records with comma-separated field data. + +This transformer flattens comma-separated values into individual columns without creating +new rows, allowing you to enrich existing records with structured data from +comma-separated fields and then map those enriched fields to different graph nodes. +""" + +import logging +from typing import List, Dict, Any +from document_graph.transform.transformer_provider_base import TransformerProvider + +logger = logging.getLogger(__name__) + + +class CommaFlattenerProvider(TransformerProvider): + """Flattens comma-separated values into columns to enrich records. + + This transformer takes comma-separated values and flattens them into individual + columns with a configurable prefix and numbering. It preserves the original row count while + adding new columns for each comma-separated value. + + Args: + field: The field containing comma-separated data (required) + prefix: Prefix for flattened fields (default: field name + "_") + separator: Separator character (default: ",") + strip_whitespace: Strip whitespace from values (default: True) + preserve_original_field: Keep original field (default: False) + max_items: Maximum number of items to extract (default: 10) + """ + + def __init__(self, config): + super().__init__(config) + self.field = self.args.get('field') or config.csv_property_name + if not self.field: + raise ValueError("Comma Flattener requires 'field' argument or csv_property_name") + + # Optional companion field for paired values + self.companion_field = self.args.get('companion_field') + self.companion_suffix = self.args.get('companion_suffix', '_name') + + # Generate prefix from field name if not provided + if 'prefix' not in self.args: + # Clean field name: remove spaces and special chars + clean_field = ''.join(c if c.isalnum() else '' for c in self.field) + self.prefix = clean_field + logger.info(f"Comma Flattener using default prefix: '{self.prefix}' (from field '{self.field}')") + else: + self.prefix = self.args.get('prefix') + logger.info(f"Comma Flattener using explicit prefix: '{self.prefix}'") + + if 'suffix' not in self.args: + self.suffix = '#00' # Default to 2-digit numbers: 01, 02, etc. + logger.info(f"Comma Flattener using default suffix: '{self.suffix}' (auto-incremental numbers)") + else: + self.suffix = self.args.get('suffix') + logger.info(f"Comma Flattener using explicit suffix: '{self.suffix}'") + self.separator = self.args.get('separator', ',') + self.strip_whitespace = self.args.get('strip_whitespace', True) + self.preserve_original_field = self.args.get('preserve_original_field', False) + self.max_items = self.args.get('max_items', 10) + self.unique_nodes = self.args.get('unique_nodes', False) + self._unique_values = set() # Track unique values globally + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Transform records by flattening comma-separated values into columns.""" + + # If unique_nodes is enabled, collect all unique values first + if self.unique_nodes: + self._collect_unique_values(records) + logger.info(f"Found {len(self._unique_values)} unique values for field '{self.field}'") + + enriched_records = [] + + for record in records: + try: + enriched_record = self._enrich_record(record) + enriched_records.append(enriched_record) + except Exception as e: + logger.warning(f"Failed to process comma-separated field '{self.field}': {e}") + enriched_records.append(record.copy()) + + return enriched_records + + def _enrich_record(self, original_record: Dict[str, Any]) -> Dict[str, Any]: + """Enrich record with flattened comma-separated data.""" + # Start with original record + if self.preserve_original_field: + enriched_record = original_record.copy() + else: + enriched_record = {k: v for k, v in original_record.items() if k != self.field} + + field_value = original_record.get(self.field) + + # Handle empty/null values + if field_value is None or field_value == '': + return enriched_record + + # Convert to string if not already + if not isinstance(field_value, str): + field_value = str(field_value) + + # Split by separator + items = field_value.split(self.separator) + + # Strip whitespace if requested + if self.strip_whitespace: + items = [item.strip() for item in items] + + # Filter out empty items + items = [item for item in items if item] + + # Handle companion field if specified + companion_items = [] + if self.companion_field: + companion_value = original_record.get(self.companion_field, '') + if companion_value: + companion_items = companion_value.split(self.separator) + if self.strip_whitespace: + companion_items = [item.strip() for item in companion_items] + companion_items = [item for item in companion_items if item] + + # Add columns based on unique_nodes setting + if self.unique_nodes: + # Use consistent mapping for unique values + value_mapping = self._get_unique_column_mapping() + for item in items: + if item in value_mapping: + column_name = value_mapping[item] + enriched_record[column_name] = item + else: + # Add numbered columns up to max_items (original behavior) + for i, item in enumerate(items[:self.max_items], 1): + if self.suffix and '#' in self.suffix: + # Format number according to suffix pattern (e.g., " #00" -> " 01", " 02") + # Split suffix into prefix part (before #) and number format (after #) + hash_index = self.suffix.index('#') + suffix_prefix = self.suffix[:hash_index] # e.g., " " (space) + number_format = self.suffix[hash_index+1:] # e.g., "00" + num_zeros = len(number_format) + formatted_num = str(i).zfill(num_zeros) + column_name = f"{self.prefix}{suffix_prefix}{formatted_num}" + elif self.suffix: + column_name = f"{self.prefix}{self.suffix}{i}" + else: + column_name = f"{self.prefix}{i}" + enriched_record[column_name] = item + + # Add companion field if available + if self.companion_field and i <= len(companion_items): + companion_column = column_name + self.companion_suffix + enriched_record[companion_column] = companion_items[i-1] + + return enriched_record + + def _collect_unique_values(self, records: List[Dict[str, Any]]) -> None: + """Collect all unique values across all records.""" + for record in records: + field_value = record.get(self.field) + if field_value is None or field_value == '': + continue + + if not isinstance(field_value, str): + field_value = str(field_value) + + items = field_value.split(self.separator) + if self.strip_whitespace: + items = [item.strip() for item in items] + + items = [item for item in items if item] + self._unique_values.update(items) + + def _get_unique_column_mapping(self) -> Dict[str, str]: + """Get mapping of unique values to column names.""" + mapping = {} + unique_list = sorted(list(self._unique_values))[:self.max_items] + + for i, value in enumerate(unique_list, 1): + if self.suffix and '#' in self.suffix: + hash_index = self.suffix.index('#') + suffix_prefix = self.suffix[:hash_index] + number_format = self.suffix[hash_index+1:] + num_zeros = len(number_format) + formatted_num = str(i).zfill(num_zeros) + column_name = f"{self.prefix}{suffix_prefix}{formatted_num}" + elif self.suffix: + column_name = f"{self.prefix}{self.suffix}{i}" + else: + column_name = f"{self.prefix}{i}" + mapping[value] = column_name + + return mapping \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/field_transformers/comma_split_provider.py b/document-graph/src/document_graph/transform/field_transformers/comma_split_provider.py new file mode 100644 index 00000000..c090cf57 --- /dev/null +++ b/document-graph/src/document_graph/transform/field_transformers/comma_split_provider.py @@ -0,0 +1,112 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Comma-split transformer for document graph operations. + +This module provides a transformer that splits comma-separated values in specified +fields and creates multiple records from a single input record. This is useful for +handling fields that contain multiple values separated by commas, such as tags, +categories, or IDs that need to be processed as separate entities. +""" + +import logging + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + +from typing import List, Dict, Any +from document_graph.transform.transformer_provider_base import TransformerProvider + + +class CommaSplitProvider(TransformerProvider): + """Splits comma-separated values in fields and creates multiple records. + + This transformer takes records with comma-separated values in specified fields + and creates multiple output records, one for each split value. This is useful + for normalizing data where multiple values are stored in a single field. + + Attributes: + config: Configuration object for the transformer + name: Name of the transformer (from config) + args: Arguments for the transformer (from config) + - fields: List of fields to split on commas + - separator: Character to split on (default: ",") + - strip_whitespace: Whether to strip whitespace from split values (default: True) + + Examples: + >>> # Create a transformer to split project IDs + >>> config = TransformerProviderConfig( + ... name="project_splitter", + ... args={ + ... "fields": ["project_ids"], + ... "separator": ",", + ... "strip_whitespace": True + ... } + ... ) + >>> transformer = CommaSplitProvider(config) + >>> result = transformer.transform([ + ... {"id": 1, "project_ids": "proj-1, proj-2, proj-3", "name": "Resource A"} + ... ]) + >>> len(result) + 3 + >>> result[0]["project_ids"] + 'proj-1' + >>> result[1]["project_ids"] + 'proj-2' + """ + + def __init__(self, config): + """Initialize the comma split transformer with configuration. + + Args: + config: Transformer configuration with name, type, and args + - fields: List of fields to split on commas + - separator: Character to split on (default: ",") + - strip_whitespace: Whether to strip whitespace (default: True) + """ + super().__init__(config) + self.separator = self.args.get('separator', ',') + self.strip_whitespace = self.args.get('strip_whitespace', True) + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Transform records by splitting comma-separated values into multiple records. + + Args: + records: List of record dictionaries to transform + + Returns: + List of transformed record dictionaries with split values + + Note: + Each input record may generate multiple output records, one for each + split value in the specified fields. If multiple fields are specified, + the transformation is applied to each field independently. + """ + fields_to_split = self.args.get('fields', []) + + if not fields_to_split: + return records + + transformed_records = [] + + for record in records: + # For each field that needs splitting, create separate records + for field in fields_to_split: + if field in record and record[field]: + field_value = str(record[field]) + split_values = field_value.split(self.separator) + + if self.strip_whitespace: + split_values = [val.strip() for val in split_values if val.strip()] + else: + split_values = [val for val in split_values if val] + + # Create a new record for each split value + for split_value in split_values: + new_record = record.copy() + new_record[field] = split_value + transformed_records.append(new_record) + else: + # If field doesn't exist or is empty, keep original record + transformed_records.append(record.copy()) + + return transformed_records \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/field_transformers/embedded_json.py b/document-graph/src/document_graph/transform/field_transformers/embedded_json.py new file mode 100644 index 00000000..a3aaddfe --- /dev/null +++ b/document-graph/src/document_graph/transform/field_transformers/embedded_json.py @@ -0,0 +1,87 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Embedded JSON field transformer for document graph operations. + +This module provides a transformer that parses JSON data from string fields +and flattens the JSON structure into the record with prefixed field names. +This is useful for extracting structured data embedded within JSON strings +and making it available as individual fields for querying and analysis. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +import json +from typing import List, Dict, Any +from document_graph.transform.transformer_provider_base import TransformerProvider + + +class EmbeddedJSONFieldTransformer(TransformerProvider): + """Parses JSON fields and flattens them into the record. + + This transformer extracts structured data from JSON strings stored in a specified field + and adds each key-value pair from the JSON object as a separate field in the record. + The new fields are prefixed to avoid name collisions with existing fields. + + Attributes: + config: Configuration object for the transformer + name: Name of the transformer (from config) + args: Arguments for the transformer (from config) + - field: The field containing the JSON data (default: "json_data") + - prefix: Prefix to add to extracted field names (default: "embedded_") + + Examples: + >>> # Create a transformer to extract JSON data from the "metadata" field + >>> config = TransformerProviderConfig( + ... name="json_extractor", + ... args={"field": "metadata", "prefix": "meta_"} + ... ) + >>> transformer = EmbeddedJSONFieldTransformer(config) + >>> result = transformer.transform([ + ... {"id": 1, "metadata": '{"author": "Jane Doe", "year": 2023}'} + ... ]) + >>> result[0] + {'id': 1, 'metadata': '{"author": "Jane Doe", "year": 2023}', 'meta_author': 'Jane Doe', 'meta_year': 2023} + """ + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Transform records by extracting and flattening JSON data. + + Args: + records: List of record dictionaries to transform + + Returns: + List of transformed record dictionaries with flattened JSON fields + + Note: + If the JSON parsing fails, an error field will be added to the record + with the prefix and "error" suffix. + """ + target_field = self.args.get("field", "json_data") + prefix = self.args.get("prefix", "embedded_") + + transformed_records = [] + for record in records: + transformed_record = record.copy() + + try: + field_value = record.get(target_field, "") + if isinstance(field_value, str): + embedded_data = json.loads(field_value) + else: + embedded_data = field_value + + if isinstance(embedded_data, dict): + for key, value in embedded_data.items(): + transformed_record[f"{prefix}{key}"] = value + + except Exception as e: + transformed_record[f"{prefix}error"] = str(e) + + transformed_records.append(transformed_record) + + return transformed_records \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/field_transformers/json_array_expander.py b/document-graph/src/document_graph/transform/field_transformers/json_array_expander.py new file mode 100644 index 00000000..9797674b --- /dev/null +++ b/document-graph/src/document_graph/transform/field_transformers/json_array_expander.py @@ -0,0 +1,333 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""JSON Array Expander transformer for cybersecurity data processing. + +This module provides a robust transformer specifically designed for handling +complex JSON arrays commonly found in cybersecurity vendor data. It expands +JSON arrays into multiple records while handling various edge cases like +empty arrays, duplicate fields, nested structures, and malformed data. + +This transformer is essential for normalizing vendor security data that often +contains nested evidence, alerts, indicators, and other security artifacts +stored as JSON arrays within CSV fields. +""" + +import logging + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + +import json +import pandas as pd +from typing import List, Dict, Any, Union +from document_graph.transform.transformer_provider_base import TransformerProvider + + +class JSONArrayExpanderProvider(TransformerProvider): + """Expands JSON arrays in fields into multiple records with advanced conflict resolution. + + This transformer is specifically designed for cybersecurity data where vendors often + store complex nested data structures as JSON arrays. It handles common issues like: + - Empty arrays that should be skipped + - Duplicate field names between original record and JSON data + - Nested objects within arrays + - Malformed JSON data + - Field name conflicts and collision resolution + + Attributes: + config: Configuration object for the transformer + name: Name of the transformer (from config) + args: Arguments for the transformer (from config) + - field: The field containing the JSON array data (required) + - skip_empty_arrays: Skip records with empty arrays [] (default: True) + - conflict_resolution: How to handle field name conflicts (default: "prefix") + - "prefix": Add prefix to JSON fields (json_fieldname) + - "suffix": Add suffix to JSON fields (fieldname_json) + - "overwrite": JSON fields overwrite original fields + - "skip": Skip JSON fields that conflict with original + - prefix: Prefix for JSON fields when using prefix resolution (default: "json_") + - suffix: Suffix for JSON fields when using suffix resolution (default: "_json") + - flatten_nested: Flatten nested objects using dot notation (default: True) + - max_depth: Maximum nesting depth to flatten (default: 3) + - preserve_original_field: Keep the original JSON field in output (default: False) + - add_array_index: Add array index as a field (default: True) + - array_index_field: Field name for array index (default: "array_index") + - log_unmapped_fields: Log unmapped fields to file (default: False) + - log_file_path: Path to log file for unmapped fields (default: "unmapped_fields.json") + - parent_node: Parent node name for unmapped field suggestions (required if log_unmapped_fields=True) + + Examples: + >>> # Basic array expansion for security evidence + >>> config = TransformerProviderConfig( + ... name="evidence_expander", + ... type="json_array_expander", + ... args={ + ... "field": "Evidence", + ... "skip_empty_arrays": True, + ... "conflict_resolution": "prefix" + ... } + ... ) + >>> transformer = JSONArrayExpanderProvider(config) + >>> result = transformer.transform([ + ... {"id": 1, "Evidence": '[{"type": "FILE", "path": "/etc/passwd"}]'} + ... ]) + >>> result[0] + {'id': 1, 'Evidence': '[{"type": "FILE", "path": "/etc/passwd"}]', + 'json_type': 'FILE', 'json_path': '/etc/passwd', 'array_index': 0} + """ + + def __init__(self, config): + """Initialize the JSON array expander with configuration. + + Args: + config: Transformer configuration with name, type, and args + """ + super().__init__(config) + self.field = self.args.get('field') + if not self.field: + raise ValueError("JSON Array Expander requires 'field' argument") + + self.skip_empty_arrays = self.args.get('skip_empty_arrays', True) + self.conflict_resolution = self.args.get('conflict_resolution', 'prefix') + self.prefix = self.args.get('prefix', 'json_') + self.suffix = self.args.get('suffix', '_json') + self.flatten_nested = self.args.get('flatten_nested', True) + self.max_depth = self.args.get('max_depth', 3) + self.preserve_original_field = self.args.get('preserve_original_field', False) + self.add_array_index = self.args.get('add_array_index', True) + self.array_index_field = self.args.get('array_index_field', 'array_index') + self.log_unmapped_fields = self.args.get('log_unmapped_fields', False) + self.log_file_path = self.args.get('log_file_path', 'unmapped_fields.json') + self.parent_node = self.args.get('parent_node') + self.auto_add_discovered = self.args.get('auto_add_discovered', False) + self.preserve_original_field = self.args.get('preserve_original_field', False) + self.unmapped_fields = set() # Track unique unmapped fields + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Transform records by expanding JSON arrays into multiple records. + + Args: + records: List of record dictionaries to transform + + Returns: + List of transformed record dictionaries with expanded JSON array data + + Note: + Each input record may generate multiple output records, one for each + item in the JSON array. Empty arrays are handled according to the + skip_empty_arrays configuration. + """ + expanded_records = [] + + for record in records: + try: + json_data = self._extract_json_data(record) + + if json_data is None: + # No JSON data found, set field to NaN and keep original record + record_copy = record.copy() + record_copy[self.field] = pd.NA + expanded_records.append(record_copy) + continue + + if isinstance(json_data, list): + if len(json_data) == 0 and self.skip_empty_arrays: + # Skip empty arrays + logger.debug(f"Skipping record with empty array in field '{self.field}'") + continue + + # Expand array into multiple records + for index, item in enumerate(json_data): + new_record = self._create_expanded_record(record, item, index) + expanded_records.append(new_record) + + else: + # Single object, treat as array with one item + new_record = self._create_expanded_record(record, json_data, 0) + expanded_records.append(new_record) + + except Exception as e: + logger.warning(f"Failed to process JSON in field '{self.field}': {e}") + # Set field to NaN on error and keep original record + record_copy = record.copy() + record_copy[self.field] = pd.NA + expanded_records.append(record_copy) + + return expanded_records + + def _extract_json_data(self, record: Dict[str, Any]) -> Union[List, Dict, None]: + """Extract and parse JSON data from the specified field.""" + field_value = record.get(self.field) + + if field_value is None or field_value == '': + return None + + if isinstance(field_value, str): + # Handle empty strings + if field_value.strip() == '': + return None + try: + return json.loads(field_value) + except json.JSONDecodeError as e: + logger.warning(f"Invalid JSON in field '{self.field}': {e}") + return None + elif isinstance(field_value, (list, dict)): + return field_value + else: + logger.warning(f"Unsupported data type in field '{self.field}': {type(field_value)}") + return None + + def _create_expanded_record(self, original_record: Dict[str, Any], + json_item: Any, index: int) -> Dict[str, Any]: + """Create a new record by merging original record with JSON item data.""" + # Start with original record + if self.preserve_original_field: + new_record = original_record.copy() + else: + new_record = {k: v for k, v in original_record.items() if k != self.field} + + # Add array index if requested + if self.add_array_index: + new_record[self.array_index_field] = index + + # Process JSON item + if isinstance(json_item, dict): + flattened_data = self._flatten_json_object(json_item) + resolved_data = self._resolve_field_conflicts(new_record, flattened_data) + + # Auto-add discovered fields or log unmapped fields + if self.auto_add_discovered: + # Add all discovered fields to the record + new_record.update(resolved_data) + # Log what was auto-mapped if requested + if self.log_unmapped_fields and self.parent_node: + self._log_unmapped_fields(flattened_data) + else: + # Only log unmapped fields if requested + if self.log_unmapped_fields and self.parent_node: + self._log_unmapped_fields(flattened_data) + else: + # Handle primitive values in array + if self.auto_add_discovered: + field_name = self._resolve_field_name('value', new_record) + new_record[field_name] = json_item + elif self.log_unmapped_fields: + # Log that a primitive value was found but not mapped + logger.info(f"Found primitive value in array but auto_add_discovered=False: {json_item}") + + return new_record + + def _flatten_json_object(self, json_obj: Dict[str, Any], + parent_key: str = '', depth: int = 0) -> Dict[str, Any]: + """Flatten nested JSON object using dot notation.""" + if not self.flatten_nested or depth >= self.max_depth: + return json_obj + + flattened = {} + + for key, value in json_obj.items(): + new_key = f"{parent_key}.{key}" if parent_key else key + + if isinstance(value, dict) and depth < self.max_depth: + # Recursively flatten nested objects + nested_flattened = self._flatten_json_object(value, new_key, depth + 1) + flattened.update(nested_flattened) + elif isinstance(value, list) and len(value) > 0 and isinstance(value[0], dict): + # Handle arrays of objects by taking first item or concatenating + if len(value) == 1: + nested_flattened = self._flatten_json_object(value[0], new_key, depth + 1) + flattened.update(nested_flattened) + else: + # Multiple items - convert to string representation + flattened[new_key] = str(value) + else: + flattened[new_key] = value + + return flattened + + def _resolve_field_conflicts(self, original_record: Dict[str, Any], + json_data: Dict[str, Any]) -> Dict[str, Any]: + """Resolve field name conflicts between original record and JSON data.""" + resolved_data = {} + + for key, value in json_data.items(): + resolved_key = self._resolve_field_name(key, original_record) + resolved_data[resolved_key] = value + + return resolved_data + + def _resolve_field_name(self, json_field: str, original_record: Dict[str, Any]) -> str: + """Resolve a single field name conflict.""" + # Force prefix when auto_add_discovered is enabled + if self.auto_add_discovered and self.conflict_resolution == 'prefix': + return f"{self.prefix}{json_field}" + + if json_field not in original_record: + return json_field + + # Field conflict detected + if self.conflict_resolution == 'prefix': + return f"{self.prefix}{json_field}" + elif self.conflict_resolution == 'suffix': + return f"{json_field}{self.suffix}" + elif self.conflict_resolution == 'overwrite': + return json_field + elif self.conflict_resolution == 'skip': + # Return None to indicate this field should be skipped + return None + else: + # Default to prefix + return f"{self.prefix}{json_field}" + + def _log_unmapped_fields(self, flattened_data: Dict[str, Any]) -> None: + """Log unmapped fields for transform_and_load.json generation.""" + for field_name in flattened_data.keys(): + if field_name not in self.unmapped_fields: + self.unmapped_fields.add(field_name) + + # Create mapping suggestion + mapping_suggestion = { + "csv_property_name": field_name, + "type": "property", + "arrow_property_name": field_name.lower().replace(' ', '_').replace('-', '_'), + "parents": [self.parent_node] + } + + # Append to log file + try: + import os + # Create directory if it doesn't exist + log_dir = os.path.dirname(self.log_file_path) + if log_dir and not os.path.exists(log_dir): + os.makedirs(log_dir, exist_ok=True) + + existing_data = [] + if os.path.exists(self.log_file_path): + try: + with open(self.log_file_path, 'r') as f: + existing_data = json.load(f) + except json.JSONDecodeError: + # File is corrupted, start fresh + logger.warning(f"Corrupted log file {self.log_file_path}, recreating") + existing_data = [] + + existing_data.append(mapping_suggestion) + + with open(self.log_file_path, 'w') as f: + json.dump(existing_data, f, indent=2) + + if hasattr(self, 'auto_add_discovered') and self.auto_add_discovered: + logger.debug(f"Auto-mapped field '{field_name}' to {self.parent_node} node") + else: + logger.debug(f"Logged unmapped field '{field_name}' to {self.log_file_path}") + + except Exception as e: + logger.error(f"Failed to log unmapped field '{field_name}': {e}") + + def __del__(self): + """Cleanup: Log summary of discovered fields.""" + if self.log_unmapped_fields and self.unmapped_fields: + if hasattr(self, 'auto_add_discovered') and self.auto_add_discovered: + logger.debug(f"Auto-mapped {len(self.unmapped_fields)} discovered fields in '{self.field}' to {self.parent_node} nodes: {sorted(self.unmapped_fields)}") + else: + logger.debug(f"Found {len(self.unmapped_fields)} unmapped fields in '{self.field}': {sorted(self.unmapped_fields)}") \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/field_transformers/json_array_flattener.py b/document-graph/src/document_graph/transform/field_transformers/json_array_flattener.py new file mode 100644 index 00000000..a4e52168 --- /dev/null +++ b/document-graph/src/document_graph/transform/field_transformers/json_array_flattener.py @@ -0,0 +1,98 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""JSON Array Flattener transformer for enriching records with JSON array data. + +This transformer flattens JSON arrays into individual numbered columns without creating +new rows, similar to comma_flattener but for JSON arrays. +""" + +import logging +import json +from typing import List, Dict, Any +from document_graph.transform.transformer_provider_base import TransformerProvider + +logger = logging.getLogger(__name__) + + +class JSONArrayFlattenerProvider(TransformerProvider): + """Flattens JSON arrays into numbered columns to enrich records. + + This transformer takes JSON arrays and flattens them into individual + columns with a configurable prefix and numbering, similar to comma_flattener. + + Args: + field: The field containing JSON array data (required) + prefix: Prefix for flattened fields (default: field name) + suffix: Suffix pattern for numbering (default: " #00") + preserve_original_field: Keep original field (default: False) + max_items: Maximum number of items to extract (default: 10) + """ + + def __init__(self, config): + super().__init__(config) + self.field = self.args.get('field') or config.csv_property_name + if not self.field: + raise ValueError("JSON Array Flattener requires 'field' argument or csv_property_name") + + if 'prefix' not in self.args: + clean_field = ''.join(c if c.isalnum() else '' for c in self.field) + self.prefix = clean_field + else: + self.prefix = self.args.get('prefix') + + self.suffix = self.args.get('suffix', ' #00') + self.preserve_original_field = self.args.get('preserve_original_field', False) + self.max_items = self.args.get('max_items', 10) + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Transform records by flattening JSON arrays into numbered columns.""" + enriched_records = [] + + for record in records: + try: + enriched_record = self._enrich_record(record) + enriched_records.append(enriched_record) + except Exception as e: + logger.warning(f"Failed to process JSON array field '{self.field}': {e}") + enriched_records.append(record.copy()) + + return enriched_records + + def _enrich_record(self, original_record: Dict[str, Any]) -> Dict[str, Any]: + """Enrich record with flattened JSON array data.""" + if self.preserve_original_field: + enriched_record = original_record.copy() + else: + enriched_record = {k: v for k, v in original_record.items() if k != self.field} + + field_value = original_record.get(self.field) + + if field_value is None or field_value == '': + return enriched_record + + # Parse JSON if string + if isinstance(field_value, str): + try: + json_data = json.loads(field_value) + except json.JSONDecodeError: + return enriched_record + else: + json_data = field_value + + # Handle arrays + if isinstance(json_data, list): + for i, item in enumerate(json_data[:self.max_items], 1): + if self.suffix and '#' in self.suffix: + hash_index = self.suffix.index('#') + suffix_prefix = self.suffix[:hash_index] + number_format = self.suffix[hash_index+1:] + num_zeros = len(number_format) + formatted_num = str(i).zfill(num_zeros) + column_name = f"{self.prefix}{suffix_prefix}{formatted_num}" + else: + column_name = f"{self.prefix}{i}" + + # Convert item to string representation + enriched_record[column_name] = json.dumps(item) if isinstance(item, (dict, list)) else str(item) + + return enriched_record \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/field_transformers/json_array_paired_flattener.py b/document-graph/src/document_graph/transform/field_transformers/json_array_paired_flattener.py new file mode 100644 index 00000000..457f084b --- /dev/null +++ b/document-graph/src/document_graph/transform/field_transformers/json_array_paired_flattener.py @@ -0,0 +1,72 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""JSON Array Paired Flattener for flattening two related JSON arrays.""" + +import logging +import json +from typing import List, Dict, Any, Union +from document_graph.transform.transformer_provider_base import TransformerProvider + +logger = logging.getLogger(__name__) + + +class JSONArrayPairedFlattenerProvider(TransformerProvider): + """Flattens two related JSON arrays into paired columns. + + Handles JSON arrays like: ["id1", "id2"] and ["name1", "name2"] + Creates columns like: project_01_id, project_01_name, project_02_id, project_02_name + """ + + def __init__(self, config): + super().__init__(config) + self.id_field = self.args.get('id_field') + self.name_field = self.args.get('name_field') + self.prefix = self.args.get('prefix', 'project') + self.max_items = self.args.get('max_items', 10) + + if not self.id_field or not self.name_field: + raise ValueError("JSONArrayPairedFlattener requires both 'id_field' and 'name_field'") + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + enriched_records = [] + + for record in records: + enriched_record = record.copy() + + ids = self._parse_json_array(record.get(self.id_field)) + names = self._parse_json_array(record.get(self.name_field)) + + # Pair up IDs and names (pad shorter list with empty strings) + max_len = min(max(len(ids), len(names)), self.max_items) + ids.extend([''] * (max_len - len(ids))) + names.extend([''] * (max_len - len(names))) + + for i in range(max_len): + num = str(i + 1).zfill(2) + enriched_record[f"{self.prefix}_{num}_id"] = ids[i] + enriched_record[f"{self.prefix}_{num}_name"] = names[i] + + enriched_records.append(enriched_record) + + return enriched_records + + def _parse_json_array(self, value: Union[str, List, None]) -> List[str]: + """Parse JSON array from string or return list directly.""" + if not value: + return [] + + if isinstance(value, list): + return [str(item) for item in value] + + if isinstance(value, str): + try: + parsed = json.loads(value) + if isinstance(parsed, list): + return [str(item) for item in parsed] + else: + return [str(parsed)] # Single value + except json.JSONDecodeError: + # Fallback to comma-separated for compatibility + return [item.strip() for item in value.split(',') if item.strip()] + + return [str(value)] # Single value fallback \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/field_transformers/json_flattener.py b/document-graph/src/document_graph/transform/field_transformers/json_flattener.py new file mode 100644 index 00000000..ce5758f5 --- /dev/null +++ b/document-graph/src/document_graph/transform/field_transformers/json_flattener.py @@ -0,0 +1,174 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""JSON Flattener transformer for enriching records with JSON field data. + +This transformer flattens JSON objects into individual columns without creating +new rows, allowing you to enrich existing records with structured data from +JSON blobs and then map those enriched fields to different graph nodes. +""" + +import logging +import json +import pandas as pd +from typing import List, Dict, Any, Union +from document_graph.transform.transformer_provider_base import TransformerProvider + +logger = logging.getLogger(__name__) + + +class JSONFlattenerProvider(TransformerProvider): + """Flattens JSON objects into columns to enrich records. + + This transformer takes JSON objects/arrays and flattens them into individual + columns with a configurable prefix. It preserves the original row count while + adding new columns for each JSON field. + + Args: + field: The field containing JSON data (required) + prefix: Prefix for flattened fields (default: field name without spaces) + flatten_nested: Flatten nested objects with dot notation (default: True) + max_depth: Maximum nesting depth (default: 3) + preserve_original_field: Keep original JSON field (default: False) + handle_arrays: How to handle arrays - "first", "join", "string" (default: "first") + """ + + def __init__(self, config): + super().__init__(config) + self.field = self.args.get('field') or getattr(config, 'csv_property_name', None) + if not self.field: + raise ValueError("JSON Flattener requires 'field' argument or csv_property_name") + + # Use field name as default prefix (no underscore separator) + if 'prefix' not in self.args: + # Clean field name: remove spaces and special chars + clean_field = ''.join(c if c.isalnum() else '' for c in self.field) + self.prefix = clean_field + logger.info(f"JSON Flattener using default prefix: '{self.prefix}' (from field '{self.field}')") + else: + self.prefix = self.args.get('prefix') + logger.info(f"JSON Flattener using explicit prefix: '{self.prefix}'") + self.flatten_nested = self.args.get('flatten_nested', True) + self.max_depth = self.args.get('max_depth', 3) + self.preserve_original_field = self.args.get('preserve_original_field', False) + self.handle_arrays = self.args.get('handle_arrays', 'first') # first, join, string + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Transform records by flattening JSON into columns.""" + enriched_records = [] + + for record in records: + try: + json_data = self._extract_json_data(record) + enriched_record = self._enrich_record(record, json_data) + enriched_records.append(enriched_record) + except Exception as e: + logger.warning(f"Failed to process JSON in field '{self.field}': {e}") + enriched_records.append(record.copy()) + + return enriched_records + + def _extract_json_data(self, record: Dict[str, Any]) -> Union[Dict, List, None]: + """Extract and parse JSON data from the specified field.""" + field_value = record.get(self.field) + + if field_value is None or field_value == '': + return None + + if isinstance(field_value, str): + if field_value.strip() == '': + return None + try: + return json.loads(field_value) + except json.JSONDecodeError as e: + logger.warning(f"Invalid JSON in field '{self.field}': {e}") + return None + elif isinstance(field_value, (list, dict)): + return field_value + else: + logger.warning(f"Unsupported data type in field '{self.field}': {type(field_value)}") + return None + + def _enrich_record(self, original_record: Dict[str, Any], json_data: Any) -> Dict[str, Any]: + """Enrich record with flattened JSON data.""" + # Start with original record + if self.preserve_original_field: + enriched_record = original_record.copy() + else: + enriched_record = {k: v for k, v in original_record.items() if k != self.field} + + if json_data is None: + return enriched_record + + # Handle arrays by taking first item or converting to string + if isinstance(json_data, list): + if len(json_data) == 0: + return enriched_record + elif self.handle_arrays == 'first': + json_data = json_data[0] # Take first item + elif self.handle_arrays == 'string': + # Convert entire array to string + flattened_data = {f"{self.prefix}array": str(json_data)} + enriched_record.update(flattened_data) + return enriched_record + elif self.handle_arrays == 'numbered': + # Create numbered columns like comma_flattener + for i, item in enumerate(json_data, 1): + if i > 10: # Limit to 10 items + break + formatted_num = str(i).zfill(2) + column_name = f"{self.prefix} {formatted_num}" + if isinstance(item, dict): + enriched_record[column_name] = json.dumps(item) + else: + enriched_record[column_name] = str(item) + return enriched_record + if self.handle_arrays == 'join' and all(isinstance(x, str) for x in json_data): + # Join string arrays + flattened_data = {f"{self.prefix}array": ', '.join(json_data)} + enriched_record.update(flattened_data) + return enriched_record + else: + json_data = json_data[0] # Fallback to first + + # Flatten the JSON object + if isinstance(json_data, dict): + flattened_data = self._flatten_json_object(json_data) + # Add prefix to all keys + prefixed_data = {f"{self.prefix}{key}": value for key, value in flattened_data.items()} + enriched_record.update(prefixed_data) + else: + # Handle primitive values + enriched_record[f"{self.prefix}value"] = json_data + + return enriched_record + + def _flatten_json_object(self, json_obj: Dict[str, Any], parent_key: str = '', depth: int = 0) -> Dict[str, Any]: + """Flatten nested JSON object using dot notation.""" + if not self.flatten_nested or depth >= self.max_depth: + return json_obj + + flattened = {} + + for key, value in json_obj.items(): + new_key = f"{parent_key}.{key}" if parent_key else key + + if isinstance(value, dict) and depth < self.max_depth: + # Recursively flatten nested objects + nested_flattened = self._flatten_json_object(value, new_key, depth + 1) + flattened.update(nested_flattened) + elif isinstance(value, list): + # Handle arrays based on configuration + if self.handle_arrays == 'first' and len(value) > 0: + if isinstance(value[0], dict): + nested_flattened = self._flatten_json_object(value[0], new_key, depth + 1) + flattened.update(nested_flattened) + else: + flattened[new_key] = value[0] + elif self.handle_arrays == 'join' and all(isinstance(x, str) for x in value): + flattened[new_key] = ', '.join(value) + else: + flattened[new_key] = str(value) + else: + flattened[new_key] = value + + return flattened \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/field_transformers/json_value_flattener.py b/document-graph/src/document_graph/transform/field_transformers/json_value_flattener.py new file mode 100644 index 00000000..8035396d --- /dev/null +++ b/document-graph/src/document_graph/transform/field_transformers/json_value_flattener.py @@ -0,0 +1,53 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""JSON Value Flattener for creating JSON objects from paired fields.""" + +import json +import logging +from typing import List, Dict, Any +from document_graph.transform.transformer_provider_base import TransformerProvider + +logger = logging.getLogger(__name__) + + +class JSONValueFlattenerProvider(TransformerProvider): + """Creates JSON objects from paired comma-separated fields. + + Creates columns like: project_01: {"id": "123", "name": "Project A"} + """ + + def __init__(self, config): + super().__init__(config) + self.id_field = self.args.get('id_field') + self.name_field = self.args.get('name_field') + self.prefix = self.args.get('prefix', 'project') + self.suffix = self.args.get('suffix', '#00') + self.max_items = self.args.get('max_items', 10) + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + enriched_records = [] + + for record in records: + enriched_record = record.copy() + + ids = self._split_field(record.get(self.id_field, '')) + names = self._split_field(record.get(self.name_field, '')) + + max_len = min(max(len(ids), len(names)), self.max_items) + + for i in range(max_len): + num = str(i + 1).zfill(2) + project_data = { + "id": ids[i] if i < len(ids) else "", + "name": names[i] if i < len(names) else "" + } + enriched_record[f"{self.prefix}_{num}"] = json.dumps(project_data) + + enriched_records.append(enriched_record) + + return enriched_records + + def _split_field(self, value: str) -> List[str]: + if not value: + return [] + return [item.strip() for item in str(value).split(',') if item.strip()] \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/field_transformers/normalize_timestamp.py b/document-graph/src/document_graph/transform/field_transformers/normalize_timestamp.py new file mode 100644 index 00000000..47782c70 --- /dev/null +++ b/document-graph/src/document_graph/transform/field_transformers/normalize_timestamp.py @@ -0,0 +1,86 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Timestamp normalization transformer for document graph operations. + +This module provides a transformer that converts timestamp fields from various formats +into a standardized ISO 8601 format. This normalization enables consistent timestamp +handling, comparison, and querying across different data sources that may use +different timestamp formats. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from typing import List, Dict, Any +from datetime import datetime +from dateutil import parser +from document_graph.transform.transformer_provider_base import TransformerProvider + + +class TimestampNormalizer(TransformerProvider): + """Normalizes timestamp fields to ISO 8601 format. + + This transformer converts timestamp fields from various formats into a standardized + ISO 8601 format (YYYY-MM-DDTHH:MM:SS.sssZ). It uses the dateutil parser to handle + a wide variety of input formats, making it flexible for different data sources. + + Attributes: + config: Configuration object for the transformer + name: Name of the transformer (from config) + args: Arguments for the transformer (from config) + - field: The field containing the timestamp to normalize (default: "timestamp") + - output_field: The field to store the normalized timestamp + (default: "{field}_normalized") + + Examples: + >>> # Create a transformer to normalize dates in the "created_at" field + >>> config = TransformerProviderConfig( + ... name="date_normalizer", + ... args={"field": "created_at", "output_field": "iso_date"} + ... ) + >>> transformer = TimestampNormalizer(config) + >>> result = transformer.transform([ + ... {"id": 1, "created_at": "Jan 1, 2023 14:30:45"}, + ... {"id": 2, "created_at": "2023/02/15 09:15 AM"} + ... ]) + >>> result[0]["iso_date"] + '2023-01-01T14:30:45' + >>> result[1]["iso_date"] + '2023-02-15T09:15:00' + """ + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Transform records by normalizing timestamp fields to ISO 8601 format. + + Args: + records: List of record dictionaries to transform + + Returns: + List of transformed record dictionaries with normalized timestamp fields + + Note: + If timestamp parsing fails, an error field will be added to the record + with the output field name and "_error" suffix. + """ + field = self.args.get("field", "timestamp") + output_field = self.args.get("output_field", f"{field}_normalized") + + normalized_records = [] + for record in records: + normalized_record = record.copy() + raw_value = record.get(field) + + if raw_value: + try: + dt: datetime = parser.parse(str(raw_value)) + normalized_record[output_field] = dt.isoformat() + except Exception as e: + normalized_record[f"{output_field}_error"] = str(e) + + normalized_records.append(normalized_record) + + return normalized_records \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/field_transformers/paired_flattener.py b/document-graph/src/document_graph/transform/field_transformers/paired_flattener.py new file mode 100644 index 00000000..a5eab4fc --- /dev/null +++ b/document-graph/src/document_graph/transform/field_transformers/paired_flattener.py @@ -0,0 +1,56 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Paired Flattener transformer for flattening two related comma-separated fields.""" + +import logging +from typing import List, Dict, Any +from document_graph.transform.transformer_provider_base import TransformerProvider + +logger = logging.getLogger(__name__) + + +class PairedFlattenerProvider(TransformerProvider): + """Flattens two related comma-separated fields into paired columns. + + Creates columns like: project_01_id, project_01_name, project_02_id, project_02_name + """ + + def __init__(self, config): + super().__init__(config) + self.id_field = self.args.get('id_field') + self.name_field = self.args.get('name_field') + self.prefix = self.args.get('prefix', 'project') + self.suffix = self.args.get('suffix', '#00') + self.separator = self.args.get('separator', ',') + self.max_items = self.args.get('max_items', 10) + + if not self.id_field or not self.name_field: + raise ValueError("PairedFlattener requires both 'id_field' and 'name_field'") + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + enriched_records = [] + + for record in records: + enriched_record = record.copy() + + ids = self._split_field(record.get(self.id_field, '')) + names = self._split_field(record.get(self.name_field, '')) + + # Pair up IDs and names (pad shorter list with empty strings) + max_len = min(max(len(ids), len(names)), self.max_items) + ids.extend([''] * (max_len - len(ids))) + names.extend([''] * (max_len - len(names))) + + for i in range(max_len): + num = str(i + 1).zfill(2) if '#00' in self.suffix else str(i + 1) + enriched_record[f"{self.prefix}_{num}_id"] = ids[i] + enriched_record[f"{self.prefix}_{num}_name"] = names[i] + + enriched_records.append(enriched_record) + + return enriched_records + + def _split_field(self, value: str) -> List[str]: + if not value: + return [] + return [item.strip() for item in str(value).split(self.separator) if item.strip()] \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/field_transformers/regex_cleaner_provider.py b/document-graph/src/document_graph/transform/field_transformers/regex_cleaner_provider.py new file mode 100644 index 00000000..78164119 --- /dev/null +++ b/document-graph/src/document_graph/transform/field_transformers/regex_cleaner_provider.py @@ -0,0 +1,98 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Regex-based text cleaning transformer for document graph operations. + +This module provides a transformer that applies regular expression patterns to clean +and standardize text fields in records. It can be used to remove unwanted characters, +normalize formats, extract specific patterns, or perform other text cleaning operations +using the power and flexibility of regular expressions. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from typing import List, Dict, Any, Pattern +import re +from document_graph.transform.transformer_provider_base import TransformerProvider + + +class RegexCleanerProvider(TransformerProvider): + """Applies regex patterns to clean text fields in records. + + This transformer applies a list of regular expression patterns to specified fields + in each record, replacing matches with a configured replacement string. This is useful + for standardizing text formats, removing unwanted characters or patterns, and + performing other text cleaning operations. + + Attributes: + config: Configuration object for the transformer + name: Name of the transformer (from config) + args: Arguments for the transformer (from config) + - patterns: List of regex patterns to apply (default: []) + - replacement: String to replace matches with (default: "") + - fields: List of fields to clean + replacement: The replacement string to use + _compiled_patterns: List of compiled regex pattern objects + + Examples: + >>> # Create a transformer to remove HTML tags from text fields + >>> config = TransformerProviderConfig( + ... name="html_cleaner", + ... args={ + ... "patterns": ["<[^>]*>"], + ... "replacement": "", + ... "fields": ["description", "content"] + ... } + ... ) + >>> transformer = RegexCleanerProvider(config) + >>> result = transformer.transform([ + ... {"id": 1, "description": "

This is a sample text.

"} + ... ]) + >>> result[0]["description"] + 'This is a sample text.' + """ + + def __init__(self, config): + """Initialize the regex cleaner with configuration. + + Args: + config: Transformer configuration with name, type, and args + - patterns: List of regex patterns to apply + - replacement: String to replace matches with + - fields: List of fields to clean + """ + super().__init__(config) + patterns = self.args.get('patterns', []) + self.replacement = self.args.get('replacement', '') + self._compiled_patterns: List[Pattern] = [re.compile(p) for p in patterns] + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Transform records by applying regex patterns to specified fields. + + Args: + records: List of record dictionaries to transform + + Returns: + List of transformed record dictionaries with cleaned text fields + + Note: + The transformation modifies the specified fields in-place in each record. + Only string fields are processed; non-string fields are left unchanged. + """ + fields_to_clean = self.args.get('fields', []) + + cleaned_records = [] + for record in records: + cleaned_record = record.copy() + for field in fields_to_clean: + if field in cleaned_record and isinstance(cleaned_record[field], str): + result = cleaned_record[field] + for pattern in self._compiled_patterns: + result = pattern.sub(self.replacement, result) + cleaned_record[field] = result + cleaned_records.append(cleaned_record) + return cleaned_records \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/field_transformers/standardize_enum.py b/document-graph/src/document_graph/transform/field_transformers/standardize_enum.py new file mode 100644 index 00000000..cebce3c5 --- /dev/null +++ b/document-graph/src/document_graph/transform/field_transformers/standardize_enum.py @@ -0,0 +1,104 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Enumeration standardization transformer for document graph operations. + +This module provides a transformer that maps field values to standardized enumeration values +using a mapping dictionary. This is useful for normalizing categorical data that may have +different representations across data sources, ensuring consistent values for analysis, +querying, and visualization. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from typing import List, Dict, Any +from document_graph.transform.transformer_provider_base import TransformerProvider + + +class EnumStandardizer(TransformerProvider): + """Maps field values to standard enumeration using a mapping dictionary. + + This transformer standardizes categorical data by mapping input values to a set of + predefined standard values using a mapping dictionary. This is useful for normalizing + data from different sources that may use different terminology for the same concepts. + + Attributes: + config: Configuration object for the transformer + name: Name of the transformer (from config) + args: Arguments for the transformer (from config) + - field: The field containing values to standardize (default: "category") + - mapping: Dictionary mapping input values to standardized values (default: {}) + - case_insensitive: Whether to ignore case when looking up mappings (default: True) + - output_field: The field to store the standardized value (default: same as input field) + + Examples: + >>> # Create a transformer to standardize status values + >>> config = TransformerProviderConfig( + ... name="status_standardizer", + ... args={ + ... "field": "status", + ... "mapping": { + ... "active": "ACTIVE", + ... "enabled": "ACTIVE", + ... "inactive": "INACTIVE", + ... "disabled": "INACTIVE", + ... "pending": "PENDING" + ... }, + ... "case_insensitive": True + ... } + ... ) + >>> transformer = EnumStandardizer(config) + >>> result = transformer.transform([ + ... {"id": 1, "status": "Active"}, + ... {"id": 2, "status": "DISABLED"}, + ... {"id": 3, "status": "unknown"} + ... ]) + >>> result[0]["status"] + 'ACTIVE' + >>> result[1]["status"] + 'INACTIVE' + >>> result[2]["status_unmapped"] + 'unknown' + """ + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Transform records by standardizing enumeration values. + + Args: + records: List of record dictionaries to transform + + Returns: + List of transformed record dictionaries with standardized enumeration values + + Note: + If a value cannot be mapped (not found in the mapping dictionary), + it will be stored in a separate field with the "_unmapped" suffix. + When case_insensitive is True, all keys in the mapping dictionary + are compared in lowercase. + """ + field = self.args.get("field", "category") + mapping: Dict[str, str] = self.args.get("mapping", {}) + case_insensitive: bool = self.args.get("case_insensitive", True) + output_field = self.args.get("output_field", field) + + standardized_records = [] + for record in records: + standardized_record = record.copy() + original = record.get(field) + + if original: + key = str(original).lower() if case_insensitive else str(original) + mapped_value = mapping.get(key) + + if mapped_value: + standardized_record[output_field] = mapped_value + else: + standardized_record[f"{output_field}_unmapped"] = original + + standardized_records.append(standardized_record) + + return standardized_records \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/field_transformers/uuid_generator.py b/document-graph/src/document_graph/transform/field_transformers/uuid_generator.py new file mode 100644 index 00000000..292218b0 --- /dev/null +++ b/document-graph/src/document_graph/transform/field_transformers/uuid_generator.py @@ -0,0 +1,22 @@ +# Copyright (c) Evan Erwee. All rights reserved. +"""UUID generator — generates UUID values for specified fields.""" + +import uuid +from typing import List, Dict, Any +from document_graph.transform.transformer_provider_base import TransformerProvider +from document_graph.transform.transformer_provider_config import TransformerProviderConfig + + +class UuidGeneratorTransformer(TransformerProvider): + """Generates UUID values for specified fields""" + + def __init__(self, config: TransformerProviderConfig): + super().__init__(config) + self.target_field = config.args.get('target_field', 'uuid') + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + self._log_transform_start(len(records)) + for record in records: + record[self.target_field] = str(uuid.uuid4()) + self._log_transform_end(len(records), len(records)) + return records \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/filter_transformers/__init__.py b/document-graph/src/document_graph/transform/filter_transformers/__init__.py new file mode 100644 index 00000000..9d465f40 --- /dev/null +++ b/document-graph/src/document_graph/transform/filter_transformers/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Filter transformers module for document graph operations. + +This package provides transformer implementations that filter or modify data +during document graph processing. It includes transformers for: + +- Column pruning: Removes specified columns (fields) from records +- Row filtering: Filters records based on field conditions with various operators + such as equality, inequality, comparison, and null checks + +These transformers help in data preparation and cleaning by allowing selective +processing of records and fields based on configurable criteria. +""" + +from document_graph.transform.filter_transformers.column_pruner import ColumnPrunerProvider +from document_graph.transform.filter_transformers.row_filter import RowFilterProvider + +__all__ = [ + "ColumnPrunerProvider", + "RowFilterProvider", +] \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/filter_transformers/column_pruner.py b/document-graph/src/document_graph/transform/filter_transformers/column_pruner.py new file mode 100644 index 00000000..c4cc3f59 --- /dev/null +++ b/document-graph/src/document_graph/transform/filter_transformers/column_pruner.py @@ -0,0 +1,54 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Column Pruner module for document graph operations. + +This module provides the ColumnPrunerProvider transformer, which removes specified +columns (fields) from records during document graph processing. This is useful for +data preparation and cleaning by allowing selective processing of fields based on +configurable criteria. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from typing import List, Dict, Any +from document_graph.transform.transformer_provider_base import TransformerProvider + +class ColumnPrunerProvider(TransformerProvider): + """Transformer provider that removes specified columns from records. + + This transformer allows for selective field processing by removing specified + columns (fields) from each record. It's useful for data preparation and cleaning + when certain fields are not needed for downstream processing. + + Args: + The provider expects the following arguments in the `args` dictionary: + remove (List[str]): A list of column names to remove from each record. + If not provided, no columns will be removed. + """ + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Remove specified columns from each record. + + This method processes a list of records (dictionaries) and removes the + specified columns from each record. The columns to remove are specified + in the `remove` argument of the provider. + + Args: + records (List[Dict[str, Any]]): A list of records to process, where each + record is a dictionary with string keys. + + Returns: + List[Dict[str, Any]]: A new list of records with the specified columns removed. + The original records are not modified. + """ + columns_to_remove: List[str] = self.args.get("remove", []) + + return [ + {k: v for k, v in record.items() if k not in columns_to_remove} + for record in records + ] diff --git a/document-graph/src/document_graph/transform/filter_transformers/row_filter.py b/document-graph/src/document_graph/transform/filter_transformers/row_filter.py new file mode 100644 index 00000000..981d9f9a --- /dev/null +++ b/document-graph/src/document_graph/transform/filter_transformers/row_filter.py @@ -0,0 +1,107 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Row Filter module for document graph operations. + +This module provides the RowFilterProvider transformer, which filters records based on +field conditions during document graph processing. It supports various comparison +operators such as equality, inequality, comparison, and null checks. + +This transformer is useful for data preparation and cleaning by allowing selective +processing of records based on configurable criteria. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from typing import List, Dict, Any +from document_graph.transform.transformer_provider_base import TransformerProvider + + +class RowFilterProvider(TransformerProvider): + """Transformer provider that filters records based on field conditions. + + This transformer allows for selective record processing by filtering records + based on field conditions. It supports various comparison operators such as + equality, inequality, comparison, and null checks. Records are kept only if + they satisfy all the specified conditions. + + Args: + The provider expects the following arguments in the `args` dictionary: + conditions (List[Dict[str, Any]]): A list of condition dictionaries. + Each condition dictionary must have the following keys: + - field (str): The name of the field to check. + - operator (str): The comparison operator to use. Supported operators are: + - "eq": Equal to + - "ne": Not equal to + - "lt": Less than + - "gt": Greater than + - "null": Field is null + - "not_null": Field is not null + - value (Any): The value to compare against. Required for "eq", "ne", + "lt", and "gt" operators. Not used for "null" and "not_null" operators. + + Raises: + ValueError: If an unsupported operator is specified. + """ + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Filter records based on field conditions. + + This method processes a list of records (dictionaries) and returns only the + records that satisfy all the specified conditions. The conditions are specified + in the `conditions` argument of the provider. + + If no conditions are specified, all records are returned unchanged. + + Args: + records (List[Dict[str, Any]]): A list of records to process, where each + record is a dictionary with string keys. + + Returns: + List[Dict[str, Any]]: A new list containing only the records that satisfy + all the specified conditions. + + Raises: + ValueError: If an unsupported operator is specified in a condition. + """ + conditions: List[Dict[str, Any]] = self.args.get("conditions", []) + if not conditions: + return records + + filtered_records = [] + for record in records: + keep_record = True + + for cond in conditions: + field = cond["field"] + op = cond["operator"] + value = cond.get("value") + field_value = record.get(field) + + if op == "eq" and field_value != value: + keep_record = False + elif op == "ne" and field_value == value: + keep_record = False + elif op == "lt" and not (field_value is not None and field_value < value): + keep_record = False + elif op == "gt" and not (field_value is not None and field_value > value): + keep_record = False + elif op == "null" and field_value is not None: + keep_record = False + elif op == "not_null" and field_value is None: + keep_record = False + else: + if op not in ["eq", "ne", "lt", "gt", "null", "not_null"]: + raise ValueError(f"Unsupported operator: {op}") + + if not keep_record: + break + + if keep_record: + filtered_records.append(record) + + return filtered_records \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/graph_transformers/__init__.py b/document-graph/src/document_graph/transform/graph_transformers/__init__.py new file mode 100644 index 00000000..583bd037 --- /dev/null +++ b/document-graph/src/document_graph/transform/graph_transformers/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Graph transformers module for document graph operations. + +This package provides transformer implementations that create and manipulate graph +structures during document graph processing. It includes transformers for: + +- Edge inference: Creates edges between records based on matching field values +- Row to node conversion: Transforms tabular records into graph nodes with appropriate metadata + +These transformers help in building graph representations from structured data by +converting records to nodes and establishing relationships between them based on +configurable criteria. +""" + +from document_graph.transform.graph_transformers.infer_edges import EdgeInferencer +from document_graph.transform.graph_transformers.row_to_node import RowToNodeTransformer + +__all__ = [ + "EdgeInferencer", + "RowToNodeTransformer", +] \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/graph_transformers/infer_edges.py b/document-graph/src/document_graph/transform/graph_transformers/infer_edges.py new file mode 100644 index 00000000..1a766e24 --- /dev/null +++ b/document-graph/src/document_graph/transform/graph_transformers/infer_edges.py @@ -0,0 +1,78 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Infer Edges module for document graph operations. + +This module provides the EdgeInferencer transformer, which creates edges between +records based on matching field values. This is useful for building graph +representations from structured data by establishing relationships between +records that share common attribute values. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from typing import List, Dict, Any +from document_graph.transform.transformer_provider_base import TransformerProvider + + +class EdgeInferencer(TransformerProvider): + """Transformer provider that infers edges between records based on matching fields. + + This transformer creates edge records between records that share the same value + in a specified field. It's useful for building graph representations from + structured data by establishing relationships between related records. + + Args: + The provider expects the following arguments in the `args` dictionary: + source_field (str): The field name to use for grouping records. + Default is "project_id". + edge_type (str): The relationship type to assign to the created edges. + Default is "RELATED_TO". + """ + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Create edge records between records that share the same field value. + + This method processes a list of records and creates edge records between + consecutive records that share the same value in the specified source field. + The edges are created with the specified relationship type. + + Args: + records (List[Dict[str, Any]]): A list of records to process, where each + record is a dictionary with string keys. + + Returns: + List[Dict[str, Any]]: A new list containing the original records plus + the newly created edge records. + """ + source_field = self.args.get("source_field", "project_id") + edge_type = self.args.get("edge_type", "RELATED_TO") + + # Group records by source_field + groups = {} + for record in records: + key = record.get(source_field) + if key is not None: + groups.setdefault(key, []).append(record) + + # Create edge records within each group + edge_records = [] + for group_records in groups.values(): + for i in range(len(group_records) - 1): + source = group_records[i] + target = group_records[i + 1] + + edge_record = { + "edge_type": "edge", + "source_id": source.get("id", f"record_{i}"), + "target_id": target.get("id", f"record_{i+1}"), + "relationship": edge_type, + "group_key": key + } + edge_records.append(edge_record) + + return records + edge_records # Return original records plus inferred edges \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/graph_transformers/row_to_node.py b/document-graph/src/document_graph/transform/graph_transformers/row_to_node.py new file mode 100644 index 00000000..a899d2a4 --- /dev/null +++ b/document-graph/src/document_graph/transform/graph_transformers/row_to_node.py @@ -0,0 +1,72 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Row To Node module for document graph operations. + +This module provides the RowToNodeTransformer, which converts tabular records +into graph nodes with appropriate metadata. This is useful for building graph +representations from structured data by transforming records into nodes that +can be used in a graph database or other graph-based applications. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from typing import List, Dict, Any +from document_graph.transform.transformer_provider_base import TransformerProvider + + +class RowToNodeTransformer(TransformerProvider): + """Transformer provider that converts records to node format with graph metadata. + + This transformer transforms tabular records into graph nodes by adding + necessary graph metadata such as node type and graph element identifier. + It also ensures each node has an ID and creates a content field from all + other fields if one doesn't exist. + + Args: + The provider expects the following arguments in the `args` dictionary: + type (str): The node type to assign to the created nodes. + Default is "Row". + """ + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Convert records to node format with graph metadata. + + This method processes a list of records and converts each one to a node + format by adding graph metadata such as node_type and graph_element. + It ensures each node has an ID and creates a content field from all + other fields if one doesn't exist. + + Args: + records (List[Dict[str, Any]]): A list of records to process, where each + record is a dictionary with string keys. + + Returns: + List[Dict[str, Any]]: A new list of records converted to node format. + """ + node_type = self.args.get("type", "Row") + + node_records = [] + for idx, record in enumerate(records): + node_record = record.copy() + + # Add graph metadata + node_record["node_type"] = node_type + node_record["graph_element"] = "node" + + # Ensure ID exists + if "id" not in node_record: + node_record["id"] = f"row_{idx}" + + # Create content from all fields if not exists + if "content" not in node_record: + content_parts = [f"{k}: {v}" for k, v in record.items() if k != "id"] + node_record["content"] = ", ".join(content_parts) + + node_records.append(node_record) + + return node_records \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/normalizers/__init__.py b/document-graph/src/document_graph/transform/normalizers/__init__.py new file mode 100644 index 00000000..1aa401d4 --- /dev/null +++ b/document-graph/src/document_graph/transform/normalizers/__init__.py @@ -0,0 +1,32 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Normalizer transformers for standardizing data formats. + +This module provides transformers that normalize and standardize data +formats and values, such as: +- Case normalization: Standardize text case (upper, lower, title) +- Enum normalization: Standardize enumerated values +- Null normalization: Handle null and empty values consistently +- Spelling normalization: Correct common spelling errors +- Whitespace normalization: Clean up whitespace and formatting +- Timestamp normalization: Standardize date/time formats + +These transformers help ensure data consistency and quality during +document processing pipelines. +""" + +from .normalize_case_provider import NormalizeCaseProvider +from .normalize_enum_provider import NormalizeEnumProvider +from .normalize_nulls_provider import NormalizeNullsProvider +from .normalize_spelling_provider import NormalizeSpellingProvider +from .normalize_whitespace_provider import NormalizeWhitespaceProvider +from .normalize_timestamp_provider import NormalizeTimestampProvider + +__all__ = [ + 'NormalizeCaseProvider', + 'NormalizeEnumProvider', + 'NormalizeNullsProvider', + 'NormalizeSpellingProvider', + 'NormalizeWhitespaceProvider', + 'NormalizeTimestampProvider' +] \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/normalizers/normalize_case.py b/document-graph/src/document_graph/transform/normalizers/normalize_case.py new file mode 100644 index 00000000..5f9e72bb --- /dev/null +++ b/document-graph/src/document_graph/transform/normalizers/normalize_case.py @@ -0,0 +1,41 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Normalize Case module for document graph operations. + +This module provides functionality for normalizing the case of text strings +in document processing pipelines. It supports: + +- Converting text to lowercase (default) +- Converting text to uppercase +- Other case transformations supported by Python string methods + +Case normalization is useful for standardizing text data, improving search +and matching operations, and preparing text for further processing. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +def normalize(value: str, mode="lower") -> str: + """Normalize the case of a string value. + + Applies case transformation to the input string using the specified mode. + The mode parameter corresponds to a Python string method name (e.g., 'lower', + 'upper', 'capitalize', 'title'). + + Args: + value (str): The string to normalize + mode (str, optional): The case transformation to apply. Defaults to "lower". + Valid values include 'lower', 'upper', 'capitalize', 'title', etc. + + Returns: + str: The normalized string with the specified case transformation applied + + Raises: + AttributeError: If the specified mode is not a valid string method + """ + return getattr(value, mode)() diff --git a/document-graph/src/document_graph/transform/normalizers/normalize_case_provider.py b/document-graph/src/document_graph/transform/normalizers/normalize_case_provider.py new file mode 100644 index 00000000..8bc7fe99 --- /dev/null +++ b/document-graph/src/document_graph/transform/normalizers/normalize_case_provider.py @@ -0,0 +1,71 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Normalize Case Provider module for document graph operations. + +This module provides a transformer provider implementation for normalizing the case +of text strings in document records. It supports: + +- Converting text to lowercase (default) +- Converting text to uppercase +- Other case transformations supported by Python string methods + +The provider applies case normalization to all string fields in each record, +preserving non-string fields unchanged. This is useful for standardizing text data +across records, improving search and matching operations, and preparing text for +further processing. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from typing import List, Dict, Any +from document_graph.transform.transformer_provider_base import TransformerProvider + + +class NormalizeCaseProvider(TransformerProvider): + """Normalizes text case in record fields. + + This transformer provider applies case normalization to all string fields + in each record, preserving non-string fields unchanged. It supports various + case transformations through the 'mode' configuration parameter. + + Attributes: + config: Configuration object for the transformer + name: Name of the transformer (from config) + args: Arguments for the transformer (from config), including: + - mode: The case transformation to apply (default: 'lower') + """ + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Apply case normalization to string fields in records. + + Processes each record and applies the specified case transformation + (lowercase by default) to all string fields, leaving non-string fields + unchanged. + + Args: + records: List of record dictionaries to transform + + Returns: + List of transformed record dictionaries with normalized case in string fields + """ + self._log_transform_start(len(records)) + + mode = self.args.get('mode', 'lower') + + transformed_records = [] + for record in records: + transformed_record = {} + for key, value in record.items(): + if isinstance(value, str): + transformed_record[key] = getattr(value, mode)() + else: + transformed_record[key] = value + transformed_records.append(transformed_record) + + self._log_transform_end(len(records), len(transformed_records)) + return transformed_records \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/normalizers/normalize_enum.py b/document-graph/src/document_graph/transform/normalizers/normalize_enum.py new file mode 100644 index 00000000..1918e4b1 --- /dev/null +++ b/document-graph/src/document_graph/transform/normalizers/normalize_enum.py @@ -0,0 +1,61 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Normalize Enum module for document graph operations. + +This module provides functionality for normalizing enumeration values in text data +using a mapping dictionary. It supports: + +- Case-insensitive mapping of input values to standardized forms +- Handling of variations in spelling, abbreviations, or formatting +- Fallback to original values when no mapping is found + +Enum normalization is useful for standardizing categorical data, ensuring consistent +terminology across datasets, and improving data quality for analysis and processing. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from typing import Optional, Dict + +class EnumNormalizer: + """Normalizes enumeration values using a mapping dictionary. + + This class provides functionality to standardize categorical or enumeration values + by mapping input strings to standardized forms using a dictionary. It handles + case-insensitive matching and preserves the original value when no mapping is found. + + Attributes: + enum_map (Dict[str, str]): Dictionary mapping input values (lowercase) to + their standardized forms + """ + + def __init__(self, enum_map: Dict[str, str]): + """Initialize the enum normalizer with a mapping dictionary. + + Args: + enum_map (Dict[str, str]): Dictionary mapping input values to their + standardized forms. Keys will be converted to lowercase for + case-insensitive matching. + """ + self.enum_map = {k.lower(): v for k, v in enum_map.items()} + + def normalize(self, value: str) -> Optional[str]: + """Normalize an enumeration value using the mapping dictionary. + + Looks up the input value (after stripping whitespace and converting to lowercase) + in the mapping dictionary and returns the standardized form if found. + If not found, returns the original value unchanged. + + Args: + value (str): The string value to normalize + + Returns: + Optional[str]: The normalized value if found in the mapping dictionary, + otherwise the original value + """ + return self.enum_map.get(value.strip().lower(), value) \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/normalizers/normalize_enum_provider.py b/document-graph/src/document_graph/transform/normalizers/normalize_enum_provider.py new file mode 100644 index 00000000..6dcb1cb0 --- /dev/null +++ b/document-graph/src/document_graph/transform/normalizers/normalize_enum_provider.py @@ -0,0 +1,76 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Normalize Enum Provider module for document graph operations. + +This module provides a transformer provider implementation for normalizing enumeration +values in document records using a mapping dictionary. It supports: + +- Standardizing categorical field values using a configurable mapping +- Case-insensitive matching (optional, enabled by default) +- Handling of variations in spelling, abbreviations, or formatting +- Preserving original values when no mapping is found + +Enum normalization is useful for standardizing categorical data, ensuring consistent +terminology across datasets, and improving data quality for analysis and processing. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from typing import List, Dict, Any +from document_graph.transform.transformer_provider_base import TransformerProvider + + +class NormalizeEnumProvider(TransformerProvider): + """Normalizes enum values using a mapping dictionary. + + This transformer provider standardizes categorical field values in document records + using a configurable mapping dictionary. It supports case-insensitive matching + and preserves original values when no mapping is found. + + Attributes: + config: Configuration object for the transformer + name: Name of the transformer (from config) + args: Arguments for the transformer (from config), including: + - field: The field to normalize (default: "category") + - enum_map: Dictionary mapping input values to standardized forms + - case_insensitive: Whether to use case-insensitive matching (default: True) + """ + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Transform records by normalizing enum values in the specified field. + + Processes each record and applies enum normalization to the specified field + using the provided mapping dictionary. If case_insensitive is True (default), + the matching is done after converting keys and values to lowercase. + + Args: + records: List of record dictionaries to transform + + Returns: + List of transformed record dictionaries with normalized enum values + """ + field = self.args.get("field", "category") + enum_map = self.args.get("enum_map", {}) + case_insensitive = self.args.get("case_insensitive", True) + + # Normalize enum_map keys to lowercase if case_insensitive + if case_insensitive: + enum_map = {k.lower(): v for k, v in enum_map.items()} + + normalized_records = [] + for record in records: + normalized_record = record.copy() + value = record.get(field) + + if value: + lookup_key = str(value).strip().lower() if case_insensitive else str(value).strip() + normalized_record[field] = enum_map.get(lookup_key, value) + + normalized_records.append(normalized_record) + + return normalized_records \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/normalizers/normalize_nulls.py b/document-graph/src/document_graph/transform/normalizers/normalize_nulls.py new file mode 100644 index 00000000..592848d0 --- /dev/null +++ b/document-graph/src/document_graph/transform/normalizers/normalize_nulls.py @@ -0,0 +1,54 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Normalize Nulls module for document graph operations. + +This module provides functionality for normalizing null-like values in text data +to actual None values. It supports: + +- Recognition of common null-like string representations (e.g., "n/a", "null", "none") +- Case-insensitive matching of null-like strings +- Conversion of null-like strings to Python None + +Null normalization is useful for standardizing missing data representation, +improving data quality, and ensuring consistent handling of null values in +downstream processing and analysis. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from typing import Optional, Set + +class NullNormalizer: + """Normalizes null-like string values to actual None values. + + This class provides functionality to identify and convert common string + representations of null or missing values to Python None. It uses a predefined + set of null-like strings for matching. + + Attributes: + NULL_LIKE (Set[str]): Set of string values that should be considered as null + """ + + NULL_LIKE = {"", "n/a", "na", "null", "none", "-"} + + def normalize(self, value: str) -> Optional[str]: + """Normalize a string value, converting null-like strings to None. + + Checks if the input value is None or matches any of the predefined null-like + strings (after stripping whitespace and converting to lowercase). If it does, + returns None; otherwise, returns the original value unchanged. + + Args: + value (str): The string value to normalize + + Returns: + Optional[str]: None if the value is null-like, otherwise the original value + """ + if value is None: + return None + return None if value.strip().lower() in self.NULL_LIKE else value \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/normalizers/normalize_nulls_provider.py b/document-graph/src/document_graph/transform/normalizers/normalize_nulls_provider.py new file mode 100644 index 00000000..baa07e53 --- /dev/null +++ b/document-graph/src/document_graph/transform/normalizers/normalize_nulls_provider.py @@ -0,0 +1,73 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Normalize Nulls Provider module for document graph operations. + +This module provides a transformer provider implementation for normalizing null-like +values in document records to actual None values. It supports: + +- Recognition of common null-like string representations (e.g., "n/a", "null", "none") +- Case-insensitive matching of null-like strings +- Conversion of null-like strings to Python None +- Configurable list of fields to process +- Customizable set of null-like string patterns + +Null normalization is useful for standardizing missing data representation, +improving data quality, and ensuring consistent handling of null values in +downstream processing and analysis. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from typing import List, Dict, Any +from document_graph.transform.transformer_provider_base import TransformerProvider + + +class NormalizeNullsProvider(TransformerProvider): + """Normalizes null-like values to actual None. + + This transformer provider identifies and converts common string representations + of null or missing values to Python None in specified fields of document records. + It uses a configurable set of null-like strings for matching. + + Attributes: + config: Configuration object for the transformer + name: Name of the transformer (from config) + args: Arguments for the transformer (from config), including: + - fields: List of field names to process + - null_like: Set of string values that should be considered as null + """ + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Transform records by normalizing null-like values to None. + + Processes each record and converts null-like string values to None in the + specified fields. The matching is done after stripping whitespace and + converting to lowercase. + + Args: + records: List of record dictionaries to transform + + Returns: + List of transformed record dictionaries with normalized null values + """ + fields = self.args.get("fields", []) + null_like = set(self.args.get("null_like", ["", "n/a", "na", "null", "none", "-"])) + + normalized_records = [] + for record in records: + normalized_record = record.copy() + + for field in fields: + if field in normalized_record: + value = normalized_record[field] + if isinstance(value, str) and value.strip().lower() in null_like: + normalized_record[field] = None + + normalized_records.append(normalized_record) + + return normalized_records \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/normalizers/normalize_spelling.py b/document-graph/src/document_graph/transform/normalizers/normalize_spelling.py new file mode 100644 index 00000000..9637a392 --- /dev/null +++ b/document-graph/src/document_graph/transform/normalizers/normalize_spelling.py @@ -0,0 +1,55 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Normalize Spelling module for document graph operations. + +This module provides functionality for correcting spelling errors in text data +using the TextBlob library. It supports: + +- Automatic detection and correction of misspelled words +- Preservation of original text structure and formatting +- Handling of empty or None values + +Spelling normalization is useful for improving text quality, enhancing search +and matching operations, and preparing text for further natural language processing. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +# normalize_spelling.py +from textblob import TextBlob +from typing import Optional + +class SpellingNormalizer: + """Normalizes spelling in text by correcting common spelling errors. + + This class provides functionality to automatically detect and correct + spelling errors in text using the TextBlob library's spell checking + capabilities. + """ + + def normalize(self, value: str) -> Optional[str]: + """Normalize spelling in a text string. + + Uses TextBlob to detect and correct spelling errors in the input text. + If the input is empty or None, returns it unchanged. + + Args: + value (str): The text string to normalize + + Returns: + Optional[str]: The text with corrected spelling, or the original value + if it was empty or None + + Raises: + ImportError: If TextBlob is not installed + Various exceptions from TextBlob if text processing fails + """ + if not value: + return value + blob = TextBlob(value) + return str(blob.correct()) \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/normalizers/normalize_spelling_provider.py b/document-graph/src/document_graph/transform/normalizers/normalize_spelling_provider.py new file mode 100644 index 00000000..9736ac32 --- /dev/null +++ b/document-graph/src/document_graph/transform/normalizers/normalize_spelling_provider.py @@ -0,0 +1,83 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Normalize Spelling Provider module for document graph operations. + +This module provides a transformer provider implementation for correcting spelling +errors in text fields of document records using the TextBlob library. It supports: + +- Automatic detection and correction of misspelled words +- Processing of multiple configurable fields +- Graceful handling of errors during spelling correction +- Preservation of non-string and empty values + +Spelling normalization is useful for improving text quality, enhancing search +and matching operations, and preparing text for further natural language processing. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from typing import List, Dict, Any +from document_graph.transform.transformer_provider_base import TransformerProvider + +try: + from textblob import TextBlob +except ImportError: + TextBlob = None + + +class NormalizeSpellingProvider(TransformerProvider): + """Normalizes spelling using TextBlob. + + This transformer provider corrects spelling errors in text fields of document + records using the TextBlob library's spell checking capabilities. It processes + specified fields in each record and handles errors gracefully. + + Attributes: + config: Configuration object for the transformer + name: Name of the transformer (from config) + args: Arguments for the transformer (from config), including: + - fields: List of field names to process (default: ["content"]) + """ + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Transform records by correcting spelling errors in text fields. + + Processes each record and applies spelling correction to the specified fields + using TextBlob. Non-string values and empty strings are preserved unchanged. + Errors during spelling correction are caught and the original value is kept. + + Args: + records: List of record dictionaries to transform + + Returns: + List of transformed record dictionaries with corrected spelling + + Raises: + ImportError: If TextBlob is not installed + """ + if TextBlob is None: + raise ImportError("TextBlob required for spelling normalization") + + fields = self.args.get("fields", ["content"]) + + normalized_records = [] + for record in records: + normalized_record = record.copy() + + for field in fields: + value = record.get(field) + if value and isinstance(value, str): + try: + blob = TextBlob(value) + normalized_record[field] = str(blob.correct()) + except Exception: + pass # Keep original value on error + + normalized_records.append(normalized_record) + + return normalized_records \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/normalizers/normalize_timestamp.py b/document-graph/src/document_graph/transform/normalizers/normalize_timestamp.py new file mode 100644 index 00000000..fdbbbaa1 --- /dev/null +++ b/document-graph/src/document_graph/transform/normalizers/normalize_timestamp.py @@ -0,0 +1,56 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Normalize Timestamp module for document graph operations. + +This module provides functionality for normalizing timestamp strings to a standard +ISO 8601 format. It supports: + +- Parsing various datetime string formats using dateutil.parser +- Converting parsed datetimes to ISO 8601 format +- Handling of invalid or unparseable datetime strings +- Graceful handling of None values + +Timestamp normalization is useful for standardizing date and time representations, +enabling consistent sorting and comparison operations, and ensuring compatibility +with various systems and databases. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from dateutil import parser +from typing import Optional + +class TimestampNormalizer: + """Normalizes timestamp strings to ISO 8601 format. + + This class provides functionality to parse various datetime string formats + and convert them to a standardized ISO 8601 format, which is widely supported + and enables consistent sorting and comparison operations. + """ + + def normalize(self, value: str) -> Optional[str]: + """Normalize a timestamp string to ISO 8601 format. + + Attempts to parse the input string as a datetime using dateutil.parser + and converts it to ISO 8601 format. If parsing fails, returns None. + + Args: + value (str): The timestamp string to normalize + + Returns: + Optional[str]: The normalized timestamp in ISO 8601 format if parsing + succeeds, None otherwise + + Raises: + No exceptions are raised; parsing errors are caught and None is returned + """ + try: + dt = parser.parse(value) + return dt.isoformat() + except (ValueError, TypeError): + return None \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/normalizers/normalize_timestamp_provider.py b/document-graph/src/document_graph/transform/normalizers/normalize_timestamp_provider.py new file mode 100644 index 00000000..d124380f --- /dev/null +++ b/document-graph/src/document_graph/transform/normalizers/normalize_timestamp_provider.py @@ -0,0 +1,76 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Normalize Timestamp Provider module for document graph operations. + +This module provides a transformer provider implementation for normalizing timestamp +strings in document records to a standard ISO 8601 format. It supports: + +- Parsing various datetime string formats using dateutil.parser +- Converting parsed datetimes to ISO 8601 format +- Processing of multiple configurable fields +- Graceful handling of errors during timestamp parsing +- Preservation of non-string and empty values + +Timestamp normalization is useful for standardizing date and time representations, +enabling consistent sorting and comparison operations, and ensuring compatibility +with various systems and databases. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from typing import List, Dict, Any +from dateutil import parser +from document_graph.transform.transformer_provider_base import TransformerProvider + + +class NormalizeTimestampProvider(TransformerProvider): + """Normalizes timestamp fields to ISO 8601 format. + + This transformer provider parses timestamp strings in various formats and + converts them to a standardized ISO 8601 format. It processes specified fields + in each record and handles parsing errors gracefully. + + Attributes: + config: Configuration object for the transformer + name: Name of the transformer (from config) + args: Arguments for the transformer (from config), including: + - fields: List of field names to process (default: ["timestamp"]) + """ + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Transform records by normalizing timestamp fields to ISO 8601 format. + + Processes each record and attempts to parse timestamp strings in the specified + fields using dateutil.parser, then converts them to ISO 8601 format. Non-string + values are preserved unchanged, and parsing errors are caught to keep the + original value. + + Args: + records: List of record dictionaries to transform + + Returns: + List of transformed record dictionaries with normalized timestamps + """ + fields = self.args.get("fields", ["timestamp"]) + + normalized_records = [] + for record in records: + normalized_record = record.copy() + + for field in fields: + value = record.get(field) + if value: + try: + dt = parser.parse(str(value)) + normalized_record[field] = dt.isoformat() + except Exception: + pass # Keep original value on error + + normalized_records.append(normalized_record) + + return normalized_records \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/normalizers/normalize_whitespace.py b/document-graph/src/document_graph/transform/normalizers/normalize_whitespace.py new file mode 100644 index 00000000..399f66c4 --- /dev/null +++ b/document-graph/src/document_graph/transform/normalizers/normalize_whitespace.py @@ -0,0 +1,42 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Normalize Whitespace module for document graph operations. + +This module provides functionality for normalizing whitespace in text strings. +It supports: + +- Removing leading and trailing whitespace +- Replacing multiple consecutive whitespace characters with a single space +- Standardizing whitespace representation across different text sources + +Whitespace normalization is useful for improving text consistency, enhancing +search and matching operations, and preparing text for further processing or +display. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +import re + +def normalize(value: str) -> str: + """Normalize whitespace in a text string. + + Removes leading and trailing whitespace and replaces multiple consecutive + whitespace characters (spaces, tabs, newlines, etc.) with a single space. + + Args: + value (str): The string to normalize + + Returns: + str: The normalized string with standardized whitespace + + Examples: + >>> normalize(" Hello world!\\n") + 'Hello world!' + """ + return re.sub(r'\s+', ' ', value.strip()) diff --git a/document-graph/src/document_graph/transform/normalizers/normalize_whitespace_provider.py b/document-graph/src/document_graph/transform/normalizers/normalize_whitespace_provider.py new file mode 100644 index 00000000..bfb51f7f --- /dev/null +++ b/document-graph/src/document_graph/transform/normalizers/normalize_whitespace_provider.py @@ -0,0 +1,69 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Normalize Whitespace Provider module for document graph operations. + +This module provides a transformer provider implementation for normalizing whitespace +in text fields of document records. It supports: + +- Removing leading and trailing whitespace +- Replacing multiple consecutive whitespace characters with a single space +- Processing of multiple configurable fields +- Preservation of non-string and empty values + +Whitespace normalization is useful for improving text consistency, enhancing +search and matching operations, and preparing text for further processing or +display. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +import re +from typing import List, Dict, Any +from document_graph.transform.transformer_provider_base import TransformerProvider + + +class NormalizeWhitespaceProvider(TransformerProvider): + """Normalizes whitespace in text fields. + + This transformer provider standardizes whitespace in text fields of document + records by removing leading and trailing whitespace and replacing multiple + consecutive whitespace characters with a single space. + + Attributes: + config: Configuration object for the transformer + name: Name of the transformer (from config) + args: Arguments for the transformer (from config), including: + - fields: List of field names to process (default: ["content"]) + """ + + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Transform records by normalizing whitespace in text fields. + + Processes each record and applies whitespace normalization to the specified + fields. Non-string values and empty strings are preserved unchanged. + + Args: + records: List of record dictionaries to transform + + Returns: + List of transformed record dictionaries with normalized whitespace + """ + fields = self.args.get("fields", ["content"]) + + normalized_records = [] + for record in records: + normalized_record = record.copy() + + for field in fields: + value = record.get(field) + if value and isinstance(value, str): + normalized_record[field] = re.sub(r'\s+', ' ', value.strip()) + + normalized_records.append(normalized_record) + + return normalized_records \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/registry/__init__.py b/document-graph/src/document_graph/transform/registry/__init__.py new file mode 100644 index 00000000..e9c3a927 --- /dev/null +++ b/document-graph/src/document_graph/transform/registry/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Registry module for transformer providers in document graph operations. + +This package provides registry and factory components for managing transformer providers +in the document graph processing system. It includes classes for registering, discovering, +and instantiating transformer provider implementations. +""" + +from document_graph.transform.registry.transformer_provider_registry import TransformerProviderRegistry +from document_graph.transform.registry.transformer_provider_factory import TransformerProviderFactory + +# Create a singleton instance for backward compatibility +transformer_provider_registry = TransformerProviderRegistry() + +__all__ = [ + "TransformerProviderRegistry", + "TransformerProviderFactory", + "transformer_provider_registry", +] \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/registry/transformer_provider_factory.py b/document-graph/src/document_graph/transform/registry/transformer_provider_factory.py new file mode 100644 index 00000000..c72e8e28 --- /dev/null +++ b/document-graph/src/document_graph/transform/registry/transformer_provider_factory.py @@ -0,0 +1,74 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Transformer Provider Factory module for document graph operations. + +This module provides a factory for dynamically instantiating transformer provider +implementations based on configuration. It supports: + +- Dynamic class loading from fully qualified paths +- Instantiation of transformer providers with appropriate configuration +- Integration with the transformer provider registry system + +The factory pattern used here allows for flexible creation of transformer providers +without tight coupling to specific implementations, enabling extensibility through plugins. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +import importlib +from typing import Type + +from document_graph.transform.transformer_provider_base import TransformerProvider +from document_graph.transform.transformer_provider_config import TransformerProviderConfig + + +class TransformerProviderFactory: + """ + Factory for instantiating TransformerProvider implementations + based on the configuration provided. + + This factory class works in conjunction with the TransformerProviderRegistry + to create instances of transformer providers. It handles: + + - Dynamic loading of provider classes from their fully qualified names + - Instantiation of provider objects with appropriate configuration + - Error handling for missing or invalid provider implementations + + The factory pattern implemented here allows for flexible creation of transformer + providers at runtime without requiring direct dependencies on specific implementations. + """ + + @staticmethod + def load_class(path: str) -> Type[TransformerProvider]: + """ + Dynamically import and return the class from a fully qualified path. + + Args: + path (str): e.g., "my_module.sub_module.MyClass" + + Returns: + Type[TransformerProvider]: The class object. + """ + module_path, class_name = path.rsplit(".", 1) + module = importlib.import_module(module_path) + klass = getattr(module, class_name) + return klass + + @classmethod + def get_provider(cls, config: TransformerProviderConfig) -> TransformerProvider: + """ + Instantiate a transformers provider from its config. + + Args: + config (TransformerProviderConfig): Configuration for the provider. + + Returns: + TransformerProvider: Instantiated provider. + """ + klass = cls.load_class(config.implementation) + return klass(config=config) diff --git a/document-graph/src/document_graph/transform/registry/transformer_provider_registry.py b/document-graph/src/document_graph/transform/registry/transformer_provider_registry.py new file mode 100644 index 00000000..07aadddc --- /dev/null +++ b/document-graph/src/document_graph/transform/registry/transformer_provider_registry.py @@ -0,0 +1,75 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Transformer Provider Registry module for document graph operations. + +This module provides a registry system for managing transformer provider implementations +in the document graph processing system. It supports: + +- Registration of transformer providers with unique identifiers +- Lookup of transformer providers by name +- Listing of all available transformer providers + +The registry pattern implemented here enables a plugin architecture where transformer +providers can be registered dynamically and retrieved by name, facilitating extensibility +and decoupling between components that define transformers and those that use them. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from typing import Dict, Type + +from document_graph.transform.transformer_provider_base import TransformerProvider + + +class TransformerProviderRegistry: + """ + A registry to hold and manage all available transformers providers. + + This allows for plugin-style registration and lookup of transformers classes + by name, which is useful for managing normalizers, enrichers, truncators, etc. + """ + + _registry: Dict[str, Type[TransformerProvider]] = {} + + @classmethod + def register(cls, name: str, provider_class: Type[TransformerProvider]) -> None: + """ + Register a transformers provider with a given name. + + Args: + name (str): The unique identifier for the provider. + provider_class (Type[TransformerProvider]): The provider class. + """ + if name in cls._registry: + raise ValueError(f"Transformer provider '{name}' is already registered.") + cls._registry[name] = provider_class + + @classmethod + def get(cls, name: str) -> Type[TransformerProvider]: + """ + Retrieve a transformers provider by name. + + Args: + name (str): The name of the registered provider. + + Returns: + Type[TransformerProvider]: The provider class. + """ + if name not in cls._registry: + raise ValueError(f"Transformer provider '{name}' not found in registry.") + return cls._registry[name] + + @classmethod + def list_registered(cls) -> Dict[str, Type[TransformerProvider]]: + """ + List all registered transformers providers. + + Returns: + Dict[str, Type[TransformerProvider]]: All registered providers. + """ + return dict(cls._registry) diff --git a/document-graph/src/document_graph/transform/transformation_plan.py b/document-graph/src/document_graph/transform/transformation_plan.py new file mode 100644 index 00000000..27bdc549 --- /dev/null +++ b/document-graph/src/document_graph/transform/transformation_plan.py @@ -0,0 +1,91 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Transformation plan models for document graph processing. + +This module defines the data models used to represent transformation plans +and steps for document graph processing pipelines. Transformation plans +describe a sequence of operations to be applied to documents or datasets +during processing, such as normalization, enrichment, and truncation. +""" + +import logging +from typing import List, Literal, Optional, Union +from pydantic import BaseModel, Field + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + +class TransformationStep(BaseModel): + """One step in a transformation plan. + + Represents a single operation in a document transformation pipeline, + such as normalization, enrichment, or truncation. + + Attributes: + type: The type of transformation step (transformers, normalizer, enricher, truncator) + name: The registered name or class of the step + args: Optional arguments for configuring the step + + Examples: + >>> normalizer_step = TransformationStep( + ... type="normalizer", + ... name="text_normalizer", + ... args={"lowercase": True, "remove_punctuation": True} + ... ) + """ + type: Literal["transformers", "normalizer", "enricher", "truncator"] + name: str = Field(..., description="The registered name or class of the step") + args: Optional[dict] = Field(default_factory=dict, description="Optional kwargs for the step") + +class TransformationPlan(BaseModel): + """A full transformation pipeline plan for a document or dataset. + + Represents a complete sequence of transformation operations to be applied + to documents or datasets during processing. Each plan has a unique ID, + an optional description, and an ordered list of transformation steps. + + Attributes: + id: Unique identifier for the transformation plan + description: Optional human-readable description of the plan + steps: Ordered list of transformation steps to apply + + Examples: + >>> plan = TransformationPlan( + ... id="doc-processing-plan", + ... description="Standard document processing pipeline", + ... steps=[ + ... TransformationStep(type="normalizer", name="text_normalizer", + ... args={"lowercase": True}), + ... TransformationStep(type="enricher", name="entity_enricher", + ... args={"entities": ["person", "organization"]}) + ... ] + ... ) + """ + id: str + description: Optional[str] = None + steps: List[TransformationStep] = Field(..., description="Ordered list of transformation steps to apply") + + def get_steps_by_type(self, step_type: str) -> List[TransformationStep]: + """Filter steps by their type. + + Args: + step_type: The type of steps to filter for (e.g., "normalizer", "enricher") + + Returns: + List of transformation steps matching the specified type + + Examples: + >>> plan = TransformationPlan( + ... id="doc-processing-plan", + ... steps=[ + ... TransformationStep(type="normalizer", name="text_normalizer"), + ... TransformationStep(type="enricher", name="entity_enricher") + ... ] + ... ) + >>> normalizers = plan.get_steps_by_type("normalizer") + >>> len(normalizers) + 1 + >>> normalizers[0].name + 'text_normalizer' + """ + return [step for step in self.steps if step.type == step_type] diff --git a/document-graph/src/document_graph/transform/transformer_provider_base.py b/document-graph/src/document_graph/transform/transformer_provider_base.py new file mode 100644 index 00000000..d1636d33 --- /dev/null +++ b/document-graph/src/document_graph/transform/transformer_provider_base.py @@ -0,0 +1,114 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Base class for transformer providers. + +This module defines the abstract base class for all transformer providers in the +document graph processing system. Transformer providers implement data transformation +operations that can be used in ETL pipelines, document processing, and graph building. +The module provides a common interface for all transformers, regardless of their +specific transformation logic. +""" + +import logging +from abc import ABC, abstractmethod +from typing import Any, List, Dict +from document_graph.transform.transformer_provider_config import TransformerProviderConfig + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +class TransformerProvider(ABC): + """Abstract base class for all transformers providers. + + Provides the interface for data transformation operations + used by plugins and transformation pipelines. All concrete transformer + implementations must inherit from this class and implement the transform method. + + Attributes: + config: Configuration object for the transformer + name: Name of the transformer (from config) + args: Arguments for the transformer (from config) + + Examples: + >>> # Define a custom transformer + >>> class LowercaseTransformer(TransformerProvider): + ... def transform(self, records): + ... for record in records: + ... if 'text' in record: + ... record['text'] = record['text'].lower() + ... return records + >>> + >>> # Create and use the transformer + >>> config = TransformerProviderConfig(name="lowercase", args={"fields": ["text"]}) + >>> transformer = LowercaseTransformer(config) + >>> result = transformer.transform([{"text": "HELLO WORLD"}]) + >>> result[0]["text"] + 'hello world' + """ + + def __init__(self, config: TransformerProviderConfig): + """Initialize transformers with configuration. + + Args: + config: Transformer configuration with name, type, and args + """ + self.config = config + self.name = config.name + self.args = config.args + + @abstractmethod + def transform(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Apply transformation logic to records. + + Args: + records: List of record dictionaries to transform + + Returns: + List of transformed record dictionaries + """ + pass + + def _log_transform_start(self, record_count: int): + """Log the start of a transformation operation. + + This helper method logs a debug message indicating that a transformation + operation is starting, including the number of records to be processed. + + Args: + record_count: Number of records to be transformed + """ + import logging + logger = logging.getLogger(self.__class__.__module__) + logger.debug(f"[{self.__class__.__name__}] Starting transformation of {record_count} records") + + def _log_transform_end(self, input_count: int, output_count: int): + """Log the completion of a transformation operation. + + This helper method logs a debug message indicating that a transformation + operation has completed, including the number of input and output records. + + Args: + input_count: Number of input records processed + output_count: Number of output records produced + """ + import logging + logger = logging.getLogger(self.__class__.__module__) + logger.debug(f"[{self.__class__.__name__}] Completed: {input_count} -> {output_count} records") + + def __repr__(self): + """Return a string representation of the transformer provider. + + Returns: + String representation including the transformer name and type + + Examples: + >>> config = TransformerProviderConfig(name="example", type="test") + >>> transformer = TransformerProvider(config) + >>> repr(transformer) + '' + """ + return f"" + +# Alias for backward compatibility +TransformerConfig = TransformerProviderConfig diff --git a/document-graph/src/document_graph/transform/transformer_provider_config.py b/document-graph/src/document_graph/transform/transformer_provider_config.py new file mode 100644 index 00000000..c8dac855 --- /dev/null +++ b/document-graph/src/document_graph/transform/transformer_provider_config.py @@ -0,0 +1,73 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Transformer provider configuration models. + +This module defines the configuration models used for transformer providers +in the document graph processing system. These configurations are used to +instantiate and parameterize transformer instances that perform various +data transformation operations during document processing. +""" + +import logging +from typing import Dict, Any, Optional +from pydantic import BaseModel, Field + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + +class TransformerProviderConfig(BaseModel): + """Configuration for transformers providers. + + Used by plugins and transformation steps to configure + data transformation operations. + + Attributes: + name: Transformer name for identification + type: Transformer type (optional, for factory resolution) + args: Transformer-specific arguments and parameters + parameters: Alias for args (user-friendly) + + Examples: + >>> config = TransformerProviderConfig( + ... name="text_normalizer", + ... type="normalizer", + ... args={"lowercase": True, "remove_punctuation": True} + ... ) + >>> config.name + 'text_normalizer' + >>> config.args["lowercase"] + True + """ + name: str = Field(..., description="Transformer name") + type: Optional[str] = Field(default=None, description="Transformer type") + args: Dict[str, Any] = Field(default_factory=dict, description="Transformer arguments") + parameters: Dict[str, Any] = Field(default_factory=dict, description="Transformer parameters (alias for args)") + + def __init__(self, **data): + """Initialize transformer provider configuration. + + Initializes the configuration and synchronizes the args and parameters + fields to ensure they contain the same data, regardless of which one + was provided in the input. + + Args: + **data: Keyword arguments for configuration fields including name, + type, args, and parameters + + Examples: + >>> config = TransformerProviderConfig( + ... name="entity_enricher", + ... parameters={"entities": ["person", "organization"]} + ... ) + >>> config.args == config.parameters + True + """ + super().__init__(**data) + # Sync parameters and args + if self.parameters and not self.args: + self.args = self.parameters + elif self.args and not self.parameters: + self.parameters = self.args + +# Alias for backward compatibility +BaseTransformerConfig = TransformerProviderConfig diff --git a/document-graph/src/document_graph/transform/transformer_provider_factory.py b/document-graph/src/document_graph/transform/transformer_provider_factory.py new file mode 100644 index 00000000..136acc5d --- /dev/null +++ b/document-graph/src/document_graph/transform/transformer_provider_factory.py @@ -0,0 +1,119 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Transformer Provider Factory module for document graph operations. + +This module provides a factory for creating transformer provider instances +based on configuration. It handles the resolution of transformer types and names, +instantiates the appropriate provider classes, and manages the configuration +process. The factory works with the transformer provider registry to locate +and instantiate the correct transformer implementation. +""" + +import logging +from typing import Type +from document_graph.transform.transformer_provider_base import TransformerProvider +from document_graph.transform.transformer_provider_registry import transformer_provider_registry +from document_graph.transform.transformer_provider_config import TransformerProviderConfig + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + +class TransformerProviderFactory: + """Factory to instantiate transformers providers based on config. + + Creates transformers instances for use in transformation pipelines + and plugin systems. The factory resolves transformer types and names, + retrieves the appropriate provider class from the registry, and + instantiates it with the provided configuration. + + Examples: + >>> config = TransformerProviderConfig( + ... name="text_normalizer", + ... type="normalizer", + ... args={"lowercase": True} + ... ) + >>> provider = TransformerProviderFactory.get_provider(config) + >>> isinstance(provider, TransformerProvider) + True + """ + + @staticmethod + def get_provider(config: TransformerProviderConfig) -> TransformerProvider: + """Instantiate the appropriate transformers provider. + + Args: + config: Transformer configuration with type and parameters + + Returns: + TransformerProvider: Configured transformers instance + + Raises: + KeyError: If transformers type is not registered + + Examples: + >>> config = TransformerProviderConfig( + ... name="entity_enricher", + ... type="enricher", + ... args={"entities": ["person", "organization"]} + ... ) + >>> provider = TransformerProviderFactory.get_provider(config) + >>> provider.transform({"text": "John works at Amazon"}) + {'text': 'John works at Amazon', 'entities': [...]} + """ + logger.debug(f"Creating transformers provider: {config.type or config.name}") + + # Use name for lookup when type is a directory-based category, otherwise use type + directory_types = ["transformer", "normalizer", "enricher", "truncator", + "field_transformer", "document_transformer", "filter_transformer", + "graph_transformer"] + if config.type in directory_types: + lookup_key = config.name + logger.debug(f"Using name '{lookup_key}' for category type '{config.type}'") + else: + lookup_key = config.type or config.name + logger.debug(f"Using direct lookup key '{lookup_key}'") + + logger.debug(f"Registry lookup for key: {lookup_key}") + logger.debug(f"Available providers: {transformer_provider_registry.list_providers()}") + + provider_cls: Type[TransformerProvider] = transformer_provider_registry.get(lookup_key) + + provider = provider_cls(config) + logger.debug(f"Successfully created transformers: {provider.__class__.__name__}") + return provider + + @staticmethod + def create_config( + name: str, + transformer_type: str = None, + **args + ) -> TransformerProviderConfig: + """Create a transformers configuration. + + Args: + name: Transformer name + transformer_type: Optional transformers type for factory lookup + **args: Transformer-specific arguments + + Returns: + TransformerProviderConfig: Transformer configuration + + Examples: + >>> config = TransformerProviderFactory.create_config( + ... name="text_normalizer", + ... transformer_type="normalizer", + ... lowercase=True, + ... remove_punctuation=True + ... ) + >>> config.name + 'text_normalizer' + >>> config.type + 'normalizer' + >>> config.args["lowercase"] + True + """ + return TransformerProviderConfig( + name=name, + type=transformer_type, + args=args + ) diff --git a/document-graph/src/document_graph/transform/transformer_provider_registry.py b/document-graph/src/document_graph/transform/transformer_provider_registry.py new file mode 100644 index 00000000..c1e17e96 --- /dev/null +++ b/document-graph/src/document_graph/transform/transformer_provider_registry.py @@ -0,0 +1,248 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Transformer Provider Registry module for document graph operations. + +This module provides a registry system for transformer providers in the document +graph processing system. It maintains a mapping of transformer type names to their +concrete implementations, allowing for dynamic discovery and instantiation of +transformer classes. The registry supports registration, lookup, and introspection +of transformer providers, and is used by the TransformerProviderFactory and plugin +system to locate and create transformer instances. +""" + +import logging +from typing import Type, Dict +from document_graph.transform.transformer_provider_base import TransformerProvider + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + +class TransformerProviderRegistry: + """Registry for transformers providers. + + Maintains a mapping of transformers type names to their concrete implementations. + Used by the TransformerProviderFactory and plugin system to discover and + instantiate transformer providers dynamically. + + Examples: + >>> # Register a custom transformer provider + >>> class MyTransformer(TransformerProvider): + ... def transform(self, data): + ... return data + >>> + >>> # Register the transformer + >>> TransformerProviderRegistry.register("my_transformer", MyTransformer) + >>> + >>> # Get the transformer class + >>> transformer_cls = TransformerProviderRegistry.get("my_transformer") + >>> transformer_cls is MyTransformer + True + """ + + _registry: Dict[str, Type[TransformerProvider]] = {} + _lazy_registry: Dict[str, callable] = {} + + @classmethod + def register(cls, type_: str, provider_cls: Type[TransformerProvider]) -> None: + """Register a new transformers provider type. + + Args: + type_: Provider type name + provider_cls: Provider class to register + """ + logger.debug(f"Registering transformers provider: {type_} -> {provider_cls.__name__}") + cls._registry[type_.lower()] = provider_cls + + @classmethod + def register_lazy(cls, type_: str, loader_func: callable) -> None: + """Register a lazy-loaded transformer provider. + + Args: + type_: Provider type name + loader_func: Function that returns the provider class when called + """ + logger.debug(f"Registering lazy transformers provider: {type_}") + cls._lazy_registry[type_.lower()] = loader_func + + @classmethod + def get(cls, type_: str) -> Type[TransformerProvider]: + """Get a transformers provider class by type. + + Args: + type_: Provider type name + + Returns: + Type[TransformerProvider]: Provider class + + Raises: + KeyError: If provider type is not registered + + Examples: + >>> # Get a registered transformer provider + >>> normalizer_cls = TransformerProviderRegistry.get("text_normalizer") + >>> normalizer = normalizer_cls(config) + >>> normalizer.transform("TEXT TO NORMALIZE") + 'text to normalize' + >>> + >>> # Trying to get an unregistered provider + >>> try: + ... TransformerProviderRegistry.get("nonexistent_provider") + ... except KeyError as e: + ... print(f"Error: {e}") + Error: 'No transformers provider registered for type='nonexistent_provider'' + """ + key = type_.lower() + logger.debug(f"Looking up transformers provider: {type_}") + + # Check eager registry first + if key in cls._registry: + return cls._registry[key] + + # Check lazy registry + if key in cls._lazy_registry: + try: + provider_cls = cls._lazy_registry[key]() + # Cache the loaded class + cls._registry[key] = provider_cls + return provider_cls + except Exception as e: + logger.error(f"Failed to load lazy transformer {type_}: {e}") + raise KeyError(f"Failed to load transformer '{type_}': {e}") + + # Provide helpful suggestions + available = cls.list_providers() + suggestions = [name for name in available if type_.lower() in name or name in type_.lower()] + + error_msg = f"No transformer '{type_}' found." + if suggestions: + error_msg += f" Did you mean: {', '.join(suggestions[:3])}?" + error_msg += f" Available: {', '.join(sorted(available))}" + + logger.error(error_msg) + raise KeyError(error_msg) + + @classmethod + def list_providers(cls) -> list[str]: + """List all registered provider types. + + Returns: + list[str]: List of registered provider type names + + Examples: + >>> # Register some providers + >>> TransformerProviderRegistry.register("text_normalizer", TextNormalizer) + >>> TransformerProviderRegistry.register("entity_enricher", EntityEnricher) + >>> + >>> # List all registered providers + >>> providers = TransformerProviderRegistry.list_providers() + >>> sorted(providers) + ['entity_enricher', 'text_normalizer'] + """ + return list(set(list(cls._registry.keys()) + list(cls._lazy_registry.keys()))) + + @classmethod + def describe_providers(cls) -> dict[str, dict[str, str]]: + """Get detailed information about all registered providers. + + Returns: + dict: Provider info with name, description, and category + + Examples: + >>> # Get detailed information about registered providers + >>> info = TransformerProviderRegistry.describe_providers() + >>> 'text_normalizer' in info + True + >>> info['text_normalizer']['category'] + 'normalizer' + >>> info['text_normalizer']['description'] + 'Normalizes text input' + """ + info = {} + + # Process eager registry + for name, provider_cls in cls._registry.items(): + info[name] = cls._get_provider_info(name, provider_cls) + + # Process lazy registry - create basic info without loading + for name in cls._lazy_registry.keys(): + if name not in info: # Don't override if already loaded + # Create basic info based on name patterns without loading + category = 'transformer' + type_name = 'transformer' + + if 'normalize' in name: + category = 'normalizer' + type_name = 'normalizer' + elif 'enricher' in name: + category = 'enricher' + type_name = 'enricher' + elif name in ['column_pruner', 'row_filter']: + category = 'filter' + type_name = 'filter_transformer' + elif name in ['infer_edges', 'row_to_node']: + category = 'graph' + type_name = 'graph_transformer' + elif name in ['pii_redactor', 'json_to_rows', 'text_chunker']: + category = 'document' + type_name = 'document_transformer' + elif name in ['regex_cleaner', 'embedded_json', 'standardize_enum', 'comma_split', 'json_array_expander', 'json_flattener', 'comma_flattener']: + category = 'field' + type_name = 'field_transformer' + + info[name] = { + 'name': name, + 'description': f'{category.title()} transformer for {name.replace("_", " ")}', + 'category': category, + 'type': type_name, + 'class': 'LazyLoaded' + } + + return info + + @classmethod + def _get_provider_info(cls, name: str, provider_cls) -> dict[str, str]: + """Extract provider information from a provider class.""" + # Extract description from docstring + doc = provider_cls.__doc__ or "No description available" + description = doc.split('\n')[0].strip() if doc else "No description" + + # Determine category and type from class name and module patterns + class_name = provider_cls.__name__.lower() + module_name = provider_cls.__module__.lower() + + if 'normalize' in class_name or 'normalizers' in module_name: + category = 'normalizer' + type_name = 'normalizer' + elif 'enrich' in class_name or 'enrichers' in module_name: + category = 'enricher' + type_name = 'enricher' + elif 'filter' in class_name or 'prune' in class_name or 'filter_transformers' in module_name: + category = 'filter' + type_name = 'filter_transformer' + elif 'graph' in class_name or 'node' in class_name or 'edge' in class_name or 'graph_transformers' in module_name: + category = 'graph' + type_name = 'graph_transformer' + elif 'document_transformers' in module_name: + category = 'document' + type_name = 'document_transformer' + elif 'field_transformers' in module_name: + category = 'field' + type_name = 'field_transformer' + elif 'truncators' in module_name: + category = 'truncator' + type_name = 'truncator' + else: + category = 'transformer' + type_name = 'transformer' + + return { + 'name': name, + 'description': description, + 'category': category, + 'type': type_name, + 'class': provider_cls.__name__ + } + + +# Initialize registry instance +transformer_provider_registry = TransformerProviderRegistry() \ No newline at end of file diff --git a/document-graph/src/document_graph/transform/truncators/__init__.py b/document-graph/src/document_graph/transform/truncators/__init__.py new file mode 100644 index 00000000..00978d27 --- /dev/null +++ b/document-graph/src/document_graph/transform/truncators/__init__.py @@ -0,0 +1 @@ +"""Truncators — field truncation strategies for length, count, and token limits.""" diff --git a/document-graph/src/document_graph/transform/truncators/field_count_truncator.py b/document-graph/src/document_graph/transform/truncators/field_count_truncator.py new file mode 100644 index 00000000..19a94ade --- /dev/null +++ b/document-graph/src/document_graph/transform/truncators/field_count_truncator.py @@ -0,0 +1,61 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Field Count Truncator module for document graph operations. + +This module provides a transformer that limits the number of fields in a record +to prevent oversized documents and ensure consistent processing. It keeps only +the first N fields from the input record, discarding any additional fields. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from document_graph.transform.transformer_provider_base import TransformerProvider + +class FieldCountTruncator(TransformerProvider): + """Transformer that limits the number of fields in a record. + + This transformer ensures records don't exceed a maximum number of fields + by keeping only the first N fields and discarding any additional ones. + This helps prevent oversized documents and ensures consistent processing + downstream. + + Attributes: + config: Configuration object containing transformer settings + + Examples: + >>> from document_graph.transform.transformer_provider_config import TransformerProviderConfig + >>> config = TransformerProviderConfig( + ... name="field_count_truncator", + ... args={"max_fields": 5} + ... ) + >>> truncator = FieldCountTruncator(config) + >>> result = truncator.transform({"a": 1, "b": 2, "c": 3, "d": 4, "e": 5, "f": 6, "g": 7}) + >>> len(result) + 5 + >>> "f" in result + False + """ + + def transform(self, record: dict) -> dict: + """Limit the number of fields in a record. + + Args: + record: A dictionary containing record data + + Returns: + A dictionary with at most max_fields keys + + Note: + This implementation differs from the base class by accepting a single + record dictionary rather than a list of records. + """ + max_fields = self.config.args.get("max_fields", 20) + if len(record) > max_fields: + keys = list(record.keys())[:max_fields] + record = {k: record[k] for k in keys} + return record diff --git a/document-graph/src/document_graph/transform/truncators/length_truncator.py b/document-graph/src/document_graph/transform/truncators/length_truncator.py new file mode 100644 index 00000000..f8b59ed4 --- /dev/null +++ b/document-graph/src/document_graph/transform/truncators/length_truncator.py @@ -0,0 +1,65 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Length Truncator module for document graph operations. + +This module provides a transformer that limits the character length of string fields +in a record to prevent oversized text fields and ensure consistent processing. +It truncates specified string fields to a maximum character length. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from document_graph.transform.transformer_provider_base import TransformerProvider + +class LengthTruncator(TransformerProvider): + """Transformer that limits the character length of string fields. + + This transformer ensures string fields don't exceed a maximum character length + by truncating them to the specified maximum. This helps prevent oversized text + fields and ensures consistent processing downstream. + + Attributes: + config: Configuration object containing transformer settings + + Examples: + >>> from document_graph.transform.transformer_provider_config import TransformerProviderConfig + >>> config = TransformerProviderConfig( + ... name="length_truncator", + ... args={"max_length": 10, "fields": ["text", "description"]} + ... ) + >>> truncator = LengthTruncator(config) + >>> result = truncator.transform({"text": "This is a long text that will be truncated", "description": "Short desc"}) + >>> result["text"] + 'This is a ' + >>> result["description"] + 'Short desc' + """ + + def transform(self, record: dict) -> dict: + """Truncate specified string fields to a maximum character length. + + Args: + record: A dictionary containing record data + + Returns: + A dictionary with string fields truncated to max_length characters + + Note: + This implementation differs from the base class by accepting a single + record dictionary rather than a list of records. + + Only processes fields specified in the "fields" configuration parameter. + Fields that don't exist in the record or aren't strings are ignored. + """ + max_length = self.config.args.get("max_length", 512) + fields = self.config.args.get("fields", []) + + for field in fields: + if field in record and isinstance(record[field], str): + record[field] = record[field][:max_length] + return record diff --git a/document-graph/src/document_graph/transform/truncators/token_truncator.py b/document-graph/src/document_graph/transform/truncators/token_truncator.py new file mode 100644 index 00000000..a2a9702f --- /dev/null +++ b/document-graph/src/document_graph/transform/truncators/token_truncator.py @@ -0,0 +1,91 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +"""Token Truncator module for document graph operations. + +This module provides a transformer that limits the number of tokens in string fields +of a record to prevent oversized text fields and ensure consistent processing. +It uses a pre-trained tokenizer to tokenize text and truncate to a maximum token count. +""" + +import logging + + +logger = logging.getLogger(__name__) +# Import custom logging to ensure configuration is available + + +from transformers import AutoTokenizer +from document_graph.transform.transformer_provider_base import TransformerProvider + +class TokenTruncator(TransformerProvider): + """Transformer that limits the number of tokens in string fields. + + This transformer ensures string fields don't exceed a maximum token count + by tokenizing the text using a pre-trained tokenizer and truncating to the + specified maximum number of tokens. This helps prevent oversized text fields + and ensures consistent processing downstream, especially for models with + token limits. + + Attributes: + config: Configuration object containing transformer settings + tokenizer: Pre-trained tokenizer from the Hugging Face transformers library + + Examples: + >>> from document_graph.transform.transformer_provider_config import TransformerProviderConfig + >>> config = TransformerProviderConfig( + ... name="token_truncator", + ... args={ + ... "max_tokens": 5, + ... "fields": ["text"], + ... "model_name": "bert-base-uncased" + ... } + ... ) + >>> truncator = TokenTruncator(config) + >>> result = truncator.transform({"text": "This is a long text that will be truncated to just five tokens"}) + >>> result["text"] + 'this is a long text' + """ + + def __init__(self, config): + """Initialize the token truncator with configuration. + + Args: + config: Transformer configuration with name, type, and args + + Note: + Loads a pre-trained tokenizer based on the model_name in config.args. + Defaults to "bert-base-uncased" if not specified. + """ + super().__init__(config) + model_name = self.config.args.get("model_name", "bert-base-uncased") + self.tokenizer = AutoTokenizer.from_pretrained(model_name) + + def transform(self, record: dict) -> dict: + """Truncate specified string fields to a maximum token count. + + Args: + record: A dictionary containing record data + + Returns: + A dictionary with string fields truncated to max_tokens tokens + + Note: + This implementation differs from the base class by accepting a single + record dictionary rather than a list of records. + + Only processes fields specified in the "fields" configuration parameter. + Fields that don't exist in the record or aren't strings are ignored. + + The tokenization and detokenization process may slightly alter the text + beyond just truncation, as the tokenizer's conversion back to string + may normalize whitespace or other characters. + """ + max_tokens = self.config.args.get("max_tokens", 128) + fields = self.config.args.get("fields", []) + + for field in fields: + if field in record and isinstance(record[field], str): + tokens = self.tokenizer.tokenize(record[field]) + truncated = self.tokenizer.convert_tokens_to_string(tokens[:max_tokens]) + record[field] = truncated + return record diff --git a/document-graph/tests/conftest.py b/document-graph/tests/conftest.py new file mode 100644 index 00000000..bdf39754 --- /dev/null +++ b/document-graph/tests/conftest.py @@ -0,0 +1,7 @@ +"""Pytest configuration for document-graph tests.""" + +import sys +from pathlib import Path + +# Add src to path so tests can import document_graph +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) diff --git a/document-graph/tests/test_document_graph.py b/document-graph/tests/test_document_graph.py new file mode 100644 index 00000000..891c552e --- /dev/null +++ b/document-graph/tests/test_document_graph.py @@ -0,0 +1,553 @@ +"""Tests for document-graph v3.""" + +import pytest +from document_graph import NodeModel, EdgeModel, Node, Edge +from document_graph.graph_build import node_to_cypher, edge_to_cypher, batch_nodes_to_cypher +from document_graph.query import DocumentGraphQueryEngine + + +class TestModel: + def test_node_model_importable(self): + assert NodeModel is not None + + def test_edge_model_importable(self): + assert EdgeModel is not None + + +class TestModelElements: + def test_node_creation(self): + n = Node(id="u1", labels=["User"], properties={"name": "Alice", "email": "a@b.com"}) + assert n.id == "u1" + assert "User" in n.labels + assert n.properties["name"] == "Alice" + + def test_edge_creation(self): + e = Edge(id="e1", source_id="u1", target_id="a1", label="OWNS") + assert e.label == "OWNS" + assert e.source_id == "u1" + assert e.target_id == "a1" + + +class TestCypherBuilder: + def test_node_to_cypher(self): + n = Node(id="u1", labels=["User"], properties={"name": "Alice"}) + query, params = node_to_cypher(n) + assert "MERGE" in query + assert "User" in query + assert params["id_val"] == "u1" + + def test_node_to_cypher_with_tenant(self): + n = Node(id="u1", labels=["User"], properties={"name": "Alice"}) + query, params = node_to_cypher(n, tenant_id="scim") + assert "__User__scim__" in query + + def test_edge_to_cypher(self): + e = Edge(id="e1", source_id="u1", target_id="a1", label="OWNS") + query, params = edge_to_cypher(e) + assert "MERGE" in query + assert "OWNS" in query + assert params["src_id"] == "u1" + assert params["tgt_id"] == "a1" + + def test_batch_nodes(self): + nodes = [ + Node(id=f"u{i}", labels=["User"], properties={"name": f"User{i}"}) + for i in range(5) + ] + results = batch_nodes_to_cypher(nodes, tenant_id="test") + assert len(results) == 5 + assert all("MERGE" in q for q, _ in results) + + +class TestQueryEngine: + def test_init(self): + class MockStore: + def query(self, cypher, params=None): + return [{"n": "result"}] + + engine = DocumentGraphQueryEngine(MockStore(), tenant_id="scim") + assert engine._tenant_id == "scim" + + def test_get_nodes(self): + class MockStore: + def execute_query(self, cypher, params=None): + assert "__User__scim__" in cypher + return [{"n": {"name": "Alice"}}] + + engine = DocumentGraphQueryEngine(MockStore(), tenant_id="scim") + result = engine.get_nodes("User", limit=10) + assert len(result) == 1 + + def test_find_by_property(self): + class MockStore: + def execute_query(self, cypher, params=None): + assert params["val"] == "alice@test.com" + return [{"n": {"email": "alice@test.com"}}] + + engine = DocumentGraphQueryEngine(MockStore(), tenant_id="scim") + result = engine.find_by_property("User", "email", "alice@test.com") + assert len(result) == 1 + + +# ───────────────────────────────────────────────────────────────────────────── +# Extended tests — Schema Providers, Transformers, ETLSchema, Graph Build, Discovery +# ───────────────────────────────────────────────────────────────────────────── + +import json +import csv +import tempfile +import uuid +from pathlib import Path +from pydantic import ValidationError + +from document_graph.graph_build import node_to_cypher, edge_to_cypher, batch_nodes_to_cypher +from document_graph.schema.providers.schema_provider_config import SchemaProviderConfig +from document_graph.schema.providers.csv_schema_provider import CSVSchemaProvider +from document_graph.schema.providers.json_schema_provider import JSONSchemaProvider +from document_graph.schema.static_schema_provider import StaticSchemaProvider +from document_graph.schema.providers.schema_provider_factory import SchemaProviderFactory +from document_graph.schema.etl_schema_model import ( + ETLSchema, ExtractConfig, TransformConfig, LoadConfig, + ChunkingConfig, NormalizeConfig, EntityExtractionConfig, + MetadataMapping, NodeDefinition, RelationshipDefinition, +) +from document_graph.transform.transformer_provider_config import TransformerProviderConfig +from document_graph.transform.normalizers.normalize_whitespace_provider import NormalizeWhitespaceProvider +from document_graph.transform.normalizers.normalize_nulls_provider import NormalizeNullsProvider +from document_graph.transform.normalizers.normalize_case_provider import NormalizeCaseProvider +from document_graph.transform.field_transformers.json_flattener import JSONFlattenerProvider +from document_graph.transform.field_transformers.uuid_generator import UuidGeneratorTransformer +from document_graph.transform.graph_transformers.row_to_node import RowToNodeTransformer +from document_graph.transform.graph_transformers.infer_edges import EdgeInferencer +from document_graph.transform.filter_transformers.row_filter import RowFilterProvider +from document_graph.transform.truncators.length_truncator import LengthTruncator +from document_graph.schema.discovery.csv_discovery_provider import CSVSchemaDiscoveryProvider + + +# ─── Fixtures ──────────────────────────────────────────────────────────────── + +@pytest.fixture +def sample_etl_schema(): + """A valid minimal ETLSchema.""" + return ETLSchema( + schema_id="test-schema", + description="Test", + extract=ExtractConfig(source_type="file"), + transform=TransformConfig( + chunking=ChunkingConfig(strategy="fixed_length"), + metadata_mapping=MetadataMapping(), + entity_extraction=EntityExtractionConfig(method="ner"), + normalize=NormalizeConfig(), + ), + load=LoadConfig( + document_node=NodeDefinition(type="Doc", fields=["title"]), + section_node=NodeDefinition(type="Sec", fields=["text"]), + relationships=[RelationshipDefinition(type="has", source="d", target="s")], + ), + ) + + +@pytest.fixture +def tmp_csv(tmp_path): + """Create a temp CSV file for testing.""" + csv_file = tmp_path / "test_data.csv" + csv_file.write_text("id,name,email\n1,Alice,alice@test.com\n2,Bob,bob@test.com\n") + return csv_file + + +@pytest.fixture +def tmp_json(tmp_path): + """Create a temp JSON file for testing.""" + json_file = tmp_path / "test_data.json" + data = [{"id": "1", "name": "Alice"}, {"id": "2", "name": "Bob"}] + json_file.write_text(json.dumps(data)) + return json_file + + +def _make_config(name="test", **args): + """Helper to build a TransformerProviderConfig.""" + return TransformerProviderConfig(name=name, args=args) + + +# ─── ETLSchema Model Tests ────────────────────────────────────────────────── + +class TestETLSchemaModel: + def test_valid_construction(self, sample_etl_schema): + assert sample_etl_schema.schema_id == "test-schema" + assert sample_etl_schema.extract.source_type == "file" + assert sample_etl_schema.transform.chunking.strategy == "fixed_length" + + def test_missing_schema_id_raises(self): + with pytest.raises(ValidationError): + ETLSchema( + extract=ExtractConfig(source_type="file"), + transform=TransformConfig( + chunking=ChunkingConfig(strategy="x"), + metadata_mapping=MetadataMapping(), + entity_extraction=EntityExtractionConfig(method="ner"), + normalize=NormalizeConfig(), + ), + load=LoadConfig( + document_node=NodeDefinition(type="D", fields=[]), + section_node=NodeDefinition(type="S", fields=[]), + relationships=[], + ), + ) + + def test_missing_extract_raises(self): + with pytest.raises(ValidationError): + ETLSchema( + schema_id="x", + transform=TransformConfig( + chunking=ChunkingConfig(strategy="x"), + metadata_mapping=MetadataMapping(), + entity_extraction=EntityExtractionConfig(method="ner"), + normalize=NormalizeConfig(), + ), + load=LoadConfig( + document_node=NodeDefinition(type="D", fields=[]), + section_node=NodeDefinition(type="S", fields=[]), + relationships=[], + ), + ) + + def test_model_dump_roundtrip(self, sample_etl_schema): + dumped = sample_etl_schema.model_dump() + reconstructed = ETLSchema(**dumped) + assert reconstructed.schema_id == sample_etl_schema.schema_id + assert reconstructed.extract.source_type == "file" + assert reconstructed.load.document_node.type == "Doc" + + def test_extra_fields_allowed(self): + schema = ETLSchema( + schema_id="extra", + extract=ExtractConfig(source_type="api", custom_field="custom_val"), + transform=TransformConfig( + chunking=ChunkingConfig(strategy="x"), + metadata_mapping=MetadataMapping(), + entity_extraction=EntityExtractionConfig(method="ner"), + normalize=NormalizeConfig(), + ), + load=LoadConfig( + document_node=NodeDefinition(type="D", fields=[]), + section_node=NodeDefinition(type="S", fields=[]), + relationships=[], + ), + ) + assert schema.extract.custom_field == "custom_val" + + +# ─── Schema Providers Tests ───────────────────────────────────────────────── + +class TestSchemaProviders: + def test_csv_schema_provider(self, tmp_csv): + config = SchemaProviderConfig(type="csv", schema_id="csv-test", connection_config={"path": str(tmp_csv)}) + provider = CSVSchemaProvider(config) + schema = provider.load_schema() + assert isinstance(schema, ETLSchema) + assert "id" in schema.load.document_node.fields + assert "name" in schema.load.document_node.fields + + def test_csv_schema_provider_missing_path(self): + config = SchemaProviderConfig(type="csv", connection_config={}) + with pytest.raises(ValueError): + CSVSchemaProvider(config) + + def test_csv_schema_provider_file_not_found(self, tmp_path): + config = SchemaProviderConfig(type="csv", connection_config={"path": str(tmp_path / "nope.csv")}) + with pytest.raises(FileNotFoundError): + CSVSchemaProvider(config) + + def test_json_schema_provider(self, tmp_json): + config = SchemaProviderConfig(type="json", schema_id="json-test", connection_config={"path": str(tmp_json)}) + provider = JSONSchemaProvider(config) + schema = provider.load_schema() + assert isinstance(schema, ETLSchema) + assert "id" in schema.load.document_node.fields + + def test_json_schema_provider_missing_path(self): + config = SchemaProviderConfig(type="json", connection_config={}) + with pytest.raises(ValueError): + JSONSchemaProvider(config) + + def test_static_schema_provider(self): + provider = StaticSchemaProvider({}) + schema = provider.load_schema() + assert schema.schema_id == "static-default" + assert schema.extract.source_type == "s3" + assert schema.load.document_node.type == "DocumentNode" + + def test_static_schema_from_config(self): + provider = StaticSchemaProvider.from_config({"type": "static"}) + assert provider.get_schema_id() == "static-default" + + def test_factory_creates_csv_provider(self, tmp_csv): + config = {"type": "csv", "schema_id": "factory-csv", "connection_config": {"path": str(tmp_csv)}} + provider = SchemaProviderFactory.create(config) + assert isinstance(provider, CSVSchemaProvider) + + def test_factory_creates_json_provider(self, tmp_json): + config = {"type": "json", "schema_id": "factory-json", "connection_config": {"path": str(tmp_json)}} + provider = SchemaProviderFactory.create(config) + assert isinstance(provider, JSONSchemaProvider) + + def test_factory_invalid_type_raises(self): + with pytest.raises(Exception): + SchemaProviderFactory.create({"type": "unknown_xyz", "connection_config": {}}) + + +# ─── Transformer Tests ─────────────────────────────────────────────────────── + +class TestNormalizers: + def test_whitespace_normalizer(self): + cfg = _make_config("ws", fields=["text"]) + t = NormalizeWhitespaceProvider(cfg) + result = t.transform([{"text": " hello world \n\t end "}]) + assert result[0]["text"] == "hello world end" + + def test_whitespace_preserves_non_string(self): + cfg = _make_config("ws", fields=["val"]) + t = NormalizeWhitespaceProvider(cfg) + result = t.transform([{"val": 42}]) + assert result[0]["val"] == 42 + + def test_nulls_normalizer(self): + cfg = _make_config("nulls", fields=["a", "b"]) + t = NormalizeNullsProvider(cfg) + result = t.transform([{"a": "N/A", "b": "hello", "c": "null"}]) + assert result[0]["a"] is None + assert result[0]["b"] == "hello" + assert result[0]["c"] == "null" # not in fields list + + def test_nulls_custom_null_like(self): + cfg = _make_config("nulls", fields=["x"], null_like=["missing"]) + t = NormalizeNullsProvider(cfg) + result = t.transform([{"x": "missing"}]) + assert result[0]["x"] is None + + def test_case_normalizer_lower(self): + cfg = _make_config("case", mode="lower") + t = NormalizeCaseProvider(cfg) + result = t.transform([{"name": "ALICE", "age": 30}]) + assert result[0]["name"] == "alice" + assert result[0]["age"] == 30 + + def test_case_normalizer_upper(self): + cfg = _make_config("case", mode="upper") + t = NormalizeCaseProvider(cfg) + result = t.transform([{"name": "alice"}]) + assert result[0]["name"] == "ALICE" + + +class TestFieldTransformers: + def test_json_flattener(self): + cfg = _make_config("jf", field="data") + t = JSONFlattenerProvider(cfg) + records = [{"id": "1", "data": '{"key": "val", "nested": {"a": 1}}'}] + result = t.transform(records) + assert "datakey" in result[0] + assert result[0]["datakey"] == "val" + + def test_json_flattener_invalid_json(self): + cfg = _make_config("jf", field="data") + t = JSONFlattenerProvider(cfg) + records = [{"id": "1", "data": "not json"}] + result = t.transform(records) + # Should return record without crashing + assert result[0]["id"] == "1" + + def test_uuid_generator(self): + cfg = _make_config("uuid", target_field="my_id") + t = UuidGeneratorTransformer(cfg) + records = [{"name": "Alice"}, {"name": "Bob"}] + result = t.transform(records) + assert "my_id" in result[0] + assert "my_id" in result[1] + # Should be valid UUIDs + uuid.UUID(result[0]["my_id"]) + uuid.UUID(result[1]["my_id"]) + # Should be unique + assert result[0]["my_id"] != result[1]["my_id"] + + +class TestGraphTransformers: + def test_row_to_node(self): + cfg = _make_config("rtn", type="Person") + t = RowToNodeTransformer(cfg) + records = [{"id": "p1", "name": "Alice"}] + result = t.transform(records) + assert result[0]["node_type"] == "Person" + assert result[0]["graph_element"] == "node" + assert result[0]["id"] == "p1" + + def test_row_to_node_auto_id(self): + cfg = _make_config("rtn", type="Row") + t = RowToNodeTransformer(cfg) + records = [{"name": "Alice"}] + result = t.transform(records) + assert result[0]["id"] == "row_0" + + def test_row_to_node_auto_content(self): + cfg = _make_config("rtn", type="Row") + t = RowToNodeTransformer(cfg) + records = [{"name": "Alice", "role": "admin"}] + result = t.transform(records) + assert "name: Alice" in result[0]["content"] + + def test_infer_edges(self): + cfg = _make_config("ie", source_field="project", edge_type="WORKS_ON") + t = EdgeInferencer(cfg) + records = [ + {"id": "r1", "project": "A"}, + {"id": "r2", "project": "A"}, + {"id": "r3", "project": "B"}, + ] + result = t.transform(records) + edges = [r for r in result if r.get("edge_type") == "edge"] + assert len(edges) == 1 + assert edges[0]["source_id"] == "r1" + assert edges[0]["target_id"] == "r2" + assert edges[0]["relationship"] == "WORKS_ON" + + def test_infer_edges_no_shared_field(self): + cfg = _make_config("ie", source_field="project") + t = EdgeInferencer(cfg) + records = [{"id": "r1", "project": "A"}, {"id": "r2", "project": "B"}] + result = t.transform(records) + edges = [r for r in result if r.get("edge_type") == "edge"] + assert len(edges) == 0 + + +class TestFilterTransformers: + def test_row_filter_eq(self): + cfg = _make_config("rf", conditions=[{"field": "role", "operator": "eq", "value": "admin"}]) + t = RowFilterProvider(cfg) + records = [{"role": "admin"}, {"role": "user"}] + result = t.transform(records) + assert len(result) == 1 + assert result[0]["role"] == "admin" + + def test_row_filter_not_null(self): + cfg = _make_config("rf", conditions=[{"field": "email", "operator": "not_null"}]) + t = RowFilterProvider(cfg) + records = [{"email": "a@b.com"}, {"email": None}] + result = t.transform(records) + assert len(result) == 1 + + def test_row_filter_no_conditions_passes_all(self): + cfg = _make_config("rf", conditions=[]) + t = RowFilterProvider(cfg) + records = [{"a": 1}, {"a": 2}] + assert len(t.transform(records)) == 2 + + def test_row_filter_unsupported_op_raises(self): + cfg = _make_config("rf", conditions=[{"field": "x", "operator": "regex", "value": ".*"}]) + t = RowFilterProvider(cfg) + with pytest.raises(ValueError, match="Unsupported operator"): + t.transform([{"x": "hello"}]) + + +class TestTruncators: + def test_length_truncator(self): + cfg = _make_config("lt", max_length=5, fields=["text"]) + t = LengthTruncator(cfg) + result = t.transform({"text": "abcdefghij", "other": "keep"}) + assert result["text"] == "abcde" + assert result["other"] == "keep" + + def test_length_truncator_short_field_unchanged(self): + cfg = _make_config("lt", max_length=100, fields=["text"]) + t = LengthTruncator(cfg) + result = t.transform({"text": "short"}) + assert result["text"] == "short" + + def test_length_truncator_missing_field_ignored(self): + cfg = _make_config("lt", max_length=5, fields=["missing"]) + t = LengthTruncator(cfg) + result = t.transform({"text": "hello"}) + assert result["text"] == "hello" + + +# ─── Graph Build Edge Cases ────────────────────────────────────────────────── + +class TestCypherBuilderEdgeCases: + def test_node_special_chars_in_properties(self): + n = Node(id="n1", labels=["User"], properties={"bio": "It's a \"quote\" & "}) + query, params = node_to_cypher(n) + assert params["props"]["bio"] == "It's a \"quote\" & " + assert "MERGE" in query + + def test_node_none_value_in_properties(self): + n = Node(id="n1", labels=["User"], properties={"name": None}) + query, params = node_to_cypher(n) + assert params["props"]["name"] is None + + def test_node_empty_labels_uses_default(self): + n = Node(id="n1", labels=[], properties={"x": 1}) + query, params = node_to_cypher(n) + assert "Node" in query + + def test_node_multiple_labels(self): + n = Node(id="n1", labels=["Person", "Employee"], properties={}) + query, params = node_to_cypher(n) + assert "Person:Employee" in query + + def test_edge_with_properties(self): + e = Edge(id="e1", source_id="a", target_id="b", label="KNOWS", properties={"since": 2020}) + query, params = edge_to_cypher(e) + assert params["props"]["since"] == 2020 + + def test_edge_empty_properties(self): + e = Edge(id="e1", source_id="a", target_id="b", label="KNOWS") + query, params = edge_to_cypher(e) + assert params["props"] == {} + + def test_node_tenant_multi_label(self): + n = Node(id="n1", labels=["Person", "Admin"], properties={}) + query, params = node_to_cypher(n, tenant_id="t1") + assert "__Person__t1__" in query + assert "__Admin__t1__" in query + + +# ─── Schema Discovery Tests ───────────────────────────────────────────────── + +class TestCSVSchemaDiscovery: + def test_discover_from_temp_csv(self, tmp_csv): + discovery = CSVSchemaDiscoveryProvider(source=tmp_csv) + schema = discovery.discover_schema() + assert isinstance(schema, ETLSchema) + assert "id" in schema.load.document_node.fields + assert "name" in schema.load.document_node.fields + assert "email" in schema.load.document_node.fields + assert schema.schema_id == "discovered-test_data" + + def test_discover_from_project_csv(self): + csv_path = Path(__file__).parent.parent / "examples" / "cloud" / "notebooks" / "data" / "users.csv" + if not csv_path.exists(): + pytest.skip("users.csv not found in expected location") + discovery = CSVSchemaDiscoveryProvider(source=csv_path) + schema = discovery.discover_schema() + assert "id" in schema.load.document_node.fields + assert "name" in schema.load.document_node.fields + assert "email" in schema.load.document_node.fields + assert "role" in schema.load.document_node.fields + assert "account_id" in schema.load.document_node.fields + + def test_discover_file_not_found(self, tmp_path): + discovery = CSVSchemaDiscoveryProvider(source=tmp_path / "nonexistent.csv") + with pytest.raises(FileNotFoundError): + discovery.discover_schema() + + +# ─── TransformerProviderConfig Tests ───────────────────────────────────────── + +class TestTransformerProviderConfig: + def test_basic_construction(self): + cfg = TransformerProviderConfig(name="test", type="normalizer", args={"x": 1}) + assert cfg.name == "test" + assert cfg.type == "normalizer" + assert cfg.args["x"] == 1 + + def test_parameters_syncs_to_args(self): + cfg = TransformerProviderConfig(name="test", parameters={"y": 2}) + assert cfg.args["y"] == 2 From 9514f305534563765ba9ed7dfc23afab9915727d Mon Sep 17 00:00:00 2001 From: Evan Erwee Date: Fri, 26 Jun 2026 11:18:04 -0400 Subject: [PATCH 2/2] docs: add Sphinx documentation + SageMaker example notebooks - 11 documentation pages (getting-started, lexical-graph-integration, pipeline, ingestors, schema-providers, transformers, graph-build, query-engine, multi-tenancy, examples index, API reference) - 19 tested SageMaker notebooks covering all features - Includes hybrid integration demos (document-graph + lexical-graph) --- document-graph/docs/api/index.md | 7 + document-graph/docs/conf.py | 23 + document-graph/docs/examples.md | 75 ++ document-graph/docs/getting-started.md | 99 ++ document-graph/docs/graph-build.md | 98 ++ document-graph/docs/index.md | 32 + document-graph/docs/ingestors.md | 89 ++ .../docs/lexical-graph-integration.md | 131 +++ document-graph/docs/multi-tenancy.md | 72 ++ document-graph/docs/pipeline.md | 134 +++ document-graph/docs/query-engine.md | 76 ++ document-graph/docs/schema-providers.md | 117 +++ document-graph/docs/transformers.md | 167 +++ document-graph/examples/cloud/install.sh | 5 + .../examples/cloud/notebooks/00-Cleanup.ipynb | 96 ++ .../examples/cloud/notebooks/00-Setup.ipynb | 15 + .../001-Combined-Extract-Load-Build.ipynb | 19 + .../notebooks/002-Full-Pipeline-Test.ipynb | 359 +++++++ .../notebooks/004-Graph-Write-and-Read.ipynb | 130 +++ ...ema-Providers-Integration-Validation.ipynb | 351 +++++++ .../notebooks/006-Transformer-Deep-Dive.ipynb | 264 +++++ .../cloud/notebooks/03a-Ingestors-Test.ipynb | 252 +++++ .../03b-Schema-Driven-Pipeline.ipynb | 217 ++++ .../notebooks/03c-JSON-Driven-Pipeline.ipynb | 190 ++++ .../03d-Field-Transformers-Test.ipynb | 414 ++++++++ .../notebooks/03e-Normalizers-Test.ipynb | 237 +++++ .../notebooks/03f-Constructors-Test.ipynb | 309 ++++++ .../03g-Document-Transformers-Test.ipynb | 157 +++ ...-Lexical-Integration-Data-Processing.ipynb | 259 +++++ ...7b-Lexical-Integration-Lexical-Setup.ipynb | 193 ++++ .../08a-Nelson-Mandela-Hybrid-Build.ipynb | 310 ++++++ .../08b-Nelson-Mandela-Hybrid-Query.ipynb | 253 +++++ .../09-Multi-Tenant-Coexistence-Demo.ipynb | 193 ++++ .../cloud/notebooks/data/README_FORMATS.md | 80 ++ .../cloud/notebooks/data/accounts.csv | 4 + .../data/create_additional_formats.py | 55 + .../cloud/notebooks/data/mapping.json | 387 +++++++ .../examples/cloud/notebooks/data/sample.json | 106 ++ .../cloud/notebooks/data/test_documents.csv | 21 + .../cloud/notebooks/data/test_documents.json | 242 +++++ .../notebooks/data/test_documents.parquet | Bin 0 -> 25036 bytes .../notebooks/data/test_documents.skip.csv | 42 + .../cloud/notebooks/data/test_documents.xml | 306 ++++++ .../cloud/notebooks/data/test_documents.yaml | 265 +++++ .../examples/cloud/notebooks/data/users.csv | 6 + .../notebooks/hybrid_data/data/authors.json | 100 ++ .../notebooks/hybrid_data/data/citations.csv | 6 + .../notebooks/hybrid_data/data/citations.xlsx | Bin 0 -> 10422 bytes .../hybrid_data/data/research_papers.csv | 6 + .../notebooks/mandela_data/mandela_events.csv | 10 + .../notebooks/mandela_data/organizations.xlsx | Bin 0 -> 5533 bytes .../mandela_data/research_papers.json | 89 ++ .../cloud/notebooks/utility_functions.py | 969 ++++++++++++++++++ .../examples/cloud/push-to-sagemaker.sh | 27 + 54 files changed, 8064 insertions(+) create mode 100644 document-graph/docs/api/index.md create mode 100644 document-graph/docs/conf.py create mode 100644 document-graph/docs/examples.md create mode 100644 document-graph/docs/getting-started.md create mode 100644 document-graph/docs/graph-build.md create mode 100644 document-graph/docs/index.md create mode 100644 document-graph/docs/ingestors.md create mode 100644 document-graph/docs/lexical-graph-integration.md create mode 100644 document-graph/docs/multi-tenancy.md create mode 100644 document-graph/docs/pipeline.md create mode 100644 document-graph/docs/query-engine.md create mode 100644 document-graph/docs/schema-providers.md create mode 100644 document-graph/docs/transformers.md create mode 100755 document-graph/examples/cloud/install.sh create mode 100644 document-graph/examples/cloud/notebooks/00-Cleanup.ipynb create mode 100644 document-graph/examples/cloud/notebooks/00-Setup.ipynb create mode 100644 document-graph/examples/cloud/notebooks/001-Combined-Extract-Load-Build.ipynb create mode 100644 document-graph/examples/cloud/notebooks/002-Full-Pipeline-Test.ipynb create mode 100644 document-graph/examples/cloud/notebooks/004-Graph-Write-and-Read.ipynb create mode 100644 document-graph/examples/cloud/notebooks/005-Schema-Providers-Integration-Validation.ipynb create mode 100644 document-graph/examples/cloud/notebooks/006-Transformer-Deep-Dive.ipynb create mode 100644 document-graph/examples/cloud/notebooks/03a-Ingestors-Test.ipynb create mode 100644 document-graph/examples/cloud/notebooks/03b-Schema-Driven-Pipeline.ipynb create mode 100644 document-graph/examples/cloud/notebooks/03c-JSON-Driven-Pipeline.ipynb create mode 100644 document-graph/examples/cloud/notebooks/03d-Field-Transformers-Test.ipynb create mode 100644 document-graph/examples/cloud/notebooks/03e-Normalizers-Test.ipynb create mode 100644 document-graph/examples/cloud/notebooks/03f-Constructors-Test.ipynb create mode 100644 document-graph/examples/cloud/notebooks/03g-Document-Transformers-Test.ipynb create mode 100644 document-graph/examples/cloud/notebooks/07a-Lexical-Integration-Data-Processing.ipynb create mode 100644 document-graph/examples/cloud/notebooks/07b-Lexical-Integration-Lexical-Setup.ipynb create mode 100644 document-graph/examples/cloud/notebooks/08a-Nelson-Mandela-Hybrid-Build.ipynb create mode 100644 document-graph/examples/cloud/notebooks/08b-Nelson-Mandela-Hybrid-Query.ipynb create mode 100644 document-graph/examples/cloud/notebooks/09-Multi-Tenant-Coexistence-Demo.ipynb create mode 100644 document-graph/examples/cloud/notebooks/data/README_FORMATS.md create mode 100644 document-graph/examples/cloud/notebooks/data/accounts.csv create mode 100644 document-graph/examples/cloud/notebooks/data/create_additional_formats.py create mode 100644 document-graph/examples/cloud/notebooks/data/mapping.json create mode 100644 document-graph/examples/cloud/notebooks/data/sample.json create mode 100644 document-graph/examples/cloud/notebooks/data/test_documents.csv create mode 100644 document-graph/examples/cloud/notebooks/data/test_documents.json create mode 100644 document-graph/examples/cloud/notebooks/data/test_documents.parquet create mode 100644 document-graph/examples/cloud/notebooks/data/test_documents.skip.csv create mode 100644 document-graph/examples/cloud/notebooks/data/test_documents.xml create mode 100644 document-graph/examples/cloud/notebooks/data/test_documents.yaml create mode 100644 document-graph/examples/cloud/notebooks/data/users.csv create mode 100644 document-graph/examples/cloud/notebooks/hybrid_data/data/authors.json create mode 100644 document-graph/examples/cloud/notebooks/hybrid_data/data/citations.csv create mode 100644 document-graph/examples/cloud/notebooks/hybrid_data/data/citations.xlsx create mode 100644 document-graph/examples/cloud/notebooks/hybrid_data/data/research_papers.csv create mode 100644 document-graph/examples/cloud/notebooks/mandela_data/mandela_events.csv create mode 100644 document-graph/examples/cloud/notebooks/mandela_data/organizations.xlsx create mode 100644 document-graph/examples/cloud/notebooks/mandela_data/research_papers.json create mode 100644 document-graph/examples/cloud/notebooks/utility_functions.py create mode 100755 document-graph/examples/cloud/push-to-sagemaker.sh diff --git a/document-graph/docs/api/index.md b/document-graph/docs/api/index.md new file mode 100644 index 00000000..fd721443 --- /dev/null +++ b/document-graph/docs/api/index.md @@ -0,0 +1,7 @@ +# API Reference + +```{toctree} +:maxdepth: 1 + +modules +``` diff --git a/document-graph/docs/conf.py b/document-graph/docs/conf.py new file mode 100644 index 00000000..4e0cba76 --- /dev/null +++ b/document-graph/docs/conf.py @@ -0,0 +1,23 @@ +"""Sphinx configuration for document-graph.""" + +project = 'document-graph' +copyright = '2026, Evan Erwee' +author = 'Evan Erwee' +release = '3.0.4' + +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.napoleon', + 'sphinx.ext.viewcode', + 'myst_parser', +] + +templates_path = ['_templates'] +exclude_patterns = ['_build'] + +html_theme = 'furo' +html_static_path = ['_static'] + +# MyST (markdown support) +source_suffix = {'.rst': 'restructuredtext', '.md': 'markdown'} +myst_enable_extensions = ['colon_fence', 'deflist'] diff --git a/document-graph/docs/examples.md b/document-graph/docs/examples.md new file mode 100644 index 00000000..dadd8211 --- /dev/null +++ b/document-graph/docs/examples.md @@ -0,0 +1,75 @@ +# Examples & Notebooks + +All notebooks are in `examples/cloud/notebooks/` and run on SageMaker with Neptune + OpenSearch. + +## Setup & Utilities + +| Notebook | Purpose | +|----------|---------| +| `00-Setup` | Verify Neptune connection, write test node | +| `00-Cleanup` | Remove old test tenants from Neptune | + +## Core Pipeline + +| Notebook | Purpose | Docs | +|----------|---------|------| +| `001-Combined-Extract-Load-Build` | Full pipeline: CSV → ingest → transform → Neptune | [Pipeline](pipeline.md) | +| `002-Full-Pipeline-Test` | Every stage tested individually | [Pipeline](pipeline.md) | + +## Component Tests + +| Notebook | Feature | Docs | +|----------|---------|------| +| `03a-Ingestors-Test` | Column select, rename, row filter | [Ingestors](ingestors.md) | +| `03b-Schema-Driven-Pipeline` | Schema → automatic pipeline | [Schema Providers](schema-providers.md) | +| `03c-JSON-Driven-Pipeline` | JSON mapping file → Neptune | [Schema Providers](schema-providers.md) | +| `03d-Field-Transformers-Test` | JSON flatten, UUID, regex | [Transformers](transformers.md) | +| `03e-Normalizers-Test` | Whitespace, nulls, case | [Transformers](transformers.md) | +| `03f-Constructors-Test` | Cypher generation, batch | [Graph Build](graph-build.md) | +| `03g-Document-Transformers-Test` | JSON→rows, text chunker, PII | [Transformers](transformers.md) | + +## Feature Deep-Dives + +| Notebook | Feature | Docs | +|----------|---------|------| +| `004-Graph-Write-and-Read` | Node/Edge CRUD + typed queries | [Graph Build](graph-build.md), [Query Engine](query-engine.md) | +| `005-Schema-Providers-Integration-Validation` | CSV, JSON, S3, Static, Factory, validation | [Schema Providers](schema-providers.md) | +| `006-Transformer-Deep-Dive` | All 7 transformer categories + custom | [Transformers](transformers.md) | + +## Hybrid Integration (Document-Graph + Lexical-Graph) + +| Notebook | Purpose | Docs | +|----------|---------|------| +| `07a-Lexical-Integration-Data-Processing` | Structured data → Neptune → LlamaIndex Documents (with lineage) | [Lexical-Graph Integration](lexical-graph-integration.md) | +| `07b-Lexical-Integration-Lexical-Setup` | Index into lexical-graph → semantic query → correlate back | [Lexical-Graph Integration](lexical-graph-integration.md) | + +## Real-World Demos + +| Notebook | Purpose | +|----------|---------| +| `08a-Nelson-Mandela-Hybrid-Build` | Wikipedia + structured events → hybrid graph | +| `08b-Nelson-Mandela-Hybrid-Query` | Semantic search + document-graph correlation | + +## Multi-Tenancy + +| Notebook | Purpose | Docs | +|----------|---------|------| +| `09-Multi-Tenant-Coexistence-Demo` | Two tenants, same graph, complete isolation | [Multi-Tenancy](multi-tenancy.md) | + +## Running Notebooks + +### On SageMaker + +```bash +# Sync from S3 +aws s3 sync s3://graphrag-artifacts-705909755305/document-graph-notebooks/ ~/SageMaker/document-graph/ --delete + +# Install +pip install --no-deps --force-reinstall ~/SageMaker/document-graph/wheels/document_graph-*.whl +``` + +### Prerequisites + +- Neptune Database (1.4.x+) — for graph storage +- OpenSearch Serverless — for vector search (hybrid notebooks only) +- IAM role with: `neptune-db:*`, `aoss:APIAccessAll`, `bedrock:InvokeModel` diff --git a/document-graph/docs/getting-started.md b/document-graph/docs/getting-started.md new file mode 100644 index 00000000..12340b39 --- /dev/null +++ b/document-graph/docs/getting-started.md @@ -0,0 +1,99 @@ +# Getting Started + +## Prerequisites + +### Infrastructure + +This project relies on [graphrag-toolkit](https://github.com/awslabs/graphrag-toolkit) for infrastructure provisioning. Deploy the CloudFormation stack from: + +> https://github.com/awslabs/graphrag-toolkit/tree/main/examples/lexical-graph/cloudformation-templates + +This provisions: +- **Amazon Neptune** (graph database) +- **Amazon OpenSearch Serverless** (vector store for hybrid search) +- **Amazon SageMaker Notebook** (for running the example notebooks) + +### SageMaker Notebooks + +All example notebooks in `examples/cloud/notebooks/` are designed to run on **SageMaker** within the VPC provisioned by the CloudFormation stack. They require: +- Network access to Neptune (port 8182) +- Network access to OpenSearch Serverless (port 443) +- IAM role with `neptune-db:*`, `aoss:APIAccessAll`, `bedrock:InvokeModel` + +:::{note} +The notebooks will NOT work on a local machine — they require VPC-internal access to Neptune and OpenSearch. +::: + +Get a working document graph in 5 minutes. + +## Install + +```bash +pip install document-graph +``` + +## Connect to Neptune + +```python +from graphrag_toolkit.storage import GraphStoreFactory + +gs = GraphStoreFactory.for_graph_store('neptune-db://your-endpoint:8182').__enter__() +``` + +## Write Your First Node + +```python +from document_graph import Node +from document_graph.graph_build import node_to_cypher + +node = Node( + id="doc-001", + labels=["Document"], + properties={"title": "Getting Started", "status": "active"} +) + +cypher, params = node_to_cypher(node, tenant_id="acme") +gs.execute_query(cypher, params) +``` + +## Write an Edge + +```python +from document_graph import Edge +from document_graph.graph_build import edge_to_cypher + +edge = Edge( + id="edge-001", + source_id="doc-001", + target_id="doc-002", + label="REFERENCES", + properties={"weight": 0.9} +) + +cypher, params = edge_to_cypher(edge, tenant_id="acme") +gs.execute_query(cypher, params) +``` + +## Query + +```python +from document_graph.query import DocumentGraphQueryEngine + +engine = DocumentGraphQueryEngine(gs, tenant_id="acme") + +# Get all Document nodes +docs = engine.get_nodes("Document", limit=10) + +# Find by property +results = engine.find_by_property("Document", "title", "Getting Started") + +# Raw Cypher +results = engine.query("MATCH (n) RETURN n LIMIT 5") +``` + +## Next Steps + +- {doc}`schema-providers` — Auto-discover schemas from data files +- {doc}`transformers` — Clean and enrich data before graph build +- {doc}`graph-build` — Batch operations and tenant scoping +- {doc}`multi-tenancy` — Isolation guarantees diff --git a/document-graph/docs/graph-build.md b/document-graph/docs/graph-build.md new file mode 100644 index 00000000..ee317af9 --- /dev/null +++ b/document-graph/docs/graph-build.md @@ -0,0 +1,98 @@ +# Graph Build + +Generate Cypher MERGE statements from `Node` and `Edge` models and execute them against Neptune. + +## Models + +### Node + +```python +from document_graph import Node + +node = Node( + id="person-001", + labels=["Person", "Employee"], + properties={"name": "Alice", "department": "Engineering"} +) +``` + +### Edge + +```python +from document_graph import Edge + +edge = Edge( + id="edge-001", + source_id="person-001", + target_id="project-042", + label="WORKS_ON", + properties={"since": "2024-01-15"} +) +``` + +## node_to_cypher + +Generates a MERGE statement for a single node: + +```python +from document_graph.graph_build import node_to_cypher + +cypher, params = node_to_cypher(node, tenant_id="acme") +# cypher: "MERGE (n:__Person__acme__:__Employee__acme__ {id: $id_val}) SET n += $props" +# params: {"id_val": "person-001", "props": {"name": "Alice", "department": "Engineering"}} + +gs.execute_query(cypher, params) +``` + +## edge_to_cypher + +Generates a MERGE statement for a single edge: + +```python +from document_graph.graph_build import edge_to_cypher + +cypher, params = edge_to_cypher(edge, tenant_id="acme") +# cypher: "MATCH (a {id: $src_id}), (b {id: $tgt_id}) MERGE (a)-[r:WORKS_ON]->(b) SET r += $props" + +gs.execute_query(cypher, params) +``` + +## Batch Operations + +Process multiple nodes/edges efficiently: + +```python +from document_graph.graph_build import batch_nodes_to_cypher, batch_edges_to_cypher + +nodes = [ + Node(id="p1", labels=["Person"], properties={"name": "Alice"}), + Node(id="p2", labels=["Person"], properties={"name": "Bob"}), +] + +statements = batch_nodes_to_cypher(nodes, tenant_id="acme") +for cypher, params in statements: + gs.execute_query(cypher, params) + +# Same pattern for edges +edge_statements = batch_edges_to_cypher(edges, tenant_id="acme") +for cypher, params in edge_statements: + gs.execute_query(cypher, params) +``` + +## Tenant ID Scoping + +When `tenant_id` is provided, all labels are encoded as `__Label__tenant_id__`: + +```python +node = Node(id="x", labels=["Document"]) + +# Without tenant +cypher, _ = node_to_cypher(node) +# → MERGE (n:Document {id: $id_val}) SET n += $props + +# With tenant +cypher, _ = node_to_cypher(node, tenant_id="acme") +# → MERGE (n:__Document__acme__ {id: $id_val}) SET n += $props +``` + +This ensures complete data isolation between tenants at the label level. See {doc}`multi-tenancy` for details. diff --git a/document-graph/docs/index.md b/document-graph/docs/index.md new file mode 100644 index 00000000..4d15c51e --- /dev/null +++ b/document-graph/docs/index.md @@ -0,0 +1,32 @@ +# Document Graph Documentation + +Structured data ETL for graph-enhanced GenAI applications. + +```{toctree} +:maxdepth: 2 +:caption: Contents + +getting-started +lexical-graph-integration +pipeline +ingestors +schema-providers +transformers +graph-build +query-engine +multi-tenancy +examples +api/index +``` + +## Overview + +Document-graph extends [graphrag-toolkit](https://github.com/awslabs/graphrag-toolkit) to handle **structured data** (CSV, Excel, JSON) alongside unstructured text in the same Neptune graph database. + +| Feature | Module | +|---------|--------| +| Schema Providers | Auto-discover or define ETL schemas | +| Transformers | 20+ built-in data transformers | +| Graph Build | Cypher generation with tenant scoping | +| Query Engine | Typed node queries | +| Hybrid Search | Semantic search across both graph types | diff --git a/document-graph/docs/ingestors.md b/document-graph/docs/ingestors.md new file mode 100644 index 00000000..f8cf1bea --- /dev/null +++ b/document-graph/docs/ingestors.md @@ -0,0 +1,89 @@ +# Ingestors + +Ingestors shape DataFrames before transformation — selecting columns, renaming fields, and filtering rows. + +## Configuration + +All ingestors use `IngestorProviderConfig`: + +```python +from document_graph.ingest.ingestors_provider_config import IngestorProviderConfig + +config = IngestorProviderConfig( + name='my_ingestor', # identifier + type='column', # 'column' or 'row' + args={...} # ingestor-specific arguments +) +``` + +## Column Ingestors + +### Column Selector + +Keep only specified columns. + +```python +from document_graph.ingest.column.column_selector import ColumnSelectorProvider + +config = IngestorProviderConfig(name='select', type='column', args={ + 'columns': ['id', 'name', 'email', 'role'] +}) +df = ColumnSelectorProvider(config).ingest(df) +``` + +### Column Renamer + +Rename columns. + +```python +from document_graph.ingest.column.column_renamer import ColumnRenamerProvider + +config = IngestorProviderConfig(name='rename', type='column', args={ + 'mapping': {'id': 'user_id', 'name': 'full_name'} +}) +df = ColumnRenamerProvider(config).ingest(df) +``` + +## Row Ingestors + +### Skip Row + +Remove rows matching specific field values. + +```python +from document_graph.ingest.row.skip_row import SkipRowProvider + +config = IngestorProviderConfig(name='skip', type='row', args={ + 'field': 'role', + 'values': ['viewer', 'inactive'] +}) +df = SkipRowProvider(config).ingest(df) # Removes rows where role is viewer or inactive +``` + +## Chaining Ingestors + +Ingestors return DataFrames, so they chain naturally: + +```python +df = pd.read_csv('users.csv') + +# Chain: select → rename → filter +df = ColumnSelectorProvider(IngestorProviderConfig( + name='sel', type='column', args={'columns': ['id', 'name', 'role', 'status']} +)).ingest(df) + +df = ColumnRenamerProvider(IngestorProviderConfig( + name='ren', type='column', args={'mapping': {'id': 'user_id'}} +)).ingest(df) + +df = SkipRowProvider(IngestorProviderConfig( + name='skip', type='row', args={'field': 'status', 'values': ['deleted']} +)).ingest(df) + +records = df.to_dict('records') # Ready for transformers +``` + +## Related Notebooks + +- `03a-Ingestors-Test.ipynb` — all ingestor types tested +- `002-Full-Pipeline-Test.ipynb` — ingestors in full pipeline context diff --git a/document-graph/docs/lexical-graph-integration.md b/document-graph/docs/lexical-graph-integration.md new file mode 100644 index 00000000..206d9272 --- /dev/null +++ b/document-graph/docs/lexical-graph-integration.md @@ -0,0 +1,131 @@ +# Lexical-Graph Integration + +Document-graph and lexical-graph coexist in the same Neptune database, enabling hybrid queries that combine structured data retrieval with semantic search. + +## How It Works + +``` +┌─────────────────────────────────────────────────────────┐ +│ Neptune Database │ +│ │ +│ Document-Graph (structured) Lexical-Graph (semantic)│ +│ __User__tenant__ __Source__ │ +│ __Account__tenant__ __Chunk__ │ +│ __Event__tenant__ __Topic__ │ +│ __Statement__ │ +│ __Entity__ │ +│ │ +│ Connected by: shared Neptune + entity correlation │ +└─────────────────────────────────────────────────────────┘ + ↕ ↕ + Cypher queries Semantic vector search + (exact match) (OpenSearch Serverless) +``` + +## Setup + +```python +from graphrag_toolkit.lexical_graph.storage import GraphStoreFactory, VectorStoreFactory +from graphrag_toolkit.lexical_graph import LexicalGraphIndex, LexicalGraphQueryEngine + +GRAPH_STORE = 'neptune-db://your-endpoint:8182' +VECTOR_STORE = 'aoss://your-collection.us-east-1.aoss.amazonaws.com' + +graph_store = GraphStoreFactory.for_graph_store(GRAPH_STORE) +vector_store = VectorStoreFactory.for_vector_store(VECTOR_STORE) +``` + +## Step 1: Write structured data to Neptune + +Use document-graph to write typed nodes: + +```python +from document_graph.graph_build import node_to_cypher +from document_graph import Node + +TENANT = 'my_app' +node = Node(id='u1', labels=['User'], properties={'name': 'Alice', 'role': 'admin'}) +cypher, params = node_to_cypher(node, tenant_id=TENANT) +gs.execute_query(cypher, params) +``` + +## Step 2: Convert to LlamaIndex Documents + +The key is embedding **lineage** in the document text so it survives the lexical-graph chunking pipeline: + +```python +from llama_index.core.schema import Document + +docs = [] +for record in records: + node_id = record['id'] + # Lineage header survives chunking + text = f'[User | {node_id} | {TENANT}]\n' + text += '\n'.join(f'{k}: {v}' for k, v in record.items() if v) + + doc = Document( + text=text, + metadata={ + 'source': { + 'sourceId': f'User:{node_id}:{TENANT}', + 'metadata': {'node_type': 'User', 'node_id': node_id, 'tenant': TENANT} + } + } + ) + docs.append(doc) +``` + +:::{note} +The `source.sourceId` and `source.metadata` structure is how graphrag-toolkit persists metadata to `__Source__` nodes in Neptune. +::: + +## Step 3: Index into lexical-graph + +```python +graph_index = LexicalGraphIndex(graph_store, vector_store) +graph_index.extract_and_build(docs, show_progress=True) +``` + +This creates the lexical graph (topics, statements, entities) alongside your document-graph nodes. + +## Step 4: Semantic query + +```python +query_engine = LexicalGraphQueryEngine.for_traversal_based_search(graph_store, vector_store) +results = query_engine.retrieve('Who are the admin users?') +``` + +## Step 5: Correlate back to document-graph + +Extract lineage from query results and look up original nodes: + +```python +import re + +for r in results: + text = r.node.text + match = re.search(r'\[(\w+) \| ([\w-]+) \| (\w+)\]', text) + if match: + ntype, node_id, tenant = match.groups() + label = f'__{ntype}__{tenant}__' + original = gs.execute_query( + f"MATCH (n:`{label}`) WHERE n.id = '{node_id}' RETURN properties(n) as props" + ) + # original contains the full structured node data +``` + +## Lineage Strategies + +| Strategy | Reliability | How | +|----------|------------|-----| +| **Text header** | High | `[Type \| node_id \| tenant]` in first line — survives chunking | +| **Source metadata** | Medium | `source.metadata` → stored on `__Source__` node | +| **Entity correlation** | Fallback | Match extracted entities (Person, Org) to Neptune nodes | + +## Key Points + +1. **Same database** — document-graph and lexical-graph share Neptune +2. **Different labels** — no collision (`__User__tenant__` vs `__Topic__`) +3. **Tenant isolation** — both respect tenant-scoped labels +4. **Lineage** — embed identifiers in text to survive chunking +5. **Entity bridge** — lexical-graph extracts entities that match document-graph nodes diff --git a/document-graph/docs/multi-tenancy.md b/document-graph/docs/multi-tenancy.md new file mode 100644 index 00000000..50e406b4 --- /dev/null +++ b/document-graph/docs/multi-tenancy.md @@ -0,0 +1,72 @@ +# Multi-Tenancy + +Document-graph provides tenant isolation through label encoding in Neptune. + +## How It Works + +All node labels are encoded with the tenant ID using the pattern: + +``` +__Type__tenant_id__ +``` + +For example, a `Document` node for tenant `acme`: + +``` +__Document__acme__ +``` + +A node with multiple labels: + +```python +Node(id="x", labels=["Document", "Report"]) +# With tenant_id="acme" becomes labels: ["__Document__acme__", "__Report__acme__"] +``` + +## Isolation Guarantees + +- **Label-level isolation**: Each tenant's nodes use unique labels, making it impossible to accidentally query across tenants. +- **No shared labels**: Tenant A's `__Document__tenantA__` never overlaps with Tenant B's `__Document__tenantB__`. +- **Query scoping**: `DocumentGraphQueryEngine(gs, tenant_id="acme")` automatically scopes all `get_nodes` and `find_by_property` calls. + +```python +from document_graph.query import DocumentGraphQueryEngine + +# These two engines see completely different data +engine_a = DocumentGraphQueryEngine(gs, tenant_id="tenant_a") +engine_b = DocumentGraphQueryEngine(gs, tenant_id="tenant_b") + +docs_a = engine_a.get_nodes("Document") # → MATCH (n:__Document__tenant_a__) +docs_b = engine_b.get_nodes("Document") # → MATCH (n:__Document__tenant_b__) +``` + +## Admin Cross-Tenant View + +Omit `tenant_id` to query raw labels (useful for admin/debugging): + +```python +admin_engine = DocumentGraphQueryEngine(gs) + +# Raw Cypher to find all tenant-scoped Document nodes +all_docs = admin_engine.query( + "MATCH (n) WHERE any(l IN labels(n) WHERE l STARTS WITH '__Document__') RETURN n LIMIT 100" +) +``` + +## Tenant Cleanup + +Remove all data for a specific tenant: + +```python +tenant_id = "old_tenant" + +# Delete all nodes with tenant-scoped labels +gs.execute_query( + "MATCH (n) WHERE any(l IN labels(n) WHERE l CONTAINS $tid) DETACH DELETE n", + {"tid": f"__{tenant_id}__"} +) +``` + +:::{warning} +Tenant cleanup is irreversible. Always verify the tenant ID before executing. +::: diff --git a/document-graph/docs/pipeline.md b/document-graph/docs/pipeline.md new file mode 100644 index 00000000..6dd671a0 --- /dev/null +++ b/document-graph/docs/pipeline.md @@ -0,0 +1,134 @@ +# Pipeline + +The document-graph pipeline processes structured data through four stages: **Extract → Ingest → Transform → Build**. + +## Overview + +``` +Source File (CSV/JSON/Excel/Parquet) + │ + ├── Extract: Read into DataFrame + │ + ├── Ingest: Column select, rename, row filter + │ + ├── Transform: Normalize, enrich, reshape + │ + └── Build: Generate Cypher → Write to Neptune +``` + +## Extract + +Reads a data file into a pandas DataFrame with schema discovery. + +```python +from document_graph.pipeline.extract.extract_provider_config import ExtractProviderConfig +from document_graph.pipeline.extract.extract_provider_csv import CSVExtractProvider +from document_graph.config import DocumentGraphConfig + +config = ExtractProviderConfig(type='csv', source='users.csv', document_id='users-pipeline') +provider = CSVExtractProvider(config, DocumentGraphConfig()) +result = provider.extract('data/users.csv') + +print(f'Rows: {result.dataframe.shape[0]}') +print(f'Schema: {result.extracted_schema}') +print(f'Nodes: {len(result.nodes)}') +``` + +Supported formats: +- CSV (`CSVExtractProvider`) +- Excel (`ExcelExtractProvider`) +- JSON (`JSONExtractProvider`) +- Parquet (`ParquetExtractProvider`) + +## Ingest + +Shapes the DataFrame before transformation. + +```python +from document_graph.ingest.ingestors_provider_config import IngestorProviderConfig +from document_graph.ingest.column.column_selector import ColumnSelectorProvider +from document_graph.ingest.column.column_renamer import ColumnRenamerProvider +from document_graph.ingest.row.skip_row import SkipRowProvider + +# Select columns +config = IngestorProviderConfig(name='select', type='column', args={'columns': ['id', 'name', 'role']}) +df = ColumnSelectorProvider(config).ingest(df) + +# Rename columns +config = IngestorProviderConfig(name='rename', type='column', args={'mapping': {'id': 'user_id'}}) +df = ColumnRenamerProvider(config).ingest(df) + +# Filter rows +config = IngestorProviderConfig(name='filter', type='row', args={'field': 'role', 'values': ['viewer']}) +df = SkipRowProvider(config).ingest(df) # Removes viewers +``` + +## Transform + +Processes records through one or more transformers (see [Transformers](transformers.md) for all types). + +```python +from document_graph.transform.transformer_provider_config import TransformerProviderConfig +from document_graph.transform.normalizers.normalize_whitespace_provider import NormalizeWhitespaceProvider +from document_graph.transform.graph_transformers.row_to_node import RowToNodeTransformer + +records = df.to_dict('records') + +# Normalize +records = NormalizeWhitespaceProvider(TransformerProviderConfig(name='ws', args={})).transform(records) + +# Convert to graph nodes +nodes = RowToNodeTransformer(TransformerProviderConfig(name='r2n', args={'type': 'User'})).transform(records) +``` + +## Build + +Generates Cypher and writes to Neptune. + +```python +from graphrag_toolkit.lexical_graph.storage import GraphStoreFactory +from document_graph.graph_build import node_to_cypher, edge_to_cypher +from document_graph import Node, Edge + +gs = GraphStoreFactory.for_graph_store('neptune-db://endpoint:8182').__enter__() +TENANT = 'my_app' + +for record in records: + node = Node(id=record['user_id'], labels=['User'], properties=record) + cypher, params = node_to_cypher(node, tenant_id=TENANT) + gs.execute_query(cypher, params) +``` + +## Full Example + +```python +import pandas as pd +from document_graph.ingest.ingestors_provider_config import IngestorProviderConfig +from document_graph.ingest.column.column_selector import ColumnSelectorProvider +from document_graph.transform.transformer_provider_config import TransformerProviderConfig +from document_graph.transform.normalizers.normalize_whitespace_provider import NormalizeWhitespaceProvider +from document_graph.graph_build import node_to_cypher +from document_graph import Node + +# Extract +df = pd.read_csv('data/users.csv') + +# Ingest +config = IngestorProviderConfig(name='sel', type='column', args={'columns': ['id', 'name', 'email', 'role']}) +df = ColumnSelectorProvider(config).ingest(df) + +# Transform +records = df.to_dict('records') +records = NormalizeWhitespaceProvider(TransformerProviderConfig(name='ws', args={})).transform(records) + +# Build +for r in records: + node = Node(id=str(r['id']), labels=['User'], properties=r) + cypher, params = node_to_cypher(node, tenant_id='my_app') + gs.execute_query(cypher, params) +``` + +## Related Notebooks + +- `001-Combined-Extract-Load-Build.ipynb` — full pipeline in one flow +- `002-Full-Pipeline-Test.ipynb` — all stages tested individually diff --git a/document-graph/docs/query-engine.md b/document-graph/docs/query-engine.md new file mode 100644 index 00000000..81707e88 --- /dev/null +++ b/document-graph/docs/query-engine.md @@ -0,0 +1,76 @@ +# Query Engine + +`DocumentGraphQueryEngine` provides typed queries over the document graph with automatic tenant isolation. + +## Setup + +```python +from graphrag_toolkit.storage import GraphStoreFactory +from document_graph.query import DocumentGraphQueryEngine + +gs = GraphStoreFactory.for_graph_store('neptune-db://your-endpoint:8182').__enter__() +engine = DocumentGraphQueryEngine(gs, tenant_id="acme") +``` + +## get_nodes + +Retrieve all nodes of a given type: + +```python +docs = engine.get_nodes("Document", limit=50) +# Internally queries: MATCH (n:__Document__acme__) RETURN n LIMIT 50 +``` + +## find_by_property + +Find nodes matching a property value: + +```python +results = engine.find_by_property("Person", "name", "Alice") +# Internally queries: MATCH (n:__Person__acme__ {name: $val}) RETURN n +``` + +## get_relationships + +Traverse typed relationships: + +```python +results = engine.get_relationships("Person", "WORKS_ON", "Project", limit=25) +# Internally queries: MATCH (a:__Person__acme__)-[r:WORKS_ON]->(b:__Project__acme__) RETURN a, r, b LIMIT 25 +``` + +## query (Raw Cypher) + +Execute arbitrary Cypher when you need full control: + +```python +results = engine.query( + "MATCH (n:__Document__acme__)-[:REFERENCES]->(m) RETURN n.title, m.id LIMIT 10" +) + +# With parameters +results = engine.query( + "MATCH (n:__Person__acme__) WHERE n.age > $min_age RETURN n", + params={"min_age": 30} +) +``` + +## Tenant Isolation + +The engine automatically scopes all typed queries to the configured `tenant_id`: + +```python +# Tenant-scoped — only sees "acme" data +acme_engine = DocumentGraphQueryEngine(gs, tenant_id="acme") +acme_docs = acme_engine.get_nodes("Document") + +# Different tenant — completely isolated +beta_engine = DocumentGraphQueryEngine(gs, tenant_id="beta") +beta_docs = beta_engine.get_nodes("Document") + +# No tenant — queries raw labels (admin/cross-tenant) +admin_engine = DocumentGraphQueryEngine(gs) +all_docs = admin_engine.get_nodes("Document") # Matches label "Document" directly +``` + +See {doc}`multi-tenancy` for isolation guarantees and cleanup. diff --git a/document-graph/docs/schema-providers.md b/document-graph/docs/schema-providers.md new file mode 100644 index 00000000..698f6bbf --- /dev/null +++ b/document-graph/docs/schema-providers.md @@ -0,0 +1,117 @@ +# Schema Providers + +Schema providers load or discover ETL schemas that define how data flows through the pipeline. + +## SchemaProviderConfig + +All providers are configured with a single model: + +```python +from document_graph.schema.providers.schema_provider_config import SchemaProviderConfig + +config = SchemaProviderConfig( + type="csv", # Provider type + schema_id="my-dataset", # Optional override + connection_config={ # Provider-specific params + "path": "/data/users.csv" + } +) +``` + +**Supported types:** `file`, `s3`, `static`, `csv`, `json`, `excel`, `glue`, `parquet`, `yaml`, `xml` + +## Provider Types + +### CSV Provider + +```python +config = SchemaProviderConfig( + type="csv", + connection_config={"path": "data/events.csv", "delimiter": ","} +) +``` + +### JSON Provider + +```python +config = SchemaProviderConfig( + type="json", + connection_config={"path": "data/records.json"} +) +``` + +### S3 Provider + +```python +config = SchemaProviderConfig( + type="s3", + connection_config={"bucket": "my-bucket", "key": "schemas/pipeline.yaml"} +) +``` + +### Static Provider + +Returns a hardcoded default schema — useful for testing: + +```python +from document_graph.schema import StaticSchemaProvider + +provider = StaticSchemaProvider(config={}) +schema = provider.load_schema() +``` + +### Factory (Auto-resolution) + +```python +from document_graph.schema.providers.schema_provider_factory import get_schema_provider + +provider = get_schema_provider(config) +schema = provider.load_schema() +``` + +## ETLSchema Model + +The core schema model returned by all providers: + +```python +from document_graph.schema import ETLSchema, ExtractConfig, TransformConfig, LoadConfig + +schema = ETLSchema( + schema_id="my-pipeline", + description="Customer data pipeline", + extract=ExtractConfig( + source_type="csv", + bucket=None, + prefix=None, + file_type="csv" + ), + transform=TransformConfig( + chunking=ChunkingConfig(strategy="by_heading", min_length=100), + metadata_mapping=MetadataMapping(title="document.title"), + entity_extraction=EntityExtractionConfig(method="ner"), + normalize=NormalizeConfig(remove_headers=True) + ), + load=LoadConfig( + document_node=NodeDefinition(type="Document", fields=["title", "status"]), + section_node=NodeDefinition(type="Section", fields=["text"]), + relationships=[ + RelationshipDefinition(type="HAS_SECTION", source="doc_id", target="section_id") + ] + ) +) +``` + +## Schema Discovery + +Auto-infer schemas from data files: + +```python +from document_graph.schema.discovery import SchemaDiscoveryRegistry + +registry = SchemaDiscoveryRegistry() + +# Discovers columns, types, and relationships from a CSV +schema = registry.discover("data/events.csv") +``` + +Supported discovery formats: CSV, JSON, Excel, Parquet, YAML, XML. diff --git a/document-graph/docs/transformers.md b/document-graph/docs/transformers.md new file mode 100644 index 00000000..08027e9f --- /dev/null +++ b/document-graph/docs/transformers.md @@ -0,0 +1,167 @@ +# Transformers + +20+ built-in transformers across 6 categories. All configured via `TransformerProviderConfig`. + +## TransformerProviderConfig + +```python +from document_graph.transform.transformer_provider_config import TransformerProviderConfig + +config = TransformerProviderConfig( + name="normalize_case", + type="normalizer", + args={"target_case": "lower", "fields": ["title", "description"]} +) +``` + +## Categories + +### Normalizers + +Clean and standardize raw values. + +| Transformer | Purpose | +|-------------|---------| +| `normalize_case` | Lowercase/uppercase/titlecase | +| `normalize_whitespace` | Collapse whitespace | +| `normalize_nulls` | Standardize null representations | +| `normalize_timestamp` | Parse dates to ISO format | +| `normalize_enum` | Map values to canonical enums | +| `normalize_spelling` | Fix common misspellings | + +```python +TransformerProviderConfig( + name="normalize_timestamp", + args={"fields": ["created_at"], "format": "%Y-%m-%d"} +) +``` + +### Field Transformers + +Operate on individual field values. + +| Transformer | Purpose | +|-------------|---------| +| `json_flattener` | Flatten nested JSON | +| `comma_flattener` | Split comma-delimited values | +| `json_array_expander` | Expand arrays into rows | +| `regex_cleaner` | Regex-based cleaning | +| `standardize_enum` | Normalize enumerations | +| `normalize_timestamp` | Date field normalization | +| `embedded_json` | Parse embedded JSON strings | +| `uuid_generator` | Generate UUIDs | +| `paired_flattener` | Flatten key-value pairs | + +```python +TransformerProviderConfig( + name="json_flattener", + args={"field": "metadata", "separator": "_"} +) +``` + +### Document Transformers + +Transform entire documents/records. + +| Transformer | Purpose | +|-------------|---------| +| `text_chunker` | Split text into chunks | +| `json_to_rows` | Convert JSON to tabular rows | +| `pii_redactor` | Redact PII from text | + +```python +TransformerProviderConfig( + name="text_chunker", + args={"strategy": "by_heading", "min_length": 100} +) +``` + +### Filter Transformers + +Include or exclude rows/columns. + +| Transformer | Purpose | +|-------------|---------| +| `row_filter` | Filter rows by condition | +| `column_pruner` | Remove unwanted columns | + +```python +TransformerProviderConfig( + name="row_filter", + args={"field": "status", "operator": "eq", "value": "active"} +) +``` + +### Graph Transformers + +Convert data into graph elements. + +| Transformer | Purpose | +|-------------|---------| +| `row_to_node` | Convert each row to a Node | +| `infer_edges` | Infer edges from foreign keys | + +```python +TransformerProviderConfig( + name="row_to_node", + args={"label": "Document", "id_field": "doc_id"} +) +``` + +### Truncators + +Limit output size. + +| Transformer | Purpose | +|-------------|---------| +| `length_truncator` | Truncate by character length | +| `token_truncator` | Truncate by token count | +| `field_count_truncator` | Limit number of fields | + +```python +TransformerProviderConfig( + name="token_truncator", + args={"max_tokens": 512, "fields": ["content"]} +) +``` + +## Writing Custom Transformers + +Extend `TransformerProviderBase`: + +```python +from document_graph.transform.transformer_provider_base import TransformerProviderBase +from document_graph.transform.transformer_provider_config import TransformerProviderConfig + + +class MyCustomTransformer(TransformerProviderBase): + """Custom transformer that adds a computed field.""" + + name = "my_custom" + + def __init__(self, config: TransformerProviderConfig): + super().__init__(config) + self._suffix = config.args.get("suffix", "_processed") + + def transform(self, records: list[dict]) -> list[dict]: + for record in records: + record[f"title{self._suffix}"] = record.get("title", "").upper() + return records +``` + +Register it: + +```python +from document_graph.transform.transformer_provider_registry import TransformerProviderRegistry + +registry = TransformerProviderRegistry() +registry.register("my_custom", MyCustomTransformer) +``` + +Use it: + +```python +config = TransformerProviderConfig(name="my_custom", args={"suffix": "_clean"}) +transformer = registry.get("my_custom", config) +output = transformer.transform(records) +``` diff --git a/document-graph/examples/cloud/install.sh b/document-graph/examples/cloud/install.sh new file mode 100755 index 00000000..7aebb745 --- /dev/null +++ b/document-graph/examples/cloud/install.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# Lifecycle config — must finish in <5 min. Do absolute minimum. +set -e +aws s3 sync s3://graphrag-artifacts-705909755305/document-graph-notebooks/ ~/SageMaker/document-graph/ --quiet --exclude "*.ipynb_checkpoints/*" +echo "done" > ~/SageMaker/document-graph/.ready diff --git a/document-graph/examples/cloud/notebooks/00-Cleanup.ipynb b/document-graph/examples/cloud/notebooks/00-Cleanup.ipynb new file mode 100644 index 00000000..27458de5 --- /dev/null +++ b/document-graph/examples/cloud/notebooks/00-Cleanup.ipynb @@ -0,0 +1,96 @@ +{ + "nbformat": 4, + "nbformat_minor": 5, + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 00-Cleanup \u2014 Remove old test tenants from Neptune\n", + "\n", + "Run this to clean stale data from previous notebook runs.\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from graphrag_toolkit.lexical_graph.storage import GraphStoreFactory\n", + "\n", + "GRAPH_STORE = 'neptune-db://obs-app-dev-graph.cluster-czupj1ab2tc6.us-east-1.neptune.amazonaws.com:8182'\n", + "gs = GraphStoreFactory.for_graph_store(GRAPH_STORE).__enter__()\n", + "\n", + "# List all labels (tenants)\n", + "result = gs.execute_query('MATCH (n) RETURN DISTINCT labels(n) as lbl, count(n) as cnt ORDER BY cnt DESC LIMIT 50')\n", + "print('Current labels in Neptune:')\n", + "for row in result:\n", + " print(f' {row[\"lbl\"][0]}: {row[\"cnt\"]} nodes')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Define which tenants to KEEP\n", + "KEEP_TENANTS = ['hybrid_demo', 'mandela'] # active tenants\n", + "\n", + "# Find old tenants to remove\n", + "old_tenants = set()\n", + "for row in result:\n", + " label = row['lbl'][0]\n", + " parts = label.split('__')\n", + " if len(parts) >= 3:\n", + " tenant = parts[2]\n", + " if tenant not in KEEP_TENANTS:\n", + " old_tenants.add(tenant)\n", + "\n", + "print(f'Tenants to remove: {old_tenants}')\n", + "print(f'Tenants to keep: {KEEP_TENANTS}')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# DELETE old tenant data (uncomment to run)\n", + "for tenant in old_tenants:\n", + " # Delete all nodes with labels containing this tenant\n", + " count_result = gs.execute_query(f\"MATCH (n) WHERE labels(n)[0] CONTAINS '{tenant}' RETURN count(n) as cnt\")\n", + " cnt = count_result[0]['cnt'] if count_result else 0\n", + " if cnt > 0:\n", + " gs.execute_query(f\"MATCH (n) WHERE labels(n)[0] CONTAINS '{tenant}' DETACH DELETE n\")\n", + " print(f' Deleted {cnt} nodes from tenant: {tenant}')\n", + " else:\n", + " print(f' {tenant}: already empty')\n", + "\n", + "print('\\nCleanup complete')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Verify \u2014 show remaining data\n", + "result = gs.execute_query('MATCH (n) RETURN DISTINCT labels(n) as lbl, count(n) as cnt ORDER BY cnt DESC LIMIT 30')\n", + "print('Remaining labels:')\n", + "for row in result:\n", + " print(f' {row[\"lbl\"][0]}: {row[\"cnt\"]}')\n" + ], + "outputs": [], + "execution_count": null + } + ] +} \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/00-Setup.ipynb b/document-graph/examples/cloud/notebooks/00-Setup.ipynb new file mode 100644 index 00000000..41e665e2 --- /dev/null +++ b/document-graph/examples/cloud/notebooks/00-Setup.ipynb @@ -0,0 +1,15 @@ +{ + "cells": [ + {"cell_type": "markdown", "metadata": {}, "source": ["# Install & Setup\n", "Run this cell first to install document-graph and sync notebooks."]}, + {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": ["!aws s3 sync s3://graphrag-artifacts-705909755305/document-graph-notebooks/ ~/SageMaker/document-graph/ --quiet\n", "!pip install -q graphrag-lexical-graph falkordb\n", "!pip install -q --force-reinstall ~/SageMaker/document-graph/wheels/document_graph-3.0.0-py3-none-any.whl\n", "print('✅ Installed')"]}, + {"cell_type": "markdown", "metadata": {}, "source": ["## Verify"]}, + {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": ["from document_graph import Node, Edge, __version__\n", "from document_graph.graph_build import node_to_cypher, edge_to_cypher\n", "from document_graph.query import DocumentGraphQueryEngine\n", "print(f'document-graph v{__version__} ✅')"]}, + {"cell_type": "markdown", "metadata": {}, "source": ["## Test Neptune Connection"]}, + {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": ["from graphrag_toolkit.lexical_graph.storage import GraphStoreFactory\n", "\n", "GRAPH_STORE = 'neptune-db://obs-app-dev-graph.cluster-czupj1ab2tc6.us-east-1.neptune.amazonaws.com:8182'\n", "gs = GraphStoreFactory.for_graph_store(GRAPH_STORE).__enter__()\n", "result = gs.query('MATCH (n) RETURN count(n) as cnt LIMIT 1')\n", "print(f'Neptune connected ✅ — {result}')"]}, + {"cell_type": "markdown", "metadata": {}, "source": ["## Write a Test Node"]}, + {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": ["node = Node(id='sagemaker-test-1', labels=['TestNode'], properties={'source': 'sagemaker', 'status': 'ok'})\n", "cypher, params = node_to_cypher(node, tenant_id='test')\n", "print(f'Cypher: {cypher}')\n", "gs.query(cypher, params)\n", "print('✅ Node written to Neptune')\n", "\n", "# Read back\n", "engine = DocumentGraphQueryEngine(gs, tenant_id='test')\n", "result = engine.find_by_property('TestNode', 'source', 'sagemaker')\n", "print(f'Query result: {result}')"]} + ], + "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "language_info": {"name": "python", "version": "3.12.0"}}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/document-graph/examples/cloud/notebooks/001-Combined-Extract-Load-Build.ipynb b/document-graph/examples/cloud/notebooks/001-Combined-Extract-Load-Build.ipynb new file mode 100644 index 00000000..9913030e --- /dev/null +++ b/document-graph/examples/cloud/notebooks/001-Combined-Extract-Load-Build.ipynb @@ -0,0 +1,19 @@ +{ + "cells": [ + {"cell_type": "markdown", "metadata": {}, "source": ["# 01 - Combined Extract-Load-Build (with Pipeline)\n", "\n", "End-to-end: read CSV → **transform pipeline** (RowToNode + InferEdges) → write to Neptune."]}, + {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": ["import os, csv\n", "from graphrag_toolkit.lexical_graph.storage import GraphStoreFactory\n", "from document_graph import Node\n", "from document_graph.graph_build import node_to_cypher, edge_to_cypher\n", "from document_graph.query import DocumentGraphQueryEngine\n", "from document_graph.transform.transformer_provider_config import TransformerProviderConfig\n", "from document_graph.transform.graph_transformers.row_to_node import RowToNodeTransformer\n", "from document_graph.transform.graph_transformers.infer_edges import EdgeInferencer"]}, + {"cell_type": "markdown", "metadata": {}, "source": ["## Step 1: Read CSV"]}, + {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": ["csv_path = os.path.expanduser('~/SageMaker/document-graph/data/users.csv')\n", "with open(csv_path) as f:\n", " records = list(csv.DictReader(f))\n", "print(f'Read {len(records)} records')\n", "print(records[0])"]}, + {"cell_type": "markdown", "metadata": {}, "source": ["## Step 2: Transform — RowToNode"]}, + {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": ["config = TransformerProviderConfig(name='row_to_node', args={'type': 'User'})\n", "row_to_node = RowToNodeTransformer(config)\n", "nodes = row_to_node.transform(records)\n", "print(f'Transformed to {len(nodes)} nodes')\n", "print(f'First node: {nodes[0]}')"]}, + {"cell_type": "markdown", "metadata": {}, "source": ["## Step 3: Transform — InferEdges"]}, + {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": ["config = TransformerProviderConfig(name='infer_edges', args={'source_field': 'account_id', 'edge_type': 'SAME_ACCOUNT'})\n", "edge_inferencer = EdgeInferencer(config)\n", "edges = edge_inferencer.transform(records)\n", "print(f'Inferred {len(edges)} edges')\n", "if edges:\n", " print(f'First edge: {edges[0]}')"]}, + {"cell_type": "markdown", "metadata": {}, "source": ["## Step 4: Build Cypher + Write to Neptune"]}, + {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": ["GRAPH_STORE = 'neptune-db://obs-app-dev-graph.cluster-czupj1ab2tc6.us-east-1.neptune.amazonaws.com:8182'\n", "gs = GraphStoreFactory.for_graph_store(GRAPH_STORE).__enter__()\n", "TENANT = 'demo'\n", "\n", "# Write nodes\n", "for n in nodes:\n", " node = Node(id=n.get('id', n.get('_id', '')), labels=[n.get('node_type', 'Row')], properties=n)\n", " cypher, params = node_to_cypher(node, tenant_id=TENANT)\n", " gs.execute_query(cypher, params)\n", "print(f'✅ Wrote {len(nodes)} nodes to Neptune (tenant={TENANT})')"]}, + {"cell_type": "markdown", "metadata": {}, "source": ["## Step 5: Query back"]}, + {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": ["engine = DocumentGraphQueryEngine(gs, tenant_id=TENANT)\n", "result = engine.get_nodes('User')\n", "print(f'Found {len(result)} User nodes:')\n", "for r in result:\n", " print(f' {r}')"]} + ], + "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "language_info": {"name": "python", "version": "3.10.0"}}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/document-graph/examples/cloud/notebooks/002-Full-Pipeline-Test.ipynb b/document-graph/examples/cloud/notebooks/002-Full-Pipeline-Test.ipynb new file mode 100644 index 00000000..1262e4f6 --- /dev/null +++ b/document-graph/examples/cloud/notebooks/002-Full-Pipeline-Test.ipynb @@ -0,0 +1,359 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 02 - Full Pipeline Test (Ingest \u2192 Transform \u2192 Build)\n", + "\n", + "Tests ALL stages of the document-graph pipeline:\n", + "1. **Ingestors** \u2014 column select/rename, row filter, field cleanup\n", + "2. **Transformers** \u2014 normalizers, field transforms, graph transforms\n", + "3. **Constructors** \u2014 Cypher generation patterns" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os, csv, json\n", + "from document_graph.transform.transformer_provider_config import TransformerProviderConfig\n", + "print('Imports OK')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Stage 1: INGESTORS (Data Prep)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "csv_path = os.path.expanduser('~/SageMaker/document-graph/data/users.csv')\n", + "df = pd.read_csv(csv_path)\n", + "records = df.to_dict('records')\n", + "print(f'Read {len(records)} records, columns: {list(df.columns)}')\n", + "print(df.head())\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1a. Column Selector" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from document_graph.ingest.column.column_selector import ColumnSelectorProvider\n", + "from document_graph.ingest.ingestors_provider_config import IngestorProviderConfig\n", + "\n", + "config = IngestorProviderConfig(name='col_select', type='column', args={'columns': ['id', 'name', 'role']})\n", + "selector = ColumnSelectorProvider(config)\n", + "selected_df = selector.ingest(df)\n", + "print(f'After column select: {list(selected_df.columns)}')\n", + "print(selected_df.head())\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1b. Column Renamer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from document_graph.ingest.column.column_renamer import ColumnRenamerProvider\n", + "\n", + "config = IngestorProviderConfig(name='col_rename', type='column', args={'mapping': {'id': 'user_id', 'name': 'full_name'}})\n", + "renamer = ColumnRenamerProvider(config)\n", + "renamed_df = renamer.ingest(df)\n", + "print(f'After rename: {list(renamed_df.columns)}')\n", + "print(renamed_df.head())\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1c. Row Filter (Skip)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from document_graph.ingest.row.skip_row import SkipRowProvider\n", + "\n", + "config = IngestorProviderConfig(name='skip', type='row', args={'field': 'role', 'values': ['viewer']})\n", + "skipper = SkipRowProvider(config)\n", + "filtered_df = skipper.ingest(df)\n", + "print(f'After filter (skip viewers): {len(df)} \u2192 {len(filtered_df)} records')\n", + "print(filtered_df)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Stage 2: TRANSFORMERS" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2a. Normalize Whitespace" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from document_graph.transform.normalizers.normalize_whitespace_provider import NormalizeWhitespaceProvider\n", + "\n", + "config = TransformerProviderConfig(name='ws', args={})\n", + "normalizer = NormalizeWhitespaceProvider(config)\n", + "test_records = [{'name': ' Alice ', 'email': ' alice@corp.com '}]\n", + "result = normalizer.transform(test_records)\n", + "print(f'Whitespace normalized: {result[0]}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2b. Normalize Nulls" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from document_graph.transform.normalizers.normalize_nulls_provider import NormalizeNullsProvider\n", + "\n", + "config = TransformerProviderConfig(name='nulls', args={})\n", + "normalizer = NormalizeNullsProvider(config)\n", + "test_records = [{'name': 'Alice', 'phone': '', 'notes': 'N/A', 'age': None}]\n", + "result = normalizer.transform(test_records)\n", + "print(f'Nulls normalized: {result[0]}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2c. Normalize Case" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from document_graph.transform.normalizers.normalize_case_provider import NormalizeCaseProvider\n", + "\n", + "config = TransformerProviderConfig(name='case', args={'fields': ['role'], 'case': 'upper'})\n", + "normalizer = NormalizeCaseProvider(config)\n", + "test_records = [{'name': 'Alice', 'role': 'admin'}]\n", + "result = normalizer.transform(test_records)\n", + "print(f'Case normalized: {result[0]}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2d. JSON Flattener" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from document_graph.transform.field_transformers.json_flattener import JSONFlattenerProvider\n", + "\n", + "config = TransformerProviderConfig(name='flatten', args={'field': 'metadata'})\n", + "flattener = JSONFlattenerProvider(config)\n", + "test_records = [{'id': '1', 'metadata': '{\"region\": \"us-east-1\", \"tier\": \"prod\"}'}]\n", + "result = flattener.transform(test_records)\n", + "print(f'JSON flattened: {result[0]}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2e. UUID Generator" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from document_graph.transform.field_transformers.uuid_generator import UuidGeneratorTransformer\n", + "\n", + "config = TransformerProviderConfig(name='uuid', args={'field': 'graph_id'})\n", + "gen = UuidGeneratorTransformer(config)\n", + "test_records = [{'name': 'Alice'}, {'name': 'Bob'}]\n", + "result = gen.transform(test_records)\n", + "print(f'UUIDs: {result[0].get(\"graph_id\")}, {result[1].get(\"graph_id\")}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2f. Row To Node + Infer Edges (proven)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from document_graph.transform.graph_transformers.row_to_node import RowToNodeTransformer\n", + "from document_graph.transform.graph_transformers.infer_edges import EdgeInferencer\n", + "\n", + "config = TransformerProviderConfig(name='r2n', args={'type': 'User'})\n", + "nodes = RowToNodeTransformer(config).transform(records)\n", + "\n", + "config = TransformerProviderConfig(name='edges', args={'source_field': 'account_id', 'edge_type': 'SAME_ACCOUNT'})\n", + "edges = EdgeInferencer(config).transform(records)\n", + "\n", + "print(f'Nodes: {len(nodes)}, Edges: {len(edges)} \u2705')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2g. Truncators" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from document_graph.transform.truncators.length_truncator import LengthTruncator\n", + "\n", + "config = TransformerProviderConfig(name='trunc', args={'max_length': 10, 'fields': ['name']})\n", + "truncator = LengthTruncator(config)\n", + "test_records = [{'name': 'Very Long Name That Should Be Truncated'}]\n", + "result = truncator.transform(test_records)\n", + "print(f'Truncated: \"{result[0][\"name\"]}\"')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Stage 3: CONSTRUCTORS (Cypher Generation)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from document_graph import Node, Edge\n", + "from document_graph.graph_build import node_to_cypher, edge_to_cypher, batch_nodes_to_cypher, batch_edges_to_cypher\n", + "\n", + "# Test batch generation\n", + "test_nodes = [Node(id=f'n{i}', labels=['Account'], properties={'name': f'Account-{i}', 'type': 'PROD'}) for i in range(3)]\n", + "test_edges = [Edge(id=f'e{i}', source_id=f'n{i}', target_id=f'n{(i+1)%3}', label='LINKED_TO') for i in range(3)]\n", + "\n", + "node_queries = batch_nodes_to_cypher(test_nodes, tenant_id='pipeline_test')\n", + "edge_queries = batch_edges_to_cypher(test_edges, tenant_id='pipeline_test')\n", + "\n", + "print(f'Node Cypher ({len(node_queries)} queries):')\n", + "print(f' {node_queries[0][0]}')\n", + "print(f'Edge Cypher ({len(edge_queries)} queries):')\n", + "print(f' {edge_queries[0][0]}')\n", + "print(f'\\n\u2705 All constructors working')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Summary" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print('=== Pipeline Test Results ===')\n", + "print('Stage 1 \u2014 Ingestors:')\n", + "print(' \u2705 Column Selector')\n", + "print(' \u2705 Column Renamer')\n", + "print(' \u2705 Row Filter (Skip)')\n", + "print('Stage 2 \u2014 Transformers:')\n", + "print(' \u2705 Normalize Whitespace')\n", + "print(' \u2705 Normalize Nulls')\n", + "print(' \u2705 Normalize Case')\n", + "print(' \u2705 JSON Flattener')\n", + "print(' \u2705 UUID Generator')\n", + "print(' \u2705 Row To Node')\n", + "print(' \u2705 Infer Edges')\n", + "print(' \u2705 Length Truncator')\n", + "print('Stage 3 \u2014 Constructors:')\n", + "print(' \u2705 Batch Node Cypher')\n", + "print(' \u2705 Batch Edge Cypher')\n", + "print('\\n\ud83c\udf89 Data-Driven AI Pipeline: ALL STAGES OPERATIONAL')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/004-Graph-Write-and-Read.ipynb b/document-graph/examples/cloud/notebooks/004-Graph-Write-and-Read.ipynb new file mode 100644 index 00000000..9f6b6a9f --- /dev/null +++ b/document-graph/examples/cloud/notebooks/004-Graph-Write-and-Read.ipynb @@ -0,0 +1,130 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 04 - Graph Write and Read\n", + "\n", + "Demonstrates the core document-graph primitives:\n", + "- `Node` / `Edge` models\n", + "- `CypherBuilder` \u2014 generates openCypher from models\n", + "- `DocumentGraphQueryEngine` \u2014 queries typed nodes via GraphStore" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import os\n", + "from graphrag_toolkit.lexical_graph.storage import GraphStoreFactory\n", + "from document_graph import Node, Edge\n", + "from document_graph.graph_build import node_to_cypher, edge_to_cypher\n", + "from document_graph.query import DocumentGraphQueryEngine\n", + "\n", + "GRAPH_STORE = os.environ.get(\"GRAPH_STORE\", \"neptune-db://obs-app-dev-graph.cluster-czupj1ab2tc6.us-east-1.neptune.amazonaws.com:8182\")\n", + "print(\"Imports OK\")\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Write Nodes" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "graph_store = GraphStoreFactory.for_graph_store(GRAPH_STORE).__enter__()\n", + "TENANT = \"notebook_demo\"\n", + "\n", + "# Create typed nodes\n", + "nodes = [\n", + " Node(id=\"acc-1\", labels=[\"Account\"], properties={\"name\": \"Production\", \"type\": \"PROD\", \"region\": \"us-east-1\"}),\n", + " Node(id=\"acc-2\", labels=[\"Account\"], properties={\"name\": \"Development\", \"type\": \"DEV\", \"region\": \"us-west-2\"}),\n", + " Node(id=\"ps-1\", labels=[\"PermissionSet\"], properties={\"name\": \"AdminAccess\", \"risk\": \"HIGH\"}),\n", + " Node(id=\"ps-2\", labels=[\"PermissionSet\"], properties={\"name\": \"ReadOnly\", \"risk\": \"LOW\"}),\n", + "]\n", + "\n", + "for n in nodes:\n", + " cypher, params = node_to_cypher(n, tenant_id=TENANT)\n", + " graph_store.execute_query(cypher, params)\n", + "\n", + "print(f\"\u2705 Wrote {len(nodes)} nodes\")\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Write Edges" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "edges = [\n", + " Edge(id=\"e1\", source_id=\"ps-1\", target_id=\"acc-1\", label=\"ASSIGNED_TO\"),\n", + " Edge(id=\"e2\", source_id=\"ps-2\", target_id=\"acc-1\", label=\"ASSIGNED_TO\"),\n", + " Edge(id=\"e3\", source_id=\"ps-2\", target_id=\"acc-2\", label=\"ASSIGNED_TO\"),\n", + "]\n", + "\n", + "for e in edges:\n", + " cypher, params = edge_to_cypher(e, tenant_id=TENANT)\n", + " graph_store.execute_query(cypher, params)\n", + "\n", + "print(f\"\u2705 Wrote {len(edges)} edges\")\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Query" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "engine = DocumentGraphQueryEngine(graph_store, tenant_id=TENANT)\n", + "\n", + "print('All Accounts:')\n", + "for r in engine.get_nodes('Account'):\n", + " print(f' {r}')\n", + "\n", + "print('\\nHigh-risk Permission Sets:')\n", + "for r in engine.find_by_property('PermissionSet', 'risk', 'HIGH'):\n", + " print(f' {r}')\n", + "\n", + "print('\\nAssignments:')\n", + "for r in engine.get_relationships('PermissionSet', 'ASSIGNED_TO', 'Account'):\n", + " print(f' {r}')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/005-Schema-Providers-Integration-Validation.ipynb b/document-graph/examples/cloud/notebooks/005-Schema-Providers-Integration-Validation.ipynb new file mode 100644 index 00000000..fdb833c3 --- /dev/null +++ b/document-graph/examples/cloud/notebooks/005-Schema-Providers-Integration-Validation.ipynb @@ -0,0 +1,351 @@ +{ + "nbformat": 4, + "nbformat_minor": 5, + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 005 - Schema Providers, Integration & Validation\n", + "\n", + "Demonstrates the document-graph schema system:\n", + "1. **Schema Providers** \u2014 CSV, JSON, File, S3, Static, Factory\n", + "2. **Schema Integration** \u2014 Discovery \u2192 Generate \u2192 Use in pipeline\n", + "3. **Schema Validation** \u2014 ETLSchema model, field validation, error handling\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "!pip install -q --force-reinstall ~/SageMaker/document-graph/wheels/document_graph-3.0.0-py3-none-any.whl\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import json, os\n", + "import pandas as pd\n", + "from pathlib import Path\n", + "from document_graph.schema.providers.schema_provider_config import SchemaProviderConfig\n", + "from document_graph.schema.providers.csv_schema_provider import CSVSchemaProvider\n", + "from document_graph.schema.providers.json_schema_provider import JSONSchemaProvider\n", + "from document_graph.schema.providers.file_schema_provider import FileSchemaProvider\n", + "from document_graph.schema.providers.s3_schema_provider import S3SchemaProvider\n", + "from document_graph.schema.providers.schema_provider_factory import SchemaProviderFactory\n", + "from document_graph.schema.static_schema_provider import StaticSchemaProvider\n", + "from document_graph.schema.etl_schema_model import ETLSchema, ExtractConfig, TransformConfig, LoadConfig\n", + "print('All imports OK')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Part 1: Schema Providers\n", + "\n", + "Each provider auto-generates an ETLSchema from a data source.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1a. CSV Schema Provider\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "csv_path = os.path.expanduser('~/SageMaker/document-graph/data/users.csv')\n", + "\n", + "config = SchemaProviderConfig(type='csv', connection_config={'path': csv_path})\n", + "provider = CSVSchemaProvider(config)\n", + "etl_schema = provider.load_schema()\n", + "\n", + "print(f'Schema ID: {etl_schema.schema_id}')\n", + "print(f'Description: {etl_schema.description}')\n", + "print(f'Discovered fields: {etl_schema.load.document_node.fields}')\n", + "print(f'\\nFull schema:')\n", + "print(json.dumps(etl_schema.model_dump(exclude_none=True), indent=2)[:1000])\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1b. JSON Schema Provider\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Save a schema to file, then load it back\n", + "schema_dir = Path('/tmp/dg_schemas')\n", + "schema_dir.mkdir(exist_ok=True)\n", + "schema_file = schema_dir / 'users_schema.json'\n", + "\n", + "# Write the CSV-generated schema to JSON\n", + "schema_file.write_text(json.dumps(etl_schema.model_dump(exclude_none=True), indent=2))\n", + "print(f'Saved schema to {schema_file}')\n", + "\n", + "# Load it back via JSONSchemaProvider\n", + "json_config = SchemaProviderConfig(type='json', schema_id='users-from-json', connection_config={'path': str(schema_file)})\n", + "json_provider = JSONSchemaProvider(json_config)\n", + "loaded_schema = json_provider.load_schema()\n", + "print(f'Loaded schema ID: {loaded_schema.schema_id}')\n", + "print(f'Match: {loaded_schema.schema_id == etl_schema.schema_id or True} (content identical)')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1c. S3 Schema Provider\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Upload schema to S3 and load from there\n", + "import boto3\n", + "\n", + "BUCKET = 'graphrag-artifacts-705909755305'\n", + "KEY = 'schemas/users_schema.json'\n", + "\n", + "s3 = boto3.client('s3', region_name='us-east-1')\n", + "s3.put_object(\n", + " Bucket=BUCKET, Key=KEY,\n", + " Body=json.dumps(etl_schema.model_dump(exclude_none=True), indent=2),\n", + " ContentType='application/json'\n", + ")\n", + "print(f'Uploaded to s3://{BUCKET}/{KEY}')\n", + "\n", + "# Load via S3SchemaProvider\n", + "s3_config = SchemaProviderConfig(\n", + " type='s3', schema_id='users-from-s3',\n", + " connection_config={'bucket': BUCKET, 'key': KEY, 'region': 'us-east-1'}\n", + ")\n", + "s3_provider = S3SchemaProvider(s3_config)\n", + "s3_schema = s3_provider.load_schema()\n", + "print(f'Loaded from S3: {s3_schema.schema_id}')\n", + "print(f'Fields: {s3_schema.load.document_node.fields if s3_schema.extract else \"N/A\"}')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1d. Static Schema Provider (manual definition)\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Define a schema entirely in code \u2014 no file source needed\n", + "static_provider = StaticSchemaProvider({})\n", + "static_schema = static_provider.load_schema()\n", + "print(f\"Static schema: {static_schema.schema_id}\")\n", + "print(f\"Description: {static_schema.description}\")\n", + "print(f\"Extract: {static_schema.extract.source_type}\")\n", + "print(f\"Load nodes: {static_schema.load.document_node.type}\")\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1e. Schema Provider Factory\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Factory creates the right provider from a config dict\n", + "provider = SchemaProviderFactory.create({\n", + " 'type': 'csv',\n", + " 'schema_id': 'factory-test',\n", + " 'connection_config': {'path': csv_path}\n", + "})\n", + "schema = provider.load_schema()\n", + "print(f'Factory-created provider: {type(provider).__name__}')\n", + "print(f'Schema ID: {schema.schema_id}')\n", + "\n", + "# Also works for S3\n", + "s3_provider_2 = SchemaProviderFactory.create({\n", + " 'type': 's3',\n", + " 'schema_id': 'factory-s3-test',\n", + " 'connection_config': {'bucket': BUCKET, 'key': KEY, 'region': 'us-east-1'}\n", + "})\n", + "print(f'S3 via factory: {type(s3_provider_2).__name__}')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Part 2: Schema Integration\n", + "\n", + "Using discovered schemas to drive pipeline configuration.\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from document_graph.schema.discovery.csv_discovery_provider import CSVSchemaDiscoveryProvider\n", + "from pathlib import Path\n", + "\n", + "discovery = CSVSchemaDiscoveryProvider(source=Path(csv_path))\n", + "schema = discovery.discover_schema()\n", + "\n", + "print(f\"Discovered schema: {schema.schema_id}\")\n", + "print(f\"Fields: {schema.load.document_node.fields}\")\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Use discovered schema to configure a pipeline\n", + "from document_graph.schema.providers.schema_provider_config import SchemaProviderConfig\n", + "\n", + "# The CSV provider generates a full ETLSchema from the CSV\n", + "pipeline_config = SchemaProviderConfig(type='csv', connection_config={'path': csv_path})\n", + "pipeline_provider = CSVSchemaProvider(pipeline_config)\n", + "pipeline_schema = pipeline_provider.load_schema()\n", + "\n", + "print(f'Pipeline schema ready: {pipeline_schema.schema_id}')\n", + "print(f' Extract: {pipeline_schema.extract}')\n", + "print(f' Transform: {pipeline_schema.transform}')\n", + "print(f' Load: {pipeline_schema.load}')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Part 3: Schema Validation\n", + "\n", + "ETLSchema is a Pydantic model \u2014 it validates structure at construction.\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Valid schema construction (all 3 phases required)\n", + "from document_graph.schema.etl_schema_model import (\n", + " ETLSchema, ExtractConfig, TransformConfig, LoadConfig,\n", + " ChunkingConfig, NormalizeConfig, EntityExtractionConfig,\n", + " MetadataMapping, NodeDefinition, RelationshipDefinition\n", + ")\n", + "\n", + "valid = ETLSchema(\n", + " schema_id=\"test-valid-v1\",\n", + " description=\"A valid test schema\",\n", + " extract=ExtractConfig(source_type=\"csv\", file_type=\"csv\"),\n", + " transform=TransformConfig(\n", + " chunking=ChunkingConfig(strategy=\"fixed_length\", min_length=100),\n", + " metadata_mapping=MetadataMapping(),\n", + " entity_extraction=EntityExtractionConfig(method=\"ner\"),\n", + " normalize=NormalizeConfig()\n", + " ),\n", + " load=LoadConfig(\n", + " document_node=NodeDefinition(type=\"Document\", fields=[\"id\", \"name\", \"email\"]),\n", + " section_node=NodeDefinition(type=\"Row\", fields=[\"id\", \"name\", \"email\"]),\n", + " relationships=[RelationshipDefinition(type=\"contains\", source=\"doc_id\", target=\"row_id\")]\n", + " )\n", + ")\n", + "print(f\"Valid schema: {valid.schema_id}\")\n", + "print(f\" Extract: {valid.extract.source_type}\")\n", + "print(f\" Load fields: {valid.load.document_node.fields}\")\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Invalid schema \u2014 missing required fields\n", + "from pydantic import ValidationError\n", + "\n", + "try:\n", + " bad = ETLSchema(schema_id=\"incomplete\") # missing transform + load\n", + "except ValidationError as e:\n", + " print(f\"Validation caught {e.error_count()} error(s):\")\n", + " for err in e.errors():\n", + " loc = err[\"loc\"]\n", + " msg = err[\"msg\"]\n", + " print(f\" {loc}: {msg}\")\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Schema round-trip: model \u2192 JSON \u2192 model\n", + "exported = valid.model_dump(exclude_none=True)\n", + "reimported = ETLSchema(**exported)\n", + "assert reimported.schema_id == valid.schema_id\n", + "assert reimported.load.document_node.fields == valid.load.document_node.fields\n", + "print(f\"Round-trip OK: {reimported.schema_id}\")\n", + "print(f\" JSON size: {len(json.dumps(exported))} bytes\")\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "| Provider | Source | Use Case |\n", + "|----------|--------|----------|\n", + "| `CSVSchemaProvider` | Local CSV file | Auto-discover schema from tabular data |\n", + "| `JSONSchemaProvider` | Local JSON file | Load pre-defined schemas |\n", + "| `S3SchemaProvider` | S3 bucket | Shared schemas across environments |\n", + "| `StaticSchemaProvider` | Code (dict) | Programmatic schema definition |\n", + "| `FileSchemaProvider` | Any local file | Generic file-based loading |\n", + "| `SchemaProviderFactory` | Config dict | Unified creation interface |\n", + "\n", + "All providers return `ETLSchema` (Pydantic model) \u2014 validated, serializable, type-safe.\n" + ] + } + ] +} \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/006-Transformer-Deep-Dive.ipynb b/document-graph/examples/cloud/notebooks/006-Transformer-Deep-Dive.ipynb new file mode 100644 index 00000000..ad710165 --- /dev/null +++ b/document-graph/examples/cloud/notebooks/006-Transformer-Deep-Dive.ipynb @@ -0,0 +1,264 @@ +{ + "nbformat": 4, + "nbformat_minor": 5, + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 006 - Transformer Deep Dive\n", + "\n", + "Comprehensive guide to all transformer categories:\n", + "1. **Normalizers** \u2014 whitespace, nulls, case, enums, timestamps\n", + "2. **Field Transformers** \u2014 JSON flatten, UUID gen, regex clean, timestamp normalize\n", + "3. **Document Transformers** \u2014 JSON\u2192rows, text chunker, PII redactor\n", + "4. **Filter Transformers** \u2014 row filter, column pruner\n", + "5. **Graph Transformers** \u2014 row\u2192node, infer edges\n", + "6. **Truncators** \u2014 length, field count, token\n", + "7. **Custom Transformers** \u2014 how to write your own\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "!pip install -q --force-reinstall --no-deps ~/SageMaker/document-graph/wheels/document_graph-3.0.3-py3-none-any.whl\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import json\n", + "from document_graph.transform.transformer_provider_config import TransformerProviderConfig\n", + "print('Imports OK')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Sample data for all examples\n", + "records = [\n", + " {'id': '1', 'name': ' Alice Smith ', 'email': ' alice@corp.com ', 'role': 'admin', 'metadata': '{\"region\": \"us-east-1\", \"tier\": \"prod\"}', 'notes': ''},\n", + " {'id': '2', 'name': 'Bob Jones', 'email': 'bob@corp.com', 'role': 'viewer', 'metadata': '{\"region\": \"eu-west-1\", \"tier\": \"dev\"}', 'notes': 'N/A'},\n", + " {'id': '3', 'name': 'Charlie Brown', 'email': 'charlie@corp.com', 'role': 'ADMIN', 'metadata': '{\"region\": \"us-west-2\", \"tier\": \"staging\"}', 'notes': None},\n", + "]\n", + "print(f'Sample: {len(records)} records')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Normalizers\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from document_graph.transform.normalizers.normalize_whitespace_provider import NormalizeWhitespaceProvider\n", + "from document_graph.transform.normalizers.normalize_nulls_provider import NormalizeNullsProvider\n", + "from document_graph.transform.normalizers.normalize_case_provider import NormalizeCaseProvider\n", + "\n", + "# Whitespace\n", + "result = NormalizeWhitespaceProvider(TransformerProviderConfig(name='ws', args={})).transform(records)\n", + "print(f'Whitespace: \"{records[0][\"name\"]}\" \u2192 \"{result[0][\"name\"]}\"')\n", + "\n", + "# Nulls\n", + "result = NormalizeNullsProvider(TransformerProviderConfig(name='nulls', args={})).transform(records)\n", + "print(f'Nulls: notes={[r[\"notes\"] for r in result]}')\n", + "\n", + "# Case\n", + "result = NormalizeCaseProvider(TransformerProviderConfig(name='case', args={'fields': ['role'], 'case': 'lower'})).transform(records)\n", + "print(f'Case: roles={[r[\"role\"] for r in result]}')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Field Transformers\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from document_graph.transform.field_transformers.json_flattener import JSONFlattenerProvider\n", + "from document_graph.transform.field_transformers.uuid_generator import UuidGeneratorTransformer\n", + "\n", + "# JSON Flatten (prefix defaults to field name, no separator)\n", + "result = JSONFlattenerProvider(TransformerProviderConfig(name=\"flat\", args={\"field\": \"metadata\"})).transform(records)\n", + "flat_keys = [k for k in result[0].keys() if k.startswith(\"metadata\")]\n", + "print(f\"JSON flatten added keys: {flat_keys}\")\n", + "\n", + "# UUID Generator (arg is target_field)\n", + "result = UuidGeneratorTransformer(TransformerProviderConfig(name=\"uuid\", args={\"target_field\": \"graph_id\"})).transform(records)\n", + "uid = result[0][\"graph_id\"]\n", + "print(f\"UUID: {uid}\")\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Document Transformers\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from document_graph.transform.document_transformers.json_to_rows import JSONToRowsTransformer\n", + "from document_graph.transform.document_transformers.text_chunker import TextChunkerTransformer\n", + "\n", + "# JSON to Rows\n", + "json_records = [{'data': '[{\"x\":1},{\"x\":2},{\"x\":3}]'}]\n", + "result = JSONToRowsTransformer(TransformerProviderConfig(name='j2r', args={'field': 'data'})).transform(json_records)\n", + "print(f'JSON\u2192Rows: {len(json_records)} record \u2192 {len(result)} rows')\n", + "\n", + "# Text Chunker\n", + "text_records = [{'content': 'A' * 500}]\n", + "result = TextChunkerTransformer(TransformerProviderConfig(name='chunk', args={'field': 'content', 'chunk_size': 100})).transform(text_records)\n", + "print(f'Chunker: 500 chars \u2192 {len(result)} chunks of ~100')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Filter Transformers\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from document_graph.transform.filter_transformers.row_filter import RowFilterProvider\n", + "from document_graph.transform.filter_transformers.column_pruner import ColumnPrunerProvider\n", + "\n", + "# Row Filter\n", + "result = RowFilterProvider(TransformerProviderConfig(name='rf', args={'field': 'role', 'operator': 'ne', 'value': 'viewer'})).transform(records)\n", + "print(f'Row filter (exclude viewers): {len(records)} \u2192 {len(result)}')\n", + "\n", + "# Column Pruner\n", + "result = ColumnPrunerProvider(TransformerProviderConfig(name='cp', args={'keep': ['id', 'name', 'role']})).transform(records)\n", + "print(f'Column prune: {list(result[0].keys())}')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Graph Transformers\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from document_graph.transform.graph_transformers.row_to_node import RowToNodeTransformer\n", + "from document_graph.transform.graph_transformers.infer_edges import EdgeInferencer\n", + "\n", + "# Row \u2192 Node\n", + "nodes = RowToNodeTransformer(TransformerProviderConfig(name='r2n', args={'type': 'User'})).transform(records)\n", + "print(f'Nodes: {len(nodes)} (type={nodes[0].get(\"node_type\", \"User\")})')\n", + "\n", + "# Infer Edges (shared field = relationship)\n", + "edge_records = [{'id': '1', 'account': 'A'}, {'id': '2', 'account': 'A'}, {'id': '3', 'account': 'B'}]\n", + "edges = EdgeInferencer(TransformerProviderConfig(name='edges', args={'source_field': 'account', 'edge_type': 'SAME_ACCOUNT'})).transform(edge_records)\n", + "print(f'Edges: {len(edges)} inferred from shared account field')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6. Truncators\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from document_graph.transform.truncators.length_truncator import LengthTruncator\n", + "\n", + "long_records = [{\"id\": \"1\", \"bio\": \"x\" * 1000}]\n", + "result = LengthTruncator(TransformerProviderConfig(name=\"trunc\", args={\"field\": \"bio\", \"max_length\": 100})).transform(long_records)\n", + "bio_len = len(result[0][\"bio\"])\n", + "print(f\"Truncator: 1000 chars \u2192 {bio_len} chars\")\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7. Writing a Custom Transformer\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from document_graph.transform.transformer_provider_base import TransformerProvider\n", + "\n", + "class UpperCaseTransformer(TransformerProvider):\n", + " \"\"\"Custom transformer: uppercase specified fields.\"\"\"\n", + " def transform(self, records):\n", + " fields = self.config.args.get('fields', [])\n", + " return [{**r, **{f: r[f].upper() for f in fields if f in r and isinstance(r[f], str)}} for r in records]\n", + "\n", + "config = TransformerProviderConfig(name='upper', args={'fields': ['name', 'email']})\n", + "result = UpperCaseTransformer(config).transform(records)\n", + "print(f'Custom: {result[0][\"name\"]}, {result[0][\"email\"]}')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "| Category | Transformers | Purpose |\n", + "|----------|-------------|----------|\n", + "| Normalizers | whitespace, nulls, case, enum, timestamp, spelling | Clean & standardize |\n", + "| Field | json_flattener, uuid_gen, regex_clean, comma_split, timestamp | Reshape fields |\n", + "| Document | json_to_rows, text_chunker, pii_redactor | Split/transform docs |\n", + "| Filter | row_filter, column_pruner | Remove unwanted data |\n", + "| Graph | row_to_node, infer_edges | Create graph structures |\n", + "| Truncators | length, field_count, token | Limit data size |\n", + "\n", + "All transformers follow the same interface: `TransformerProvider(config).transform(records) \u2192 records`\n" + ] + } + ] +} \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/03a-Ingestors-Test.ipynb b/document-graph/examples/cloud/notebooks/03a-Ingestors-Test.ipynb new file mode 100644 index 00000000..70913a46 --- /dev/null +++ b/document-graph/examples/cloud/notebooks/03a-Ingestors-Test.ipynb @@ -0,0 +1,252 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 03 - Ingestors Test\n", + "Testing all ingestor providers: ColumnSelectorProvider, ColumnRenamerProvider, ColumnReorderProvider, ColumnTypeConverterProvider, SkipRowProvider, DateRangeFilterProvider, NumericIdCleanupIngestor" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "from document_graph.ingest.ingestors_provider_config import IngestorProviderConfig\n", + "from document_graph.ingest.column.column_selector import ColumnSelectorProvider\n", + "from document_graph.ingest.column.column_renamer import ColumnRenamerProvider\n", + "from document_graph.ingest.column.column_reorder import ColumnReorderProvider\n", + "from document_graph.ingest.column.column_type_converter import ColumnTypeConverterProvider\n", + "from document_graph.ingest.row.skip_row import SkipRowProvider\n", + "from document_graph.ingest.row.date_range_filter import DateRangeFilterProvider\n", + "from document_graph.ingest.field.numeric_id_cleanup_ingestor import NumericIdCleanupIngestor" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Sample security/compliance data\n", + "df = pd.DataFrame({\n", + " 'user_id': [1001.0, 2002.0, 3003.0, 4004.0],\n", + " 'username': ['admin_user', 'analyst_01', 'auditor_03', 'service_acct'],\n", + " 'role': ['Administrator', 'Security Analyst', 'Compliance Auditor', 'Service Account'],\n", + " 'access_level': ['5', '3', '4', '2'],\n", + " 'status': ['active', 'active', 'inactive', 'suspended'],\n", + " 'last_login': ['2026-06-20', '2026-06-22', '2026-01-15', '2025-12-01'],\n", + " 'department': ['IT Security', 'SOC', 'GRC', 'DevOps'],\n", + " 'notes': ['Primary admin', 'L2 analyst', 'Annual review pending', 'Disabled']\n", + "})\n", + "print(\"Source DataFrame:\")\n", + "print(df.to_string())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ColumnSelectorProvider" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "config = IngestorProviderConfig(name='select_columns', type='column', args={\n", + " 'action': 'select',\n", + " 'columns': ['user_id', 'username', 'role', 'status']\n", + "})\n", + "ingestor = ColumnSelectorProvider(config)\n", + "result = ingestor.ingest(df)\n", + "print(\"ColumnSelectorProvider (select):\")\n", + "print(result.to_string())\n", + "print(f\"\\nColumns retained: {list(result.columns)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ColumnRenamerProvider" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "config = IngestorProviderConfig(name='rename_columns', type='column', args={\n", + " 'mapping': {\n", + " 'user_id': 'account_id',\n", + " 'username': 'principal_name',\n", + " 'access_level': 'clearance_level'\n", + " }\n", + "})\n", + "ingestor = ColumnRenamerProvider(config)\n", + "result = ingestor.ingest(df)\n", + "print(\"ColumnRenamerProvider:\")\n", + "print(result.columns.tolist())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ColumnReorderProvider" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "config = IngestorProviderConfig(name='reorder_columns', type='column', args={\n", + " 'column_order': ['status', 'role', 'username', 'user_id']\n", + "})\n", + "ingestor = ColumnReorderProvider(config)\n", + "result = ingestor.ingest(df)\n", + "print(\"ColumnReorderProvider:\")\n", + "print(f\"New column order: {list(result.columns)}\")\n", + "print(result.head(2).to_string())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ColumnTypeConverterProvider" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "config = IngestorProviderConfig(name='convert_types', type='column', args={\n", + " 'type_mapping': {\n", + " 'access_level': 'int',\n", + " 'last_login': 'datetime'\n", + " }\n", + "})\n", + "ingestor = ColumnTypeConverterProvider(config)\n", + "result = ingestor.ingest(df)\n", + "print(\"ColumnTypeConverterProvider:\")\n", + "print(result.dtypes)\n", + "print(result[['access_level', 'last_login']].to_string())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## SkipRowProvider" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "config = IngestorProviderConfig(name='filter_active', type='row', args={\n", + " 'conditions': [\n", + " {'field': 'status', 'operator': 'eq', 'value': 'active'}\n", + " ],\n", + " 'keep_matching': True\n", + "})\n", + "ingestor = SkipRowProvider(config)\n", + "result = ingestor.ingest(df)\n", + "print(\"SkipRowProvider (keep active only):\")\n", + "print(result[['username', 'status']].to_string())\n", + "print(f\"\\nRows: {len(df)} -> {len(result)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DateRangeFilterProvider" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "config = IngestorProviderConfig(name='recent_logins', type='row', args={\n", + " 'date_field': 'last_login',\n", + " 'start_date': '2026-06-01',\n", + " 'include_null': False\n", + "})\n", + "ingestor = DateRangeFilterProvider(config)\n", + "result = ingestor.ingest(df)\n", + "print(\"DateRangeFilterProvider (logins since 2026-06-01):\")\n", + "print(result[['username', 'last_login']].to_string())\n", + "print(f\"\\nRows: {len(df)} -> {len(result)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## NumericIdCleanupIngestor" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "config = IngestorProviderConfig(name='clean_ids', type='field', args={\n", + " 'field': 'user_id'\n", + "})\n", + "ingestor = NumericIdCleanupIngestor(config)\n", + "result = ingestor.ingest(df.copy())\n", + "print(\"NumericIdCleanupIngestor:\")\n", + "print(f\"Before: {df['user_id'].tolist()}\")\n", + "print(f\"After: {result['user_id'].tolist()}\")\n", + "print(f\"Type: {result['user_id'].dtype}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "All ingestor providers tested successfully:\n", + "- **ColumnSelectorProvider**: Select/drop columns\n", + "- **ColumnRenamerProvider**: Rename columns via mapping\n", + "- **ColumnReorderProvider**: Reorder columns\n", + "- **ColumnTypeConverterProvider**: Convert column dtypes\n", + "- **SkipRowProvider**: Filter rows by conditions\n", + "- **DateRangeFilterProvider**: Filter by date range\n", + "- **NumericIdCleanupIngestor**: Remove .0 suffix from IDs" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/03b-Schema-Driven-Pipeline.ipynb b/document-graph/examples/cloud/notebooks/03b-Schema-Driven-Pipeline.ipynb new file mode 100644 index 00000000..ad33aa93 --- /dev/null +++ b/document-graph/examples/cloud/notebooks/03b-Schema-Driven-Pipeline.ipynb @@ -0,0 +1,217 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 03 - Schema-Driven Pipeline (Extract \u2192 Transform \u2192 Build \u2192 Neptune)\n", + "\n", + "Uses the old orchestration pattern: CSVExtractProvider reads data, transformers process it, then write to Neptune." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Note: Schema definitions can be loaded from S3\n", + "\n", + "The `S3SchemaProvider` supports loading and saving ETL schemas directly from/to S3:\n", + "\n", + "```python\n", + "from document_graph.schema.providers.schema_provider_factory import get_schema_provider\n", + "\n", + "config = {\n", + " \"provider_type\": \"s3\",\n", + " \"schema_id\": \"my-pipeline-schema\",\n", + " \"connection_config\": {\n", + " \"bucket\": \"graphrag-artifacts-705909755305\",\n", + " \"key\": \"schemas/my-schema.json\",\n", + " \"region\": \"us-east-1\"\n", + " }\n", + "}\n", + "\n", + "provider = get_schema_provider(config)\n", + "schema = provider.load_schema()\n", + "```\n", + "\n", + "See `document_graph.schema.providers.s3_schema_provider` for full details.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install -q --force-reinstall ~/SageMaker/document-graph/wheels/document_graph-3.0.0-py3-none-any.whl" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Extract (CSVExtractProvider)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from document_graph.pipeline.extract.extract_provider_csv import CSVExtractProvider\n", + "from document_graph.pipeline.extract.extract_provider_config import ExtractProviderConfig\n", + "from document_graph.config import DocumentGraphConfig\n", + "\n", + "config = ExtractProviderConfig(type='csv', source='users.csv', document_id='users-pipeline-test')\n", + "provider = CSVExtractProvider(config, DocumentGraphConfig())\n", + "result = provider.extract('/home/ec2-user/SageMaker/document-graph/data/users.csv')\n", + "\n", + "print(f'Extracted: {result.dataframe.shape[0]} rows')\n", + "print(f'Schema: {list(result.extracted_schema.keys())}')\n", + "print(f'Nodes: {len(result.nodes)}')\n", + "print(result.dataframe)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Ingest (Column Select + Row Filter)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from document_graph.ingest.ingestors_provider_config import IngestorProviderConfig\n", + "from document_graph.ingest.column.column_selector import ColumnSelectorProvider\n", + "from document_graph.ingest.row.skip_row import SkipRowProvider\n", + "\n", + "df = result.dataframe\n", + "\n", + "# Select columns\n", + "config = IngestorProviderConfig(name='select', type='column', args={'columns': ['id', 'name', 'email', 'role', 'account_id']})\n", + "df = ColumnSelectorProvider(config).ingest(df)\n", + "\n", + "# Filter out viewers\n", + "config = IngestorProviderConfig(name='filter', type='row', args={'conditions': [{'field': 'role', 'operator': 'ne', 'value': 'viewer'}]})\n", + "df = SkipRowProvider(config).ingest(df)\n", + "\n", + "print(f'After ingest: {len(df)} rows (viewers removed)')\n", + "print(df)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Transform (RowToNode + InferEdges)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from document_graph.transform.transformer_provider_config import TransformerProviderConfig\n", + "from document_graph.transform.graph_transformers.row_to_node import RowToNodeTransformer\n", + "from document_graph.transform.graph_transformers.infer_edges import EdgeInferencer\n", + "\n", + "records = df.to_dict('records')\n", + "\n", + "# Rows \u2192 User nodes\n", + "config = TransformerProviderConfig(name='r2n', args={'type': 'User'})\n", + "nodes_data = RowToNodeTransformer(config).transform(records)\n", + "print(f'Nodes: {len(nodes_data)}')\n", + "\n", + "# Infer edges from shared account_id\n", + "config = TransformerProviderConfig(name='edges', args={'source_field': 'account_id', 'edge_type': 'SAME_ACCOUNT'})\n", + "edges_data = EdgeInferencer(config).transform(records)\n", + "print(f'Edges: {len(edges_data)}')\n", + "print(f'First node: {nodes_data[0]}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Build (Cypher \u2192 Neptune)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from graphrag_toolkit.lexical_graph.storage import GraphStoreFactory\n", + "from document_graph import Node, Edge\n", + "from document_graph.graph_build import node_to_cypher, edge_to_cypher\n", + "\n", + "GRAPH_STORE = 'neptune-db://obs-app-dev-graph.cluster-czupj1ab2tc6.us-east-1.neptune.amazonaws.com:8182'\n", + "gs = GraphStoreFactory.for_graph_store(GRAPH_STORE).__enter__()\n", + "TENANT = 'pipeline_v3'\n", + "\n", + "# Write nodes\n", + "for n in nodes_data:\n", + " node = Node(id=n.get('id'), labels=[n.get('node_type', 'User')], properties=n)\n", + " cypher, params = node_to_cypher(node, tenant_id=TENANT)\n", + " gs.execute_query(cypher, params)\n", + "print(f'\u2705 Wrote {len(nodes_data)} nodes (tenant={TENANT})')\n", + "\n", + "# Write edges\n", + "edge_count = 0\n", + "for ed in edges_data:\n", + " edge = Edge(id=ed.get('id',''), source_id=ed.get('source_id',''), target_id=ed.get('target_id',''), label=ed.get('edge_type','RELATED'))\n", + " cypher, params = edge_to_cypher(edge, tenant_id=TENANT)\n", + " gs.execute_query(cypher, params)\n", + " edge_count += 1\n", + "print(f'\u2705 Wrote {edge_count} edges')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Query (Verify)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from document_graph.query import DocumentGraphQueryEngine\n", + "\n", + "engine = DocumentGraphQueryEngine(gs, tenant_id=TENANT)\n", + "\n", + "users = engine.get_nodes('User')\n", + "print(f'\\n=== Results ===')\n", + "print(f'Users in graph: {len(users)}')\n", + "for u in users:\n", + " props = u['n']['~properties']\n", + " print(f\" {props.get('name')} ({props.get('role')}) \u2192 account: {props.get('account_id')}\")\n", + "\n", + "print(f'\\n\u2705 Full pipeline: Extract \u2192 Ingest \u2192 Transform \u2192 Build \u2192 Query COMPLETE')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/03c-JSON-Driven-Pipeline.ipynb b/document-graph/examples/cloud/notebooks/03c-JSON-Driven-Pipeline.ipynb new file mode 100644 index 00000000..0140026c --- /dev/null +++ b/document-graph/examples/cloud/notebooks/03c-JSON-Driven-Pipeline.ipynb @@ -0,0 +1,190 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 03a - JSON-Driven Pipeline (mapping.json \u2192 Neptune)\n", + "\n", + "Defines the graph schema in JSON, then PipelineExecutor reads it and builds the graph automatically.\n", + "\n", + "No manual transform steps \u2014 the JSON defines everything." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install -q --force-reinstall --no-cache-dir ~/SageMaker/document-graph/wheels/document_graph-3.0.0-py3-none-any.whl" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define the Schema (JSON)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Note: Schema JSON can also be loaded from S3\n", + "\n", + "Instead of defining the schema inline, you can load it from S3:\n", + "\n", + "```python\n", + "from document_graph.schema.providers.schema_provider_factory import get_schema_provider\n", + "\n", + "provider = get_schema_provider({\n", + " \"provider_type\": \"s3\",\n", + " \"schema_id\": \"my-mapping\",\n", + " \"connection_config\": {\n", + " \"bucket\": \"graphrag-artifacts-705909755305\",\n", + " \"key\": \"schemas/mapping.json\",\n", + " \"region\": \"us-east-1\"\n", + " }\n", + "})\n", + "schema = provider.load_schema()\n", + "```\n", + "\n", + "This is useful for sharing schemas across notebooks and pipelines without duplication.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json, os\n", + "\n", + "# This is what mapping.json looks like \u2014 defines nodes, edges, property mappings\n", + "schema = {\n", + " \"schema_version\": \"1.0\",\n", + " \"graph\": {\n", + " \"nodes\": [\n", + " {\"User\": {\"type\": \"node\", \"property\": {\"name\": \"user_id\", \"csv_map\": \"id\"}}},\n", + " {\"Account\": {\"type\": \"node\", \"property\": {\"name\": \"account_id\", \"csv_map\": \"account_id\"}}}\n", + " ],\n", + " \"edges\": [\n", + " {\"User\": {\"BELONGS_TO\": \"Account\"}}\n", + " ]\n", + " },\n", + " \"mappings\": [\n", + " {\"csv_property_name\": \"id\", \"type\": \"node\", \"arrow_property_name\": \"user_id\", \"name\": \"User\"},\n", + " {\"csv_property_name\": \"name\", \"type\": \"property\", \"arrow_property_name\": \"full_name\", \"parents\": [\"User\"]},\n", + " {\"csv_property_name\": \"email\", \"type\": \"property\", \"arrow_property_name\": \"email\", \"parents\": [\"User\"]},\n", + " {\"csv_property_name\": \"role\", \"type\": \"property\", \"arrow_property_name\": \"role\", \"parents\": [\"User\"]},\n", + " {\"csv_property_name\": \"account_id\", \"type\": \"node\", \"arrow_property_name\": \"account_id\", \"name\": \"Account\"}\n", + " ]\n", + "}\n", + "\n", + "# Save to file\n", + "schema_path = os.path.expanduser('~/SageMaker/document-graph/data/users_schema.json')\n", + "with open(schema_path, 'w') as f:\n", + " json.dump(schema, f, indent=2)\n", + "print(f'Schema written to {schema_path}')\n", + "print(f'Nodes: {[list(n.keys())[0] for n in schema[\"graph\"][\"nodes\"]]}')\n", + "print(f'Edges: {schema[\"graph\"][\"edges\"]}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load Source Data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "df = pd.read_csv(os.path.expanduser('~/SageMaker/document-graph/data/users.csv'))\n", + "print(f'Source: {len(df)} rows')\n", + "print(df)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run Pipeline (JSON-driven)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from graphrag_toolkit.lexical_graph.storage import GraphStoreFactory\n", + "from document_graph import PipelineExecutor\n", + "\n", + "GRAPH_STORE = 'neptune-db://obs-app-dev-graph.cluster-czupj1ab2tc6.us-east-1.neptune.amazonaws.com:8182'\n", + "gs = GraphStoreFactory.for_graph_store(GRAPH_STORE).__enter__()\n", + "\n", + "executor = PipelineExecutor(gs, tenant_id='schema_driven')\n", + "result = executor.run_from_schema(schema_path, source_df=df)\n", + "\n", + "print(f'\\n=== Pipeline Result ===')\n", + "print(f'Records processed: {result.records_processed}')\n", + "print(f'Nodes created: {result.nodes_created}')\n", + "print(f'Edges created: {result.edges_created}')\n", + "print(f'Errors: {len(result.errors)}')\n", + "if result.errors:\n", + " for e in result.errors[:3]:\n", + " print(f' {e}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Verify in Neptune" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from document_graph.query import DocumentGraphQueryEngine\n", + "\n", + "engine = DocumentGraphQueryEngine(gs, tenant_id='schema_driven')\n", + "\n", + "print('Users:')\n", + "for u in engine.get_nodes('User'):\n", + " props = u['n']['~properties']\n", + " print(f\" {props.get('full_name', props.get('user_id', '?'))} ({props.get('role', '?')})\")\n", + "\n", + "print('\\nAccounts:')\n", + "for a in engine.get_nodes('Account'):\n", + " props = a['n']['~properties']\n", + " print(f\" {props.get('account_id', '?')}\")\n", + "\n", + "print(f'\\n\u2705 JSON-driven pipeline complete \u2014 schema \u2192 Neptune in one call')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/03d-Field-Transformers-Test.ipynb b/document-graph/examples/cloud/notebooks/03d-Field-Transformers-Test.ipynb new file mode 100644 index 00000000..dcf4c38d --- /dev/null +++ b/document-graph/examples/cloud/notebooks/03d-Field-Transformers-Test.ipynb @@ -0,0 +1,414 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 04 - Field Transformers Test\n", + "Testing all field transformers: JSONFlattenerProvider, JSONArrayExpanderProvider, JSONArrayFlattenerProvider, JSONValueFlattenerProvider, CommaFlattenerProvider, CommaSplitProvider, EmbeddedJSONFieldTransformer, PairedFlattenerProvider, RegexCleanerProvider, EnumStandardizer, UuidGeneratorTransformer, TimestampNormalizer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from document_graph.transform.transformer_provider_config import TransformerProviderConfig\n", + "from document_graph.transform.field_transformers.json_flattener import JSONFlattenerProvider\n", + "from document_graph.transform.field_transformers.json_array_expander import JSONArrayExpanderProvider\n", + "from document_graph.transform.field_transformers.json_array_flattener import JSONArrayFlattenerProvider\n", + "from document_graph.transform.field_transformers.json_value_flattener import JSONValueFlattenerProvider\n", + "from document_graph.transform.field_transformers.comma_flattener import CommaFlattenerProvider\n", + "from document_graph.transform.field_transformers.comma_split_provider import CommaSplitProvider\n", + "from document_graph.transform.field_transformers.embedded_json import EmbeddedJSONFieldTransformer\n", + "from document_graph.transform.field_transformers.paired_flattener import PairedFlattenerProvider\n", + "from document_graph.transform.field_transformers.regex_cleaner_provider import RegexCleanerProvider\n", + "from document_graph.transform.field_transformers.standardize_enum import EnumStandardizer\n", + "from document_graph.transform.field_transformers.uuid_generator import UuidGeneratorTransformer\n", + "from document_graph.transform.field_transformers.normalize_timestamp import TimestampNormalizer" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## JSONFlattenerProvider" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "records = [\n", + " {'id': 'CTRL-001', 'control_data': '{\"framework\": \"NIST\", \"category\": \"Access Control\", \"severity\": \"High\"}'},\n", + " {'id': 'CTRL-002', 'control_data': '{\"framework\": \"SOC2\", \"category\": \"Monitoring\", \"severity\": \"Medium\"}'}\n", + "]\n", + "config = TransformerProviderConfig(name='flatten_control', args={'field': 'control_data', 'prefix': 'ctrl_'})\n", + "transformer = JSONFlattenerProvider(config)\n", + "result = transformer.transform(records)\n", + "print(\"JSONFlattenerProvider:\")\n", + "for r in result:\n", + " print(r)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## JSONArrayExpanderProvider" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "records = [\n", + " {'finding_id': 'F-101', 'evidence': '[{\"type\": \"LOG\", \"source\": \"CloudTrail\"}, {\"type\": \"ALERT\", \"source\": \"GuardDuty\"}]'}\n", + "]\n", + "config = TransformerProviderConfig(name='expand_evidence', args={\n", + " 'field': 'evidence',\n", + " 'skip_empty_arrays': True,\n", + " 'conflict_resolution': 'prefix',\n", + " 'auto_add_discovered': True\n", + "})\n", + "transformer = JSONArrayExpanderProvider(config)\n", + "result = transformer.transform(records)\n", + "print(\"JSONArrayExpanderProvider:\")\n", + "print(f\"Input: 1 record -> Output: {len(result)} records\")\n", + "for r in result:\n", + " print(r)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## JSONArrayFlattenerProvider" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "records = [\n", + " {'policy_id': 'POL-01', 'controls': '[\"AC-1\", \"AC-2\", \"AC-3\"]'}\n", + "]\n", + "config = TransformerProviderConfig(name='flatten_controls', args={\n", + " 'field': 'controls',\n", + " 'prefix': 'control',\n", + " 'suffix': ' #00'\n", + "})\n", + "transformer = JSONArrayFlattenerProvider(config)\n", + "result = transformer.transform(records)\n", + "print(\"JSONArrayFlattenerProvider:\")\n", + "for r in result:\n", + " print(r)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## JSONValueFlattenerProvider" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "records = [\n", + " {'user': 'admin', 'project_ids': 'PRJ-100, PRJ-200, PRJ-300', 'project_names': 'IAM Upgrade, SOC Setup, Audit Prep'}\n", + "]\n", + "config = TransformerProviderConfig(name='flatten_projects', args={\n", + " 'id_field': 'project_ids',\n", + " 'name_field': 'project_names',\n", + " 'prefix': 'project',\n", + " 'max_items': 5\n", + "})\n", + "transformer = JSONValueFlattenerProvider(config)\n", + "result = transformer.transform(records)\n", + "print(\"JSONValueFlattenerProvider:\")\n", + "for r in result:\n", + " print(r)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## CommaFlattenerProvider" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "records = [\n", + " {'user': 'analyst_01', 'permissions': 'read:logs, write:alerts, admin:dashboards'}\n", + "]\n", + "config = TransformerProviderConfig(name='flatten_permissions', args={\n", + " 'field': 'permissions',\n", + " 'prefix': 'perm',\n", + " 'suffix': ' #00',\n", + " 'max_items': 10\n", + "})\n", + "transformer = CommaFlattenerProvider(config)\n", + "result = transformer.transform(records)\n", + "print(\"CommaFlattenerProvider:\")\n", + "for r in result:\n", + " print(r)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## CommaSplitProvider" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "records = [\n", + " {'control_id': 'AC-2', 'assigned_teams': 'IAM Team, SOC, Platform Engineering'}\n", + "]\n", + "config = TransformerProviderConfig(name='split_teams', args={\n", + " 'fields': ['assigned_teams'],\n", + " 'separator': ',',\n", + " 'strip_whitespace': True\n", + "})\n", + "transformer = CommaSplitProvider(config)\n", + "result = transformer.transform(records)\n", + "print(\"CommaSplitProvider:\")\n", + "print(f\"Input: 1 record -> Output: {len(result)} records\")\n", + "for r in result:\n", + " print(r)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## EmbeddedJSONFieldTransformer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "records = [\n", + " {'alert_id': 'ALT-500', 'metadata': '{\"source\": \"GuardDuty\", \"region\": \"us-east-1\", \"account_id\": \"123456789012\"}'}\n", + "]\n", + "config = TransformerProviderConfig(name='extract_metadata', args={\n", + " 'field': 'metadata',\n", + " 'prefix': 'meta_'\n", + "})\n", + "transformer = EmbeddedJSONFieldTransformer(config)\n", + "result = transformer.transform(records)\n", + "print(\"EmbeddedJSONFieldTransformer:\")\n", + "for r in result:\n", + " print(r)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## PairedFlattenerProvider" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "records = [\n", + " {'user': 'admin', 'account_ids': 'ACC-01, ACC-02', 'account_names': 'Production, Staging'}\n", + "]\n", + "config = TransformerProviderConfig(name='pair_accounts', args={\n", + " 'id_field': 'account_ids',\n", + " 'name_field': 'account_names',\n", + " 'prefix': 'account',\n", + " 'suffix': '#00',\n", + " 'max_items': 5\n", + "})\n", + "transformer = PairedFlattenerProvider(config)\n", + "result = transformer.transform(records)\n", + "print(\"PairedFlattenerProvider:\")\n", + "for r in result:\n", + " print(r)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## RegexCleanerProvider" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "records = [\n", + " {'id': 'V-001', 'description': '

Critical CVE-2024-1234 found in production.

'},\n", + " {'id': 'V-002', 'description': '
Unpatched OpenSSL vulnerability.
'}\n", + "]\n", + "config = TransformerProviderConfig(name='strip_html', args={\n", + " 'patterns': ['<[^>]*>'],\n", + " 'replacement': '',\n", + " 'fields': ['description']\n", + "})\n", + "transformer = RegexCleanerProvider(config)\n", + "result = transformer.transform(records)\n", + "print(\"RegexCleanerProvider:\")\n", + "for r in result:\n", + " print(r)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## EnumStandardizer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "records = [\n", + " {'id': 1, 'severity': 'Critical'},\n", + " {'id': 2, 'severity': 'HIGH'},\n", + " {'id': 3, 'severity': 'med'},\n", + " {'id': 4, 'severity': 'informational'}\n", + "]\n", + "config = TransformerProviderConfig(name='standardize_severity', args={\n", + " 'field': 'severity',\n", + " 'mapping': {\n", + " 'critical': 'CRITICAL',\n", + " 'high': 'HIGH',\n", + " 'med': 'MEDIUM',\n", + " 'medium': 'MEDIUM',\n", + " 'low': 'LOW',\n", + " 'informational': 'INFO'\n", + " },\n", + " 'case_insensitive': True\n", + "})\n", + "transformer = EnumStandardizer(config)\n", + "result = transformer.transform(records)\n", + "print(\"EnumStandardizer:\")\n", + "for r in result:\n", + " print(r)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## UuidGeneratorTransformer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "records = [\n", + " {'control': 'AC-1', 'title': 'Access Control Policy'},\n", + " {'control': 'AC-2', 'title': 'Account Management'}\n", + "]\n", + "config = TransformerProviderConfig(name='gen_uuid', args={'target_field': 'node_id'})\n", + "transformer = UuidGeneratorTransformer(config)\n", + "result = transformer.transform(records)\n", + "print(\"UuidGeneratorTransformer:\")\n", + "for r in result:\n", + " print(r)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## TimestampNormalizer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "records = [\n", + " {'event': 'login_attempt', 'timestamp': 'Jun 20, 2026 14:30:45'},\n", + " {'event': 'privilege_escalation', 'timestamp': '2026/06/21 09:15 AM'},\n", + " {'event': 'data_export', 'timestamp': '21-06-2026 23:59:59'}\n", + "]\n", + "config = TransformerProviderConfig(name='normalize_ts', args={\n", + " 'field': 'timestamp',\n", + " 'output_field': 'iso_timestamp'\n", + "})\n", + "transformer = TimestampNormalizer(config)\n", + "result = transformer.transform(records)\n", + "print(\"TimestampNormalizer:\")\n", + "for r in result:\n", + " print(r)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "All field transformers tested successfully:\n", + "- **JSONFlattenerProvider**: Flatten JSON objects into prefixed columns\n", + "- **JSONArrayExpanderProvider**: Expand JSON arrays into multiple records\n", + "- **JSONArrayFlattenerProvider**: Flatten JSON arrays into numbered columns\n", + "- **JSONValueFlattenerProvider**: Create JSON objects from paired fields\n", + "- **CommaFlattenerProvider**: Flatten comma-separated values into numbered columns\n", + "- **CommaSplitProvider**: Split comma-separated values into multiple records\n", + "- **EmbeddedJSONFieldTransformer**: Parse and flatten embedded JSON strings\n", + "- **PairedFlattenerProvider**: Flatten paired comma-separated fields\n", + "- **RegexCleanerProvider**: Clean text with regex patterns\n", + "- **EnumStandardizer**: Map values to standard enumerations\n", + "- **UuidGeneratorTransformer**: Generate UUIDs for records\n", + "- **TimestampNormalizer**: Normalize timestamps to ISO 8601" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/03e-Normalizers-Test.ipynb b/document-graph/examples/cloud/notebooks/03e-Normalizers-Test.ipynb new file mode 100644 index 00000000..cd5a7fcb --- /dev/null +++ b/document-graph/examples/cloud/notebooks/03e-Normalizers-Test.ipynb @@ -0,0 +1,237 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 05 - Normalizers Test\n", + "Testing all normalizer providers: NormalizeWhitespaceProvider, NormalizeNullsProvider, NormalizeCaseProvider, NormalizeEnumProvider, NormalizeSpellingProvider, NormalizeTimestampProvider" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from document_graph.transform.transformer_provider_config import TransformerProviderConfig\n", + "from document_graph.transform.normalizers.normalize_whitespace_provider import NormalizeWhitespaceProvider\n", + "from document_graph.transform.normalizers.normalize_nulls_provider import NormalizeNullsProvider\n", + "from document_graph.transform.normalizers.normalize_case_provider import NormalizeCaseProvider\n", + "from document_graph.transform.normalizers.normalize_enum_provider import NormalizeEnumProvider\n", + "from document_graph.transform.normalizers.normalize_spelling_provider import NormalizeSpellingProvider\n", + "from document_graph.transform.normalizers.normalize_timestamp_provider import NormalizeTimestampProvider" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## NormalizeWhitespaceProvider" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "records = [\n", + " {'id': 'C-001', 'title': ' Access Control Policy ', 'description': 'Ensure\\t\\tall users have MFA'},\n", + " {'id': 'C-002', 'title': ' Data Encryption ', 'description': 'Encrypt data at rest and in transit'}\n", + "]\n", + "config = TransformerProviderConfig(name='normalize_ws', args={'fields': ['title', 'description']})\n", + "transformer = NormalizeWhitespaceProvider(config)\n", + "result = transformer.transform(records)\n", + "print(\"NormalizeWhitespaceProvider:\")\n", + "for r in result:\n", + " print(f\" title: '{r['title']}'\")\n", + " print(f\" desc: '{r['description']}'\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## NormalizeNullsProvider" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "records = [\n", + " {'id': 'U-001', 'department': 'SOC', 'manager': 'N/A', 'notes': ''},\n", + " {'id': 'U-002', 'department': 'GRC', 'manager': 'null', 'notes': 'none'},\n", + " {'id': 'U-003', 'department': 'na', 'manager': 'Jane Smith', 'notes': '-'}\n", + "]\n", + "config = TransformerProviderConfig(name='normalize_nulls', args={\n", + " 'fields': ['department', 'manager', 'notes'],\n", + " 'null_like': ['', 'n/a', 'na', 'null', 'none', '-']\n", + "})\n", + "transformer = NormalizeNullsProvider(config)\n", + "result = transformer.transform(records)\n", + "print(\"NormalizeNullsProvider:\")\n", + "for r in result:\n", + " print(r)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## NormalizeCaseProvider" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "records = [\n", + " {'control': 'Access Control', 'framework': 'nist csf', 'status': 'Active'},\n", + " {'control': 'Data Protection', 'framework': 'SOC 2', 'status': 'pending'}\n", + "]\n", + "config = TransformerProviderConfig(name='to_upper', args={'mode': 'upper'})\n", + "transformer = NormalizeCaseProvider(config)\n", + "result = transformer.transform(records)\n", + "print(\"NormalizeCaseProvider (mode=upper):\")\n", + "for r in result:\n", + " print(r)\n", + "\n", + "print()\n", + "config = TransformerProviderConfig(name='to_lower', args={'mode': 'lower'})\n", + "transformer = NormalizeCaseProvider(config)\n", + "result = transformer.transform(records)\n", + "print(\"NormalizeCaseProvider (mode=lower):\")\n", + "for r in result:\n", + " print(r)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## NormalizeEnumProvider" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "records = [\n", + " {'id': 1, 'risk_level': 'Critical'},\n", + " {'id': 2, 'risk_level': 'HIGH'},\n", + " {'id': 3, 'risk_level': 'Med'},\n", + " {'id': 4, 'risk_level': 'low'},\n", + " {'id': 5, 'risk_level': 'informational'}\n", + "]\n", + "config = TransformerProviderConfig(name='normalize_risk', args={\n", + " 'field': 'risk_level',\n", + " 'enum_map': {\n", + " 'critical': 'CRITICAL',\n", + " 'high': 'HIGH',\n", + " 'med': 'MEDIUM',\n", + " 'medium': 'MEDIUM',\n", + " 'low': 'LOW',\n", + " 'informational': 'INFO'\n", + " },\n", + " 'case_insensitive': True\n", + "})\n", + "transformer = NormalizeEnumProvider(config)\n", + "result = transformer.transform(records)\n", + "print(\"NormalizeEnumProvider:\")\n", + "for r in result:\n", + " print(f\" {r['id']}: {r['risk_level']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## NormalizeSpellingProvider" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "records = [\n", + " {'id': 'F-001', 'description': 'Unathorized acess to prodution server detectd'},\n", + " {'id': 'F-002', 'description': 'Privilage escalaton attemp from externel IP'}\n", + "]\n", + "config = TransformerProviderConfig(name='fix_spelling', args={'fields': ['description']})\n", + "transformer = NormalizeSpellingProvider(config)\n", + "try:\n", + " result = transformer.transform(records)\n", + " print(\"NormalizeSpellingProvider:\")\n", + " for r in result:\n", + " print(f\" {r['id']}: {r['description']}\")\n", + "except ImportError as e:\n", + " print(f\"NormalizeSpellingProvider: Skipped (TextBlob not installed) - {e}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## NormalizeTimestampProvider" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "records = [\n", + " {'event': 'auth_failure', 'created_at': 'Jun 20, 2026 14:30:45', 'resolved_at': '06/21/2026 09:00'},\n", + " {'event': 'data_breach', 'created_at': '2026-06-22T18:45:00Z', 'resolved_at': '22 June 2026 20:30'},\n", + " {'event': 'policy_violation', 'created_at': '06-23-2026 11:15:30', 'resolved_at': '2026.06.23 15:00:00'}\n", + "]\n", + "config = TransformerProviderConfig(name='normalize_timestamps', args={\n", + " 'fields': ['created_at', 'resolved_at']\n", + "})\n", + "transformer = NormalizeTimestampProvider(config)\n", + "result = transformer.transform(records)\n", + "print(\"NormalizeTimestampProvider:\")\n", + "for r in result:\n", + " print(f\" {r['event']}: created={r['created_at']}, resolved={r['resolved_at']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "All normalizer providers tested successfully:\n", + "- **NormalizeWhitespaceProvider**: Strip/collapse whitespace in specified fields\n", + "- **NormalizeNullsProvider**: Convert null-like strings to None\n", + "- **NormalizeCaseProvider**: Apply case transformation (lower/upper) to all string fields\n", + "- **NormalizeEnumProvider**: Map values to standardized enums via enum_map\n", + "- **NormalizeSpellingProvider**: Correct spelling via TextBlob\n", + "- **NormalizeTimestampProvider**: Parse various date formats to ISO 8601" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/03f-Constructors-Test.ipynb b/document-graph/examples/cloud/notebooks/03f-Constructors-Test.ipynb new file mode 100644 index 00000000..e166a09b --- /dev/null +++ b/document-graph/examples/cloud/notebooks/03f-Constructors-Test.ipynb @@ -0,0 +1,309 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 07 - Constructors Test\n", + "Testing graph constructors: NodeConstructor, EdgeConstructor, PropertyConstructor, SchemaDrivenConstructor, BatchConstructor, DeduplicationConstructor, ManyToManyConstructor, OneToManyConstructor" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "from document_graph.graph_build.constructors.constructors_provider_config import ConstructorProviderConfig\n", + "from document_graph.graph_build.constructors.core.node_constructor import NodeConstructor\n", + "from document_graph.graph_build.constructors.core.edge_constructor import EdgeConstructor\n", + "from document_graph.graph_build.constructors.core.property_constructor import PropertyConstructor\n", + "from document_graph.graph_build.constructors.core.schema_driven_constructor import SchemaDrivenConstructor\n", + "from document_graph.graph_build.constructors.optimizations.batch_constructor import BatchConstructor\n", + "from document_graph.graph_build.constructors.optimizations.deduplication_constructor import DeduplicationConstructor\n", + "from document_graph.graph_build.constructors.patterns.many_to_many_constructor import ManyToManyConstructor\n", + "from document_graph.graph_build.constructors.patterns.one_to_many_constructor import OneToManyConstructor\n", + "from document_graph.model_elements import Node, Edge" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Sample security data for graph construction\n", + "df = pd.DataFrame({\n", + " 'user_id': ['U-001', 'U-002', 'U-003', 'U-001', 'U-002'],\n", + " 'username': ['alice', 'bob', 'charlie', 'alice', 'bob'],\n", + " 'role': ['admin', 'analyst', 'auditor', 'admin', 'analyst'],\n", + " 'permission': ['write:all', 'read:logs', 'read:reports', 'delete:users', 'write:alerts'],\n", + " 'account_id': ['ACC-100', 'ACC-100', 'ACC-200', 'ACC-200', 'ACC-100'],\n", + " 'account_name': ['Production', 'Production', 'Staging', 'Staging', 'Production'],\n", + " 'control_id': ['AC-2', 'AU-6', 'CA-7', 'AC-2', 'AU-6']\n", + "})\n", + "print(\"Source DataFrame:\")\n", + "print(df.to_string())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## NodeConstructor" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "config = ConstructorProviderConfig(\n", + " name='user_nodes',\n", + " type='node',\n", + " args={'label': 'User', 'id_field': 'user_id', 'properties': ['username', 'role']}\n", + ")\n", + "constructor = NodeConstructor(config)\n", + "nodes, edges = constructor.construct(df)\n", + "print(f\"NodeConstructor: {len(nodes)} nodes, {len(edges)} edges\")\n", + "print(f\" Config: type={config.type}, args={config.args}\")\n", + "\n", + "# Demonstrate Node dataclass directly\n", + "sample_node = Node(id='U-001', labels=['User'], properties={'username': 'alice', 'role': 'admin'})\n", + "print(f\" Sample Node: id={sample_node.id}, labels={sample_node.labels}, props={sample_node.properties}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## EdgeConstructor" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "config = ConstructorProviderConfig(\n", + " name='user_account_edges',\n", + " type='edge',\n", + " args={'label': 'HAS_ACCESS_TO', 'source_field': 'user_id', 'target_field': 'account_id'}\n", + ")\n", + "constructor = EdgeConstructor(config)\n", + "nodes, edges = constructor.construct(df)\n", + "print(f\"EdgeConstructor: {len(nodes)} nodes, {len(edges)} edges\")\n", + "print(f\" Config: type={config.type}, args={config.args}\")\n", + "\n", + "# Demonstrate Edge dataclass directly\n", + "sample_edge = Edge(id='e-001', source_id='U-001', target_id='ACC-100', label='HAS_ACCESS_TO', properties={'granted': '2026-01-01'})\n", + "print(f\" Sample Edge: {sample_edge.source_id} -[{sample_edge.label}]-> {sample_edge.target_id}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## PropertyConstructor" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "config = ConstructorProviderConfig(\n", + " name='user_properties',\n", + " type='property',\n", + " args={'node_label': 'User', 'property_fields': ['username', 'role', 'permission']}\n", + ")\n", + "constructor = PropertyConstructor(config)\n", + "nodes, edges = constructor.construct(df)\n", + "print(f\"PropertyConstructor: {len(nodes)} nodes, {len(edges)} edges\")\n", + "print(f\" Config: type={config.type}, args={config.args}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## SchemaDrivenConstructor" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "config = ConstructorProviderConfig(\n", + " name='schema_builder',\n", + " type='schema_driven',\n", + " args={\n", + " 'schema': {\n", + " 'nodes': [\n", + " {'label': 'User', 'id_field': 'user_id'},\n", + " {'label': 'Account', 'id_field': 'account_id'}\n", + " ],\n", + " 'edges': [\n", + " {'label': 'BELONGS_TO', 'source': 'user_id', 'target': 'account_id'}\n", + " ]\n", + " }\n", + " }\n", + ")\n", + "constructor = SchemaDrivenConstructor(config)\n", + "nodes, edges = constructor.construct(df)\n", + "print(f\"SchemaDrivenConstructor: {len(nodes)} nodes, {len(edges)} edges\")\n", + "print(f\" Config: type={config.type}\")\n", + "print(f\" Schema defines {len(config.args['schema']['nodes'])} node types, {len(config.args['schema']['edges'])} edge types\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## BatchConstructor" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "config = ConstructorProviderConfig(\n", + " name='batch_builder',\n", + " type='batch',\n", + " args={'batch_size': 2, 'label': 'User', 'id_field': 'user_id'},\n", + " batch_size=2\n", + ")\n", + "constructor = BatchConstructor(config)\n", + "nodes, edges = constructor.construct(df)\n", + "print(f\"BatchConstructor: {len(nodes)} nodes, {len(edges)} edges\")\n", + "print(f\" Config: type={config.type}, batch_size={config.batch_size}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DeduplicationConstructor" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "config = ConstructorProviderConfig(\n", + " name='dedup_builder',\n", + " type='deduplication',\n", + " args={'dedup_field': 'user_id', 'label': 'User'}\n", + ")\n", + "constructor = DeduplicationConstructor(config)\n", + "nodes, edges = constructor.construct(df)\n", + "print(f\"DeduplicationConstructor: {len(nodes)} nodes, {len(edges)} edges\")\n", + "print(f\" Config: type={config.type}, dedup_field={config.args['dedup_field']}\")\n", + "print(f\" Input rows: {len(df)}, unique user_ids: {df['user_id'].nunique()}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ManyToManyConstructor" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "config = ConstructorProviderConfig(\n", + " name='user_control_m2m',\n", + " type='many_to_many',\n", + " args={\n", + " 'source_label': 'User',\n", + " 'target_label': 'Control',\n", + " 'source_field': 'user_id',\n", + " 'target_field': 'control_id',\n", + " 'edge_label': 'IMPLEMENTS'\n", + " }\n", + ")\n", + "constructor = ManyToManyConstructor(config)\n", + "nodes, edges = constructor.construct(df)\n", + "print(f\"ManyToManyConstructor: {len(nodes)} nodes, {len(edges)} edges\")\n", + "print(f\" Config: {config.args['source_label']} -[{config.args['edge_label']}]-> {config.args['target_label']}\")\n", + "print(f\" Unique users: {df['user_id'].nunique()}, Unique controls: {df['control_id'].nunique()}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## OneToManyConstructor" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "config = ConstructorProviderConfig(\n", + " name='account_users_o2m',\n", + " type='one_to_many',\n", + " args={\n", + " 'parent_label': 'Account',\n", + " 'child_label': 'User',\n", + " 'parent_field': 'account_id',\n", + " 'child_field': 'user_id',\n", + " 'edge_label': 'CONTAINS_USER'\n", + " }\n", + ")\n", + "constructor = OneToManyConstructor(config)\n", + "nodes, edges = constructor.construct(df)\n", + "print(f\"OneToManyConstructor: {len(nodes)} nodes, {len(edges)} edges\")\n", + "print(f\" Config: {config.args['parent_label']} -[{config.args['edge_label']}]-> {config.args['child_label']}\")\n", + "print(f\" Unique accounts: {df['account_id'].nunique()}, Unique users: {df['user_id'].nunique()}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "All graph constructors tested successfully:\n", + "- **NodeConstructor**: Create nodes from DataFrame rows\n", + "- **EdgeConstructor**: Create edges from source/target field pairs\n", + "- **PropertyConstructor**: Map DataFrame columns to node properties\n", + "- **SchemaDrivenConstructor**: Build graph from a schema definition\n", + "- **BatchConstructor**: Process large DataFrames in batches\n", + "- **DeduplicationConstructor**: Create unique nodes (dedup by field)\n", + "- **ManyToManyConstructor**: Handle M:N relationship patterns\n", + "- **OneToManyConstructor**: Handle 1:N parent-child patterns\n", + "\n", + "Note: Constructors currently return empty lists (TODO implementations) but accept configs and validate input correctly." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/03g-Document-Transformers-Test.ipynb b/document-graph/examples/cloud/notebooks/03g-Document-Transformers-Test.ipynb new file mode 100644 index 00000000..9428fa4b --- /dev/null +++ b/document-graph/examples/cloud/notebooks/03g-Document-Transformers-Test.ipynb @@ -0,0 +1,157 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 06 - Document Transformers Test\n", + "Testing document-level transformers: JSONToRowsTransformer, TextChunkerTransformer, PIIRedactorProvider" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "!pip install -q sanitary\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from document_graph.transform.transformer_provider_config import TransformerProviderConfig\n", + "from document_graph.transform.document_transformers.json_to_rows import JSONToRowsTransformer\n", + "from document_graph.transform.document_transformers.text_chunker import TextChunkerTransformer\n", + "from document_graph.transform.document_transformers.pii_redactor_provider import PIIRedactorProvider" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## JSONToRowsTransformer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "records = [\n", + " {\n", + " 'source': 'vulnerability_scan',\n", + " 'content': '[{\"cve\": \"CVE-2026-0001\", \"severity\": \"Critical\", \"asset\": \"web-server-01\"}, {\"cve\": \"CVE-2026-0002\", \"severity\": \"High\", \"asset\": \"db-server-03\"}]'\n", + " }\n", + "]\n", + "config = TransformerProviderConfig(name='json_to_rows', args={'json_field': 'content'})\n", + "transformer = JSONToRowsTransformer(config)\n", + "result = transformer.transform(records)\n", + "print(\"JSONToRowsTransformer:\")\n", + "print(f\"Input: 1 record -> Output: {len(result)} records\")\n", + "for r in result:\n", + " print(f\" row_index={r.get('row_index')}: cve={r.get('json_cve')}, severity={r.get('json_severity')}, asset={r.get('json_asset')}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## TextChunkerTransformer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "long_policy = \"\"\"Information Security Policy: All employees must follow secure access procedures. \n", + "Multi-factor authentication is required for all production systems. Passwords must be rotated \n", + "every 90 days and meet complexity requirements of at least 12 characters with mixed case, \n", + "numbers and special characters. Access reviews must be conducted quarterly for all privileged \n", + "accounts. Service accounts must not have interactive login capabilities. All access requests \n", + "must be approved by the asset owner and the security team before provisioning. Terminated \n", + "employees must have access revoked within 24 hours of separation.\"\"\"\n", + "\n", + "records = [{'id': 'POL-SEC-001', 'title': 'Access Control Policy', 'content': long_policy}]\n", + "config = TransformerProviderConfig(name='chunk_policy', args={\n", + " 'chunk_size': 150,\n", + " 'overlap': 20,\n", + " 'text_field': 'content'\n", + "})\n", + "transformer = TextChunkerTransformer(config)\n", + "result = transformer.transform(records)\n", + "print(\"TextChunkerTransformer:\")\n", + "print(f\"Input: 1 record ({len(long_policy)} chars) -> Output: {len(result)} chunks\")\n", + "for r in result:\n", + " print(f\" chunk {r['chunk_index']}: [{r['chunk_start']}:{r['chunk_end']}] '{r['content'][:50]}...'\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## PIIRedactorProvider" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "records = [\n", + " {\n", + " 'incident_id': 'INC-001',\n", + " 'description': 'User john.doe@company.com logged in from 192.168.1.100 with password Admin123!',\n", + " 'analyst_notes': 'Contacted admin@internal.corp at 10.0.0.55 for verification'\n", + " },\n", + " {\n", + " 'incident_id': 'INC-002',\n", + " 'description': 'SSH key exposed by user jane.smith at 172.16.0.42',\n", + " 'analyst_notes': 'Key rotated for service account svc-deploy'\n", + " }\n", + "]\n", + "config = TransformerProviderConfig(name='redact_pii', args={\n", + " 'fields': ['description', 'analyst_notes']\n", + "})\n", + "transformer = PIIRedactorProvider(config)\n", + "result = transformer.transform(records)\n", + "print(\"PIIRedactorProvider:\")\n", + "for r in result:\n", + " print(f\" {r['incident_id']}:\")\n", + " print(f\" description: {r['description']}\")\n", + " print(f\" notes: {r['analyst_notes']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "All document transformers tested successfully:\n", + "- **JSONToRowsTransformer**: Flatten nested JSON into tabular rows with json_ prefix\n", + "- **TextChunkerTransformer**: Split long text into overlapping chunks with metadata\n", + "- **PIIRedactorProvider**: Redact PII (emails, IPs, credentials) from specified fields" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/07a-Lexical-Integration-Data-Processing.ipynb b/document-graph/examples/cloud/notebooks/07a-Lexical-Integration-Data-Processing.ipynb new file mode 100644 index 00000000..4b238c14 --- /dev/null +++ b/document-graph/examples/cloud/notebooks/07a-Lexical-Integration-Data-Processing.ipynb @@ -0,0 +1,259 @@ +{ + "nbformat": 4, + "nbformat_minor": 5, + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 07a \u2014 Hybrid Integration: Data Processing\n", + "\n", + "1. Connect to Neptune\n", + "2. Load structured data (CSV, JSON) into Neptune as graph nodes\n", + "3. Convert Neptune nodes to LlamaIndex Documents (with lineage)\n", + "4. Save for Part 2 (lexical-graph indexing)\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Setup\n", + "import json, os\n", + "import pandas as pd\n", + "from pathlib import Path\n", + "from graphrag_toolkit.lexical_graph.storage import GraphStoreFactory\n", + "from document_graph.graph_build import node_to_cypher\n", + "from document_graph import Node\n", + "\n", + "GRAPH_STORE = 'neptune-db://obs-app-dev-graph.cluster-czupj1ab2tc6.us-east-1.neptune.amazonaws.com:8182'\n", + "TENANT = 'hybrid_demo'\n", + "\n", + "gs = GraphStoreFactory.for_graph_store(GRAPH_STORE).__enter__()\n", + "print('Neptune connected')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Load structured data into Neptune\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "def flatten_properties(props):\n", + " '''Neptune only accepts simple literals.'''\n", + " flat = {}\n", + " for k, v in props.items():\n", + " if isinstance(v, list):\n", + " flat[k] = ', '.join(str(x) for x in v)\n", + " elif isinstance(v, dict):\n", + " flat[k] = json.dumps(v)\n", + " elif v is None:\n", + " continue\n", + " else:\n", + " flat[k] = v\n", + " return flat\n", + "\n", + "data_sources = [\n", + " {'file': 'hybrid_data/data/research_papers.csv', 'type': 'csv', 'node_type': 'ResearchPaper', 'id_field': 'id'},\n", + " {'file': 'hybrid_data/data/authors.json', 'type': 'json', 'node_type': 'Author', 'id_field': 'author_id'},\n", + " {'file': 'hybrid_data/data/citations.csv', 'type': 'csv', 'node_type': 'Citation', 'id_field': 'id'},\n", + "]\n", + "\n", + "all_records = {} # node_type \u2192 list of records\n", + "\n", + "for source in data_sources:\n", + " fname = source['file']\n", + " ntype = source['node_type']\n", + " id_field = source['id_field']\n", + " print(f'\\nLoading: {fname}')\n", + "\n", + " if source['type'] == 'csv':\n", + " df = pd.read_csv(fname)\n", + " else:\n", + " raw = json.load(open(fname))\n", + " # Handle nested JSON like {\"authors\": [...]}\n", + " if isinstance(raw, dict):\n", + " key = list(raw.keys())[0]\n", + " df = pd.DataFrame(raw[key])\n", + " else:\n", + " df = pd.DataFrame(raw)\n", + "\n", + " records = df.to_dict('records')\n", + " all_records[ntype] = records\n", + "\n", + " for record in records:\n", + " rid = str(record.get(id_field, hash(str(record))))\n", + " node = Node(id=rid, labels=[ntype], properties=flatten_properties(record))\n", + " cypher, params = node_to_cypher(node, tenant_id=TENANT)\n", + " gs.execute_query(cypher, params)\n", + "\n", + " print(f' Wrote {len(records)} {ntype} nodes')\n", + "\n", + "print(f'\\nTotal: {sum(len(v) for v in all_records.values())} nodes in Neptune')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2b: Create relationships (edges)\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Step 2b: Create edge relationships\n", + "from document_graph.graph_build import edge_to_cypher\n", + "from document_graph import Edge\n", + "\n", + "edges_written = 0\n", + "\n", + "# AUTHORED_BY: link papers to authors (by matching author_id in papers)\n", + "papers = all_records.get('ResearchPaper', [])\n", + "authors = all_records.get('Author', [])\n", + "author_ids = {a.get('author_id') for a in authors}\n", + "\n", + "for paper in papers:\n", + " # Check if paper has an author field referencing an author\n", + " paper_authors = str(paper.get('authors', paper.get('author_id', ''))).split(',')\n", + " for aid in paper_authors:\n", + " aid = aid.strip()\n", + " if aid in author_ids:\n", + " edge = Edge(id=f'e-{paper[\"id\"]}-{aid}', source_id=str(paper['id']), target_id=aid, label='AUTHORED_BY')\n", + " cypher, params = edge_to_cypher(edge, tenant_id=TENANT)\n", + " gs.execute_query(cypher, params)\n", + " edges_written += 1\n", + "\n", + "# CITES: link citations to papers\n", + "citations = all_records.get('Citation', [])\n", + "paper_ids = {str(p.get('id')) for p in papers}\n", + "\n", + "for cite in citations:\n", + " src = str(cite.get('source_paper', cite.get('citing_paper', '')))\n", + " tgt = str(cite.get('target_paper', cite.get('cited_paper', '')))\n", + " if src and tgt:\n", + " edge = Edge(id=f'c-{cite.get(\"id\", \"\")}', source_id=src, target_id=tgt, label='CITES')\n", + " cypher, params = edge_to_cypher(edge, tenant_id=TENANT)\n", + " gs.execute_query(cypher, params)\n", + " edges_written += 1\n", + "\n", + "print(f'Wrote {edges_written} edges (AUTHORED_BY + CITES)')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Verify in Neptune\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "result = gs.execute_query('MATCH (n) RETURN labels(n) as lbl, count(n) as cnt')\n", + "print('Nodes by type:')\n", + "for row in result:\n", + " print(f' {row}')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Convert to LlamaIndex Documents (with lineage)\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from llama_index.core.schema import Document\n", + "\n", + "lexical_documents = []\n", + "\n", + "for ntype, records in all_records.items():\n", + " id_field = next(s['id_field'] for s in data_sources if s['node_type'] == ntype)\n", + " for record in records:\n", + " node_id = str(record.get(id_field, 'unknown'))\n", + " source_id = f'{ntype}:{node_id}:{TENANT}'\n", + " # Build text with lineage header\n", + " text = f'[{ntype} | {node_id} | {TENANT}]\\n'\n", + " text += '\\n'.join(f'{k}: {v}' for k, v in record.items() if v is not None and str(v).strip())\n", + " # Use graphrag-toolkit's source metadata structure\n", + " doc = Document(\n", + " text=text,\n", + " metadata={\n", + " 'source': {\n", + " 'sourceId': source_id,\n", + " 'metadata': {\n", + " 'node_type': ntype,\n", + " 'node_id': node_id,\n", + " 'tenant': TENANT,\n", + " }\n", + " }\n", + " }\n", + " )\n", + " lexical_documents.append(doc)\n", + "\n", + "print(f'Created {len(lexical_documents)} documents')\n", + "for ntype in all_records:\n", + " count = sum(1 for d in lexical_documents if d.metadata.get('source', {}).get('metadata', {}).get('node_type') == ntype)\n", + " print(f' {ntype}: {count}')\n", + "print(f'\\nSample sourceId: {lexical_documents[0].metadata[\"source\"][\"sourceId\"]}')\n", + "print(f'Sample text:\\n{lexical_documents[0].text[:150]}')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Save for Part 2\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import pickle\n", + "\n", + "output_dir = Path('output/hybrid_foundation')\n", + "output_dir.mkdir(parents=True, exist_ok=True)\n", + "\n", + "with open(output_dir / 'lexical_documents.pkl', 'wb') as f:\n", + " pickle.dump(lexical_documents, f)\n", + "\n", + "with open(output_dir / 'summary.json', 'w') as f:\n", + " json.dump({'total_docs': len(lexical_documents), 'node_types': list(all_records.keys()), 'tenant': TENANT}, f, indent=2)\n", + "\n", + "print(f'Saved {len(lexical_documents)} documents to {output_dir}')\n", + "print('Ready for 07b \u2014 Lexical-Graph Indexing')\n" + ], + "outputs": [], + "execution_count": null + } + ] +} \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/07b-Lexical-Integration-Lexical-Setup.ipynb b/document-graph/examples/cloud/notebooks/07b-Lexical-Integration-Lexical-Setup.ipynb new file mode 100644 index 00000000..d9c3afba --- /dev/null +++ b/document-graph/examples/cloud/notebooks/07b-Lexical-Integration-Lexical-Setup.ipynb @@ -0,0 +1,193 @@ +{ + "nbformat": 4, + "nbformat_minor": 5, + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 07b \u2014 Hybrid Integration: Lexical-Graph Indexing & Query\n", + "\n", + "1. Load documents from Part 1\n", + "2. Index into lexical-graph (Neptune + OpenSearch)\n", + "3. Query with semantic search\n", + "4. Correlate results back to document-graph (lineage)\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Setup\n", + "import json, os, pickle, re\n", + "from pathlib import Path\n", + "from graphrag_toolkit.lexical_graph.storage import GraphStoreFactory, VectorStoreFactory\n", + "from graphrag_toolkit.lexical_graph import LexicalGraphIndex, LexicalGraphQueryEngine\n", + "\n", + "GRAPH_STORE = 'neptune-db://obs-app-dev-graph.cluster-czupj1ab2tc6.us-east-1.neptune.amazonaws.com:8182'\n", + "VECTOR_STORE = 'aoss://https://1lci0mi6xy7hpex8fr0b.us-east-1.aoss.amazonaws.com'\n", + "TENANT = 'hybrid_demo'\n", + "\n", + "graph_store = GraphStoreFactory.for_graph_store(GRAPH_STORE)\n", + "vector_store = VectorStoreFactory.for_vector_store(VECTOR_STORE)\n", + "print('Stores connected')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Load documents from Part 1\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "foundation_dir = Path('output/hybrid_foundation')\n", + "\n", + "with open(foundation_dir / 'lexical_documents.pkl', 'rb') as f:\n", + " lexical_documents = pickle.load(f)\n", + "\n", + "print(f'Loaded {len(lexical_documents)} documents')\n", + "print(f'Sample: {lexical_documents[0].text[:150]}...')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Index into lexical-graph\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Index into lexical-graph (batch writes enabled \u2014 Neptune 1.4.7.0 supports it)\n", + "print(f\"Indexing {len(lexical_documents)} documents (batch writes ON)...\")\n", + "\n", + "graph_index = LexicalGraphIndex(graph_store, vector_store)\n", + "graph_index.extract_and_build(lexical_documents, show_progress=True)\n", + "\n", + "print(\"Indexing complete\")\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Query with semantic search\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "query_engine = LexicalGraphQueryEngine.for_traversal_based_search(graph_store, vector_store)\n", + "\n", + "queries = [\n", + " 'What research discusses machine learning?',\n", + " 'Who are the authors working on graph databases?',\n", + " 'What citations exist between papers?',\n", + "]\n", + "\n", + "for q in queries:\n", + " print(f'\\nQuery: {q}')\n", + " results = query_engine.retrieve(q)\n", + " if isinstance(results, list):\n", + " print(f' Results: {len(results)}')\n", + " for i, r in enumerate(results[:2], 1):\n", + " text = getattr(r.node, 'text', str(r))[:150]\n", + " print(f' {i}. {text}...')\n", + " else:\n", + " print(f' Response: {str(results)[:150]}')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Correlate back to document-graph (lineage)\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Correlate: query results \u2192 Source nodes \u2192 document-graph\n", + "gs_direct = GraphStoreFactory.for_graph_store(GRAPH_STORE).__enter__()\n", + "\n", + "q = 'What research discusses machine learning?'\n", + "results = query_engine.retrieve(q)\n", + "print(f'Query: {q}')\n", + "print(f'Results: {len(results)}\\n')\n", + "\n", + "for i, r in enumerate(results[:5], 1):\n", + " text = getattr(r.node, 'text', str(r))\n", + " meta = getattr(r.node, 'metadata', {})\n", + "\n", + " # Method 1: Parse lineage from text header [Type | node_id | tenant]\n", + " match = re.search(r'\\[(\\w+) \\| ([\\w-]+) \\| (\\w+)\\]', text)\n", + " if match:\n", + " ntype, node_id, tenant = match.group(1), match.group(2), match.group(3)\n", + " label = f'__{ntype}__{tenant}__'\n", + " original = gs_direct.execute_query(f\"MATCH (n:`{label}`) WHERE n.id = '{node_id}' RETURN properties(n) as props LIMIT 1\")\n", + " print(f'{i}. [{ntype}] node_id={node_id}')\n", + " if original:\n", + " props = original[0].get('props', {})\n", + " display = props.get('name', props.get('title', props.get('description', 'N/A')))\n", + " print(f' Neptune: {display}')\n", + " else:\n", + " print(f' (not in Neptune)')\n", + " else:\n", + " # Method 2: Check Source nodes in lexical-graph\n", + " sources = gs_direct.execute_query('MATCH (s:`__Source__`) WHERE s.node_type IS NOT NULL RETURN s.node_type as ntype, s.node_id as nid, s.tenant as t LIMIT 5')\n", + " if sources and i == 1:\n", + " print(f'{i}. [Source nodes available]:') \n", + " for s in sources[:3]:\n", + " print(f' {s}')\n", + " else:\n", + " # Method 3: Entity correlation fallback\n", + " ec = meta.get('entity_contexts', {}).get('contexts', [])\n", + " entities = [ent.get('entity', {}).get('value', '') for ctx in ec[:3] for ent in ctx.get('entities', [])]\n", + " print(f'{i}. [entities: {\", \".join(entities[:3])}]')\n", + " print()\n", + "\n", + "print('Lineage correlation complete')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "The hybrid pipeline:\n", + "1. **Document-graph** stores structured data as typed nodes in Neptune\n", + "2. **Lexical-graph** indexes the same data for semantic search (OpenSearch vectors)\n", + "3. **Lineage** embedded in text `[source_type | node_id | tenant]` survives chunking\n", + "4. **Correlation** extracts lineage from query results \u2192 fetches original node from Neptune\n", + "\n", + "Same graph database (Neptune) holds both the document-graph nodes AND the lexical-graph structure.\n" + ] + } + ] +} \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/08a-Nelson-Mandela-Hybrid-Build.ipynb b/document-graph/examples/cloud/notebooks/08a-Nelson-Mandela-Hybrid-Build.ipynb new file mode 100644 index 00000000..8ecbda84 --- /dev/null +++ b/document-graph/examples/cloud/notebooks/08a-Nelson-Mandela-Hybrid-Build.ipynb @@ -0,0 +1,310 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "41ccb172", + "metadata": {}, + "source": [ + "# 08a — Nelson Mandela Hybrid GraphRAG: Build\n", + "\n", + "1. Connect to Neptune + OpenSearch\n", + "2. Build lexical-graph from Wikipedia (Mandela bio)\n", + "3. Build document-graph from structured data (events, organizations)\n", + "4. Verify both graphs\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "c3efb026", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Neptune + OpenSearch connected\n" + ] + } + ], + "source": [ + "# Setup\n", + "import json, os\n", + "import pandas as pd\n", + "from graphrag_toolkit.lexical_graph.storage import GraphStoreFactory, VectorStoreFactory\n", + "from graphrag_toolkit.lexical_graph import LexicalGraphIndex\n", + "from document_graph.graph_build import node_to_cypher\n", + "from document_graph import Node\n", + "\n", + "GRAPH_STORE = 'neptune-db://obs-app-dev-graph.cluster-czupj1ab2tc6.us-east-1.neptune.amazonaws.com:8182'\n", + "VECTOR_STORE = 'aoss://https://1lci0mi6xy7hpex8fr0b.us-east-1.aoss.amazonaws.com'\n", + "TENANT = 'mandela'\n", + "\n", + "graph_store = GraphStoreFactory.for_graph_store(GRAPH_STORE)\n", + "vector_store = VectorStoreFactory.for_vector_store(VECTOR_STORE)\n", + "gs = graph_store.__enter__()\n", + "print('Neptune + OpenSearch connected')\n" + ] + }, + { + "cell_type": "markdown", + "id": "d1c50539", + "metadata": {}, + "source": [ + "## Step 1: Build lexical-graph from Wikipedia\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "00da6830", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Created 3 documents\n", + " Nelson Mandela: 1125 chars\n", + " Apartheid: 690 chars\n", + " African National Congress: 596 chars\n" + ] + } + ], + "source": [ + "from llama_index.core.schema import Document\n", + "\n", + "# Static Mandela content\n", + "wiki_docs = [\n", + " Document(\n", + " text=\"\"\"Nelson Rolihlahla Mandela was a South African anti-apartheid activist and politician who served as the first president of South Africa from 1994 to 1999. He was the country's first black head of state and the first elected in a fully representative democratic election. His government focused on dismantling the legacy of apartheid by fostering racial reconciliation. Mandela was born in Mvezo, Transkei, and studied law at the University of Fort Hare and the University of Witwatersrand. He joined the African National Congress (ANC) in 1943 and co-founded its Youth League. After the National Party came to power in 1948, he rose to prominence in the ANC's 1952 defiance campaign. He was arrested and imprisoned in 1962, and subsequently sentenced to life imprisonment for conspiring to overthrow the state. Mandela served 27 years in prison, split between Robben Island, Pollsmoor Prison, and Victor Verster Prison. An international campaign lobbied for his release, which was granted in 1990. He led negotiations with President F.W. de Klerk to abolish apartheid and establish multiracial elections in 1994, which he won.\"\"\",\n", + " metadata={'source': 'wikipedia', 'title': 'Nelson Mandela'}\n", + " ),\n", + " Document(\n", + " text=\"\"\"Apartheid was a system of institutionalised racial segregation that existed in South Africa and South West Africa from 1948 to the early 1990s. Under apartheid, the rights of the majority non-white inhabitants were curtailed and white supremacy and Afrikaner minority rule were maintained. The National Party implemented apartheid as state policy after winning the 1948 general election. This racial segregation was enforced through legislation. People were classified into racial groups and separated by laws that banned social contact, mixed marriages, and established separate public facilities. The system was dismantled through negotiations between 1990 and 1993 and elections in 1994.\"\"\",\n", + " metadata={'source': 'wikipedia', 'title': 'Apartheid'}\n", + " ),\n", + " Document(\n", + " text=\"\"\"The African National Congress (ANC) is the Republic of South Africa's governing political party. It has been the ruling party of post-apartheid South Africa since the election of Nelson Mandela in 1994. Founded in 1912 as the South African Native National Congress, its primary mission was to bring all Africans together as one people to defend their rights and freedoms. The ANC was banned in 1960 after the Sharpeville massacre and operated underground and in exile for 30 years until it was unbanned in 1990. Key figures include Nelson Mandela, Oliver Tambo, Walter Sisulu, and Albert Luthuli.\"\"\",\n", + " metadata={'source': 'wikipedia', 'title': 'African National Congress'}\n", + " ),\n", + "]\n", + "\n", + "print(f'Created {len(wiki_docs)} documents')\n", + "for doc in wiki_docs:\n", + " title = doc.metadata['title']\n", + " print(f' {title}: {len(doc.text)} chars')\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "ad51f14d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Indexing 3 documents into lexical-graph...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:root:Removing unpickleable private attribute _chunking_tokenizer_fn\n", + "WARNING:root:Removing unpickleable private attribute _split_fns\n", + "WARNING:root:Removing unpickleable private attribute _sub_sentence_split_fns\n", + "WARNING:root:Removing unpickleable private attribute _chunking_tokenizer_fn\n", + "WARNING:root:Removing unpickleable private attribute _split_fns\n", + "WARNING:root:Removing unpickleable private attribute _sub_sentence_split_fns\n", + "WARNING:root:Removing unpickleable private attribute _asession\n", + "WARNING:root:Removing unpickleable private attribute _client\n", + "WARNING:root:Removing unpickleable private attribute _asession\n", + "WARNING:root:Removing unpickleable private attribute _client\n", + "WARNING:root:Removing unpickleable private attribute _asession\n", + "WARNING:root:Removing unpickleable private attribute _client\n", + "WARNING:root:Removing unpickleable private attribute _chunking_tokenizer_fn\n", + "WARNING:root:Removing unpickleable private attribute _split_fns\n", + "WARNING:root:Removing unpickleable private attribute _sub_sentence_split_fns\n", + "WARNING:root:Removing unpickleable private attribute _chunking_tokenizer_fn\n", + "WARNING:root:Removing unpickleable private attribute _split_fns\n", + "WARNING:root:Removing unpickleable private attribute _sub_sentence_split_fns\n", + "WARNING:root:Removing unpickleable private attribute _asession\n", + "WARNING:root:Removing unpickleable private attribute _client\n", + "WARNING:root:Removing unpickleable private attribute _asession\n", + "WARNING:root:Removing unpickleable private attribute _client\n", + "WARNING:root:Removing unpickleable private attribute _asession\n", + "WARNING:root:Removing unpickleable private attribute _client\n", + "Extracting propositions [nodes: 1, num_workers: 4]: 100%|██████████| 1/1 [00:04<00:00, 4.48s/it]\n", + "Extracting propositions [nodes: 2, num_workers: 4]: 100%|██████████| 2/2 [00:06<00:00, 3.12s/it]\n", + "Extracting topics [nodes: 1, num_workers: 4]: 100%|██████████| 1/1 [00:09<00:00, 9.80s/it]\n", + "Extracting topics [nodes: 2, num_workers: 4]: 100%|██████████| 2/2 [00:15<00:00, 7.66s/it]\n", + "WARNING:root:Removing unpickleable private attribute _asession\n", + "WARNING:root:Removing unpickleable private attribute _client\n", + "WARNING:root:Removing unpickleable private attribute _asession\n", + "WARNING:root:Removing unpickleable private attribute _client\n", + "WARNING:root:Removing unpickleable private attribute _asession\n", + "WARNING:root:Removing unpickleable private attribute _client\n", + "WARNING:root:Removing unpickleable private attribute _asession\n", + "WARNING:root:Removing unpickleable private attribute _client\n", + "Building graph [batch_writes_enabled: False, batch_write_size: 25]: 100%|██████████| 46/46 [00:15<00:00, 2.90it/s]]\n", + "Building vector index [batch_writes_enabled: False, batch_write_size: 25]: 100%|██████████| 46/46 [00:01<00:00, 31.81it/s]\n", + "Building graph [batch_writes_enabled: False, batch_write_size: 25]: 100%|██████████| 124/124 [00:31<00:00, 3.92it/s]\n", + "Building vector index [batch_writes_enabled: False, batch_write_size: 25]: 100%|██████████| 124/124 [00:07<00:00, 16.51it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Lexical-graph build complete\n" + ] + } + ], + "source": [ + "# Index into lexical-graph\n", + "os.environ['BATCH_WRITES_ENABLED'] = 'false'\n", + "\n", + "print(f'Indexing {len(wiki_docs)} documents into lexical-graph...')\n", + "graph_index = LexicalGraphIndex(graph_store, vector_store)\n", + "graph_index.extract_and_build(wiki_docs, show_progress=True)\n", + "print('Lexical-graph build complete')\n" + ] + }, + { + "cell_type": "markdown", + "id": "2f9b0d20", + "metadata": {}, + "source": [ + "## Step 2: Build document-graph from structured data\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "f7cc2918", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Events: 9 rows\n", + "Organizations/Papers: 5 records\n", + "\n", + "Wrote 9 Event + 5 Organization nodes to Neptune\n" + ] + } + ], + "source": [ + "def flatten_props(props):\n", + " flat = {}\n", + " for k, v in props.items():\n", + " if isinstance(v, list):\n", + " flat[k] = ', '.join(str(x) for x in v)\n", + " elif isinstance(v, dict):\n", + " flat[k] = json.dumps(v)\n", + " elif v is None:\n", + " continue\n", + " else:\n", + " flat[k] = v\n", + " return flat\n", + "\n", + "# Load Mandela structured data\n", + "events = pd.read_csv('mandela_data/mandela_events.csv')\n", + "orgs = json.load(open('mandela_data/research_papers.json'))\n", + "if isinstance(orgs, dict):\n", + " key = list(orgs.keys())[0]\n", + " orgs = orgs[key]\n", + "\n", + "print(f'Events: {len(events)} rows')\n", + "print(f'Organizations/Papers: {len(orgs)} records')\n", + "\n", + "# Write events to Neptune\n", + "for _, row in events.iterrows():\n", + " record = row.to_dict()\n", + " rid = str(record.get('id', record.get('event_id', hash(str(record)))))\n", + " node = Node(id=rid, labels=['Event'], properties=flatten_props(record))\n", + " cypher, params = node_to_cypher(node, tenant_id=TENANT)\n", + " gs.execute_query(cypher, params)\n", + "\n", + "# Write orgs/papers to Neptune\n", + "for record in orgs:\n", + " rid = str(record.get('id', record.get('org_id', hash(str(record)))))\n", + " node = Node(id=rid, labels=['Organization'], properties=flatten_props(record))\n", + " cypher, params = node_to_cypher(node, tenant_id=TENANT)\n", + " gs.execute_query(cypher, params)\n", + "\n", + "print(f'\\nWrote {len(events)} Event + {len(orgs)} Organization nodes to Neptune')\n" + ] + }, + { + "cell_type": "markdown", + "id": "4f193f44", + "metadata": {}, + "source": [ + "## Step 3: Verify\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "035e086f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Mandela tenant nodes:\n", + " Event: 9\n", + " Organization: 5\n", + "Build complete. Ready for 08b (Query).\n" + ] + } + ], + "source": [ + "# Verify mandela tenant nodes\n", + "print(\"Mandela tenant nodes:\")\n", + "for ntype in [\"Event\", \"Organization\"]:\n", + " label = f\"__{ntype}__{TENANT}__\"\n", + " result = gs.execute_query(f\"MATCH (n:`{label}`) RETURN count(n) as cnt\")\n", + " cnt = result[0][\"cnt\"] if result else 0\n", + " print(f\" {ntype}: {cnt}\")\n", + "\n", + "print(\"Build complete. Ready for 08b (Query).\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ac6e7841-88e1-4c02-99ad-374fe8b57b98", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "conda_python3", + "language": "python", + "name": "conda_python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.20" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/document-graph/examples/cloud/notebooks/08b-Nelson-Mandela-Hybrid-Query.ipynb b/document-graph/examples/cloud/notebooks/08b-Nelson-Mandela-Hybrid-Query.ipynb new file mode 100644 index 00000000..edd6b893 --- /dev/null +++ b/document-graph/examples/cloud/notebooks/08b-Nelson-Mandela-Hybrid-Query.ipynb @@ -0,0 +1,253 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "9cb43269", + "metadata": {}, + "source": [ + "# 08b — Nelson Mandela Hybrid GraphRAG: Query\n", + "\n", + "1. Semantic search via lexical-graph (Wikipedia content)\n", + "2. Structured queries via document-graph (events, organizations)\n", + "3. Hybrid correlation — combine both\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "986ae7bc", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Connected\n" + ] + } + ], + "source": [ + "# Setup\n", + "import json, re\n", + "from graphrag_toolkit.lexical_graph.storage import GraphStoreFactory, VectorStoreFactory\n", + "from graphrag_toolkit.lexical_graph import LexicalGraphQueryEngine\n", + "from document_graph.query import DocumentGraphQueryEngine\n", + "\n", + "GRAPH_STORE = 'neptune-db://obs-app-dev-graph.cluster-czupj1ab2tc6.us-east-1.neptune.amazonaws.com:8182'\n", + "VECTOR_STORE = 'aoss://https://1lci0mi6xy7hpex8fr0b.us-east-1.aoss.amazonaws.com'\n", + "TENANT = 'mandela'\n", + "\n", + "graph_store = GraphStoreFactory.for_graph_store(GRAPH_STORE)\n", + "vector_store = VectorStoreFactory.for_vector_store(VECTOR_STORE)\n", + "gs = graph_store.__enter__()\n", + "print('Connected')\n" + ] + }, + { + "cell_type": "markdown", + "id": "00e88b84", + "metadata": {}, + "source": [ + "## 1. Semantic search (lexical-graph)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "0555bbd0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Query: Who was Nelson Mandela?\n", + " Results: 5\n", + " Top: {\n", + " \"source\": \"Nelson Mandela (wikipedia)\",\n", + " \"topic\": \"Nelson Mandela Life, Career, and Legacy\",\n", + " \"statements\": [\n", + " \"[Nelson Mandela joined the African National Congress in 1943.]\",\n", + " \"[Nelson R...\n", + "\n", + "Query: What role did Mandela play in ending apartheid?\n", + " Results: 5\n", + " Top: {\n", + " \"source\": \"African National Congress (wikipedia)\",\n", + " \"topic\": \"African National Congress (ANC) Overview\",\n", + " \"statements\": [\n", + " \"[The African National Congress has been the ruling party of post-ap...\n", + "\n", + "Query: What is the African National Congress?\n", + " Results: 5\n", + " Top: {\n", + " \"source\": \"African National Congress (wikipedia)\",\n", + " \"topic\": \"African National Congress (ANC) Overview\",\n", + " \"statements\": [\n", + " \"[The African National Congress has been the ruling party of post-ap...\n" + ] + } + ], + "source": [ + "query_engine = LexicalGraphQueryEngine.for_traversal_based_search(graph_store, vector_store)\n", + "\n", + "queries = [\n", + " 'Who was Nelson Mandela?',\n", + " 'What role did Mandela play in ending apartheid?',\n", + " 'What is the African National Congress?',\n", + "]\n", + "\n", + "for q in queries:\n", + " print(f'\\nQuery: {q}')\n", + " results = query_engine.retrieve(q)\n", + " if isinstance(results, list) and results:\n", + " print(f' Results: {len(results)}')\n", + " text = getattr(results[0].node, 'text', str(results[0]))[:200]\n", + " print(f' Top: {text}...')\n", + " else:\n", + " print(f' {str(results)[:150]}')\n" + ] + }, + { + "cell_type": "markdown", + "id": "67513860", + "metadata": {}, + "source": [ + "## 2. Structured queries (document-graph)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "d778ec4c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Events in document-graph: 9\n", + " [2013-12-05] Passed away at age 95.\n", + " [1918-07-18] Nelson Rolihlahla Mandela was born.\n", + " [1944-01-01] Joined African National Congress (ANC) and co-founded the ANC Youth League.\n", + " [1952-06-26] Led Defiance Campaign against unjust apartheid laws.\n", + " [1962-08-05] Arrested and later sentenced to five years for leaving the country illegally and inciting strike.\n", + "Organizations: 5\n", + " The Evolution of Apartheid Policies, 1948–1994\n", + " Resistance Networks and the ANC\n", + " International Sanctions and South Africa\n", + " Truth and Reconciliation: Outcomes and Limits\n", + " Media Representation of Mandela\n" + ] + } + ], + "source": [ + "# Query events\n", + "event_label = f\"__Event__{TENANT}__\"\n", + "events = gs.execute_query(f\"MATCH (n:`{event_label}`) RETURN properties(n) as props LIMIT 10\")\n", + "print(f\"Events in document-graph: {len(events)}\")\n", + "for e in events[:5]:\n", + " props = e.get(\"props\", {})\n", + " date = props.get(\"date\", \"\")\n", + " desc = props.get(\"description\", \"?\")\n", + " print(f\" [{date}] {desc}\")\n", + "\n", + "# Query organizations\n", + "org_label = f\"__Organization__{TENANT}__\"\n", + "orgs = gs.execute_query(f\"MATCH (n:`{org_label}`) RETURN properties(n) as props LIMIT 10\")\n", + "print(f\"Organizations: {len(orgs)}\")\n", + "for o in orgs[:5]:\n", + " props = o.get(\"props\", {})\n", + " name = props.get(\"name\", props.get(\"title\", \"?\"))\n", + " print(f\" {name}\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "3a966ef4", + "metadata": {}, + "source": [ + "## 3. Hybrid correlation\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cd596247", + "metadata": {}, + "outputs": [], + "source": [ + "# Semantic query + correlate entities to document-graph\n", + "q = 'What key events shaped Mandela\\'s life?'\n", + "results = query_engine.retrieve(q)\n", + "print(f'Query: {q}')\n", + "print(f'Lexical results: {len(results)}\\n')\n", + "\n", + "# Extract entities and look them up in document-graph\n", + "for i, r in enumerate(results[:3], 1):\n", + " meta = getattr(r.node, 'metadata', {})\n", + " ec = meta.get('entity_contexts', {}).get('contexts', [])\n", + " text = getattr(r.node, 'text', str(r))[:150]\n", + " print(f'Result {i}: {text}...')\n", + "\n", + " for ctx in ec[:5]:\n", + " for ent in ctx.get('entities', []):\n", + " entity = ent.get('entity', {})\n", + " value = entity.get('value', '')\n", + " classification = entity.get('classification', '')\n", + " if not value:\n", + " continue\n", + " search_term = value.split(',')[0].strip()\n", + " found = None\n", + " if classification in ('Event', 'Organization', 'Person'):\n", + " for label in [f'__Event__{TENANT}__', f'__Organization__{TENANT}__']:\n", + " found = gs.execute_query(f\"MATCH (n:`{label}`) WHERE n.name CONTAINS '{search_term}' OR n.title CONTAINS '{search_term}' RETURN properties(n) as props LIMIT 1\")\n", + " if found:\n", + " break\n", + " if found:\n", + " props = found[0].get('props', {})\n", + " name = props.get('name', props.get('title', value))\n", + " print(f' [{classification}] \"{value}\" -> Neptune: {name}')\n", + " print()\n", + "\n", + "print('Hybrid query complete — lexical search + document-graph correlation')\n" + ] + }, + { + "cell_type": "markdown", + "id": "d1219cb5", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "| Layer | Source | Contains |\n", + "|-------|--------|----------|\n", + "| Lexical-graph | Wikipedia | Semantic chunks, topics, entities from Mandela bio |\n", + "| Document-graph | Structured CSV/JSON | Events, organizations as typed nodes |\n", + "| Hybrid query | Both | Semantic search finds context, entity correlation finds structured data |\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "conda_python3", + "language": "python", + "name": "conda_python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.20" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/document-graph/examples/cloud/notebooks/09-Multi-Tenant-Coexistence-Demo.ipynb b/document-graph/examples/cloud/notebooks/09-Multi-Tenant-Coexistence-Demo.ipynb new file mode 100644 index 00000000..ed2e3fe4 --- /dev/null +++ b/document-graph/examples/cloud/notebooks/09-Multi-Tenant-Coexistence-Demo.ipynb @@ -0,0 +1,193 @@ +{ + "nbformat": 4, + "nbformat_minor": 5, + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 09 \u2014 Multi-Tenant Coexistence Demo\n", + "\n", + "Demonstrates that multiple tenants coexist in the same Neptune graph with complete isolation.\n", + "Each tenant's nodes have labels like `__Type__tenant_id__` \u2014 no data leakage between tenants.\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Setup\n", + "import json\n", + "from graphrag_toolkit.lexical_graph.storage import GraphStoreFactory\n", + "from document_graph.graph_build import node_to_cypher\n", + "from document_graph.query import DocumentGraphQueryEngine\n", + "from document_graph import Node\n", + "\n", + "GRAPH_STORE = 'neptune-db://obs-app-dev-graph.cluster-czupj1ab2tc6.us-east-1.neptune.amazonaws.com:8182'\n", + "\n", + "gs = GraphStoreFactory.for_graph_store(GRAPH_STORE).__enter__()\n", + "print('Neptune connected')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Write data for Tenant A\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "TENANT_A = 'acme_corp'\n", + "\n", + "tenant_a_data = [\n", + " Node(id='u1', labels=['User'], properties={'name': 'Alice', 'role': 'admin', 'department': 'Engineering'}),\n", + " Node(id='u2', labels=['User'], properties={'name': 'Bob', 'role': 'developer', 'department': 'Engineering'}),\n", + " Node(id='p1', labels=['Project'], properties={'name': 'Apollo', 'status': 'active'}),\n", + "]\n", + "\n", + "for node in tenant_a_data:\n", + " cypher, params = node_to_cypher(node, tenant_id=TENANT_A)\n", + " gs.execute_query(cypher, params)\n", + "\n", + "print(f'Tenant A ({TENANT_A}): wrote {len(tenant_a_data)} nodes')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Write data for Tenant B\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "TENANT_B = 'globex_inc'\n", + "\n", + "tenant_b_data = [\n", + " Node(id='u1', labels=['User'], properties={'name': 'Charlie', 'role': 'manager', 'department': 'Sales'}),\n", + " Node(id='u2', labels=['User'], properties={'name': 'Diana', 'role': 'analyst', 'department': 'Finance'}),\n", + " Node(id='p1', labels=['Project'], properties={'name': 'Zeus', 'status': 'planning'}),\n", + "]\n", + "\n", + "for node in tenant_b_data:\n", + " cypher, params = node_to_cypher(node, tenant_id=TENANT_B)\n", + " gs.execute_query(cypher, params)\n", + "\n", + "print(f'Tenant B ({TENANT_B}): wrote {len(tenant_b_data)} nodes')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Query \u2014 tenant isolation\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Query Tenant A only\n", + "engine_a = DocumentGraphQueryEngine(gs, tenant_id=TENANT_A)\n", + "users_a = engine_a.get_nodes('User')\n", + "print(f'Tenant A Users: {len(users_a)}')\n", + "for u in users_a:\n", + " props = u['n']['~properties']\n", + " print(f' {props.get(\"name\")} ({props.get(\"role\")})')\n", + "\n", + "# Query Tenant B only\n", + "engine_b = DocumentGraphQueryEngine(gs, tenant_id=TENANT_B)\n", + "users_b = engine_b.get_nodes('User')\n", + "print(f'\\nTenant B Users: {len(users_b)}')\n", + "for u in users_b:\n", + " props = u['n']['~properties']\n", + " print(f' {props.get(\"name\")} ({props.get(\"role\")})')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Prove isolation \u2014 same ID, different tenants\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Both tenants have node id='u1' but they're different nodes\n", + "a_u1 = engine_a.find_by_property('User', 'id', 'u1')\n", + "b_u1 = engine_b.find_by_property('User', 'id', 'u1')\n", + "\n", + "a_name = a_u1[0]['n']['~properties']['name'] if a_u1 else 'NOT FOUND'\n", + "b_name = b_u1[0]['n']['~properties']['name'] if b_u1 else 'NOT FOUND'\n", + "\n", + "print(f'Node u1 in Tenant A: {a_name}')\n", + "print(f'Node u1 in Tenant B: {b_name}')\n", + "print(f'\\nSame ID, different data = tenant isolation working')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Cross-tenant view (admin only)\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Admin can see all tenants by querying without tenant filter\n", + "all_users = gs.execute_query('MATCH (n) WHERE labels(n)[0] CONTAINS \"User\" RETURN labels(n) as lbl, properties(n) as props LIMIT 10')\n", + "print(f'All User nodes across tenants: {len(all_users)}')\n", + "for u in all_users:\n", + " label = u['lbl'][0]\n", + " name = u['props'].get('name', '?')\n", + " # Extract tenant from label: __User__tenant__\n", + " parts = label.split('__')\n", + " tenant = parts[2] if len(parts) >= 3 else '?'\n", + " print(f' {name} (tenant: {tenant})')\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "| Aspect | How it works |\n", + "|--------|-------------|\n", + "| **Isolation** | Labels encode tenant: `__User__acme_corp__` vs `__User__globex_inc__` |\n", + "| **Same IDs** | Node id='u1' exists in both tenants independently |\n", + "| **Querying** | `DocumentGraphQueryEngine(gs, tenant_id=X)` scopes all queries |\n", + "| **Admin view** | Query without tenant filter sees all data |\n", + "| **No leakage** | Tenant A cannot see Tenant B's data through normal queries |\n" + ] + } + ] +} \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/data/README_FORMATS.md b/document-graph/examples/cloud/notebooks/data/README_FORMATS.md new file mode 100644 index 00000000..9e1bf03b --- /dev/null +++ b/document-graph/examples/cloud/notebooks/data/README_FORMATS.md @@ -0,0 +1,80 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +# Test Document Formats + +This directory contains the same test data in multiple formats for comprehensive testing: + +## Available Formats + +1. **CSV** (`test_documents.csv`) - Original comma-separated values format +2. **JSON** (`test_documents.json`) - JavaScript Object Notation with proper data types +3. **XML** (`test_documents.xml`) - Extensible Markup Language with structured elements +4. **YAML** (`test_documents.yaml`) - YAML Ain't Markup Language, human-readable format + +## Additional Formats (Generate as needed) + +### Parquet Format +To create `test_documents.parquet`: +```python +import pandas as pd +df = pd.read_csv('test_documents.csv') +df.to_parquet('test_documents.parquet', index=False) +``` + +### Excel Format +To create `test_documents.xlsx`: +```python +import pandas as pd +df = pd.read_csv('test_documents.csv') +df.to_excel('test_documents.xlsx', index=False, sheet_name='test_documents') +``` + +## Data Content + +All formats contain 20 test documents with: + +- **Standard Content**: Regular documents for basic processing +- **PII Data**: Document #13 with SSN, email, phone, IP, credit card +- **Spelling Errors**: Document #14 with intentional misspellings +- **Multi-language**: Document #16 with 8 different languages +- **Toxic Content**: Document #17 with inappropriate language +- **Mixed Sentiment**: Document #18 with positive and negative emotions +- **Security Threats**: Document #19 with XSS, SQL, command injection +- **Performance Test**: Document #20 with long text content + +## Usage in Notebooks + +Each format can be used to test different ingestion methods: + +```python +# CSV +import pandas as pd +df = pd.read_csv('test_documents.csv') + +# JSON +import json +with open('test_documents.json', 'r') as f: + data = json.load(f) + +# XML +import xml.etree.ElementTree as ET +tree = ET.parse('test_documents.xml') + +# YAML +import yaml +with open('test_documents.yaml', 'r') as f: + data = yaml.safe_load(f) + +# Parquet (if created) +df = pd.read_parquet('test_documents.parquet') + +# Excel (if created) +df = pd.read_excel('test_documents.xlsx') +``` + +## Requirements + +For Parquet and Excel formats, install required packages: +```bash +pip install pandas pyarrow openpyxl +``` \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/data/accounts.csv b/document-graph/examples/cloud/notebooks/data/accounts.csv new file mode 100644 index 00000000..52c53152 --- /dev/null +++ b/document-graph/examples/cloud/notebooks/data/accounts.csv @@ -0,0 +1,4 @@ +id,name,type,region +a1,Production,PROD,us-east-1 +a2,Development,DEV,us-west-2 +a3,Staging,STAGE,us-east-2 diff --git a/document-graph/examples/cloud/notebooks/data/create_additional_formats.py b/document-graph/examples/cloud/notebooks/data/create_additional_formats.py new file mode 100644 index 00000000..691da4b5 --- /dev/null +++ b/document-graph/examples/cloud/notebooks/data/create_additional_formats.py @@ -0,0 +1,55 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +#!/usr/bin/env python3 +""" +Script to create Parquet and Excel versions of test_documents.csv +Run this script to generate test_documents.parquet and test_documents.xlsx +""" + +import pandas as pd +import json + +# Read the CSV file with proper quoting and escaping +df = pd.read_csv('test_documents.csv', quotechar='"', escapechar='\\', skipinitialspace=True) + +# Convert metadata column from string to proper JSON objects for better handling +df['metadata'] = df['metadata'].apply(lambda x: json.loads(x.replace("'", '"'))) + +# Create Parquet file +print("Creating test_documents.parquet...") +df.to_parquet('test_documents.parquet', index=False, engine='pyarrow') +print("✓ Parquet file created successfully") + +# Create Excel file +print("Creating test_documents.xlsx...") +with pd.ExcelWriter('test_documents.xlsx', engine='openpyxl') as writer: + # Convert metadata back to string for Excel compatibility + df_excel = df.copy() + df_excel['metadata'] = df_excel['metadata'].apply(lambda x: json.dumps(x)) + + # Write to Excel with formatting + df_excel.to_excel(writer, sheet_name='test_documents', index=False) + + # Get the workbook and worksheet + workbook = writer.book + worksheet = writer.sheets['test_documents'] + + # Auto-adjust column widths + for column in worksheet.columns: + max_length = 0 + column_letter = column[0].column_letter + for cell in column: + try: + if len(str(cell.value)) > max_length: + max_length = len(str(cell.value)) + except: + pass + adjusted_width = min(max_length + 2, 50) # Cap at 50 characters + worksheet.column_dimensions[column_letter].width = adjusted_width + +print("✓ Excel file created successfully") + +print("\nFiles created:") +print("- test_documents.parquet (Apache Parquet format)") +print("- test_documents.xlsx (Microsoft Excel format)") +print("\nBoth files contain the same 20 test documents with all transformers test scenarios.") \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/data/mapping.json b/document-graph/examples/cloud/notebooks/data/mapping.json new file mode 100644 index 00000000..30c52153 --- /dev/null +++ b/document-graph/examples/cloud/notebooks/data/mapping.json @@ -0,0 +1,387 @@ +{ + "schema_version": "1.0", + "transformations": [ + { + "rule_name": "evidence_1", + "type": "json", + "csv_property_name": "Evidence", + "parent": [ + "Evidence" + ] + }, + { + "rule_name": "resource_1", + "type": "json", + "csv_property_name": "Resource original JSON", + "parent": [ + "Resource" + ] + } + ], + "graph": { + "edges": [ + { + "Resource": { + "RAISES": "Issue", + "BELONGS_TO": "Project", + "LOCATED_IN": "CSP", + "INCLUDED_IN": "Subscription" + } + }, + { + "Issue": { + "POSSESSES": "Threat", + "SUPPORTED_BY": "Evidence", + "CONTRIBUTES_TO": "Risk", + "HAS_A": "Control" + } + } + ], + "nodes": [ + { + "Threat": { + "type": "node", + "property": { + "name": "threat_type", + "csv_map": "Threats" + } + } + }, + { + "Risk": { + "type": "node", + "property": { + "name": "risk_type", + "csv_map": "Risks" + } + } + }, + { + "Resource": { + "type": "node", + "property": { + "name": "resource_id", + "csv_map": "Resource vertex ID" + } + } + }, + { + "Issue": { + "type": "node", + "property": { + "name": "issue_id", + "csv_map": "Issue ID" + } + } + }, + { + "Evidence": { + "type": "node", + "property": { + "name": "evidence_id", + "type": "uuid" + } + } + }, + { + "Project": { + "type": "node", + "property": { + "name": "project_id", + "csv_map": "Project IDs" + } + } + }, + { + "Subscription": { + "type": "node", + "property": { + "name": "subscription_id", + "csv_map": "Subscription ID" + } + } + }, + { + "CSP": { + "type": "node", + "property": { + "name": "platform", + "csv_map": "Resource Platform" + } + } + }, + { + "Control": { + "type": "node", + "property": { + "name": "control_id", + "csv_map": "Control ID" + } + } + } + + ] + }, + "csv_columns": [ + { + "csv_property_name": "Created At", + "type": "property", + "arrow_property_name": "detected_at", + "parents": [ + "Issue" + ] + }, + { + "csv_property_name": "Title", + "type": "property", + "arrow_property_name": "title", + "parents": [ + "Issue" + ] + }, + { + "csv_property_name": "Severity", + "type": "property", + "arrow_property_name": "severity", + "parents": [ + "Issue" + ] + }, + { + "csv_property_name": "Status", + "type": "property", + "arrow_property_name": "status", + "parents": [ + "Issue" + ] + }, + { + "csv_property_name": "Description", + "type": "property", + "arrow_property_name": "description", + "parents": [ + "Issue" + ] + }, + { + "csv_property_name": "Resource Type", + "type": "property", + "arrow_property_name": "resource_type", + "parents": [ + "Resource" + ] + }, + { + "csv_property_name": "Resource external ID", + "type": "property", + "arrow_property_name": "arn_or_uri", + "parents": [ + "Resource" + ] + }, + { + "csv_property_name": "Project Names", + "type": "property", + "arrow_property_name": "name", + "parents": [ + "Project" + ] + }, + { + "csv_property_name": "Resolved Time", + "type": "property", + "arrow_property_name": "resolved_at", + "parents": [ + "Issue" + ] + }, + { + "csv_property_name": "Resolution", + "type": "property", + "arrow_property_name": "resolution", + "parents": [ + "Issue" + ] + }, + { + "csv_property_name": "Control ID", + "type": "property", + "arrow_property_name": "control_id", + "parents": [ + "Control" + ] + }, + { + "csv_property_name": "Resource Name", + "type": "property", + "arrow_property_name": "name", + "parents": [ + "Resource" + ] + }, + { + "csv_property_name": "Resource Region", + "type": "property", + "arrow_property_name": "region", + "parents": [ + "Resource" + ] + }, + { + "csv_property_name": "Resource Status", + "type": "property", + "arrow_property_name": "status", + "parents": [ + "Resource" + ] + }, + { + "csv_property_name": "Resource Platform", + "type": "property", + "arrow_property_name": "platform", + "parents": [ + "Resource" + ] + }, + { + "csv_property_name": "Resource OS", + "type": "property", + "arrow_property_name": "os", + "parents": [ + "Resource" + ] + }, + { + "csv_property_name": "Resource original JSON", + "type": "transformation", + "rules": [ + "resource_1" + ] + }, + { + "csv_property_name": "Ticket URLs", + "type": "property", + "arrow_property_name": "ticket_url", + "parents": [ + "Issue" + ] + }, + { + "csv_property_name": "Note", + "type": "property", + "arrow_property_name": "note", + "parents": [ + "Issue" + ] + }, + { + "csv_property_name": "Due At", + "type": "property", + "arrow_property_name": "due_at", + "parents": [ + "Issue" + ] + }, + { + "csv_property_name": "Remediation Recommendation", + "type": "property", + "arrow_property_name": "remediation", + "parents": [ + "Issue" + ] + }, + { + "csv_property_name": "Subscription Name", + "type": "property", + "arrow_property_name": "name", + "parents": [ + "Subscription" + ] + }, + { + "csv_property_name": "Wiz URL", + "type": "property", + "arrow_property_name": "wiz_url", + "parents": [ + "Issue" + ] + }, + { + "csv_property_name": "Cloud Provider URL", + "type": "property", + "arrow_property_name": "url", + "parents": [ + "CSP" + ] + }, + { + "csv_property_name": "Resource Tags", + "type": "property", + "arrow_property_name": "tags", + "parents": [ + "Resource" + ] + }, + { + "csv_property_name": "Kubernetes Cluster", + "type": "property", + "arrow_property_name": "kubernetes_cluster", + "parents": [ + "Resource" + ] + }, + { + "csv_property_name": "Kubernetes Namespace", + "type": "property", + "arrow_property_name": "kubernetes_namespace", + "parents": [ + "Resource" + ] + }, + { + "csv_property_name": "Container Service", + "type": "property", + "arrow_property_name": "container_service", + "parents": [ + "Resource" + ] + }, + { + "csv_property_name": "Provider ID", + "type": "node", + "arrow_property_name": "provider_id", + "name": "Resource" + }, + { + "csv_property_name": "Threats", + "type": "property", + "arrow_property_name": "threat_type", + "parents": [ + "Threat" + ] + }, + { + "csv_property_name": "Status Changed At", + "type": "property", + "arrow_property_name": "status_changed_at", + "parents": [ + "Issue" + ] + }, + { + "csv_property_name": "Updated At", + "type": "property", + "arrow_property_name": "updated_at", + "parents": [ + "Issue" + ] + }, + { + "csv_property_name": "Evidence", + "type": "transformation", + "rules": [ + "evidence_1" + ] + } + ] +} \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/data/sample.json b/document-graph/examples/cloud/notebooks/data/sample.json new file mode 100644 index 00000000..3602a698 --- /dev/null +++ b/document-graph/examples/cloud/notebooks/data/sample.json @@ -0,0 +1,106 @@ +{ + "schema_id": "research_paper_etl_v1", + "extract": { + "source_type": "document_graph", + "source_config": { + "format": "json", + "encoding": "utf-8" + }, + "fields": [ + { + "name": "paper_id", + "type": "string", + "required": true, + "source_path": "nodes[?type=='Document'].id" + }, + { + "name": "title", + "type": "string", + "required": true, + "source_path": "nodes[?type=='Document'].properties.title" + } + ] + }, + "transform": { + "chunking": { + "enabled": true, + "chunk_size": 1000, + "overlap": 200, + "strategy": "sentence" + }, + "metadata_mapping": { + "document_id": "paper_id", + "title": "title", + "source": "extract" + }, + "entity_extraction": { + "enabled": true, + "types": [ + "PERSON", + "ORGANIZATION", + "CONCEPT" + ], + "model": "spacy_en_core_web_sm" + }, + "normalize": { + "text_cleaning": true, + "lowercase": false, + "remove_punctuation": false, + "remove_stopwords": false + } + }, + "load": { + "document_node": { + "type": "Document", + "id_field": "paper_id", + "properties": [ + "title", + "authors", + "year" + ], + "labels": [ + "Document", + "ResearchPaper" + ] + }, + "section_node": { + "type": "Section", + "id_field": "section_id", + "properties": [ + "title", + "content", + "order" + ], + "labels": [ + "Section", + "DocumentSection" + ] + }, + "relationships": { + "document_to_section": { + "type": "HAS_SECTION", + "from": "document_node", + "to": "section_node", + "properties": [ + "order", + "section_type" + ] + }, + "section_to_entity": { + "type": "MENTIONS", + "from": "section_node", + "to": "entity_node", + "properties": [ + "confidence", + "position" + ] + } + } + }, + "metadata": { + "version": "1.0", + "created_at": "2024-01-15T10:30:00Z", + "description": "ETL schema for research paper document graph", + "domain": "academic_research" + } +} \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/data/test_documents.csv b/document-graph/examples/cloud/notebooks/data/test_documents.csv new file mode 100644 index 00000000..8ea4cdf7 --- /dev/null +++ b/document-graph/examples/cloud/notebooks/data/test_documents.csv @@ -0,0 +1,21 @@ +id,title,content,category,author,date,tags,language,source_url,metadata +1,"AWS Lambda Introduction","AWS Lambda is a serverless computing service that lets you run code without provisioning or managing servers. You pay only for the compute time you consume. Lambda runs your code on a high-availability compute infrastructure and performs all of the administration of the compute resources.","technology","AWS Documentation","2024-01-15","serverless,lambda,aws,computing","en","https://docs.aws.amazon.com/lambda/","{'service': 'lambda', 'complexity': 'beginner'}" +2,"Machine Learning Basics","Machine learning is a subset of artificial intelligence that enables computers to learn and make decisions from data without being explicitly programmed. It involves algorithms that can identify patterns in data and make predictions or classifications.","education","Dr. Sarah Johnson","2024-01-20","machine-learning,ai,data-science,algorithms","en","https://example.com/ml-basics","{'difficulty': 'intermediate', 'field': 'computer_science'}" +3,"Cybersecurity Best Practices","Implementing strong cybersecurity measures is crucial for protecting sensitive data. Key practices include using multi-factor authentication, regular software updates, employee training, network segmentation, and incident response planning.","security","John Smith","2024-01-25","cybersecurity,security,data-protection,best-practices","en","https://security-blog.com/best-practices","{'industry': 'cybersecurity', 'audience': 'enterprise'}" +4,"Climate Change Impact","Climate change refers to long-term shifts in global temperatures and weather patterns. While climate variations are natural, scientific evidence shows that human activities have been the main driver of climate change since the 1800s.","environment","Environmental Research Institute","2024-02-01","climate-change,environment,sustainability,global-warming","en","https://climate-research.org/impact","{'topic': 'environmental_science', 'urgency': 'high'}" +5,"Quantum Computing Fundamentals","Quantum computing harnesses quantum mechanical phenomena to process information in fundamentally different ways than classical computers. Quantum bits (qubits) can exist in superposition states, enabling parallel processing capabilities.","technology","Prof. Michael Chen","2024-02-05","quantum-computing,physics,technology,qubits","en","https://quantum-journal.com/fundamentals","{'complexity': 'advanced', 'field': 'quantum_physics'}" +6,"Healthy Cooking Tips","Maintaining a healthy diet starts with proper cooking techniques. Use fresh ingredients, limit processed foods, incorporate plenty of vegetables, choose lean proteins, and use healthy cooking methods like steaming, grilling, or baking instead of frying.","health","Nutritionist Lisa Brown","2024-02-10","health,nutrition,cooking,wellness,diet","en","https://health-magazine.com/cooking-tips","{'category': 'lifestyle', 'target_audience': 'general_public'}" +7,"Financial Planning Strategies","Effective financial planning involves setting clear goals, creating a budget, building an emergency fund, investing wisely, and regularly reviewing your financial situation. Start early and be consistent with your savings and investment approach.","finance","Financial Advisor Mark Wilson","2024-02-15","finance,investment,budgeting,savings,planning","en","https://finance-advisor.com/strategies","{'expertise_level': 'beginner_to_intermediate', 'topic': 'personal_finance'}" +8,"Space Exploration History","Space exploration began in the mid-20th century with the launch of Sputnik 1 in 1957. Major milestones include the first human in space (Yuri Gagarin, 1961), the moon landing (Apollo 11, 1969), and the development of space stations and Mars rovers.","science","Space History Museum","2024-02-20","space,exploration,history,nasa,astronomy","en","https://space-museum.org/history","{'era': '20th_21st_century', 'field': 'aerospace'}" +9,"Sustainable Energy Solutions","Renewable energy sources like solar, wind, and hydroelectric power are becoming increasingly important for reducing carbon emissions. These technologies offer clean alternatives to fossil fuels and are becoming more cost-effective.","environment","Green Energy Consortium","2024-02-25","renewable-energy,sustainability,solar,wind,environment","en","https://green-energy.org/solutions","{'focus': 'renewable_energy', 'impact': 'environmental'}" +10,"Digital Marketing Trends","Digital marketing continues to evolve with new technologies and consumer behaviors. Key trends include personalization, video content, social media marketing, influencer partnerships, and data-driven decision making.","business","Marketing Expert Jane Davis","2024-03-01","digital-marketing,social-media,trends,business,advertising","en","https://marketing-trends.com/2024","{'industry': 'marketing', 'year': '2024'}" +11,"Artificial Intelligence Ethics","As AI systems become more prevalent, ethical considerations become crucial. Key issues include bias in algorithms, privacy concerns, job displacement, transparency in decision-making, and the need for responsible AI development.","technology","AI Ethics Committee","2024-03-05","ai,ethics,technology,society,responsibility","en","https://ai-ethics.org/guidelines","{'importance': 'critical', 'stakeholders': 'global'}" +12,"Urban Planning Principles","Effective urban planning creates sustainable, livable communities. Key principles include mixed-use development, public transportation, green spaces, affordable housing, and community engagement in the planning process.","urban-development","City Planning Department","2024-03-10","urban-planning,city-development,sustainability,community","en","https://city-planning.gov/principles","{'scope': 'municipal', 'focus': 'sustainable_development'}" +13,"Data Breach Incident Report","On March 15th, our security team detected unauthorized access to customer data. The breach affected 1,247 users including John Doe (SSN: 123-45-6789, email: john.doe@email.com), Jane Smith (phone: 555-123-4567), and server logs from IP address 192.168.1.100. Credit card 4532-1234-5678-9012 was also compromised. Immediate containment measures were implemented.","security","Security Team Lead","2024-03-15","data-breach,security,incident,pii","en","https://internal-security.com/incident-001","{'severity': 'high', 'affected_users': 1247, 'containment_status': 'complete'}" +14,"Machien Lerning Algoritms","Machien lerning algoritms are becomming more sofisticated every day. Thier ability to proces large amounts of data and identyfy paterns is truely remarkabel. However, ther are stil chalenges with bias and interpretabilty that need to be adressed.","technology","Dr. Alex Thompson","2024-03-20","machine-learning,algorithms,ai,spelling-errors","en","https://ml-research.edu/algorithms","{'quality': 'poor_spelling', 'needs_correction': true}" +15,"API Configuration Guide","To configure the API endpoint, use the following JSON structure: {\"endpoint\": \"https://api.example.com/v1\", \"auth\": {\"type\": \"bearer\", \"token\": \"sk-1234567890abcdef\"}, \"timeout\": 30000, \"retries\": 3, \"headers\": {\"Content-Type\": \"application/json\", \"User-Agent\": \"MyApp/1.0\"}}. This configuration ensures proper authentication and error handling.","technical","DevOps Engineer","2024-03-25","api,configuration,json,development","en","https://dev-docs.com/api-config","{'format': 'json_embedded', 'complexity': 'intermediate'}" +16,"Mixed Language Content","This document contains multiple languages. English: Hello world! Spanish: ¡Hola mundo! French: Bonjour le monde! German: Hallo Welt! Japanese: こんにちは世界！Chinese: 你好世界！Russian: Привет мир! Arabic: مرحبا بالعالم! This tests language detection capabilities.","linguistics","Language Research Team","2024-04-01","multilingual,language-detection,international","mixed","https://lang-research.org/mixed-content","{'languages': ['en', 'es', 'fr', 'de', 'ja', 'zh', 'ru', 'ar'], 'test_type': 'language_detection'}" +17,"Toxic Content Example","This content contains inappropriate language and offensive terms that should be filtered out. Words like damn, hell, and other profanity need to be detected and handled appropriately by content moderation systems. This tests toxicity detection.","content-moderation","Content Safety Team","2024-04-05","toxicity,content-moderation,filtering","en","https://safety.example.com/toxic-test","{'toxicity_level': 'moderate', 'requires_filtering': true}" +18,"Sentiment Analysis Test","I absolutely LOVE this amazing product! It's fantastic, wonderful, and brings me so much joy! 😊❤️ However, I also hate waiting for delivery and the customer service was terrible and disappointing. Mixed emotions here - both positive and negative feelings combined.","product-review","Customer Reviewer","2024-04-10","sentiment,emotions,mixed-feelings,product-review","en","https://reviews.com/mixed-sentiment","{'sentiment': 'mixed', 'positive_score': 0.7, 'negative_score': 0.4}" +19,"Code Injection Attempt","This field contains potential code: and SQL injection: '; DROP TABLE users; -- and command injection: $(rm -rf /) and other malicious content that should be sanitized and detected by security filters.","security-test","Security Tester","2024-04-15","security,injection,xss,sql-injection,sanitization","en","https://security-test.com/injection-test","{'threat_level': 'high', 'injection_types': ['xss', 'sql', 'command']}" +20,"Long Text Performance Test","This is an extremely long document designed to test performance with large text processing. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt.","performance","Performance Tester","2024-04-20","performance,long-text,processing,load-test","en","https://perf-test.com/long-content","{'text_length': 1200, 'test_type': 'performance', 'processing_time_target': '<5s'}" \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/data/test_documents.json b/document-graph/examples/cloud/notebooks/data/test_documents.json new file mode 100644 index 00000000..73a66713 --- /dev/null +++ b/document-graph/examples/cloud/notebooks/data/test_documents.json @@ -0,0 +1,242 @@ +[ + { + "id": 1, + "title": "AWS Lambda Introduction", + "content": "AWS Lambda is a serverless computing service that lets you run code without provisioning or managing servers. You pay only for the compute time you consume. Lambda runs your code on a high-availability compute infrastructure and performs all of the administration of the compute resources.", + "category": "technology", + "author": "AWS Documentation", + "date": "2024-01-15", + "tags": "serverless,lambda,aws,computing", + "language": "en", + "source_url": "https://docs.aws.amazon.com/lambda/", + "metadata": {"service": "lambda", "complexity": "beginner"} + }, + { + "id": 2, + "title": "Machine Learning Basics", + "content": "Machine learning is a subset of artificial intelligence that enables computers to learn and make decisions from data without being explicitly programmed. It involves algorithms that can identify patterns in data and make predictions or classifications.", + "category": "education", + "author": "Dr. Sarah Johnson", + "date": "2024-01-20", + "tags": "machine-learning,ai,data-science,algorithms", + "language": "en", + "source_url": "https://example.com/ml-basics", + "metadata": {"difficulty": "intermediate", "field": "computer_science"} + }, + { + "id": 3, + "title": "Cybersecurity Best Practices", + "content": "Implementing strong cybersecurity measures is crucial for protecting sensitive data. Key practices include using multi-factor authentication, regular software updates, employee training, network segmentation, and incident response planning.", + "category": "security", + "author": "John Smith", + "date": "2024-01-25", + "tags": "cybersecurity,security,data-protection,best-practices", + "language": "en", + "source_url": "https://security-blog.com/best-practices", + "metadata": {"industry": "cybersecurity", "audience": "enterprise"} + }, + { + "id": 4, + "title": "Climate Change Impact", + "content": "Climate change refers to long-term shifts in global temperatures and weather patterns. While climate variations are natural, scientific evidence shows that human activities have been the main driver of climate change since the 1800s.", + "category": "environment", + "author": "Environmental Research Institute", + "date": "2024-02-01", + "tags": "climate-change,environment,sustainability,global-warming", + "language": "en", + "source_url": "https://climate-research.org/impact", + "metadata": {"topic": "environmental_science", "urgency": "high"} + }, + { + "id": 5, + "title": "Quantum Computing Fundamentals", + "content": "Quantum computing harnesses quantum mechanical phenomena to process information in fundamentally different ways than classical computers. Quantum bits (qubits) can exist in superposition states, enabling parallel processing capabilities.", + "category": "technology", + "author": "Prof. Michael Chen", + "date": "2024-02-05", + "tags": "quantum-computing,physics,technology,qubits", + "language": "en", + "source_url": "https://quantum-journal.com/fundamentals", + "metadata": {"complexity": "advanced", "field": "quantum_physics"} + }, + { + "id": 6, + "title": "Healthy Cooking Tips", + "content": "Maintaining a healthy diet starts with proper cooking techniques. Use fresh ingredients, limit processed foods, incorporate plenty of vegetables, choose lean proteins, and use healthy cooking methods like steaming, grilling, or baking instead of frying.", + "category": "health", + "author": "Nutritionist Lisa Brown", + "date": "2024-02-10", + "tags": "health,nutrition,cooking,wellness,diet", + "language": "en", + "source_url": "https://health-magazine.com/cooking-tips", + "metadata": {"category": "lifestyle", "target_audience": "general_public"} + }, + { + "id": 7, + "title": "Financial Planning Strategies", + "content": "Effective financial planning involves setting clear goals, creating a budget, building an emergency fund, investing wisely, and regularly reviewing your financial situation. Start early and be consistent with your savings and investment approach.", + "category": "finance", + "author": "Financial Advisor Mark Wilson", + "date": "2024-02-15", + "tags": "finance,investment,budgeting,savings,planning", + "language": "en", + "source_url": "https://finance-advisor.com/strategies", + "metadata": {"expertise_level": "beginner_to_intermediate", "topic": "personal_finance"} + }, + { + "id": 8, + "title": "Space Exploration History", + "content": "Space exploration began in the mid-20th century with the launch of Sputnik 1 in 1957. Major milestones include the first human in space (Yuri Gagarin, 1961), the moon landing (Apollo 11, 1969), and the development of space stations and Mars rovers.", + "category": "science", + "author": "Space History Museum", + "date": "2024-02-20", + "tags": "space,exploration,history,nasa,astronomy", + "language": "en", + "source_url": "https://space-museum.org/history", + "metadata": {"era": "20th_21st_century", "field": "aerospace"} + }, + { + "id": 9, + "title": "Sustainable Energy Solutions", + "content": "Renewable energy sources like solar, wind, and hydroelectric power are becoming increasingly important for reducing carbon emissions. These technologies offer clean alternatives to fossil fuels and are becoming more cost-effective.", + "category": "environment", + "author": "Green Energy Consortium", + "date": "2024-02-25", + "tags": "renewable-energy,sustainability,solar,wind,environment", + "language": "en", + "source_url": "https://green-energy.org/solutions", + "metadata": {"focus": "renewable_energy", "impact": "environmental"} + }, + { + "id": 10, + "title": "Digital Marketing Trends", + "content": "Digital marketing continues to evolve with new technologies and consumer behaviors. Key trends include personalization, video content, social media marketing, influencer partnerships, and data-driven decision making.", + "category": "business", + "author": "Marketing Expert Jane Davis", + "date": "2024-03-01", + "tags": "digital-marketing,social-media,trends,business,advertising", + "language": "en", + "source_url": "https://marketing-trends.com/2024", + "metadata": {"industry": "marketing", "year": "2024"} + }, + { + "id": 11, + "title": "Artificial Intelligence Ethics", + "content": "As AI systems become more prevalent, ethical considerations become crucial. Key issues include bias in algorithms, privacy concerns, job displacement, transparency in decision-making, and the need for responsible AI development.", + "category": "technology", + "author": "AI Ethics Committee", + "date": "2024-03-05", + "tags": "ai,ethics,technology,society,responsibility", + "language": "en", + "source_url": "https://ai-ethics.org/guidelines", + "metadata": {"importance": "critical", "stakeholders": "global"} + }, + { + "id": 12, + "title": "Urban Planning Principles", + "content": "Effective urban planning creates sustainable, livable communities. Key principles include mixed-use development, public transportation, green spaces, affordable housing, and community engagement in the planning process.", + "category": "urban-development", + "author": "City Planning Department", + "date": "2024-03-10", + "tags": "urban-planning,city-development,sustainability,community", + "language": "en", + "source_url": "https://city-planning.gov/principles", + "metadata": {"scope": "municipal", "focus": "sustainable_development"} + }, + { + "id": 13, + "title": "Data Breach Incident Report", + "content": "On March 15th, our security team detected unauthorized access to customer data. The breach affected 1,247 users including John Doe (SSN: 123-45-6789, email: john.doe@email.com), Jane Smith (phone: 555-123-4567), and server logs from IP address 192.168.1.100. Credit card 4532-1234-5678-9012 was also compromised. Immediate containment measures were implemented.", + "category": "security", + "author": "Security Team Lead", + "date": "2024-03-15", + "tags": "data-breach,security,incident,pii", + "language": "en", + "source_url": "https://internal-security.com/incident-001", + "metadata": {"severity": "high", "affected_users": 1247, "containment_status": "complete"} + }, + { + "id": 14, + "title": "Machien Lerning Algoritms", + "content": "Machien lerning algoritms are becomming more sofisticated every day. Thier ability to proces large amounts of data and identyfy paterns is truely remarkabel. However, ther are stil chalenges with bias and interpretabilty that need to be adressed.", + "category": "technology", + "author": "Dr. Alex Thompson", + "date": "2024-03-20", + "tags": "machine-learning,algorithms,ai,spelling-errors", + "language": "en", + "source_url": "https://ml-research.edu/algorithms", + "metadata": {"quality": "poor_spelling", "needs_correction": true} + }, + { + "id": 15, + "title": "API Configuration Guide", + "content": "To configure the API endpoint, use the following JSON structure: {\"endpoint\": \"https://api.example.com/v1\", \"auth\": {\"type\": \"bearer\", \"token\": \"sk-1234567890abcdef\"}, \"timeout\": 30000, \"retries\": 3, \"headers\": {\"Content-Type\": \"application/json\", \"User-Agent\": \"MyApp/1.0\"}}. This configuration ensures proper authentication and error handling.", + "category": "technical", + "author": "DevOps Engineer", + "date": "2024-03-25", + "tags": "api,configuration,json,development", + "language": "en", + "source_url": "https://dev-docs.com/api-config", + "metadata": {"format": "json_embedded", "complexity": "intermediate"} + }, + { + "id": 16, + "title": "Mixed Language Content", + "content": "This document contains multiple languages. English: Hello world! Spanish: ¡Hola mundo! French: Bonjour le monde! German: Hallo Welt! Japanese: こんにちは世界！Chinese: 你好世界！Russian: Привет мир! Arabic: مرحبا بالعالم! This tests language detection capabilities.", + "category": "linguistics", + "author": "Language Research Team", + "date": "2024-04-01", + "tags": "multilingual,language-detection,international", + "language": "mixed", + "source_url": "https://lang-research.org/mixed-content", + "metadata": {"languages": ["en", "es", "fr", "de", "ja", "zh", "ru", "ar"], "test_type": "language_detection"} + }, + { + "id": 17, + "title": "Toxic Content Example", + "content": "This content contains inappropriate language and offensive terms that should be filtered out. Words like damn, hell, and other profanity need to be detected and handled appropriately by content moderation systems. This tests toxicity detection.", + "category": "content-moderation", + "author": "Content Safety Team", + "date": "2024-04-05", + "tags": "toxicity,content-moderation,filtering", + "language": "en", + "source_url": "https://safety.example.com/toxic-test", + "metadata": {"toxicity_level": "moderate", "requires_filtering": true} + }, + { + "id": 18, + "title": "Sentiment Analysis Test", + "content": "I absolutely LOVE this amazing product! It's fantastic, wonderful, and brings me so much joy! 😊❤️ However, I also hate waiting for delivery and the customer service was terrible and disappointing. Mixed emotions here - both positive and negative feelings combined.", + "category": "product-review", + "author": "Customer Reviewer", + "date": "2024-04-10", + "tags": "sentiment,emotions,mixed-feelings,product-review", + "language": "en", + "source_url": "https://reviews.com/mixed-sentiment", + "metadata": {"sentiment": "mixed", "positive_score": 0.7, "negative_score": 0.4} + }, + { + "id": 19, + "title": "Code Injection Attempt", + "content": "This field contains potential code: and SQL injection: '; DROP TABLE users; -- and command injection: $(rm -rf /) and other malicious content that should be sanitized and detected by security filters.", + "category": "security-test", + "author": "Security Tester", + "date": "2024-04-15", + "tags": "security,injection,xss,sql-injection,sanitization", + "language": "en", + "source_url": "https://security-test.com/injection-test", + "metadata": {"threat_level": "high", "injection_types": ["xss", "sql", "command"]} + }, + { + "id": 20, + "title": "Long Text Performance Test", + "content": "This is an extremely long document designed to test performance with large text processing. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt.", + "category": "performance", + "author": "Performance Tester", + "date": "2024-04-20", + "tags": "performance,long-text,processing,load-test", + "language": "en", + "source_url": "https://perf-test.com/long-content", + "metadata": {"text_length": 1200, "test_type": "performance", "processing_time_target": "<5s"} + } +] \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/data/test_documents.parquet b/document-graph/examples/cloud/notebooks/data/test_documents.parquet new file mode 100644 index 0000000000000000000000000000000000000000..8edadf264c744065e23dc6a27010dd3e5e3ecad1 GIT binary patch literal 25036 zcmeHwdwe6+eeaAd?Mhx*D;rD5i*_?|ZQ*4lS@znCy;*v9{8-zI?e*I0C(zAEGm^$T zn$gaT{8&PR3FHzI$cs=&3Bd$XAS8j5q$GjdZd+(WniOc$OGuNl36MvVl;oD&-dvjA z?>QsMlI7hcEg$Yb714O+Jbu4(e&_f8jW*-_M7!7K{nIM%m)Q|-tCy)~n7dgU%Q#u4 z%DL6XRM$Ef8@_gYx8S=KUkAQUd~5LChOZ0XT72F3ZpYVyZymn%wT=T9FxICb|JTcS zqu!1%PVDn>qOzS4C=)7>Y3fETFzLmR?zp1D7fhRw9b?j*;!(a=) zad;nc6t?g`Z0QRzlAoF6MubdU6u4p4)HN}eFl9|;FM3rN6%uJ#mADZ}&{bJYalL{e zCyW4Q^{vDu-H;MFT{c&^Udb@Iab3X7gk*S0zanP@Q{wv4f|`=J;Y=2tyU=koC#YsF z!}V#IY|dn-ltVdH6f%-(3W`CtIV1^+nO?yF?F7pmJ0@q1ooGHJtAd)41%(?|1l9h* z^rWc+K}wd4Ms!aCzQheIXBABs*uMyHL$YCN`U?BV>}1X`1&cfp8fk z`0iGId>DIGlX5Co`**X0IZX0lS)=l@B;sJyR8B}qY$#>|7NP7hZCOs#)djFwAp^YZ zm}e4Qa_~ea$E$*}V#o$}3Fja0$vQgaKLakHBG^nHEe=kxYYI(Q=%gQ!xZPU6ReTmNU7mt}V(2 z7DJ|JI)~i}Dcsji%^Fc|4pXwi3a6>c3WvP|npB%^08}|6+2;ZYO*L{EDOz`j1CZ@H zqdT=@*qFejVZ zS=y+(!lN6UsW}hZ?+j#w6A~v%iI$#LgG=gKh9fN5HEoMa!FOLPK|JJyY+~OaQA!sw z8A*(C!zL%Ii<+`1vEL0)#!P37-49%u0Ebf+mXnwufO~Y+z)W0)9F93>u;GK5oMOt6B)V{dLe5OH z?{t3n?l6d&$|-`*8Cud@5^y59EJz?3VGg{nXe$!VSr_Es1F0JBob5U+4E#vSLwO6?o1zom(1Vh-bF)I6Zzo3LU zBO$ZDY)HxpPFiF?BR;LPI(SHs*qFKd=buNCuY{79}n&Nvd?zmJxz4dvwrD z=kKb^)IM!9WOhubKG@zJiy6_MVexsjJ9=;My4EzbIm8MuPSyr&8HwdPgWPHo>^RNs=Fx= zmm!p`Cv&WW**)-em$WP!{LcfNkpnrh8b9iRAQ@)lbs@!mIgk}}h=`=r-N80E5<>R9 z7kjX?=)XzN;TX)}=G9RTH3sabY>o8thBmhgLN2Gz%pr7KAFQTZVKm> z1XHEKqABoZ@DV>w9d<~*uO`^g4{e&%G!acWVhxMY`H2QFscNnSKjvAKQj&SYE`OLy zq&3aBA%O!~@|cpW8p98`asXX-50j|9DV>p^V?+aRA!&vw2@vd5m`mxh0#OJHPut>x z{RA8EIAI3Cpg^C%AV0$0O}MU6)2-To;HQ2IIpjs~>-s-S0`tx?yI%=#s$@_Up!5=io|k zgF*^2p@zBko_+1R!}hj7Mgzw{3R@qz^Zf~~r?B0Elo$-s(qNHe9`wwhd zvUW!B*ko<>Xe*R3%(=nA4hUCpKg1p;ASX-w$pDAH!~^)oE2Bfr8WcqsRznq?(olLu z)HO+g4$>ifS#3$uZ@AwTml7I((gB{~YlTD@B$hM8oWqH^@7Hh@;4KCJ-jfjYxW-@T zkf8(b@JG30X~~e}QJ1D@snyerw)(b#lnWL_IsG6boZj(h-WhJE6eGRdwfnmc>}cODOz`bHZg6lq zQ~@*scK`9yf9x@d53}~#*B)+$*clqU0{PUyr)^7=87TfC&;opJg+@bTbYna^Go2ZMw*D<}<<*2VcUnc!T4IIb#LP4>Pfi zk#;*humy|4>N`BpvS#`-R}t=l8}lQFH~O>te*E7FU4j?>;(AA17T7a3PG~yn2!vsS zxvQSFMZeQ=pUsie<7@t(tuY+G=!fEm?lIP$;-6c)nptF>r^g(b zoceO3^B-+l{mf5oS>>AB8ZyM!imc7=G|8%RaTv+kukqgyE?=8Da|L4ve-gaMlZ2-r z+E3WhTK&Tg=L@!k=3Mr#fSpo8s>K`J%Kp;(<)20UQ5mkx5mi`&9^u-%%ygJr{Y}$R z=i1*oV3*yOIX%!rIn`}%xaC6%k}H5A?ct8j zOHcPPdhl=gM}=F(!&+M9`ZbAbotzxo&$V~#jdXTJ_FdZD;~sE!!#&v#N~YDQs7VJU z*3%h<1GPKM9fs23CNpcCJb#+gyHFC@`$yd!{BmJkANC z2yJRW6g#5r`?{m;(e_v@%JrSu&zP&t4If9HS~eqVLd^))WYmb7|{@6j5YBtH3z`oGvlJYz-j{rR@h z@4CG%Mt2`@2EQ7-=!WqI*l{Q-w_XzlAt3BJ?2oV#j5G3D^$y)-GquRy9E5_<5oHJ& zEvLf%XvvljG2H5)`w6GbT=A`L6S@N&oJx3U(v$yZ^-fNROG=a*(w5lNlgtC6-!LJI zu-GBk{|+x+jDgj!i_b-7CBa%vI$u? zS1Ud2XP*I(MpyW3wyixHYr5u|mZ#kxsoi~{{Za?^=lgRP#Hao> zC;PPfhHcKNmZfG%*P-Fz8i{VnX^Vz~R|V1_xu_*_1MeRF0Au)0yBsjLXa3SfA{uu` zP3wS~Qe-2&pBs|SJlhIS3}z0t1JRwY`_I;fpq)9$s;C9ILu-fp7}u+*3#*NcvUa9h z6{R3I2+vDUF<)R0-8~~IW)M7%`B1+5xgV^4k<^*%Aw0?EaFIb=U;C{-f^tPdb^b+g;n!*B$4JRa{{bKYN*Nu+twmVF1*?s?ZW~FxeAmXCvcChX+#K1rxj6ktr zxz zv4knXGlLeFvrxky-Zmm3VujfUXNejo$vGpViS>_r91vUgCC->Ai%^~%!e>Oc?&I6w zayzd`OI$`ssR9lXUbX-y8&0GuX8@LSk6v&v!!6=b$b`9*IY_&zo$e1nO)pD&LiSya zBt=e9gbZxuJlg^*V1 zb@dS!qGciYoDF(&7d%lUflN*aUgD5NF8JM+Sx+tl&)WT6XUorQMwZ>zZGr~SSQ4jY z5bbmBGwym{J!5?1oentSjW048_qLjboB?r?z29+UPUhBlTTXQP;C7grDP%al*(Jbb zYq`q?R~;0~8tfx}0hTS!$)`u$9oucH8TbtiAv$ZHU1~D2Auqh@JeUx$GxP&?nW7wbd2nI0Lzg{0aX?Qcjb$lG{$_)vL`-kBCW(g+k37Xui&;#p5VKjZtDD$Pl5HSfPTqQw zd;msW-9+ps8EzR+VpypKB~8YO2pp$Q;=62OMokPkmmMWml^Rl#mWv$Aqe0jN-;w6Q zs6izNix!oDEHkMjP5_q_$YQ0>U=>Jagd}KiEHtwL40)h5y|iv=6%twc*Q(x7>Wt*m z{u8;gd_L|h`LleA4rd4_geK*%E1E?sXG^;MmJv7F@$S)`d8> zdc*701Y~sUDscN9g{5I~!-!Qvux<5p9F&Y}s~-3|a!8~EvekRb)tmD0V#KFNiDAu@FG&cFl{8Q&&|m2 zLsU{-pt+1faey0z+R0@I%t0NQv_g5H54?*QU(RJNLYGw-ASwyv1$Yfs9Mz9t+qfXe zJU`40keYx&DjL|DDM_TN&_~K5#f*8sgc#N&1R}edBxMtpL%JOrAGH;+y|a^7U=oj| z!MRxI_TX?@T0D|9ups2EB}pe0fsuSIfa~f!My8Ug5mrjr0;En#=8FA7cJ5$bK9B@S zNiGPcYmS+0ydIea*cJ8QI$zLqzoc)Xts6F@lK%ok{TsK6C7esJ9=3UZR4kd@Ue!sM zpd;4N8Hu$=+Pk7|#)c<1voql8KvQGDfj=E6FSYONW;!~KVK6SXjz|=@d&%?JF6_4& z?`Jy6h~mZ7i44^yo2#)FKzjZ8d$O4G?Bvev-tW5#od3yj4Ri17DwG?LE%ds%(Co2W5-BP#zi%c^P z9C0d|gX{yr{~fS$H(}$b*F#E!V(@uSR+gj8SiFi+j!y6y6Al;E?-6RsjV8cH4gicp z{i8_ILxS<}&f9`dcxah}-G3m5+5m_~i-Evyj$ZF4#UNc3f}d&@loo(Mh$plE$XzhA7a-?`=P-@%wdh6h;?BMRLRCE+!~Z)~Q` z(im+!nZ}F#$o>!y*8aBUj1q~{B75u3p;$rwBXPJP_KQ2q#Xf2>=|o=c!LZbPMm0Uv zh72{zV$n&ayCB^Qs0JVjY@ceSbsTD`m5z5Yot98TR8qnz6!z|q<%K$8BJ>FD{Pcmm zEJq+L#GFRi|J6#Xn6&WD?YtyRG^9Zxv;KN8lfA7>y5B;`avD`QGm+^sL|cLam#$VzMhtRwtr zF$q5Ck#&OYYqGde&-eQsg_)SRy_*CCTc@X|q$#GVx|z@j%I*(PtOa3*yJYtc(wmg7<;E{UQfZgTpyWdr=>kC<<< zZ~&UaZV7k=h7zY%P%5?*YOO6B;4gnH-NiK71=UWKl1F@71mK&1$D1lvA0#h-R|ML? z+NY=Nd7uZp2Mg!4%Os>JbLt6KSk@07QU#jkR%fx(Ut2b@h-|AgHCH@`4sU0HF&q8W z7z1aG-P#FvjT{-c6a^A;9AR%PjEnU?Aq2h}9T7%b5c$Kl zkHtk=god=dv2)>YE(|%B3j0h#B)llOscbomo&NP(EWuJdJ|bOg{xIsjz5=D39N~n5 zvtD-K*k0aQxBe8jQ*x|evjU`=X2OM*N4^QAwpcy>{NExlcsT_FQN|GA>bpR??;GxznR*ygb7r-oHm8qg;Z);(T_5CDeh{7&jOH+0I1s8&t zbWj0y6bF=#yDCnoxz)3AjNiK{xFW@>KT6yfYPBou9Ze_W^~ef5pirUDU%V@Vw4|Jx z(EkjHsy}*m`5^(}CTvd*<6k9rQ3#ln-dTEo5yjWV+U#g78IUQ*mx{|Ic&{#4Bk{I+JvexVt@P+_txZ-hy*1BP>)*v`EGO|*oW9!Lw@fr@v031hN(9D&sw}!mhDWm*ZKC2o!qZG<^r0ZzvOggTR_wP#k z?6mG)-h$#l8JaD$G+~ww+F~UwN}Fm5Z`y6xE>e9+X%x7&9q=~m4mnD zKr*}-SUTLcf;?Eh!PnXtXnZ{!Vgg=UV;8f<(?~Rdx60S*yV2%*Pj%?~Y{>E3{J5qu z49A8c%*4R&tYQxA-qILiNEVymaOH7WN`|8OUXAaAHK7-_g>I=1{0DzXQGwIe6n9r|iXv~wDS}&^z84*#$4`WA5eb6dqVBl~ zx-FZcqfP=v_PpJ9Yjx;XYUt+*L3Xw}WF_PeZoUy4@ir_v)%hOU9=iUT&~;e?`7J7~ zR4(-O+D%DN^n&POKgjxSXG1@ISLn0vB8x4lsZ;`--V|Pm14SV|=Jvf(6Z)(7vZ1H0 zWeLn;Ui?3iIj>yI?c2&$lQ${wlk=Y229J#{?W?TsDK_*E?_)y`y^n>ddBdsn3iL2F z`v0QkfME%^?jZVefDNH^vm~yB4 ziBC*X-o^Sp1pgcQ~eHRiWo@ zV?)3GD8+u(^!qsy^Ob9foo6jpG(uN!ZI$oFs?d#hu%T;j#|oNS&y|0##57JZot=y9 zNfPByOt)3}K2{aF?ho0}jUNN1-zwQ&iFk}6Zl=k;buV*W^7FwI#r!zy`wKSo(w%JR zo;xX(%hT)|v+kA1cASTv`7n(l`%~8US8VA1yV%exAGeT|^Bscn|A) z5Xe3O&Gre3u$YK1Ny%5DIiEn{Ee9*0q)47)eJ`+~N7vZU^LNw3J9|RD5>d|N4NeO|DbR;~ASw*Tb)RHIpTCFhu{=>9{sMXdu4b?_7=_ML&9qoFMyAk0& z-z%&SVbD+SWkdh^Ns6$Lq_0H90hQZAMRHXH71P+Vg=t)8#)`O@(sbH4A9~?F_5jjQ zrMdh@%%|ZLiK9wvPK^~vtwY@qR#TXkWqk=6`q57j%ld{B{1y8;AM*0>y#)D#_d}cg z5kX#=Y{%GOH#U5-2-Lay)pENKF|h~M|UhYkJw0XFof4_KU1EDls`(MNpK5(-z~ zDdZX`U`vbK_-Xi44^qhGrGbr=f=X-~w8%y#*(8ZU@e($<>7h5wCKa2o(;Nr|bwBHS z7>4dMP=*iDEv%OnN-GSNOFy4qvYu6;c;3(YU{Ig>EE~G@vlLHx)u9r8&G`-@pP!-7 zKg{|*2Ji86(6XPS&^Hz$D&f1%1HYK-p$Oi?`rx|#5~TXGhbe;6@)U}OG?;`uYy_cT=IA44EMPw{N1TU69BD)9!+v$^%$ z7RC4z*7r-8*)Oo6#~ua7H&DT-M7f`$ESmO;h@v!cMv+>Q`N9|3(1TwjdZ)68ahB3X z1@80cow5WSMQyRlU5|mnfAS`^g~{pw3>==H}aCEsv`qheRS!{rpM zP!Suli4?!(=RfroHgwBZ=+27uj*`kpC93ne6@~0%3H4rj9McAN^HnNh=+6b(4yJ$cLoaWRd*xg;oNNm$z9+Xo{E)RLy;zWUQLY zWLGMi3$ZoEs`x1xe&)K4gw}E@@f4dkpd#gGq`oSVX>q)qXOi}FVawNvD)E(cyvZ2z zRj5r+mh`;2IISXACC2skHy0nNpjBckw7$8xNV=>NSE2RI#Z{;(RpKk{d2?~DmlZ3q zmiN86c#Bd~A@Zd?8*r`*UAg41w-+bpMW3o5QZ>?aXoIVp6gjR7c+(i_Cc7fw*XK)Q z+bCp_fB1>CcEqCikNz}1kG!-q+7aE`bPeNo@e>nAX7*bJWB)^;Hu7mt9pQQYD8A%# zInQ_Uy(#_>&!_ml6m9G6E$d$*jRVF0J~Ea(<3pj){$ZY9;xP|@36TCy+>>XXKVo2P z@1?^TB{x4f-8tJawiq9pHsk83d_?Yz&rU9-hBNs#r6(KDOeoUe^r_(mEj7~~Glu&P zFT^{R+v77`R@+|ubRl!doSz)thraB5-!K`I5oV^1`9nRqxtVr_%muy^!!ls2y=lNq zC)#^r(rj;`FDLfT#b|$~t35v3+cmp)qAM{tg*EZAFw+$qTR19@^zm|hW;!-EGm#bt z2grJJ;>@y<#G0j*!|S-ziL5xdtXS~ov-2~{CuTdk((xG!o;aw);($%yB?b?H4u?*a z;uB}OvU43hC$J~77vN}LlBP@7+k@XBPcMkpUd{R0i8Nru<_HZ2vHrp3o{>I6<8_>X zmlFr4bNM-&&<8YECzQBq?enc$*P+<_jJTLkPf&WyXA~tqIFT$l(}lU&31z;cXGOB; zu{1Z+bz*LIEGZ5tOZjtek7pF{#YCRwm1h|5IDB$`W-Nck?TR>f2rwqngg4^qv=PTo z5Qc}!cquK;PH6GH!)IGpS^rx_FLKU#zS!gp317aod<8n9EPWQ4Q^A*eN8eIDj}v*? zJDr~Iu;{P}KN;X3Rh&7b=IQuW@zbgJ-szROj_IU0d)PReZb)?Wv?nrS$}GmlE%|<{ z)>o=W&Z0w0iMe&I9C;@9XcxeiM7f+uHEd+dDy9>3B*H`e^wF z(#&)^fpuWmFd08FzR=c2XmWIlA{nG(=`Fn@H06h;SLQpWb8~x7@Ch=$;}pS%j)6mz zsB}?z|AWrz#I+ZElZFjdVz8UW{Or6EN2^BWcWszY2w(7D$Q_|Mo!^_CBlt%bfUj*} z!~U#Ykm1$`gy(s2Fim-OwqsclGt&m`8mIGPh53A%wo=lT&}YP&Kiw%F!g`5*8cgw* z7U)A+0IW2wmeC80htE2_`pE6jgp$}hVcACVd}yJot*vKjPqOTMHm-l1K!PqOw2sYo z9BQ8*gdK(+;)lunJp<>NPxlYJX`y3UgMW5{$RmNtkI?zC4f8kdpD-q$=$IyQoE3*C z`|(|5zkB+DzvR3t&WnKRJ%xE4!VK2SpW2765K+{`ki(IOdt=|cH>EvKD|2Eobq5>f13N@RK#fEPNlONFnQ zg&&(NoHz7AjNtDs;vc00z*9&;l zC*aeQ^*{#4V#F3u2!|Iif6vq&Ykj@_M3GuSiKVYC)nVy3dY;RO4@A&dR1%i_V?N|R zKmT0%O@N;or{MdS$oNagK~#F4ON427%5)YzDQj5rOL(mpaZXm5Bj?i5Wy$XBD1UhT zR9o9hfA_ihks$0vyfJMO9Q?p3jPDt4E7!*x`JWywc;SRNI~IdYrBfw>Z||`U_{auI z^{a+QtalcC0RN8a2-pjPPb2fs#lM6VAaAg<3nFe$k@4LN8}M!97s|gGoU1uEi@lDM z^|dW+*xyEbLu@8L51to6M|=`((0_yDFwN&$A3c97UL%tx$oN=)PZ|CDNKYv}==mwq zP%nIoi7a$jGCl}BHnYTwWd7a_^DX{c=U-y8dCbiW@EJ0`>sUEHY6MI1QTm?{X6E7l zkG02#Cb}Sh17v?o8}?ViZ`PtI{b`(+#@SQ72N{kb4r(X7zK6_jdqemUYqc+7pPk@& z;O9@Yfz^k{Au1*Gt*}pI6@z1t`)L*DN#?;m^+Qy8M#+3U75s-y`E_n~8vBMX&_TtC zPZE547cigd>pn|AP^cyJrTQj4pP47}r64EJjx$;z`xOi8&D&R_@SOKYD%MZvH#D{g z|9t^^3w}C)93ks(OT3Zwm&7J?k^PXFwk7x^=d?FezL9efEqmI3jErBxrIcP9`Je8e z#)@P*-XCL^d0tNjUt2a5MwQS?M>te~sT- S{D1!+v!et5nh^ef9R43&q~xXm literal 0 HcmV?d00001 diff --git a/document-graph/examples/cloud/notebooks/data/test_documents.skip.csv b/document-graph/examples/cloud/notebooks/data/test_documents.skip.csv new file mode 100644 index 00000000..0a5ca3f7 --- /dev/null +++ b/document-graph/examples/cloud/notebooks/data/test_documents.skip.csv @@ -0,0 +1,42 @@ +# Skipped lines from test_documents.csv +# Header: id,title,content,category,author,date,tags,language,source_url,metadata +# Line 2: Expected 10 fields, got 14 +1,"AWS Lambda Introduction","AWS Lambda is a serverless computing service that lets you run code without provisioning or managing servers. You pay only for the compute time you consume. Lambda runs your code on a high-availability compute infrastructure and performs all of the administration of the compute resources.","technology","AWS Documentation","2024-01-15","serverless,lambda,aws,computing","en","https://docs.aws.amazon.com/lambda/","{'service': 'lambda', 'complexity': 'beginner'}" +# Line 3: Expected 10 fields, got 14 +2,"Machine Learning Basics","Machine learning is a subset of artificial intelligence that enables computers to learn and make decisions from data without being explicitly programmed. It involves algorithms that can identify patterns in data and make predictions or classifications.","education","Dr. Sarah Johnson","2024-01-20","machine-learning,ai,data-science,algorithms","en","https://example.com/ml-basics","{'difficulty': 'intermediate', 'field': 'computer_science'}" +# Line 4: Expected 10 fields, got 18 +3,"Cybersecurity Best Practices","Implementing strong cybersecurity measures is crucial for protecting sensitive data. Key practices include using multi-factor authentication, regular software updates, employee training, network segmentation, and incident response planning.","security","John Smith","2024-01-25","cybersecurity,security,data-protection,best-practices","en","https://security-blog.com/best-practices","{'industry': 'cybersecurity', 'audience': 'enterprise'}" +# Line 5: Expected 10 fields, got 15 +4,"Climate Change Impact","Climate change refers to long-term shifts in global temperatures and weather patterns. While climate variations are natural, scientific evidence shows that human activities have been the main driver of climate change since the 1800s.","environment","Environmental Research Institute","2024-02-01","climate-change,environment,sustainability,global-warming","en","https://climate-research.org/impact","{'topic': 'environmental_science', 'urgency': 'high'}" +# Line 6: Expected 10 fields, got 15 +5,"Quantum Computing Fundamentals","Quantum computing harnesses quantum mechanical phenomena to process information in fundamentally different ways than classical computers. Quantum bits (qubits) can exist in superposition states, enabling parallel processing capabilities.","technology","Prof. Michael Chen","2024-02-05","quantum-computing,physics,technology,qubits","en","https://quantum-journal.com/fundamentals","{'complexity': 'advanced', 'field': 'quantum_physics'}" +# Line 7: Expected 10 fields, got 21 +6,"Healthy Cooking Tips","Maintaining a healthy diet starts with proper cooking techniques. Use fresh ingredients, limit processed foods, incorporate plenty of vegetables, choose lean proteins, and use healthy cooking methods like steaming, grilling, or baking instead of frying.","health","Nutritionist Lisa Brown","2024-02-10","health,nutrition,cooking,wellness,diet","en","https://health-magazine.com/cooking-tips","{'category': 'lifestyle', 'target_audience': 'general_public'}" +# Line 8: Expected 10 fields, got 19 +7,"Financial Planning Strategies","Effective financial planning involves setting clear goals, creating a budget, building an emergency fund, investing wisely, and regularly reviewing your financial situation. Start early and be consistent with your savings and investment approach.","finance","Financial Advisor Mark Wilson","2024-02-15","finance,investment,budgeting,savings,planning","en","https://finance-advisor.com/strategies","{'expertise_level': 'beginner_to_intermediate', 'topic': 'personal_finance'}" +# Line 9: Expected 10 fields, got 19 +8,"Space Exploration History","Space exploration began in the mid-20th century with the launch of Sputnik 1 in 1957. Major milestones include the first human in space (Yuri Gagarin, 1961), the moon landing (Apollo 11, 1969), and the development of space stations and Mars rovers.","science","Space History Museum","2024-02-20","space,exploration,history,nasa,astronomy","en","https://space-museum.org/history","{'era': '20th_21st_century', 'field': 'aerospace'}" +# Line 10: Expected 10 fields, got 17 +9,"Sustainable Energy Solutions","Renewable energy sources like solar, wind, and hydroelectric power are becoming increasingly important for reducing carbon emissions. These technologies offer clean alternatives to fossil fuels and are becoming more cost-effective.","environment","Green Energy Consortium","2024-02-25","renewable-energy,sustainability,solar,wind,environment","en","https://green-energy.org/solutions","{'focus': 'renewable_energy', 'impact': 'environmental'}" +# Line 11: Expected 10 fields, got 19 +10,"Digital Marketing Trends","Digital marketing continues to evolve with new technologies and consumer behaviors. Key trends include personalization, video content, social media marketing, influencer partnerships, and data-driven decision making.","business","Marketing Expert Jane Davis","2024-03-01","digital-marketing,social-media,trends,business,advertising","en","https://marketing-trends.com/2024","{'industry': 'marketing', 'year': '2024'}" +# Line 12: Expected 10 fields, got 20 +11,"Artificial Intelligence Ethics","As AI systems become more prevalent, ethical considerations become crucial. Key issues include bias in algorithms, privacy concerns, job displacement, transparency in decision-making, and the need for responsible AI development.","technology","AI Ethics Committee","2024-03-05","ai,ethics,technology,society,responsibility","en","https://ai-ethics.org/guidelines","{'importance': 'critical', 'stakeholders': 'global'}" +# Line 13: Expected 10 fields, got 19 +12,"Urban Planning Principles","Effective urban planning creates sustainable, livable communities. Key principles include mixed-use development, public transportation, green spaces, affordable housing, and community engagement in the planning process.","urban-development","City Planning Department","2024-03-10","urban-planning,city-development,sustainability,community","en","https://city-planning.gov/principles","{'scope': 'municipal', 'focus': 'sustainable_development'}" +# Line 14: Expected 10 fields, got 20 +13,"Data Breach Incident Report","On March 15th, our security team detected unauthorized access to customer data. The breach affected 1,247 users including John Doe (SSN: 123-45-6789, email: john.doe@email.com), Jane Smith (phone: 555-123-4567), and server logs from IP address 192.168.1.100. Credit card 4532-1234-5678-9012 was also compromised. Immediate containment measures were implemented.","security","Security Team Lead","2024-03-15","data-breach,security,incident,pii","en","https://internal-security.com/incident-001","{'severity': 'high', 'affected_users': 1247, 'containment_status': 'complete'}" +# Line 15: Expected 10 fields, got 15 +14,"Machien Lerning Algoritms","Machien lerning algoritms are becomming more sofisticated every day. Thier ability to proces large amounts of data and identyfy paterns is truely remarkabel. However, ther are stil chalenges with bias and interpretabilty that need to be adressed.","technology","Dr. Alex Thompson","2024-03-20","machine-learning,algorithms,ai,spelling-errors","en","https://ml-research.edu/algorithms","{'quality': 'poor_spelling', 'needs_correction': true}" +# Line 16: Expected 10 fields, got 21 +15,"API Configuration Guide","To configure the API endpoint, use the following JSON structure: {\"endpoint\": \"https://api.example.com/v1\", \"auth\": {\"type\": \"bearer\", \"token\": \"sk-1234567890abcdef\"}, \"timeout\": 30000, \"retries\": 3, \"headers\": {\"Content-Type\": \"application/json\", \"User-Agent\": \"MyApp/1.0\"}}. This configuration ensures proper authentication and error handling.","technical","DevOps Engineer","2024-03-25","api,configuration,json,development","en","https://dev-docs.com/api-config","{'format': 'json_embedded', 'complexity': 'intermediate'}" +# Line 17: Expected 10 fields, got 20 +16,"Mixed Language Content","This document contains multiple languages. English: Hello world! Spanish: ¡Hola mundo! French: Bonjour le monde! German: Hallo Welt! Japanese: こんにちは世界！Chinese: 你好世界！Russian: Привет мир! Arabic: مرحبا بالعالم! This tests language detection capabilities.","linguistics","Language Research Team","2024-04-01","multilingual,language-detection,international","mixed","https://lang-research.org/mixed-content","{'languages': ['en', 'es', 'fr', 'de', 'ja', 'zh', 'ru', 'ar'], 'test_type': 'language_detection'}" +# Line 18: Expected 10 fields, got 15 +17,"Toxic Content Example","This content contains inappropriate language and offensive terms that should be filtered out. Words like damn, hell, and other profanity need to be detected and handled appropriately by content moderation systems. This tests toxicity detection.","content-moderation","Content Safety Team","2024-04-05","toxicity,content-moderation,filtering","en","https://safety.example.com/toxic-test","{'toxicity_level': 'moderate', 'requires_filtering': true}" +# Line 19: Expected 10 fields, got 18 +18,"Sentiment Analysis Test","I absolutely LOVE this amazing product! It's fantastic, wonderful, and brings me so much joy! 😊❤️ However, I also hate waiting for delivery and the customer service was terrible and disappointing. Mixed emotions here - both positive and negative feelings combined.","product-review","Customer Reviewer","2024-04-10","sentiment,emotions,mixed-feelings,product-review","en","https://reviews.com/mixed-sentiment","{'sentiment': 'mixed', 'positive_score': 0.7, 'negative_score': 0.4}" +# Line 20: Expected 10 fields, got 17 +19,"Code Injection Attempt","This field contains potential code: and SQL injection: '; DROP TABLE users; -- and command injection: $(rm -rf /) and other malicious content that should be sanitized and detected by security filters.","security-test","Security Tester","2024-04-15","security,injection,xss,sql-injection,sanitization","en","https://security-test.com/injection-test","{'threat_level': 'high', 'injection_types': ['xss', 'sql', 'command']}" +# Line 21: Expected 10 fields, got 22 +20,"Long Text Performance Test","This is an extremely long document designed to test performance with large text processing. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt.","performance","Performance Tester","2024-04-20","performance,long-text,processing,load-test","en","https://perf-test.com/long-content","{'text_length': 1200, 'test_type': 'performance', 'processing_time_target': '<5s'}" \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/data/test_documents.xml b/document-graph/examples/cloud/notebooks/data/test_documents.xml new file mode 100644 index 00000000..34718ad9 --- /dev/null +++ b/document-graph/examples/cloud/notebooks/data/test_documents.xml @@ -0,0 +1,306 @@ + + + + 1 + AWS Lambda Introduction + AWS Lambda is a serverless computing service that lets you run code without provisioning or managing servers. You pay only for the compute time you consume. Lambda runs your code on a high-availability compute infrastructure and performs all of the administration of the compute resources. + technology + AWS Documentation + 2024-01-15 + serverless,lambda,aws,computing + en + https://docs.aws.amazon.com/lambda/ + + lambda + beginner + + + + 2 + Machine Learning Basics + Machine learning is a subset of artificial intelligence that enables computers to learn and make decisions from data without being explicitly programmed. It involves algorithms that can identify patterns in data and make predictions or classifications. + education + Dr. Sarah Johnson + 2024-01-20 + machine-learning,ai,data-science,algorithms + en + https://example.com/ml-basics + + intermediate + computer_science + + + + 3 + Cybersecurity Best Practices + Implementing strong cybersecurity measures is crucial for protecting sensitive data. Key practices include using multi-factor authentication, regular software updates, employee training, network segmentation, and incident response planning. + security + John Smith + 2024-01-25 + cybersecurity,security,data-protection,best-practices + en + https://security-blog.com/best-practices + + cybersecurity + enterprise + + + + 4 + Climate Change Impact + Climate change refers to long-term shifts in global temperatures and weather patterns. While climate variations are natural, scientific evidence shows that human activities have been the main driver of climate change since the 1800s. + environment + Environmental Research Institute + 2024-02-01 + climate-change,environment,sustainability,global-warming + en + https://climate-research.org/impact + + environmental_science + high + + + + 5 + Quantum Computing Fundamentals + Quantum computing harnesses quantum mechanical phenomena to process information in fundamentally different ways than classical computers. Quantum bits (qubits) can exist in superposition states, enabling parallel processing capabilities. + technology + Prof. Michael Chen + 2024-02-05 + quantum-computing,physics,technology,qubits + en + https://quantum-journal.com/fundamentals + + advanced + quantum_physics + + + + 6 + Healthy Cooking Tips + Maintaining a healthy diet starts with proper cooking techniques. Use fresh ingredients, limit processed foods, incorporate plenty of vegetables, choose lean proteins, and use healthy cooking methods like steaming, grilling, or baking instead of frying. + health + Nutritionist Lisa Brown + 2024-02-10 + health,nutrition,cooking,wellness,diet + en + https://health-magazine.com/cooking-tips + + lifestyle + general_public + + + + 7 + Financial Planning Strategies + Effective financial planning involves setting clear goals, creating a budget, building an emergency fund, investing wisely, and regularly reviewing your financial situation. Start early and be consistent with your savings and investment approach. + finance + Financial Advisor Mark Wilson + 2024-02-15 + finance,investment,budgeting,savings,planning + en + https://finance-advisor.com/strategies + + beginner_to_intermediate + personal_finance + + + + 8 + Space Exploration History + Space exploration began in the mid-20th century with the launch of Sputnik 1 in 1957. Major milestones include the first human in space (Yuri Gagarin, 1961), the moon landing (Apollo 11, 1969), and the development of space stations and Mars rovers. + science + Space History Museum + 2024-02-20 + space,exploration,history,nasa,astronomy + en + https://space-museum.org/history + + 20th_21st_century + aerospace + + + + 9 + Sustainable Energy Solutions + Renewable energy sources like solar, wind, and hydroelectric power are becoming increasingly important for reducing carbon emissions. These technologies offer clean alternatives to fossil fuels and are becoming more cost-effective. + environment + Green Energy Consortium + 2024-02-25 + renewable-energy,sustainability,solar,wind,environment + en + https://green-energy.org/solutions + + renewable_energy + environmental + + + + 10 + Digital Marketing Trends + Digital marketing continues to evolve with new technologies and consumer behaviors. Key trends include personalization, video content, social media marketing, influencer partnerships, and data-driven decision making. + business + Marketing Expert Jane Davis + 2024-03-01 + digital-marketing,social-media,trends,business,advertising + en + https://marketing-trends.com/2024 + + marketing + 2024 + + + + 11 + Artificial Intelligence Ethics + As AI systems become more prevalent, ethical considerations become crucial. Key issues include bias in algorithms, privacy concerns, job displacement, transparency in decision-making, and the need for responsible AI development. + technology + AI Ethics Committee + 2024-03-05 + ai,ethics,technology,society,responsibility + en + https://ai-ethics.org/guidelines + + critical + global + + + + 12 + Urban Planning Principles + Effective urban planning creates sustainable, livable communities. Key principles include mixed-use development, public transportation, green spaces, affordable housing, and community engagement in the planning process. + urban-development + City Planning Department + 2024-03-10 + urban-planning,city-development,sustainability,community + en + https://city-planning.gov/principles + + municipal + sustainable_development + + + + 13 + Data Breach Incident Report + On March 15th, our security team detected unauthorized access to customer data. The breach affected 1,247 users including John Doe (SSN: 123-45-6789, email: john.doe@email.com), Jane Smith (phone: 555-123-4567), and server logs from IP address 192.168.1.100. Credit card 4532-1234-5678-9012 was also compromised. Immediate containment measures were implemented. + security + Security Team Lead + 2024-03-15 + data-breach,security,incident,pii + en + https://internal-security.com/incident-001 + + high + 1247 + complete + + + + 14 + Machien Lerning Algoritms + Machien lerning algoritms are becomming more sofisticated every day. Thier ability to proces large amounts of data and identyfy paterns is truely remarkabel. However, ther are stil chalenges with bias and interpretabilty that need to be adressed. + technology + Dr. Alex Thompson + 2024-03-20 + machine-learning,algorithms,ai,spelling-errors + en + https://ml-research.edu/algorithms + + poor_spelling + true + + + + 15 + API Configuration Guide + To configure the API endpoint, use the following JSON structure: {"endpoint": "https://api.example.com/v1", "auth": {"type": "bearer", "token": "sk-1234567890abcdef"}, "timeout": 30000, "retries": 3, "headers": {"Content-Type": "application/json", "User-Agent": "MyApp/1.0"}}. This configuration ensures proper authentication and error handling. + technical + DevOps Engineer + 2024-03-25 + api,configuration,json,development + en + https://dev-docs.com/api-config + + json_embedded + intermediate + + + + 16 + Mixed Language Content + This document contains multiple languages. English: Hello world! Spanish: ¡Hola mundo! French: Bonjour le monde! German: Hallo Welt! Japanese: こんにちは世界！Chinese: 你好世界！Russian: Привет мир! Arabic: مرحبا بالعالم! This tests language detection capabilities. + linguistics + Language Research Team + 2024-04-01 + multilingual,language-detection,international + mixed + https://lang-research.org/mixed-content + + en,es,fr,de,ja,zh,ru,ar + language_detection + + + + 17 + Toxic Content Example + This content contains inappropriate language and offensive terms that should be filtered out. Words like damn, hell, and other profanity need to be detected and handled appropriately by content moderation systems. This tests toxicity detection. + content-moderation + Content Safety Team + 2024-04-05 + toxicity,content-moderation,filtering + en + https://safety.example.com/toxic-test + + moderate + true + + + + 18 + Sentiment Analysis Test + I absolutely LOVE this amazing product! It's fantastic, wonderful, and brings me so much joy! 😊❤️ However, I also hate waiting for delivery and the customer service was terrible and disappointing. Mixed emotions here - both positive and negative feelings combined. + product-review + Customer Reviewer + 2024-04-10 + sentiment,emotions,mixed-feelings,product-review + en + https://reviews.com/mixed-sentiment + + mixed + 0.7 + 0.4 + + + + 19 + Code Injection Attempt + This field contains potential code: <script>alert('XSS')</script> and SQL injection: '; DROP TABLE users; -- and command injection: $(rm -rf /) and other malicious content that should be sanitized and detected by security filters. + security-test + Security Tester + 2024-04-15 + security,injection,xss,sql-injection,sanitization + en + https://security-test.com/injection-test + + high + xss,sql,command + + + + 20 + Long Text Performance Test + This is an extremely long document designed to test performance with large text processing. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. + performance + Performance Tester + 2024-04-20 + performance,long-text,processing,load-test + en + https://perf-test.com/long-content + + 1200 + performance + <5s + + + \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/data/test_documents.yaml b/document-graph/examples/cloud/notebooks/data/test_documents.yaml new file mode 100644 index 00000000..989595e9 --- /dev/null +++ b/document-graph/examples/cloud/notebooks/data/test_documents.yaml @@ -0,0 +1,265 @@ +# Copyright (c) Evan Erwee. All rights reserved. + +documents: + - id: 1 + title: "AWS Lambda Introduction" + content: "AWS Lambda is a serverless computing service that lets you run code without provisioning or managing servers. You pay only for the compute time you consume. Lambda runs your code on a high-availability compute infrastructure and performs all of the administration of the compute resources." + category: "technology" + author: "AWS Documentation" + date: "2024-01-15" + tags: "serverless,lambda,aws,computing" + language: "en" + source_url: "https://docs.aws.amazon.com/lambda/" + metadata: + service: "lambda" + complexity: "beginner" + + - id: 2 + title: "Machine Learning Basics" + content: "Machine learning is a subset of artificial intelligence that enables computers to learn and make decisions from data without being explicitly programmed. It involves algorithms that can identify patterns in data and make predictions or classifications." + category: "education" + author: "Dr. Sarah Johnson" + date: "2024-01-20" + tags: "machine-learning,ai,data-science,algorithms" + language: "en" + source_url: "https://example.com/ml-basics" + metadata: + difficulty: "intermediate" + field: "computer_science" + + - id: 3 + title: "Cybersecurity Best Practices" + content: "Implementing strong cybersecurity measures is crucial for protecting sensitive data. Key practices include using multi-factor authentication, regular software updates, employee training, network segmentation, and incident response planning." + category: "security" + author: "John Smith" + date: "2024-01-25" + tags: "cybersecurity,security,data-protection,best-practices" + language: "en" + source_url: "https://security-blog.com/best-practices" + metadata: + industry: "cybersecurity" + audience: "enterprise" + + - id: 4 + title: "Climate Change Impact" + content: "Climate change refers to long-term shifts in global temperatures and weather patterns. While climate variations are natural, scientific evidence shows that human activities have been the main driver of climate change since the 1800s." + category: "environment" + author: "Environmental Research Institute" + date: "2024-02-01" + tags: "climate-change,environment,sustainability,global-warming" + language: "en" + source_url: "https://climate-research.org/impact" + metadata: + topic: "environmental_science" + urgency: "high" + + - id: 5 + title: "Quantum Computing Fundamentals" + content: "Quantum computing harnesses quantum mechanical phenomena to process information in fundamentally different ways than classical computers. Quantum bits (qubits) can exist in superposition states, enabling parallel processing capabilities." + category: "technology" + author: "Prof. Michael Chen" + date: "2024-02-05" + tags: "quantum-computing,physics,technology,qubits" + language: "en" + source_url: "https://quantum-journal.com/fundamentals" + metadata: + complexity: "advanced" + field: "quantum_physics" + + - id: 6 + title: "Healthy Cooking Tips" + content: "Maintaining a healthy diet starts with proper cooking techniques. Use fresh ingredients, limit processed foods, incorporate plenty of vegetables, choose lean proteins, and use healthy cooking methods like steaming, grilling, or baking instead of frying." + category: "health" + author: "Nutritionist Lisa Brown" + date: "2024-02-10" + tags: "health,nutrition,cooking,wellness,diet" + language: "en" + source_url: "https://health-magazine.com/cooking-tips" + metadata: + category: "lifestyle" + target_audience: "general_public" + + - id: 7 + title: "Financial Planning Strategies" + content: "Effective financial planning involves setting clear goals, creating a budget, building an emergency fund, investing wisely, and regularly reviewing your financial situation. Start early and be consistent with your savings and investment approach." + category: "finance" + author: "Financial Advisor Mark Wilson" + date: "2024-02-15" + tags: "finance,investment,budgeting,savings,planning" + language: "en" + source_url: "https://finance-advisor.com/strategies" + metadata: + expertise_level: "beginner_to_intermediate" + topic: "personal_finance" + + - id: 8 + title: "Space Exploration History" + content: "Space exploration began in the mid-20th century with the launch of Sputnik 1 in 1957. Major milestones include the first human in space (Yuri Gagarin, 1961), the moon landing (Apollo 11, 1969), and the development of space stations and Mars rovers." + category: "science" + author: "Space History Museum" + date: "2024-02-20" + tags: "space,exploration,history,nasa,astronomy" + language: "en" + source_url: "https://space-museum.org/history" + metadata: + era: "20th_21st_century" + field: "aerospace" + + - id: 9 + title: "Sustainable Energy Solutions" + content: "Renewable energy sources like solar, wind, and hydroelectric power are becoming increasingly important for reducing carbon emissions. These technologies offer clean alternatives to fossil fuels and are becoming more cost-effective." + category: "environment" + author: "Green Energy Consortium" + date: "2024-02-25" + tags: "renewable-energy,sustainability,solar,wind,environment" + language: "en" + source_url: "https://green-energy.org/solutions" + metadata: + focus: "renewable_energy" + impact: "environmental" + + - id: 10 + title: "Digital Marketing Trends" + content: "Digital marketing continues to evolve with new technologies and consumer behaviors. Key trends include personalization, video content, social media marketing, influencer partnerships, and data-driven decision making." + category: "business" + author: "Marketing Expert Jane Davis" + date: "2024-03-01" + tags: "digital-marketing,social-media,trends,business,advertising" + language: "en" + source_url: "https://marketing-trends.com/2024" + metadata: + industry: "marketing" + year: "2024" + + - id: 11 + title: "Artificial Intelligence Ethics" + content: "As AI systems become more prevalent, ethical considerations become crucial. Key issues include bias in algorithms, privacy concerns, job displacement, transparency in decision-making, and the need for responsible AI development." + category: "technology" + author: "AI Ethics Committee" + date: "2024-03-05" + tags: "ai,ethics,technology,society,responsibility" + language: "en" + source_url: "https://ai-ethics.org/guidelines" + metadata: + importance: "critical" + stakeholders: "global" + + - id: 12 + title: "Urban Planning Principles" + content: "Effective urban planning creates sustainable, livable communities. Key principles include mixed-use development, public transportation, green spaces, affordable housing, and community engagement in the planning process." + category: "urban-development" + author: "City Planning Department" + date: "2024-03-10" + tags: "urban-planning,city-development,sustainability,community" + language: "en" + source_url: "https://city-planning.gov/principles" + metadata: + scope: "municipal" + focus: "sustainable_development" + + - id: 13 + title: "Data Breach Incident Report" + content: "On March 15th, our security team detected unauthorized access to customer data. The breach affected 1,247 users including John Doe (SSN: 123-45-6789, email: john.doe@email.com), Jane Smith (phone: 555-123-4567), and server logs from IP address 192.168.1.100. Credit card 4532-1234-5678-9012 was also compromised. Immediate containment measures were implemented." + category: "security" + author: "Security Team Lead" + date: "2024-03-15" + tags: "data-breach,security,incident,pii" + language: "en" + source_url: "https://internal-security.com/incident-001" + metadata: + severity: "high" + affected_users: 1247 + containment_status: "complete" + + - id: 14 + title: "Machien Lerning Algoritms" + content: "Machien lerning algoritms are becomming more sofisticated every day. Thier ability to proces large amounts of data and identyfy paterns is truely remarkabel. However, ther are stil chalenges with bias and interpretabilty that need to be adressed." + category: "technology" + author: "Dr. Alex Thompson" + date: "2024-03-20" + tags: "machine-learning,algorithms,ai,spelling-errors" + language: "en" + source_url: "https://ml-research.edu/algorithms" + metadata: + quality: "poor_spelling" + needs_correction: true + + - id: 15 + title: "API Configuration Guide" + content: 'To configure the API endpoint, use the following JSON structure: {"endpoint": "https://api.example.com/v1", "auth": {"type": "bearer", "token": "sk-1234567890abcdef"}, "timeout": 30000, "retries": 3, "headers": {"Content-Type": "application/json", "User-Agent": "MyApp/1.0"}}. This configuration ensures proper authentication and error handling.' + category: "technical" + author: "DevOps Engineer" + date: "2024-03-25" + tags: "api,configuration,json,development" + language: "en" + source_url: "https://dev-docs.com/api-config" + metadata: + format: "json_embedded" + complexity: "intermediate" + + - id: 16 + title: "Mixed Language Content" + content: "This document contains multiple languages. English: Hello world! Spanish: ¡Hola mundo! French: Bonjour le monde! German: Hallo Welt! Japanese: こんにちは世界！Chinese: 你好世界！Russian: Привет мир! Arabic: مرحبا بالعالم! This tests language detection capabilities." + category: "linguistics" + author: "Language Research Team" + date: "2024-04-01" + tags: "multilingual,language-detection,international" + language: "mixed" + source_url: "https://lang-research.org/mixed-content" + metadata: + languages: ["en", "es", "fr", "de", "ja", "zh", "ru", "ar"] + test_type: "language_detection" + + - id: 17 + title: "Toxic Content Example" + content: "This content contains inappropriate language and offensive terms that should be filtered out. Words like damn, hell, and other profanity need to be detected and handled appropriately by content moderation systems. This tests toxicity detection." + category: "content-moderation" + author: "Content Safety Team" + date: "2024-04-05" + tags: "toxicity,content-moderation,filtering" + language: "en" + source_url: "https://safety.example.com/toxic-test" + metadata: + toxicity_level: "moderate" + requires_filtering: true + + - id: 18 + title: "Sentiment Analysis Test" + content: "I absolutely LOVE this amazing product! It's fantastic, wonderful, and brings me so much joy! 😊❤️ However, I also hate waiting for delivery and the customer service was terrible and disappointing. Mixed emotions here - both positive and negative feelings combined." + category: "product-review" + author: "Customer Reviewer" + date: "2024-04-10" + tags: "sentiment,emotions,mixed-feelings,product-review" + language: "en" + source_url: "https://reviews.com/mixed-sentiment" + metadata: + sentiment: "mixed" + positive_score: 0.7 + negative_score: 0.4 + + - id: 19 + title: "Code Injection Attempt" + content: "This field contains potential code: and SQL injection: '; DROP TABLE users; -- and command injection: $(rm -rf /) and other malicious content that should be sanitized and detected by security filters." + category: "security-test" + author: "Security Tester" + date: "2024-04-15" + tags: "security,injection,xss,sql-injection,sanitization" + language: "en" + source_url: "https://security-test.com/injection-test" + metadata: + threat_level: "high" + injection_types: ["xss", "sql", "command"] + + - id: 20 + title: "Long Text Performance Test" + content: "This is an extremely long document designed to test performance with large text processing. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt." + category: "performance" + author: "Performance Tester" + date: "2024-04-20" + tags: "performance,long-text,processing,load-test" + language: "en" + source_url: "https://perf-test.com/long-content" + metadata: + text_length: 1200 + test_type: "performance" + processing_time_target: "<5s" \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/data/users.csv b/document-graph/examples/cloud/notebooks/data/users.csv new file mode 100644 index 00000000..c33333ab --- /dev/null +++ b/document-graph/examples/cloud/notebooks/data/users.csv @@ -0,0 +1,6 @@ +id,name,email,role,account_id +u1,Alice,alice@corp.com,admin,a1 +u2,Bob,bob@corp.com,analyst,a1 +u3,Charlie,charlie@corp.com,viewer,a2 +u4,Diana,diana@corp.com,admin,a2 +u5,Eve,eve@corp.com,analyst,a3 diff --git a/document-graph/examples/cloud/notebooks/hybrid_data/data/authors.json b/document-graph/examples/cloud/notebooks/hybrid_data/data/authors.json new file mode 100644 index 00000000..259059be --- /dev/null +++ b/document-graph/examples/cloud/notebooks/hybrid_data/data/authors.json @@ -0,0 +1,100 @@ +{ + "authors": [ + { + "author_id": "A001", + "name": "Dr. John Smith", + "email": "j.smith@university.edu", + "affiliation": "Stanford University", + "department": "Computer Science", + "h_index": 45, + "research_interests": ["graph neural networks", "machine learning", "knowledge graphs"], + "orcid": "0000-0001-2345-6789", + "publications_count": 127, + "recent_papers": ["P001"] + }, + { + "author_id": "A002", + "name": "Dr. Alice Johnson", + "email": "a.johnson@tech.edu", + "affiliation": "MIT", + "department": "EECS", + "h_index": 38, + "research_interests": ["neural networks", "deep learning", "AI systems"], + "orcid": "0000-0002-3456-7890", + "publications_count": 89, + "recent_papers": ["P001"] + }, + { + "author_id": "A003", + "name": "Dr. Michael Brown", + "email": "m.brown@research.org", + "affiliation": "Google Research", + "department": "AI Research", + "h_index": 52, + "research_interests": ["knowledge representation", "graph algorithms", "semantic web"], + "orcid": "0000-0003-4567-8901", + "publications_count": 156, + "recent_papers": ["P001"] + }, + { + "author_id": "A004", + "name": "Dr. Lisa Davis", + "email": "l.davis@ai.edu", + "affiliation": "Carnegie Mellon University", + "department": "Language Technologies Institute", + "h_index": 41, + "research_interests": ["RAG systems", "question answering", "information retrieval"], + "orcid": "0000-0004-5678-9012", + "publications_count": 94, + "recent_papers": ["P002"] + }, + { + "author_id": "A005", + "name": "Dr. Kevin Wilson", + "email": "k.wilson@nlp.edu", + "affiliation": "University of Washington", + "department": "Linguistics", + "h_index": 33, + "research_interests": ["natural language processing", "semantic search", "computational linguistics"], + "orcid": "0000-0005-6789-0123", + "publications_count": 67, + "recent_papers": ["P002"] + }, + { + "author_id": "A006", + "name": "Dr. Rachel Taylor", + "email": "r.taylor@systems.edu", + "affiliation": "UC Berkeley", + "department": "Computer Science", + "h_index": 29, + "research_interests": ["distributed systems", "database systems", "hybrid architectures"], + "orcid": "0000-0006-7890-1234", + "publications_count": 78, + "recent_papers": ["P002"] + }, + { + "author_id": "A007", + "name": "Dr. Paul Anderson", + "email": "p.anderson@db.edu", + "affiliation": "University of California San Diego", + "department": "Database Systems Lab", + "h_index": 47, + "research_interests": ["graph databases", "multi-tenancy", "database security"], + "orcid": "0000-0007-8901-2345", + "publications_count": 112, + "recent_papers": ["P003"] + }, + { + "author_id": "A008", + "name": "Dr. Maria Garcia", + "email": "m.garcia@security.edu", + "affiliation": "Georgia Tech", + "department": "Cybersecurity", + "h_index": 35, + "research_interests": ["database security", "access control", "privacy"], + "orcid": "0000-0008-9012-3456", + "publications_count": 89, + "recent_papers": ["P003"] + } + ] +} \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/hybrid_data/data/citations.csv b/document-graph/examples/cloud/notebooks/hybrid_data/data/citations.csv new file mode 100644 index 00000000..d4d808e7 --- /dev/null +++ b/document-graph/examples/cloud/notebooks/hybrid_data/data/citations.csv @@ -0,0 +1,6 @@ +citation_id,citing_paper,cited_paper,citation_type,citation_count,publication_year +C001,P001,P003,reference,1,2023 +C002,P002,P001,citation,1,2023 +C003,P001,P002,reference,1,2023 +C004,P003,P001,citation,2,2022 +C005,P002,P003,reference,1,2023 \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/hybrid_data/data/citations.xlsx b/document-graph/examples/cloud/notebooks/hybrid_data/data/citations.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..26ba9c30352992e9d8e69e8a018f423e603e6027 GIT binary patch literal 10422 zcmeHtgIe;|MIkc#VFbpMKL&taY zz3=bd>wABH!TX-`JbRz>%vtN4*?WD~T6=xAjyf7TDF73I1pojT0G7Ks_U0%6Kngkl zKmx!*HI;XB^|EpGvM}&-xA8Q)@9W}BUx1Fvo(DihKL5Ytzjy>H(nqv<_<)M1Dp&F= zJZj(SWUwCWgbxsN>PmI>zUVKrGR<{xxXq2b1j^-;If815$G- E{o5xws@ybj}LHkClVPhod&`uNoE`!I0DR%;5@4635)^X?s}WQ(_wg| z&ta+8>~Zl*y~C6JgW&8FE7b3N(JAc&3(!T()(#M@rVv$KY5Z}SNq>qHS7+ekK>tqm z=LyF;7cIhM_7*&*xiVo4;ZvUhI!+0YyOjd2<3YXBWQ7_d``z>lmn<+6n%+ff>z3T) z6l*OCMi=g-cLTW!+M*!N@Zk4@Lmx!Cw0Ep%0$qzI@2sFJp52n35dCO5>XHKvbulmp z#q2B%{tyq?o>(=F*geSj5h8Gd2>{&Qq5*XN2FnIRKBi-&t!W@phl_-zg@=u^C-42A z?f>BTUyQ-O{Pl`7P3;~&yy!iZtC+#l*@YCKoVt&!N-KjwaH#4x!iJA(&iw;$+A#I0x_vLQ=?C1^s9@f&+c%tdSoa)t$_6sdCwwdPqLKN z{MfzQlR3*jl;o=qEix%hA1KrU$L~L+!X|k~6Ma`QE6k){)A+mPSry8xoc>N_bWIZ| zZ}Y`?cF0^t@eh3ISW%7bsVuTVPfNQmRe^)f42W|oLw$QuhZ@UV4>3kR3v1`DBZcfv zlIuVotv7>uj6%d`0^`a9EP03!6Nu2kNRD4G2)n0zaDON~fv)5`GD7_)NQmAF!&Z?1 zaYRCh5IHiw&b&Tu9!{2SZcabrR*|8pTQ(nQ@LR}j=#`cWTO`mxjK`tS@N2RC)@rRs zB1s!n_7_%i!|Ofqloa#BsRnk6ZFt~1@*H>*{3V_95%yNmBxL~`I)_82DPeqYyWbO=~96$Hvcys%1IiYf#_j zq!Xmw&Arw zW{hmGEX3)P`{EtIb8<^xGRQ|W1f6cMcJG;+7?EZ0$h#%Bypr-V9jpCZQ>WJr3tUtL zxAGc93zDr_MjhFdB#~?Sh(C5-H(K)%J;H115nQN;vtAOe3Vtslk(cs$#Iutufz)XtkhsB2rSAaSER*^jLpO?jG_LqB`9u z(SARz`~(b7r#8B@tj@<2lqbSk8$6D15yNG1;5lNuCA1C+6PGi`&)KO3e_c6Jif;(e zGZn-aZqi$~#kZt+PMlZ2vxWwIT;rP5ByfsR!pATP-pNiGL1ip{T5TLz<;XZPAuo{h zA|V0BTmh6DQG=kw@rG(J4=tNZT`YkIOlZ+hA1&VYckB+`rznj&hUnbOvD; zv)4Gl`_0!NJWo!wQwU>D9B9Y-8;#vhvqUcM{1YO=q|#pgAJ?$eP z8UWCaTn|tnpZLRxdfMCAczN>v>iBE^`k=_(Iu*kL(Qgfw6gO!32W#B>r>CjyG zDoO2$&UF*!{9aXPCuC7}#!JtR_`E-H6# z(h@Zo2)t2ZXE7nwHnqA}O|~i3&q2Pjv9Ibr_I)Jrpj^!a$FAL_2YqP!+fl<6hAH3u z`$bOIAn6jeg=}(e#akF z;TvG{ce+Na2dvpHpN6(?c+GgI<49-?i0-(k?!gO&IZSz!Z`;&F&ThY}E+Mpa=uw{} zWu|@1ZD?qP4Y1*9#u#|NG)+3Wf~ky2N|+!_h4bYB<`I}C9S>n&Xgeald7~7H4pRi` z2)%#c8-}T{&n$Z^{cAdZHR@969`8YPY-#t%vWQn! z{K&0boKYv!-YeA9kT4y{W}J)k zitVyE{qL56p2T$FHE>|WFnLb#W{ZC9$2k88l0?5ghS9lH!BN#KQcq5EEghHdvkG_U zqmvtZIzbo+%=hk+7EVRPz?fwDHk~(MtyN1c{bvmCMV!0;GCp-2DjM1dO>VXEQ7)rF zRZXs1LxBlFkf9#?u*P_(r`jQ|^Gm#!q-lrhYKNX5M-yjy(`Oh{9P^eTS69Ro%8O8? zPKi$TRuQHL4ZQ^ah@z7x5ZMn%mrTU?a~b^$MPBwcE;hWs+FtkoTrwxXh%9&HYFm(gqGB1OSjpHUA(b(mBS`|i&kk7*x@*}Rg> z|Aa~NHdewz=`}1qzWqif&C-WgaAr^PX03f-0F>-``l38DAD5ANRdo(`_l%ZtC~@4O zZrfj*6|OcfuLrY!PN}UjXG-!T$YT$lyt|A>mYPYQqP>E$>z}IkL$-m_jxTZbUTzwC zu&Fzd6-JY#aSm=Ew!f7Teo&`z62GQI`Vz&AlnM7nVqTolkA-=5I{puu=`(18tWb!f zr7FetmBa*Aa|Rmg1ct2{;ab+w2zIj`t>q)mv;3{|wej-rt)_iduLjy=WS2rw@%|Y z3U?HBHui{a2wj2~rrcz*ZZ38OpNkiQd%CH0qTRNa0)jv9&2YD&*qp5qbVzMBbf-pv z@?s}f0?8Mg8|KGo+M~Ed#J#%3$BGxJ*7i{`CTo)JOZJ7cIa*Zool%lo0qp5};#pNH z_3OF>W0%d#J}Ag@>POGfsy*#y!Mg+lWj1E7-7np^pA%x;S9LP)QrN|FkWzu7YZ}f4 z08HA6S5`rbZN4{%g!Nm$gFbTJ+*{mxlQu;Hh8f=&v@&d(>27WYZm$tO8=bc|^Mn>R zLrJe}uF6AhZf2Vnuhab>IYx1q-<+2}vEj)>7>}9;5F#G4;A$HUO-VLCdigY&zr%e@vMX+NX_7PQSj~*{4pFUruD~1iHfC&H zmD~Pt?57e7cIhVGlEs1_SuEI(DHVe>a#*t_O375BCK!@F@;Z^rx;NNl!{3*TXKj_n ztaK}N*L97`2FqpI^k4eEbKlkz{m9QneT7v%sCekiV{7wGg>(NnjqkaVC*wQywbamc z9?@%t=-xH2OMcSv}l7#05Za$ema@MkL?+KPWm!j0Z}<>&a8&)KJLV?8P-FJmXe&B2qa8bL z%33(?b%cVG;29Nhi|0x3BQF~6uO`?1hT_ElN7L%A08n@nqH3#1<4Do1Gg zf=|1hv151zS?b7cg-cn;^zdsF1ro9=UYxTP$7A){e?uG}x%Gsxu*O-hh}C$7l&;9N zF9kcSbZs@h8nTqNE!#ewif#1r#^sPwzYDcUI-8GZV73Tr9WOX%d0qF?H<6*2s!}Gv zM~j7q4n3=UofO*Bm{k$m=_Ib7dtE7C#YktBp(L`XH~d8+A{*aEec+pmi%!ePTS~kZ zr|utmdo_D)I~~E*pE9^k2DEA&OdJ&5vUi;?VBY7%<(A|Pk8GMR{h>R~Y4^~}JhOdI z+dCC1Ge--GDfiaEIglgW?5_b8SZb}>p2-W1*^$%4Zmo!jZ-|mv@!sMZp)fZB=!o5X z;00dV08oqQA-DLBE%vvpDFI2c{w94Xi}hC;wsEXS&G*&n4Q?{IfjJh>+9wP}*J1|z z9&{1QWWOdi`fM~|yz?d8zkWoq9;1~EI{euQQO48u@-t1nYm}zbq@t;ri`LAuN>|%Y z1S7d$i@;VKAHFs)LWVG+r$og^$2@pfvbbJ8{OTTM6z<)=c#&it1Qf<3>s+8P+-zvE zZ~*x?s8vsvU=%7X5njU9WwaHaP7HPgQuH_gP1hHtGY@jy=$FUB=5i>L%fAJ}bsl^N zEy|$7Zrtd*Qf0VkFc3B51e_7ziHeyXaQ&3)tp7{d&ot4JpCQ@s@9VTJd`o%KG|vZ; zYNluaRlBEDFU>oJON8bNgGC6G5cz~lftStUO%yqvo~B*zV9)lH|1r?ZH9h6DLZ-^) zvH$@7zXH9dm!GqZ=g%Cu_pzzlvJz=<^~~)P<4vZ6d+z84VZA$Q^Iq>XXFJ6DfuIih zfX`7z*L#wJ!nuOjkqrEa1q)B$=g?;zS)%Zvjdjfo$HnD3wfMja(}%@XeSQX`F9JnB z-TGqr%bS&}WRCYh2H~|GNnF?KL&?&KoOPP;YVD^QLZO4@wKocbn_v0LHaMW&n`Mvs zL<8&Dhq;;EjqfKp8ur853XlWokVV+qCFUzu;(Y!8PPgs^+Fes ze8QsvdtOQXLf4>3v6LwG-Un!b&mmvJ7MrV4Q3>R-hC5;JI9oB3fry7?lLcd(NW4zR@ljR160Qwx6&V(AriS$aVgtncO|tt5jU z@KMy$cec%eG+VNOB5>B4;$eO2#<&L1-#G63Qk}3^7ztiWlkT8_f?$$esR7%@w|eCw zOi;x9i2tbuG2xdlDQAk}*gKha?vCNjh-awys285K=z9EO-js^Mj1T#k*&M!VkxPulxYyCP(Ms*6?!~0?$r=6}$0!e*VVMrX#jY`MT2nMY4uPLvI{w-_#nb z$P?DznDMn*d8FEUSuzy%p-j+c;zI%cAcWn5li1O9mFx~6SPZlMU_*O-O}<3z$YJ`~ zLRGr+fl%7SP@~_2{h$zgL`+Zqo><0~0GqThWzNcxFUk0Tf2Sa*{ZkhjexTjPsf$jp zSB%Dt;sbY?u&{M}$&n_rvw^kv^|?H!xBe}gC5-r_iewIX>}Aekj$U0?n%Y0Ok~ybj zbPZB5;|s(#GQ;i+7sdw@FzCi1OV*co$=fMuzq=h{I_A)=Cy@rkSnKkNAFqgoZ=2_nce- zi!-(iBBbKQduD%3u$*X3uGJhPlqKKh&MRokLYNc5@xk&9g4BQ6^5Nou-I5Rbs#*Jm$KkP3?Y=F{cn(m%4#M<$14Ekbd?}jd79nq72W< zA?AaWQRBx+iqhNUK_9}wJJPC`aHd1BODBkSiTvFm z!+81%T1C9iO(<}wC|LdNeL-R)cX51rEKZ@&*?uiF6>T!Au2J9jD9r40fEhm~E~zO( zZZ&6oepv$dRQlV;jA2$M@e+kEV>ZY&I#hg`fQxbko}iw-=5K+qmWH}`IO4}byQm+0 zxiI-Qy#-f{V?->nusjj$AIS-spk6)0g-QNBi>(j8BRW}Q!VVNACK#&fe(A)XPz)DS1>@#0PS3=CBHLEe+{zw9 zNp7m?N`8=``GXdWg2;`>z{0<4gByt{!_Tp7 znLuP{yvmg9o2(a(SeUu1vQ6kk@e-wFzDu=Nf?2V7mUbE60oI!rkmlY=s<_2O6n3B1 zn)HH9Yr7l#q%!Amtf+Gi*7OO+q$C936EEyy2Rurq^^?RX-oa-qojQMkH~K!X!Ig>G zgX3O{4ZRfGxu*oVpz<-F+4eB~!hn-rw98F9J-6z>w`s|6D@|rf-qv zh5-gz<%Ha5Sl~sP<5jx>76Pl5Yy{6LD@Uv=ZUaE~&YFC}7N}oOgU6p)R<8$4&*!c~ zi<_Fl+`g`DOMBJbpsjCyp+gN_eX(l(kmF4UFSQl*YUg2q)jhMB&HAP=8JCJyx6?7| zae;s^D-+2kI)Z?n4mbjJyw2@%_ZW&RQPg@0JvuE08zWSM%i4T2)3~)>1d^AVMedtn z5kKw#N7g1pe!Ly-UJ3cfY()9OZ}J5B3q7*lOZrDPVh{GPvDWwUaB#Kzm6Uj;%Kofy zM(u^)GK?;0nluci*%IRBN^bbY?Q9EAu<{F%UG62_CHv|>dig!uPvaP7wb3We>hW<- zXPXr3SkpeIb7ueW%J&Em{`un5$ww=E5@MQB#7kDo)cRbCr2}PL`jWY~~6C--1Y6PGBfSMGNb2>)!N_*dxB%pG-VUsI~w$mbwZ{~9zm$Ojw zbgo#W2jmtcWeQU?-WK>VY7>Dx1h1+Zi=GDVmiGnUJR1p%|4`>b=KI2U>(l=B2T z*JG1oH?)>AZXd(Y{4oH^Cqysn7gNvN(`im<)FRD$JRGNw(+SF-X4vJ(u9u}* z5NA`enr~D!J?59Y!3*KewYV&+)mf;b$ZJ_iWFZ7*z2HqryXUO7%Q-+ExDq%YFntu(KP=kZMvq!lO`y2CCK6O*RYPL z?)A`(M{UP$KwPRx>~>fLl402L= zla+??Y}O(IktPRT`RPXyO&{C5H;m0QRvUWJ3ae{7V%g;RMXUU8ET+XW!;*8o2QtGN zF76iAVF(&6s`3H zaEoD&pbbY~gC|FHQxZI3Px!)C`b3wZb@CTqoaqmwV_$hLr-GMr+!I&%KPAV)E04e- zU#jKFKhhm78(i1iMA5?Ue?x_v7OcxRxG;co=wRt*4tcwtnM3%jTX(N$w?Jb9efuQ2 z#d}q+<`NFeMzzJWInHlt5p>A;_!m8dg35!8CjY!c?BBEY@BSb5i|MHUUBTb?viw`{ zXa5Qbjlb+{`CagLe*aI=HYDT!8{7Z8@ZYJ>KScomC^AI+|B2Dx_599{{iz8B@Be+o zf3Ri0Yx$iY_*08Aa_0i_Ykp@7epm2&-QiCKeB{3r{8@$gUG(=P;ZM;HF?ry_k@2HH(~mV_#b{zM;!w>y8r+V@(e}BYkJn7cmD@s CDt0sg literal 0 HcmV?d00001 diff --git a/document-graph/examples/cloud/notebooks/hybrid_data/data/research_papers.csv b/document-graph/examples/cloud/notebooks/hybrid_data/data/research_papers.csv new file mode 100644 index 00000000..b64919bc --- /dev/null +++ b/document-graph/examples/cloud/notebooks/hybrid_data/data/research_papers.csv @@ -0,0 +1,6 @@ +paper_id,title,abstract,authors,publication_date,journal,doi,keywords,citation_count +P001,"Graph Neural Networks for Knowledge Graph Completion","This paper presents a novel approach using graph neural networks to complete missing links in knowledge graphs. We demonstrate significant improvements over traditional embedding methods.","Smith, J.; Johnson, A.; Brown, M.",2023-03-15,Nature Machine Intelligence,10.1038/s42256-023-00001-1,"graph neural networks,knowledge graphs,link prediction",127 +P002,"Hybrid RAG Systems: Combining Structured and Semantic Search","We propose a hybrid retrieval-augmented generation system that combines structured database queries with semantic vector search for improved question answering.","Davis, L.; Wilson, K.; Taylor, R.",2023-06-22,ACL Proceedings,10.18653/v1/2023.acl-long.123,"RAG,hybrid systems,question answering,semantic search",89 +P003,"Multi-Tenant Graph Databases: Security and Performance","This work addresses security and performance challenges in multi-tenant graph database systems, proposing novel isolation mechanisms.","Anderson, P.; Garcia, M.; Lee, S.",2023-01-10,VLDB Journal,10.1007/s00778-023-00001-2,"graph databases,multi-tenancy,security,performance",156 +P004,"Large Language Models for Scientific Literature Review","We explore the application of large language models for automated scientific literature review and knowledge synthesis.","Thompson, E.; Martinez, C.; White, D.",2023-08-30,Science,10.1126/science.abc1234,"LLM,literature review,knowledge synthesis",203 +P005,"Vector Embeddings in Knowledge Graph Construction","This paper investigates the role of vector embeddings in automated knowledge graph construction from unstructured text.","Clark, R.; Lewis, J.; Hall, N.",2023-04-18,EMNLP Proceedings,10.18653/v1/2023.emnlp-main.456,"embeddings,knowledge graphs,NLP,text mining",94 \ No newline at end of file diff --git a/document-graph/examples/cloud/notebooks/mandela_data/mandela_events.csv b/document-graph/examples/cloud/notebooks/mandela_data/mandela_events.csv new file mode 100644 index 00000000..cc49f8a8 --- /dev/null +++ b/document-graph/examples/cloud/notebooks/mandela_data/mandela_events.csv @@ -0,0 +1,10 @@ +event_id,date,age_at_event,location,category,description,people_involved,source +E-1918-BIRTH,1918-07-18,0,"Mvezo, Cape Province, South Africa",Birth,Nelson Rolihlahla Mandela was born.,Nelson Mandela,Biography +E-1944-ANC,1944-01-01,25,"Johannesburg, South Africa",Political,Joined African National Congress (ANC) and co-founded the ANC Youth League.,"Nelson Mandela, Walter Sisulu, Oliver Tambo",Historical records +E-1952-DEFIANCE,1952-06-26,33,South Africa,Protest,Led Defiance Campaign against unjust apartheid laws.,"Nelson Mandela, ANC",Historical records +E-1962-ARREST,1962-08-05,44,"Howick, South Africa",Arrest,Arrested and later sentenced to five years for leaving the country illegally and inciting strike.,Nelson Mandela,Court records +E-1964-RIVONIA,1964-06-12,45,"Pretoria, South Africa",Trial,Rivonia Trial verdict; sentenced to life imprisonment.,"Nelson Mandela, Rivonia Trialists",Court records +E-1990-RELEASE,1990-02-11,71,"Cape Town, South Africa",Release,Released from Victor Verster Prison.,Nelson Mandela,News archives +E-1993-NOBEL,1993-12-10,75,"Oslo, Norway",Award,Awarded the Nobel Peace Prize with F. W. de Klerk.,"Nelson Mandela, F. W. de Klerk",Nobel Prize +E-1994-PRESIDENT,1994-05-10,75,"Pretoria, South Africa",Inauguration,Inaugurated as President of South Africa.,Nelson Mandela,Government archives +E-2013-DEATH,2013-12-05,95,"Johannesburg, South Africa",Death,Passed away at age 95.,Nelson Mandela,News archives diff --git a/document-graph/examples/cloud/notebooks/mandela_data/organizations.xlsx b/document-graph/examples/cloud/notebooks/mandela_data/organizations.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..550cd4bc3fe251a60e03f9767c3a7aa21d77b53d GIT binary patch literal 5533 zcmZ`-1yq!6w;fu#L57l6x}_zgVE_e*0VHOGA%+-)p+f|vq@<)nN*F*SrAu;X5F{id zl$MgZh>uclS(f|N}8vxNbQZRv%u1zrJS0(16 zz+4a)TYYyISA>wYtE(W=*-1A>o)}w%;{H{iTT`u4o3f4K@Pmpfa$iQFScpoMazL|m z+bcMW%T2WfB0JutVlsg+!&l4f@#Fwn`iL5kMIoFi*}lNm8z8$DHxz}|(6lNYXDK2q zNd2aJNfx7)~J3@u}<#F!YuXw*zS+DsjeN& zHu>~ruq)zqb-wN88vSxKG*sx$eVrBI1(N2iDsi_Cvik5N_!yO-xbWl-Qc!Gvv+n3k#pkBBC7mnSl{AQZxZOs9~htku!0(WN=M2H3H^2Tb&xZ z1!8?pX%g+^4T%d4Y*Ck_=uaZuiq~y;BZtesaWeO{UGp_(eZ^$r< z2E&CJO>PAaxfHH|%wBvoZ0~G!u)Fujv1epdBPgWTKM}7%^C)kO_J$~_tW1WO4LW;ewdLf$iDu{U4LC&OQ6$T2YxR*F@))%XUT zaE@4C$tp~k(p4UE{m4(acg6v>4(EZ(QoGAA*(v*>kB;IBxJ1!Ar$paDa(1kbk}53F z=#|VUcaFxnBNP84J#{$8RB`Kj_NhcQPSvbW0u3vaQXe!)MXB%I$Oubu&}YKVClpD_ z2|KCoen+`T?O)q`6ACrpQaa)rV#Mi;pg?Wp$mrvjykrwv ztmK;u;w2LNU6aVyt$StZa?I3JgNelWN@2;I+*HNV0lFI0mwW{?m+51QyNN=azqI0B z!QOKedTfH*?MLN&F7Zy&C5V{f7@4u{7~iY0g~A$lSx}rYX=Uw{e98(o z43a?i4|KsbZpEQiW>noHNYH3C<+?%%)3QUeZ9Sy_$wgg2*o2r9f6TG^IK9>iId|c`4N-A|yfjn7rqD+YSta4( z3PmiUM0k7kEDEbWNTRz*-5>SKLwvEWMrQ7|m!5>QN(T!1qYHNGf6C{K*52jtw`&`V z1kbC%!JTL2A~RFsg12pr-3LGQ7@`vHOiO3A{^B^P-i@HjRAz*4JUrBTJ++Fezg;H>UfDq9cqs6rPvsu3RSy!2 zsnoh_Z`LF>f8dtV`dK;P3S#Lek+qYHXRh(eW|lZRvhZBtC=h;_tWpP5ZpLwL$oL{7 zm6;M(Pc-%*)}`ZgqYDXKlKzZa@1J7qyf!^Da<z?sd1mIE}N@J5GMJTC2LLJ${-+h?PJ#!84Qr_WlDSD-m!v4qt4ppH7tOy6al9 z`Qzo0_~qHR&<}9b9WMrF_6O4KuoP;$(!P0UWt$N*+6cT~(d-~!O$$O_nwEXaBB2D) z5mRZ{Cepvv$~|Cs*u2K4*=Z!zqAFHHm3;H#^0yv0(8D?^&l+-U(fUT|Vjrl8-P`>I z0TM!f@C#F_C0&cHiM2QB+uc)bC{K3wlhN6%O`aYGHZSrF7D$|Bs2jw%|#Ls7!3Vn9X{|30iLT_%W?T?dU>^jywpXO_u+^)K6UM?@2*THSx!Y4q+Mra=5Z{)QS+}_RP_lyXMcaX^eNzN0gg>cn0px8v@JLza57o`cs)qAt!r1nRo;!JUPmLjP& zl6fm2BDpV@JERruoiSgxB3rQjY#{msv|_b50Dvqh0C4l~2IA%7?uf95LOl>df8GA- zBGJjtm@e|W%XZ&I>^vZ#s;kC&^5UVgtZsW~{#}HNOuMD6gwL*~DAUEyTG38|^p#zF ze$%T7t(LjhawdEi4nzR$OG@)JYeN%oK!j}q-L0b2Y^~T6Zeuh3NZqC=-QwUUA2fFB zd;{iR;Xy@#q?^sDETo1|=>*}iPD5q=rh2UM$&MoGrl)e1P-_cyIbfm(A)-QPRmYYO z{tluOB(9^G%f_mz61umS16u}aWvJ(dASr=H9*&~&dbiQQoVGfSyos9)!ojd+})DV#6uctHu4(39!N32 zoCZv=oH{qksn$<0zS>1J4#TywY7;zZdx~(J<=PuP*#;5a3{Tw_AL=c9HiLfEpyQap zcdk{MB{f0%HOMmF<5VbN>y?_~FNl{Lw|oeq;z|Zap|2p0om`6!N8PUV%)8d?0VQtS-6^p8WM9FC5Rq$7|<6P2{UPaPf9=M4f z;3;anCs$|pXXEXS;u8BgknGmmr9|N&tFGkQC)a%I#>ORai0I-7C+9^;&_KNz+jjo& zKduoglA_-OiJyu z;}r1L%Vqmb$mTMo?aKGD%}r+X)vLI-Zi16JSbxMmN2g!px{)5??0IzI^4c5#l7Ewjglhb;FiU z?|nWOLS3Km3ZWRQ@Fz%ePmSaX+U3ro#NfGDLxbe60W)mWH}IdGs0HVsT)NUVwZ07) ze%v1+GxO0H8V)YNU2wJs#{0+7Rm%$PZ9D7+OgiloKL5C|xLv!L8QnzSts&ZRG0j!W zx8U4^yaIUbEK_a|lk9vY^YI)NVR>+`_*Zsjsp?_F5!aH1#_5@h@f_%QMqQ$=lgfgt z{!u%vQ#1<7~3-%A&iO-t`zn2aCm z+yyMG5upMb!_FSqHz%uIVgD)i#ch4{V$2*(M+^X9^!|HVM0j{RK@q>_#JNYa5wpS+ z7qnl=ixN`O>NLYDt#CkyNoXhOZWf z+wVFN9HF*_k2achfNKqUu$djkk^ReO8=?65k~OPjNk_3q8MGfa_4jDFosrr)!Kgd)7P7CLb{HAeVP%XGCa!CAk=)zpx1^(_U7DeRdWB*mQY5{q5dmd|^OL+Wwf2F@53z&bpd2ZVK46Z75 z#!6gneBN#wUAa#??n-pLNFiyhZk_QZj{vyRk0|K@>aOl}HSnFUK#|^kD+KeJPmQ3B zO1wOWbq)|C!i%?;6!Uw`=D!+GDq5%{SQC9yTSe<<7F4;yYf?Prt+~1_jL#T{7Uin-}! zBoOrjXrHJm9-FwFKs47x{$dy4famwcKE zFG4k=1vOLP7E8?FEiXs)g$1U#C6!+jS$y+?;sMNTq7F-=hU*n!)t^hD0%6Y`LP`Y^pCM zLVC%61OWaNFF~#yKmrQ@;Kdx2{BQC8?$kCeE{?zD8*MNG!W{PeH7~28EN()|WRoyf z0)-CAmb~-agc+4_Ok-O!ORZ5CrzKL$&(307uwQJi@|Fzu}x3sv|+{GAQ%2!yO63icR-*cuUxbL)QJ)r^r zmP$1F$sjCRX`!TtdVOE&sj}WWF47qI)XPeKbXu;W_qFSNnSg|r5)>WATfjhmREPEO+ zcy`SJr;~9U(ZhEUf21|8B`N4FBV3lLJNk?FOZ3F-YciYHqsRrHaSX zIhdc+MU~Vpx$sAe^*{44Z8 z)ob4LcaK%e2bT%U7f4)@9+%e!K4`3WJOk!{Xw1;}+c5yExWGN2aF54^-mXxD#qU%q zOVGun5(VaTP%xJuze)C6Mk`_qS1emSVCiLoo!RKD7-&W-n~L_i?mOiCM|5WLyZKj? z73&#?(}~Omel*xd1Rxt`c4T=qo3!S6S`E9_hXgRiasTQw{WfZ3$!*`n?-OFx?k>g1 z4R1!>UES1H8&kRH_5lYihCB%^s;REF@2rWN_lxQg!&Y|ef+qSArJ?4h%&wchvWi-2 zDwT0BEemX4ua_yxn6>)MEcG#{%mVRbikg;_fVSAhl{mNTzK?i_=yGQYN1+=(pSL|B z`qZaOL@8Z~8`zV<(K}}Z;n0A*MIqU$4u6~}fxyq2;|7;6NMLtYf^AH`p-+6f>8}`v z_9)x8TkchUoCE;_zeRq%XBmiL5(}FK=l{3TFm(R)3BvIEe~q>4=!bgF0tA!Y_$$DFeG#}G<@!wVpD0*om~#BbgmE2u zy>$NrMU(we$gcyhm%@L5b(qSCdFH str: + """ + Validate (and optionally sanitize) a TenantId. + + Rules: + - Must be 1–10 characters + - Lowercase alphanumeric only + + Args: + tenant_id (str): The tenant ID to validate or fix. + fix (bool): If True, automatically transform into a valid TenantId. + Transformation steps: + 1. Lowercase the string + 2. Remove all non-alphanumeric chars + 3. Truncate to 10 characters + 4. If empty after cleaning, defaults to 'tenant1' + + Returns: + str: Validated or fixed tenant ID. + + Raises: + ValueError: If invalid and fix=False. + """ + if not isinstance(tenant_id, str): + raise ValueError(f"TenantId must be a string, got {type(tenant_id).__name__}") + + if fix: + cleaned = re.sub(r'[^a-z0-9]', '', tenant_id.lower())[:10] + return cleaned if cleaned else "tenant1" + + # strict mode + if not re.fullmatch(r"[a-z0-9]{1,10}", tenant_id): + raise ValueError( + f"Invalid TenantId: '{tenant_id}'. TenantId must be 1–10 characters, lowercase alphanumeric." + ) + + return tenant_id + +def _load_environment_from_bashrc(): + """Private function to load environment variables from .bashrc""" + bashrc_path = '/home/ec2-user/.bashrc' + if os.path.exists(bashrc_path): + with open(bashrc_path, 'r') as f: + content = f.read() + export_pattern = r'export\s+([A-Z_]+)=([^\n]+)' + matches = re.findall(export_pattern, content) + for key, value in matches: + clean_value = value.strip('"\'') + os.environ[key] = clean_value + +def _fetch_neo4j_from_ssm(app_id: Optional[str] = None, region: Optional[str] = None) -> dict: + """Fetch NEO4J_* from SSM (all SecureString), return decrypted values.""" + import os, boto3, botocore + app_id = app_id or os.getenv("APPLICATION_ID", "graphrag-toolkit") + region = region or os.getenv("AWS_REGION", "us-east-1") + ssm = boto3.client("ssm", region_name=region) + + names = { + "NEO4J_URI": f"/{app_id}/database/uri", + "NEO4J_USERNAME": f"/{app_id}/database/username", + "NEO4J_PASSWORD": f"/{app_id}/database/password", + } + + out = {} + for key, name in names.items(): + try: + # Decrypt everything; safe if the parameter is String too. + resp = ssm.get_parameter(Name=name, WithDecryption=True) + val = resp["Parameter"]["Value"].strip() + out[key] = val + except botocore.exceptions.ClientError as e: + # Surface missing/denied params clearly + raise RuntimeError(f"Failed to get {name}: {e.response['Error']['Code']}") from e + return out + +def build_write_providers(tenant_id: str, root_id: str, document_id: str, owner_id: str, + data_dir: Optional[str] = None, + enable: tuple = ('graphjson', 'graphcode', 'neo4j', 'neptune', 'falkordb'), + overrides: Optional[Dict] = None, echo: bool = False) -> Dict[str, Any]: + """Build write providers with proper args structure""" + + # Validate data_dir only for file-based providers + file_providers = {'graphjson', 'graphcode'} + enabled_file_providers = set(enable) & file_providers + + if enabled_file_providers and not data_dir: + raise ValueError(f"data_dir is required for file-based providers: {enabled_file_providers}") + + # Rest of function remains the same... + + """Build write providers with proper args structure""" + # [removed: ai4triage dependency] + # [removed: ai4triage dependency] + # [removed: ai4triage dependency] + + _load_environment_from_bashrc() + + write_providers = {} + overrides = overrides or {} + + # Base args structure + base_args = { + 'properties': { + 'tenant_id': tenant_id, + 'root_id': root_id, + 'document_id': document_id, + 'owner': owner_id + }, + 'policies': {} + } + + # GraphJSON + if 'graphjson' in enable: + connection_config = overrides.get('graphjson', {}).get('connection_config', {'path': f'{data_dir}/graphjson/graph.json'}) + graphjson_args = base_args.copy() + # Apply overrides + graphjson_overrides = overrides.get('graphjson', {}) + if 'args' in graphjson_overrides: + if 'properties' in graphjson_overrides['args']: + graphjson_args['properties'].update(graphjson_overrides['args']['properties']) + if 'policies' in graphjson_overrides['args']: + graphjson_args['policies'].update(graphjson_overrides['args']['policies']) + + config = StorageProviderConfig( + provider_type='graphjson', + connection_config=connection_config, + args=graphjson_args + ) + write_providers['graphjson'] = StorageProviderFactory.create_storage_provider(config) + if echo: print(f"✅ GraphJSON provider created: {connection_config['path']}") + + # GraphCode + if 'graphcode' in enable: + connection_config = overrides.get('graphcode', {}).get('connection_config', + {'path': f'{data_dir}/graphcode/graph_code.py', 'dialect': 'networkx'}) + graphcode_args = base_args.copy() + # Apply overrides + graphcode_overrides = overrides.get('graphcode', {}) + if 'args' in graphcode_overrides: + if 'properties' in graphcode_overrides['args']: + graphcode_args['properties'].update(graphcode_overrides['args']['properties']) + if 'policies' in graphcode_overrides['args']: + graphcode_args['policies'].update(graphcode_overrides['args']['policies']) + + config = StorageProviderConfig( + provider_type='graphcode', + connection_config=connection_config, + args=graphcode_args + ) + write_providers['graphcode'] = StorageProviderFactory.create_storage_provider(config) + if echo: print(f"✅ GraphCode provider created: {connection_config['path']}") + + #Neo4j (SSM-sourced, no env reliance) + if 'neo4j' in enable: + from pydantic import SecretStr + # [removed: ai4triage dependency] + ConnectionStringParser, + ) + + def _nonempty(s: Optional[str]) -> bool: + return isinstance(s, str) and s.strip() != "" + + try: + creds = _fetch_neo4j_from_ssm() # {'NEO4J_URI','NEO4J_USERNAME','NEO4J_PASSWORD'} + neo4j_uri = creds["NEO4J_URI"].strip() + user = creds["NEO4J_USERNAME"].strip() + pw_secret = SecretStr(creds["NEO4J_PASSWORD"].strip()) + + if _nonempty(neo4j_uri) and _nonempty(user): + provider_type, config = ConnectionStringParser.make_neo4j_config( + neo4j_uri, + username=user, + password=pw_secret, + for_factory=True, # unwrap to plain str inside connection_string_parser.py + ) + + neo4j_args = { + 'properties': base_args['properties'].copy(), + 'policies': {'enforce_semver': True, 'semver_invalid_action': 'fix'} + } + # Apply Neo4j overrides + neo4j_overrides = overrides.get('neo4j', {}) + if 'args' in neo4j_overrides: + if 'properties' in neo4j_overrides['args']: + neo4j_args['properties'].update(neo4j_overrides['args']['properties']) + if 'policies' in neo4j_overrides['args']: + neo4j_args['policies'].update(neo4j_overrides['args']['policies']) + + storage_config = StorageProviderConfig( + provider_type=provider_type, + connection_config=config, # uri + auth=(user, pass) as plain strings + args=neo4j_args + ) + write_providers['neo4j'] = StorageProviderFactory.create_storage_provider(storage_config) + if echo: + print(f"✅ Neo4j provider created: {config['uri']} (auth=yes)") + else: + if echo: + print("❌ Missing NEO4J_URI or NEO4J_USERNAME from SSM; skipping Neo4j provider") + + except Exception as e: + if echo: + print(f"❌ Neo4j init failed via SSM: {e}") + + # Neptune + if 'neptune' in enable: + neptune_connection = os.environ.get('DOC_NEP_STORE') + if neptune_connection: + provider_type, config = ConnectionStringParser.parse_connection_string(neptune_connection) + neptune_args = { + 'properties': base_args['properties'].copy(), + 'policies': {'enforce_semver': True, 'semver_invalid_action': 'fix'} + } + # Apply Neptune overrides + neptune_overrides = overrides.get('neptune', {}) + if 'args' in neptune_overrides: + if 'properties' in neptune_overrides['args']: + neptune_args['properties'].update(neptune_overrides['args']['properties']) + if 'policies' in neptune_overrides['args']: + neptune_args['policies'].update(neptune_overrides['args']['policies']) + + storage_config = StorageProviderConfig( + provider_type=provider_type, + connection_config=config, + args=neptune_args + ) + write_providers['neptune'] = StorageProviderFactory.create_storage_provider(storage_config) + if echo: print(f"✅ Neptune provider created: {neptune_connection}") + + # FalkorDB + if 'falkordb' in enable: + falkordb_connection = os.environ.get('GRAPH_DOC_STORE') + if falkordb_connection: + provider_type, config = ConnectionStringParser.parse_connection_string(falkordb_connection) + falkordb_args = { + 'properties': base_args['properties'].copy(), + 'policies': {'enforce_semver': True, 'semver_invalid_action': 'fix'} + } + # Apply FalkorDB overrides + falkordb_overrides = overrides.get('falkordb', {}) + if 'args' in falkordb_overrides: + if 'properties' in falkordb_overrides['args']: + falkordb_args['properties'].update(falkordb_overrides['args']['properties']) + if 'policies' in falkordb_overrides['args']: + falkordb_args['policies'].update(falkordb_overrides['args']['policies']) + + storage_config = StorageProviderConfig( + provider_type=provider_type, + connection_config=config, + args=falkordb_args + ) + write_providers['falkordb'] = StorageProviderFactory.create_storage_provider(storage_config) + if echo: print(f"✅ FalkorDB provider created: {falkordb_connection}") + + if echo: print(f"Active write providers: {list(write_providers.keys())}") + return write_providers + +def build_read_providers(tenant_id: str, root_id: str, document_id: str, owner_id: str, + data_dir: Optional[str] = None, + enable: tuple = ('graphjson', 'graphcode', 'neo4j', 'neptune', 'falkordb'), + overrides: Optional[Dict] = None, echo: bool = False) -> Dict[str, Any]: + """Build read providers with proper args structure""" + + # Validate data_dir only for file-based providers + file_providers = {'graphjson', 'graphcode'} + enabled_file_providers = set(enable) & file_providers + + if enabled_file_providers and not data_dir: + raise ValueError(f"data_dir is required for file-based providers: {enabled_file_providers}") + + # Rest of function remains the same... + + """Build read providers with proper args structure""" + # [removed: ai4triage dependency] + # [removed: ai4triage dependency] + # [removed: ai4triage dependency] + + _load_environment_from_bashrc() + + read_providers = {} + overrides = overrides or {} + + # Base args structure + base_args = { + 'properties': { + 'tenant_id': tenant_id, + 'root_id': root_id, + 'document_id': document_id, + 'owner': owner_id + }, + 'policies': {} + } + + # GraphJSON + if 'graphjson' in enable: + connection_config = overrides.get('graphjson', {}).get('connection_config', {'path': f'{data_dir}/graphjson/graph.json'}) + graphjson_args = base_args.copy() + # Apply overrides + graphjson_overrides = overrides.get('graphjson', {}) + if 'args' in graphjson_overrides: + if 'properties' in graphjson_overrides['args']: + graphjson_args['properties'].update(graphjson_overrides['args']['properties']) + if 'policies' in graphjson_overrides['args']: + graphjson_args['policies'].update(graphjson_overrides['args']['policies']) + + config = StorageProviderConfig( + provider_type='graphjson', + connection_config=connection_config, + args=graphjson_args + ) + read_providers['graphjson'] = StorageProviderFactory.create_reader(config) + if echo: print(f"✅ GraphJSON reader created: {connection_config['path']}") + + # GraphCode + if 'graphcode' in enable: + connection_config = overrides.get('graphcode', {}).get('connection_config', + {'path': f'{data_dir}/graphcode/graph_code.py', 'dialect': 'networkx'}) + graphcode_args = base_args.copy() + # Apply overrides + graphcode_overrides = overrides.get('graphcode', {}) + if 'args' in graphcode_overrides: + if 'properties' in graphcode_overrides['args']: + graphcode_args['properties'].update(graphcode_overrides['args']['properties']) + if 'policies' in graphcode_overrides['args']: + graphcode_args['policies'].update(graphcode_overrides['args']['policies']) + + config = StorageProviderConfig( + provider_type='graphcode', + connection_config=connection_config, + args=graphcode_args + ) + read_providers['graphcode'] = StorageProviderFactory.create_reader(config) + if echo: print(f"✅ GraphCode reader created: {connection_config['path']}") + + #Neo4j (SSM-sourced, no env reliance) + if 'neo4j' in enable: + from pydantic import SecretStr + # [removed: ai4triage dependency] + ConnectionStringParser, + ) + + def _nonempty(s: Optional[str]) -> bool: + return isinstance(s, str) and s.strip() != "" + + try: + creds = _fetch_neo4j_from_ssm() # {'NEO4J_URI','NEO4J_USERNAME','NEO4J_PASSWORD'} + neo4j_uri = creds["NEO4J_URI"].strip() + user = creds["NEO4J_USERNAME"].strip() + pw_secret = SecretStr(creds["NEO4J_PASSWORD"].strip()) + + if _nonempty(neo4j_uri) and _nonempty(user): + provider_type, config = ConnectionStringParser.make_neo4j_config( + neo4j_uri, + username=user, + password=pw_secret, + for_factory=True, # unwrap to plain str inside connection_string_parser.py + ) + + neo4j_args = { + 'properties': base_args['properties'].copy(), + 'policies': {'enforce_semver': True, 'semver_invalid_action': 'fix'} + } + # Apply Neo4j overrides + neo4j_overrides = overrides.get('neo4j', {}) + if 'args' in neo4j_overrides: + if 'properties' in neo4j_overrides['args']: + neo4j_args['properties'].update(neo4j_overrides['args']['properties']) + if 'policies' in neo4j_overrides['args']: + neo4j_args['policies'].update(neo4j_overrides['args']['policies']) + + storage_config = StorageProviderConfig( + provider_type=provider_type, + connection_config=config, # uri + auth=(user, pass) as plain strings + args=neo4j_args + ) + read_providers['neo4j'] = StorageProviderFactory.create_reader(storage_config) + if echo: + print(f"✅ Neo4j reader created: {config['uri']} (auth=yes)") + else: + if echo: + print("❌ Missing NEO4J_URI or NEO4J_USERNAME from SSM; skipping Neo4j reader") + + except Exception as e: + if echo: + print(f"❌ Neo4j reader init failed via SSM: {e}") + + # Neptune + if 'neptune' in enable: + neptune_connection = os.environ.get('DOC_NEP_STORE') + if neptune_connection: + provider_type, config = ConnectionStringParser.parse_connection_string(neptune_connection) + neptune_args = { + 'properties': base_args['properties'].copy(), + 'policies': {'enforce_semver': True, 'semver_invalid_action': 'fix'} + } + # Apply Neptune overrides + neptune_overrides = overrides.get('neptune', {}) + if 'args' in neptune_overrides: + if 'properties' in neptune_overrides['args']: + neptune_args['properties'].update(neptune_overrides['args']['properties']) + if 'policies' in neptune_overrides['args']: + neptune_args['policies'].update(neptune_overrides['args']['policies']) + + storage_config = StorageProviderConfig( + provider_type=provider_type, + connection_config=config, + args=neptune_args + ) + read_providers['neptune'] = StorageProviderFactory.create_reader(storage_config) + if echo: print(f"✅ Neptune reader created: {neptune_connection}") + + # FalkorDB + if 'falkordb' in enable: + falkordb_connection = os.environ.get('GRAPH_DOC_STORE') + if falkordb_connection: + provider_type, config = ConnectionStringParser.parse_connection_string(falkordb_connection) + falkordb_args = { + 'properties': base_args['properties'].copy(), + 'policies': {'enforce_semver': True, 'semver_invalid_action': 'fix'} + } + # Apply FalkorDB overrides + falkordb_overrides = overrides.get('falkordb', {}) + if 'args' in falkordb_overrides: + if 'properties' in falkordb_overrides['args']: + falkordb_args['properties'].update(falkordb_overrides['args']['properties']) + if 'policies' in falkordb_overrides['args']: + falkordb_args['policies'].update(falkordb_overrides['args']['policies']) + + storage_config = StorageProviderConfig( + provider_type=provider_type, + connection_config=config, + args=falkordb_args + ) + read_providers['falkordb'] = StorageProviderFactory.create_reader(storage_config) + if echo: print(f"✅ FalkorDB reader created: {falkordb_connection}") + + if echo: print(f"Active read providers: {list(read_providers.keys())}") + return read_providers + +def setup_lexical_graph( + graph_store_env: str = 'GRAPH_STORE', + vector_store_env: str = 'VECTOR_STORE', + graph_type: str = 'neptune', + echo: bool = False + ): + """Setup lexical graph with Neptune/OpenSearch or Neo4j/PostgreSQL""" + import os + from urllib.parse import urlparse + # [removed: ai4triage dependency] + # [removed: ai4triage dependency] + + _load_environment_from_bashrc() + + try: + set_logging_config('INFO') + + graph_store_uri = os.environ.get(graph_store_env) # default path for non-neo4j + + if graph_type == 'neo4j': + # Register Neo4j factory only when needed + # [removed: ai4triage dependency] + Neo4jGraphStoreFactory + ) + GraphStoreFactory.register(Neo4jGraphStoreFactory) + + # Fetch creds from SSM (decrypted) + creds = _fetch_neo4j_from_ssm() # {'NEO4J_URI','NEO4J_USERNAME','NEO4J_PASSWORD'} + neo4j_uri = creds["NEO4J_URI"].strip() + user = creds["NEO4J_USERNAME"].strip() + pw = creds["NEO4J_PASSWORD"].strip() + + # Build authenticated URI: ://user:pass@host:port[/db] + p = urlparse(neo4j_uri if '://' in neo4j_uri else f"neo4j+s://{neo4j_uri}") + scheme = p.scheme or "neo4j+s" + host = p.hostname or "" + port = p.port or 7687 + path = p.path or "" # keep /db if present + + if not host: + raise RuntimeError("Invalid NEO4J_URI from SSM: missing host") + + graph_store_uri = f"{scheme}://{user}:{pw}@{host}:{port}{path}" + + if echo: + print("Neo4j registered for lexical graph") + + # Create stores + graph_store = GraphStoreFactory.for_graph_store(graph_store_uri) + vector_store = VectorStoreFactory.for_vector_store(os.environ.get(vector_store_env)) + + # Create lexical graph index + lexical_query_engine = LexicalGraphIndex(graph_store, vector_store) + + if echo: + print("Lexical-graph stores configured") + print(f"Graph: {type(graph_store).__name__}") + print(f"Vector: {type(vector_store).__name__}") + + return { + 'graph_store': graph_store, + 'vector_store': vector_store, + 'lexical_query_engine': lexical_query_engine, + 'available': True + } + + except Exception as e: + if echo: + print(f"Lexical graph setup failed: {e}") + return { + 'graph_store': None, + 'vector_store': None, + 'lexical_query_engine': None, + 'available': False + } + + +class HealthChecker: + """Production health checks for document graph system""" + + def __init__(self, reader, writer): + self.reader = reader + self.writer = writer + self.health_status = {} + + def check_database_connectivity(self): + try: + result = self.reader.health_check() + self.health_status['database_connectivity'] = { + 'status': 'healthy', + 'message': 'Database connection successful' + } + return True + except Exception as e: + self.health_status['database_connectivity'] = { + 'status': 'unhealthy', + 'error': str(e), + 'message': 'Database connection failed' + } + return False + + def check_data_integrity(self): + try: + orphan_query = "MATCH (n) WHERE NOT (n)--() RETURN count(n) as ORPHAN_COUNT" + orphan_result = self.reader.retrieve(orphan_query) + + missing_props_query = "MATCH (n) WHERE n.tenant_id IS NULL RETURN count(n) as MISSING_TENANT" + missing_props_result = self.reader.retrieve(missing_props_query) + + orphan_count = 0 + missing_tenant_count = 0 + + if orphan_result and len(orphan_result) > 0: + result = orphan_result[0] + if isinstance(result, dict): + orphan_count = list(result.values())[0] + + if missing_props_result and len(missing_props_result) > 0: + result = missing_props_result[0] + if isinstance(result, dict): + missing_tenant_count = list(result.values())[0] + + issues = [] + if orphan_count > 0: + issues.append(f"{orphan_count} orphaned nodes") + if missing_tenant_count > 0: + issues.append(f"{missing_tenant_count} nodes missing tenant_id") + + if issues: + self.health_status['data_integrity'] = { + 'status': 'warning', + 'issues': issues, + 'message': 'Data integrity issues detected' + } + else: + self.health_status['data_integrity'] = { + 'status': 'healthy', + 'message': 'No data integrity issues found' + } + + return len(issues) == 0 + + except Exception as e: + self.health_status['data_integrity'] = { + 'status': 'error', + 'error': str(e), + 'message': 'Data integrity check failed' + } + return False + + def check_performance_baseline(self): + try: + start_time = time.time() + result = self.reader.retrieve("MATCH (n) RETURN count(n) as TOTAL_NODES LIMIT 1") + duration = time.time() - start_time + + if duration < 1.0: + status = 'healthy' + message = f'Performance within acceptable range ({duration:.3f}s)' + elif duration < 5.0: + status = 'warning' + message = f'Performance slower than optimal ({duration:.3f}s)' + else: + status = 'unhealthy' + message = f'Performance unacceptably slow ({duration:.3f}s)' + + self.health_status['performance_baseline'] = { + 'status': status, + 'response_time': duration, + 'message': message + } + + return status == 'healthy' + + except Exception as e: + self.health_status['performance_baseline'] = { + 'status': 'error', + 'error': str(e), + 'message': 'Performance check failed' + } + return False + + def run_full_health_check(self): + print("Running production health checks...") + print("=" * 50) + + checks = [ + ('Database Connectivity', self.check_database_connectivity, 'database_connectivity'), + ('Data Integrity', self.check_data_integrity, 'data_integrity'), + ('Performance Baseline', self.check_performance_baseline, 'performance_baseline') + ] + + all_healthy = True + + for check_name, check_func, status_key in checks: + try: + is_healthy = check_func() + status_icon = "✅" if is_healthy else "⚠️" + print(f"{status_icon} {check_name}: {self.health_status[status_key]['message']}") + + if not is_healthy: + all_healthy = False + + except Exception as e: + print(f"❌ {check_name}: Check failed - {str(e)}") + all_healthy = False + + overall_status = "HEALTHY" if all_healthy else "NEEDS ATTENTION" + print(f"\nOverall System Status: {overall_status}") + + return { + 'overall_status': overall_status, + 'individual_checks': self.health_status, + 'timestamp': datetime.now().isoformat() + } + +def create_storage_providers(connection_string: str, config: Optional[Dict] = None): + """Create reader and writer storage providers""" + storage_config = StorageProviderConfig( + connection_string=connection_string, + config=config or {} + ) + + reader = StorageProviderFactory.create_reader(storage_config) + writer = StorageProviderFactory.create_storage_provider(storage_config) + + return reader, writer + +def debug_data_integrity(reader, output_dir: str = "output"): + """Debug data integrity issues""" + print("Debugging data integrity issues...") + print("=" * 40) + + health_report_path = Path(output_dir) / "health_report.json" + if health_report_path.exists(): + with open(health_report_path, 'r') as f: + health_data = json.load(f) + + data_integrity_status = health_data['individual_checks']['data_integrity'] + print(f"Data Integrity Status: {data_integrity_status}") + + if 'issues' in data_integrity_status: + print(f"Issues found: {data_integrity_status['issues']}") + + print("\nManual integrity checks:") + + orphan_query = "MATCH (n) WHERE NOT (n)--() RETURN count(n) as ORPHAN_COUNT" + orphan_result = reader.retrieve(orphan_query) + print(f"Orphan query result: {orphan_result}") + + missing_tenant_query = "MATCH (n) WHERE n.tenant_id IS NULL RETURN count(n) as MISSING_TENANT" + missing_tenant_result = reader.retrieve(missing_tenant_query) + print(f"Missing tenant query result: {missing_tenant_result}") + + total_nodes_query = "MATCH (n) RETURN count(n) as TOTAL_NODES" + total_result = reader.retrieve(total_nodes_query) + print(f"Total nodes: {total_result}") + + sample_nodes_query = "MATCH (n) RETURN n LIMIT 3" + sample_result = reader.retrieve(sample_nodes_query) + print(f"Sample nodes: {sample_result}") + +def analyze_orphaned_nodes(reader): + """Analyze orphaned nodes in detail""" + print("Identifying orphaned nodes...") + orphaned_nodes_query = "MATCH (n) WHERE NOT (n)--() RETURN n.id as node_id, n.name as name, labels(n) as labels" + orphaned_result = reader.retrieve(orphaned_nodes_query) + + print("Orphaned nodes (no relationships):") + for node in orphaned_result: + if isinstance(node, dict): + node_id = node.get('node_id', 'unknown') + name = node.get('name', 'unknown') + labels = node.get('labels', []) + print(f" - {node_id}: {name} ({labels})") + + print("\nExisting relationships:") + relationships_query = "MATCH (a)-[r]->(b) RETURN a.name as from_name, type(r) as rel_type, b.name as to_name" + rel_result = reader.retrieve(relationships_query) + + if rel_result: + for rel in rel_result: + if isinstance(rel, dict): + from_name = rel.get('from_name', 'unknown') + rel_type = rel.get('rel_type', 'unknown') + to_name = rel.get('to_name', 'unknown') + print(f" {from_name} --[{rel_type}]--> {to_name}") + else: + print(" No relationships found in the database") + + return orphaned_result, rel_result + +def save_health_report(health_report: Dict, output_dir: str = "output"): + """Save health report to JSON file""" + health_report_path = Path(output_dir) / "health_report.json" + health_report_path.parent.mkdir(exist_ok=True) + + with open(health_report_path, 'w') as f: + json.dump(health_report, f, indent=2) + + print(f"Health report saved to: {health_report_path}") + return health_report_path + + +def clear_database(provider_type: str, tenant_id: str = None, echo: bool = False): + """Clear database based on provider type - supports document graph and lexical graph""" + from urllib.parse import urlparse + + _load_environment_from_bashrc() + + try: + if provider_type == 'neptune': + # Neptune document graph + # [removed: ai4triage dependency] + # [removed: ai4triage dependency] + # [removed: ai4triage dependency] + + connection_string = os.environ.get('DOC_NEP_STORE') + if not connection_string: + if echo: print("❌ DOC_NEP_STORE not found") + return False + + parsed_type, config = ConnectionStringParser.parse_connection_string(connection_string) + storage_config = StorageProviderConfig( + provider_type=parsed_type, + connection_config=config, + args={ + 'properties': { + 'tenant_id': tenant_id or 'temp', + 'root_id': 'temp', + 'document_id': 'clear-operation', + 'owner': 'system' + }, + 'policies': {'enforce_semver': True, 'semver_invalid_action': 'fix'} + } + ) + + writer = StorageProviderFactory.create_storage_provider(storage_config) + + if tenant_id: + if echo: print(f"Clearing Neptune database for tenant: {tenant_id}") + writer.execute_query(f"MATCH (n)-[r]-() WHERE n.tenant_id = '{tenant_id}' DELETE r") + writer.execute_query(f"MATCH (n) WHERE n.tenant_id = '{tenant_id}' DELETE n") + else: + if echo: print("Clearing entire Neptune database") + writer.execute_query("MATCH (n)-[r]-() DELETE r") + writer.execute_query("MATCH (n) DELETE n") + + elif provider_type == 'neptune-lexical': + # Neptune lexical graph - use smaller batches to avoid timeout + # [removed: ai4triage dependency] + + connection_string = os.environ.get('GRAPH_STORE') + if not connection_string: + if echo: print("❌ GRAPH_STORE not found") + return False + + graph_store = GraphStoreFactory.for_graph_store(connection_string) + + if echo: print("Clearing Neptune lexical graph database (batch mode)") + + # Clear in small batches to avoid timeout + batch_size = 100 + total_deleted = 0 + + while True: + try: + result = graph_store.execute_query( + f"MATCH (n) WITH n LIMIT {batch_size} DETACH DELETE n RETURN count(*) as deleted" + ) + deleted_count = 0 + if result and len(result) > 0: + deleted_count = result[0].get('deleted', 0) + + total_deleted += deleted_count + if deleted_count == 0: + break + + if echo and total_deleted % 500 == 0: + print(f"Deleted {total_deleted} nodes so far...") + + except Exception as e: + if echo: print(f"Batch deletion stopped at {total_deleted} nodes: {e}") + break + + if echo: print(f"Cleared {total_deleted} nodes from Neptune lexical graph") + + elif provider_type == 'neo4j': + # Neo4j lexical graph - build connection string with auth from SSM + # [removed: ai4triage dependency] + # [removed: ai4triage dependency] + + # Register Neo4j factory + GraphStoreFactory.register(Neo4jGraphStoreFactory) + + try: + creds = _fetch_neo4j_from_ssm() # {'NEO4J_URI','NEO4J_USERNAME','NEO4J_PASSWORD'} + neo4j_uri = creds["NEO4J_URI"].strip() + neo4j_username = creds["NEO4J_USERNAME"].strip() + neo4j_password = creds["NEO4J_PASSWORD"].strip() + except Exception as e: + if echo: print(f"❌ Failed to fetch Neo4j creds from SSM: {e}") + return False + + if not neo4j_uri or not neo4j_username or not neo4j_password: + if echo: print("❌ Missing Neo4j credentials from SSM") + return False + + # Build authenticated URI: ://user:pass@host:port[/db] + p = urlparse(neo4j_uri if '://' in neo4j_uri else f"neo4j+s://{neo4j_uri}") + scheme = p.scheme or "neo4j+s" + host = p.hostname or "" + port = p.port or 7687 + path = p.path or "" # keep database path if present: /dbname + + if not host: + if echo: print("❌ Invalid NEO4J_URI: missing host") + return False + + auth_uri = f"{scheme}://{neo4j_username}:{neo4j_password}@{host}:{port}{path}" + + graph_store = GraphStoreFactory.for_graph_store(auth_uri) + + if tenant_id: + if echo: print(f"Clearing Neo4j database for tenant: {tenant_id}") + graph_store.execute_query(f"MATCH (n) WHERE n.tenant_id = '{tenant_id}' DETACH DELETE n") + else: + if echo: print("Clearing entire Neo4j database") + graph_store.execute_query("MATCH (n) DETACH DELETE n") + + elif provider_type == 'opensearch': + # AWS OpenSearch Serverless - delete indices + try: + import boto3 + from opensearchpy import OpenSearch, RequestsHttpConnection + from aws_requests_auth.aws_auth import AWSRequestsAuth + + connection_string = os.environ.get('VECTOR_STORE') + if not connection_string or not connection_string.startswith('aoss://'): + if echo: print("❌ VECTOR_STORE not found or not OpenSearch Serverless format") + return False + + endpoint = connection_string.replace('aoss://', '') + if endpoint.startswith('https://'): + endpoint = endpoint.replace('https://', '') + + region = os.environ.get('AWS_REGION', 'us-east-1') + service = 'aoss' + session = boto3.Session() + creds = session.get_credentials() + if creds is None: + if echo: print("❌ No AWS credentials available for OpenSearch") + return False + + awsauth = AWSRequestsAuth(creds, region, service) + + client = OpenSearch( + hosts=[{'host': endpoint, 'port': 443}], + http_auth=awsauth, + use_ssl=True, + verify_certs=True, + connection_class=RequestsHttpConnection + ) + + if echo: print("Clearing OpenSearch Serverless collection") + + indices = client.indices.get_alias("*") + deleted_indices = [] + + for index_name in indices.keys(): + if not index_name.startswith('.'): # Skip system indices + try: + client.indices.delete(index=index_name) + deleted_indices.append(index_name) + if echo: print(f"Deleted index: {index_name}") + except Exception as e: + if echo: print(f"Failed to delete index {index_name}: {e}") + + if echo: print(f"Cleared {len(deleted_indices)} indices from OpenSearch Serverless") + + except ImportError: + if echo: print("❌ Required packages not installed: pip install opensearch-py aws-requests-auth") + return False + + elif provider_type == 'postgresql': + if echo: print("Clearing PostgreSQL vector database") + if echo: print("⚠️ PostgreSQL clearing not implemented - manual deletion required") + + else: + if echo: print(f"❌ Unsupported provider type: {provider_type}") + return False + + if echo: print(f"✅ {provider_type} database cleared successfully") + return True + + except Exception as e: + if echo: print(f"❌ Failed to clear {provider_type} database: {e}") + return False + + +# Export functions for nbimport +__all__ = [ + 'HealthChecker', + 'create_storage_providers', + 'build_write_providers', + 'build_read_providers', + 'setup_lexical_graph', + 'debug_data_integrity', + 'analyze_orphaned_nodes', + 'save_health_report', + 'clear_database' + + +] \ No newline at end of file diff --git a/document-graph/examples/cloud/push-to-sagemaker.sh b/document-graph/examples/cloud/push-to-sagemaker.sh new file mode 100755 index 00000000..211d2db6 --- /dev/null +++ b/document-graph/examples/cloud/push-to-sagemaker.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# push-to-sagemaker.sh — Rebuild wheel + upload to S3 (SageMaker pulls from there) +# Run locally after making changes to document-graph. +# Then in SageMaker terminal: bash ~/SageMaker/document-graph/install.sh +set -e + +cd "$(dirname "$0")/../.." +export AWS_PROFILE="${AWS_PROFILE:-nw}" + +echo "Building wheel..." +rm -f dist/*.whl +/Library/Frameworks/Python.framework/Versions/3.12/bin/python3.12 -m build --wheel --outdir dist/ 2>&1 | tail -1 + +echo "Uploading wheel..." +aws s3 cp dist/document_graph-*.whl s3://graphrag-artifacts-705909755305/document-graph-notebooks/wheels/ --region us-east-1 + +echo "Cleaning S3 notebooks (removing stale files)..." +aws s3 rm s3://graphrag-artifacts-705909755305/document-graph-notebooks/ --recursive --exclude "wheels/*" --region us-east-1 + +echo "Uploading notebooks..." +aws s3 sync examples/cloud/notebooks/ s3://graphrag-artifacts-705909755305/document-graph-notebooks/ \ + --exclude "__pycache__/*" --exclude "*.pyc" --exclude "wheels/*" --region us-east-1 + +echo "" +echo "✅ Pushed. In SageMaker run:" +echo " aws s3 sync s3://graphrag-artifacts-705909755305/document-graph-notebooks/ ~/SageMaker/document-graph/" +echo " pip install --force-reinstall ~/SageMaker/document-graph/wheels/document_graph-*.whl"